diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 542db7e..02011ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,3 +37,12 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - run: cargo check -p shuru-cli + + linux-test: + name: Linux tests + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: cargo test -p shuru-proto -p shuru-linux -p shuru-node -p shuru-vm -p shuru-guest diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml index b2a41e6..40a8088 100644 --- a/.github/workflows/publish-crates.yml +++ b/.github/workflows/publish-crates.yml @@ -28,7 +28,7 @@ jobs: echo "Published $crate" echo "Waiting for crates.io to index..." sleep 30 - elif echo "$output" | grep -q "already exists"; then + elif echo "$output" | grep -qE "already (exists|uploaded)"; then echo "$crate already published, skipping" else echo "$output" >&2 @@ -60,7 +60,7 @@ jobs: echo "Published $crate" echo "Waiting for crates.io to index..." sleep 30 - elif echo "$output" | grep -q "already exists"; then + elif echo "$output" | grep -qE "already (exists|uploaded)"; then echo "$crate already published, skipping" else echo "$output" >&2 @@ -91,7 +91,7 @@ jobs: echo "Published $crate" echo "Waiting for crates.io to index..." sleep 30 - elif echo "$output" | grep -q "already exists"; then + elif echo "$output" | grep -qE "already (exists|uploaded)"; then echo "$crate already published, skipping" else echo "$output" >&2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dafd62e..6666275 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,7 +42,7 @@ jobs: DATA_DIR="${HOME}/.local/share/shuru" tar czf "shuru-os-${VERSION}-aarch64.tar.gz" \ -C "${DATA_DIR}" \ - Image initramfs.cpio.gz rootfs.ext4 + Image initramfs.cpio rootfs.ext4 - name: Upload artifact uses: actions/upload-artifact@v4 @@ -127,12 +127,16 @@ jobs: uses: softprops/action-gh-release@v2 with: generate_release_notes: true + # A tag with a hyphen (v0.8.0-rc.1) is a rehearsal: published as a + # pre-release, never marked latest, never pushed to Homebrew. + prerelease: ${{ contains(github.ref_name, '-') }} files: artifacts/* update-homebrew: name: Update Homebrew tap runs-on: ubuntu-latest needs: release + if: ${{ !contains(github.ref_name, '-') }} steps: - uses: actions/checkout@v4 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index 62a476e..539f59b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,374 @@ # Changelog +## 0.11.0 + +### Fast boot + +`shuru run -- true` on an M-series Mac goes from about 300 ms to about 160 +ms. The guest agent is now `/init` in an uncompressed initramfs and mounts +the root itself. The old initramfs gunzip, `e2fsck` and offline +`resize2fs` are gone from the boot path. The kernel replays an old journal +when it mounts, and a larger disk grows online with `EXT4_IOC_RESIZE_FS`. +Networking is configured by the agent. The rootfs no longer carries a copy +of the agent. + +The kernel tick goes from 100 Hz to 1000 Hz. virtio_pci_modern sleeps one +millisecond at a time while a device resets, and on the asynchronous Apple +reset each of those sleeps cost two jiffies, 10 to 20 ms per device at 100 +Hz. Guest uptime when the command starts drops from 150 ms to 70 ms. + +On APFS, growing a clone of a fragmented file costs time in proportion to +its extent count, 0.5 to 1.5 s for the base image built by +`prepare-rootfs.sh`. The base is now rewritten sequentially and extended to +4096 MB once, after download or on the first boot that finds it short, and +a clone then costs well under a millisecond. + +### Sealed checkpoints + +Every checkpoint path now freezes ext4 in the guest before the host copies +the disk, so a saved disk is clean and needs no journal replay. A seal +carries an id and a lease; the guest thaws itself when the lease runs out, +and the host publishes the copy only after a matching thaw. A failed save +keeps the previous checkpoint. A save that died leaves a staging directory +that the next save of the same name reuses. CAS checkpoints are written as +standalone indexes, so replacing one cannot form a parent cycle or depend +on the base image path. + +The root is mounted `errors=remount-ro`. A disk with recorded ext4 errors +refuses to boot and refuses to seal. + +### Freed space goes back to the host + +A seal runs `FITRIM` for free extents of 1 MiB and up, and `trim` is +available on a running sandbox from the CLI stdio protocol and both SDKs. +The Linux block device supports discard and write zeroes, mapped to +`fallocate` hole punching, with every range bounds checked. + +### Disk size + +Disks default to 4096 MB, which is now also the minimum. A disk never +shrinks below its source. + +### macOS: a VM that powers off during boot no longer hangs the host + +Connecting to the guest waited forever if the VM stopped while the connect +was pending, because the framework never calls the completion handler. It +now notices the stopped VM and fails. + +### Watch events carry their names again + +`file_change` notifications sent `"2"` and `"3"` instead of `"modify"` and +`"delete"`, and the guest reported every new file as modified. Events are +`create`, `modify`, `delete` and `rename` again, a rename arrives once for +each path, and `shuru_proto::watch_kind::name` maps a kind to its name. + +### Files of 1 MB and up can be read and written + +`read_file` and `write_file` sent the whole file in one frame, and frames +stop at 1 MB, so larger files failed with a broken pipe. Contents now +travel in 256 KiB frames. A failed write reports the guest error, and the +TypeScript SDK no longer rescans a long response line for every chunk it +receives, which made a 64 MB read take 1.7 s instead of 0.5 s. + +### Host memory no longer grows with what the guest does + +A guest could grow the host shuru process without limit: by writing to a +socket the remote never read, by flooding a socket the guest never read, or +by printing to a spawned process stdout the client never consumed. The old +build reached 2 GB in five seconds on the last case. The proxy now holds a +fixed window per connection each way and its smoltcp buffers are halved to +512 KiB and 256 KiB at the same measured throughput. stdio and SDK event +queues are bounded, and buffered exec output and file reads fail past 64 MiB +with an error that points at streaming instead. On Linux the VMM validates virtqueue +sizes, ring addresses and descriptor chains, and caps virtio-fs handles, +inodes and request sizes, so a guest cannot loop, crash or inflate it either. +RFC 0011 has the rule, the remaining per-VM ceilings and the cgroup envelope +for hosted nodes. + +### Kill reaches a process that stopped reading stdin + +Stdin and the kill signal shared one connection, so a process that stopped +reading stdin left the kill queued behind stdin nobody would take, and the +SDK kept piling stdin up in the shuru process. A kill now sends SIGTERM as +before and, if the process is still running two seconds later, SIGKILL on a +connection of its own. The guest also kills a process group when the host +hangs up on it. In the TypeScript SDK, `write()` on a spawned process now +returns a promise that resolves once the data reaches the VM; up to 1 MiB +is in flight and the rest waits in the calling process. The stdio server +refuses more than 4 MiB of unacknowledged stdin per process. A single write +over 1 MiB used to break the stream; it is now split. + +### Short connections no longer get refused at random + +The network proxy added a listener for each guest SYN it inspected, but +smoltcp also read frames straight from the guest while it polled. A SYN that +arrived during a poll skipped inspection and was answered with a reset, so a +loop of short connections failed with "Connection refused" somewhere between +150 and 475 connections in. smoltcp now only sees inspected frames, and 600 +back to back connections succeed. + +### Linux: the guest console works + +The Linux backend emulates a PL011 serial port and boots the guest with +`console=ttyAMA0`, but the kernel was built without the PL011 driver, so a +Linux guest printed nothing, `--verbose` showed no kernel log and +`--console` had no console to attach to. The driver is now built in. + +The Linux bootloader also dropped `quiet` from every command line. With a +working console that would print the whole kernel log through the emulated +UART on every boot, about 230 ms on a1.metal, only to discard it. Linux now +keeps `quiet` unless `--verbose` is set, as macOS does. + +### Checkpoint saves report stopped services again + +`shuru checkpoint create --verbose` prints the services that were stopped +before the save, and every save prints services that ignored SIGTERM. + +## 0.10.0 + +### Linux: a VM can no longer reach another VM + +On a Linux host every VM asked vhost-vsock for guest CID 3. vhost CIDs are +global to the host kernel, so the second VM got EADDRINUSE, the error was +printed and ignored, and its host side connected to CID 3 anyway: the first +VM. Reproduced on a1.metal with 0.9.3, a second `shuru run` executed inside +the first sandbox and printed its marker file and kernel boot_id. macOS was +not affected. + +The Linux backend now emulates virtio-vsock in the VMM instead of using +vhost-vsock. Each VM has its own device and its own CID space, the host end +of every connection is a socketpair inside the shuru process, and no other +process on the host can reach the guest agent. Host data stays in that +socket until the guest posts receive buffers, then one read puts it +straight into guest memory across as many buffers as the credit the guest +grants allows, so nothing queues in the VMM. Guest data goes from guest +memory to the socket in one write and is only copied when the socket is +full. Connections opened by the guest are refused. The vhost_vsock module +is no longer needed. + +### Linux: devices no longer share a lock + +Every virtio device used to sit behind one lock, and disk and shared folder +requests ran on the vCPU that rang the doorbell while holding it, so one +busy device stalled every vCPU touching any other. Each device now has its +own lock and its own thread. Doorbells go to an eventfd inside KVM, so +ringing one no longer exits to the VMM, and interrupts fire only when +buffers were used and the guest asked for them. Raising and acknowledging +an interrupt now happen under one lock, which closes a window where an +acknowledgement could lower the line over a newly raised reason. + +### The guest checks who is connecting + +Every boot gets a random 32 byte token and a 16 byte instance id on the +kernel command line. The guest agent requires the token as the first frame +of every connection, on the control channel and on port forwards, and +answers with the instance id, which the host checks. A connection that +reaches the wrong VM is refused instead of served, and a guest booted +without a token stops. The token is redacted from `--verbose` and +`--console` output. This applies on macOS too. + +The protocol version is 2. An OS image older than this release does not +answer the hello, and `shuru upgrade` fetches the matching one. + +Measured on a1.metal against 0.9.3 on the same host, three runs each: + +| | 0.9.3 (vhost) | 0.10.0 | +|---|---|---| +| exec round trip p50 | 1.6 ms | 1.8 ms | +| 16 parallel execs | 35 ms | 36 ms | +| guest to host, 128 MB | 117 to 191 MB/s | 434 to 451 MB/s | +| host to guest, 128 MB | 353 to 366 MB/s | 315 to 360 MB/s | +| shared folder read, 128 MB | 241 to 245 MB/s | 343 to 384 MB/s | + +### Crates + +- `shuru-proto` 0.5.0: `hello::Identity`, the `HELLO` and `HELLO_OK` + frames, protocol version 2. +- `shuru-linux` 0.4.0: breaking. Userspace virtio-vsock, + `VirtualMachine::new` returns a `Result`, `GUEST_CID` and `AF_VSOCK` are + gone. `kvm::devices`: the bus dispatches to `MmioHandler`, devices take + an `Interrupt`, and `VirtioBackend::process_queue` returns whether it + used a buffer. +- `shuru-vm` 0.6.0 and `shuru-sdk` 0.8.0: the hello on every connection. + They need a 0.10.0 OS image. +- `shuru-guest` 0.5.0. + +## 0.9.3 + +### Guest memory goes back to the host + +Every VM now carries a virtio memory balloon (RFC 0006). On a Linux host +the guest kernel reports the pages it has freed and the host discards +them, so an idle sandbox costs what the guest is using, not what it has +touched since boot. Measured on a1.metal: a guest that filled and freed +1 GB took the host process from 1109 MB resident back to 87 MB within +seven seconds, with nobody asking. + +A caller can also set a memory target. The guest gives up the difference +through the balloon and takes it back when the target rises, on Linux and +on macOS. `shuru run` is unchanged. + +### Crates + +- `shuru-linux` 0.3.2: the balloon backend, `set_memory_target` and + `ballooned_bytes` on `VirtualMachine`. +- `shuru-darwin` 0.2.2: `set_memory_target` through the framework balloon. +- `shuru-vm` 0.5.2: the balloon is attached on both platforms; + `Sandbox::set_memory_target`. +- `shuru-sdk` 0.7.1: `AsyncSandbox::set_memory_target` and the `balloon` + example. + +## 0.9.2 + +### The working disk can no longer outlive its run + +Every run clones the base image, and until now the clone was a named file +under `instances/` that the process deleted on a clean exit. A run +ended by SIGTERM, a kill, a crash or an SDK program dying left the clone +behind, and `shuru prune` could not tell: SDK directories were never +matched, and a reused PID looked alive. Disk usage grew with every +interrupted run. + +The clone is anonymous now. On Linux it is created with `O_TMPFILE` and +never has a name; on macOS the framework needs a path, so the clone is +named for the two syscalls between creation and open, then unlinked and +attached through `/dev/fd`. The kernel frees the blocks when the last +descriptor closes, however the process ends. Checkpoints are cloned out of +the open descriptor, instantly on APFS, XFS and btrfs. The `instances` +directory, the PID naming and `shuru prune` are gone; `shuru upgrade` +removes a leftover `instances` directory from earlier versions. + +Measured on macOS: a run killed with SIGKILL while writing 600 MB returned +the space to the volume within seconds and left no file behind. + +### Crates + +- `shuru-darwin` 0.2.1: `clone_path` and `clone_fd` (APFS clones). +- `shuru-linux` 0.3.1: `open_anonymous` and `reflink`. +- `shuru-vm` 0.5.1: `WorkDisk`, the anonymous clone, with `attach_path` + and `save_to`. Stays free of unsafe code. +- `shuru-sdk` 0.7.0: breaking. `AsyncSandbox::instance_dir` is removed, + there is no directory any more. The Linux SDK path now reflinks instead + of copying the base image. + +## 0.9.1 + +### CLI downloads trust the platform certificate store + +`shuru upgrade`, `shuru init` and `shuru recipe` verified TLS against a +bundled root list, so a corporate TLS interceptor such as Netskope failed +with `UnknownIssuer` (#27). They now use the platform verifier: the system +keychain on macOS, the system store plus `SSL_CERT_FILE` and `SSL_CERT_DIR` +on Linux. The connection the proxy opens to a host a secret is bound to +already honours `SSL_CERT_FILE` and `SSL_CERT_DIR` on both platforms, and +that is the way to trust an interceptor there; documented in the README. + +### Docs + +The secrets section now says what responses from a bound host do: they come +back untouched, so a host that reflects the request shows the real value to +the guest. Bind a secret only to hosts you trust with it. + +## 0.9.0 + +### Network throughput inside the VM, about 80 times faster + +The proxy netstack that backs `--allow-net` capped every guest connection +at about 1 MB/s whatever the host could do. Contributed by rsclarke in #35: +the smoltcp poll loop no longer sleeps 1 ms between passes but waits on the +host socket, the proxy command channel and the stack timer; the TCP buffers +grow so window scaling can fill the pipe; and the VM network attachment +runs at a 65535 byte IP-MTU so the stack handles one 64 KiB frame per +syscall instead of forty at 1500 bytes. `apt-get`, `npm install` and +release downloads inside the VM now run close to host speed. + +On top of that: per-connection buffers are 1 MiB receive and 512 KiB send +rather than 4 MiB each way, with a ceiling of 512 live sockets, so a guest +opening hundreds of connections costs the host a bounded amount of memory. +The host socket buffers are read back after being set and a socket too +small to carry one frame is an error rather than a silent stall. The Linux +backend receives frames at the new size. + +### Crates + +- `shuru-proxy` 0.4.0: larger frames and buffers, `DrainStatus` on the + device, `create_socketpair` fails when the kernel caps the buffers. +- `shuru-darwin` 0.2.0 and `shuru-linux` 0.3.0: breaking. + `FileHandleNetworkAttachment::new` takes the IP-MTU. +- `shuru-vm` 0.5.0 and `shuru-sdk` 0.6.0: pick up the above. + +## 0.8.1 + +### Linux: reflink clones instead of copies + +The Linux backend copied the full disk image for every boot and every +checkpoint, where macOS has always used an APFS clonefile. The clone now +goes through the FICLONE ioctl, so on XFS with `reflink=1` or btrfs a boot +and a checkpoint are copy-on-write and cost no disk. Measured on an +a1.metal host against the same 0.8.0 image: boot 1.3s instead of 9s, +`checkpoint create` 1.3s instead of 51s, `run --from` 1.0s instead of 35s, +and a checkpoint allocates 0 bytes instead of 4 GB. Filesystems without +reflinks still work through the old copy, with a one-time note naming the +directory so the fix is obvious. + +## 0.8.0 + +### Guest services + +An image can declare long running processes in +`/etc/shuru/services/.json`. The guest init starts them at boot, +waits for their readiness check before running the command, and stops them +with SIGTERM before the VM powers off. The host now asks the guest to shut +down over vsock, which also syncs the disk, so checkpoints are cloned from +a consistent image on every stop path. `Sandbox::stop()` is graceful and +`Sandbox::kill()` is the old hard stop. See docs/rfcs/0004-guest-services.md. + +### Recipes + +`shuru run --recipe REF` and `shuru checkpoint create NAME --recipe REF` +build a checkpoint from a recipe: a directory with `recipe.json` and shell +steps, fetched from GitHub at an exact commit or read from a local path. +`shuru recipe list` shows the official catalog in superhq-ai/recipes and +`shuru recipe show REF` prints a recipe before it is built. Checkpoints built +this way record their source and revision, visible in `shuru checkpoint +list`, and a changed recipe rebuilds on the next run. `--checkpoint NAME` +on `run` picks the checkpoint name instead of the recipe name. Third-party +recipes ask for confirmation or `--yes`. The catalog ships `docker` and +`browser`, headless Chromium as a guest service with the DevTools protocol +on port 9222, which the Browser Use example now uses. See +docs/rfcs/0005-recipes.md. + +### checkpoint update + +`shuru checkpoint update NAME ...` takes the same arguments as `create` and +replaces an existing checkpoint, or creates it when missing. The new +checkpoint is built under a staging name and swapped in only when the +command exits 0, so a failed update leaves the old one in place. +`--from NAME` layers the command onto the existing checkpoint and +`--recipe REF` rebuilds from a recipe. Closes #44. + +### Docker Engine runs inside the VM + +The `docker` recipe installs pinned docker-ce 29.9.2 and containerd 2.4.2, +declares dockerd as a guest service, and pre-pulls one image, so +`shuru checkpoint create docker --recipe docker` followed by +`shuru run --from docker -- docker run ...` just works, as does +`shuru run --recipe docker` on its own. Containers get the same boundary as +any guest process: no egress without `--allow-net`, the host allowlist +applied by the proxy, host ports only through `-p`. + +To support it, the guest kernel gains cgroup v2 controllers, cgroup BPF, +seccomp filters, veth, and the netfilter targets dockerd installs, and moves +to Linux 6.12.114. The guest init now mounts cgroup2 on /sys/fs/cgroup and +tmpfs on /run and /dev/shm. `/run` is therefore no longer persisted into +checkpoints. + +`SHURU_DATA_DIR` overrides the data directory so builds and tests can use +their own kernel, initramfs and checkpoints. `scripts/build-kernel-hf.sh` +builds the kernel on a Hugging Face CPU job. + +See docs/rfcs/0003-docker-on-shuru.md. + ## 0.7.0 ### Secrets can be refreshed while the VM runs diff --git a/Cargo.lock b/Cargo.lock index ab4a423..c9094c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -231,6 +231,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cexpr" version = "0.6.0" @@ -312,6 +318,16 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -347,6 +363,22 @@ dependencies = [ "url", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -887,6 +919,50 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys 0.3.1", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -1203,6 +1279,12 @@ dependencies = [ "syn", ] +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "pem" version = "3.0.6" @@ -1386,6 +1468,18 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pki-types" version = "1.14.0" @@ -1395,6 +1489,33 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.9" @@ -1416,6 +1537,47 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "seccompiler" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae55de56877481d112a559bbc12667635fdaf5e005712fd4e2b2fa50ffc884" +dependencies = [ + "libc", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.11.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.228" @@ -1476,7 +1638,7 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "shuru-cli" -version = "0.7.0" +version = "0.11.0" dependencies = [ "anyhow", "base64", @@ -1499,7 +1661,7 @@ dependencies = [ [[package]] name = "shuru-darwin" -version = "0.1.1" +version = "0.2.3" dependencies = [ "block2", "crossbeam-channel", @@ -1512,7 +1674,7 @@ dependencies = [ [[package]] name = "shuru-guest" -version = "0.3.6" +version = "0.6.0" dependencies = [ "flate2", "ignore", @@ -1526,7 +1688,7 @@ dependencies = [ [[package]] name = "shuru-linux" -version = "0.2.1" +version = "0.4.1" dependencies = [ "crossbeam-channel", "kvm-bindings", @@ -1538,17 +1700,35 @@ dependencies = [ "vmm-sys-util", ] +[[package]] +name = "shuru-node" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "getrandom 0.3.4", + "kvm-bindings", + "libc", + "seccompiler", + "serde", + "serde_json", + "shuru-linux", + "shuru-proto", + "shuru-proxy", +] + [[package]] name = "shuru-proto" -version = "0.3.3" +version = "0.6.0" dependencies = [ "serde", "serde_json", + "subtle", ] [[package]] name = "shuru-proxy" -version = "0.3.0" +version = "0.5.0" dependencies = [ "anyhow", "boring", @@ -1571,7 +1751,7 @@ dependencies = [ [[package]] name = "shuru-sdk" -version = "0.4.0" +version = "0.9.0" dependencies = [ "anyhow", "blake3", @@ -1586,7 +1766,7 @@ dependencies = [ [[package]] name = "shuru-store" -version = "0.1.3" +version = "0.2.0" dependencies = [ "anyhow", "blake3", @@ -1598,10 +1778,11 @@ dependencies = [ [[package]] name = "shuru-vm" -version = "0.3.8" +version = "0.7.0" dependencies = [ "anyhow", "crossbeam-channel", + "getrandom 0.3.4", "serde", "serde_json", "shuru-darwin", @@ -1964,6 +2145,7 @@ dependencies = [ "percent-encoding", "rustls", "rustls-pki-types", + "rustls-platform-verifier", "serde", "serde_json", "ureq-proto", @@ -2077,6 +2259,15 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "1.0.6" @@ -2123,13 +2314,22 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets", + "windows-targets 0.52.6", ] [[package]] @@ -2141,34 +2341,67 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -2181,24 +2414,48 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.52.6" diff --git a/Cargo.toml b/Cargo.toml index b7c893e..d8fafb2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "crates/shuru-cli", "crates/shuru-darwin", "crates/shuru-linux", + "crates/shuru-node", "crates/shuru-proto", "crates/shuru-proxy", "crates/shuru-sdk", @@ -12,8 +13,9 @@ members = [ "crates/shuru-guest", ] # shuru-darwin and shuru-linux are platform-specific — only pulled in -# transitively via shuru-vm's cfg'd deps. Excluded from default builds -# so `cargo check` doesn't fail on the wrong OS. +# transitively via shuru-vm's cfg'd deps. shuru-node runs on Linux nodes +# only. Excluded from default builds so `cargo check` doesn't fail on the +# wrong OS. default-members = [ "crates/shuru-cli", "crates/shuru-proto", diff --git a/README.md b/README.md index 0131ef1..e971c10 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ Shuru boots lightweight Linux VMs for AI agents. On macOS it uses Apple's Virtua - macOS 14 (Sonoma) or later on Apple Silicon - Linux ARM64 with KVM access (`/dev/kvm`) for experimental testing only +On Linux, keep the data directory (`~/.local/share/shuru` or `SHURU_DATA_DIR`) on a filesystem with reflinks, XFS with `reflink=1` or btrfs. Every boot and every checkpoint is then a copy-on-write clone that costs no disk and no time, as on APFS. On ext4 shuru falls back to copying the disk image, which turns a one second boot into ten and makes each checkpoint a full copy. + ## Install ```sh @@ -51,6 +53,12 @@ shuru run --allow-net --dns-resolver 1.1.1.1 -- curl https://example.com shuru run --cpus 4 --memory 4096 --disk-size 8192 -- make -j4 ``` +Disks default to 4096 MB, which is also the minimum. Larger disks grow in +the guest at boot and cannot shrink below their source image. Sparse files +and reflinks keep unused capacity from becoming an allocation. Checkpoints +freeze ext4 before copying and replace the saved disk only after the seal +is confirmed. + ### Directory mounts Share host directories into the VM using VirtioFS. By default the host directory is read-only; guest writes go to a tmpfs overlay layer (discarded when the VM exits). Append `:rw` to make the mount read-write — guest writes go directly to the host filesystem. @@ -105,6 +113,12 @@ shuru run --from myenv -- python3 script.py # Branch from an existing checkpoint shuru checkpoint create myenv2 --from myenv --allow-net -- sh -c 'pip install numpy' +# Rebuild in place: the old checkpoint stays until the new build succeeds +shuru checkpoint update myenv --allow-net -- sh -c 'apt-get install -y python3 gcc make' + +# Or layer onto the existing one +shuru checkpoint update myenv --from myenv --allow-net -- sh -c 'pip install numpy' + # List and delete shuru checkpoint list shuru checkpoint delete myenv @@ -112,7 +126,7 @@ shuru checkpoint delete myenv ### Secrets -Secrets keep API keys on the host. The guest receives a random placeholder token; the proxy substitutes the real value only on HTTPS requests to the specified hosts. The real secret never enters the VM. +Secrets keep API keys on the host. The guest receives a random placeholder token; the proxy substitutes the real value only on HTTPS requests to the specified hosts. The real secret is never placed in the VM. Bind a secret only to hosts you trust with it: responses come back untouched, so a bound host that reflects the request, as echo and debug endpoints do, shows the real value to the guest. ```sh # Inject a secret via CLI @@ -156,6 +170,21 @@ Shuru loads `shuru.json` from the current directory (or `--config PATH`). All fi The `network.allow` list restricts which hosts the guest can reach. Omit it to allow all hosts. +### Environment variables + +Per-machine settings live in the environment rather than in `shuru.json`, so +a project file never decides where assets live or how checkpoints are stored +on someone else. None of these can grant a permission; networking, mounts and +host writes stay on flags and in the project file. + +| Variable | Effect | +|---|---| +| `SHURU_DATA_DIR` | Where assets, checkpoints and instance data live. Default `~/.local/share/shuru`. | +| `SHURU_KERNEL`, `SHURU_INITRD`, `SHURU_ROOTFS` | Boot from these files instead of the installed assets. Same as `--kernel`, `--initrd` and `--rootfs`, which win when both are given. | +| `SHURU_STORAGE` | `cas` saves new checkpoints as `.idx` indexes into the chunk store under the data dir. Unset or any other value keeps the default flat `.ext4` copies. A `.idx` checkpoint always boots through the chunk store. | +| `SHURU_BOOT_TRACE` | When set, prints boot timings to stderr. | +| `SSL_CERT_FILE`, `SSL_CERT_DIR` | Extra roots for the proxy when it opens its own TLS connection to a host a secret is bound to. Needed behind a TLS interceptor such as Netskope, since that connection does not read the macOS keychain. CLI downloads use the platform store and need this only on Linux. | + ## SDK Use shuru programmatically from TypeScript with the [`@superhq/shuru`](https://www.npmjs.com/package/@superhq/shuru) package. @@ -193,19 +222,88 @@ Once installed, agents will use `shuru run` whenever they need sandboxed executi ## Browser Use -Create a reusable Chromium checkpoint once, then let Browser Use drive that -isolated browser through Shuru's localhost port forwarding: +The `browser` recipe installs Chromium from Debian and runs it headless as a +guest service, under an unprivileged user, with the DevTools protocol on +port 9222 inside the VM. Forward that port and point any CDP client at it: ```sh -shuru checkpoint create browser-use --allow-net -- \ - sh -lc 'apt-get update && apt-get install -y chromium' +shuru run --recipe browser -p 9222:9222 -- sleep infinity +curl -s http://127.0.0.1:9222/json/version +``` +The Browser Use example does exactly that. Chromium stays inside the +microVM and only its CDP port reaches the host: + +```sh OPENAI_API_KEY=... \ uv run --with browser-use --with openai examples/browser-use.py ``` -The example keeps Chromium inside the microVM and exposes only its CDP port to -the host. +## Recipes + +A recipe is a versioned description of a checkpoint: a `recipe.json` with +metadata and services, plus shell steps that run in fresh boots. The official +catalog lives in [superhq-ai/recipes](https://github.com/superhq-ai/recipes). + +```sh +shuru recipe list # the official catalog +shuru recipe show docker # metadata, services and every step +shuru checkpoint create docker --recipe docker # build the checkpoint under that name +shuru run --from docker -- docker ps # and start from it like any other +shuru run --recipe docker -- docker ps # or both at once: builds on first use, reuses after +shuru checkpoint create mydocker --recipe docker # the same recipe under a name of your own +shuru checkpoint update docker --recipe docker # force a rebuild at the current revision +shuru run --recipe docker --checkpoint ci -- ... # the one-liner under a name of your own +shuru run --recipe owner/repo/path@v1 --yes -- ... # third-party, pinned to a tag +shuru run --recipe ./my-recipe -- ... # local directory while developing +``` + +References resolve to an exact commit, which is printed and recorded next to +the checkpoint. `shuru checkpoint list` shows where each one came from, and a +changed recipe rebuilds on the next `shuru run --recipe`. Builds use the +network allowlist the recipe declares, never the project `shuru.json`. See +docs/rfcs/0005-recipes.md. + +## Docker + +Docker Engine runs inside the VM through the `docker` recipe. Build the +checkpoint once, then start from it; after that containers run with +networking disabled: + +```sh +shuru checkpoint create docker --recipe docker + +# Serve the pre-pulled nginx-unprivileged image on host port 8080 +shuru run --from docker -p 8080:8080 -- \ + docker run --rm -p 8080:8080 nginxinc/nginx-unprivileged:1.31.7-alpine-slim +curl http://127.0.0.1:8080/ +``` + +`shuru run --recipe docker -- ...` does both: it builds the checkpoint on +first use and reuses it afterwards. + +dockerd is a guest service: it starts when the VM boots, the command waits +until `docker info` answers, and the daemon is stopped and the disk synced +before the VM powers off. Pulling other images needs `--allow-net`, and the +usual `--allow-host` allowlist applies to container traffic too: containers +reach the internet only through the Shuru proxy, and nothing at all without +`--allow-net`. + +Any image can declare services the same way. A JSON file under +`/etc/shuru/services/` names the command to run, an optional readiness +check, and the stop timeout: + +```json +{"exec": ["dockerd"], "ready": ["docker", "info"], "stop_timeout_secs": 30} +``` + +See docs/rfcs/0004-guest-services.md for the full contract. + +Docker isolation inside the VM stays at its defaults (seccomp, no new +privileges). Do not use `--privileged` or mount the Docker socket into a +container; neither crosses the VM boundary, but there is no reason to weaken +the inner layer. See docs/rfcs/0003-docker-on-shuru.md for the full design, +limits, and verification evidence. ## Changelog diff --git a/crates/shuru-cli/Cargo.toml b/crates/shuru-cli/Cargo.toml index ab903f2..a604065 100644 --- a/crates/shuru-cli/Cargo.toml +++ b/crates/shuru-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuru-cli" -version = "0.7.0" +version = "0.11.0" edition = "2021" [[bin]] @@ -25,5 +25,5 @@ serde_json = "1" tar = "0.4.45" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } -ureq = { version = "3", features = ["json"] } +ureq = { version = "3", features = ["json", "platform-verifier"] } base64 = "0.22.1" diff --git a/crates/shuru-cli/src/assets.rs b/crates/shuru-cli/src/assets.rs index 7f04b6b..37ac48a 100644 --- a/crates/shuru-cli/src/assets.rs +++ b/crates/shuru-cli/src/assets.rs @@ -14,7 +14,7 @@ pub const CURRENT_VERSION: &str = env!("CARGO_PKG_VERSION"); pub fn assets_ready(data_dir: &str) -> bool { let kernel = format!("{}/Image", data_dir); let rootfs = format!("{}/rootfs.ext4", data_dir); - let initramfs = format!("{}/initramfs.cpio.gz", data_dir); + let initramfs = format!("{}/initramfs.cpio", data_dir); if !Path::new(&kernel).exists() || !Path::new(&rootfs).exists() @@ -52,7 +52,8 @@ fn download_os_image_version(data_dir: &str, version: &str) -> Result<()> { eprintln!("shuru: downloading OS image ({})...", tag); eprintln!("shuru: {}", url); - let response = ureq::get(&url) + let response = crate::http::agent() + .get(&url) .call() .with_context(|| format!("download failed — is version {} released?", tag))?; @@ -72,6 +73,8 @@ fn download_os_image_version(data_dir: &str, version: &str) -> Result<()> { eprintln!(); // newline after progress + shuru_vm::WorkDisk::prepare_base(&format!("{}/rootfs.ext4", data_dir))?; + // Write VERSION file let version_file = format!("{}/VERSION", data_dir); fs::write(&version_file, format!("{}\n", version)).context("failed to write VERSION file")?; @@ -102,6 +105,8 @@ pub fn upgrade(data_dir: &str) -> Result<()> { bail!("This copy was installed via Homebrew. Please run `brew upgrade shuru` instead"); } } + // Working disks lived here before 0.9.2 and could outlive a crashed run. + let _ = std::fs::remove_dir_all(format!("{}/instances", data_dir)); eprintln!("shuru: checking for updates..."); @@ -110,7 +115,8 @@ pub fn upgrade(data_dir: &str) -> Result<()> { GITHUB_REPO ); - let response = ureq::get(&api_url) + let response = crate::http::agent() + .get(&api_url) .header("Accept", "application/vnd.github+json") .header("User-Agent", "shuru") .call() @@ -145,7 +151,8 @@ pub fn upgrade(data_dir: &str) -> Result<()> { eprintln!("shuru: downloading CLI ({})...", latest); eprintln!("shuru: {}", cli_url); - let response = ureq::get(&cli_url) + let response = crate::http::agent() + .get(&cli_url) .call() .with_context(|| format!("failed to download CLI v{}", latest))?; diff --git a/crates/shuru-cli/src/checkpoint.rs b/crates/shuru-cli/src/checkpoint.rs index b44ef42..fd3e816 100644 --- a/crates/shuru-cli/src/checkpoint.rs +++ b/crates/shuru-cli/src/checkpoint.rs @@ -5,7 +5,8 @@ use anyhow::{bail, Result}; use shuru_vm::default_data_dir; use crate::cli::VmArgs; -use crate::config::load_config; +use crate::config::{load_config, ShuruConfig}; +use crate::recipe; use crate::vm; pub(crate) fn create( @@ -15,7 +16,18 @@ pub(crate) fn create( command: Vec, ) -> Result { let cfg = load_config(vm_args.config.as_deref())?; + create_with_config(name, vm_args, &cfg, from, command) +} +/// Create a checkpoint with an explicit config. Recipe builds pass an empty +/// one so the project shuru.json never reaches a build. +pub(crate) fn create_with_config( + name: String, + vm_args: &VmArgs, + cfg: &ShuruConfig, + from: Option<&str>, + command: Vec, +) -> Result { let command = if !command.is_empty() { command } else { @@ -28,30 +40,81 @@ pub(crate) fn create( let checkpoints_dir = format!("{}/checkpoints", data_dir); // Check both formats if checkpoint_exists(&checkpoints_dir, &name) { - bail!("checkpoint '{}' already exists, delete it first", name); + bail!( + "checkpoint '{}' already exists; use `shuru checkpoint update` to replace it", + name + ); } - let mut prepared = vm::prepare_vm(vm_args, &cfg, from)?; - prepared.sync_before_stop = true; - let result = vm::run_command(&prepared, &command)?; - + let prepared = vm::prepare_vm(vm_args, cfg, from)?; std::fs::create_dir_all(&checkpoints_dir)?; - eprintln!("shuru: saving checkpoint '{}'...", name); - - if let Some(ref nbd_handle) = result.nbd_handle { - let index_path = format!("{}/{}.idx", checkpoints_dir, name); - nbd_handle.save_checkpoint(&index_path)?; + let ext = if vm::uses_cas(&prepared) { + "idx" } else { - let ext4_path = format!("{}/{}.ext4", checkpoints_dir, name); - vm::clone_file(&prepared.work_rootfs, &ext4_path)?; - } + "ext4" + }; + let dest = format!("{checkpoints_dir}/{name}.{ext}"); + let result = vm::run_command(&prepared, &command, Some(&dest))?; eprintln!("shuru: checkpoint '{}' saved", name); drop(result.nbd_handle); - let _ = std::fs::remove_dir_all(&prepared.instance_dir); Ok(result.exit_code) } +/// `shuru checkpoint update NAME`: the new checkpoint is built under a +/// staging name and swapped in once the command succeeds, so a failed build +/// leaves the old checkpoint untouched. `--from NAME` layers the command on +/// top of the existing checkpoint instead of starting from the base image. +pub(crate) fn update( + name: String, + vm_args: &VmArgs, + from: Option<&str>, + command: Vec, +) -> Result { + shuru_vm::validate_checkpoint_name(&name).map_err(|e| anyhow::anyhow!(e))?; + if !exists(&name) { + return create(name, vm_args, from, command); + } + let cfg = load_config(vm_args.config.as_deref())?; + let staging = staging_name(&name); + if exists(&staging) { + delete(&staging)?; + } + let code = create_with_config(staging.clone(), vm_args, &cfg, from, command)?; + if code != 0 { + let _ = delete(&staging); + bail!( + "command exited with status {}, checkpoint '{}' unchanged", + code, + name + ); + } + replace(&name, &staging)?; + Ok(0) +} + +/// Where an update builds before it replaces `name`. +pub(crate) fn staging_name(name: &str) -> String { + format!("{}.update", name) +} + +/// Move the files of `staging` over those of `name`, in either format, +/// including the recipe sidecar. +pub(crate) fn replace(name: &str, staging: &str) -> Result<()> { + let checkpoints_dir = format!("{}/checkpoints", default_data_dir()); + for ext in ["idx", "ext4", "json"] { + let target = format!("{}/{}.{}", checkpoints_dir, name, ext); + let source = format!("{}/{}.{}", checkpoints_dir, staging, ext); + if std::path::Path::new(&source).exists() { + shuru_vm::check_checkpoint_format(&target)?; + std::fs::rename(&source, &target)?; + } + } + std::fs::File::open(&checkpoints_dir)?.sync_all()?; + eprintln!("shuru: checkpoint '{}' updated", name); + Ok(()) +} + pub(crate) fn list() -> Result<()> { let data_dir = default_data_dir(); let checkpoints_dir = format!("{}/checkpoints", data_dir); @@ -101,8 +164,17 @@ pub(crate) fn list() -> Result<()> { checkpoints.sort_by_key(|(_, _, t, _)| *t); - println!("{:<20} {:>10} {}", "NAME", "SIZE", "CREATED"); + println!("NAME SIZE CREATED RECIPE"); for (name, size, mtime, is_cas) in &checkpoints { + let origin = recipe::read_meta(name) + .map(|m| { + format!( + "{}@{}", + m.recipe.source, + &m.recipe.revision[..m.recipe.revision.len().min(7)] + ) + }) + .unwrap_or_default(); let size_str = if *is_cas { if *size >= 1024 * 1024 { format!("{} MB (cas)", size / (1024 * 1024)) @@ -127,12 +199,18 @@ pub(crate) fn list() -> Result<()> { } else { format!("{}d ago", elapsed / 86400) }; - println!("{:<20} {:>10} {}", name, size_str, age); + println!("{:<20} {:>10} {:<12} {}", name, size_str, age, origin); } Ok(()) } +/// Whether a checkpoint exists in either format. +pub(crate) fn exists(name: &str) -> bool { + let checkpoints_dir = format!("{}/checkpoints", default_data_dir()); + checkpoint_exists(&checkpoints_dir, name) +} + pub(crate) fn delete(name: &str) -> Result<()> { shuru_vm::validate_checkpoint_name(name).map_err(|e| anyhow::anyhow!(e))?; @@ -150,6 +228,7 @@ pub(crate) fn delete(name: &str) -> Result<()> { } else { bail!("Checkpoint '{}' not found", name); } + recipe::remove_meta(name); eprintln!("shuru: checkpoint '{}' deleted", name); Ok(()) diff --git a/crates/shuru-cli/src/cli.rs b/crates/shuru-cli/src/cli.rs index 7f6ee19..f4e78b6 100644 --- a/crates/shuru-cli/src/cli.rs +++ b/crates/shuru-cli/src/cli.rs @@ -12,7 +12,7 @@ pub(crate) struct VmArgs { #[arg(long)] pub memory: Option, - /// Disk size in MB (default: 4096) + /// Disk size in MB (minimum and default: 4096) #[arg(long)] pub disk_size: Option, @@ -24,7 +24,7 @@ pub(crate) struct VmArgs { #[arg(long, env = "SHURU_ROOTFS")] pub rootfs: Option, - /// Path to initramfs (for loading VirtIO modules) + /// Path to the uncompressed initramfs containing the guest agent #[arg(long, env = "SHURU_INITRD")] pub initrd: Option, @@ -87,6 +87,19 @@ pub(crate) enum Commands { #[arg(long)] from: Option, + /// Build a checkpoint from a recipe on first use and start from it + /// (NAME, OWNER/REPO[/PATH][@REF], or a local directory) + #[arg(long, conflicts_with = "from", value_name = "REF")] + recipe: Option, + + /// Build a third-party recipe without the confirmation prompt + #[arg(long)] + yes: bool, + + /// Name of the checkpoint --recipe builds and reuses (default: the recipe name) + #[arg(long, requires = "recipe", value_name = "NAME")] + checkpoint: Option, + /// Attach to raw serial console instead of running a command #[arg(long)] console: bool, @@ -116,10 +129,31 @@ pub(crate) enum Commands { action: CheckpointCommands, }, - /// Remove leftover instance data from crashed VMs - Prune, + /// Browse and inspect recipes (RFC 0005) + Recipe { + #[command(subcommand)] + action: RecipeCommands, + }, } +#[derive(clap::Subcommand)] +pub(crate) enum RecipeCommands { + /// List the official recipes from github.com/superhq-ai/recipes + List { + /// Ignore the cached index + #[arg(long)] + refresh: bool, + }, + + /// Print a recipe, its services and every build step + Show { + /// NAME, OWNER/REPO[/PATH][@REF], or a local directory + recipe: String, + }, +} + +// Parsed once; the flattened VmArgs in Create dwarfs the other variants. +#[allow(clippy::large_enum_variant)] #[derive(clap::Subcommand)] pub(crate) enum CheckpointCommands { /// Run a command and save the resulting disk state as a checkpoint @@ -134,6 +168,42 @@ pub(crate) enum CheckpointCommands { #[arg(long)] from: Option, + /// Build from a recipe instead of running a command + /// (NAME, OWNER/REPO[/PATH][@REF], or a local directory) + #[arg(long, conflicts_with = "from", value_name = "REF")] + recipe: Option, + + /// Build a third-party recipe without the confirmation prompt + #[arg(long)] + yes: bool, + + /// Command and arguments to run inside the VM + #[arg(trailing_var_arg = true, allow_hyphen_values = true)] + command: Vec, + }, + + /// Rebuild a checkpoint in place: like delete then create, except the + /// old one stays until the new build succeeds + Update { + /// Checkpoint name + name: String, + + #[command(flatten)] + vm: VmArgs, + + /// Start from an existing checkpoint; NAME itself layers the command on top + #[arg(long)] + from: Option, + + /// Build from a recipe instead of running a command + /// (NAME, OWNER/REPO[/PATH][@REF], or a local directory) + #[arg(long, conflicts_with = "from", value_name = "REF")] + recipe: Option, + + /// Build a third-party recipe without the confirmation prompt + #[arg(long)] + yes: bool, + /// Command and arguments to run inside the VM #[arg(trailing_var_arg = true, allow_hyphen_values = true)] command: Vec, diff --git a/crates/shuru-cli/src/config.rs b/crates/shuru-cli/src/config.rs index 8aa7cbe..4990b80 100644 --- a/crates/shuru-cli/src/config.rs +++ b/crates/shuru-cli/src/config.rs @@ -118,6 +118,9 @@ impl ShuruConfig { /// Reject a malformed config up front, so `shuru run` fails the same way /// whether or not networking is enabled. pub fn validate(&self) -> Result<()> { + if let Some(size) = self.disk_size { + shuru_vm::validate_disk_size(size)?; + } if let Some(ref secrets) = self.secrets { for (name, entry) in secrets { entry.to_secret_config(name)?; diff --git a/crates/shuru-cli/src/http.rs b/crates/shuru-cli/src/http.rs new file mode 100644 index 0000000..bf47a3a --- /dev/null +++ b/crates/shuru-cli/src/http.rs @@ -0,0 +1,27 @@ +//! The HTTP client behind every download the CLI makes. +//! +//! Trust comes from the platform store rather than a bundled root list, so a +//! TLS interceptor whose root is installed on the machine works the same way +//! it does for curl and git: the keychain on macOS, the system store plus +//! SSL_CERT_FILE and SSL_CERT_DIR on Linux. + +use std::sync::OnceLock; + +use ureq::tls::{RootCerts, TlsConfig}; +use ureq::Agent; + +pub fn agent() -> Agent { + static AGENT: OnceLock = OnceLock::new(); + AGENT + .get_or_init(|| { + Agent::config_builder() + .tls_config( + TlsConfig::builder() + .root_certs(RootCerts::PlatformVerifier) + .build(), + ) + .build() + .new_agent() + }) + .clone() +} diff --git a/crates/shuru-cli/src/main.rs b/crates/shuru-cli/src/main.rs index 6f75c36..0d07121 100644 --- a/crates/shuru-cli/src/main.rs +++ b/crates/shuru-cli/src/main.rs @@ -2,6 +2,8 @@ mod assets; mod checkpoint; mod cli; mod config; +mod http; +mod recipe; mod stdio; mod vm; @@ -12,7 +14,7 @@ use clap::Parser; use shuru_vm::{default_data_dir, VmState}; -use cli::{CheckpointCommands, Cli, Commands}; +use cli::{CheckpointCommands, Cli, Commands, RecipeCommands}; use config::load_config; fn main() -> Result<()> { @@ -29,12 +31,26 @@ fn main() -> Result<()> { Commands::Run { vm, from, + recipe, + yes, + checkpoint, console, stdio, command, } => { let cfg = load_config(vm.config.as_deref())?; + // A recipe is built (once) before the run, with its own policy. + let from = match recipe { + Some(ref reference) => Some(recipe::ensure_checkpoint( + reference, + &vm, + yes, + checkpoint.as_deref(), + )?), + None => from, + }; + // Command resolution: CLI args > config > default /bin/sh let command = if !command.is_empty() { command @@ -51,10 +67,9 @@ fn main() -> Result<()> { } else if console { run_console(&prepared) } else { - vm::run_command(&prepared, &command).map(|r| r.exit_code) + vm::run_command(&prepared, &command, None).map(|r| r.exit_code) }; - let _ = std::fs::remove_dir_all(&prepared.instance_dir); process::exit(result?); } Commands::Init { force } => { @@ -75,49 +90,43 @@ fn main() -> Result<()> { let data_dir = default_data_dir(); assets::upgrade(&data_dir)?; } - Commands::Prune => { - let data_dir = default_data_dir(); - let instances_dir = format!("{}/instances", data_dir); - let entries = match std::fs::read_dir(&instances_dir) { - Ok(entries) => entries, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - eprintln!("shuru: no orphaned instances found"); - return Ok(()); - } - Err(e) => return Err(e.into()), - }; - - let mut removed = 0u32; - for entry in entries { - let entry = entry?; - let name = entry.file_name(); - let Some(pid) = name.to_str().and_then(|s| s.parse::().ok()) else { - continue; - }; - // Check if the process is still running - let alive = unsafe { libc::kill(pid, 0) } == 0; - if !alive { - std::fs::remove_dir_all(entry.path())?; - removed += 1; - } - } - - if removed == 0 { - eprintln!("shuru: no orphaned instances found"); - } else { - eprintln!("shuru: removed {} orphaned instance(s)", removed); - } - } Commands::Checkpoint { action } => match action { CheckpointCommands::Create { name, vm, from, + recipe, + yes, command, } => { + if let Some(ref reference) = recipe { + if !command.is_empty() { + anyhow::bail!("--recipe builds from the recipe steps; drop the command"); + } + recipe::create_checkpoint(&name, reference, &vm, yes)?; + return Ok(()); + } let exit_code = checkpoint::create(name, &vm, from.as_deref(), command)?; process::exit(exit_code); } + CheckpointCommands::Update { + name, + vm, + from, + recipe, + yes, + command, + } => { + if let Some(ref reference) = recipe { + if !command.is_empty() { + anyhow::bail!("--recipe builds from the recipe steps; drop the command"); + } + recipe::update_checkpoint(&name, reference, &vm, yes)?; + return Ok(()); + } + let exit_code = checkpoint::update(name, &vm, from.as_deref(), command)?; + process::exit(exit_code); + } CheckpointCommands::List => checkpoint::list()?, CheckpointCommands::Delete { name } => checkpoint::delete(&name)?, CheckpointCommands::Push { name: _ } => { @@ -127,6 +136,10 @@ fn main() -> Result<()> { anyhow::bail!("checkpoint pull is not yet implemented") } }, + Commands::Recipe { action } => match action { + RecipeCommands::List { refresh } => recipe::list(refresh)?, + RecipeCommands::Show { recipe: reference } => recipe::show(&reference)?, + }, } Ok(()) @@ -135,7 +148,11 @@ fn main() -> Result<()> { /// Run the VM in raw serial console mode (for debugging). fn run_console(prepared: &vm::PreparedVm) -> Result { eprintln!("shuru: kernel={}", prepared.kernel_path); - eprintln!("shuru: rootfs={} (work copy)", prepared.work_rootfs); + eprintln!( + "shuru: rootfs={} (anonymous clone of {})", + prepared.work_disk.attach_path(), + prepared.source_rootfs + ); eprintln!( "shuru: booting VM ({}cpus, {}MB RAM, {}MB disk)...", prepared.cpus, prepared.memory, prepared.disk_size diff --git a/crates/shuru-cli/src/recipe.rs b/crates/shuru-cli/src/recipe.rs new file mode 100644 index 0000000..8ec3657 --- /dev/null +++ b/crates/shuru-cli/src/recipe.rs @@ -0,0 +1,733 @@ +//! Recipes (RFC 0005): a directory with recipe.json and one or more step +//! scripts, fetched from GitHub at an exact commit or read from a local +//! path, and built into a checkpoint through the normal checkpoint flow. + +use std::collections::BTreeMap; +use std::io::{IsTerminal, Write}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime}; + +use anyhow::{bail, Context, Result}; +use base64::Engine; +use serde::{Deserialize, Serialize}; + +use shuru_vm::default_data_dir; + +use crate::assets; +use crate::checkpoint; +use crate::cli::VmArgs; +use crate::config::ShuruConfig; + +pub(crate) const OFFICIAL_OWNER: &str = "superhq-ai"; +pub(crate) const OFFICIAL_REPO: &str = "recipes"; +const INDEX_TTL: Duration = Duration::from_secs(24 * 3600); +const USER_AGENT: &str = concat!("shuru/", env!("CARGO_PKG_VERSION")); + +#[derive(Debug, Clone, Deserialize)] +pub(crate) struct RecipeSpec { + pub name: String, + #[serde(default)] + pub description: String, + #[serde(default)] + pub min_shuru_version: Option, + #[serde(default)] + pub from: Option, + #[serde(default)] + pub cpus: Option, + #[serde(default)] + pub memory: Option, + #[serde(default)] + pub disk_size: Option, + #[serde(default)] + pub network: Option, + #[serde(default)] + pub services: BTreeMap, + #[serde(default = "default_steps")] + pub steps: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +pub(crate) struct RecipeNetwork { + #[serde(default)] + pub allow: Vec, +} + +fn default_steps() -> Vec { + vec!["build.sh".to_string()] +} + +/// Where a recipe reference points. +#[derive(Debug, Clone)] +enum Source { + Local(PathBuf), + GitHub { + owner: String, + repo: String, + path: String, + reference: String, + }, +} + +/// A recipe with its step scripts loaded and its revision fixed. +pub(crate) struct Recipe { + pub spec: RecipeSpec, + /// (file name, contents) in run order. + pub steps: Vec<(String, String)>, + /// Human readable origin, e.g. `superhq-ai/recipes/docker` or a path. + pub source: String, + /// Commit SHA for GitHub, content hash for local directories. + pub revision: String, + /// Official catalog or local path: no confirmation needed. + pub trusted: bool, +} + +impl Recipe { + pub fn short_revision(&self) -> &str { + &self.revision[..self.revision.len().min(7)] + } +} + +/// Sidecar written next to a checkpoint built from a recipe. +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct CheckpointMeta { + pub recipe: RecipeMeta, +} + +#[derive(Debug, Serialize, Deserialize)] +pub(crate) struct RecipeMeta { + pub name: String, + pub source: String, + pub revision: String, +} + +#[derive(Debug, Deserialize)] +struct IndexEntry { + name: String, + #[serde(default)] + description: String, +} + +// --- Reference parsing --- + +/// Only an explicit path form is local. A bare name is never looked up in +/// the working directory: a cloned repository must not be able to shadow +/// an official recipe and inherit its trust. +fn parse_reference(reference: &str) -> Result { + let looks_local = reference.starts_with("./") + || reference.starts_with("../") + || reference.starts_with('/') + || reference.starts_with("~/") + || reference == "." + || reference == ".."; + if looks_local { + let path = if let Some(rest) = reference.strip_prefix("~/") { + let home = std::env::var("HOME").context("HOME is not set")?; + PathBuf::from(home).join(rest) + } else { + PathBuf::from(reference) + }; + return Ok(Source::Local(path)); + } + + let (locator, git_ref) = match reference.rsplit_once('@') { + Some((l, r)) if !r.is_empty() && !r.contains('/') => (l, r.to_string()), + _ => (reference, "HEAD".to_string()), + }; + let parts: Vec<&str> = locator.split('/').collect(); + if parts.iter().any(|p| p.is_empty() || *p == "..") { + bail!("invalid recipe reference: {}", reference); + } + let (owner, repo, path) = match parts.len() { + 1 => { + if Path::new(parts[0]).join("recipe.json").is_file() { + eprintln!( + "shuru: note: ./{} has a recipe.json, but a bare name means the official catalog; pass ./{} to build the local one", + parts[0], parts[0] + ); + } + ( + OFFICIAL_OWNER.to_string(), + OFFICIAL_REPO.to_string(), + parts[0].to_string(), + ) + } + 2 => (parts[0].to_string(), parts[1].to_string(), String::new()), + _ => ( + parts[0].to_string(), + parts[1].to_string(), + parts[2..].join("/"), + ), + }; + Ok(Source::GitHub { + owner, + repo, + path, + reference: git_ref, + }) +} + +// --- Fetching --- + +fn http_get(url: &str, accept: &str) -> Result { + let mut req = crate::http::agent() + .get(url) + .header("User-Agent", USER_AGENT) + .header("Accept", accept); + if url.starts_with("https://api.github.com/") { + if let Ok(token) = std::env::var("GITHUB_TOKEN").or_else(|_| std::env::var("GH_TOKEN")) { + if !token.is_empty() { + req = req.header("Authorization", &format!("Bearer {}", token)); + } + } + } + let mut resp = req.call().with_context(|| format!("GET {} failed", url))?; + resp.body_mut() + .read_to_string() + .with_context(|| format!("reading {}", url)) +} + +fn resolve_sha(owner: &str, repo: &str, reference: &str) -> Result { + let url = format!( + "https://api.github.com/repos/{}/{}/commits/{}", + owner, repo, reference + ); + let sha = http_get(&url, "application/vnd.github.sha") + .with_context(|| format!("resolving {}/{}@{}", owner, repo, reference))?; + let sha = sha.trim().to_string(); + if sha.len() != 40 || !sha.chars().all(|c| c.is_ascii_hexdigit()) { + bail!( + "unexpected commit id from GitHub for {}/{}@{}: {}", + owner, + repo, + reference, + sha + ); + } + Ok(sha) +} + +fn raw_url(owner: &str, repo: &str, sha: &str, path: &str, file: &str) -> String { + if path.is_empty() { + format!( + "https://raw.githubusercontent.com/{}/{}/{}/{}", + owner, repo, sha, file + ) + } else { + format!( + "https://raw.githubusercontent.com/{}/{}/{}/{}/{}", + owner, repo, sha, path, file + ) + } +} + +fn parse_spec(text: &str, origin: &str) -> Result { + let spec: RecipeSpec = + serde_json::from_str(text).with_context(|| format!("invalid recipe.json in {}", origin))?; + validate_spec(&spec, origin)?; + Ok(spec) +} + +fn valid_name(name: &str, allow_dots: bool) -> bool { + let mut chars = name.chars(); + match chars.next() { + Some(c) if c.is_ascii_lowercase() || c.is_ascii_digit() => {} + _ => return false, + } + chars.all(|c| { + c.is_ascii_lowercase() + || c.is_ascii_digit() + || c == '-' + || c == '_' + || (allow_dots && c == '.') + }) +} + +fn validate_spec(spec: &RecipeSpec, origin: &str) -> Result<()> { + if let Some(size) = spec.disk_size { + shuru_vm::validate_disk_size(size)?; + } + if !valid_name(&spec.name, true) { + bail!( + "{}: recipe name {:?} must match [a-z0-9][a-z0-9._-]*", + origin, + spec.name + ); + } + if spec.steps.is_empty() { + bail!("{}: steps must list at least one script", origin); + } + for step in &spec.steps { + if step.contains('/') || step.contains("..") || step.is_empty() { + bail!( + "{}: step {:?} must be a file name in the recipe directory", + origin, + step + ); + } + } + for (name, value) in &spec.services { + if !valid_name(name, false) { + bail!( + "{}: service name {:?} must match [a-z0-9][a-z0-9-]*", + origin, + name + ); + } + if !value.is_object() { + bail!("{}: service {} must be a JSON object", origin, name); + } + } + if let Some(ref from) = spec.from { + shuru_vm::validate_checkpoint_name(from) + .map_err(|e| anyhow::anyhow!("{}: {}", origin, e))?; + } + if let Some(ref min) = spec.min_shuru_version { + let need = parse_version(min) + .with_context(|| format!("{}: invalid min_shuru_version {:?}", origin, min))?; + let have = parse_version(assets::CURRENT_VERSION)?; + if have < need { + bail!( + "{}: recipe needs shuru {} or newer, this is {}. Run `shuru upgrade`.", + origin, + min, + assets::CURRENT_VERSION + ); + } + } + Ok(()) +} + +fn parse_version(v: &str) -> Result<(u64, u64, u64)> { + let core = v.trim().trim_start_matches('v'); + let core = core.split(['-', '+']).next().unwrap_or(core); + let mut parts = core.split('.').map(|p| p.parse::()); + let major = parts.next().transpose()?.context("missing major version")?; + let minor = parts.next().transpose()?.unwrap_or(0); + let patch = parts.next().transpose()?.unwrap_or(0); + Ok((major, minor, patch)) +} + +fn load_local(dir: &Path) -> Result { + let dir = dir + .canonicalize() + .with_context(|| format!("recipe directory {} not found", dir.display()))?; + let origin = dir.display().to_string(); + let spec_text = std::fs::read_to_string(dir.join("recipe.json")) + .with_context(|| format!("reading {}/recipe.json", origin))?; + let spec = parse_spec(&spec_text, &origin)?; + let mut hasher = blake3::Hasher::new(); + hasher.update(spec_text.as_bytes()); + let mut steps = Vec::new(); + for step in &spec.steps { + let text = std::fs::read_to_string(dir.join(step)) + .with_context(|| format!("reading step {} in {}", step, origin))?; + hasher.update(step.as_bytes()); + hasher.update(text.as_bytes()); + steps.push((step.clone(), text)); + } + Ok(Recipe { + spec, + steps, + source: origin, + revision: hasher.finalize().to_hex().to_string(), + trusted: true, + }) +} + +fn load_github(owner: &str, repo: &str, path: &str, reference: &str) -> Result { + let sha = resolve_sha(owner, repo, reference)?; + let origin = if path.is_empty() { + format!("{}/{}", owner, repo) + } else { + format!("{}/{}/{}", owner, repo, path) + }; + let spec_text = http_get( + &raw_url(owner, repo, &sha, path, "recipe.json"), + "text/plain", + ) + .with_context(|| format!("fetching recipe.json for {}@{}", origin, &sha[..7]))?; + let spec = parse_spec(&spec_text, &origin)?; + let mut steps = Vec::new(); + for step in &spec.steps { + let text = http_get(&raw_url(owner, repo, &sha, path, step), "text/plain") + .with_context(|| format!("fetching step {} for {}@{}", step, origin, &sha[..7]))?; + steps.push((step.clone(), text)); + } + Ok(Recipe { + spec, + steps, + source: origin, + revision: sha, + trusted: owner == OFFICIAL_OWNER && repo == OFFICIAL_REPO, + }) +} + +/// Resolve a reference to a recipe with its scripts and exact revision. +pub(crate) fn resolve(reference: &str) -> Result { + match parse_reference(reference)? { + Source::Local(dir) => load_local(&dir), + Source::GitHub { + owner, + repo, + path, + reference, + } => load_github(&owner, &repo, &path, &reference), + } +} + +// --- Display and trust --- + +fn print_summary(recipe: &Recipe, out: &mut impl Write) -> std::io::Result<()> { + writeln!(out, "name: {}", recipe.spec.name)?; + if !recipe.spec.description.is_empty() { + writeln!(out, "description: {}", recipe.spec.description)?; + } + writeln!( + out, + "source: {}@{}", + recipe.source, + recipe.short_revision() + )?; + if let Some(ref from) = recipe.spec.from { + writeln!(out, "from: checkpoint {}", from)?; + } + match &recipe.spec.network { + None => writeln!(out, "network: offline during build")?, + Some(n) if n.allow.is_empty() => writeln!(out, "network: unrestricted during build")?, + Some(n) => writeln!(out, "network: {}", n.allow.join(", "))?, + } + if !recipe.spec.services.is_empty() { + let names: Vec<&str> = recipe.spec.services.keys().map(String::as_str).collect(); + writeln!(out, "services: {}", names.join(", "))?; + } + let steps: Vec = recipe + .steps + .iter() + .map(|(n, t)| format!("{} ({} bytes)", n, t.len())) + .collect(); + writeln!(out, "steps: {}", steps.join(", "))?; + Ok(()) +} + +/// Official catalog and local directories build without asking. Anything +/// else prints its provenance and needs a yes, interactive or by flag. +fn confirm(recipe: &Recipe, yes: bool) -> Result<()> { + if recipe.trusted { + return Ok(()); + } + let mut err = std::io::stderr(); + writeln!(err, "shuru: third-party recipe")?; + print_summary(recipe, &mut err)?; + if yes { + return Ok(()); + } + if !std::io::stdin().is_terminal() { + bail!( + "refusing to build third-party recipe {} without confirmation; pass --yes", + recipe.source + ); + } + write!(err, "Build it? [y/N] ")?; + err.flush()?; + let mut answer = String::new(); + std::io::stdin().read_line(&mut answer)?; + if !matches!(answer.trim(), "y" | "Y" | "yes") { + bail!("aborted"); + } + Ok(()) +} + +// --- Commands --- + +pub(crate) fn show(reference: &str) -> Result<()> { + let recipe = resolve(reference)?; + let mut out = std::io::stdout(); + print_summary(&recipe, &mut out)?; + if !recipe.spec.services.is_empty() { + writeln!(out)?; + for (name, value) in &recipe.spec.services { + writeln!(out, "--- /etc/shuru/services/{}.json ---", name)?; + writeln!(out, "{}", serde_json::to_string_pretty(value)?)?; + } + } + for (name, text) in &recipe.steps { + writeln!(out)?; + writeln!(out, "--- {} ---", name)?; + write!(out, "{}", text)?; + if !text.ends_with('\n') { + writeln!(out)?; + } + } + Ok(()) +} + +fn index_cache_path() -> String { + format!("{}/recipes-index.json", default_data_dir()) +} + +fn fetch_index(refresh: bool) -> Result> { + let cache = index_cache_path(); + if !refresh { + if let Ok(meta) = std::fs::metadata(&cache) { + let fresh = meta + .modified() + .ok() + .and_then(|m| SystemTime::now().duration_since(m).ok()) + .map(|age| age < INDEX_TTL) + .unwrap_or(false); + if fresh { + if let Ok(text) = std::fs::read_to_string(&cache) { + if let Ok(entries) = serde_json::from_str::>(&text) { + return Ok(entries); + } + } + } + } + } + let url = format!( + "https://raw.githubusercontent.com/{}/{}/HEAD/index.json", + OFFICIAL_OWNER, OFFICIAL_REPO + ); + let text = http_get(&url, "application/json").context("fetching the recipe index")?; + let entries: Vec = + serde_json::from_str(&text).context("parsing the recipe index")?; + if let Some(parent) = Path::new(&cache).parent() { + let _ = std::fs::create_dir_all(parent); + } + let _ = std::fs::write(&cache, &text); + Ok(entries) +} + +pub(crate) fn list(refresh: bool) -> Result<()> { + let entries = fetch_index(refresh)?; + if entries.is_empty() { + eprintln!("No recipes published yet."); + return Ok(()); + } + let width = entries + .iter() + .map(|e| e.name.len()) + .max() + .unwrap_or(4) + .max(4); + println!("{: String { + base64::engine::general_purpose::STANDARD.encode(s.as_bytes()) +} + +/// The guest command for one step: write service files on the first step, +/// then run the script from tmpfs so it never lands in the checkpoint. +fn step_command(recipe: &Recipe, index: usize, script: &str) -> Result> { + let mut payload = String::from("set -e\n"); + if index == 0 && !recipe.spec.services.is_empty() { + payload.push_str("mkdir -p /etc/shuru/services\n"); + for (name, value) in &recipe.spec.services { + let json = serde_json::to_string_pretty(value)?; + payload.push_str(&format!( + "echo {} | base64 -d > /etc/shuru/services/{}.json\n", + b64(&json), + name + )); + } + } + payload.push_str(&format!( + "echo {} | base64 -d > /tmp/shuru-recipe-step.sh\n", + b64(script) + )); + payload.push_str("exec sh /tmp/shuru-recipe-step.sh\n"); + Ok(vec![ + "sh".to_string(), + "-c".to_string(), + format!("echo {} | base64 -d | sh", b64(&payload)), + ]) +} + +/// VM settings for a build: the recipe decides size and network, the user +/// may enlarge the VM, and nothing from the project config or the user +/// mounts, ports and secrets reaches the build. +fn build_args(vm: &VmArgs, spec: &RecipeSpec) -> VmArgs { + VmArgs { + cpus: vm.cpus.or(spec.cpus), + memory: vm.memory.or(spec.memory), + disk_size: vm.disk_size.or(spec.disk_size), + kernel: vm.kernel.clone(), + rootfs: vm.rootfs.clone(), + initrd: vm.initrd.clone(), + allow_net: spec.network.is_some(), + dns_resolver: vm.dns_resolver, + allow_host_writes: false, + port: Vec::new(), + mount: Vec::new(), + secret: Vec::new(), + allow_host: spec + .network + .as_ref() + .map(|n| n.allow.clone()) + .unwrap_or_default(), + expose_host: Vec::new(), + config: None, + verbose: vm.verbose, + } +} + +fn meta_path(name: &str) -> String { + format!("{}/checkpoints/{}.json", default_data_dir(), name) +} + +pub(crate) fn read_meta(name: &str) -> Option { + let text = std::fs::read_to_string(meta_path(name)).ok()?; + serde_json::from_str(&text).ok() +} + +pub(crate) fn remove_meta(name: &str) { + let _ = std::fs::remove_file(meta_path(name)); +} + +/// Build `name` from the recipe, one checkpoint per step. +pub(crate) fn build(name: &str, recipe: &Recipe, vm: &VmArgs) -> Result<()> { + shuru_vm::validate_checkpoint_name(name).map_err(|e| anyhow::anyhow!(e))?; + let total = recipe.steps.len(); + eprintln!( + "shuru: building recipe {} from {}@{} ({} step{})", + recipe.spec.name, + recipe.source, + recipe.short_revision(), + total, + if total == 1 { "" } else { "s" } + ); + let args = build_args(vm, &recipe.spec); + let cfg = ShuruConfig::default(); + let mut from = recipe.spec.from.clone(); + let mut intermediates: Vec = Vec::new(); + + for (i, (step_name, script)) in recipe.steps.iter().enumerate() { + let last = i + 1 == total; + let target = if last { + name.to_string() + } else { + format!("{}-step{}", name, i + 1) + }; + eprintln!("shuru: step {}/{}: {}", i + 1, total, step_name); + let command = step_command(recipe, i, script)?; + let code = + checkpoint::create_with_config(target.clone(), &args, &cfg, from.as_deref(), command)?; + if code != 0 { + let _ = checkpoint::delete(&target); + if let Some(kept) = intermediates.last() { + eprintln!( + "shuru: intermediate checkpoint {} kept for debugging; delete it with `shuru checkpoint delete {}`", + kept, kept + ); + } + bail!("recipe step {} exited with status {}", step_name, code); + } + if !last { + intermediates.push(target.clone()); + } + from = Some(target); + } + + for t in &intermediates { + checkpoint::delete(t)?; + } + + let meta = CheckpointMeta { + recipe: RecipeMeta { + name: recipe.spec.name.clone(), + source: recipe.source.clone(), + revision: recipe.revision.clone(), + }, + }; + std::fs::write(meta_path(name), serde_json::to_string_pretty(&meta)?) + .context("writing checkpoint metadata")?; + Ok(()) +} + +/// `shuru checkpoint create NAME --recipe REF`. +pub(crate) fn create_checkpoint(name: &str, reference: &str, vm: &VmArgs, yes: bool) -> Result<()> { + let recipe = resolve(reference)?; + confirm(&recipe, yes)?; + if checkpoint::exists(name) { + bail!( + "checkpoint {} already exists; use `shuru checkpoint update` to replace it", + name + ); + } + build(name, &recipe, vm) +} + +/// `shuru checkpoint update NAME --recipe REF`: rebuild even when the +/// revision matches. The old checkpoint stays until the build succeeds. +pub(crate) fn update_checkpoint(name: &str, reference: &str, vm: &VmArgs, yes: bool) -> Result<()> { + let recipe = resolve(reference)?; + confirm(&recipe, yes)?; + if !checkpoint::exists(name) { + return build(name, &recipe, vm); + } + let staging = checkpoint::staging_name(name); + if checkpoint::exists(&staging) { + checkpoint::delete(&staging)?; + } + build(&staging, &recipe, vm)?; + checkpoint::replace(name, &staging) +} + +/// `shuru run --recipe REF [--checkpoint NAME]`: make sure a checkpoint +/// named after the recipe, or as given, exists at the resolved revision, and +/// return its name. +pub(crate) fn ensure_checkpoint( + reference: &str, + vm: &VmArgs, + yes: bool, + name: Option<&str>, +) -> Result { + let recipe = resolve(reference)?; + confirm(&recipe, yes)?; + let name = name + .map(str::to_string) + .unwrap_or_else(|| recipe.spec.name.clone()); + if checkpoint::exists(&name) { + match read_meta(&name) { + Some(meta) if meta.recipe.revision == recipe.revision => { + eprintln!( + "shuru: using checkpoint {} ({}@{})", + name, + meta.recipe.source, + recipe.short_revision() + ); + return Ok(name); + } + Some(meta) => { + eprintln!( + "shuru: recipe changed ({}@{} -> {}@{}), rebuilding checkpoint {}", + meta.recipe.source, + &meta.recipe.revision[..meta.recipe.revision.len().min(7)], + recipe.source, + recipe.short_revision(), + name + ); + checkpoint::delete(&name)?; + } + None => bail!( + "checkpoint {} exists but was not built from a recipe; delete it or run with --from {}", + name, + name + ), + } + } + build(&name, &recipe, vm)?; + Ok(name) +} diff --git a/crates/shuru-cli/src/stdio.rs b/crates/shuru-cli/src/stdio.rs index bbf1efd..e2c17f6 100644 --- a/crates/shuru-cli/src/stdio.rs +++ b/crates/shuru-cli/src/stdio.rs @@ -1,12 +1,14 @@ use std::collections::HashMap; use std::io::{self, BufRead, Write}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; +use std::time::Duration; use anyhow::Result; use base64::Engine; use serde::{Deserialize, Serialize}; -use shuru_proto::{frame, WatchEvent}; +use shuru_proto::{frame, watch_kind, WatchEvent}; use shuru_vm::Sandbox; use crate::vm::{self, PreparedVm}; @@ -21,6 +23,7 @@ mod method { pub const WATCH: &str = "watch"; pub const READ_FILE: &str = "read_file"; pub const WRITE_FILE: &str = "write_file"; + pub const TRIM: &str = "trim"; pub const CHECKPOINT: &str = "checkpoint"; pub const MKDIR: &str = "mkdir"; pub const READ_DIR: &str = "read_dir"; @@ -37,6 +40,16 @@ const METHOD_NOT_FOUND: i32 = -32601; const INVALID_PARAMS: i32 = -32602; const SERVER_ERROR: i32 = -32000; +const EVENT_QUEUE: usize = 64; + +/// Stdin a client may have queued for one process before writes fail. The +/// SDK waits for each write to be acknowledged, so it stays well under this. +const STDIN_BACKLOG: usize = 4 * 1024 * 1024; + +/// How long a killed process gets to exit before its connection is closed, +/// which makes the guest kill it outright. +const KILL_GRACE: Duration = Duration::from_secs(2); + // --- JSON-RPC 2.0 types --- #[derive(Deserialize)] @@ -260,12 +273,55 @@ enum Event { // --- Process handle for stdin/kill --- enum ProcessInput { - Stdin(Vec), + Stdin { + data: Vec, + id: serde_json::Value, + }, Kill, } struct ProcessHandle { input_tx: std::sync::mpsc::Sender, + state: Arc, +} + +#[derive(Default)] +struct ProcessState { + /// Stdin bytes accepted from the client and not yet written to the guest. + backlog: AtomicUsize, + killed: AtomicBool, + exited: AtomicBool, + exec_id: Mutex>, +} + +impl ProcessState { + /// SIGKILL the process if it outlives the grace period. A guest blocked + /// writing stdin to a process that never reads it cannot see the KILL + /// frame queued behind that stdin, so this goes on its own connection. + fn kill_after_grace(self: Arc, sandbox: Arc) { + std::thread::spawn(move || { + std::thread::sleep(KILL_GRACE); + if self.exited.load(Ordering::SeqCst) { + return; + } + if let Some(id) = *self.exec_id.lock().unwrap() { + if let Err(e) = sandbox.kill_exec(id) { + eprintln!("shuru: kill failed: {e}"); + } + } + }); + } +} + +/// Reply to a request that carried an id; notifications get no reply. +fn reply_input(out: &SharedWriter, id: serde_json::Value, result: Result<(), String>) { + if id.is_null() { + return; + } + let _ = match result { + Ok(()) => send_result_shared(out, id, EmptyResult {}), + Err(msg) => send_error_shared(out, id, SERVER_ERROR, msg), + }; } // --- Shared writer --- @@ -358,8 +414,9 @@ pub(crate) fn run_stdio(prepared: &PreparedVm) -> Result { None }; - // Event channel: background threads -> main loop - let (event_tx, event_rx) = std::sync::mpsc::channel::(); + // Event channel: background threads -> main loop. Bounded, so a client + // that stops reading stalls the guest instead of growing this process. + let (event_tx, event_rx) = std::sync::mpsc::sync_channel::(EVENT_QUEUE); // Send ready notification send_json_shared( @@ -480,20 +537,34 @@ pub(crate) fn run_stdio(prepared: &PreparedVm) -> Result { pid_counter += 1; let pid = format!("p{}", pid_counter); - // Channel for sending stdin/kill to the process + // Channel for sending stdin/kill to the process. Its bytes + // are bounded by STDIN_BACKLOG, checked before each send. let (input_tx, input_rx) = std::sync::mpsc::channel::(); - processes.insert(pid.clone(), ProcessHandle { input_tx }); + let state = Arc::new(ProcessState::default()); + // Dropping a finished process ends its input thread. + processes.retain(|_, h| !h.state.exited.load(Ordering::SeqCst)); + processes.insert( + pid.clone(), + ProcessHandle { + input_tx, + state: state.clone(), + }, + ); let sb = sandbox.clone(); let tx = event_tx.clone(); + let out_for_input = out.clone(); let pid_clone = pid.clone(); let mut spawn_env = secret_env.clone(); - spawn_env.extend(params.env.into_iter()); + spawn_env.extend(params.env); bg_threads.push(std::thread::spawn(move || { let argv: Vec<&str> = params.argv.iter().map(|s| s.as_str()).collect(); let stream = match sb.open_exec(&argv, &spawn_env, params.cwd.as_deref()) { - Ok(s) => s, + Ok((s, id)) => { + *state.exec_id.lock().unwrap() = Some(id); + s + } Err(e) => { eprintln!("shuru: spawn failed: {}", e); let _ = tx.send(Event::Exit { @@ -514,19 +585,41 @@ pub(crate) fn run_stdio(prepared: &PreparedVm) -> Result { // Returns vsock_writer so it isn't dropped before the read // loop finishes — on Apple vsock, dropping the original fd // closes the entire connection even if a dup'd reader exists. + let input_state = state.clone(); let input_thread = std::thread::spawn(move || -> std::net::TcpStream { + let state = input_state; + let mut closed: Option<&str> = None; for msg in input_rx { match msg { - ProcessInput::Stdin(data) => { - if frame::write_frame(&mut vsock_writer, frame::STDIN, &data) - .is_err() - { - break; + ProcessInput::Stdin { data, id } => { + state.backlog.fetch_sub(data.len(), Ordering::SeqCst); + if state.killed.load(Ordering::SeqCst) { + closed = Some("process was killed"); } + let result = match closed { + Some(why) => Err(why.to_string()), + None => data + .chunks(frame::FILE_CHUNK) + .try_for_each(|chunk| { + frame::write_frame( + &mut vsock_writer, + frame::STDIN, + chunk, + ) + }) + .map_err(|e| { + closed = Some("process stdin is closed"); + format!("writing stdin: {e}") + }), + }; + reply_input(&out_for_input, id, result); } ProcessInput::Kill => { - let _ = frame::write_frame(&mut vsock_writer, frame::KILL, &[]); - break; + if closed.is_none() { + let _ = + frame::write_frame(&mut vsock_writer, frame::KILL, &[]); + } + closed = Some("process was killed"); } } } @@ -580,6 +673,7 @@ pub(crate) fn run_stdio(prepared: &PreparedVm) -> Result { } } } + state.exited.store(true, Ordering::SeqCst); let _ = input_thread.join(); })); @@ -601,20 +695,55 @@ pub(crate) fn run_stdio(prepared: &PreparedVm) -> Result { } }; if let Some(handle) = processes.get(¶ms.pid) { - let _ = handle.input_tx.send(ProcessInput::Kill); + if !handle.state.killed.swap(true, Ordering::SeqCst) { + let _ = handle.input_tx.send(ProcessInput::Kill); + handle.state.clone().kill_after_grace(sandbox.clone()); + } } send_result_shared(&out, req.id, EmptyResult {})?; } method::INPUT => { - // Fire-and-forget notification (may have no id) + // A request is answered once the data reaches the guest; a + // notification (no id) gets no answer and no backpressure. + let id = req.id; let params: InputParams = match serde_json::from_value(req.params) { Ok(p) => p, - Err(_) => continue, + Err(e) => { + reply_input(&out, id, Err(format!("invalid params: {e}"))); + continue; + } }; - if let Some(handle) = processes.get(¶ms.pid) { - if let Ok(data) = BASE64.decode(¶ms.data) { - let _ = handle.input_tx.send(ProcessInput::Stdin(data)); + let Some(handle) = processes.get(¶ms.pid) else { + reply_input(&out, id, Err(format!("no process {}", params.pid))); + continue; + }; + let data = match BASE64.decode(¶ms.data) { + Ok(d) => d, + Err(e) => { + reply_input(&out, id, Err(format!("invalid base64: {e}"))); + continue; + } + }; + let len = data.len(); + let backlog = &handle.state.backlog; + if backlog.load(Ordering::SeqCst) + len > STDIN_BACKLOG { + let msg = format!( + "stdin backlog for {} is over {} MiB; wait for earlier writes", + params.pid, + STDIN_BACKLOG >> 20 + ); + if id.is_null() { + eprintln!("shuru: {msg}, dropped {len} bytes"); + } + reply_input(&out, id, Err(msg)); + continue; + } + backlog.fetch_add(len, Ordering::SeqCst); + if let Err(e) = handle.input_tx.send(ProcessInput::Stdin { data, id }) { + if let ProcessInput::Stdin { id, .. } = e.0 { + backlog.fetch_sub(len, Ordering::SeqCst); + reply_input(&out, id, Err("process has exited".into())); } } } @@ -647,17 +776,16 @@ pub(crate) fn run_stdio(prepared: &PreparedVm) -> Result { } }; let mut reader = stream; - loop { - match frame::read_frame(&mut reader) { - Ok(Some((frame::WATCH_EVENT, data))) => { - if let Some(evt) = WatchEvent::decode(&data) { - let _ = tx.send(Event::FileChange { - path: evt.path, - event: evt.kind.to_string(), - }); - } - } - _ => break, + while let Ok(Some((frame::WATCH_EVENT, data))) = frame::read_frame(&mut reader) + { + let Some(evt) = WatchEvent::decode(&data) else { + continue; + }; + if let Some(name) = watch_kind::name(evt.kind) { + let _ = tx.send(Event::FileChange { + path: evt.path, + event: name.to_string(), + }); } } })); @@ -697,6 +825,10 @@ pub(crate) fn run_stdio(prepared: &PreparedVm) -> Result { handle_write_file(&sandbox, req.id, ¶ms.path, ¶ms.content, &out)?; } + method::TRIM => match sandbox.trim() { + Ok(response) => send_result_shared(&out, req.id, response)?, + Err(e) => send_error_shared(&out, req.id, SERVER_ERROR, e.to_string())?, + }, method::CHECKPOINT => { let params: CheckpointParams = match serde_json::from_value(req.params) { Ok(p) => p, @@ -710,7 +842,14 @@ pub(crate) fn run_stdio(prepared: &PreparedVm) -> Result { continue; } }; - handle_checkpoint(&sandbox, prepared, req.id, ¶ms.name, &out)?; + handle_checkpoint( + &sandbox, + prepared, + nbd_handle.as_ref(), + req.id, + ¶ms.name, + &out, + )?; let _ = sandbox.stop(); drop(event_tx); let _ = event_thread.join(); @@ -889,7 +1028,8 @@ pub(crate) fn run_stdio(prepared: &PreparedVm) -> Result { // any background threads stuck on read_frame(). let _ = sandbox.stop(); - // Wait briefly for background threads to notice and exit + // Input threads run until their process entry goes away. + drop(processes); for thread in bg_threads { let _ = thread.join(); } @@ -906,13 +1046,10 @@ fn handle_exec( env: &HashMap, out: &SharedWriter, ) -> Result<()> { - let mut stdout_buf = Vec::new(); - let mut stderr_buf = Vec::new(); - - let exit_code = match sandbox.exec_with_env(argv, env, &mut stdout_buf, &mut stderr_buf) { - Ok(code) => code, + let output = match sandbox.exec_buffered(argv, env) { + Ok(output) => output, Err(e) => { - return send_error_shared(out, id, SERVER_ERROR, format!("exec failed: {}", e)); + return send_error_shared(out, id, SERVER_ERROR, format!("exec failed: {:#}", e)); } }; @@ -920,9 +1057,9 @@ fn handle_exec( out, id, ExecResultPayload { - stdout: String::from_utf8_lossy(&stdout_buf).into_owned(), - stderr: String::from_utf8_lossy(&stderr_buf).into_owned(), - exit_code, + stdout: String::from_utf8_lossy(&output.stdout).into_owned(), + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + exit_code: output.exit_code, }, ) } @@ -968,28 +1105,19 @@ fn handle_write_file( fn handle_checkpoint( sandbox: &Sandbox, prepared: &PreparedVm, + nbd: Option<&shuru_store::NbdHandle>, id: serde_json::Value, name: &str, out: &SharedWriter, ) -> Result<()> { - let mut discard_out = Vec::new(); - let mut discard_err = Vec::new(); - if let Err(e) = sandbox.exec(&["sync"], &mut discard_out, &mut discard_err) { - return send_error_shared( - out, - id, - SERVER_ERROR, - format!("checkpoint sync failed: {}", e), - ); - } - if let Err(msg) = shuru_vm::validate_checkpoint_name(name) { return send_error_shared(out, id, SERVER_ERROR, msg); } let data_dir = shuru_vm::default_data_dir(); let checkpoints_dir = format!("{}/checkpoints", data_dir); - let checkpoint_path = format!("{}/{}.ext4", checkpoints_dir, name); + let ext = if nbd.is_some() { "idx" } else { "ext4" }; + let checkpoint_path = format!("{checkpoints_dir}/{name}.{ext}"); if let Err(e) = std::fs::create_dir_all(&checkpoints_dir) { return send_error_shared( @@ -1000,7 +1128,13 @@ fn handle_checkpoint( ); } - if let Err(e) = vm::clone_file(&prepared.work_rootfs, &checkpoint_path) { + if let Err(e) = sandbox.save_disk(&checkpoint_path, shuru_vm::AfterSave::PowerOff, |path| { + if let Some(handle) = nbd { + handle.save_checkpoint(path) + } else { + prepared.work_disk.save_to(path) + } + }) { return send_error_shared( out, id, diff --git a/crates/shuru-cli/src/vm.rs b/crates/shuru-cli/src/vm.rs index a6d2431..267a2ed 100644 --- a/crates/shuru-cli/src/vm.rs +++ b/crates/shuru-cli/src/vm.rs @@ -1,37 +1,8 @@ use std::collections::HashMap; -use std::ffi::CString; use std::io::IsTerminal; use anyhow::{bail, Context, Result}; -#[cfg(target_os = "macos")] -extern "C" { - fn clonefile( - src: *const libc::c_char, - dst: *const libc::c_char, - flags: u32, - ) -> libc::c_int; -} - -#[cfg(target_os = "macos")] -pub(crate) fn clone_file(src: &str, dst: &str) -> Result<()> { - let c_src = CString::new(src).context("invalid source path")?; - let c_dst = CString::new(dst).context("invalid destination path")?; - let ret = unsafe { clonefile(c_src.as_ptr(), c_dst.as_ptr(), 0) }; - if ret != 0 { - let err = std::io::Error::last_os_error(); - bail!("clonefile({} -> {}) failed: {}", src, dst, err); - } - Ok(()) -} - -#[cfg(target_os = "linux")] -pub(crate) fn clone_file(src: &str, dst: &str) -> Result<()> { - std::fs::copy(src, dst) - .with_context(|| format!("failed to copy {} -> {}", src, dst))?; - Ok(()) -} - use shuru_vm::{MountConfig, PortMapping, Sandbox}; use crate::assets; @@ -53,9 +24,10 @@ fn trace_boot(label: &str) { } pub(crate) struct PreparedVm { - pub instance_dir: String, pub source_rootfs: String, - pub work_rootfs: String, + /// Anonymous clone of the source, alive for the run and freed by the + /// kernel whenever the process ends. + pub work_disk: shuru_vm::WorkDisk, /// If restoring from a CAS checkpoint, the index path. pub cas_index: Option, pub kernel_path: String, @@ -64,24 +36,17 @@ pub(crate) struct PreparedVm { pub memory: u64, pub disk_size: u64, pub proxy_config: Option, - /// Flush guest page cache to disk before stopping. Set for checkpoint - /// creation, where the disk is cloned after the hard stop and unsynced - /// writes would silently vanish from the saved image. - pub sync_before_stop: bool, pub verbose: bool, pub forwards: Vec, pub mounts: Vec, } -pub(crate) fn prepare_vm( - vm: &VmArgs, - cfg: &ShuruConfig, - from: Option<&str>, -) -> Result { +pub(crate) fn prepare_vm(vm: &VmArgs, cfg: &ShuruConfig, from: Option<&str>) -> Result { trace_boot("prepare_vm start"); let cpus = vm.cpus.or(cfg.cpus).unwrap_or(2); let memory = vm.memory.or(cfg.memory).unwrap_or(2048); let disk_size = vm.disk_size.or(cfg.disk_size).unwrap_or(4096); + shuru_vm::validate_disk_size(disk_size)?; let allow_net = vm.allow_net || cfg.allow_net.unwrap_or(false); let allow_host_writes = vm.allow_host_writes || cfg.allow_host_writes.unwrap_or(false); let verbose = vm.verbose; @@ -95,8 +60,9 @@ pub(crate) fn prepare_vm( // Merge --secret flags: NAME=ENV_VAR@host1,host2 for s in &vm.secret { - let (name, from, hosts) = parse_secret_flag(s) - .with_context(|| format!("invalid --secret: '{}' (expected NAME=ENV@host1,host2)", s))?; + let (name, from, hosts) = parse_secret_flag(s).with_context(|| { + format!("invalid --secret: '{}' (expected NAME=ENV@host1,host2)", s) + })?; proxy.secrets.insert( name, shuru_proxy::config::SecretConfig::from_env(from, hosts), @@ -129,8 +95,8 @@ pub(crate) fn prepare_vm( } let mut forwards = Vec::new(); for s in &port_strs { - let mapping = parse_port_mapping(s) - .with_context(|| format!("invalid port mapping: '{}'", s))?; + let mapping = + parse_port_mapping(s).with_context(|| format!("invalid port mapping: '{}'", s))?; forwards.push(mapping); } @@ -143,8 +109,7 @@ pub(crate) fn prepare_vm( } let mut mounts = Vec::new(); for s in &mount_strs { - let mc = parse_mount_spec(s) - .with_context(|| format!("invalid mount spec: '{}'", s))?; + let mc = parse_mount_spec(s).with_context(|| format!("invalid mount spec: '{}'", s))?; mounts.push(mc); } @@ -174,7 +139,7 @@ pub(crate) fn prepare_vm( let initrd_path_str = vm .initrd .clone() - .unwrap_or_else(|| format!("{}/initramfs.cpio.gz", data_dir)); + .unwrap_or_else(|| format!("{}/initramfs.cpio", data_dir)); if !std::path::Path::new(&kernel_path).exists() { bail!( @@ -188,8 +153,7 @@ pub(crate) fn prepare_vm( let mut cas_index: Option = None; let source = match from { Some(name) => { - shuru_vm::validate_checkpoint_name(name) - .map_err(|e| anyhow::anyhow!(e))?; + shuru_vm::validate_checkpoint_name(name).map_err(|e| anyhow::anyhow!(e))?; // Check .idx (CAS) first, then .ext4 (legacy) let idx_path = format!("{}/{}.idx", checkpoints_dir, name); let ext4_path = format!("{}/{}.ext4", checkpoints_dir, name); @@ -214,60 +178,32 @@ pub(crate) fn prepare_vm( }; trace_boot("prepare: config + assets checked"); - // Create per-instance working copy (clean any stale dir from PID reuse) - let instance_dir = format!("{}/instances/{}", data_dir, std::process::id()); - let _ = std::fs::remove_dir_all(&instance_dir); - std::fs::create_dir_all(&instance_dir)?; - let work_rootfs = format!("{}/rootfs.ext4", instance_dir); - trace_boot("prepare: instance dir ready"); - - // CAS checkpoints don't need a file copy — the NBD server reads from the chunk store. - // We still need a rootfs file for the VM builder (kernel cmdline root=), but it can be - // a dummy for CAS mode since I/O goes through NBD. - if cas_index.is_none() { + // A CAS checkpoint is served over NBD, so its disk is only a placeholder. + if from.is_none() && vm.rootfs.is_none() { + if let Err(e) = shuru_vm::WorkDisk::prepare_base(&source) { + eprintln!("shuru: note: {:#}", e); + } + } + let work_disk = if cas_index.is_none() { if verbose { eprintln!("shuru: creating working copy..."); } - clone_file(&source, &work_rootfs)?; + shuru_vm::WorkDisk::clone_from(&source, &data_dir, disk_size)? } else { - // Create a minimal placeholder so the VM builder doesn't fail - std::fs::File::create(&work_rootfs)?; - } + shuru_vm::WorkDisk::empty(&data_dir)? + }; trace_boot("prepare: rootfs cloned"); - // Extend to requested disk size - let f = std::fs::OpenOptions::new() - .write(true) - .open(&work_rootfs)?; - let target = disk_size * 1024 * 1024; - let current = f.metadata()?.len(); - if target < current { - bail!( - "--disk-size {}MB is smaller than the base image ({}MB)", - disk_size, - current / 1024 / 1024 - ); - } - if target > current { - f.set_len(target)?; - } - drop(f); - let initrd_path = if std::path::Path::new(&initrd_path_str).exists() { Some(initrd_path_str) } else { - eprintln!( - "shuru: warning: initramfs not found at {}, booting without it", - initrd_path_str - ); - None + bail!("required initramfs not found at {}", initrd_path_str); }; trace_boot("prepare_vm done (disk cloned + extended)"); Ok(PreparedVm { - instance_dir, source_rootfs: source, - work_rootfs, + work_disk, cas_index, kernel_path, initrd_path, @@ -275,7 +211,6 @@ pub(crate) fn prepare_vm( memory, disk_size, proxy_config, - sync_before_stop: false, verbose, forwards, mounts, @@ -290,7 +225,7 @@ pub(crate) fn build_sandbox( ) -> Result { let mut builder = Sandbox::builder() .kernel(&prepared.kernel_path) - .rootfs(&prepared.work_rootfs) + .rootfs(prepared.work_disk.attach_path()) .cpus(prepared.cpus) .memory_mb(prepared.memory) .console(console) @@ -319,15 +254,24 @@ pub(crate) fn build_sandbox( /// default; CAS is opt-in via SHURU_STORAGE=cas. Booting from a CAS /// checkpoint always uses NBD since the chunk store is the only place its /// content exists. +pub(crate) fn uses_cas(prepared: &PreparedVm) -> bool { + prepared.cas_index.is_some() + || matches!(std::env::var("SHURU_STORAGE").as_deref(), Ok("cas" | "nbd")) +} + pub(crate) fn start_nbd(prepared: &PreparedVm) -> Result> { - let storage = std::env::var("SHURU_STORAGE").unwrap_or_default(); - let want_cas = storage == "cas" || storage == "nbd"; - if prepared.cas_index.is_none() && !want_cas { + if !uses_cas(prepared) { return Ok(None); } - let socket_path = format!("{}/nbd.sock", prepared.instance_dir); let data_dir = shuru_vm::default_data_dir(); let cas_dir = format!("{}/cas", data_dir); + std::fs::create_dir_all(&cas_dir)?; + let socket_path = format!( + "{}/nbd-{}-{}.sock", + cas_dir, + std::process::id(), + prepared.work_disk.as_raw_fd() + ); let index_path = if let Some(ref idx) = prepared.cas_index { idx.clone() } else { @@ -349,10 +293,18 @@ pub(crate) struct RunResult { pub nbd_handle: Option, } -pub(crate) fn run_command(prepared: &PreparedVm, command: &[String]) -> Result { +pub(crate) fn run_command( + prepared: &PreparedVm, + command: &[String], + checkpoint: Option<&str>, +) -> Result { if prepared.verbose { eprintln!("shuru: kernel={}", prepared.kernel_path); - eprintln!("shuru: rootfs={} (work copy)", prepared.work_rootfs); + eprintln!( + "shuru: rootfs={} (anonymous clone of {})", + prepared.work_disk.attach_path(), + prepared.source_rootfs + ); } eprintln!( "shuru: booting VM ({}cpus, {}MB RAM, {}MB disk)...", @@ -424,17 +376,37 @@ pub(crate) fn run_command(prepared: &PreparedVm, command: &[String]) -> Result report_services(&report, prepared.verbose), + Err(e) => eprintln!("shuru: guest shutdown failed, powering off: {}", e), + } + trace_boot("guest shut down"); + sandbox.kill()?; } - drop(proxy_handle); - let _ = sandbox.stop(); - Ok(RunResult { exit_code, nbd_handle }) + Ok(RunResult { + exit_code, + nbd_handle, + }) } /// Pure string validation — no filesystem access. Separated from `parse_mount_spec` @@ -472,9 +444,22 @@ fn parse_mount_spec(s: &str) -> Result { parse_mount_parts(&host_path, guest, mode) } +fn report_services(report: &shuru_proto::ShutdownResponse, verbose: bool) { + if verbose && !report.stopped.is_empty() { + eprintln!("shuru: stopped services: {}", report.stopped.join(", ")); + } + if !report.killed.is_empty() { + eprintln!( + "shuru: services ignored SIGTERM and were killed: {}", + report.killed.join(", ") + ); + } +} + fn validate_mounts(mounts: &[MountConfig], allow_host_writes: bool) -> Result<()> { let cwd = std::env::current_dir().context("failed to determine current working directory")?; - let cwd = std::fs::canonicalize(&cwd).context("failed to canonicalize current working directory")?; + let cwd = + std::fs::canonicalize(&cwd).context("failed to canonicalize current working directory")?; validate_mounts_with_cwd(mounts, allow_host_writes, &cwd) } @@ -627,7 +612,9 @@ mod tests { read_only: true, }]; let err = validate_mounts_with_cwd(&mounts, false, cwd).unwrap_err(); - assert!(err.to_string().contains("outside the current working directory")); + assert!(err + .to_string() + .contains("outside the current working directory")); } #[test] diff --git a/crates/shuru-cli/tests/checkpoint_update.rs b/crates/shuru-cli/tests/checkpoint_update.rs new file mode 100644 index 0000000..088e04b --- /dev/null +++ b/crates/shuru-cli/tests/checkpoint_update.rs @@ -0,0 +1,152 @@ +//! `shuru checkpoint update` (issue #44). Every test boots real VMs, so all +//! are #[ignore]d. Run with: +//! +//! SHURU_BIN=target/debug/shuru cargo test -p shuru-cli --test checkpoint_update -- --ignored +//! +//! SHURU_BIN must be a codesigned binary (see tests/docker.rs). SHURU_DATA_DIR, +//! SHURU_KERNEL and SHURU_INITRD are inherited by the CLI. + +use std::process::{Command, Output, Stdio}; + +fn shuru_bin() -> String { + std::env::var("SHURU_BIN") + .expect("SHURU_BIN not set: point it at a codesigned shuru binary (e.g. just build)") +} + +fn text(out: &Output) -> String { + format!( + "--- stdout ---\n{}\n--- stderr ---\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ) +} + +fn stdout(out: &Output) -> String { + String::from_utf8_lossy(&out.stdout).to_string() +} + +fn stderr(out: &Output) -> String { + String::from_utf8_lossy(&out.stderr).to_string() +} + +fn shuru(args: &[&str]) -> Output { + let out = Command::new(shuru_bin()) + .args(args) + .stdin(Stdio::null()) + .output() + .expect("failed to spawn shuru"); + eprintln!("$ shuru {}\n{}", args.join(" "), text(&out)); + out +} + +fn unique(prefix: &str) -> String { + format!("{}-{}", prefix, std::process::id()) +} + +struct Cleanup(Vec); + +impl Drop for Cleanup { + fn drop(&mut self) { + for name in &self.0 { + let _ = Command::new(shuru_bin()) + .args(["checkpoint", "delete", name]) + .output(); + } + } +} + +fn files_in(name: &str) -> String { + let out = shuru(&["run", "--from", name, "--", "ls", "/opt/m"]); + assert!(out.status.success(), "{}", text(&out)); + stdout(&out).trim().to_string() +} + +#[test] +#[ignore] +fn update_creates_replaces_layers_and_keeps_the_old_one_on_failure() { + let name = unique("shuru-test-update"); + let _cleanup = Cleanup(vec![name.clone(), format!("{}.update", name)]); + + // Missing checkpoint: update behaves like create. + let out = shuru(&[ + "checkpoint", + "update", + &name, + "--", + "sh", + "-c", + "mkdir -p /opt/m && touch /opt/m/one", + ]); + assert!(out.status.success(), "{}", text(&out)); + assert!( + stderr(&out).contains(&format!("checkpoint '{}' saved", name)), + "{}", + text(&out) + ); + assert_eq!(files_in(&name), "one"); + + // Plain update starts from the base image again. + let out = shuru(&[ + "checkpoint", + "update", + &name, + "--", + "sh", + "-c", + "mkdir -p /opt/m && touch /opt/m/two", + ]); + assert!(out.status.success(), "{}", text(&out)); + assert!( + stderr(&out).contains(&format!("checkpoint '{}' updated", name)), + "{}", + text(&out) + ); + assert_eq!(files_in(&name), "two"); + + // --from NAME layers the command on top of the existing checkpoint. + let out = shuru(&[ + "checkpoint", + "update", + &name, + "--from", + &name, + "--", + "touch", + "/opt/m/three", + ]); + assert!(out.status.success(), "{}", text(&out)); + assert_eq!(files_in(&name), "three\ntwo"); + + // A failing command leaves the old checkpoint in place and no staging copy. + let out = shuru(&[ + "checkpoint", + "update", + &name, + "--from", + &name, + "--", + "sh", + "-c", + "touch /opt/m/four; exit 3", + ]); + assert!(!out.status.success(), "{}", text(&out)); + let e = stderr(&out); + assert!(e.contains("exited with status 3"), "{}", e); + assert!( + e.contains(&format!("checkpoint '{}' unchanged", name)), + "{}", + e + ); + assert_eq!(files_in(&name), "three\ntwo"); + let list = stdout(&shuru(&["checkpoint", "list"])); + assert!(!list.contains(&format!("{}.update", name)), "{}", list); + + // create still refuses to overwrite and points at update. + let out = shuru(&["checkpoint", "create", &name, "--", "true"]); + assert!(!out.status.success(), "{}", text(&out)); + assert!( + stderr(&out).contains("shuru checkpoint update"), + "{}", + text(&out) + ); +} diff --git a/crates/shuru-cli/tests/docker.rs b/crates/shuru-cli/tests/docker.rs new file mode 100644 index 0000000..a7e99a0 --- /dev/null +++ b/crates/shuru-cli/tests/docker.rs @@ -0,0 +1,464 @@ +//! Docker Engine inside a Shuru VM (RFC 0003). Boots real VMs from the +//! checkpoint built by the docker recipe (`shuru checkpoint create docker +//! --recipe docker`, or `--recipe /docker` from a local clone of +//! superhq-ai/recipes), so every test is #[ignore]d. Run with: +//! +//! SHURU_BIN=target/debug/shuru cargo test -p shuru-cli --test docker -- --ignored +//! +//! Environment: +//! SHURU_BIN codesigned shuru binary (required). cargo test +//! relinks target/debug/shuru and drops its +//! signature, so sign a copy elsewhere and point +//! SHURU_BIN at that. +//! SHURU_DOCKER_CHECKPOINT checkpoint name (default "docker") +//! SHURU_DOCKER_IMAGE image reference baked into the checkpoint +//! SHURU_TEST_NET=1 also run the tests that need real egress +//! +//! SHURU_DATA_DIR, SHURU_KERNEL and SHURU_INITRD are inherited by the CLI, +//! so a run can use an isolated kernel and checkpoint directory. +//! +//! dockerd runs as a guest service (RFC 0004): it is up before the command +//! starts and is stopped cleanly before the VM powers off, so the scripts +//! below call `docker` directly. + +use std::io::{Read, Write}; +use std::net::{TcpListener, TcpStream}; +use std::process::{Command, Output, Stdio}; +use std::time::{Duration, Instant}; + +use base64::Engine; +const DEFAULT_IMAGE: &str = "nginxinc/nginx-unprivileged:1.31.7-alpine-slim@sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7"; + +fn shuru_bin() -> String { + std::env::var("SHURU_BIN") + .expect("SHURU_BIN not set: point it at a codesigned shuru binary (e.g. just build)") +} + +fn checkpoint() -> String { + std::env::var("SHURU_DOCKER_CHECKPOINT").unwrap_or_else(|_| "docker".to_string()) +} + +fn image() -> String { + std::env::var("SHURU_DOCKER_IMAGE").unwrap_or_else(|_| DEFAULT_IMAGE.to_string()) +} + +fn net_tests_enabled() -> bool { + std::env::var("SHURU_TEST_NET") + .map(|v| v == "1") + .unwrap_or(false) +} + +/// Build the guest command: the script is base64 encoded so no quoting +/// survives the trip through argv into `sh -c`. +fn guest_cmd(script: &str) -> Vec { + let script = script.replace("IMAGE", &image()); + let b64 = base64::engine::general_purpose::STANDARD.encode(script.as_bytes()); + vec![ + "sh".into(), + "-c".into(), + format!("echo {} | base64 -d | sh", b64), + ] +} + +/// Runs the script in a fresh VM and echoes the output so a `--nocapture` +/// run doubles as an evidence log. +fn shuru_run(extra: &[&str], script: &str) -> Output { + let mut cmd = Command::new(shuru_bin()); + cmd.args(["run", "--from", &checkpoint()]); + cmd.args(extra); + cmd.arg("--"); + cmd.args(guest_cmd(script)); + cmd.stdin(Stdio::null()); + let out = cmd.output().expect("failed to spawn shuru"); + eprintln!( + "$ shuru run --from {} {}\n{}", + checkpoint(), + extra.join(" "), + text(&out) + ); + out +} + +fn text(out: &Output) -> String { + format!( + "--- stdout ---\n{}\n--- stderr ---\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ) +} + +fn stdout(out: &Output) -> String { + String::from_utf8_lossy(&out.stdout).to_string() +} + +fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port() +} + +fn http_get(port: u16, path: &str) -> Option { + let mut s = TcpStream::connect_timeout( + &format!("127.0.0.1:{}", port).parse().unwrap(), + Duration::from_secs(2), + ) + .ok()?; + s.set_read_timeout(Some(Duration::from_secs(5))).ok()?; + write!( + s, + "GET {} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n", + path + ) + .ok()?; + let mut body = String::new(); + s.read_to_string(&mut body).ok()?; + Some(body) +} + +#[test] +#[ignore] +fn dockerd_runs_pinned_image_as_non_root_with_default_isolation() { + let out = shuru_run( + &[], + r#" +set -e +docker info --format "{{.CgroupDriver}} cgroup-v{{.CgroupVersion}} storage={{.Driver}} kernel={{.KernelVersion}} security={{.SecurityOptions}}" +docker version --format "docker={{.Server.Version}} containerd={{(index .Server.Components 1).Version}} runc={{(index .Server.Components 2).Version}}" +docker image inspect --format "digest={{index .RepoDigests 0}} user={{.Config.User}}" IMAGE +echo "uid=$(docker run --rm IMAGE id -u)" +docker run -d --name web IMAGE > /dev/null +sleep 1 +PID=$(docker inspect --format "{{.State.Pid}}" web) +echo "nginx-master-$(grep ^Uid: /proc/$PID/status | tr -s "\t" " ")" +echo "privileged=$(docker inspect --format "{{.HostConfig.Privileged}}" web)" +echo "seccomp=$(docker inspect --format "{{.HostConfig.SecurityOpt}}" web)" +echo "no-new-privs=$(docker exec web grep NoNewPrivs /proc/1/status)" +echo "seccomp-mode=$(docker exec web grep Seccomp: /proc/1/status)" +docker rm -f web > /dev/null +"#, + ); + assert!(out.status.success(), "{}", text(&out)); + let s = stdout(&out); + assert!(s.contains("cgroup-v2"), "{}", s); + assert!(s.contains("storage=overlayfs"), "{}", s); + assert!(s.contains("name=seccomp,profile=builtin"), "{}", s); + assert!(s.contains("name=no-new-privileges"), "{}", s); + assert!( + s.contains("digest=nginxinc/nginx-unprivileged@sha256:a8cb0f33"), + "{}", + s + ); + assert!(s.contains("user=101"), "{}", s); + assert!(s.contains("\nuid=101\n"), "{}", s); + assert!(s.contains("nginx-master-Uid: 101 101 101 101"), "{}", s); + assert!(s.contains("privileged=false"), "{}", s); + assert!(s.contains("seccomp=[]"), "{}", s); + assert!(s.contains("NoNewPrivs:\t1"), "{}", s); + assert!(s.contains("Seccomp:\t2"), "{}", s); +} + +#[test] +#[ignore] +fn http_through_port_forward_reaches_container() { + let port = free_port(); + let token = format!("shuru-docker-{}", std::process::id()); + let script = format!( + r#" +set -e +mkdir -p /srv/www +echo "{token}" > /srv/www/index.html +docker run -d --name web -p 8080:8080 -v /srv/www:/usr/share/nginx/html:ro IMAGE > /dev/null +echo READY +i=0 +until docker logs web 2>&1 | grep -q "GET /done"; do + sleep 0.2 + i=$((i + 1)) + if [ "$i" -ge 300 ]; then echo TIMEOUT; break; fi +done +docker rm -f web > /dev/null +echo STOPPED +"# + ); + + let child = Command::new(shuru_bin()) + .args([ + "run", + "--from", + &checkpoint(), + "-p", + &format!("{}:8080", port), + "--", + ]) + .args(guest_cmd(&script)) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("failed to spawn shuru"); + + let deadline = Instant::now() + Duration::from_secs(90); + let mut body = None; + while Instant::now() < deadline { + if let Some(b) = http_get(port, "/") { + if b.contains(&token) { + body = Some(b); + break; + } + } + std::thread::sleep(Duration::from_millis(300)); + } + let _ = http_get(port, "/done"); + let out = child.wait_with_output().unwrap(); + eprintln!( + "$ shuru run --from {} -p {}:8080\n{}", + checkpoint(), + port, + text(&out) + ); + if let Some(b) = &body { + eprintln!("$ GET http://127.0.0.1:{}/\n{}", port, b); + } + + let body = body.unwrap_or_else(|| { + panic!( + "no HTTP response with token on host port {}\n{}", + port, + text(&out) + ) + }); + assert!(body.starts_with("HTTP/1.1 200"), "{}", body); + assert!(body.contains("Server: nginx"), "{}", body); + assert!(stdout(&out).contains("STOPPED"), "{}", text(&out)); + assert!(!stdout(&out).contains("TIMEOUT"), "{}", text(&out)); +} + +#[test] +#[ignore] +fn container_has_no_egress_without_allow_net() { + let out = shuru_run( + &[], + r#" +echo "guest-eth0=$(ip -o link show eth0 2>&1 | head -c 40)" +docker run --rm IMAGE sh -c "wget -T 5 -qO- http://example.com && echo EGRESS_OK || echo EGRESS_BLOCKED" +docker run --rm IMAGE sh -c "nslookup example.com 2>&1 | tail -n 1; wget -T 5 -qO- http://93.184.215.14 && echo IP_OK || echo IP_BLOCKED" +"#, + ); + let s = stdout(&out); + assert!( + s.contains("guest-eth0=Device \"eth0\" does not exist"), + "{}", + text(&out) + ); + assert!(s.contains("EGRESS_BLOCKED"), "{}", text(&out)); + assert!(s.contains("IP_BLOCKED"), "{}", text(&out)); + assert!(!s.contains("EGRESS_OK"), "{}", text(&out)); +} + +#[test] +#[ignore] +fn container_egress_follows_host_allowlist() { + if !net_tests_enabled() { + eprintln!("skipped: set SHURU_TEST_NET=1"); + return; + } + let out = shuru_run( + &["--allow-net", "--allow-host", "example.com"], + r#" +docker run --rm IMAGE sh -c "wget -T 15 -qO- http://example.com | grep -o \"[^<]*\" && echo ALLOWED_OK || echo ALLOWED_FAIL" +docker run --rm IMAGE sh -c "wget -T 5 -qO- http://www.iana.org/ > /dev/null 2>&1 && echo DOMAIN_LEAK || echo DOMAIN_BLOCKED" +docker run --rm IMAGE sh -c "wget -T 5 -qO- http://1.1.1.1/ > /dev/null 2>&1 && echo IP_LEAK || echo IP_BLOCKED" +docker run --rm --dns 8.8.8.8 IMAGE sh -c "wget -T 5 -qO- http://example.com > /dev/null 2>&1 && echo DNS_LEAK || echo DNS_BLOCKED" +"#, + ); + let s = stdout(&out); + assert!(s.contains("Example Domain"), "{}", text(&out)); + assert!(s.contains("ALLOWED_OK"), "{}", text(&out)); + assert!(s.contains("DOMAIN_BLOCKED"), "{}", text(&out)); + assert!(s.contains("IP_BLOCKED"), "{}", text(&out)); + assert!(s.contains("DNS_BLOCKED"), "{}", text(&out)); + assert!(!s.contains("LEAK"), "{}", text(&out)); +} + +/// A listener on the host loopback must be unreachable from a container, +/// with and without networking. The gateway address and host.shuru.internal +/// are the only names a guest could try. +#[test] +#[ignore] +fn container_cannot_reach_host_loopback_services() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + listener.set_nonblocking(true).unwrap(); + + let script = format!( + r#" +docker run --rm IMAGE sh -c "wget -T 5 -qO- http://10.0.0.1:{port}/ > /dev/null 2>&1 && echo GATEWAY_REACHED || echo GATEWAY_BLOCKED" +docker run --rm IMAGE sh -c "wget -T 5 -qO- http://host.shuru.internal:{port}/ > /dev/null 2>&1 && echo HOSTNAME_REACHED || echo HOSTNAME_BLOCKED" +"# + ); + + for extra in [&[][..], &["--allow-net"][..]] { + let out = shuru_run(extra, &script); + let s = stdout(&out); + assert!(s.contains("GATEWAY_BLOCKED"), "{:?}\n{}", extra, text(&out)); + assert!( + s.contains("HOSTNAME_BLOCKED"), + "{:?}\n{}", + extra, + text(&out) + ); + assert!(!s.contains("REACHED"), "{:?}\n{}", extra, text(&out)); + } + assert!( + listener.accept().is_err(), + "host listener on {} received a connection from the sandbox", + port + ); +} + +#[test] +#[ignore] +fn container_cannot_write_to_host_through_default_mount() { + // Mounts must live under the CWD, which cargo sets to the crate dir. + let tmp = tempfile::tempdir_in(std::env::current_dir().unwrap()).unwrap(); + std::fs::write(tmp.path().join("host.txt"), "from-host\n").unwrap(); + let spec = format!("{}:/workspace", tmp.path().to_str().unwrap()); + + let out = shuru_run( + &["--mount", &spec], + r#" +docker run --rm --user 0 -v /workspace:/w IMAGE sh -c "cat /w/host.txt; echo escaped > /w/escaped.txt && echo WROTE_IN_GUEST" +docker run --rm -v /:/vmroot IMAGE sh -c "ls /vmroot/Users > /dev/null 2>&1 && echo HOST_FS_VISIBLE || echo HOST_FS_ABSENT" +"#, + ); + let s = stdout(&out); + assert!(s.contains("from-host"), "{}", text(&out)); + assert!(s.contains("WROTE_IN_GUEST"), "{}", text(&out)); + assert!(s.contains("HOST_FS_ABSENT"), "{}", text(&out)); + assert!( + !tmp.path().join("escaped.txt").exists(), + "container write reached the host through a read-only mount" + ); +} + +#[test] +#[ignore] +fn resource_limits_apply_inside_containers() { + let out = shuru_run( + &[], + r#" +docker run --rm --memory 64m --cpus 0.5 --pids-limit 20 IMAGE sh -c "echo memory.max=\$(cat /sys/fs/cgroup/memory.max); echo cpu.max=\$(cat /sys/fs/cgroup/cpu.max); echo pids.max=\$(cat /sys/fs/cgroup/pids.max)" +docker run --rm --memory 32m IMAGE sh -c "a=\$(head -c 200m /dev/zero | tr \"\\0\" a); echo survived" +echo "oom-exit=$?" +docker run --rm --pids-limit 5 IMAGE sh -c "for i in 1 2 3 4 5 6 7 8 9 10; do sleep 3 & done; wait; echo forked-all" 2>&1 | tail -n 1 +"#, + ); + let s = stdout(&out); + assert!(s.contains("memory.max=67108864"), "{}", text(&out)); + assert!(s.contains("cpu.max=50000 100000"), "{}", text(&out)); + assert!(s.contains("pids.max=20"), "{}", text(&out)); + assert!(s.contains("oom-exit=137"), "{}", text(&out)); + assert!(!s.contains("survived"), "{}", text(&out)); + assert!(!s.contains("forked-all"), "{}", text(&out)); +} + +#[test] +#[ignore] +fn named_volume_persists_across_container_restart() { + let token = format!("persisted-{}", std::process::id()); + let script = format!( + r#" +set -e +docker volume create vdata > /dev/null +docker create --name keeper -v vdata:/tmp IMAGE sleep 120 > /dev/null +docker start keeper > /dev/null +docker exec keeper sh -c "echo {token} > /tmp/marker" +docker stop -t 1 keeper > /dev/null +docker start keeper > /dev/null +echo "after-container-restart=$(docker exec keeper cat /tmp/marker)" +docker rm -f keeper > /dev/null +echo "after-new-container=$(docker run --rm -v vdata:/tmp IMAGE cat /tmp/marker)" +"# + ); + let out = shuru_run(&[], &script); + assert!(out.status.success(), "{}", text(&out)); + let s = stdout(&out); + assert!( + s.contains(&format!("after-container-restart={}", token)), + "{}", + s + ); + assert!( + s.contains(&format!("after-new-container={}", token)), + "{}", + s + ); +} + +/// Shuru resets the disk on every run, so persistence across a VM stop and +/// start goes through a checkpoint. The guest stops dockerd and syncs the +/// disk during the shutdown handshake, before the checkpoint is cloned, so +/// the volume store is consistent and the daemon restarts cleanly on the +/// next boot. +#[test] +#[ignore] +fn named_volume_persists_across_vm_stop_start_via_checkpoint() { + let name = format!("shuru-test-docker-data-{}", std::process::id()); + let token = format!("vm-persisted-{}", std::process::id()); + + let write = format!( + r#" +set -e +docker volume create vdata > /dev/null +docker run --rm -v vdata:/tmp IMAGE sh -c "echo {token} > /tmp/marker" +"# + ); + let create = Command::new(shuru_bin()) + .args([ + "checkpoint", + "create", + &name, + "--verbose", + "--from", + &checkpoint(), + "--", + ]) + .args(guest_cmd(&write)) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!(create.status.success(), "{}", text(&create)); + + let read = Command::new(shuru_bin()) + .args(["run", "--from", &name, "--"]) + .args(guest_cmd( + r#"echo "after-vm-restart=$(docker run --rm -v vdata:/tmp IMAGE cat /tmp/marker)""#, + )) + .stdin(Stdio::null()) + .output() + .unwrap(); + + let _ = Command::new(shuru_bin()) + .args(["checkpoint", "delete", &name]) + .output(); + eprintln!( + "$ shuru checkpoint create {} --verbose --from {}\n{}", + name, + checkpoint(), + text(&create) + ); + eprintln!("$ shuru run --from {}\n{}", name, text(&read)); + + assert!( + String::from_utf8_lossy(&create.stderr).contains("stopped services: docker"), + "{}", + text(&create) + ); + assert!(read.status.success(), "{}", text(&read)); + assert!( + stdout(&read).contains(&format!("after-vm-restart={}", token)), + "{}", + text(&read) + ); +} diff --git a/crates/shuru-cli/tests/recipes.rs b/crates/shuru-cli/tests/recipes.rs new file mode 100644 index 0000000..86559f2 --- /dev/null +++ b/crates/shuru-cli/tests/recipes.rs @@ -0,0 +1,601 @@ +//! Recipes (RFC 0005). Local recipe directories are written on the fly and +//! built into throwaway checkpoints in real VMs, so every test but the +//! first is #[ignore]d. Run with: +//! +//! SHURU_BIN=target/debug/shuru cargo test -p shuru-cli --test recipes -- --ignored +//! +//! SHURU_BIN must be a codesigned binary (see tests/docker.rs). SHURU_DATA_DIR, +//! SHURU_KERNEL and SHURU_INITRD are inherited by the CLI. +//! +//! SHURU_DOCKER_RECIPE path to the docker recipe directory from a checkout +//! of superhq-ai/recipes; with SHURU_TEST_NET=1 it is +//! built for real and runs a container +//! SHURU_BROWSER_RECIPE same for the browser recipe; built for real, then +//! the DevTools endpoint is checked inside the VM and +//! through a forwarded port + +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; + +fn shuru_bin() -> String { + std::env::var("SHURU_BIN") + .expect("SHURU_BIN not set: point it at a codesigned shuru binary (e.g. just build)") +} + +fn text(out: &Output) -> String { + format!( + "--- stdout ---\n{}\n--- stderr ---\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ) +} + +fn stdout(out: &Output) -> String { + String::from_utf8_lossy(&out.stdout).to_string() +} + +fn stderr(out: &Output) -> String { + String::from_utf8_lossy(&out.stderr).to_string() +} + +fn shuru(args: &[&str]) -> Output { + let out = Command::new(shuru_bin()) + .args(args) + .stdin(Stdio::null()) + .output() + .expect("failed to spawn shuru"); + eprintln!("$ shuru {}\n{}", args.join(" "), text(&out)); + out +} + +fn unique(prefix: &str) -> String { + format!("{}-{}", prefix, std::process::id()) +} + +/// A local recipe directory. Steps are plain files so a test can edit one +/// and observe the rebuild. +struct LocalRecipe { + dir: tempfile::TempDir, +} + +impl LocalRecipe { + fn new(name: &str, spec_json: &str, steps: &[(&str, &str)]) -> Self { + let dir = tempfile::tempdir().unwrap(); + let spec = spec_json.replace("NAME", name); + std::fs::write(dir.path().join("recipe.json"), spec).unwrap(); + for (file, body) in steps { + std::fs::write(dir.path().join(file), body).unwrap(); + } + LocalRecipe { dir } + } + + fn path(&self) -> &Path { + self.dir.path() + } + + fn path_str(&self) -> String { + self.dir.path().to_str().unwrap().to_string() + } +} + +struct Cleanup(Vec); + +impl Drop for Cleanup { + fn drop(&mut self) { + for name in &self.0 { + let _ = Command::new(shuru_bin()) + .args(["checkpoint", "delete", name]) + .output(); + } + } +} + +const TWO_STEP_SPEC: &str = r#"{ + "name": "NAME", + "description": "two steps and a service", + "disk_size": 4096, + "services": { + "hello": { + "exec": ["sh", "-c", "touch /run/hello-ran; exec sleep infinity"], + "ready": ["test", "-f", "/run/hello-ran"] + } + }, + "steps": ["first.sh", "second.sh"] +}"#; + +const FIRST_STEP: &str = "mkdir -p /opt/recipe\necho step1 >> /opt/recipe/log\ntest -f /etc/shuru/services/hello.json && echo service-file-present >> /opt/recipe/log\n"; +const SECOND_STEP: &str = "echo step2 >> /opt/recipe/log\ntest -f /run/hello-ran && echo step2-saw-service >> /opt/recipe/log\n"; + +#[test] +fn show_prints_metadata_and_steps_without_booting() { + let recipe = LocalRecipe::new( + "show-test", + TWO_STEP_SPEC, + &[("first.sh", FIRST_STEP), ("second.sh", SECOND_STEP)], + ); + let out = shuru(&["recipe", "show", &recipe.path_str()]); + assert!(out.status.success(), "{}", text(&out)); + let s = stdout(&out); + assert!(s.contains("name: show-test"), "{}", s); + assert!(s.contains("description: two steps and a service"), "{}", s); + assert!(s.contains("network: offline during build"), "{}", s); + assert!(s.contains("services: hello"), "{}", s); + assert!( + s.contains("--- /etc/shuru/services/hello.json ---"), + "{}", + s + ); + assert!(s.contains("--- first.sh ---"), "{}", s); + assert!(s.contains("echo step2 >> /opt/recipe/log"), "{}", s); +} + +#[test] +fn show_rejects_bad_recipes() { + let bad_name = LocalRecipe::new( + "Bad Name", + TWO_STEP_SPEC, + &[("first.sh", ""), ("second.sh", "")], + ); + let out = shuru(&["recipe", "show", &bad_name.path_str()]); + assert!(!out.status.success()); + assert!( + stderr(&out).contains("must match [a-z0-9]"), + "{}", + text(&out) + ); + + let missing_step = LocalRecipe::new("ok", TWO_STEP_SPEC, &[("first.sh", "")]); + let out = shuru(&["recipe", "show", &missing_step.path_str()]); + assert!(!out.status.success()); + assert!( + stderr(&out).contains("reading step second.sh"), + "{}", + text(&out) + ); + + let future = LocalRecipe::new( + "future", + r#"{"name": "NAME", "min_shuru_version": "99.0.0"}"#, + &[("build.sh", "true\n")], + ); + let out = shuru(&["recipe", "show", &future.path_str()]); + assert!(!out.status.success()); + assert!( + stderr(&out).contains("needs shuru 99.0.0 or newer"), + "{}", + text(&out) + ); +} + +#[test] +#[ignore] +fn run_builds_once_reuses_and_rebuilds_when_the_recipe_changes() { + let name = unique("shuru-test-recipe"); + let _cleanup = Cleanup(vec![name.clone()]); + let recipe = LocalRecipe::new( + &name, + TWO_STEP_SPEC, + &[("first.sh", FIRST_STEP), ("second.sh", SECOND_STEP)], + ); + let check = "cat /opt/recipe/log; test -f /run/hello-ran && echo RUN_SAW_SERVICE"; + + let out = shuru(&[ + "run", + "--recipe", + &recipe.path_str(), + "--", + "sh", + "-c", + check, + ]); + assert!(out.status.success(), "{}", text(&out)); + let s = stdout(&out); + let e = stderr(&out); + assert!( + e.contains(&format!("building recipe {} from", name)), + "{}", + e + ); + assert!(e.contains("step 1/2: first.sh"), "{}", e); + assert!(e.contains("step 2/2: second.sh"), "{}", e); + assert!( + s.contains("step1\nservice-file-present\nstep2\nstep2-saw-service\n"), + "{}", + s + ); + assert!(s.contains("RUN_SAW_SERVICE"), "{}", s); + + let out = shuru(&[ + "run", + "--recipe", + &recipe.path_str(), + "--", + "sh", + "-c", + check, + ]); + assert!(out.status.success(), "{}", text(&out)); + let e = stderr(&out); + assert!(e.contains(&format!("using checkpoint {}", name)), "{}", e); + assert!(!e.contains("building recipe"), "{}", e); + + let list = shuru(&["checkpoint", "list"]); + let l = stdout(&list); + let row = l.lines().find(|line| line.starts_with(&name)).unwrap_or(""); + assert!(row.contains(&recipe.path_str()), "{}", l); + + std::fs::write( + recipe.path().join("second.sh"), + format!("{}echo step2-v2 >> /opt/recipe/log\n", SECOND_STEP), + ) + .unwrap(); + let out = shuru(&[ + "run", + "--recipe", + &recipe.path_str(), + "--", + "sh", + "-c", + check, + ]); + assert!(out.status.success(), "{}", text(&out)); + let e = stderr(&out); + assert!(e.contains("recipe changed"), "{}", e); + assert!(e.contains("building recipe"), "{}", e); + assert!(stdout(&out).contains("step2-v2"), "{}", text(&out)); + + let leftovers = stdout(&shuru(&["checkpoint", "list"])); + assert!( + !leftovers.contains(&format!("{}-step1", name)), + "{}", + leftovers + ); +} + +#[test] +fn bare_name_never_resolves_from_the_working_directory() { + // A cloned repository with NAME/recipe.json must not shadow the official + // recipe of the same name, which would inherit its trust. + let cwd = tempfile::tempdir().unwrap(); + let shadow = cwd.path().join("shadow-test"); + std::fs::create_dir(&shadow).unwrap(); + std::fs::write( + shadow.join("recipe.json"), + r#"{"name": "shadow-test", "description": "local-shadow", "steps": ["evil.sh"]}"#, + ) + .unwrap(); + std::fs::write(shadow.join("evil.sh"), "true\n").unwrap(); + + let out = Command::new(shuru_bin()) + .args(["recipe", "show", "shadow-test"]) + .current_dir(cwd.path()) + .stdin(Stdio::null()) + .output() + .expect("failed to spawn shuru"); + eprintln!("{}", text(&out)); + assert!(!out.status.success(), "{}", text(&out)); + assert!(!stdout(&out).contains("local-shadow"), "{}", text(&out)); + assert!( + stderr(&out).contains("pass ./shadow-test"), + "{}", + text(&out) + ); + + let out = Command::new(shuru_bin()) + .args(["recipe", "show", "./shadow-test"]) + .current_dir(cwd.path()) + .stdin(Stdio::null()) + .output() + .expect("failed to spawn shuru"); + assert!(out.status.success(), "{}", text(&out)); + assert!(stdout(&out).contains("local-shadow"), "{}", text(&out)); +} + +#[test] +fn checkpoint_flag_requires_recipe() { + let out = shuru(&["run", "--checkpoint", "x", "--", "true"]); + assert!(!out.status.success(), "{}", text(&out)); + assert!(stderr(&out).contains("--recipe"), "{}", text(&out)); +} + +#[test] +#[ignore] +fn run_recipe_under_a_chosen_checkpoint_name() { + let name = unique("shuru-test-recipe-named-run"); + let _cleanup = Cleanup(vec![name.clone()]); + let recipe = LocalRecipe::new( + "some-recipe", + r#"{"name": "NAME", "steps": ["build.sh"]}"#, + &[("build.sh", "echo named > /opt/marker\n")], + ); + + let args = [ + "run", + "--recipe", + &recipe.path_str(), + "--checkpoint", + &name, + "--", + "cat", + "/opt/marker", + ]; + let out = shuru(&args); + assert!(out.status.success(), "{}", text(&out)); + assert!( + stderr(&out).contains("building recipe some-recipe"), + "{}", + text(&out) + ); + assert!(stdout(&out).contains("named"), "{}", text(&out)); + + let out = shuru(&args); + assert!(out.status.success(), "{}", text(&out)); + assert!( + stderr(&out).contains(&format!("using checkpoint {}", name)), + "{}", + text(&out) + ); + + let list = stdout(&shuru(&["checkpoint", "list"])); + assert!(list.lines().any(|l| l.starts_with(&name)), "{}", list); + assert!( + !list.lines().any(|l| l.starts_with("some-recipe ")), + "{}", + list + ); +} + +#[test] +#[ignore] +fn create_builds_under_a_chosen_name_and_refuses_a_command() { + let name = unique("shuru-test-recipe-named"); + let _cleanup = Cleanup(vec![name.clone()]); + let recipe = LocalRecipe::new( + "irrelevant", + r#"{"name": "NAME", "steps": ["build.sh"]}"#, + &[("build.sh", "echo built-by-recipe > /opt/marker\n")], + ); + + let out = shuru(&[ + "checkpoint", + "create", + &name, + "--recipe", + &recipe.path_str(), + "--", + "echo", + "no", + ]); + assert!(!out.status.success(), "{}", text(&out)); + assert!(stderr(&out).contains("drop the command"), "{}", text(&out)); + + let out = shuru(&[ + "checkpoint", + "create", + &name, + "--recipe", + &recipe.path_str(), + ]); + assert!(out.status.success(), "{}", text(&out)); + + let out = shuru(&["run", "--from", &name, "--", "cat", "/opt/marker"]); + assert!(out.status.success(), "{}", text(&out)); + assert!(stdout(&out).contains("built-by-recipe"), "{}", text(&out)); + + let out = shuru(&[ + "checkpoint", + "create", + &name, + "--recipe", + &recipe.path_str(), + ]); + assert!(!out.status.success(), "{}", text(&out)); + assert!(stderr(&out).contains("already exists"), "{}", text(&out)); +} + +#[test] +#[ignore] +fn update_rebuilds_a_recipe_checkpoint_at_the_same_revision() { + let name = unique("shuru-test-recipe-update"); + let _cleanup = Cleanup(vec![name.clone(), format!("{}.update", name)]); + let recipe = LocalRecipe::new( + "irrelevant", + r#"{"name": "NAME", "steps": ["build.sh"]}"#, + &[("build.sh", "date +%s%N > /opt/built\n")], + ); + + let out = shuru(&[ + "checkpoint", + "create", + &name, + "--recipe", + &recipe.path_str(), + ]); + assert!(out.status.success(), "{}", text(&out)); + let first = stdout(&shuru(&["run", "--from", &name, "--", "cat", "/opt/built"])); + assert!(!first.trim().is_empty(), "{}", first); + + let out = shuru(&[ + "checkpoint", + "update", + &name, + "--recipe", + &recipe.path_str(), + ]); + assert!(out.status.success(), "{}", text(&out)); + let e = stderr(&out); + assert!(e.contains("building recipe"), "{}", e); + assert!( + e.contains(&format!("checkpoint '{}' updated", name)), + "{}", + e + ); + let second = stdout(&shuru(&["run", "--from", &name, "--", "cat", "/opt/built"])); + assert_ne!(first.trim(), second.trim(), "{}", second); + + let list = stdout(&shuru(&["checkpoint", "list"])); + let row = list.lines().find(|l| l.starts_with(&name)).unwrap_or(""); + assert!(row.contains(&recipe.path_str()), "{}", list); + assert!(!list.contains(&format!("{}.update", name)), "{}", list); +} + +#[test] +#[ignore] +fn failing_step_keeps_no_final_checkpoint() { + let name = unique("shuru-test-recipe-fail"); + let _cleanup = Cleanup(vec![name.clone(), format!("{}-step1", name)]); + let recipe = LocalRecipe::new( + &name, + r#"{"name": "NAME", "steps": ["ok.sh", "bad.sh"]}"#, + &[ + ("ok.sh", "echo fine\n"), + ("bad.sh", "echo boom >&2\nexit 7\n"), + ], + ); + let out = shuru(&["run", "--recipe", &recipe.path_str(), "--", "true"]); + assert!(!out.status.success(), "{}", text(&out)); + let e = stderr(&out); + assert!(e.contains("step bad.sh exited with status 7"), "{}", e); + assert!( + e.contains(&format!("intermediate checkpoint {}-step1 kept", name)), + "{}", + e + ); + let list = stdout(&shuru(&["checkpoint", "list"])); + assert!( + !list.lines().any(|l| l.starts_with(&format!("{} ", name))), + "{}", + list + ); +} + +#[test] +#[ignore] +fn browser_recipe_serves_cdp_inside_and_through_a_forwarded_port() { + use std::io::{Read, Write}; + + let Ok(dir) = std::env::var("SHURU_BROWSER_RECIPE") else { + eprintln!("skipped: set SHURU_BROWSER_RECIPE to the browser recipe directory"); + return; + }; + if !std::env::var("SHURU_TEST_NET") + .map(|v| v == "1") + .unwrap_or(false) + { + eprintln!("skipped: set SHURU_TEST_NET=1"); + return; + } + let name = unique("shuru-test-recipe-browser"); + let _cleanup = Cleanup(vec![name.clone()]); + + let out = shuru(&["checkpoint", "create", &name, "--recipe", &dir]); + assert!(out.status.success(), "{}", text(&out)); + assert!( + stderr(&out).contains("step 2/2: check.sh"), + "{}", + text(&out) + ); + assert!(stdout(&out).contains("screenshot ok"), "{}", text(&out)); + + // Inside the VM the service is ready before the command runs. + let out = shuru(&[ + "run", + "--from", + &name, + "--", + "sh", + "-c", + "id -un; curl -sf http://127.0.0.1:9222/json/version", + ]); + assert!(out.status.success(), "{}", text(&out)); + let s = stdout(&out); + assert!(s.contains("Chrome/"), "{}", s); + assert!(s.contains("webSocketDebuggerUrl"), "{}", s); + + // Through -p the host reaches the same endpoint, which is how CDP clients attach. + let port = 19000 + (std::process::id() % 1000) as u16; + let mut child = Command::new(shuru_bin()) + .args([ + "run", + "--from", + &name, + "-p", + &format!("{}:9222", port), + "--", + "sleep", + "30", + ]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("failed to spawn shuru"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + let mut body = String::new(); + while std::time::Instant::now() < deadline { + if let Ok(mut stream) = std::net::TcpStream::connect(("127.0.0.1", port)) { + let _ = stream.set_read_timeout(Some(std::time::Duration::from_secs(5))); + // The DevTools server ignores HTTP/1.0 requests. + let _ = stream.write_all( + b"GET /json/version HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n", + ); + let _ = stream.read_to_string(&mut body); + if body.contains("Chrome/") { + break; + } + body.clear(); + } + std::thread::sleep(std::time::Duration::from_millis(250)); + } + let _ = child.kill(); + let _ = child.wait(); + assert!( + body.contains("Chrome/"), + "no CDP answer on host port {}: {:?}", + port, + body + ); +} + +#[test] +#[ignore] +fn docker_recipe_builds_and_runs_a_container() { + let Ok(dir) = std::env::var("SHURU_DOCKER_RECIPE") else { + eprintln!("skipped: set SHURU_DOCKER_RECIPE to the docker recipe directory"); + return; + }; + if !std::env::var("SHURU_TEST_NET") + .map(|v| v == "1") + .unwrap_or(false) + { + eprintln!("skipped: set SHURU_TEST_NET=1"); + return; + } + let dir = PathBuf::from(dir); + let name = unique("shuru-test-recipe-docker"); + let _cleanup = Cleanup(vec![name.clone()]); + + let out = shuru(&[ + "checkpoint", + "create", + &name, + "--recipe", + dir.to_str().unwrap(), + ]); + assert!(out.status.success(), "{}", text(&out)); + assert!(stderr(&out).contains("step 2/2: pull.sh"), "{}", text(&out)); + + let out = shuru(&[ + "run", + "--from", + &name, + "--", + "sh", + "-c", + "docker info --format '{{.ServerVersion}}' && docker run --rm nginxinc/nginx-unprivileged:1.31.7-alpine-slim id -u", + ]); + assert!(out.status.success(), "{}", text(&out)); + let s = stdout(&out); + assert!(s.contains("29.9.2"), "{}", s); + assert!(s.contains("\n101\n"), "{}", s); +} diff --git a/crates/shuru-cli/tests/rw_mounts.rs b/crates/shuru-cli/tests/rw_mounts.rs index 9c97c17..b27995d 100644 --- a/crates/shuru-cli/tests/rw_mounts.rs +++ b/crates/shuru-cli/tests/rw_mounts.rs @@ -2,6 +2,7 @@ //! All #[ignore]d by default. Run with: //! SHURU_BIN=target/debug/shuru cargo test -p shuru-cli -- --ignored +use std::path::Path; use std::process::Command; use tempfile::tempdir; @@ -10,11 +11,25 @@ fn shuru_bin() -> String { .expect("SHURU_BIN not set — point it at a codesigned shuru binary (e.g. just build)") } -fn run_in_vm(mount_spec: &str, guest_cmd: &str) -> std::process::Output { - Command::new(shuru_bin()) - .args(["run", "--mount", mount_spec, "--", "sh", "-c", guest_cmd]) +/// Runs from inside `host_dir`, since mounts must sit under the working +/// directory. A `:rw` spec also passes `--allow-host-writes`. +fn run_in_vm(host_dir: &Path, mount_spec: &str, guest_cmd: &str) -> std::process::Output { + let mut cmd = Command::new(shuru_bin()); + cmd.current_dir(host_dir) + .args(["run", "--mount", mount_spec]); + if mount_spec.ends_with(":rw") { + cmd.arg("--allow-host-writes"); + } + let output = cmd + .args(["--", "sh", "-c", guest_cmd]) .output() - .expect("failed to spawn shuru — is SHURU_BIN correct?") + .expect("failed to spawn shuru — is SHURU_BIN correct?"); + assert!( + output.status.success(), + "shuru run failed; stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + output } #[test] @@ -24,7 +39,7 @@ fn ro_mount_default_discards_guest_writes() { let host_dir = tmp.path().to_str().unwrap(); let spec = format!("{}:/workspace", host_dir); - run_in_vm(&spec, "echo guest > /workspace/test.txt"); + run_in_vm(tmp.path(), &spec, "echo guest > /workspace/test.txt"); assert!( !tmp.path().join("test.txt").exists(), @@ -39,7 +54,7 @@ fn rw_mount_writes_land_on_host() { let host_dir = tmp.path().to_str().unwrap(); let spec = format!("{}:/workspace:rw", host_dir); - run_in_vm(&spec, "echo guest > /workspace/test.txt"); + run_in_vm(tmp.path(), &spec, "echo guest > /workspace/test.txt"); let written = std::fs::read_to_string(tmp.path().join("test.txt")) .expect("guest write should land on host"); @@ -54,7 +69,11 @@ fn rw_mount_guest_reads_host_files() { let host_dir = tmp.path().to_str().unwrap(); let spec = format!("{}:/workspace:rw", host_dir); - run_in_vm(&spec, "cp /workspace/original.txt /workspace/copy.txt"); + run_in_vm( + tmp.path(), + &spec, + "cp /workspace/original.txt /workspace/copy.txt", + ); let copied = std::fs::read_to_string(tmp.path().join("copy.txt")) .expect("guest should be able to read and copy host files"); @@ -68,7 +87,7 @@ fn explicit_ro_suffix_discards_guest_writes() { let host_dir = tmp.path().to_str().unwrap(); let spec = format!("{}:/workspace:ro", host_dir); - run_in_vm(&spec, "echo ghost > /workspace/test.txt"); + run_in_vm(tmp.path(), &spec, "echo ghost > /workspace/test.txt"); assert!( !tmp.path().join("test.txt").exists(), @@ -83,13 +102,8 @@ fn rw_mount_creates_missing_guest_dirs() { let host_dir = tmp.path().to_str().unwrap(); let spec = format!("{}:/mnt/does/not/exist:rw", host_dir); - let output = run_in_vm(&spec, "echo ok > /mnt/does/not/exist/proof.txt"); + run_in_vm(tmp.path(), &spec, "echo ok > /mnt/does/not/exist/proof.txt"); - assert!( - output.status.success(), - "rw mount to non-existent guest path should succeed; stderr: {}", - String::from_utf8_lossy(&output.stderr) - ); let written = std::fs::read_to_string(tmp.path().join("proof.txt")) .expect("write through rw mount at auto-created guest path should land on host"); assert_eq!(written.trim(), "ok"); @@ -103,13 +117,8 @@ fn ro_mount_creates_missing_guest_dirs() { let host_dir = tmp.path().to_str().unwrap(); let spec = format!("{}:/mnt/does/not/exist:ro", host_dir); - let output = run_in_vm(&spec, "cat /mnt/does/not/exist/hello.txt"); + let output = run_in_vm(tmp.path(), &spec, "cat /mnt/does/not/exist/hello.txt"); - assert!( - output.status.success(), - "ro mount to non-existent guest path should succeed; stderr: {}", - String::from_utf8_lossy(&output.stderr) - ); assert!( String::from_utf8_lossy(&output.stdout).contains("from-host"), "should read host file through overlay at auto-created guest path" diff --git a/crates/shuru-cli/tests/services.rs b/crates/shuru-cli/tests/services.rs new file mode 100644 index 0000000..16668cd --- /dev/null +++ b/crates/shuru-cli/tests/services.rs @@ -0,0 +1,191 @@ +//! Guest services (RFC 0004). Each test builds a throwaway checkpoint from +//! the base rootfs with a small shell service and boots real VMs from it, +//! so every test is #[ignore]d. Run with: +//! +//! SHURU_BIN=target/debug/shuru cargo test -p shuru-cli --test services -- --ignored +//! +//! SHURU_BIN must be a codesigned binary (see tests/docker.rs). SHURU_DATA_DIR, +//! SHURU_KERNEL and SHURU_INITRD are inherited by the CLI. + +use std::process::{Command, Output, Stdio}; + +use base64::Engine; + +fn shuru_bin() -> String { + std::env::var("SHURU_BIN") + .expect("SHURU_BIN not set: point it at a codesigned shuru binary (e.g. just build)") +} + +fn guest_cmd(script: &str) -> Vec { + let b64 = base64::engine::general_purpose::STANDARD.encode(script.as_bytes()); + vec![ + "sh".into(), + "-c".into(), + format!("echo {} | base64 -d | sh", b64), + ] +} + +fn text(out: &Output) -> String { + format!( + "--- stdout ---\n{}\n--- stderr ---\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ) +} + +fn stdout(out: &Output) -> String { + String::from_utf8_lossy(&out.stdout).to_string() +} + +fn stderr(out: &Output) -> String { + String::from_utf8_lossy(&out.stderr).to_string() +} + +fn shuru(args: &[&str], script: &str) -> Output { + let out = Command::new(shuru_bin()) + .args(args) + .arg("--") + .args(guest_cmd(script)) + .stdin(Stdio::null()) + .output() + .expect("failed to spawn shuru"); + eprintln!("$ shuru {}\n{}", args.join(" "), text(&out)); + out +} + +/// A checkpoint that is deleted when dropped, so a failed assertion does +/// not leave it behind. +struct Checkpoint(String); + +impl Checkpoint { + fn create(suffix: &str, from: Option<&str>, setup: &str) -> Self { + let name = format!("shuru-test-svc-{}-{}", suffix, std::process::id()); + let mut args = vec!["checkpoint", "create", name.as_str(), "--verbose"]; + if let Some(from) = from { + args.extend(["--from", from]); + } + let out = shuru(&args, setup); + assert!( + out.status.success(), + "checkpoint create failed\n{}", + text(&out) + ); + Checkpoint(name) + } +} + +impl Drop for Checkpoint { + fn drop(&mut self) { + let _ = Command::new(shuru_bin()) + .args(["checkpoint", "delete", &self.0]) + .output(); + } +} + +/// Writes a service file. The JSON is passed through base64 so the shell +/// never sees its quotes. +fn declare(name: &str, json: &str) -> String { + let b64 = base64::engine::general_purpose::STANDARD.encode(json.as_bytes()); + format!( + "mkdir -p /etc/shuru/services && echo {} | base64 -d > /etc/shuru/services/{}.json\n", + b64, name + ) +} + +#[test] +#[ignore] +fn service_with_delayed_ready_check_gates_the_command() { + let setup = declare( + "slow", + r#"{"exec": ["sh", "-c", "sleep 2; touch /run/slow-ready; exec sleep infinity"], + "ready": ["test", "-f", "/run/slow-ready"], + "ready_timeout_secs": 20, "stop_timeout_secs": 5}"#, + ) + &declare( + "fast", + r#"{"exec": ["sh", "-c", "echo fast-started; exec sleep infinity"]}"#, + ); + let ckpt = Checkpoint::create("gate", None, &setup); + + let out = shuru( + &["run", "--from", &ckpt.0, "--verbose"], + r#" +if [ -f /run/slow-ready ]; then echo GATED_OK; else echo GATED_FAIL; fi +echo "fast-log=$(cat /run/shuru/log/fast.log)" +echo "slow-alive=$(pgrep -c -f "sleep infinity")" +"#, + ); + assert!(out.status.success(), "{}", text(&out)); + let s = stdout(&out); + assert!(s.contains("GATED_OK"), "{}", text(&out)); + assert!(s.contains("fast-log=fast-started"), "{}", text(&out)); + assert!(s.contains("slow-alive=2"), "{}", text(&out)); + assert!( + stderr(&out).contains("stopped services: fast, slow"), + "{}", + text(&out) + ); +} + +#[test] +#[ignore] +fn service_that_fails_blocks_the_command_with_its_log() { + let setup = declare( + "bad", + r#"{"exec": ["sh", "-c", "echo booting bad service; sleep 1; echo giving up; exit 3"], + "ready": ["false"], "ready_timeout_secs": 10}"#, + ); + let ckpt = Checkpoint::create("fail", None, &setup); + + let out = shuru(&["run", "--from", &ckpt.0], "echo SHOULD_NOT_RUN"); + assert!(!out.status.success(), "{}", text(&out)); + assert!(!stdout(&out).contains("SHOULD_NOT_RUN"), "{}", text(&out)); + let e = stderr(&out); + assert!( + e.contains("service bad exited with status 3"), + "{}", + text(&out) + ); + assert!(e.contains("giving up"), "{}", text(&out)); +} + +#[test] +#[ignore] +fn shutdown_delivers_sigterm_and_syncs_before_the_checkpoint_is_cloned() { + let setup = declare( + "trap", + r#"{"exec": ["sh", "-c", "trap \"echo got-term > /var/lib/trap-marker; exit 0\" TERM; while true; do sleep 1; done"], + "stop_timeout_secs": 10}"#, + ); + let base = Checkpoint::create("term", None, &setup); + let derived = Checkpoint::create("term-derived", Some(&base.0), "echo RAN_IN_BASE"); + + let out = shuru( + &["run", "--from", &derived.0], + "cat /var/lib/trap-marker 2>/dev/null || echo NO_MARKER", + ); + assert!(out.status.success(), "{}", text(&out)); + assert!(stdout(&out).contains("got-term"), "{}", text(&out)); +} + +#[test] +#[ignore] +fn service_ignoring_sigterm_is_killed_after_its_stop_timeout() { + let setup = declare( + "stubborn", + r#"{"exec": ["sh", "-c", "trap \"\" TERM; while true; do sleep 1; done"], + "stop_timeout_secs": 2}"#, + ); + let ckpt = Checkpoint::create("kill", None, &setup); + + let start = std::time::Instant::now(); + let out = shuru(&["run", "--from", &ckpt.0], "echo RAN"); + let elapsed = start.elapsed(); + assert!(out.status.success(), "{}", text(&out)); + assert!(stdout(&out).contains("RAN"), "{}", text(&out)); + assert!( + stderr(&out).contains("services ignored SIGTERM and were killed: stubborn"), + "{}", + text(&out) + ); + assert!(elapsed.as_secs() < 30, "stop took {:?}", elapsed); +} diff --git a/crates/shuru-darwin/Cargo.toml b/crates/shuru-darwin/Cargo.toml index 0582ca7..444c148 100644 --- a/crates/shuru-darwin/Cargo.toml +++ b/crates/shuru-darwin/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuru-darwin" -version = "0.1.1" +version = "0.2.3" edition = "2021" description = "Virtualization.framework bindings for shuru" license = "Apache-2.0" @@ -27,5 +27,7 @@ objc2-virtualization = { version = "0.3", features = [ "dispatch2", "VZFileHandleNetworkDeviceAttachment", "VZNetworkBlockDeviceStorageDeviceAttachment", + "VZMemoryBalloonDevice", + "VZVirtioTraditionalMemoryBalloonDevice", ] } diff --git a/crates/shuru-darwin/src/clone.rs b/crates/shuru-darwin/src/clone.rs new file mode 100644 index 0000000..319aa15 --- /dev/null +++ b/crates/shuru-darwin/src/clone.rs @@ -0,0 +1,41 @@ +//! APFS copy-on-write clones. + +use std::ffi::CString; +use std::io; +use std::os::unix::io::RawFd; + +extern "C" { + fn clonefile(src: *const libc::c_char, dst: *const libc::c_char, flags: u32) -> libc::c_int; + fn fclonefileat( + srcfd: libc::c_int, + dst_dirfd: libc::c_int, + dst: *const libc::c_char, + flags: u32, + ) -> libc::c_int; +} + +fn c_path(path: &str) -> io::Result { + CString::new(path).map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path holds a NUL")) +} + +/// Clone the file at `src` to a new file at `dst`, which must not exist. +pub fn clone_path(src: &str, dst: &str) -> io::Result<()> { + let c_src = c_path(src)?; + let c_dst = c_path(dst)?; + let ret = unsafe { clonefile(c_src.as_ptr(), c_dst.as_ptr(), 0) }; + if ret != 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} + +/// Clone the open file `fd` to a new file at `dst`, which must not exist. +/// Works on a file that has no name any more. +pub fn clone_fd(fd: RawFd, dst: &str) -> io::Result<()> { + let c_dst = c_path(dst)?; + let ret = unsafe { fclonefileat(fd, libc::AT_FDCWD, c_dst.as_ptr(), 0) }; + if ret != 0 { + return Err(io::Error::last_os_error()); + } + Ok(()) +} diff --git a/crates/shuru-darwin/src/lib.rs b/crates/shuru-darwin/src/lib.rs index 0a843e9..3fdd40e 100644 --- a/crates/shuru-darwin/src/lib.rs +++ b/crates/shuru-darwin/src/lib.rs @@ -1,6 +1,7 @@ mod sys; mod bootloader; +mod clone; mod configuration; mod directory_sharing; mod entropy; @@ -14,6 +15,7 @@ pub mod terminal; mod vm; pub use bootloader::LinuxBootLoader; +pub use clone::{clone_fd, clone_path}; pub use configuration::VirtualMachineConfiguration; pub use directory_sharing::{SharedDirectory, VirtioFileSystemDevice}; pub use entropy::VirtioEntropyDevice; diff --git a/crates/shuru-darwin/src/network.rs b/crates/shuru-darwin/src/network.rs index 5a91f41..b36b2d8 100644 --- a/crates/shuru-darwin/src/network.rs +++ b/crates/shuru-darwin/src/network.rs @@ -20,19 +20,25 @@ impl FileHandleNetworkAttachment { /// Creates a network attachment from a connected datagram socket fd. /// The fd should be one end of `socketpair(AF_UNIX, SOCK_DGRAM)`. /// VZ takes ownership of this fd (closes on dealloc). - pub fn new(fd: RawFd) -> Self { + /// + /// `mtu` is the IP-layer MTU (1500..=65535). VZ defaults to 1500 if + /// `setMaximumTransmissionUnit:` is never called. Apple requires the + /// associated socket's `SO_SNDBUF`/`SO_RCVBUF` to be sized for the + /// chosen MTU (`SO_RCVBUF` >= 2× `SO_SNDBUF`, recommended 4×) — see + /// `shuru_proxy::create_socketpair`. + pub fn new(fd: RawFd, mtu: i64) -> Self { unsafe { let file_handle = NSFileHandle::initWithFileDescriptor_closeOnDealloc( NSFileHandle::alloc(), fd, true, ); - FileHandleNetworkAttachment { - inner: VZFileHandleNetworkDeviceAttachment::initWithFileHandle( - VZFileHandleNetworkDeviceAttachment::alloc(), - &file_handle, - ), - } + let inner = VZFileHandleNetworkDeviceAttachment::initWithFileHandle( + VZFileHandleNetworkDeviceAttachment::alloc(), + &file_handle, + ); + inner.setMaximumTransmissionUnit(mtu as isize); + FileHandleNetworkAttachment { inner } } } } diff --git a/crates/shuru-darwin/src/vm.rs b/crates/shuru-darwin/src/vm.rs index 0d08e01..c1e9227 100644 --- a/crates/shuru-darwin/src/vm.rs +++ b/crates/shuru-darwin/src/vm.rs @@ -11,7 +11,8 @@ use objc2_foundation::{ NSKeyValueObservingOptions, NSObject, NSObjectNSKeyValueObserverRegistration, NSString, }; use objc2_virtualization::{ - VZVirtioSocketConnection, VZVirtioSocketDevice, VZVirtualMachine, VZVirtualMachineState, + VZVirtioSocketConnection, VZVirtioSocketDevice, VZVirtioTraditionalMemoryBalloonDevice, + VZVirtualMachine, VZVirtualMachineState, }; use crate::configuration::VirtualMachineConfiguration; @@ -218,6 +219,32 @@ impl VirtualMachine { .map_err(|_| VzError::new("VM stop channel closed"))? } + /// How much memory the guest should keep. The framework rounds the + /// value down to a megabyte and clamps it to the configured size; the + /// guest responds in its own time. Fails without a balloon device. + pub fn set_memory_target(&self, bytes: u64) -> Result<()> { + let (tx, rx) = std::sync::mpsc::channel(); + let machine = ThreadSafe(self.ctx.machine.0.clone()); + + let dispatch_block = RcBlock::new(move || { + let devices = unsafe { machine.memoryBalloonDevices() }; + let result = match devices.firstObject() { + Some(device) => { + let device: Retained = + unsafe { Retained::cast_unchecked(device) }; + unsafe { device.setTargetVirtualMachineMemorySize(bytes) }; + Ok(()) + } + None => Err(VzError::new("no memory balloon device")), + }; + let _ = tx.send(result); + }); + + self.queue.exec_block_async(&dispatch_block); + + rx.recv().map_err(|_| VzError::new("VM queue closed"))? + } + pub fn can_start(&self) -> bool { self.queue .exec_sync(move || unsafe { self.ctx.machine.canStart() }) @@ -297,8 +324,21 @@ impl VirtualMachine { self.queue.exec_block_async(&dispatch_block); - rx.recv() - .map_err(|_| VzError::new("vsock connection channel closed"))? + // The framework never calls the completion handler when the VM + // stops while a connect is pending. + loop { + match rx.recv_timeout(std::time::Duration::from_millis(50)) { + Ok(result) => return result, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { + if matches!(self.state(), VmState::Stopped | VmState::Error) { + return Err(VzError::new("VM stopped during vsock connect")); + } + } + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => { + return Err(VzError::new("vsock connection channel closed")); + } + } + } } pub fn state(&self) -> VmState { diff --git a/crates/shuru-guest/Cargo.toml b/crates/shuru-guest/Cargo.toml index 3d82ae5..e7a33ae 100644 --- a/crates/shuru-guest/Cargo.toml +++ b/crates/shuru-guest/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuru-guest" -version = "0.3.6" +version = "0.6.0" edition = "2021" [[bin]] diff --git a/crates/shuru-guest/src/disk.rs b/crates/shuru-guest/src/disk.rs new file mode 100644 index 0000000..d78a118 --- /dev/null +++ b/crates/shuru-guest/src/disk.rs @@ -0,0 +1,270 @@ +use std::fs::File; +use std::io; +use std::os::fd::AsRawFd; +use std::os::unix::fs::FileExt; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use shuru_proto::{SealRequest, SealResponse, TrimResponse}; + +const FIFREEZE: libc::c_ulong = 0xc0045877; +const FITHAW: libc::c_ulong = 0xc0045878; +const FITRIM: libc::c_ulong = 0xc0185879; +const EXT4_IOC_RESIZE_FS: libc::c_ulong = 0x40086610; +const BLKGETSIZE64: libc::c_ulong = 0x80081272; + +static BOOT_ERRORS: AtomicU32 = AtomicU32::new(0); +static BUSY: AtomicBool = AtomicBool::new(false); +static LEASE: Mutex> = Mutex::new(None); + +struct Lease { + id: u64, + deadline: Instant, + root: File, + expired: bool, +} + +pub fn busy() -> bool { + BUSY.load(Ordering::Acquire) +} + +fn check(ret: libc::c_int) -> io::Result<()> { + if ret < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + } +} + +fn invalid(message: &str) -> io::Error { + io::Error::other(message) +} + +fn superblock() -> io::Result { + let mut sb = [0; 1024]; + File::open("/dev/vda")?.read_exact_at(&mut sb, 1024)?; + crate::ext4::parse(&sb) +} + +fn writable(root: &File) -> io::Result<()> { + let mut st: libc::statvfs = unsafe { std::mem::zeroed() }; + check(unsafe { libc::fstatvfs(root.as_raw_fd(), &mut st) })?; + if st.f_flag & libc::ST_RDONLY != 0 { + return Err(invalid("root ext4 is read-only")); + } + superblock()?; + let count: u32 = std::fs::read_to_string("/sys/fs/ext4/vda/errors_count")? + .trim() + .parse() + .map_err(|_| invalid("invalid ext4 errors_count"))?; + if count != BOOT_ERRORS.load(Ordering::Acquire) { + return Err(invalid("root ext4 recorded an error during this boot")); + } + Ok(()) +} + +#[repr(C)] +struct TrimRange { + start: u64, + len: u64, + minlen: u64, +} + +fn trim_root(root: &File) -> io::Result { + let mut range = TrimRange { + start: 0, + len: u64::MAX, + minlen: 1024 * 1024, + }; + if unsafe { libc::ioctl(root.as_raw_fd(), FITRIM as _, &mut range) } < 0 { + let e = io::Error::last_os_error(); + if e.raw_os_error() == Some(libc::EOPNOTSUPP) { + return Ok(TrimResponse { + supported: false, + bytes: 0, + }); + } + return Err(e); + } + Ok(TrimResponse { + supported: true, + bytes: range.len, + }) +} + +pub fn check_root() -> io::Result<()> { + writable(&File::open("/")?) +} + +pub fn sync() -> io::Result<()> { + let root = File::open("/")?; + check(unsafe { libc::syncfs(root.as_raw_fd()) }) +} + +pub fn trim() -> io::Result { + let _guard = LEASE.lock().unwrap(); + if busy() { + return Err(invalid("root seal is active")); + } + let root = File::open("/")?; + writable(&root)?; + check(unsafe { libc::syncfs(root.as_raw_fd()) })?; + trim_root(&root) +} + +pub fn seal(req: SealRequest) -> io::Result { + if !(1000..=300_000).contains(&req.lease_ms) { + return Err(invalid("seal lease must be 1000 to 300000 ms")); + } + let mut state = LEASE.lock().unwrap(); + if busy() { + return Err(invalid("root seal is active")); + } + BUSY.store(true, Ordering::Release); + let result = (|| { + let root = File::open("/")?; + let services = if req.stop_services { + crate::services::shutdown() + } else { + Default::default() + }; + writable(&root)?; + check(unsafe { libc::syncfs(root.as_raw_fd()) })?; + trim_root(&root)?; + superblock()?; + check(unsafe { libc::ioctl(root.as_raw_fd(), FIFREEZE as _, 0) })?; + let verified = writable(&root).and_then(|_| { + if superblock()?.needs_recovery { + return Err(invalid("frozen root still needs journal recovery")); + } + Ok(()) + }); + if let Err(e) = verified { + if unsafe { libc::ioctl(root.as_raw_fd(), FITHAW as _, 0) } < 0 { + unsafe { + libc::reboot(libc::LINUX_REBOOT_CMD_POWER_OFF); + } + } + return Err(e); + } + *state = Some(Lease { + id: req.id, + deadline: Instant::now() + Duration::from_millis(req.lease_ms), + root, + expired: false, + }); + Ok(SealResponse::Sealed(services)) + })(); + if result.is_err() { + BUSY.store(false, Ordering::Release); + } + result +} + +pub fn thaw(id: u64) -> io::Result { + let mut state = LEASE.lock().unwrap(); + let lease = state.as_mut().ok_or_else(|| invalid("no seal to thaw"))?; + if lease.id != id { + return Err(invalid("seal id does not match")); + } + let expired = lease.expired || Instant::now() >= lease.deadline; + if !lease.expired { + check(unsafe { libc::ioctl(lease.root.as_raw_fd(), FITHAW as _, 0) })?; + } + *state = None; + BUSY.store(false, Ordering::Release); + Ok(if expired { + SealResponse::Expired + } else { + SealResponse::Thawed + }) +} + +pub fn start_watchdog() { + std::thread::spawn(|| loop { + std::thread::sleep(Duration::from_millis(50)); + // A blocked freeze is not cancellable. The lease starts after it returns. + if let Ok(mut state) = LEASE.try_lock() { + if let Some(lease) = state.as_mut() { + if !lease.expired + && Instant::now() >= lease.deadline + && unsafe { libc::ioctl(lease.root.as_raw_fd(), FITHAW as _, 0) } == 0 + { + lease.expired = true; + BUSY.store(false, Ordering::Release); + } + } + } + }); +} + +fn mount( + source: &std::ffi::CStr, + target: &std::ffi::CStr, + kind: &std::ffi::CStr, + flags: libc::c_ulong, + data: *const libc::c_void, +) -> io::Result<()> { + check(unsafe { libc::mount(source.as_ptr(), target.as_ptr(), kind.as_ptr(), flags, data) }) +} + +pub fn boot() -> io::Result<()> { + if unsafe { libc::getpid() } != 1 { + return Err(invalid("agent must run as PID 1")); + } + let mut fs: libc::statfs = unsafe { std::mem::zeroed() }; + check(unsafe { libc::statfs(c"/".as_ptr(), &mut fs) })?; + if fs.f_type != 0x858458f6 && fs.f_type != 0x01021994 { + return Err(invalid("agent must boot from the image initramfs")); + } + mount(c"devtmpfs", c"/dev", c"devtmpfs", 0, std::ptr::null())?; + let initial = superblock()?; + BOOT_ERRORS.store(initial.errors, Ordering::Release); + mount( + c"/dev/vda", + c"/newroot", + c"ext4", + libc::MS_RDONLY, + c"errors=remount-ro,noinit_itable".as_ptr().cast(), + )?; + // A read-only mount can replay the journal and surface a recorded error. + let sb = superblock()?; + mount( + c"/dev/vda", + c"/newroot", + c"ext4", + libc::MS_REMOUNT, + c"errors=remount-ro,noinit_itable".as_ptr().cast(), + )?; + let root = File::open("/newroot")?; + let device = File::open("/dev/vda")?; + let mut size: u64 = 0; + check(unsafe { libc::ioctl(device.as_raw_fd(), BLKGETSIZE64 as _, &mut size) })?; + check(unsafe { libc::fstatfs(root.as_raw_fd(), &mut fs) })?; + let count = sb.blocks; + let blocks = size / fs.f_bsize as u64; + if blocks < count { + return Err(invalid("root filesystem exceeds device size")); + } + if blocks > count { + check(unsafe { libc::ioctl(root.as_raw_fd(), EXT4_IOC_RESIZE_FS as _, &blocks) })?; + check(unsafe { libc::syncfs(root.as_raw_fd()) })?; + let grown = superblock()?; + let actual = grown.blocks; + if actual != blocks { + return Err(invalid("ext4 could not grow to the requested device size")); + } + } + superblock()?; + drop(device); + drop(root); + check(unsafe { libc::umount(c"/dev".as_ptr()) })?; + std::fs::remove_file("/init")?; + std::fs::remove_dir_all("/dev")?; + check(unsafe { libc::chdir(c"/newroot".as_ptr()) })?; + mount(c".", c"/", c"", libc::MS_MOVE, std::ptr::null())?; + check(unsafe { libc::chroot(c".".as_ptr()) })?; + check(unsafe { libc::chdir(c"/".as_ptr()) })?; + Ok(()) +} diff --git a/crates/shuru-guest/src/ext4.rs b/crates/shuru-guest/src/ext4.rs new file mode 100644 index 0000000..7c9ec59 --- /dev/null +++ b/crates/shuru-guest/src/ext4.rs @@ -0,0 +1,62 @@ +use std::io; + +pub struct Superblock { + pub blocks: u64, + pub errors: u32, + pub needs_recovery: bool, +} + +pub fn parse(sb: &[u8; 1024]) -> io::Result { + let u16_at = |n| u16::from_le_bytes(sb[n..n + 2].try_into().unwrap()); + let u32_at = |n| u32::from_le_bytes(sb[n..n + 4].try_into().unwrap()); + if u16_at(56) != 0xef53 { + return Err(io::Error::other("root device is not ext4")); + } + if u16_at(58) & 2 != 0 { + return Err(io::Error::other("root ext4 EXT4_ERROR_FS flag is set")); + } + let incompat = u32_at(96); + let high = if incompat & 0x80 != 0 { + u32_at(336) as u64 + } else { + 0 + }; + Ok(Superblock { + blocks: u32_at(4) as u64 | (high << 32), + errors: u32_at(404), + needs_recovery: incompat & 4 != 0, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn old_dirty_disks_are_allowed_but_error_flags_are_not() { + let mut sb = [0; 1024]; + assert!(parse(&sb).is_err()); + sb[56..58].copy_from_slice(&0xef53u16.to_le_bytes()); + sb[96] = 4; + assert!(parse(&sb).unwrap().needs_recovery); + sb[58] = 2; + assert!(parse(&sb) + .err() + .unwrap() + .to_string() + .contains("EXT4_ERROR_FS")); + } + + #[test] + fn high_block_count_requires_64bit_feature() { + let mut sb = [0; 1024]; + sb[56..58].copy_from_slice(&0xef53u16.to_le_bytes()); + sb[4] = 1; + sb[336] = 2; + assert_eq!(parse(&sb).unwrap().blocks, 1); + sb[96] = 0x80; + assert_eq!(parse(&sb).unwrap().blocks, (2u64 << 32) + 1); + sb[404] = 3; + assert_eq!(parse(&sb).unwrap().errors, 3); + } +} diff --git a/crates/shuru-guest/src/identity.rs b/crates/shuru-guest/src/identity.rs new file mode 100644 index 0000000..3d76ab3 --- /dev/null +++ b/crates/shuru-guest/src/identity.rs @@ -0,0 +1,59 @@ +use std::io; + +use shuru_proto::hello::Identity; + +fn invalid() -> io::Error { + io::Error::new( + io::ErrorKind::InvalidData, + "missing or invalid guest identity", + ) +} + +fn parameter(cmdline: &str, name: &str) -> io::Result<[u8; N]> { + let mut values = cmdline.split_ascii_whitespace().filter_map(|arg| { + let (key, value) = arg.split_once("=")?; + (key == name).then_some(value) + }); + let value = values.next().ok_or_else(invalid)?; + if value.len() != N * 2 + || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) + || values.next().is_some() + { + return Err(invalid()); + } + let mut bytes = [0; N]; + for (out, pair) in bytes.iter_mut().zip(value.as_bytes().chunks_exact(2)) { + let hex = std::str::from_utf8(pair).map_err(|_| invalid())?; + *out = u8::from_str_radix(hex, 16).map_err(|_| invalid())?; + } + Ok(bytes) +} + +pub fn parse(cmdline: &str) -> io::Result { + Ok(Identity { + token: parameter(cmdline, "shuru.token")?, + instance: parameter(cmdline, "shuru.instance")?, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn parse_identity() { + let cmdline = format!( + "quiet shuru.token={} shuru.instance={} root=/dev/vda", + "ab".repeat(32), + "CD".repeat(16) + ); + let identity = parse(&cmdline).unwrap(); + assert_eq!(identity.token, [0xab; 32]); + assert_eq!(identity.instance, [0xcd; 16]); + assert!(parse(&format!("{cmdline} shuru.token={}", "ab".repeat(32))).is_err()); + assert!(parse(&cmdline.replace("ab", "zz")).is_err()); + assert!(parse(&cmdline.replace("ab", "+1")).is_err()); + assert!(parse(&cmdline.replace("CD", "C")).is_err()); + assert!(parse("").is_err()); + assert!(parse(&format!("shuru.token={}", "00".repeat(32))).is_err()); + } +} diff --git a/crates/shuru-guest/src/main.rs b/crates/shuru-guest/src/main.rs index 179772f..7e6eeac 100644 --- a/crates/shuru-guest/src/main.rs +++ b/crates/shuru-guest/src/main.rs @@ -1,3 +1,15 @@ +#[cfg(any(target_os = "linux", test))] +mod identity; + +#[cfg(any(target_os = "linux", test))] +mod ext4; + +#[cfg(target_os = "linux")] +mod services; + +#[cfg(target_os = "linux")] +mod disk; + #[cfg(target_os = "linux")] mod guest { use std::io::{Read, Write}; @@ -15,6 +27,10 @@ mod guest { }; use shuru_proto::{VSOCK_PORT, VSOCK_PORT_FORWARD}; + /// Process groups of running execs, by the id the host started them with. + static EXECS: std::sync::LazyLock>> = + std::sync::LazyLock::new(Default::default); + fn uptime_ms() -> f64 { let mut ts = libc::timespec { tv_sec: 0, @@ -38,7 +54,7 @@ mod guest { if libc::fcntl(2, libc::F_GETFD) >= 0 { return; } - let fd = libc::open(b"/dev/null\0".as_ptr() as *const libc::c_char, libc::O_RDWR); + let fd = libc::open(c"/dev/null".as_ptr(), libc::O_RDWR); if fd >= 0 { for t in 0..3 { libc::dup2(fd, t); @@ -92,9 +108,9 @@ mod guest { } fn mount_filesystems() { - mount_fs("proc", "/proc", "proc", None); - mount_fs("sysfs", "/sys", "sysfs", None); - mount_fs("devtmpfs", "/dev", "devtmpfs", None); + assert!(mount_fs("proc", "/proc", "proc", None)); + assert!(mount_fs("sysfs", "/sys", "sysfs", None)); + assert!(mount_fs("devtmpfs", "/dev", "devtmpfs", None)); std::fs::create_dir_all("/dev/pts").ok(); mount_fs( "devpts", @@ -103,6 +119,48 @@ mod guest { Some("newinstance,ptmxmode=0666"), ); mount_fs("tmpfs", "/tmp", "tmpfs", None); + // Runtime state lives on tmpfs so pid files and sockets never land in + // a checkpoint. A stale /run/docker.pid otherwise blocks dockerd on + // the next boot. + // Both are guest RAM, so cap them well below the VM memory. + assert!(mount_fs( + "tmpfs", + "/run", + "tmpfs", + Some("mode=0755,size=64m") + )); + std::fs::create_dir_all("/run/lock").ok(); + std::fs::create_dir_all("/dev/shm").ok(); + mount_fs("tmpfs", "/dev/shm", "tmpfs", Some("mode=1777,size=256m")); + mount_cgroup2(); + } + + /// Unified cgroup hierarchy with every controller delegated below the + /// root, which is what runc and containerd expect to find. + fn mount_cgroup2() { + if !mount_fs( + "cgroup2", + "/sys/fs/cgroup", + "cgroup2", + Some("nsdelegate,memory_recursiveprot"), + ) { + return; + } + let controllers = match std::fs::read_to_string("/sys/fs/cgroup/cgroup.controllers") { + Ok(c) => c, + Err(_) => return, + }; + for name in controllers.split_whitespace() { + if let Err(e) = std::fs::write( + "/sys/fs/cgroup/cgroup.subtree_control", + format!("+{}", name), + ) { + eprintln!( + "shuru-guest: failed to enable cgroup controller {}: {}", + name, e + ); + } + } } fn process_mount(req: &MountRequest) -> MountResponse { @@ -189,7 +247,7 @@ mod guest { let copy_len = name.len().min(libc::IFNAMSIZ); std::ptr::copy_nonoverlapping( name.as_ptr(), - ifr.ifr_name.as_mut_ptr() as *mut u8, + ifr.ifr_name.as_mut_ptr().cast::(), copy_len, ); @@ -207,63 +265,61 @@ mod guest { } // --- Networking setup --- - // Network is configured by initramfs before switch_root (static IP for proxy). - // By the time we get here, eth0 already has an IP if --allow-net was used. - fn setup_networking() { unsafe { let sock = libc::socket(libc::AF_INET, libc::SOCK_DGRAM, 0); if sock < 0 { - eprintln!("shuru-guest: failed to create socket for networking setup"); - return; + panic!("network socket: {}", std::io::Error::last_os_error()); } - bring_up_interface(sock, b"lo\0"); - - // Check if eth0 exists (network device present) - let has_eth0 = { - let mut ifr: libc::ifreq = std::mem::zeroed(); - std::ptr::copy_nonoverlapping( - b"eth0\0".as_ptr(), - ifr.ifr_name.as_mut_ptr() as *mut u8, - 5, - ); - libc::ioctl(sock, libc::SIOCGIFFLAGS as _, &mut ifr) == 0 - }; - - if !has_eth0 { + let mut ifr: libc::ifreq = std::mem::zeroed(); + std::ptr::copy_nonoverlapping(c"eth0".as_ptr(), ifr.ifr_name.as_mut_ptr(), 5); + if libc::ioctl(sock, libc::SIOCGIFFLAGS as _, &mut ifr) < 0 { libc::close(sock); - eprintln!("shuru-guest: no network device (sandbox mode)"); return; } - - // Check if eth0 already has an IP (configured by initramfs) - let has_ip = { - let mut ifr: libc::ifreq = std::mem::zeroed(); - std::ptr::copy_nonoverlapping( - b"eth0\0".as_ptr(), - ifr.ifr_name.as_mut_ptr() as *mut u8, - 5, - ); - libc::ioctl(sock, libc::SIOCGIFADDR as _, &mut ifr) == 0 - }; - - libc::close(sock); - - if has_ip { - eprintln!("shuru-guest: network already configured (by initramfs)"); - } else { - eprintln!("shuru-guest: eth0 present but no IP configured"); + fn address(bytes: [u8; 4]) -> libc::sockaddr { + let addr = libc::sockaddr_in { + sin_family: libc::AF_INET as _, + sin_port: 0, + sin_addr: libc::in_addr { + s_addr: u32::from_ne_bytes(bytes), + }, + sin_zero: [0; 8], + }; + unsafe { std::mem::transmute(addr) } } + for (op, addr) in [ + (libc::SIOCSIFADDR, [10, 0, 0, 2]), + (libc::SIOCSIFNETMASK, [255, 255, 255, 0]), + ] { + ifr.ifr_ifru.ifru_addr = address(addr); + if libc::ioctl(sock, op as _, &ifr) < 0 { + panic!("network address: {}", std::io::Error::last_os_error()); + } + } + bring_up_interface(sock, b"eth0\0"); + let mut route: libc::rtentry = std::mem::zeroed(); + route.rt_dst = address([0; 4]); + route.rt_genmask = address([0; 4]); + route.rt_gateway = address([10, 0, 0, 1]); + route.rt_flags = (libc::RTF_UP | libc::RTF_GATEWAY) as _; + if libc::ioctl(sock, libc::SIOCADDRT as _, &route) < 0 { + panic!("network route: {}", std::io::Error::last_os_error()); + } + libc::close(sock); } + std::fs::write("/etc/resolv.conf", "nameserver 10.0.0.1\n").expect("configure DNS"); } fn reap_zombies() { loop { - let ret = unsafe { libc::waitpid(-1, std::ptr::null_mut(), libc::WNOHANG) }; + let mut status: libc::c_int = 0; + let ret = unsafe { libc::waitpid(-1, &mut status, libc::WNOHANG) }; if ret <= 0 { break; } + crate::services::reaped(ret, status); } } @@ -346,20 +402,75 @@ mod guest { } } - fn handle_connection(fd: i32) { + fn authenticate( + stream: &mut std::net::TcpStream, + identity: &shuru_proto::hello::Identity, + ) -> std::io::Result<()> { + let timeout = Some(std::time::Duration::from_secs(2)); + stream.set_read_timeout(timeout)?; + stream.set_write_timeout(timeout)?; + identity.accept(stream)?; + stream.set_read_timeout(None)?; + stream.set_write_timeout(None) + } + + fn handle_connection(fd: i32, identity: shuru_proto::hello::Identity) { // SAFETY: fd is a valid socket from accept() - let stream = unsafe { std::net::TcpStream::from_raw_fd(fd) }; + let mut stream = unsafe { std::net::TcpStream::from_raw_fd(fd) }; + if authenticate(&mut stream, &identity).is_err() { + return; + } let _ = stream.set_nodelay(true); let mut reader = stream.try_clone().expect("failed to clone stream"); let mut writer = stream; - loop { - let (msg_type, payload) = match frame::read_frame(&mut reader) { - Ok(Some(f)) => f, - _ => break, // EOF or error - }; - + // Ends on EOF or a read error. + while let Ok(Some((msg_type, payload))) = frame::read_frame(&mut reader) { + if crate::disk::busy() && !matches!(msg_type, frame::THAW_REQ | frame::SEAL_REQ) { + let _ = frame::write_frame(&mut writer, frame::ERROR, b"root seal is active"); + return; + } match msg_type { + frame::SEAL_REQ | frame::THAW_REQ => { + let result = if msg_type == frame::SEAL_REQ { + serde_json::from_slice::(&payload) + .map_err(std::io::Error::other) + .and_then(crate::disk::seal) + } else { + serde_json::from_slice::(&payload) + .map_err(std::io::Error::other) + .and_then(|r| crate::disk::thaw(r.id)) + }; + match result { + Ok(resp) => { + let _ = frame::send_json(&mut writer, msg_type + 1, &resp); + } + Err(e) => { + let _ = frame::write_frame( + &mut writer, + frame::ERROR, + e.to_string().as_bytes(), + ); + } + } + } + frame::KILL_REQ => { + if let Ok(req) = serde_json::from_slice::(&payload) { + if let Some(pgid) = EXECS.lock().unwrap().get(&req.id) { + unsafe { libc::kill(-pgid, libc::SIGKILL) }; + } + } + let _ = frame::write_frame(&mut writer, frame::KILL_RESP, &[]); + } + frame::TRIM_REQ => match crate::disk::trim() { + Ok(resp) => { + let _ = frame::send_json(&mut writer, frame::TRIM_RESP, &resp); + } + Err(e) => { + let _ = + frame::write_frame(&mut writer, frame::ERROR, e.to_string().as_bytes()); + } + }, frame::MOUNT_REQ => { let mount_req: MountRequest = match serde_json::from_slice(&payload) { Ok(r) => r, @@ -387,6 +498,13 @@ mod guest { continue; } + // Services declared by the image must be ready before a + // command runs against them (RFC 0004). + if let Err(msg) = crate::services::wait_ready() { + let _ = frame::write_frame(&mut writer, frame::ERROR, msg.as_bytes()); + continue; + } + if req.tty.unwrap_or(false) { // TTY mode: hand off the raw fd let raw_fd = std::os::unix::io::AsRawFd::as_raw_fd(&writer); @@ -534,72 +652,86 @@ mod guest { }; handle_download(&req, &mut writer); } + frame::SHUTDOWN_REQ => { + let resp = crate::services::shutdown(); + match crate::disk::sync() { + Ok(()) => { + let _ = frame::send_json(&mut writer, frame::SHUTDOWN_RESP, &resp); + } + Err(e) => { + let _ = frame::write_frame( + &mut writer, + frame::ERROR, + e.to_string().as_bytes(), + ); + } + } + } _ => {} // unknown type, skip } } } fn handle_read_file(req: &ReadFileRequest, writer: &mut impl Write) { - match std::fs::read(&req.path) { - Ok(data) => { - let _ = frame::write_frame(writer, frame::READ_FILE_RESP, &data); - } - Err(e) => { - let msg = format!("read_file {}: {}", req.path, e); - let _ = frame::write_frame(writer, frame::ERROR, msg.as_bytes()); + let fail = |writer: &mut _, e: std::io::Error| { + let msg = format!("read_file {}: {}", req.path, e); + let _ = frame::write_frame(writer, frame::ERROR, msg.as_bytes()); + }; + let mut file = match std::fs::File::open(&req.path) { + Ok(f) => f, + Err(e) => return fail(writer, e), + }; + let mut buf = vec![0u8; frame::FILE_CHUNK]; + loop { + match file.read(&mut buf) { + Ok(n) => { + if frame::write_frame(writer, frame::READ_FILE_RESP, &buf[..n]).is_err() + || n == 0 + { + return; + } + } + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + Err(e) => return fail(writer, e), } } } fn handle_write_file(req: &WriteFileRequest, reader: &mut impl Read, writer: &mut impl Write) { - let data = match frame::read_frame(reader) { - Ok(Some((frame::WRITE_FILE_DATA, payload))) => payload, - _ => { - let resp = WriteFileResponse { - ok: false, - error: Some("expected WRITE_FILE_DATA frame".into()), - }; - let _ = frame::send_json(writer, frame::WRITE_FILE_RESP, &resp); - return; - } - }; - - if data.len() as u64 != req.len { + let respond = |writer: &mut _, error: Option| { let resp = WriteFileResponse { - ok: false, - error: Some(format!( - "length mismatch: expected {}, got {}", - req.len, - data.len() - )), + ok: error.is_none(), + error, }; let _ = frame::send_json(writer, frame::WRITE_FILE_RESP, &resp); - return; - } + }; if let Some(parent) = std::path::Path::new(&req.path).parent() { let _ = std::fs::create_dir_all(parent); } - - match std::fs::write(&req.path, &data) { - Ok(()) => { - unsafe { - libc::sync(); - } - let resp = WriteFileResponse { - ok: true, - error: None, - }; - let _ = frame::send_json(writer, frame::WRITE_FILE_RESP, &resp); + let mut file = + std::fs::File::create(&req.path).map_err(|e| format!("write_file {}: {}", req.path, e)); + + // Every chunk is read even after a failure, so the host gets the + // error rather than a closed connection. + let mut received = 0u64; + while received < req.len { + let chunk = match frame::read_frame(reader) { + Ok(Some((frame::WRITE_FILE_DATA, chunk))) if !chunk.is_empty() => chunk, + _ => return respond(writer, Some("expected WRITE_FILE_DATA frame".into())), + }; + received += chunk.len() as u64; + if received > req.len { + let msg = format!("length mismatch: expected {}, got {}", req.len, received); + return respond(writer, Some(msg)); } - Err(e) => { - let resp = WriteFileResponse { - ok: false, - error: Some(format!("write_file {}: {}", req.path, e)), - }; - let _ = frame::send_json(writer, frame::WRITE_FILE_RESP, &resp); + if let Ok(f) = &mut file { + if let Err(e) = f.write_all(&chunk) { + file = Err(format!("write_file {}: {}", req.path, e)); + } } } + respond(writer, file.err()); } fn send_fs_ok(writer: &mut impl Write) { @@ -1048,6 +1180,9 @@ mod guest { match cmd.spawn() { Ok(mut child) => { let child_pid = child.id() as i32; + if let Some(id) = req.id { + EXECS.lock().unwrap().insert(id, child_pid); + } // Channel serializes all frame writes to prevent interleaving let (tx, rx) = std::sync::mpsc::channel::<(u8, Vec)>(); @@ -1098,6 +1233,18 @@ mod guest { } }); + // The input thread below can block writing to a child that + // never reads its stdin, and then cannot see the host hang + // up. Watch for that separately and kill the group. + let hangup_watch = match (vsock_reader.try_clone(), std::io::pipe()) { + (Ok(conn), Ok((done_rx, done_tx))) => { + let t = + std::thread::spawn(move || kill_on_hangup(conn, child_pid, done_rx)); + Some((t, done_tx)) + } + _ => None, + }; + // Thread: vsock STDIN/KILL frames -> child stdin let child_stdin = child.stdin.take().unwrap(); let input_thread = std::thread::spawn(move || { @@ -1124,12 +1271,27 @@ mod guest { // Wait for output to drain, then wait for child let _ = stdout_thread.join(); let _ = stderr_thread.join(); - let status = child.wait().expect("failed to wait on child"); - let exit_code = status.code().unwrap_or(-1); - + // Forget the id and stop the hangup watch once the child has + // exited but before it is reaped, so a late kill cannot reach + // a reused pid. + let mut info: libc::siginfo_t = unsafe { std::mem::zeroed() }; unsafe { - libc::sync(); + libc::waitid( + libc::P_PID, + child_pid as libc::id_t, + &mut info, + libc::WEXITED | libc::WNOWAIT, + ) + }; + if let Some(id) = req.id { + EXECS.lock().unwrap().remove(&id); + } + if let Some((t, done_tx)) = hangup_watch { + drop(done_tx); + let _ = t.join(); } + let status = child.wait().expect("failed to wait on child"); + let exit_code = status.code().unwrap_or(-1); let _ = tx.send((frame::EXIT, frame::exit_payload(exit_code).to_vec())); drop(tx); @@ -1146,6 +1308,40 @@ mod guest { } } + /// Kill the group when the host hangs up. Returns without killing once + /// the write end of `done` is closed. + fn kill_on_hangup(conn: std::net::TcpStream, pgid: i32, done: std::io::PipeReader) { + use std::os::unix::io::AsRawFd; + let mut fds = [ + libc::pollfd { + fd: done.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }, + libc::pollfd { + fd: conn.as_raw_fd(), + events: libc::POLLRDHUP, + revents: 0, + }, + ]; + loop { + let n = unsafe { libc::poll(fds.as_mut_ptr(), 2, -1) }; + if n < 0 { + if std::io::Error::last_os_error().raw_os_error() == Some(libc::EINTR) { + continue; + } + return; + } + if fds[0].revents != 0 { + return; + } + if fds[1].revents & (libc::POLLRDHUP | libc::POLLHUP | libc::POLLERR) != 0 { + unsafe { libc::kill(-pgid, libc::SIGKILL) }; + return; + } + } + } + /// Find the overlay upper dir for a mount point by parsing /proc/mounts. /// Returns (watch_path, prefix_to_strip, prefix_to_add) if overlay found. fn resolve_overlay_upper(mount_point: &str) -> Option<(String, String, String)> { @@ -1231,6 +1427,15 @@ mod guest { if matches!(event.kind, notify::EventKind::Access(_)) { continue; } + // A rename already arrived as its From and To halves. + if matches!( + event.kind, + notify::EventKind::Modify(notify::event::ModifyKind::Name( + notify::event::RenameMode::Both + )) + ) { + continue; + } for path in &event.paths { if gitignore.matched(path, path.is_dir()).is_ignore() { continue; @@ -1242,10 +1447,13 @@ mod guest { Some(p) => p, None => continue, }; - let kind = if event.kind.is_remove() { - shuru_proto::watch_kind::DELETE - } else { - shuru_proto::watch_kind::MODIFY + let kind = match event.kind { + notify::EventKind::Create(_) => shuru_proto::watch_kind::CREATE, + notify::EventKind::Remove(_) => shuru_proto::watch_kind::DELETE, + notify::EventKind::Modify(notify::event::ModifyKind::Name(_)) => { + shuru_proto::watch_kind::RENAME + } + _ => shuru_proto::watch_kind::MODIFY, }; let evt = WatchEvent { kind, @@ -1507,13 +1715,6 @@ mod guest { libc::waitpid(child_pid, &mut status, 0); } - // Flush all filesystem writes to disk before reporting exit. - // Without this, data can be lost if the VM is stopped immediately - // after the exit code is sent (e.g. during checkpoint create). - unsafe { - libc::sync(); - } - let exit_code = if libc::WIFEXITED(status) { libc::WEXITSTATUS(status) } else if libc::WIFSIGNALED(status) { @@ -1525,7 +1726,7 @@ mod guest { write_frame_fd(vsock_fd, frame::EXIT, &frame::exit_payload(exit_code)); } - fn forward_accept_loop(listener_fd: i32) { + fn forward_accept_loop(listener_fd: i32, identity: shuru_proto::hello::Identity) { loop { let client_fd = unsafe { libc::accept(listener_fd, std::ptr::null_mut(), std::ptr::null_mut()) }; @@ -1535,13 +1736,16 @@ mod guest { } std::thread::spawn(move || { - handle_forward_connection(client_fd); + handle_forward_connection(client_fd, identity); }); } } - fn handle_forward_connection(fd: i32) { + fn handle_forward_connection(fd: i32, identity: shuru_proto::hello::Identity) { let mut stream = unsafe { std::net::TcpStream::from_raw_fd(fd) }; + if authenticate(&mut stream, &identity).is_err() { + return; + } let _ = stream.set_nodelay(true); // Read the forward request frame @@ -1628,10 +1832,36 @@ mod guest { "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", ); + if let Err(e) = crate::disk::boot() { + stage(&format!("root boot failed: {e}")); + unsafe { + libc::reboot(libc::LINUX_REBOOT_CMD_POWER_OFF); + } + loop { + std::thread::park(); + } + } mount_filesystems(); + if let Err(e) = crate::disk::check_root() { + stage(&format!("root boot failed: {e}")); + unsafe { + libc::reboot(libc::LINUX_REBOOT_CMD_POWER_OFF); + } + loop { + std::thread::park(); + } + } + crate::disk::start_watchdog(); ensure_stdio(); stage("filesystems mounted"); + let identity = std::fs::read_to_string("/proc/cmdline") + .and_then(|cmdline| crate::identity::parse(&cmdline)) + .unwrap_or_else(|_| { + eprintln!("shuru-guest: missing or invalid guest identity"); + std::process::exit(1); + }); + // Set hostname let hostname = b"shuru\0"; unsafe { @@ -1641,6 +1871,11 @@ mod guest { setup_networking(); stage("networking ready"); + let started = crate::services::start_all(); + if started > 0 { + stage(&format!("{} service(s) starting", started)); + } + // Register signal handlers (PID 1 has no default signal dispositions) unsafe { libc::signal( @@ -1666,7 +1901,7 @@ mod guest { VSOCK_PORT_FORWARD )); std::thread::spawn(move || { - forward_accept_loop(fwd_listener_fd); + forward_accept_loop(fwd_listener_fd, identity); }); loop { @@ -1681,7 +1916,7 @@ mod guest { eprintln!("shuru-guest: accepted vsock connection"); std::thread::spawn(move || { - handle_connection(client_fd); + handle_connection(client_fd, identity); }); reap_zombies(); diff --git a/crates/shuru-guest/src/services.rs b/crates/shuru-guest/src/services.rs new file mode 100644 index 0000000..54448f0 --- /dev/null +++ b/crates/shuru-guest/src/services.rs @@ -0,0 +1,483 @@ +//! Guest services (RFC 0004): long running processes declared by the image +//! under /etc/shuru/services/.json. Init starts them at boot, gates +//! exec requests on their readiness, and stops them on SHUTDOWN_REQ. + +use std::collections::HashMap; +use std::os::unix::process::CommandExt; +use std::process::{Command, Stdio}; +use std::sync::{Condvar, Mutex}; +use std::time::{Duration, Instant}; + +use shuru_proto::ShutdownResponse; + +const SERVICES_DIR: &str = "/etc/shuru/services"; +const LOG_DIR: &str = "/run/shuru/log"; +const READY_POLL: Duration = Duration::from_millis(250); +const DEFAULT_READY_TIMEOUT: u64 = 30; +const DEFAULT_STOP_TIMEOUT: u64 = 15; +const LOG_TAIL_LINES: usize = 20; + +#[derive(Clone, Debug, PartialEq)] +enum State { + Starting, + Ready, + Failed(String), + Stopped, +} + +struct Spec { + name: String, + exec: Vec, + ready: Option>, + ready_timeout: Duration, + stop_timeout: Duration, + env: HashMap, + cwd: String, +} + +struct Service { + name: String, + pid: i32, + state: State, + stop_timeout: Duration, +} + +struct Registry { + services: Vec, + shutting_down: bool, +} + +static REGISTRY: Mutex = Mutex::new(Registry { + services: Vec::new(), + shutting_down: false, +}); +static CHANGED: Condvar = Condvar::new(); + +fn log_line(msg: &str) { + let s = format!("shuru-guest: {}\n", msg); + unsafe { libc::write(2, s.as_ptr() as *const libc::c_void, s.len()) }; +} + +fn registry() -> std::sync::MutexGuard<'static, Registry> { + REGISTRY.lock().unwrap_or_else(|e| e.into_inner()) +} + +fn valid_name(name: &str) -> bool { + let mut chars = name.chars(); + match chars.next() { + Some(c) if c.is_ascii_lowercase() || c.is_ascii_digit() => {} + _ => return false, + } + chars.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') +} + +fn argv_field(value: &serde_json::Value, key: &str) -> Result>, String> { + match value.get(key) { + None | Some(serde_json::Value::Null) => Ok(None), + Some(serde_json::Value::Array(items)) => { + let mut argv = Vec::with_capacity(items.len()); + for item in items { + match item.as_str() { + Some(s) => argv.push(s.to_string()), + None => return Err(format!("{} must be an array of strings", key)), + } + } + if argv.is_empty() { + return Err(format!("{} must not be empty", key)); + } + Ok(Some(argv)) + } + Some(_) => Err(format!("{} must be an array of strings", key)), + } +} + +fn secs_field(value: &serde_json::Value, key: &str, default: u64) -> Result { + match value.get(key) { + None | Some(serde_json::Value::Null) => Ok(Duration::from_secs(default)), + Some(v) => match v.as_u64() { + Some(n) => Ok(Duration::from_secs(n)), + None => Err(format!("{} must be a non-negative integer", key)), + }, + } +} + +fn parse_spec(name: &str, text: &str) -> Result { + let value: serde_json::Value = serde_json::from_str(text).map_err(|e| e.to_string())?; + if !value.is_object() { + return Err("service file must be a JSON object".into()); + } + let exec = argv_field(&value, "exec")?.ok_or("exec is required")?; + let ready = argv_field(&value, "ready")?; + let ready_timeout = secs_field(&value, "ready_timeout_secs", DEFAULT_READY_TIMEOUT)?; + let stop_timeout = secs_field(&value, "stop_timeout_secs", DEFAULT_STOP_TIMEOUT)?; + let mut env = HashMap::new(); + if let Some(obj) = value.get("env") { + match obj.as_object() { + Some(map) => { + for (k, v) in map { + match v.as_str() { + Some(s) => { + env.insert(k.clone(), s.to_string()); + } + None => return Err("env values must be strings".into()), + } + } + } + None if obj.is_null() => {} + None => return Err("env must be an object".into()), + } + } + let cwd = match value.get("cwd") { + None | Some(serde_json::Value::Null) => "/".to_string(), + Some(v) => v + .as_str() + .map(str::to_string) + .ok_or("cwd must be a string")?, + }; + Ok(Spec { + name: name.to_string(), + exec, + ready, + ready_timeout, + stop_timeout, + env, + cwd, + }) +} + +fn load_specs() -> Vec { + let entries = match std::fs::read_dir(SERVICES_DIR) { + Ok(e) => e, + Err(_) => return Vec::new(), + }; + let mut specs = Vec::new(); + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) != Some("json") { + continue; + } + let name = match path.file_stem().and_then(|s| s.to_str()) { + Some(n) if valid_name(n) => n.to_string(), + _ => { + log_line(&format!( + "ignoring service file {}: bad name", + path.display() + )); + continue; + } + }; + let text = match std::fs::read_to_string(&path) { + Ok(t) => t, + Err(e) => { + log_line(&format!("ignoring service {}: {}", name, e)); + continue; + } + }; + match parse_spec(&name, &text) { + Ok(spec) => specs.push(spec), + Err(e) => log_line(&format!("ignoring service {}: {}", name, e)), + } + } + specs.sort_by(|a, b| a.name.cmp(&b.name)); + specs +} + +fn log_path(name: &str) -> String { + format!("{}/{}.log", LOG_DIR, name) +} + +fn spawn(spec: &Spec) -> std::io::Result { + std::fs::create_dir_all(LOG_DIR)?; + let log = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(log_path(&spec.name))?; + let log_err = log.try_clone()?; + let mut cmd = Command::new(&spec.exec[0]); + cmd.args(&spec.exec[1..]); + cmd.envs(&spec.env); + cmd.current_dir(&spec.cwd); + cmd.stdin(Stdio::null()); + cmd.stdout(Stdio::from(log)); + cmd.stderr(Stdio::from(log_err)); + // Own process group so a stop can reach anything the service forked. + unsafe { + cmd.pre_exec(|| { + libc::setpgid(0, 0); + Ok(()) + }); + } + let child = cmd.spawn()?; + Ok(child.id() as i32) +} + +/// Non-blocking exit check. `None` while running, `Some(status)` once the +/// process is gone. A status of -1 means the main loop reaped it first. +fn poll_exit(pid: i32) -> Option { + let mut status: libc::c_int = 0; + let r = unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) }; + if r == pid { + Some(status) + } else if r < 0 { + Some(-1) + } else { + None + } +} + +fn describe_exit(status: i32) -> String { + if status < 0 { + "exited".to_string() + } else if libc::WIFEXITED(status) { + format!("exited with status {}", libc::WEXITSTATUS(status)) + } else if libc::WIFSIGNALED(status) { + format!("killed by signal {}", libc::WTERMSIG(status)) + } else { + "exited".to_string() + } +} + +fn set_state(name: &str, state: State) { + let mut reg = registry(); + if let Some(svc) = reg.services.iter_mut().find(|s| s.name == name) { + if matches!(svc.state, State::Stopped) { + return; + } + svc.state = state; + } + drop(reg); + CHANGED.notify_all(); +} + +/// Called by the PID 1 reaper for every child it collects, so a service +/// that dies at any point is recorded even when no thread is polling it. +pub fn reaped(pid: i32, status: i32) { + let name = { + let reg = registry(); + if reg.shutting_down { + return; + } + reg.services + .iter() + .find(|s| s.pid == pid && s.state != State::Stopped) + .map(|s| s.name.clone()) + }; + if let Some(name) = name { + log_line(&format!("service {} {}", name, describe_exit(status))); + set_state(&name, State::Failed(describe_exit(status))); + } +} + +fn ready_check_passes(spec: &Spec, ready: &[String]) -> bool { + Command::new(&ready[0]) + .args(&ready[1..]) + .envs(&spec.env) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map(|s| s.success()) + .unwrap_or(false) +} + +fn watch_until_ready(spec: Spec, pid: i32) { + let deadline = Instant::now() + spec.ready_timeout; + loop { + if let Some(status) = poll_exit(pid) { + set_state(&spec.name, State::Failed(describe_exit(status))); + log_line(&format!("service {} {}", spec.name, describe_exit(status))); + return; + } + match &spec.ready { + None => { + set_state(&spec.name, State::Ready); + return; + } + Some(ready) => { + if ready_check_passes(&spec, ready) { + set_state(&spec.name, State::Ready); + log_line(&format!("service {} ready", spec.name)); + return; + } + } + } + if Instant::now() >= deadline { + let msg = format!("not ready after {}s", spec.ready_timeout.as_secs()); + log_line(&format!("service {} {}", spec.name, msg)); + set_state(&spec.name, State::Failed(msg)); + return; + } + std::thread::sleep(READY_POLL); + } +} + +/// Spawn every declared service. Returns the number started. +pub fn start_all() -> usize { + let specs = load_specs(); + let mut started = 0; + for spec in specs { + let pid = match spawn(&spec) { + Ok(pid) => pid, + Err(e) => { + log_line(&format!("service {} failed to start: {}", spec.name, e)); + registry().services.push(Service { + name: spec.name.clone(), + pid: 0, + state: State::Failed(format!("failed to start: {}", e)), + stop_timeout: spec.stop_timeout, + }); + continue; + } + }; + registry().services.push(Service { + name: spec.name.clone(), + pid, + state: State::Starting, + stop_timeout: spec.stop_timeout, + }); + started += 1; + std::thread::spawn(move || watch_until_ready(spec, pid)); + } + started +} + +fn log_tail(name: &str) -> String { + let text = std::fs::read_to_string(log_path(name)).unwrap_or_default(); + let lines: Vec<&str> = text.lines().collect(); + let start = lines.len().saturating_sub(LOG_TAIL_LINES); + lines[start..].join("\n") +} + +/// Pick up exits the reaper has not seen yet, so a service that died +/// between two connections is reported on the next exec. +fn poll_all() { + let pids: Vec<(String, i32)> = registry() + .services + .iter() + .filter(|s| s.pid > 0 && matches!(s.state, State::Starting | State::Ready)) + .map(|s| (s.name.clone(), s.pid)) + .collect(); + for (name, pid) in pids { + if let Some(status) = poll_exit(pid) { + if status >= 0 { + log_line(&format!("service {} {}", name, describe_exit(status))); + set_state(&name, State::Failed(describe_exit(status))); + } + } + } +} + +/// Block until no service is still starting. Ok when every service is ready +/// (or none exist), Err with a message naming the first failed service. +pub fn wait_ready() -> Result<(), String> { + if registry().shutting_down { + return Err("guest is shutting down".into()); + } + poll_all(); + let mut reg = registry(); + loop { + if reg.shutting_down { + return Err("guest is shutting down".into()); + } + if reg.services.iter().any(|s| s.state == State::Starting) { + reg = CHANGED + .wait_timeout(reg, Duration::from_millis(200)) + .unwrap_or_else(|e| e.into_inner()) + .0; + continue; + } + let failed = reg.services.iter().find_map(|s| match &s.state { + State::Failed(reason) => Some((s.name.clone(), reason.clone())), + _ => None, + }); + drop(reg); + return match failed { + None => Ok(()), + Some((name, reason)) => { + let tail = log_tail(&name); + let mut msg = format!("service {} {}", name, reason); + if !tail.is_empty() { + msg.push_str(&format!("\n--- {} ---\n{}\n", log_path(&name), tail)); + } + Err(msg) + } + }; + } +} + +/// Stop every running service (SIGTERM, then SIGKILL to its process group +/// after its stop timeout). The disk control path handles synchronization. +pub fn shutdown() -> ShutdownResponse { + let targets: Vec<(String, i32, Duration)> = { + let mut reg = registry(); + reg.shutting_down = true; + reg.services + .iter() + .filter(|s| s.pid > 0 && matches!(s.state, State::Starting | State::Ready)) + .map(|s| (s.name.clone(), s.pid, s.stop_timeout)) + .collect() + }; + CHANGED.notify_all(); + + let mut resp = ShutdownResponse::default(); + if !targets.is_empty() { + let start = Instant::now(); + for (_, pid, _) in &targets { + unsafe { libc::kill(*pid, libc::SIGTERM) }; + } + let mut pending: Vec<(String, i32, Duration)> = targets; + loop { + pending.retain(|(name, pid, _)| match poll_exit(*pid) { + Some(_) => { + resp.stopped.push(name.clone()); + false + } + None => true, + }); + if pending.is_empty() { + break; + } + let elapsed = start.elapsed(); + let mut expired = Vec::new(); + pending.retain(|(name, pid, timeout)| { + if elapsed >= *timeout { + unsafe { + libc::kill(-*pid, libc::SIGKILL); + libc::kill(*pid, libc::SIGKILL); + } + expired.push((name.clone(), *pid)); + false + } else { + true + } + }); + for (name, pid) in expired { + log_line(&format!("service {} ignored SIGTERM, killed", name)); + // Reap the killed process so it does not linger as a zombie. + for _ in 0..100 { + if poll_exit(pid).is_some() { + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + resp.killed.push(name); + } + if pending.is_empty() { + break; + } + std::thread::sleep(Duration::from_millis(50)); + } + resp.stopped.sort(); + resp.killed.sort(); + for name in resp.stopped.iter().chain(resp.killed.iter()) { + set_state(name, State::Stopped); + } + log_line(&format!( + "services stopped in {:.1}s (stopped: {:?}, killed: {:?})", + start.elapsed().as_secs_f64(), + resp.stopped, + resp.killed + )); + } + resp +} diff --git a/crates/shuru-linux/Cargo.toml b/crates/shuru-linux/Cargo.toml index 919810f..26b83cd 100644 --- a/crates/shuru-linux/Cargo.toml +++ b/crates/shuru-linux/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuru-linux" -version = "0.2.1" +version = "0.4.1" edition = "2021" description = "KVM backend for shuru" license = "Apache-2.0" diff --git a/crates/shuru-linux/src/anonfile.rs b/crates/shuru-linux/src/anonfile.rs new file mode 100644 index 0000000..909775b --- /dev/null +++ b/crates/shuru-linux/src/anonfile.rs @@ -0,0 +1,44 @@ +//! Nameless files and reflinks. + +use std::ffi::CString; +use std::fs::File; +use std::io; +use std::os::unix::io::{AsRawFd, FromRawFd}; + +/// A read-write file with no name on the filesystem holding `dir`, via +/// `O_TMPFILE`. `Ok(None)` when the filesystem does not support it. +pub fn open_anonymous(dir: &str) -> io::Result> { + let c_dir = CString::new(dir) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path holds a NUL"))?; + let fd = unsafe { + libc::open( + c_dir.as_ptr(), + libc::O_TMPFILE | libc::O_RDWR | libc::O_CLOEXEC, + 0o600, + ) + }; + if fd >= 0 { + return Ok(Some(unsafe { File::from_raw_fd(fd) })); + } + let err = io::Error::last_os_error(); + match err.raw_os_error() { + Some(libc::EOPNOTSUPP) | Some(libc::EISDIR) | Some(libc::EINVAL) => Ok(None), + _ => Err(err), + } +} + +/// Share the blocks of `src` with `dst` (`FICLONE`). `Ok(false)` when the +/// filesystem cannot, so the caller copies instead. +pub fn reflink(dst: &File, src: &File) -> io::Result { + let ret = unsafe { libc::ioctl(dst.as_raw_fd(), libc::FICLONE as _, src.as_raw_fd()) }; + if ret == 0 { + return Ok(true); + } + let err = io::Error::last_os_error(); + match err.raw_os_error() { + Some(libc::EOPNOTSUPP) | Some(libc::ENOTTY) | Some(libc::EXDEV) | Some(libc::EINVAL) => { + Ok(false) + } + _ => Err(err), + } +} diff --git a/crates/shuru-linux/src/bootloader.rs b/crates/shuru-linux/src/bootloader.rs index a38d70a..2dcc6ff 100644 --- a/crates/shuru-linux/src/bootloader.rs +++ b/crates/shuru-linux/src/bootloader.rs @@ -34,9 +34,7 @@ impl LinuxBootLoader { pub fn set_command_line(&self, command_line: &str) { // On KVM we use PL011 UART (ttyAMA0) instead of virtio console (hvc0). // Add earlycon for immediate output from the first instruction. - let adjusted = command_line - .replace("console=hvc0", "console=ttyAMA0") - .replace(" quiet", ""); + let adjusted = command_line.replace("console=hvc0", "console=ttyAMA0"); let with_earlycon = format!("{} earlycon=pl011,mmio,0x09000000", adjusted); *self.command_line.borrow_mut() = Some(with_earlycon); } diff --git a/crates/shuru-linux/src/configuration.rs b/crates/shuru-linux/src/configuration.rs index 3076400..39646dd 100644 --- a/crates/shuru-linux/src/configuration.rs +++ b/crates/shuru-linux/src/configuration.rs @@ -25,6 +25,7 @@ pub(crate) struct ConfigData { pub network_mac: Option<[u8; 6]>, pub has_socket: bool, pub mounts: Vec<(String, String, bool)>, // (tag, host_path, read_only) + pub has_balloon: bool, } pub struct VirtualMachineConfiguration { @@ -52,6 +53,7 @@ impl VirtualMachineConfiguration { network_mac: None, has_socket: false, mounts: Vec::new(), + has_balloon: false, }), } } @@ -119,8 +121,8 @@ impl VirtualMachineConfiguration { // Entropy device is a no-op on KVM — /dev/urandom is available. } - pub fn set_memory_balloon_devices(&self, _devices: &[VirtioMemoryBalloonDevice]) { - // Memory balloon is not yet implemented for KVM. + pub fn set_memory_balloon_devices(&self, devices: &[VirtioMemoryBalloonDevice]) { + self.inner.borrow_mut().has_balloon = !devices.is_empty(); } pub fn validate(&self) -> Result<()> { diff --git a/crates/shuru-linux/src/kvm/balloon.rs b/crates/shuru-linux/src/kvm/balloon.rs new file mode 100644 index 0000000..8a8ac8f --- /dev/null +++ b/crates/shuru-linux/src/kvm/balloon.rs @@ -0,0 +1,274 @@ +//! Virtio memory balloon (device ID 5). +//! +//! Memory goes back to the host two ways: +//! +//! - Free page reporting (`VIRTIO_BALLOON_F_REPORTING`). The guest kernel +//! hands over runs of free pages a couple of seconds after they free up +//! and the host discards them with `MADV_DONTNEED`. Nothing has to ask +//! and the guest keeps its full memory size. +//! - A memory target. The host lowers how much memory the guest keeps, +//! the guest driver allocates the difference and sends the page numbers +//! over the inflate queue, and the host discards those pages too. +//! Raising the target gives them back through the deflate queue. +//! +//! A discarded page reads as zero when the guest touches it again, so the +//! only cost to the guest is a page fault. The guest may also deflate on +//! its own under memory pressure (`VIRTIO_BALLOON_F_DEFLATE_ON_OOM`). + +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::{Arc, Mutex}; + +use vm_memory::{Bytes, GuestAddress, GuestMemory, GuestMemoryMmap}; + +use super::devices::{ + avail_ring_entry, write_used, Interrupt, VirtioBackend, VirtioQueueState, INTERRUPT_CONFIG, + VIRTIO_F_VERSION_1, VRING_DESC_F_NEXT, VRING_DESC_F_WRITE, +}; + +const VIRTIO_BALLOON_F_STATS_VQ: u64 = 1 << 1; +const VIRTIO_BALLOON_F_DEFLATE_ON_OOM: u64 = 1 << 2; +const VIRTIO_BALLOON_F_FREE_PAGE_HINT: u64 = 1 << 3; +const VIRTIO_BALLOON_F_REPORTING: u64 = 1 << 5; + +const OFFERED_FEATURES: u64 = + VIRTIO_F_VERSION_1 | VIRTIO_BALLOON_F_DEFLATE_ON_OOM | VIRTIO_BALLOON_F_REPORTING; + +/// Balloon pages are always 4 KiB, whatever the guest page size. +const PAGE_SHIFT: u32 = 12; +const MAX_INFLATE_PFNS: u32 = 256; +const PAGE_SIZE: u64 = 1 << PAGE_SHIFT; + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum QueueRole { + Inflate, + Deflate, + Stats, + FreePageHint, + Reporting, +} + +/// The optional queues in spec order. Each exists only when its feature is +/// negotiated, and a queue that does not exist takes no index, so the ones +/// after it move up. The spec numbers them 2, 3 and 4, which is their +/// position only when all three are negotiated. +const OPTIONAL_QUEUES: [(u64, QueueRole); 3] = [ + (VIRTIO_BALLOON_F_STATS_VQ, QueueRole::Stats), + (VIRTIO_BALLOON_F_FREE_PAGE_HINT, QueueRole::FreePageHint), + (VIRTIO_BALLOON_F_REPORTING, QueueRole::Reporting), +]; + +struct QueueLayout { + roles: Vec, +} + +impl QueueLayout { + fn for_features(features: u64) -> Self { + let mut roles = vec![QueueRole::Inflate, QueueRole::Deflate]; + roles.extend( + OPTIONAL_QUEUES + .iter() + .filter(|(bit, _)| features & bit != 0) + .map(|(_, role)| *role), + ); + QueueLayout { roles } + } + + fn role(&self, idx: u16) -> Option { + self.roles.get(usize::from(idx)).copied() + } + + fn len(&self) -> usize { + self.roles.len() + } +} + +/// Host side of the balloon: set a target, read what the guest gave up. +pub struct BalloonControl { + total_bytes: u64, + target_pages: AtomicU32, + actual_pages: AtomicU32, + interrupt: Mutex>>, +} + +impl BalloonControl { + /// How much memory the guest should keep. Rounded down to a page and + /// clamped to the configured size; the guest responds in its own time. + pub fn set_memory_target(&self, bytes: u64) { + let keep = bytes.min(self.total_bytes); + let pages = ((self.total_bytes - keep) / PAGE_SIZE) as u32; + self.target_pages.store(pages, Ordering::Release); + if let Some(interrupt) = self.interrupt.lock().unwrap().as_ref() { + interrupt.raise(INTERRUPT_CONFIG); + } + } + + /// Bytes the guest has handed over through the target so far. Free + /// page reporting is not counted: those pages still belong to the guest. + pub fn ballooned_bytes(&self) -> u64 { + self.actual_pages.load(Ordering::Acquire) as u64 * PAGE_SIZE + } +} + +pub struct BalloonBackend { + control: Arc, + host_page: u64, + layout: QueueLayout, +} + +impl BalloonBackend { + pub fn new(total_bytes: u64) -> Self { + let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + BalloonBackend { + control: Arc::new(BalloonControl { + total_bytes, + target_pages: AtomicU32::new(0), + actual_pages: AtomicU32::new(0), + interrupt: Mutex::new(None), + }), + host_page: if page > 0 { page as u64 } else { PAGE_SIZE }, + layout: QueueLayout::for_features(OFFERED_FEATURES), + } + } + + pub fn control(&self) -> Arc { + self.control.clone() + } + + /// Give the host pages fully inside `[gpa, gpa + len)` back. A guest + /// page smaller than a host page cannot go alone, since the rest of the + /// host page may still be in use. + fn discard(&self, mem: &GuestMemoryMmap, gpa: u64, len: u64) { + let mask = !(self.host_page - 1); + let start = (gpa + self.host_page - 1) & mask; + let end = gpa.saturating_add(len) & mask; + if end <= start { + return; + } + let len = (end - start) as usize; + if !mem.check_range(GuestAddress(start), len) { + return; + } + if let Ok(ptr) = mem.get_host_address(GuestAddress(start)) { + unsafe { + libc::madvise(ptr as *mut libc::c_void, len, libc::MADV_DONTNEED); + } + } + } + + /// An inflate buffer is an array of 4 KiB page numbers. Consecutive + /// pages are discarded as one run. + fn discard_pfns(&self, mem: &GuestMemoryMmap, addr: u64, len: u32) { + let mut run: Option<(u64, u64)> = None; + for i in 0..(len / 4) as u64 { + let pfn = match mem.read_obj::(GuestAddress(addr + i * 4)) { + Ok(pfn) => pfn as u64, + Err(_) => break, + }; + let gpa = pfn << PAGE_SHIFT; + run = match run { + Some((start, run_len)) if start + run_len == gpa => { + Some((start, run_len + PAGE_SIZE)) + } + Some((start, run_len)) => { + self.discard(mem, start, run_len); + Some((gpa, PAGE_SIZE)) + } + None => Some((gpa, PAGE_SIZE)), + }; + } + if let Some((start, run_len)) = run { + self.discard(mem, start, run_len); + } + } + + fn drain(&self, role: QueueRole, q: &mut VirtioQueueState, mem: &GuestMemoryMmap) -> bool { + let current = match q.pending(mem) { + Some(idx) => idx, + None => return false, + }; + let used = q.last_avail_idx != current; + while q.last_avail_idx != current { + let head = avail_ring_entry(mem, q.avail_addr, q.size, q.last_avail_idx); + let chain = q + .chain(mem, head, VRING_DESC_F_NEXT | VRING_DESC_F_WRITE) + .unwrap_or_default(); + for desc in chain { + match role { + // Linux sends at most 256 page numbers per buffer. + QueueRole::Inflate => { + self.discard_pfns(mem, desc.addr, desc.len.min(MAX_INFLATE_PFNS * 4)) + } + QueueRole::Deflate => {} + QueueRole::Reporting => self.discard(mem, desc.addr, desc.len as u64), + QueueRole::Stats | QueueRole::FreePageHint => { + unreachable!("the balloon does not offer stats or free page hints") + } + } + } + write_used(mem, q.used_addr, q.size, q.last_avail_idx, head as u32, 0); + q.last_avail_idx = q.last_avail_idx.wrapping_add(1); + } + used + } +} + +impl VirtioBackend for BalloonBackend { + fn device_id(&self) -> u32 { + 5 + } + + fn device_features(&self) -> u64 { + OFFERED_FEATURES + } + + fn queue_count(&self) -> usize { + QueueLayout::for_features(OFFERED_FEATURES).len() + } + + fn queue_max_size(&self) -> u16 { + 128 + } + + fn config_read(&self, offset: u64) -> u32 { + match offset { + 0 => self.control.target_pages.load(Ordering::Acquire), + 4 => self.control.actual_pages.load(Ordering::Acquire), + _ => 0, + } + } + + fn config_write(&mut self, offset: u64, value: u32) { + if offset == 4 { + self.control.actual_pages.store(value, Ordering::Release); + } + } + + fn features_ok(&mut self, features: u64) { + self.layout = QueueLayout::for_features(features); + } + + fn activate(&mut self, _: &[VirtioQueueState], _: &GuestMemoryMmap, interrupt: Arc) { + *self.control.interrupt.lock().unwrap() = Some(interrupt); + } + + fn process_queue( + &mut self, + queue_idx: u16, + queues: &mut [VirtioQueueState], + mem: &GuestMemoryMmap, + ) -> bool { + let Some(role) = self.layout.role(queue_idx) else { + return false; + }; + match queues.get_mut(usize::from(queue_idx)) { + Some(q) if q.ready => self.drain(role, q, mem), + _ => false, + } + } + + fn reset(&mut self) { + *self.control.interrupt.lock().unwrap() = None; + self.control.actual_pages.store(0, Ordering::Release); + self.layout = QueueLayout::for_features(OFFERED_FEATURES); + } +} diff --git a/crates/shuru-linux/src/kvm/block_ranges.rs b/crates/shuru-linux/src/kvm/block_ranges.rs new file mode 100644 index 0000000..578dc53 --- /dev/null +++ b/crates/shuru-linux/src/kvm/block_ranges.rs @@ -0,0 +1,109 @@ +pub(super) const VIRTIO_BLK_T_IN: u32 = 0; +pub(super) const VIRTIO_BLK_T_OUT: u32 = 1; +pub(super) const VIRTIO_BLK_T_DISCARD: u32 = 11; +pub(super) const VIRTIO_BLK_T_WRITE_ZEROES: u32 = 13; +pub(super) const VIRTIO_BLK_S_UNSUPP: u8 = 2; +pub(super) const MAX_RANGES: usize = 32; +pub(super) const MAX_RANGE_SECTORS: u32 = 2 * 1024 * 1024; + +pub(super) fn decode_ranges( + payload: &[u8], + capacity: u64, + kind: u32, +) -> Result, u8> { + if payload.is_empty() || !payload.len().is_multiple_of(16) || payload.len() / 16 > MAX_RANGES { + return Err(VIRTIO_BLK_S_IOERR); + } + let mut ranges = Vec::new(); + for bytes in payload.chunks_exact(16) { + let sector = u64::from_le_bytes(bytes[..8].try_into().unwrap()); + let count = u32::from_le_bytes(bytes[8..12].try_into().unwrap()); + let flags = u32::from_le_bytes(bytes[12..16].try_into().unwrap()); + if (kind == VIRTIO_BLK_T_DISCARD && flags != 0) || flags & !1 != 0 { + return Err(VIRTIO_BLK_S_UNSUPP); + } + if count > MAX_RANGE_SECTORS || sector > capacity || count as u64 > capacity - sector { + return Err(VIRTIO_BLK_S_IOERR); + } + let end = sector + .checked_add(count as u64) + .and_then(|s| s.checked_mul(512)) + .filter(|n| *n <= i64::MAX as u64) + .ok_or(VIRTIO_BLK_S_IOERR)?; + let len = count as i64 * 512; + ranges.push((end as i64 - len, len, flags & 1 != 0)); + } + Ok(ranges) +} + +pub(super) const VIRTIO_BLK_S_OK: u8 = 0; +pub(super) const VIRTIO_BLK_S_IOERR: u8 = 1; + +#[cfg(test)] +mod tests { + use super::*; + + fn range(sector: u64, count: u32, flags: u32) -> Vec { + [ + sector.to_le_bytes().as_slice(), + count.to_le_bytes().as_slice(), + flags.to_le_bytes().as_slice(), + ] + .concat() + } + + #[test] + fn validates_all_ranges_and_signed_host_offsets() { + assert_eq!( + decode_ranges(&range(8, 8, 1), 16, VIRTIO_BLK_T_WRITE_ZEROES), + Ok(vec![(4096, 4096, true)]) + ); + assert_eq!( + decode_ranges(&range(8, 9, 0), 16, VIRTIO_BLK_T_DISCARD), + Err(VIRTIO_BLK_S_IOERR) + ); + assert_eq!( + decode_ranges(&range(u64::MAX, 1, 0), u64::MAX, VIRTIO_BLK_T_DISCARD), + Err(VIRTIO_BLK_S_IOERR) + ); + assert_eq!( + decode_ranges( + &range(i64::MAX as u64 / 512, 1, 0), + u64::MAX, + VIRTIO_BLK_T_DISCARD + ), + Err(VIRTIO_BLK_S_IOERR) + ); + let mut payload = range(0, 8, 0); + payload.extend(range(15, 2, 0)); + assert_eq!( + decode_ranges(&payload, 16, VIRTIO_BLK_T_DISCARD), + Err(VIRTIO_BLK_S_IOERR) + ); + } + + #[test] + fn enforces_segment_count_length_flags_and_limits() { + for payload in [ + vec![], + vec![0; 15], + vec![0; (MAX_RANGES + 1) * 16], + range(0, MAX_RANGE_SECTORS + 1, 0), + ] { + assert_eq!( + decode_ranges(&payload, u64::MAX, VIRTIO_BLK_T_DISCARD), + Err(VIRTIO_BLK_S_IOERR) + ); + } + assert_eq!( + decode_ranges(&range(0, 8, 1), 16, VIRTIO_BLK_T_DISCARD), + Err(VIRTIO_BLK_S_UNSUPP) + ); + assert_eq!( + decode_ranges(&range(0, 8, 2), 16, VIRTIO_BLK_T_WRITE_ZEROES), + Err(VIRTIO_BLK_S_UNSUPP) + ); + assert!(decode_ranges(&vec![0; MAX_RANGES * 16], 16, VIRTIO_BLK_T_DISCARD).is_ok()); + assert!(decode_ranges(&range(16, 0, 0), 16, VIRTIO_BLK_T_DISCARD).is_ok()); + } +} diff --git a/crates/shuru-linux/src/kvm/devices.rs b/crates/shuru-linux/src/kvm/devices.rs index a87ab58..6ddc0a7 100644 --- a/crates/shuru-linux/src/kvm/devices.rs +++ b/crates/shuru-linux/src/kvm/devices.rs @@ -1,26 +1,45 @@ +use std::io; use std::os::fd::{AsRawFd, RawFd}; -use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; -use std::sync::Arc; +use std::sync::atomic::{fence, AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; -use kvm_ioctls::VmFd; -use vm_memory::{Address, Bytes, GuestAddress, GuestMemory, GuestMemoryMmap}; +use kvm_ioctls::{IoEventAddress, VmFd}; +use vm_memory::{Bytes, GuestAddress, GuestMemory, GuestMemoryMmap}; use vmm_sys_util::eventfd::EventFd; use super::layout; // =========================================================================== -// MMIO device trait + bus +// MMIO device traits + bus // =========================================================================== +/// A device whose registers are guarded as a whole. pub trait MmioDevice: Send { fn read(&mut self, offset: u64, data: &mut [u8]); fn write(&mut self, offset: u64, data: &[u8]); } +/// What the bus dispatches to. Every device does its own locking, so vCPUs +/// touching different devices never wait on each other. +pub trait MmioHandler: Send + Sync { + fn read(&self, offset: u64, data: &mut [u8]); + fn write(&self, offset: u64, data: &[u8]); +} + +impl MmioHandler for Mutex { + fn read(&self, offset: u64, data: &mut [u8]) { + self.lock().unwrap().read(offset, data); + } + + fn write(&self, offset: u64, data: &[u8]) { + self.lock().unwrap().write(offset, data); + } +} + pub struct MmioEntry { pub base: u64, pub size: u64, - pub device: Box, + pub device: Box, } pub struct MmioBus { @@ -34,12 +53,12 @@ impl MmioBus { } } - pub fn add(&mut self, base: u64, size: u64, device: Box) { + pub fn add(&mut self, base: u64, size: u64, device: Box) { self.devices.push(MmioEntry { base, size, device }); } - pub fn handle_read(&mut self, addr: u64, data: &mut [u8]) { - for entry in &mut self.devices { + pub fn handle_read(&self, addr: u64, data: &mut [u8]) { + for entry in &self.devices { if addr >= entry.base && addr < entry.base + entry.size { entry.device.read(addr - entry.base, data); return; @@ -50,8 +69,8 @@ impl MmioBus { } } - pub fn handle_write(&mut self, addr: u64, data: &[u8]) { - for entry in &mut self.devices { + pub fn handle_write(&self, addr: u64, data: &[u8]) { + for entry in &self.devices { if addr >= entry.base && addr < entry.base + entry.size { entry.device.write(addr - entry.base, data); return; @@ -284,6 +303,11 @@ const VIRTIO_MMIO_CONFIG: u64 = 0x100; const VIRTIO_STATUS_FEATURES_OK: u32 = 8; const VIRTIO_STATUS_DRIVER_OK: u32 = 4; +const VRING_AVAIL_F_NO_INTERRUPT: u16 = 1; + +/// Upper bound for every device's queue size, so chain walks can track +/// visited descriptors in a fixed array. +const MAX_QUEUE_SIZE: u16 = 256; #[derive(Clone)] pub struct VirtioQueueState { @@ -298,6 +322,7 @@ pub struct VirtioQueueState { impl VirtioQueueState { fn new(max_size: u16) -> Self { + assert!(max_size <= MAX_QUEUE_SIZE); VirtioQueueState { max_size, size: 0, @@ -308,6 +333,106 @@ impl VirtioQueueState { last_avail_idx: 0, } } + + /// The avail index to process up to, or None when the queue is not set + /// up in a way the device can use: size outside 1..=max_size, rings + /// outside guest memory, or an avail index more than one ring ahead. + pub fn pending(&self, mem: &GuestMemoryMmap) -> Option { + if !self.ready || self.size == 0 || self.size > self.max_size { + return None; + } + let size = self.size as usize; + if !mem.check_range(GuestAddress(self.desc_addr), size * 16) + || !mem.check_range(GuestAddress(self.avail_addr), 6 + size * 2) + || !mem.check_range(GuestAddress(self.used_addr), 6 + size * 8) + { + return None; + } + let current = avail_idx(mem, self.avail_addr); + if current.wrapping_sub(self.last_avail_idx) > self.size { + return None; + } + Some(current) + } + + /// The descriptors of the chain at `head`, or None if it leaves the + /// table, loops, or carries a flag outside `flags`. Only valid after + /// `pending` accepted the queue. + pub(super) fn chain( + &self, + mem: &GuestMemoryMmap, + head: u16, + flags: u16, + ) -> Option> { + let mut chain = Vec::new(); + let mut seen = [false; MAX_QUEUE_SIZE as usize]; + let mut next = head; + loop { + if next >= self.size || seen[next as usize] { + return None; + } + seen[next as usize] = true; + let desc = read_desc(mem, self.desc_addr, next); + if desc.flags & !flags != 0 { + return None; + } + chain.push(desc); + if desc.flags & VRING_DESC_F_NEXT == 0 { + return Some(chain); + } + next = desc.next; + } + } + + /// Whether the driver wants an interrupt for buffers put on this used ring. + pub fn wants_interrupt(&self, mem: &GuestMemoryMmap) -> bool { + // The used index must be visible to the driver before its flag is read. + fence(Ordering::SeqCst); + mem.read_obj::(GuestAddress(self.avail_addr)) + .map_or(true, |flags| { + u16::from_le(flags) & VRING_AVAIL_F_NO_INTERRUPT == 0 + }) + } +} + +pub const INTERRUPT_USED: u32 = 1; +pub const INTERRUPT_CONFIG: u32 = 2; + +/// A device interrupt line and the status register that says why it fired. +/// Both change under one lock so an acknowledgement cannot lower the line +/// over a reason raised in the meantime. +pub struct Interrupt { + vm_fd: Arc, + irq: u32, + status: Mutex, +} + +impl Interrupt { + fn new(vm_fd: Arc, irq: u32) -> Self { + Interrupt { + vm_fd, + irq, + status: Mutex::new(0), + } + } + + pub fn raise(&self, reason: u32) { + let mut status = self.status.lock().unwrap(); + *status |= reason; + let _ = self.vm_fd.set_irq_line(self.irq, true); + } + + fn status(&self) -> u32 { + *self.status.lock().unwrap() + } + + fn ack(&self, reasons: u32) { + let mut status = self.status.lock().unwrap(); + *status &= !reasons; + if *status == 0 { + let _ = self.vm_fd.set_irq_line(self.irq, false); + } + } } pub trait VirtioBackend: Send { @@ -317,204 +442,328 @@ pub trait VirtioBackend: Send { fn queue_max_size(&self) -> u16; fn config_read(&self, offset: u64) -> u32; fn config_write(&mut self, offset: u64, value: u32); + /// The feature set the driver accepted, once it sets FEATURES_OK. + fn features_ok(&mut self, _features: u64) {} fn activate( &mut self, queues: &[VirtioQueueState], mem: &GuestMemoryMmap, - vm_fd: &Arc, - irq: u32, - interrupt_status: Arc, + interrupt: Arc, ); + /// Returns whether any buffer went onto a used ring. fn process_queue( &mut self, queue_idx: u16, queues: &mut [VirtioQueueState], mem: &GuestMemoryMmap, - vm_fd: &Arc, - irq: u32, - ); + ) -> bool; + /// A backend with a worker of its own takes the queue kicks, one per + /// queue in queue order, and the transport leaves its queues alone. + fn take_kicks(&mut self, _kicks: &[EventFd]) -> io::Result { + Ok(false) + } fn reset(&mut self); } -pub struct VirtioMmioDevice { +struct Core { backend: Box, queues: Vec, + mem: GuestMemoryMmap, dev_features_sel: u32, drv_features: u64, drv_features_sel: u32, queue_sel: u32, status: u32, - interrupt_status: Arc, - vm_fd: Arc, - mem: GuestMemoryMmap, - irq: u32, activated: bool, } +impl Core { + fn selected_queue(&self) -> Option<&VirtioQueueState> { + self.queues.get(self.queue_sel as usize) + } + + fn selected_queue_mut(&mut self) -> Option<&mut VirtioQueueState> { + self.queues.get_mut(self.queue_sel as usize) + } +} + +/// Guest kicks land on an eventfd per queue inside KVM, so a kick never +/// leaves the kernel and never holds up the vCPU that made it. The queues +/// are served by the backend worker or by a thread per device. +pub struct VirtioMmioDevice { + core: Arc>, + interrupt: Arc, + vm_fd: Arc, + notify: IoEventAddress, + kicks: Vec, + stop: EventFd, + worker: Option>, +} + impl VirtioMmioDevice { pub fn new( backend: Box, + base: u64, spi: u32, vm_fd: Arc, mem: GuestMemoryMmap, - ) -> Self { + ) -> io::Result { let queue_count = backend.queue_count(); let queue_max = backend.queue_max_size(); - let queues = (0..queue_count) - .map(|_| VirtioQueueState::new(queue_max)) - .collect(); - - VirtioMmioDevice { - backend, - queues, - dev_features_sel: 0, - drv_features: 0, - drv_features_sel: 0, - queue_sel: 0, - status: 0, - interrupt_status: Arc::new(AtomicU32::new(0)), - vm_fd, - mem, + let mut device = VirtioMmioDevice { + core: Arc::new(Mutex::new(Core { + backend, + queues: (0..queue_count) + .map(|_| VirtioQueueState::new(queue_max)) + .collect(), + mem, + dev_features_sel: 0, + drv_features: 0, + drv_features_sel: 0, + queue_sel: 0, + status: 0, + activated: false, + })), // KVM_IRQ_LINE on ARM: SPI N maps to intid N+32 - irq: layout::spi_to_irq(spi), - activated: false, + interrupt: Arc::new(Interrupt::new(vm_fd.clone(), layout::spi_to_irq(spi))), + vm_fd, + notify: IoEventAddress::Mmio(base + VIRTIO_MMIO_QUEUE_NOTIFY), + kicks: Vec::with_capacity(queue_count), + stop: EventFd::new(libc::EFD_CLOEXEC)?, + worker: None, + }; + for index in 0..queue_count { + let kick = EventFd::new(libc::EFD_NONBLOCK | libc::EFD_CLOEXEC)?; + device + .vm_fd + .register_ioevent(&kick, &device.notify, index as u32) + .map_err(io::Error::other)?; + device.kicks.push(kick); } + let owned = device + .core + .lock() + .unwrap() + .backend + .take_kicks(&device.kicks)?; + if !owned { + let kicks = device + .kicks + .iter() + .map(EventFd::try_clone) + .collect::>>()?; + let stop = device.stop.try_clone()?; + let core = device.core.clone(); + let interrupt = device.interrupt.clone(); + device.worker = Some( + std::thread::Builder::new() + .name("shuru-virtio".into()) + .spawn(move || serve(core, interrupt, kicks, stop))?, + ); + } + Ok(device) } - fn selected_queue(&self) -> Option<&VirtioQueueState> { - self.queues.get(self.queue_sel as usize) + fn kick_all(&self) { + for kick in &self.kicks { + let _ = kick.write(1); + } } +} - fn selected_queue_mut(&mut self) -> Option<&mut VirtioQueueState> { - self.queues.get_mut(self.queue_sel as usize) +fn serve(core: Arc>, interrupt: Arc, kicks: Vec, stop: EventFd) { + let mut fds: Vec = std::iter::once(&stop) + .chain(&kicks) + .map(|event| libc::pollfd { + fd: event.as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }) + .collect(); + loop { + let n = unsafe { libc::poll(fds.as_mut_ptr(), fds.len() as libc::nfds_t, -1) }; + if n < 0 { + if io::Error::last_os_error().kind() == io::ErrorKind::Interrupted { + continue; + } + return; + } + if fds[0].revents != 0 { + return; + } + for (index, kick) in kicks.iter().enumerate() { + if fds[index + 1].revents & libc::POLLIN == 0 { + continue; + } + let _ = kick.read(); + let mut core = core.lock().unwrap(); + let Core { + backend, + queues, + mem, + activated, + .. + } = &mut *core; + if *activated + && backend.process_queue(index as u16, queues, mem) + && queues[index].wants_interrupt(mem) + { + interrupt.raise(INTERRUPT_USED); + } + } } } -impl MmioDevice for VirtioMmioDevice { - fn read(&mut self, offset: u64, data: &mut [u8]) { - let val: u32 = match offset { - VIRTIO_MMIO_MAGIC => 0x7472_6976, - VIRTIO_MMIO_VERSION => 2, - VIRTIO_MMIO_DEVICE_ID => self.backend.device_id(), - VIRTIO_MMIO_VENDOR_ID => 0x554D_4551, - VIRTIO_MMIO_DEV_FEATURES => { - let features = self.backend.device_features(); - if self.dev_features_sel == 0 { - features as u32 - } else { - (features >> 32) as u32 +impl Drop for VirtioMmioDevice { + fn drop(&mut self) { + let _ = self.stop.write(1); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + for (index, kick) in self.kicks.iter().enumerate() { + let _ = self + .vm_fd + .unregister_ioevent(kick, &self.notify, index as u32); + } + } +} + +impl MmioHandler for VirtioMmioDevice { + fn read(&self, offset: u64, data: &mut [u8]) { + let val: u32 = if offset == VIRTIO_MMIO_INTERRUPT_STATUS { + self.interrupt.status() + } else { + let core = self.core.lock().unwrap(); + match offset { + VIRTIO_MMIO_MAGIC => 0x7472_6976, + VIRTIO_MMIO_VERSION => 2, + VIRTIO_MMIO_DEVICE_ID => core.backend.device_id(), + VIRTIO_MMIO_VENDOR_ID => 0x554D_4551, + VIRTIO_MMIO_DEV_FEATURES => { + let features = core.backend.device_features(); + if core.dev_features_sel == 0 { + features as u32 + } else { + (features >> 32) as u32 + } } + VIRTIO_MMIO_QUEUE_NUM_MAX => core.selected_queue().map_or(0, |q| q.max_size as u32), + VIRTIO_MMIO_QUEUE_READY => core.selected_queue().map_or(0, |q| q.ready as u32), + VIRTIO_MMIO_STATUS => core.status, + VIRTIO_MMIO_CONFIG_GENERATION => 0, + o if o >= VIRTIO_MMIO_CONFIG => core.backend.config_read(o - VIRTIO_MMIO_CONFIG), + _ => 0, } - VIRTIO_MMIO_QUEUE_NUM_MAX => self.selected_queue().map_or(0, |q| q.max_size as u32), - VIRTIO_MMIO_QUEUE_READY => self.selected_queue().map_or(0, |q| q.ready as u32), - VIRTIO_MMIO_INTERRUPT_STATUS => self.interrupt_status.load(Ordering::Acquire), - VIRTIO_MMIO_STATUS => self.status, - VIRTIO_MMIO_CONFIG_GENERATION => 0, - o if o >= VIRTIO_MMIO_CONFIG => self.backend.config_read(o - VIRTIO_MMIO_CONFIG), - _ => 0, }; let len = data.len().min(4); data[..len].copy_from_slice(&val.to_le_bytes()[..len]); } - fn write(&mut self, offset: u64, data: &[u8]) { + fn write(&self, offset: u64, data: &[u8]) { let mut bytes = [0u8; 4]; let len = data.len().min(4); bytes[..len].copy_from_slice(&data[..len]); let val = u32::from_le_bytes(bytes); match offset { - VIRTIO_MMIO_DEV_FEATURES_SEL => self.dev_features_sel = val, + VIRTIO_MMIO_INTERRUPT_ACK => return self.interrupt.ack(val), + // Matching kicks never get here; KVM signals the eventfd itself. + VIRTIO_MMIO_QUEUE_NOTIFY => return, + _ => {} + } + + let mut core = self.core.lock().unwrap(); + match offset { + VIRTIO_MMIO_DEV_FEATURES_SEL => core.dev_features_sel = val, VIRTIO_MMIO_DRV_FEATURES => { - if self.drv_features_sel == 0 { - self.drv_features = (self.drv_features & 0xFFFF_FFFF_0000_0000) | val as u64; + if core.drv_features_sel == 0 { + core.drv_features = (core.drv_features & 0xFFFF_FFFF_0000_0000) | val as u64; } else { - self.drv_features = - (self.drv_features & 0x0000_0000_FFFF_FFFF) | ((val as u64) << 32); + core.drv_features = + (core.drv_features & 0x0000_0000_FFFF_FFFF) | ((val as u64) << 32); } } - VIRTIO_MMIO_DRV_FEATURES_SEL => self.drv_features_sel = val, - VIRTIO_MMIO_QUEUE_SEL => self.queue_sel = val, + VIRTIO_MMIO_DRV_FEATURES_SEL => core.drv_features_sel = val, + VIRTIO_MMIO_QUEUE_SEL => core.queue_sel = val, VIRTIO_MMIO_QUEUE_NUM => { - if let Some(q) = self.selected_queue_mut() { - q.size = val as u16; + if let Some(q) = core.selected_queue_mut() { + q.size = if (1..=q.max_size as u32).contains(&val) { + val as u16 + } else { + 0 + }; } } VIRTIO_MMIO_QUEUE_READY => { - if let Some(q) = self.selected_queue_mut() { + if let Some(q) = core.selected_queue_mut() { q.ready = val == 1; } } - VIRTIO_MMIO_QUEUE_NOTIFY => { - self.backend.process_queue( - val as u16, - &mut self.queues, - &self.mem, - &self.vm_fd, - self.irq, - ); - self.interrupt_status.fetch_or(1, Ordering::Release); - let _ = self.vm_fd.set_irq_line(self.irq, true); - } - VIRTIO_MMIO_INTERRUPT_ACK => { - self.interrupt_status.fetch_and(!val, Ordering::AcqRel); - if self.interrupt_status.load(Ordering::Acquire) == 0 { - let _ = self.vm_fd.set_irq_line(self.irq, false); - } - } VIRTIO_MMIO_STATUS => { - self.status = val; + let prev = core.status; + core.status = val; if val == 0 { - self.backend.reset(); - self.activated = false; - self.interrupt_status.store(0, Ordering::Release); + core.backend.reset(); + core.activated = false; + for q in &mut core.queues { + *q = VirtioQueueState::new(q.max_size); + } + self.interrupt.ack(u32::MAX); + } + if (val & VIRTIO_STATUS_FEATURES_OK) != 0 && (prev & VIRTIO_STATUS_FEATURES_OK) == 0 + { + let features = core.drv_features & core.backend.device_features(); + core.backend.features_ok(features); } - if !self.activated + if !core.activated && (val & VIRTIO_STATUS_DRIVER_OK) != 0 && (val & VIRTIO_STATUS_FEATURES_OK) != 0 { - self.activated = true; - self.backend.activate( - &self.queues, - &self.mem, - &self.vm_fd, - self.irq, - self.interrupt_status.clone(), - ); + core.activated = true; + let Core { + backend, + queues, + mem, + .. + } = &mut *core; + backend.activate(queues, mem, self.interrupt.clone()); + drop(core); + // Anything queued before the driver finished setting up. + self.kick_all(); } } VIRTIO_MMIO_QUEUE_DESC_LOW => { - if let Some(q) = self.selected_queue_mut() { + if let Some(q) = core.selected_queue_mut() { q.desc_addr = (q.desc_addr & !0xFFFF_FFFF) | val as u64; } } VIRTIO_MMIO_QUEUE_DESC_HIGH => { - if let Some(q) = self.selected_queue_mut() { + if let Some(q) = core.selected_queue_mut() { q.desc_addr = (q.desc_addr & 0xFFFF_FFFF) | ((val as u64) << 32); } } VIRTIO_MMIO_QUEUE_DRIVER_LOW => { - if let Some(q) = self.selected_queue_mut() { + if let Some(q) = core.selected_queue_mut() { q.avail_addr = (q.avail_addr & !0xFFFF_FFFF) | val as u64; } } VIRTIO_MMIO_QUEUE_DRIVER_HIGH => { - if let Some(q) = self.selected_queue_mut() { + if let Some(q) = core.selected_queue_mut() { q.avail_addr = (q.avail_addr & 0xFFFF_FFFF) | ((val as u64) << 32); } } VIRTIO_MMIO_QUEUE_DEVICE_LOW => { - if let Some(q) = self.selected_queue_mut() { + if let Some(q) = core.selected_queue_mut() { q.used_addr = (q.used_addr & !0xFFFF_FFFF) | val as u64; } } VIRTIO_MMIO_QUEUE_DEVICE_HIGH => { - if let Some(q) = self.selected_queue_mut() { + if let Some(q) = core.selected_queue_mut() { q.used_addr = (q.used_addr & 0xFFFF_FFFF) | ((val as u64) << 32); } } o if o >= VIRTIO_MMIO_CONFIG => { - self.backend.config_write(o - VIRTIO_MMIO_CONFIG, val); + core.backend.config_write(o - VIRTIO_MMIO_CONFIG, val); } _ => {} } @@ -525,20 +774,24 @@ impl MmioDevice for VirtioMmioDevice { // Virtio descriptor chain helpers // =========================================================================== -const VRING_DESC_F_NEXT: u16 = 1; -const VRING_DESC_F_WRITE: u16 = 2; +pub(super) const VRING_DESC_F_NEXT: u16 = 1; +pub(super) const VRING_DESC_F_WRITE: u16 = 2; #[derive(Clone, Copy)] -struct VringDesc { - addr: u64, - len: u32, - flags: u16, - next: u16, +pub(super) struct VringDesc { + pub(super) addr: u64, + pub(super) len: u32, + pub(super) flags: u16, + pub(super) next: u16, } pub fn avail_idx(mem: &GuestMemoryMmap, avail_addr: u64) -> u16 { - mem.read_obj::(GuestAddress(avail_addr + 2)) - .unwrap_or(0) + let idx = mem + .read_obj::(GuestAddress(avail_addr + 2)) + .unwrap_or(0); + // Ring entries the index covers must not be read before the index. + fence(Ordering::Acquire); + idx } pub fn avail_ring_entry(mem: &GuestMemoryMmap, avail_addr: u64, queue_size: u16, idx: u16) -> u16 { @@ -547,7 +800,7 @@ pub fn avail_ring_entry(mem: &GuestMemoryMmap, avail_addr: u64, queue_size: u16, .unwrap_or(0) } -fn read_desc(mem: &GuestMemoryMmap, desc_addr: u64, idx: u16) -> VringDesc { +pub(super) fn read_desc(mem: &GuestMemoryMmap, desc_addr: u64, idx: u16) -> VringDesc { let off = idx as u64 * 16; VringDesc { addr: mem.read_obj(GuestAddress(desc_addr + off)).unwrap_or(0), @@ -572,6 +825,7 @@ pub fn write_used( let ring_off = 4 + (used_idx % queue_size) as u64 * 8; let _ = mem.write_obj(desc_id, GuestAddress(used_addr + ring_off)); let _ = mem.write_obj(len, GuestAddress(used_addr + ring_off + 4)); + fence(Ordering::Release); let _ = mem.write_obj(used_idx.wrapping_add(1), GuestAddress(used_addr + 2)); } @@ -579,33 +833,134 @@ pub fn write_used( // Virtio block backend (device ID 2) // =========================================================================== -use std::fs::{File, OpenOptions}; +use std::fs::File; use std::io::{Read, Seek, SeekFrom, Write}; -const VIRTIO_BLK_T_IN: u32 = 0; -const VIRTIO_BLK_T_OUT: u32 = 1; -const VIRTIO_BLK_S_OK: u8 = 0; -const VIRTIO_BLK_S_IOERR: u8 = 1; -const VIRTIO_F_VERSION_1: u64 = 1 << 32; +#[path = "block_ranges.rs"] +mod block_ranges; +use block_ranges::*; +pub(super) const VIRTIO_F_VERSION_1: u64 = 1 << 32; pub struct BlockBackend { disk: File, capacity: u64, read_only: bool, + negotiated: u64, buf: Vec, } impl BlockBackend { - pub fn new(path: &str, read_only: bool) -> std::io::Result { - let disk = OpenOptions::new().read(true).write(!read_only).open(path)?; + pub fn new(disk: File, read_only: bool) -> std::io::Result { let capacity = disk.metadata()?.len() / 512; Ok(BlockBackend { disk, capacity, read_only, + negotiated: 0, buf: vec![0u8; 65536], }) } + fn request(&mut self, mem: &GuestMemoryMmap, chain: &[VringDesc]) -> Result { + let fail = VIRTIO_BLK_S_IOERR; + if chain.len() < 2 { + return Err(fail); + } + let header = &chain[0]; + if header.len != 16 || header.flags & VRING_DESC_F_WRITE != 0 { + return Err(fail); + } + let mut bytes = [0; 16]; + mem.read_slice(&mut bytes, GuestAddress(header.addr)) + .map_err(|_| fail)?; + let kind = u32::from_le_bytes(bytes[..4].try_into().unwrap()); + let sector = u64::from_le_bytes(bytes[8..].try_into().unwrap()); + let data = &chain[1..chain.len() - 1]; + let mut total = 0usize; + for desc in data { + if (desc.flags & VRING_DESC_F_WRITE != 0) != (kind == VIRTIO_BLK_T_IN) { + return Err(fail); + } + if desc.len > 1024 * 1024 + || !mem.check_range(GuestAddress(desc.addr), desc.len as usize) + { + return Err(fail); + } + total = total.checked_add(desc.len as usize).ok_or(fail)?; + } + match kind { + VIRTIO_BLK_T_DISCARD | VIRTIO_BLK_T_WRITE_ZEROES => { + if self.read_only + || total == 0 + || total > MAX_RANGES * 16 + || !total.is_multiple_of(16) + || sector != 0 + { + return Err(fail); + } + let feature = if kind == VIRTIO_BLK_T_DISCARD { 13 } else { 14 }; + if self.negotiated & (1 << feature) == 0 { + return Err(VIRTIO_BLK_S_UNSUPP); + } + let mut payload = [0u8; MAX_RANGES * 16]; + let mut at = 0; + for desc in data { + let end = at + desc.len as usize; + mem.read_slice(&mut payload[at..end], GuestAddress(desc.addr)) + .map_err(|_| fail)?; + at = end; + } + let ranges = decode_ranges(&payload[..total], self.capacity, kind)?; + for (offset, len, unmap) in ranges { + if len == 0 { + continue; + } + let mode = if kind == VIRTIO_BLK_T_DISCARD || unmap { + libc::FALLOC_FL_PUNCH_HOLE | libc::FALLOC_FL_KEEP_SIZE + } else { + libc::FALLOC_FL_ZERO_RANGE | libc::FALLOC_FL_KEEP_SIZE + }; + if unsafe { libc::fallocate(self.disk.as_raw_fd(), mode, offset, len) } < 0 { + return Err(fail); + } + } + Ok(0) + } + VIRTIO_BLK_T_IN | VIRTIO_BLK_T_OUT => { + if kind == VIRTIO_BLK_T_OUT && self.read_only { + return Err(fail); + } + if !total.is_multiple_of(512) + || sector > self.capacity + || total as u64 / 512 > self.capacity - sector + { + return Err(fail); + } + self.disk + .seek(SeekFrom::Start(sector.checked_mul(512).ok_or(fail)?)) + .map_err(|_| fail)?; + let mut written = 0; + for desc in data { + let mut at = 0usize; + while at < desc.len as usize { + let n = self.buf.len().min(desc.len as usize - at); + let buf = &mut self.buf[..n]; + let addr = GuestAddress(desc.addr.checked_add(at as u64).ok_or(fail)?); + if kind == VIRTIO_BLK_T_IN { + self.disk.read_exact(buf).map_err(|_| fail)?; + mem.write_slice(buf, addr).map_err(|_| fail)?; + written += n as u32; + } else { + mem.read_slice(buf, addr).map_err(|_| fail)?; + self.disk.write_all(buf).map_err(|_| fail)?; + } + at += n; + } + } + Ok(written) + } + _ => Err(VIRTIO_BLK_S_UNSUPP), + } + } } impl VirtioBackend for BlockBackend { @@ -614,9 +969,11 @@ impl VirtioBackend for BlockBackend { } fn device_features(&self) -> u64 { - let mut f = VIRTIO_F_VERSION_1; + let mut f = VIRTIO_F_VERSION_1 | (1 << 1) | (1 << 2); if self.read_only { f |= 1 << 5; + } else { + f |= (1 << 13) | (1 << 14); } f } @@ -632,102 +989,62 @@ impl VirtioBackend for BlockBackend { match offset { 0 => self.capacity as u32, 4 => (self.capacity >> 32) as u32, + 8 => 1024 * 1024, + 12 => 254, + 36 | 48 if !self.read_only => MAX_RANGE_SECTORS, + 40 | 52 if !self.read_only => MAX_RANGES as u32, + 44 if !self.read_only => 8, + 56 if !self.read_only => 1, _ => 0, } } fn config_write(&mut self, _: u64, _: u32) {} - fn activate( - &mut self, - _: &[VirtioQueueState], - _: &GuestMemoryMmap, - _: &Arc, - _: u32, - _: Arc, - ) { + fn features_ok(&mut self, features: u64) { + self.negotiated = features; } + fn activate(&mut self, _: &[VirtioQueueState], _: &GuestMemoryMmap, _: Arc) {} + fn process_queue( &mut self, _queue_idx: u16, queues: &mut [VirtioQueueState], mem: &GuestMemoryMmap, - _vm_fd: &Arc, - _irq: u32, - ) { + ) -> bool { let q = match queues.get_mut(0) { - Some(q) if q.ready => q, - _ => return, + Some(q) => q, + None => return false, }; - - let current_avail = avail_idx(mem, q.avail_addr); - + let current_avail = match q.pending(mem) { + Some(idx) => idx, + None => return false, + }; + let used = q.last_avail_idx != current_avail; while q.last_avail_idx != current_avail { let head = avail_ring_entry(mem, q.avail_addr, q.size, q.last_avail_idx); - - let header_desc = read_desc(mem, q.desc_addr, head); - let req_type = mem - .read_obj::(GuestAddress(header_desc.addr)) - .unwrap_or(u32::MAX); - let sector = mem - .read_obj::(GuestAddress(header_desc.addr + 8)) - .unwrap_or(0); - - // Walk the descriptor chain inline — no Vec allocation. - // Chain: header → data desc(s) → status (last, no NEXT flag). - let mut disk_offset = sector * 512; - let mut total_data_len: u32 = 0; - let mut status = VIRTIO_BLK_S_OK; - let mut status_addr: Option = None; - let mut next = header_desc.next; - let mut has_next = header_desc.flags & VRING_DESC_F_NEXT != 0; - - while has_next { - let desc = read_desc(mem, q.desc_addr, next); - let is_last = desc.flags & VRING_DESC_F_NEXT == 0; - - if is_last { - status_addr = Some(desc.addr); - } else if status == VIRTIO_BLK_S_OK { - let len = desc.len as usize; - if self.buf.len() < len { - self.buf.resize(len, 0); + let chain = q + .chain(mem, head, VRING_DESC_F_NEXT | VRING_DESC_F_WRITE) + .unwrap_or_default(); + let status_desc = chain.last().filter(|d| { + chain.len() >= 2 + && d.flags == VRING_DESC_F_WRITE + && d.len >= 1 + && mem.check_range(GuestAddress(d.addr), 1) + }); + let mut used_len = 0; + if let Some(desc) = status_desc { + let result = self.request(mem, &chain); + let status = match result { + Ok(n) => { + used_len = n; + VIRTIO_BLK_S_OK } - let buf = &mut self.buf[..len]; - - match req_type { - VIRTIO_BLK_T_IN => { - if self.disk.seek(SeekFrom::Start(disk_offset)).is_err() - || self.disk.read_exact(buf).is_err() - { - status = VIRTIO_BLK_S_IOERR; - } else { - let _ = mem.write_slice(buf, GuestAddress(desc.addr)); - } - } - VIRTIO_BLK_T_OUT if !self.read_only => { - if mem.read_slice(buf, GuestAddress(desc.addr)).is_err() - || self.disk.seek(SeekFrom::Start(disk_offset)).is_err() - || self.disk.write_all(buf).is_err() - { - status = VIRTIO_BLK_S_IOERR; - } - } - _ => { - status = VIRTIO_BLK_S_IOERR; - } - } - disk_offset += len as u64; - total_data_len += desc.len; - } - - next = desc.next; - has_next = !is_last; - } - - if let Some(addr) = status_addr { - let _ = mem.write_obj(status, GuestAddress(addr)); + Err(status) => status, + }; + let _ = mem.write_obj(status, GuestAddress(desc.addr)); + used_len += 1; } write_used( mem, @@ -735,13 +1052,16 @@ impl VirtioBackend for BlockBackend { q.size, q.last_avail_idx, head as u32, - total_data_len + 1, + used_len, ); q.last_avail_idx = q.last_avail_idx.wrapping_add(1); } + used } - fn reset(&mut self) {} + fn reset(&mut self) { + self.negotiated = 0; + } } // =========================================================================== @@ -756,14 +1076,19 @@ pub struct NetBackend { mac: [u8; 6], rx_running: Arc, rx_thread: Option>, + tx_buf: Vec, } +// The virtio-net header plus one Ethernet frame at the 65535-byte IP MTU. +const MAX_TX_FRAME: usize = VIRTIO_NET_HDR_SIZE + 14 + 65535; + impl NetBackend { pub fn new(fd: RawFd, mac: [u8; 6]) -> Self { NetBackend { fd, mac, rx_running: Arc::new(AtomicBool::new(false)), + tx_buf: vec![0; MAX_TX_FRAME], rx_thread: None, } } @@ -799,9 +1124,7 @@ impl VirtioBackend for NetBackend { &mut self, queues: &[VirtioQueueState], mem: &GuestMemoryMmap, - vm_fd: &Arc, - irq: u32, - interrupt_status: Arc, + interrupt: Arc, ) { // Set fd to non-blocking unsafe { @@ -819,13 +1142,12 @@ impl VirtioBackend for NetBackend { let running = self.rx_running.clone(); let fd = self.fd; let mem = mem.clone(); - let vm_fd = vm_fd.clone(); self.rx_thread = Some( std::thread::Builder::new() .name("shuru-net-rx".into()) .spawn(move || { - net_rx_loop(fd, mem, rx_q, vm_fd, irq, interrupt_status, running); + net_rx_loop(fd, mem, rx_q, interrupt, running); }) .expect("failed to spawn net-rx thread"), ); @@ -836,36 +1158,41 @@ impl VirtioBackend for NetBackend { queue_idx: u16, queues: &mut [VirtioQueueState], mem: &GuestMemoryMmap, - _vm_fd: &Arc, - _irq: u32, - ) { + ) -> bool { if queue_idx != 1 { - return; // RX handled by thread + return false; // RX handled by thread } let q = match queues.get_mut(1) { - Some(q) if q.ready => q, - _ => return, + Some(q) => q, + None => return false, }; - - let current_avail = avail_idx(mem, q.avail_addr); + let current_avail = match q.pending(mem) { + Some(idx) => idx, + None => return false, + }; + let used = q.last_avail_idx != current_avail; while q.last_avail_idx != current_avail { let head = avail_ring_entry(mem, q.avail_addr, q.size, q.last_avail_idx); - let mut packet = Vec::new(); - let mut desc = read_desc(mem, q.desc_addr, head); - loop { - let mut buf = vec![0u8; desc.len as usize]; - let _ = mem.read_slice(&mut buf, GuestAddress(desc.addr)); - packet.extend_from_slice(&buf); - if desc.flags & VRING_DESC_F_NEXT == 0 { + // A chain that loops, is longer than one frame, or points + // outside guest memory is dropped. + let mut len = 0; + for desc in q.chain(mem, head, VRING_DESC_F_NEXT).unwrap_or_default() { + let end = len + desc.len as usize; + if end > MAX_TX_FRAME + || mem + .read_slice(&mut self.tx_buf[len..end], GuestAddress(desc.addr)) + .is_err() + { + len = 0; break; } - desc = read_desc(mem, q.desc_addr, desc.next); + len = end; } // Skip virtio-net header, send raw Ethernet frame - if packet.len() > VIRTIO_NET_HDR_SIZE { - let frame = &packet[VIRTIO_NET_HDR_SIZE..]; + if len > VIRTIO_NET_HDR_SIZE { + let frame = &self.tx_buf[VIRTIO_NET_HDR_SIZE..len]; unsafe { libc::send( self.fd, @@ -879,6 +1206,7 @@ impl VirtioBackend for NetBackend { write_used(mem, q.used_addr, q.size, q.last_avail_idx, head as u32, 0); q.last_avail_idx = q.last_avail_idx.wrapping_add(1); } + used } fn reset(&mut self) { @@ -900,14 +1228,15 @@ fn net_rx_loop( fd: RawFd, mem: GuestMemoryMmap, rx_q: VirtioQueueState, - vm_fd: Arc, - irq: u32, - interrupt_status: Arc, + interrupt: Arc, running: Arc, ) { let mut last_avail = rx_q.last_avail_idx; let mut used_idx: u16 = 0; - let mut buf = vec![0u8; 65535]; + // One Ethernet frame at the 65535-byte IP-MTU the proxy stack uses: + // 14 bytes of header plus the IP payload. A datagram longer than the + // buffer would be truncated by recv. + let mut buf = vec![0u8; 14 + 65535]; while running.load(Ordering::Acquire) { let mut pfd = libc::pollfd { @@ -927,12 +1256,21 @@ fn net_rx_loop( let frame = &buf[..n as usize]; // Check for available RX descriptors - let current_avail = avail_idx(&mem, rx_q.avail_addr); + let mut ring = rx_q.clone(); + ring.last_avail_idx = last_avail; + let current_avail = match ring.pending(&mem) { + Some(idx) => idx, + None => continue, + }; if last_avail == current_avail { continue; // drop frame — no buffers } let head = avail_ring_entry(&mem, rx_q.avail_addr, rx_q.size, last_avail); + if head >= rx_q.size { + last_avail = last_avail.wrapping_add(1); + continue; + } let desc = read_desc(&mem, rx_q.desc_addr, head); let total = VIRTIO_NET_HDR_SIZE + frame.len(); @@ -953,348 +1291,241 @@ fn net_rx_loop( used_idx = used_idx.wrapping_add(1); last_avail = last_avail.wrapping_add(1); - // Signal the guest - interrupt_status.fetch_or(1, Ordering::Release); - let _ = vm_fd.set_irq_line(irq, true); + if rx_q.wants_interrupt(&mem) { + interrupt.raise(INTERRUPT_USED); + } } } } -// =========================================================================== -// Vhost-vsock backend (device ID 19) -// =========================================================================== - -pub struct VhostVsockBackend { - guest_cid: u64, - vhost_fd: Option, - kick_evts: Vec, - call_evts: Vec, - irq_running: Arc, - irq_thread: Option>, -} +#[cfg(test)] +mod block_tests { + use super::*; + use std::os::unix::fs::FileExt; + + fn backend() -> BlockBackend { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let path = std::env::temp_dir().join(format!( + "shuru-block-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&path) + .unwrap(); + std::fs::remove_file(path).unwrap(); + file.set_len(16384).unwrap(); + file.write_all_at(&vec![7; 16384], 0).unwrap(); + let mut disk = BlockBackend::new(file, false).unwrap(); + disk.features_ok(disk.device_features()); + disk + } -impl VhostVsockBackend { - pub fn new(guest_cid: u64) -> Self { - VhostVsockBackend { - guest_cid, - vhost_fd: None, - kick_evts: Vec::new(), - call_evts: Vec::new(), - irq_running: Arc::new(AtomicBool::new(false)), - irq_thread: None, + fn request(mem: &GuestMemoryMmap, kind: u32, ranges: &[(u64, u32, u32)]) -> Vec { + mem.write_obj(kind.to_le(), GuestAddress(0x1000)).unwrap(); + mem.write_obj(0u64, GuestAddress(0x1008)).unwrap(); + for (i, (sector, count, flags)) in ranges.iter().enumerate() { + let at = 0x2000 + i as u64 * 16; + mem.write_obj(sector.to_le(), GuestAddress(at)).unwrap(); + mem.write_obj(count.to_le(), GuestAddress(at + 8)).unwrap(); + mem.write_obj(flags.to_le(), GuestAddress(at + 12)).unwrap(); } + vec![ + VringDesc { + addr: 0x1000, + len: 16, + flags: 1, + next: 1, + }, + VringDesc { + addr: 0x2000, + len: ranges.len() as u32 * 16, + flags: 1, + next: 2, + }, + VringDesc { + addr: 0x3000, + len: 1, + flags: 2, + next: 0, + }, + ] } -} - -// Vhost ioctl encoding -const VHOST_VIRTIO: u64 = 0xAF; - -const fn vhost_io(nr: u64) -> u64 { - (VHOST_VIRTIO << 8) | nr -} - -const fn vhost_iow_size(nr: u64, size: usize) -> u64 { - (1u64 << 30) | ((size as u64) << 16) | (VHOST_VIRTIO << 8) | nr -} -const VHOST_SET_OWNER: u64 = vhost_io(0x01); -const VHOST_SET_FEATURES: u64 = vhost_iow_size(0x00, 8); // sizeof(u64) -const VHOST_SET_MEM_TABLE: u64 = vhost_iow_size(0x03, 8); // sizeof(vhost_memory header) -const VHOST_SET_VRING_NUM: u64 = vhost_iow_size(0x10, 8); // sizeof(vhost_vring_state) -const VHOST_SET_VRING_ADDR: u64 = vhost_iow_size(0x11, 40); // sizeof(vhost_vring_addr) = 4+4+8+8+8+8 -const VHOST_SET_VRING_BASE: u64 = vhost_iow_size(0x12, 8); -const VHOST_SET_VRING_KICK: u64 = vhost_iow_size(0x20, 8); // sizeof(vhost_vring_file) -const VHOST_SET_VRING_CALL: u64 = vhost_iow_size(0x21, 8); -const VHOST_VSOCK_SET_GUEST_CID: u64 = vhost_iow_size(0x60, 8); -// _IOW(0xAF, 0x61, int) — starts/stops the vhost worker -const VHOST_VSOCK_SET_RUNNING: u64 = vhost_iow_size(0x61, 4); - -#[repr(C)] -struct VhostVringState { - index: u32, - num: u32, -} - -#[repr(C)] -struct VhostVringAddr { - index: u32, - flags: u32, - desc_user_addr: u64, - used_user_addr: u64, - avail_user_addr: u64, - log_guest_addr: u64, -} - -#[repr(C)] -struct VhostVringFile { - index: u32, - fd: i32, -} + #[test] + fn invalid_later_range_leaves_disk_untouched() { + let mem = GuestMemoryMmap::from_ranges(&[(GuestAddress(0), 65536)]).unwrap(); + let mut disk = backend(); + let chain = request(&mem, VIRTIO_BLK_T_DISCARD, &[(0, 8, 0), (31, 2, 0)]); + assert_eq!(disk.request(&mem, &chain), Err(VIRTIO_BLK_S_IOERR)); + let mut bytes = [0; 16384]; + disk.disk.read_exact_at(&mut bytes, 0).unwrap(); + assert!(bytes.iter().all(|b| *b == 7)); + } -#[repr(C)] -struct VhostMemoryHeader { - nregions: u32, - _padding: u32, -} + #[test] + fn discard_and_both_zero_modes_preserve_size_and_neighbors() { + let mem = GuestMemoryMmap::from_ranges(&[(GuestAddress(0), 65536)]).unwrap(); + for (kind, flags) in [ + (VIRTIO_BLK_T_DISCARD, 0), + (VIRTIO_BLK_T_WRITE_ZEROES, 0), + (VIRTIO_BLK_T_WRITE_ZEROES, 1), + ] { + let mut disk = backend(); + let chain = request(&mem, kind, &[(8, 8, flags)]); + assert_eq!(disk.request(&mem, &chain), Ok(0)); + let mut bytes = [0; 16384]; + disk.disk.read_exact_at(&mut bytes, 0).unwrap(); + assert!(bytes[..4096].iter().all(|b| *b == 7)); + assert!(bytes[4096..8192].iter().all(|b| *b == 0)); + assert!(bytes[8192..].iter().all(|b| *b == 7)); + assert_eq!(disk.disk.metadata().unwrap().len(), 16384); + } + } -#[repr(C)] -struct VhostMemoryRegion { - guest_phys_addr: u64, - memory_size: u64, - userspace_addr: u64, - _flags_padding: u64, -} + #[test] + fn descriptor_cycles_and_bad_heads_finish_without_io() { + let mem = GuestMemoryMmap::from_ranges(&[(GuestAddress(0), 65536)]).unwrap(); + let mut disk = backend(); + let mut q = VirtioQueueState::new(8); + q.size = 8; + q.ready = true; + q.desc_addr = 0x1000; + q.avail_addr = 0x2000; + q.used_addr = 0x3000; + mem.write_obj(1u16, GuestAddress(q.desc_addr + 12)).unwrap(); + mem.write_obj(0u16, GuestAddress(q.desc_addr + 14)).unwrap(); + mem.write_obj(2u16, GuestAddress(q.avail_addr + 2)).unwrap(); + mem.write_obj(0u16, GuestAddress(q.avail_addr + 4)).unwrap(); + mem.write_obj(100u16, GuestAddress(q.avail_addr + 6)) + .unwrap(); + assert!(disk.process_queue(0, std::slice::from_mut(&mut q), &mem)); + assert_eq!(q.last_avail_idx, 2); + assert_eq!( + mem.read_obj::(GuestAddress(q.used_addr + 2)).unwrap(), + 2 + ); + } -#[repr(C)] -struct VhostMemTable { - header: VhostMemoryHeader, - region: VhostMemoryRegion, + #[test] + fn readonly_and_bad_data_direction_are_rejected() { + let mem = GuestMemoryMmap::from_ranges(&[(GuestAddress(0), 65536)]).unwrap(); + let mut disk = backend(); + let mut chain = request(&mem, VIRTIO_BLK_T_DISCARD, &[(0, 8, 0)]); + chain[1].flags |= VRING_DESC_F_WRITE; + assert_eq!(disk.request(&mem, &chain), Err(VIRTIO_BLK_S_IOERR)); + chain[1].flags &= !VRING_DESC_F_WRITE; + disk.read_only = true; + assert_eq!(disk.request(&mem, &chain), Err(VIRTIO_BLK_S_IOERR)); + assert_eq!(disk.device_features() & ((1 << 13) | (1 << 14)), 0); + } } -fn guest_to_host(mem: &GuestMemoryMmap, guest_addr: u64) -> u64 { - mem.get_host_address(GuestAddress(guest_addr)) - .map(|p| p as u64) - .unwrap_or(0) -} +#[cfg(test)] +mod queue_tests { + use super::*; -impl VirtioBackend for VhostVsockBackend { - fn device_id(&self) -> u32 { - 19 - } + const DESC: u64 = 0x1000; + const AVAIL: u64 = 0x2000; + const USED: u64 = 0x3000; - fn device_features(&self) -> u64 { - VIRTIO_F_VERSION_1 + fn queue(size: u16) -> VirtioQueueState { + VirtioQueueState { + size, + ready: true, + desc_addr: DESC, + avail_addr: AVAIL, + used_addr: USED, + ..VirtioQueueState::new(256) + } } - fn queue_count(&self) -> usize { - 3 - } - fn queue_max_size(&self) -> u16 { - 256 + fn desc(mem: &GuestMemoryMmap, i: u16, addr: u64, len: u32, flags: u16, next: u16) { + let at = DESC + i as u64 * 16; + mem.write_obj(addr, GuestAddress(at)).unwrap(); + mem.write_obj(len, GuestAddress(at + 8)).unwrap(); + mem.write_obj(flags, GuestAddress(at + 12)).unwrap(); + mem.write_obj(next, GuestAddress(at + 14)).unwrap(); } - fn config_read(&self, offset: u64) -> u32 { - match offset { - 0 => self.guest_cid as u32, - 4 => (self.guest_cid >> 32) as u32, - _ => 0, + fn offer(mem: &GuestMemoryMmap, heads: &[u16]) { + for (i, head) in heads.iter().enumerate() { + mem.write_obj(*head, GuestAddress(AVAIL + 4 + i as u64 * 2)) + .unwrap(); } + mem.write_obj(heads.len() as u16, GuestAddress(AVAIL + 2)) + .unwrap(); } - fn config_write(&mut self, _: u64, _: u32) {} - - fn activate( - &mut self, - queues: &[VirtioQueueState], - mem: &GuestMemoryMmap, - vm_fd: &Arc, - irq: u32, - _interrupt_status: Arc, - ) { - let vhost_fd = unsafe { - libc::open( - b"/dev/vhost-vsock\0".as_ptr() as *const libc::c_char, - libc::O_RDWR | libc::O_CLOEXEC, - ) - }; - if vhost_fd < 0 { - eprintln!( - "shuru: failed to open /dev/vhost-vsock: {}", - std::io::Error::last_os_error() - ); - return; - } - self.vhost_fd = Some(vhost_fd); - - unsafe { - // Exact order matching Cloud Hypervisor / Firecracker: - // 1. OWNER → 2. FEATURES → 3. MEM_TABLE → 4. queues (NUM, ADDR, BASE, CALL, KICK) → 5. CID → 6. IRQFD - if libc::ioctl(vhost_fd, VHOST_SET_OWNER) < 0 { - eprintln!( - "shuru: VHOST_SET_OWNER failed: {}", - std::io::Error::last_os_error() - ); - return; - } - - // Negotiate features — get supported, intersect, set - let get_features: u64 = (2u64 << 30) | (8u64 << 16) | (VHOST_VIRTIO << 8) | 0x00; - let mut avail_features: u64 = 0; - libc::ioctl(vhost_fd, get_features, &mut avail_features); - let features = avail_features & VIRTIO_F_VERSION_1; - if libc::ioctl(vhost_fd, VHOST_SET_FEATURES, &features) < 0 { - eprintln!( - "shuru: VHOST_SET_FEATURES failed: {}", - std::io::Error::last_os_error() - ); - return; - } - - // Memory table - let host_addr = guest_to_host(mem, layout::DRAM_BASE); - let region = mem.find_region(GuestAddress(layout::DRAM_BASE)); - if let Some(region) = region { - use vm_memory::GuestMemoryRegion; - let mem_table = VhostMemTable { - header: VhostMemoryHeader { - nregions: 1, - _padding: 0, - }, - region: VhostMemoryRegion { - guest_phys_addr: region.start_addr().raw_value(), - memory_size: region.len(), - userspace_addr: host_addr, - _flags_padding: 0, - }, - }; - if libc::ioctl(vhost_fd, VHOST_SET_MEM_TABLE, &mem_table) < 0 { - eprintln!( - "shuru: VHOST_SET_MEM_TABLE failed: {}", - std::io::Error::last_os_error() - ); - return; - } - } - - // Set up vrings — only RX (0) and TX (1) - for (i, q) in queues.iter().enumerate().take(2) { - if !q.ready || q.size == 0 { - continue; - } - - let vring_num = VhostVringState { - index: i as u32, - num: q.size as u32, - }; - libc::ioctl(vhost_fd, VHOST_SET_VRING_NUM, &vring_num); - - let vring_addr = VhostVringAddr { - index: i as u32, - flags: 0, - desc_user_addr: guest_to_host(mem, q.desc_addr), - used_user_addr: guest_to_host(mem, q.used_addr), - avail_user_addr: guest_to_host(mem, q.avail_addr), - log_guest_addr: 0, - }; - libc::ioctl(vhost_fd, VHOST_SET_VRING_ADDR, &vring_addr); - - let vring_base = VhostVringState { - index: i as u32, - num: 0, - }; - libc::ioctl(vhost_fd, VHOST_SET_VRING_BASE, &vring_base); - - let kick_evt = EventFd::new(0).expect("eventfd"); - let call_evt = EventFd::new(0).expect("eventfd"); - - let call_file = VhostVringFile { - index: i as u32, - fd: call_evt.as_raw_fd(), - }; - libc::ioctl(vhost_fd, VHOST_SET_VRING_CALL, &call_file); - - // KICK last — starts the vhost data plane - let kick_file = VhostVringFile { - index: i as u32, - fd: kick_evt.as_raw_fd(), - }; - libc::ioctl(vhost_fd, VHOST_SET_VRING_KICK, &kick_file); - - self.kick_evts.push(kick_evt); - self.call_evts.push(call_evt); - } - - // CID AFTER queue setup - if libc::ioctl(vhost_fd, VHOST_VSOCK_SET_GUEST_CID, &self.guest_cid) < 0 { - eprintln!( - "shuru: VHOST_VSOCK_SET_GUEST_CID failed: {}", - std::io::Error::last_os_error() - ); - return; - } - - // START the vhost worker — without this, no queues are processed! - let running: libc::c_int = 1; - if libc::ioctl(vhost_fd, VHOST_VSOCK_SET_RUNNING, &running) < 0 { - eprintln!( - "shuru: VHOST_VSOCK_SET_RUNNING failed: {}", - std::io::Error::last_os_error() - ); - return; - } - // Vhost worker is now running and processing queues. - - // Can't use KVM_IRQFD because we need to set InterruptStatus - // BEFORE the interrupt reaches the guest. - self.irq_running.store(true, Ordering::Release); - let irq_running = self.irq_running.clone(); - let vm_fd_clone = vm_fd.clone(); - let call_fd_0 = self.call_evts.get(0).map(|e| e.as_raw_fd()).unwrap_or(-1); - let call_fd_1 = self.call_evts.get(1).map(|e| e.as_raw_fd()).unwrap_or(-1); - self.irq_thread = std::thread::Builder::new() - .name("vsock-irq".into()) - .spawn(move || { - let mut pfds = [ - libc::pollfd { - fd: call_fd_0, - events: libc::POLLIN, - revents: 0, - }, - libc::pollfd { - fd: call_fd_1, - events: libc::POLLIN, - revents: 0, - }, - ]; - while irq_running.load(Ordering::Acquire) { - let ret = libc::poll(pfds.as_mut_ptr(), 2, 500); - if ret > 0 { - for pfd in &mut pfds { - if pfd.revents & (libc::POLLIN | libc::POLLNVAL) == libc::POLLIN { - let mut val: u64 = 0; - libc::read(pfd.fd, &mut val as *mut _ as *mut libc::c_void, 8); - _interrupt_status.fetch_or(1, Ordering::Release); - let _ = vm_fd_clone.set_irq_line(irq, true); - } - } - } - } - }) - .ok(); - } + fn memory() -> GuestMemoryMmap { + GuestMemoryMmap::from_ranges(&[(GuestAddress(0), 0x20000)]).unwrap() } - fn process_queue( - &mut self, - queue_idx: u16, - _queues: &mut [VirtioQueueState], - _mem: &GuestMemoryMmap, - _vm_fd: &Arc, - _irq: u32, - ) { - // Signal the kick eventfd so the vhost kernel module processes the queue. - if let Some(evt) = self.kick_evts.get(queue_idx as usize) { - let _ = evt.write(1); - } + #[test] + fn pending_rejects_unusable_queues() { + let mem = memory(); + offer(&mem, &[0]); + assert_eq!(queue(8).pending(&mem), Some(1)); + assert_eq!(queue(0).pending(&mem), None); + assert_eq!(queue(257).pending(&mem), None); + let mut far = queue(8); + far.desc_addr = 0x1FFC0; + assert_eq!(far.pending(&mem), None); + mem.write_obj(9u16, GuestAddress(AVAIL + 2)).unwrap(); + assert_eq!(queue(8).pending(&mem), None); } - fn reset(&mut self) { - self.irq_running.store(false, Ordering::Release); - if let Some(h) = self.irq_thread.take() { - let _ = h.join(); - } - self.kick_evts.clear(); - self.call_evts.clear(); - if let Some(fd) = self.vhost_fd.take() { - unsafe { - libc::close(fd); - } - } + #[test] + fn chain_rejects_loops_out_of_range_and_foreign_flags() { + let mem = memory(); + let q = queue(8); + desc(&mem, 0, 0x8000, 4, VRING_DESC_F_NEXT, 1); + desc(&mem, 1, 0x8000, 4, 0, 0); + assert_eq!( + q.chain(&mem, 0, VRING_DESC_F_NEXT).map(|c| c.len()), + Some(2) + ); + desc(&mem, 1, 0x8000, 4, VRING_DESC_F_NEXT, 0); + assert!(q.chain(&mem, 0, VRING_DESC_F_NEXT).is_none()); + desc(&mem, 1, 0x8000, 4, VRING_DESC_F_NEXT, 8); + assert!(q.chain(&mem, 0, VRING_DESC_F_NEXT).is_none()); + desc(&mem, 1, 0x8000, 4, VRING_DESC_F_WRITE, 0); + assert!(q.chain(&mem, 0, VRING_DESC_F_NEXT).is_none()); + assert!(q.chain(&mem, 8, VRING_DESC_F_NEXT).is_none()); } -} -impl Drop for VhostVsockBackend { - fn drop(&mut self) { - self.reset(); + #[test] + fn net_tx_drops_bad_chains_and_sends_good_frames() { + let mem = memory(); + let (host, guest) = std::os::unix::net::UnixDatagram::pair().unwrap(); + host.set_nonblocking(true).unwrap(); + let mut net = NetBackend::new(guest.as_raw_fd(), [0; 6]); + let mut queues = [queue(8), queue(8)]; + + // A chain that points at itself, then one descriptor of 4 GiB. + desc(&mem, 0, 0x8000, 64, VRING_DESC_F_NEXT, 0); + desc(&mem, 1, 0x8000, u32::MAX, 0, 0); + // A frame split over two descriptors. + mem.write_slice(&[0xAB; 64], GuestAddress(0x9000)).unwrap(); + desc( + &mem, + 2, + 0x9000, + VIRTIO_NET_HDR_SIZE as u32, + VRING_DESC_F_NEXT, + 3, + ); + desc(&mem, 3, 0x9000 + VIRTIO_NET_HDR_SIZE as u64, 20, 0, 0); + offer(&mem, &[0, 1, 2]); + + assert!(net.process_queue(1, &mut queues, &mem)); + assert_eq!(queues[1].last_avail_idx, 3); + let mut frame = [0u8; 128]; + assert_eq!(host.recv(&mut frame).unwrap(), 20); + assert!(host.recv(&mut frame).is_err()); + drop(guest); } } diff --git a/crates/shuru-linux/src/kvm/layout.rs b/crates/shuru-linux/src/kvm/layout.rs index 48c5115..d5b3646 100644 --- a/crates/shuru-linux/src/kvm/layout.rs +++ b/crates/shuru-linux/src/kvm/layout.rs @@ -1,7 +1,7 @@ -/// aarch64 memory map for the KVM virtual machine. -/// -/// Below DRAM_BASE is MMIO space for devices. -/// DRAM starts at 1 GiB and extends for the configured memory size. +//! aarch64 memory map for the KVM virtual machine. +//! +//! Below DRAM_BASE is MMIO space for devices. +//! DRAM starts at 1 GiB and extends for the configured memory size. // RAM pub const DRAM_BASE: u64 = 0x4000_0000; // 1 GiB @@ -59,6 +59,3 @@ pub const PSTATE_FAULT_BITS_64: u64 = 0x3c5; // crate but we also reference them directly for GIC setup) pub const KVM_ARM_VCPU_PSCI_0_2: u32 = 2; pub const KVM_ARM_VCPU_POWER_OFF: u32 = 0; - -pub const GUEST_CID: u64 = 3; -pub const AF_VSOCK: i32 = 40; diff --git a/crates/shuru-linux/src/kvm/mod.rs b/crates/shuru-linux/src/kvm/mod.rs index c39dae2..ed7428e 100644 --- a/crates/shuru-linux/src/kvm/mod.rs +++ b/crates/shuru-linux/src/kvm/mod.rs @@ -1,6 +1,8 @@ +pub mod balloon; pub mod devices; pub mod layout; pub mod virtio_fs; +pub mod vsock; use std::fs::File; use std::io::Read; @@ -28,39 +30,53 @@ use kvm_bindings::{KVM_DEV_ARM_VGIC_GRP_ADDR, KVM_DEV_ARM_VGIC_GRP_CTRL}; pub struct KvmVm { #[allow(dead_code)] pub kvm: Kvm, + #[allow(dead_code)] pub vm_fd: Arc, + #[allow(dead_code)] pub mem: GuestMemoryMmap, - pub bus: Arc>, + pub bus: Arc, pub vcpus: Vec, pub vcpu_threads: Vec>, pub running: Arc, - pub guest_cid: u64, + pub vsock: Option>, + pub balloon: Option>, #[allow(dead_code)] _gic: DeviceFd, } pub struct VmCreateConfig { + /// An open /dev/kvm; opened here when absent. + pub kvm: Option, pub cpu_count: usize, pub memory_bytes: u64, - pub kernel_path: String, - pub initrd_path: Option, + pub kernel: File, + pub initrd: Option, pub command_line: String, pub serial_write_fd: Option, pub serial_read_fd: Option, - pub disk_path: Option, + pub disk: Option, pub disk_read_only: bool, pub network_fd: Option, pub network_mac: Option<[u8; 6]>, pub has_vsock: bool, - pub guest_cid: u64, + pub vsock_connections: usize, /// Virtio-fs mounts: (tag, host_path, read_only). pub mounts: Vec<(String, String, bool)>, + pub has_balloon: bool, } impl KvmVm { - pub fn create(config: VmCreateConfig) -> Result { - let kvm = - Kvm::new().map_err(|e| VzError::new(format!("failed to open /dev/kvm: {}", e)))?; + pub fn create(mut config: VmCreateConfig) -> Result { + let kvm = match config.kvm.take() { + // SAFETY: the descriptor is an open /dev/kvm handed over to us. + Some(fd) => unsafe { + use std::os::fd::{FromRawFd, IntoRawFd}; + Kvm::from_raw_fd(fd.into_raw_fd()) + }, + None => { + Kvm::new().map_err(|e| VzError::new(format!("failed to open /dev/kvm: {}", e)))? + } + }; let vm_fd = kvm .create_vm() .map_err(|e| VzError::new(format!("failed to create VM: {}", e)))?; @@ -91,21 +107,24 @@ impl KvmVm { } // Load kernel (PE format for aarch64) - let mut kernel_file = File::open(&config.kernel_path) - .map_err(|e| VzError::new(format!("failed to open kernel: {}", e)))?; // kernel_offset must be 2MB-aligned. The PE loader adds text_offset // from the image header internally, so pass DRAM_BASE (1 GiB aligned). - let kernel_result = PE::load(&mem, Some(GuestAddress(DRAM_BASE)), &mut kernel_file, None) - .map_err(|e| VzError::new(format!("failed to load kernel: {}", e)))?; + let kernel_result = PE::load( + &mem, + Some(GuestAddress(DRAM_BASE)), + &mut config.kernel, + None, + ) + .map_err(|e| VzError::new(format!("failed to load kernel: {}", e)))?; let entry_addr = kernel_result.kernel_load.raw_value(); // Load initrd let mut initrd_addr: Option = None; let mut initrd_size: u64 = 0; - if let Some(ref initrd_path) = config.initrd_path { + if let Some(mut initrd) = config.initrd.take() { let mut initrd_data = Vec::new(); - File::open(initrd_path) - .and_then(|mut f| f.read_to_end(&mut initrd_data)) + initrd + .read_to_end(&mut initrd_data) .map_err(|e| VzError::new(format!("failed to read initrd: {}", e)))?; // Place after FDT reservation, page-aligned @@ -126,48 +145,29 @@ impl KvmVm { bus.add( UART_BASE, UART_SIZE, - Box::new(Pl011::new(config.serial_write_fd, config.serial_read_fd)), + Box::new(Mutex::new(Pl011::new( + config.serial_write_fd, + config.serial_read_fd, + ))), ); // PL031 RTC - needed so the guest boots with the host wall clock // (otherwise TLS certs are "not yet valid" since 1970-01-01). - bus.add(RTC_BASE, RTC_SIZE, Box::new(Pl031::new())); + bus.add(RTC_BASE, RTC_SIZE, Box::new(Mutex::new(Pl031::new()))); // Virtio block - if let Some(ref disk_path) = config.disk_path { - let blk = BlockBackend::new(disk_path, config.disk_read_only) + if let Some(disk) = config.disk.take() { + let blk = BlockBackend::new(disk, config.disk_read_only) .map_err(|e| VzError::new(format!("failed to open disk: {}", e)))?; - let spi = VIRTIO_SPI_BASE + virtio_idx; - let base = VIRTIO_MMIO_BASE + (virtio_idx as u64) * VIRTIO_MMIO_GAP; - bus.add( - base, - VIRTIO_MMIO_SIZE, - Box::new(VirtioMmioDevice::new( - Box::new(blk), - spi, - vm_fd.clone(), - mem.clone(), - )), - ); - virtio_idx += 1; + add_virtio(&mut bus, &mut virtio_idx, Box::new(blk), &vm_fd, &mem)?; } - // Vhost-vsock - let guest_cid = config.guest_cid; + let mut vsock_control = None; if config.has_vsock { - let spi = VIRTIO_SPI_BASE + virtio_idx; - let base = VIRTIO_MMIO_BASE + (virtio_idx as u64) * VIRTIO_MMIO_GAP; - bus.add( - base, - VIRTIO_MMIO_SIZE, - Box::new(VirtioMmioDevice::new( - Box::new(VhostVsockBackend::new(guest_cid)), - spi, - vm_fd.clone(), - mem.clone(), - )), - ); - virtio_idx += 1; + let backend = vsock::VsockBackend::new(config.vsock_connections) + .map_err(|e| VzError::new(format!("failed to create vsock device: {e}")))?; + vsock_control = Some(backend.control()); + add_virtio(&mut bus, &mut virtio_idx, Box::new(backend), &vm_fd, &mem)?; } // Virtio-fs mounts (one device per mount) @@ -178,19 +178,7 @@ impl KvmVm { tag, e )) })?; - let spi = VIRTIO_SPI_BASE + virtio_idx; - let base = VIRTIO_MMIO_BASE + (virtio_idx as u64) * VIRTIO_MMIO_GAP; - bus.add( - base, - VIRTIO_MMIO_SIZE, - Box::new(VirtioMmioDevice::new( - Box::new(fs), - spi, - vm_fd.clone(), - mem.clone(), - )), - ); - virtio_idx += 1; + add_virtio(&mut bus, &mut virtio_idx, Box::new(fs), &vm_fd, &mem)?; } // Virtio net @@ -198,19 +186,21 @@ impl KvmVm { let mac = config .network_mac .unwrap_or([0x02, 0x00, 0x00, 0x00, 0x00, 0x01]); - let spi = VIRTIO_SPI_BASE + virtio_idx; - let base = VIRTIO_MMIO_BASE + (virtio_idx as u64) * VIRTIO_MMIO_GAP; - bus.add( - base, - VIRTIO_MMIO_SIZE, - Box::new(VirtioMmioDevice::new( - Box::new(NetBackend::new(net_fd, mac)), - spi, - vm_fd.clone(), - mem.clone(), - )), - ); - virtio_idx += 1; + add_virtio( + &mut bus, + &mut virtio_idx, + Box::new(NetBackend::new(net_fd, mac)), + &vm_fd, + &mem, + )?; + } + + // Virtio balloon + let mut balloon_control = None; + if config.has_balloon { + let backend = balloon::BalloonBackend::new(config.memory_bytes); + balloon_control = Some(backend.control()); + add_virtio(&mut bus, &mut virtio_idx, Box::new(backend), &vm_fd, &mem)?; } // Create vCPUs (must exist before GIC init) @@ -253,11 +243,12 @@ impl KvmVm { kvm, vm_fd, mem, - bus: Arc::new(Mutex::new(bus)), + bus: Arc::new(bus), vcpus, vcpu_threads: Vec::new(), running: Arc::new(AtomicBool::new(false)), - guest_cid, + vsock: vsock_control, + balloon: balloon_control, _gic: gic, }) } @@ -291,6 +282,9 @@ impl KvmVm { for handle in self.vcpu_threads.drain(..) { let _ = handle.join(); } + if let Some(vsock) = &self.vsock { + vsock.deactivate(); + } } } @@ -547,11 +541,27 @@ fn generate_fdt( // vCPU run loop // =========================================================================== -fn vcpu_run_loop(mut vcpu: VcpuFd, bus: Arc>, running: Arc) { +fn add_virtio( + bus: &mut MmioBus, + virtio_idx: &mut u32, + backend: Box, + vm_fd: &Arc, + mem: &GuestMemoryMmap, +) -> Result<(), VzError> { + let spi = VIRTIO_SPI_BASE + *virtio_idx; + let base = VIRTIO_MMIO_BASE + (*virtio_idx as u64) * VIRTIO_MMIO_GAP; + let device = VirtioMmioDevice::new(backend, base, spi, vm_fd.clone(), mem.clone()) + .map_err(|e| VzError::new(format!("failed to set up virtio device: {e}")))?; + bus.add(base, VIRTIO_MMIO_SIZE, Box::new(device)); + *virtio_idx += 1; + Ok(()) +} + +fn vcpu_run_loop(mut vcpu: VcpuFd, bus: Arc, running: Arc) { // Install empty handler for SIGRTMIN so KVM_RUN returns EINTR on stop unsafe { let mut sa: libc::sigaction = std::mem::zeroed(); - sa.sa_sigaction = empty_signal_handler as usize; + sa.sa_sigaction = empty_signal_handler as *const () as usize; sa.sa_flags = libc::SA_SIGINFO; libc::sigaction(libc::SIGRTMIN(), &sa, std::ptr::null_mut()); } @@ -563,10 +573,10 @@ fn vcpu_run_loop(mut vcpu: VcpuFd, bus: Arc>, running: Arc { // data is &mut [u8] — write device response directly - bus.lock().unwrap().handle_read(addr, data); + bus.handle_read(addr, data); } VcpuExit::MmioWrite(addr, data) => { - bus.lock().unwrap().handle_write(addr, data); + bus.handle_write(addr, data); } VcpuExit::SystemEvent(..) | VcpuExit::Shutdown => { running.store(false, Ordering::Release); diff --git a/crates/shuru-linux/src/kvm/virtio_fs.rs b/crates/shuru-linux/src/kvm/virtio_fs.rs index a12b233..9b26109 100644 --- a/crates/shuru-linux/src/kvm/virtio_fs.rs +++ b/crates/shuru-linux/src/kvm/virtio_fs.rs @@ -12,17 +12,16 @@ //! gracefully. use std::collections::HashMap; -use std::ffi::{CStr, CString, OsStr, OsString}; -use std::os::unix::ffi::{OsStrExt, OsStringExt}; +use std::ffi::{CString, OsStr}; +use std::os::unix::ffi::OsStrExt; use std::os::unix::fs::MetadataExt; -use std::path::{Component, Path, PathBuf}; -use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; -use kvm_ioctls::VmFd; -use vm_memory::{Address, Bytes, GuestAddress, GuestMemoryMmap}; +use vm_memory::{Bytes, GuestAddress, GuestMemoryMmap}; -use super::devices::{avail_idx, avail_ring_entry, write_used, VirtioBackend, VirtioQueueState}; +use super::devices::{avail_ring_entry, write_used, Interrupt, VirtioBackend, VirtioQueueState}; // =========================================================================== // FUSE wire protocol (subset) @@ -81,6 +80,8 @@ const FUSE_READDIRPLUS_AUTO: u32 = 1 << 14; const FUSE_ASYNC_DIO: u32 = 1 << 15; const MAX_WRITE: u32 = 128 * 1024; +const MAX_OPEN_FILES: usize = 512; +const MAX_INODES: usize = 1 << 18; // =========================================================================== // Plain-data wire structs (repr(C, packed) so we can reinterpret raw bytes) @@ -174,14 +175,6 @@ struct FuseAttrOut { attr: FuseAttr, } -#[repr(C)] -#[derive(Default, Clone, Copy)] -struct FuseGetattrIn { - getattr_flags: u32, - dummy: u32, - fh: u64, -} - #[repr(C)] #[derive(Default, Clone, Copy)] struct FuseSetattrIn { @@ -328,20 +321,6 @@ struct FuseDirent { // name follows, padded to 8 bytes } -#[repr(C)] -#[derive(Default, Clone, Copy)] -struct FuseGetxattrIn { - size: u32, - padding: u32, -} - -#[repr(C)] -#[derive(Default, Clone, Copy)] -struct FuseAccessIn { - mask: u32, - padding: u32, -} - #[repr(C)] #[derive(Default, Clone, Copy)] struct FuseFsyncIn { @@ -373,35 +352,16 @@ struct DescSpan { writable: bool, } -fn collect_desc_chain( - mem: &GuestMemoryMmap, - desc_addr: u64, - head: u16, - max_descs: u16, -) -> Vec { - let mut out = Vec::new(); - let mut idx = head; - for _ in 0..max_descs { - let off = idx as u64 * 16; - let addr: u64 = mem.read_obj(GuestAddress(desc_addr + off)).unwrap_or(0); - let len: u32 = mem.read_obj(GuestAddress(desc_addr + off + 8)).unwrap_or(0); - let flags: u16 = mem - .read_obj(GuestAddress(desc_addr + off + 12)) - .unwrap_or(0); - let next: u16 = mem - .read_obj(GuestAddress(desc_addr + off + 14)) - .unwrap_or(0); - out.push(DescSpan { - addr, - len, - writable: flags & VRING_DESC_F_WRITE != 0, - }); - if flags & VRING_DESC_F_NEXT == 0 { - break; - } - idx = next; - } - out +fn collect_desc_chain(mem: &GuestMemoryMmap, q: &VirtioQueueState, head: u16) -> Vec { + q.chain(mem, head, VRING_DESC_F_NEXT | VRING_DESC_F_WRITE) + .unwrap_or_default() + .into_iter() + .map(|desc| DescSpan { + addr: desc.addr, + len: desc.len, + writable: desc.flags & VRING_DESC_F_WRITE != 0, + }) + .collect() } // =========================================================================== @@ -477,6 +437,9 @@ impl<'a> RequestReader<'a> { if ch[0] == 0 { return CString::new(bytes).ok(); } + if bytes.len() >= libc::PATH_MAX as usize { + return None; + } bytes.push(ch[0]); } } @@ -517,14 +480,6 @@ impl<'a> ResponseWriter<'a> { } } - fn capacity(&self) -> u32 { - self.spans - .iter() - .filter(|s| s.writable) - .map(|s| s.len) - .sum() - } - fn write_bytes(&mut self, input: &[u8]) -> bool { let mut written = 0; while written < input.len() { @@ -623,13 +578,18 @@ impl InodeTable { } /// Ensure (dev, ino) is mapped; bump its refcount and return the nodeid. - fn intern(&mut self, path: PathBuf, meta: &std::fs::Metadata) -> u64 { + /// None once MAX_INODES are mapped and this one is not among them: the + /// guest holds a reference to every mapped inode until it sends FORGET. + fn intern(&mut self, path: PathBuf, meta: &std::fs::Metadata) -> Option { let key = (meta.dev(), meta.ino()); if let Some(&id) = self.by_hostid.get(&key) { if let Some(entry) = self.by_id.get_mut(&id) { entry.refcount = entry.refcount.saturating_add(1); } - return id; + return Some(id); + } + if self.full() { + return None; } let id = self.next_id.fetch_add(1, Ordering::Relaxed); self.by_id.insert( @@ -642,7 +602,11 @@ impl InodeTable { }, ); self.by_hostid.insert(key, id); - id + Some(id) + } + + fn full(&self) -> bool { + self.by_id.len() >= MAX_INODES } fn forget(&mut self, nodeid: u64, n: u64) { @@ -685,10 +649,19 @@ impl FhTable { } } - fn insert(&mut self, file: std::fs::File) -> u64 { + /// None once the guest holds MAX_OPEN_FILES handles, so it cannot take + /// the descriptors the VMM needs for its own devices. + fn insert(&mut self, file: std::fs::File) -> Option { + if self.full() { + return None; + } let fh = self.next.fetch_add(1, Ordering::Relaxed); self.files.insert(fh, file); - fh + Some(fh) + } + + fn full(&self) -> bool { + self.files.len() >= MAX_OPEN_FILES } fn get(&self, fh: u64) -> Option<&std::fs::File> { @@ -764,49 +737,41 @@ impl VirtioBackend for VirtioFsBackend { fn config_write(&mut self, _: u64, _: u32) {} - fn activate( - &mut self, - _queues: &[VirtioQueueState], - _mem: &GuestMemoryMmap, - _vm_fd: &Arc, - _irq: u32, - _interrupt_status: Arc, - ) { - } + fn activate(&mut self, _: &[VirtioQueueState], _: &GuestMemoryMmap, _: Arc) {} fn process_queue( &mut self, - _queue_idx: u16, + queue_idx: u16, queues: &mut [VirtioQueueState], mem: &GuestMemoryMmap, - _vm_fd: &Arc, - _irq: u32, - ) { + ) -> bool { // Both queues (hiprio = 0, request = 1) are handled the same way. - for q_idx in 0..queues.len() { - let q = match queues.get_mut(q_idx) { - Some(q) if q.ready && q.size > 0 => q, - _ => continue, - }; - - let current_avail = avail_idx(mem, q.avail_addr); - while q.last_avail_idx != current_avail { - let head = avail_ring_entry(mem, q.avail_addr, q.size, q.last_avail_idx); - let spans = collect_desc_chain(mem, q.desc_addr, head, q.size); - - let written = self.handle_request(mem, &spans); - - write_used( - mem, - q.used_addr, - q.size, - q.last_avail_idx, - head as u32, - written, - ); - q.last_avail_idx = q.last_avail_idx.wrapping_add(1); - } + let q = match queues.get_mut(usize::from(queue_idx)) { + Some(q) => q, + None => return false, + }; + let current_avail = match q.pending(mem) { + Some(idx) => idx, + None => return false, + }; + let used = q.last_avail_idx != current_avail; + while q.last_avail_idx != current_avail { + let head = avail_ring_entry(mem, q.avail_addr, q.size, q.last_avail_idx); + let spans = collect_desc_chain(mem, q, head); + + let written = self.handle_request(mem, &spans); + + write_used( + mem, + q.used_addr, + q.size, + q.last_avail_idx, + head as u32, + written, + ); + q.last_avail_idx = q.last_avail_idx.wrapping_add(1); } + used } fn reset(&mut self) { @@ -967,7 +932,10 @@ impl VirtioFsBackend { if !inodes.contains(&child_path) { return (-libc::EACCES, 0); } - let nodeid = inodes.intern(child_path.clone(), &meta); + let nodeid = match inodes.intern(child_path.clone(), &meta) { + Some(id) => id, + None => return (-libc::ENFILE, 0), + }; let out = FuseEntryOut { nodeid, generation: 0, @@ -1036,10 +1004,10 @@ impl VirtioFsBackend { Err(_) => return (-libc::EINVAL, 0), }; - if req.valid & FATTR_MODE != 0 { - if unsafe { libc::chmod(cpath.as_ptr(), req.mode as libc::mode_t) } < 0 { - return (-errno(), 0); - } + if req.valid & FATTR_MODE != 0 + && unsafe { libc::chmod(cpath.as_ptr(), req.mode as libc::mode_t) } < 0 + { + return (-errno(), 0); } if req.valid & (FATTR_UID | FATTR_GID) != 0 { let uid = if req.valid & FATTR_UID != 0 { @@ -1056,10 +1024,10 @@ impl VirtioFsBackend { return (-errno(), 0); } } - if req.valid & FATTR_SIZE != 0 { - if unsafe { libc::truncate(cpath.as_ptr(), req.size as libc::off_t) } < 0 { - return (-errno(), 0); - } + if req.valid & FATTR_SIZE != 0 + && unsafe { libc::truncate(cpath.as_ptr(), req.size as libc::off_t) } < 0 + { + return (-errno(), 0); } if req.valid & (FATTR_ATIME | FATTR_MTIME | FATTR_ATIME_NOW | FATTR_MTIME_NOW) != 0 { let now = libc::timespec { @@ -1074,7 +1042,7 @@ impl VirtioFsBackend { now } else if req.valid & FATTR_ATIME != 0 { libc::timespec { - tv_sec: req.atime as libc::time_t, + tv_sec: req.atime as _, tv_nsec: req.atimensec as i64, } } else { @@ -1084,7 +1052,7 @@ impl VirtioFsBackend { now } else if req.valid & FATTR_MTIME != 0 { libc::timespec { - tv_sec: req.mtime as libc::time_t, + tv_sec: req.mtime as _, tv_nsec: req.mtimensec as i64, } } else { @@ -1127,7 +1095,10 @@ impl VirtioFsBackend { drop(inodes); match std::fs::File::open(&path) { Ok(f) => { - let fh = self.fhs.lock().unwrap().insert(f); + let fh = match self.fhs.lock().unwrap().insert(f) { + Some(fh) => fh, + None => return (-libc::EMFILE, 0), + }; let out = FuseOpenOut { fh, open_flags: 0, @@ -1186,17 +1157,7 @@ impl VirtioFsBackend { }; let child_path = ent.path(); - let this_nodeid = if plus { - let mut inodes = self.inodes.lock().unwrap(); - if !inodes.contains(&child_path) { - continue; - } - inodes.intern(child_path.clone(), &meta) - } else { - meta.ino() - }; - - let kind = mode_to_dt(meta.mode() as u32); + let kind = mode_to_dt(meta.mode()); let name_bytes = name.as_bytes(); let namelen = name_bytes.len(); let entry_size = if plus { @@ -1210,6 +1171,21 @@ impl VirtioFsBackend { break; } + // An entry is interned only once it is sure to be reported, since + // the guest only forgets the nodeids it was sent. + let this_nodeid = if plus { + let mut inodes = self.inodes.lock().unwrap(); + if !inodes.contains(&child_path) { + continue; + } + match inodes.intern(child_path.clone(), &meta) { + Some(id) => id, + None => break, + } + } else { + meta.ino() + }; + if plus { let e = FuseEntryOut { nodeid: this_nodeid, @@ -1289,7 +1265,10 @@ impl VirtioFsBackend { return (-errno(), 0); } let file = unsafe { std::os::unix::io::FromRawFd::from_raw_fd(fd) }; - let fh = self.fhs.lock().unwrap().insert(file); + let fh = match self.fhs.lock().unwrap().insert(file) { + Some(fh) => fh, + None => return (-libc::EMFILE, 0), + }; let out = FuseOpenOut { fh, open_flags: 0, @@ -1319,6 +1298,12 @@ impl VirtioFsBackend { if inodes.read_only { return (-libc::EROFS, 0); } + if inodes.full() { + return (-libc::ENFILE, 0); + } + if self.fhs.lock().unwrap().full() { + return (-libc::EMFILE, 0); + } let parent_path = match inodes.path_of(parent) { Some(p) => p, None => return (-libc::ENOENT, 0), @@ -1351,11 +1336,12 @@ impl VirtioFsBackend { return (-io_errno(&e), 0); } }; - let nodeid = inodes.intern(child.clone(), &meta); + // Room for both was checked above, under the same locks' owner. + let nodeid = inodes.intern(child.clone(), &meta).unwrap_or(0); drop(inodes); let file = unsafe { std::os::unix::io::FromRawFd::from_raw_fd(fd) }; - let fh = self.fhs.lock().unwrap().insert(file); + let fh = self.fhs.lock().unwrap().insert(file).unwrap_or(0); let entry = FuseEntryOut { nodeid, @@ -1396,7 +1382,7 @@ impl VirtioFsBackend { let raw_fd = std::os::unix::io::AsRawFd::as_raw_fd(file); drop(fhs); - let mut buf = vec![0u8; req.size as usize]; + let mut buf = vec![0u8; req.size.min(MAX_WRITE) as usize]; let n = unsafe { libc::pread( raw_fd, @@ -1422,6 +1408,9 @@ impl VirtioFsBackend { if self.inodes.lock().unwrap().read_only { return (-libc::EROFS, 0); } + if req.size > MAX_WRITE { + return (-libc::EINVAL, 0); + } let data = match reader.read_vec(req.size as usize) { Some(v) => v, None => return (-libc::EIO, 0), @@ -1502,11 +1491,11 @@ impl VirtioFsBackend { return (-errno(), 0); } let out = FuseStatfsOut { - blocks: stat.f_blocks as u64, - bfree: stat.f_bfree as u64, - bavail: stat.f_bavail as u64, - files: stat.f_files as u64, - ffree: stat.f_ffree as u64, + blocks: stat.f_blocks, + bfree: stat.f_bfree, + bavail: stat.f_bavail, + files: stat.f_files, + ffree: stat.f_ffree, bsize: stat.f_bsize as u32, namelen: stat.f_namemax as u32, frsize: stat.f_frsize as u32, @@ -1593,7 +1582,10 @@ impl VirtioFsBackend { Ok(m) => m, Err(e) => return (-io_errno(&e), 0), }; - let nodeid = inodes.intern(child.clone(), &meta); + let nodeid = match inodes.intern(child.clone(), &meta) { + Some(id) => id, + None => return (-libc::ENFILE, 0), + }; let out = FuseEntryOut { nodeid, generation: 0, @@ -1695,7 +1687,7 @@ fn attr_from_meta(ino: u64, meta: &std::fs::Metadata) -> FuseAttr { atimensec: meta.atime_nsec() as u32, mtimensec: meta.mtime_nsec() as u32, ctimensec: meta.ctime_nsec() as u32, - mode: meta.mode() as u32, + mode: meta.mode(), nlink: meta.nlink() as u32, uid: meta.uid(), gid: meta.gid(), @@ -1706,14 +1698,14 @@ fn attr_from_meta(ino: u64, meta: &std::fs::Metadata) -> FuseAttr { } fn mode_to_dt(mode: u32) -> u32 { - match mode & libc::S_IFMT as u32 { - m if m == libc::S_IFREG as u32 => libc::DT_REG as u32, - m if m == libc::S_IFDIR as u32 => libc::DT_DIR as u32, - m if m == libc::S_IFLNK as u32 => libc::DT_LNK as u32, - m if m == libc::S_IFIFO as u32 => libc::DT_FIFO as u32, - m if m == libc::S_IFSOCK as u32 => libc::DT_SOCK as u32, - m if m == libc::S_IFCHR as u32 => libc::DT_CHR as u32, - m if m == libc::S_IFBLK as u32 => libc::DT_BLK as u32, + match mode & libc::S_IFMT { + m if m == libc::S_IFREG => libc::DT_REG as u32, + m if m == libc::S_IFDIR => libc::DT_DIR as u32, + m if m == libc::S_IFLNK => libc::DT_LNK as u32, + m if m == libc::S_IFIFO => libc::DT_FIFO as u32, + m if m == libc::S_IFSOCK => libc::DT_SOCK as u32, + m if m == libc::S_IFCHR => libc::DT_CHR as u32, + m if m == libc::S_IFBLK => libc::DT_BLK as u32, _ => libc::DT_UNKNOWN as u32, } } @@ -1739,3 +1731,37 @@ fn errno() -> i32 { fn io_errno(e: &std::io::Error) -> i32 { e.raw_os_error().unwrap_or(libc::EIO) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_without_a_nul_stop_at_path_max() { + let mem = GuestMemoryMmap::from_ranges(&[(GuestAddress(0), 0x10000)]).unwrap(); + mem.write_slice(&[b'a'; 0x10000], GuestAddress(0)).unwrap(); + // Every span aliases the same guest bytes and claims 4 GiB. + let spans = vec![ + DescSpan { + addr: 0, + len: u32::MAX, + writable: false, + }; + 128 + ]; + assert!(RequestReader::new(&mem, &spans).read_cstr().is_none()); + } + + #[test] + fn open_handles_are_capped() { + let mut fhs = FhTable::new(); + for _ in 0..MAX_OPEN_FILES { + assert!(fhs + .insert(std::fs::File::open("/dev/null").unwrap()) + .is_some()); + } + assert!(fhs + .insert(std::fs::File::open("/dev/null").unwrap()) + .is_none()); + } +} diff --git a/crates/shuru-linux/src/kvm/vsock.rs b/crates/shuru-linux/src/kvm/vsock.rs new file mode 100644 index 0000000..49a16f8 --- /dev/null +++ b/crates/shuru-linux/src/kvm/vsock.rs @@ -0,0 +1,936 @@ +mod connection; +mod queue; + +use std::collections::VecDeque; +use std::io::{self, Read}; +use std::net::TcpStream; +use std::os::fd::{AsRawFd, FromRawFd, IntoRawFd, OwnedFd}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{mpsc, Arc, Mutex}; +use std::time::{Duration, Instant}; + +use vm_memory::{GuestAddress, GuestMemory, GuestMemoryMmap, VolatileSlice}; +use vmm_sys_util::eventfd::EventFd; + +use super::devices::{ + Interrupt, VirtioBackend, VirtioQueueState, INTERRUPT_USED, VIRTIO_F_VERSION_1, +}; +pub use connection::MAX_CONNECTIONS; +use connection::{ + readv, sendv, Header, Mux, Opener, Payload, RxBuffers, GUEST_CID, HEADER_LEN, MAX_PACKET, +}; +use queue::{invalid_queue, Queue}; + +// A finite batch keeps reset and host connects responsive under guest load. +const BATCH: usize = 256; +const IOV_MAX: usize = 1024; +// Epoll tokens: 0 is the wake eventfd, host ports fit in 32 bits, kicks +// sit above them. +const KICK: u64 = 1 << 40; +const CONNECT_TIMEOUT: Duration = Duration::from_secs(2); +// "CONNECT 4294967295\n" with room to spare. +const MAX_CONNECT_LINE: usize = 32; +// Listener clients that have not sent their CONNECT line yet. More are +// closed at once. +const MAX_HANDSHAKES: usize = 16; + +/// Splits a descriptor chain into its header bytes and the payload after. +fn split<'a>( + mem: &'a GuestMemoryMmap, + descriptors: &[(u64, usize)], +) -> io::Result<(Vec>, Vec>, usize)> { + let mut header = Vec::new(); + let mut payload = Vec::new(); + let mut missing = HEADER_LEN; + let mut len = 0; + for &(address, size) in descriptors { + if size == 0 { + continue; + } + let slice = mem + .get_slice(GuestAddress(address), size) + .map_err(|_| invalid_queue())?; + let take = missing.min(size); + if take > 0 { + header.push(slice.subslice(0, take).map_err(|_| invalid_queue())?); + missing -= take; + } + if size > take { + payload.push(slice.offset(take).map_err(|_| invalid_queue())?); + len += size - take; + } + } + if missing > 0 { + return Err(invalid_queue()); + } + Ok((header, payload, len)) +} + +struct TxPacket<'a> { + slices: Vec>, + len: usize, +} + +impl<'a> TxPacket<'a> { + fn parse(mem: &'a GuestMemoryMmap, descriptors: &[(u64, usize)]) -> io::Result<(Header, Self)> { + let (header, slices, len) = split(mem, descriptors)?; + let mut bytes = [0; HEADER_LEN]; + let mut offset = 0; + for slice in &header { + offset += slice.copy_to(&mut bytes[offset..]); + } + let header = Header::decode(&bytes).ok_or_else(invalid_queue)?; + Ok((header, Self { slices, len })) + } + + /// The slices holding the bytes from `offset` on. + fn from(&self, mut offset: usize) -> impl Iterator> + '_ { + self.slices.iter().filter_map(move |slice| { + if offset >= slice.len() { + offset -= slice.len(); + return None; + } + let rest = slice.offset(offset).ok(); + offset = 0; + rest + }) + } +} + +impl Payload for TxPacket<'_> { + fn len(&self) -> usize { + self.len + } + + fn send(&self, socket: &UnixStream, offset: usize) -> io::Result { + let slices: Vec<_> = self.from(offset).take(IOV_MAX).collect(); + let guards: Vec<_> = slices.iter().map(|slice| slice.ptr_guard()).collect(); + let iov: Vec = guards + .iter() + .zip(&slices) + .map(|(guard, slice)| libc::iovec { + iov_base: guard.as_ptr().cast_mut().cast(), + iov_len: slice.len(), + }) + .collect(); + sendv(socket, &iov) + } + + fn copy(&self, offset: usize, out: &mut VecDeque) { + let start = out.len(); + out.resize(start + self.len - offset, 0); + let (front, back) = out.as_mut_slices(); + let mut parts = if start >= front.len() { + vec![&mut back[start - front.len()..]] + } else { + vec![&mut front[start..], back] + } + .into_iter(); + let mut dst: &mut [u8] = parts.next().unwrap_or_default(); + for slice in self.from(offset) { + let mut done = 0; + while done < slice.len() { + if dst.is_empty() { + match parts.next() { + Some(next) => dst = next, + None => return, + } + } + let n = slice.offset(done).map_or(0, |rest| rest.copy_to(dst)); + if n == 0 { + return; + } + dst = &mut std::mem::take(&mut dst)[n..]; + done += n; + } + } + } +} + +struct RxChain<'a> { + head: u16, + header: Vec>, + payload: Vec>, + room: usize, + len: u32, +} + +// Buffers are read from the ring only as packets need them, so a small +// message does not pay for every buffer the guest posted. +struct RxBatch<'a> { + mem: &'a GuestMemoryMmap, + queue: &'a Queue, + chains: Vec>, + done: bool, + // A malformed chain at the front, to hand back empty. + invalid: Option, + error: Option, +} + +impl<'a> RxBatch<'a> { + fn new(mem: &'a GuestMemoryMmap, queue: &'a Queue) -> Self { + Self { + mem, + queue, + chains: Vec::new(), + done: false, + invalid: None, + error: None, + } + } + + fn load(&mut self, index: usize) -> bool { + while self.chains.len() <= index && !self.done { + if self.chains.len() == BATCH { + self.done = true; + break; + } + match self.queue.head_at(self.mem, self.chains.len()) { + Ok(Some(head)) => match self + .queue + .descriptors(self.mem, head, true) + .and_then(|descriptors| split(self.mem, &descriptors)) + { + Ok((header, payload, room)) => self.chains.push(RxChain { + head, + header, + payload, + room, + len: 0, + }), + Err(_) => { + if self.chains.is_empty() { + self.invalid = Some(head); + } + self.done = true; + } + }, + Ok(None) => self.done = true, + Err(error) => { + self.error = Some(error); + self.done = true; + } + } + } + index < self.chains.len() + } +} + +impl RxBuffers for RxBatch<'_> { + fn room(&mut self, index: usize) -> Option { + self.load(index) + .then(|| self.chains[index].room.min(MAX_PACKET)) + } + + fn put_header(&mut self, index: usize, header: Header) { + let chain = &mut self.chains[index]; + let bytes = header.encode(); + let mut offset = 0; + for slice in &chain.header { + slice.copy_from(&bytes[offset..offset + slice.len()]); + offset += slice.len(); + } + chain.len = (HEADER_LEN + header.len as usize) as u32; + } + + fn recv(&mut self, socket: &UnixStream, first: usize, max: usize) -> io::Result { + let mut parts = Vec::new(); + let mut left = max; + 'chains: for index in first.. { + let mut room = self.room(index).unwrap_or(0).min(left); + if room == 0 { + break; + } + left -= room; + for slice in &self.chains[index].payload { + if room == 0 { + break; + } + if parts.len() == IOV_MAX { + break 'chains; + } + let len = slice.len().min(room); + parts.push((slice.ptr_guard_mut(), len)); + room -= len; + } + } + let iov: Vec = parts + .iter() + .map(|(guard, len)| libc::iovec { + iov_base: guard.as_ptr().cast(), + iov_len: *len, + }) + .collect(); + readv(socket, &iov) + } +} + +struct Transport { + rx: Queue, + tx: Queue, + mem: GuestMemoryMmap, + interrupt: Arc, +} + +impl Transport { + // One pass over both queues with guest kicks off. Returns whether there + // is more to do right away. + fn process(&mut self, mux: &mut Mux) -> io::Result { + let Transport { + rx, + tx, + mem, + interrupt, + } = self; + tx.set_notify(mem, false)?; + rx.set_notify(mem, false)?; + + let mut tx_used = false; + for _ in 0..BATCH { + if !mux.can_receive() { + break; + } + let Some(head) = tx.head(mem)? else { + break; + }; + if let Ok(descriptors) = tx.descriptors(mem, head, false) { + if let Ok((header, payload)) = TxPacket::parse(mem, &descriptors) { + mux.receive(header, &payload); + } + } + tx.complete(mem, head, 0)?; + tx_used = true; + } + mux.pump(); + + let mut batch = RxBatch::new(mem, rx); + let used = mux.fill(&mut batch); + let filled: Vec<(u16, u32)> = batch.chains[..used] + .iter() + .map(|chain| (chain.head, chain.len)) + .collect(); + let invalid = batch.invalid; + if let Some(error) = batch.error.take() { + return Err(error); + } + drop(batch); + for (head, len) in filled { + rx.complete(mem, head, len)?; + } + if let Some(head) = invalid { + rx.complete(mem, head, 0)?; + } + let rx_used = used > 0 || invalid.is_some(); + let rx_drained = rx.head(mem)?.is_none(); + + tx.set_notify(mem, true)?; + rx.set_notify(mem, true)?; + if (tx_used && tx.wants_interrupt(mem)) || (rx_used && rx.wants_interrupt(mem)) { + interrupt.raise(INTERRUPT_USED); + } + // Buffers posted while kicks were off came without one. + Ok(tx_used + || rx_used + || (mux.can_receive() && tx.head(mem)?.is_some()) + || (rx_drained && rx.head(mem)?.is_some())) + } +} + +struct State { + transport: Option, + mux: Mux, + failed: bool, +} + +pub struct VsockControl { + state: Mutex, + wake: EventFd, + kicks: Mutex>, + epoll: OwnedFd, + stopping: AtomicBool, + listeners: Mutex>>, + handshakes: AtomicUsize, +} + +impl VsockControl { + pub fn deactivate(&self) { + let mut state = self.state.lock().unwrap(); + state.transport = None; + state.mux.clear(); + state.failed = false; + self.notify(); + } + + fn notify(&self) { + let _ = self.wake.write(1); + } + + fn register(&self, fd: i32, token: u64, events: i32) -> io::Result<()> { + let mut event = libc::epoll_event { + events: events as u32, + u64: token, + }; + let ret = + unsafe { libc::epoll_ctl(self.epoll.as_raw_fd(), libc::EPOLL_CTL_ADD, fd, &mut event) }; + if ret < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + } + } + + /// Hands `device` to the mux as the host end of a connect to `port`. + fn attach(&self, device: UnixStream, port: u32, opener: Opener) -> io::Result { + let host_port; + { + let mut state = self.state.lock().unwrap(); + if state.failed || state.transport.is_none() || self.stopping.load(Ordering::Acquire) { + return Err(io::Error::new( + io::ErrorKind::ConnectionRefused, + "vsock device is not ready", + )); + } + let fd = device.as_raw_fd(); + host_port = state.mux.add(device, port, opener)?; + let events = libc::EPOLLIN | libc::EPOLLOUT | libc::EPOLLRDHUP | libc::EPOLLET; + if let Err(error) = self.register(fd, host_port as u64, events) { + state.mux.cancel(host_port); + return Err(error); + } + } + self.notify(); + Ok(host_port) + } + + pub fn connect(&self, port: u32) -> io::Result { + let (client, device) = UnixStream::pair()?; + let (sender, receiver) = mpsc::sync_channel(1); + let host_port = self.attach(device, port, Opener::Caller(sender))?; + match receiver.recv_timeout(CONNECT_TIMEOUT) { + Ok(Ok(())) => { + // The public backend API uses TcpStream for any connected stream fd. + Ok(unsafe { TcpStream::from_raw_fd(client.into_raw_fd()) }) + } + result => { + self.state.lock().unwrap().mux.cancel(host_port); + self.notify(); + match result { + Ok(Err(error)) => Err(error), + _ => Err(io::Error::new( + io::ErrorKind::TimedOut, + "vsock connect timed out", + )), + } + } + } + } + + /// Accepts host connects on `listener`. A client writes + /// `CONNECT \n` and reads `OK \n` once the guest + /// accepts; any other outcome closes the socket. + pub fn serve(self: &Arc, listener: UnixListener) -> io::Result<()> { + let listener = Arc::new(listener); + self.listeners.lock().unwrap().push(listener.clone()); + let control = self.clone(); + std::thread::Builder::new() + .name("shuru-vsock-listen".into()) + .spawn(move || control.accept(&listener))?; + Ok(()) + } + + fn accept(self: &Arc, listener: &UnixListener) { + loop { + let stream = match listener.accept() { + Ok((stream, _)) => stream, + Err(_) if self.stopping.load(Ordering::Acquire) => return, + Err(e) => match e.raw_os_error() { + Some(libc::EINTR | libc::ECONNABORTED | libc::EPROTO) => continue, + // Out of descriptors or memory: wait for connections to close. + Some(libc::EMFILE | libc::ENFILE | libc::ENOBUFS | libc::ENOMEM) => { + std::thread::sleep(Duration::from_millis(10)); + continue; + } + _ => return, + }, + }; + if self.handshakes.fetch_add(1, Ordering::AcqRel) >= MAX_HANDSHAKES { + self.handshakes.fetch_sub(1, Ordering::AcqRel); + continue; + } + // Each client gets its own thread, so a silent one holds up + // nobody else. + let control = self.clone(); + let handshake = std::thread::Builder::new() + .name("shuru-vsock-connect".into()) + .spawn(move || { + if let Ok(port) = read_connect(&stream) { + let _ = control.attach(stream, port, Opener::Listener); + } + control.handshakes.fetch_sub(1, Ordering::AcqRel); + }); + if handshake.is_err() { + self.handshakes.fetch_sub(1, Ordering::AcqRel); + } + } + } +} + +/// Reads `CONNECT \n` without consuming anything after it. +fn read_connect(stream: &UnixStream) -> io::Result { + let invalid = || io::Error::new(io::ErrorKind::InvalidData, "bad vsock connect line"); + stream.set_read_timeout(Some(CONNECT_TIMEOUT))?; + let mut line = Vec::new(); + let mut byte = [0]; + while line.len() < MAX_CONNECT_LINE { + if (&*stream).read(&mut byte)? == 0 { + return Err(invalid()); + } + if byte[0] == b'\n' { + return std::str::from_utf8(&line) + .ok() + .and_then(|line| line.strip_prefix("CONNECT ")) + .filter(|port| port.bytes().all(|b| b.is_ascii_digit())) + .and_then(|port| port.parse().ok()) + .ok_or_else(invalid); + } + line.push(byte[0]); + } + Err(invalid()) +} + +pub struct VsockBackend { + control: Arc, + worker: Option>, + negotiated: bool, +} + +impl VsockBackend { + pub fn new(max_connections: usize) -> io::Result { + let wake = EventFd::new(libc::EFD_NONBLOCK | libc::EFD_CLOEXEC)?; + let fd = unsafe { libc::epoll_create1(libc::EPOLL_CLOEXEC) }; + if fd < 0 { + return Err(io::Error::last_os_error()); + } + let control = Arc::new(VsockControl { + state: Mutex::new(State { + transport: None, + mux: Mux::with_limit(max_connections), + failed: false, + }), + wake, + kicks: Mutex::new(Vec::new()), + epoll: unsafe { OwnedFd::from_raw_fd(fd) }, + stopping: AtomicBool::new(false), + listeners: Mutex::new(Vec::new()), + handshakes: AtomicUsize::new(0), + }); + control.register(control.wake.as_raw_fd(), 0, libc::EPOLLIN)?; + let worker_control = control.clone(); + let worker = std::thread::Builder::new() + .name("shuru-vsock".into()) + .spawn(move || run(worker_control))?; + Ok(Self { + control, + worker: Some(worker), + negotiated: false, + }) + } + + pub fn control(&self) -> Arc { + self.control.clone() + } +} + +/// Milliseconds until `deadline`, rounded up so it has passed on waking. +fn wait_ms(deadline: Option) -> i32 { + deadline.map_or(-1, |deadline| { + let left = deadline.saturating_duration_since(Instant::now()); + left.as_micros().div_ceil(1000).min(i32::MAX as u128) as i32 + }) +} + +fn run(control: Arc) { + let mut events = [libc::epoll_event { events: 0, u64: 0 }; 128]; + let mut busy = false; + while !control.stopping.load(Ordering::Acquire) { + let timeout = if busy { + 0 + } else { + wait_ms(control.state.lock().unwrap().mux.next_deadline()) + }; + let n = unsafe { + libc::epoll_wait( + control.epoll.as_raw_fd(), + events.as_mut_ptr(), + events.len() as i32, + timeout, + ) + }; + if n < 0 && io::Error::last_os_error().kind() == io::ErrorKind::Interrupted { + continue; + } + let mut state = control.state.lock().unwrap(); + if n < 0 { + state.failed = true; + state.mux.clear(); + state.transport = None; + break; + } + for event in &events[..n as usize] { + let token = event.u64; + if token == 0 { + let _ = control.wake.read(); + continue; + } + if token & KICK != 0 { + if let Some(kick) = control.kicks.lock().unwrap().get((token & !KICK) as usize) { + let _ = kick.read(); + } + continue; + } + let ready = event.events as i32; + let closed = ready & (libc::EPOLLHUP | libc::EPOLLERR) != 0; + state.mux.wake( + token as u32, + closed || ready & (libc::EPOLLIN | libc::EPOLLRDHUP) != 0, + closed || ready & libc::EPOLLOUT != 0, + ); + } + busy = false; + if let Some(mut transport) = state.transport.take() { + match transport.process(&mut state.mux) { + Ok(more) => { + state.transport = Some(transport); + busy = more; + } + Err(_) => { + state.failed = true; + state.mux.clear(); + } + } + } + } +} + +impl VirtioBackend for VsockBackend { + fn device_id(&self) -> u32 { + 19 + } + fn device_features(&self) -> u64 { + VIRTIO_F_VERSION_1 + } + fn queue_count(&self) -> usize { + 3 + } + fn queue_max_size(&self) -> u16 { + 256 + } + fn config_read(&self, offset: u64) -> u32 { + match offset { + 0 => GUEST_CID as u32, + 4 => (GUEST_CID >> 32) as u32, + _ => 0, + } + } + fn config_write(&mut self, _: u64, _: u32) {} + fn features_ok(&mut self, features: u64) { + self.negotiated = features & VIRTIO_F_VERSION_1 != 0; + } + fn activate( + &mut self, + queues: &[VirtioQueueState], + mem: &GuestMemoryMmap, + interrupt: Arc, + ) { + let mut state = self.control.state.lock().unwrap(); + let queues = queues + .iter() + .map(|queue| { + if !queue.ready { + return Err(invalid_queue()); + } + Queue::new( + queue.size, + queue.desc_addr, + queue.avail_addr, + queue.used_addr, + mem, + ) + }) + .collect::>>(); + match queues { + Ok(mut queues) if queues.len() == 3 && !state.failed && self.negotiated => { + queues.pop(); + let tx = queues.pop().unwrap(); + let rx = queues.pop().unwrap(); + state.transport = Some(Transport { + rx, + tx, + mem: mem.clone(), + interrupt, + }); + } + _ => { + state.failed = true; + } + } + self.control.notify(); + } + fn process_queue(&mut self, _: u16, _: &mut [VirtioQueueState], _: &GuestMemoryMmap) -> bool { + false + } + fn take_kicks(&mut self, kicks: &[EventFd]) -> io::Result { + let mut owned = self.control.kicks.lock().unwrap(); + for (index, kick) in kicks.iter().enumerate() { + let kick = kick.try_clone()?; + self.control + .register(kick.as_raw_fd(), KICK | index as u64, libc::EPOLLIN)?; + owned.push(kick); + } + Ok(true) + } + fn reset(&mut self) { + self.negotiated = false; + self.control.deactivate(); + } +} + +impl Drop for VsockBackend { + fn drop(&mut self) { + self.control.stopping.store(true, Ordering::Release); + self.control.notify(); + // Wakes the acceptors out of accept. + for listener in self.control.listeners.lock().unwrap().drain(..) { + unsafe { libc::shutdown(listener.as_raw_fd(), libc::SHUT_RDWR) }; + } + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + self.reset(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Read, Write}; + use vm_memory::Bytes; + + fn memory() -> GuestMemoryMmap { + GuestMemoryMmap::from_ranges(&[(GuestAddress(0), 1024 * 1024)]).unwrap() + } + + fn header(len: usize) -> Header { + Header { + len: len as u32, + op: connection::RW, + ..Header::default() + } + } + + #[test] + fn tx_payload_after_a_header_that_shares_its_descriptor() { + let mem = memory(); + let payload: Vec = (0..110).collect(); + mem.write_slice(&header(110).encode(), GuestAddress(0x1000)) + .unwrap(); + mem.write_slice(&payload[..10], GuestAddress(0x1000 + HEADER_LEN as u64)) + .unwrap(); + mem.write_slice(&payload[10..], GuestAddress(0x2000)) + .unwrap(); + let descriptors = [(0x1000, HEADER_LEN + 10), (0x2000, 100)]; + let (decoded, packet) = TxPacket::parse(&mem, &descriptors).unwrap(); + assert_eq!(decoded, header(110)); + assert_eq!(Payload::len(&packet), 110); + + let (mut host, device) = UnixStream::pair().unwrap(); + assert_eq!(packet.send(&device, 5).unwrap(), 105); + let mut sent = vec![0; 105]; + host.read_exact(&mut sent).unwrap(); + assert_eq!(sent, payload[5..]); + + // Start the copy at the wrap point of the deque. + let mut out = VecDeque::with_capacity(128); + out.extend([0xee; 100]); + out.drain(..90); + packet.copy(3, &mut out); + assert_eq!(out.len(), 10 + 107); + assert!(out.iter().take(10).all(|&b| b == 0xee)); + assert!(out + .iter() + .skip(10) + .copied() + .eq(payload[3..].iter().copied())); + } + + #[test] + fn rx_header_split_across_descriptors_and_one_read_for_several_buffers() { + let mem = memory(); + let first = [(0x4000, 20), (0x5000, HEADER_LEN - 20 + 100)]; + let second = [(0x6000, HEADER_LEN + 50)]; + let queue = Queue::new(8, 0x1000, 0x2000, 0x3000, &mem).unwrap(); + let mut batch = RxBatch::new(&mem, &queue); + batch.done = true; + for (head, descriptors) in [(0, &first[..]), (1, &second[..])] { + let (header, payload, room) = split(&mem, descriptors).unwrap(); + batch.chains.push(RxChain { + head, + header, + payload, + room, + len: 0, + }); + } + assert_eq!( + (batch.room(0), batch.room(1), batch.room(2)), + (Some(100), Some(50), None) + ); + + let (mut host, device) = UnixStream::pair().unwrap(); + let data: Vec = (0..130).collect(); + host.write_all(&data).unwrap(); + assert_eq!(batch.recv(&device, 0, 1000).unwrap(), 130); + batch.put_header(0, header(100)); + batch.put_header(1, header(30)); + assert_eq!((batch.chains[0].len, batch.chains[1].len), (144, 74)); + + let mut wire = [0; HEADER_LEN]; + mem.read_slice(&mut wire[..20], GuestAddress(0x4000)) + .unwrap(); + mem.read_slice(&mut wire[20..], GuestAddress(0x5000)) + .unwrap(); + assert_eq!(Header::decode(&wire), Some(header(100))); + let mut got = vec![0; 100]; + mem.read_slice(&mut got, GuestAddress(0x5000 + 24)).unwrap(); + assert_eq!(got, data[..100]); + let mut got = vec![0; 30]; + mem.read_slice(&mut got, GuestAddress(0x6000 + HEADER_LEN as u64)) + .unwrap(); + assert_eq!(got, data[100..]); + } + + #[test] + fn chains_shorter_than_a_header_are_rejected() { + let mem = memory(); + assert!(split(&mem, &[(0x1000, HEADER_LEN - 1)]).is_err()); + assert!(split(&mem, &[(0x1000, 10), (0x2000, 0), (0x3000, 33)]).is_err()); + assert_eq!(split(&mem, &[(0x1000, HEADER_LEN)]).unwrap().2, 0); + } + + // Posts one writable descriptor per head, then makes them available. + fn post(mem: &GuestMemoryMmap, buffers: &[(u64, u32)]) { + for (index, &(address, len)) in buffers.iter().enumerate() { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&address.to_le_bytes()); + bytes.extend_from_slice(&len.to_le_bytes()); + bytes.extend_from_slice(&2u16.to_le_bytes()); + bytes.extend_from_slice(&0u16.to_le_bytes()); + mem.write_slice(&bytes, GuestAddress(0x1000 + index as u64 * 16)) + .unwrap(); + mem.write_obj( + (index as u16).to_le(), + GuestAddress(0x2004 + index as u64 * 2), + ) + .unwrap(); + } + mem.write_obj((buffers.len() as u16).to_le(), GuestAddress(0x2002)) + .unwrap(); + } + + #[test] + fn buffers_are_read_from_the_ring_only_when_needed() { + let mem = memory(); + let queue = Queue::new(8, 0x1000, 0x2000, 0x3000, &mem).unwrap(); + post(&mem, &[(0x4000, 3776), (0x5000, 3776), (0x6000, 3776)]); + let mut batch = RxBatch::new(&mem, &queue); + assert_eq!(batch.room(0), Some(3776 - HEADER_LEN)); + assert_eq!(batch.chains.len(), 1); + assert_eq!(batch.room(2), Some(3776 - HEADER_LEN)); + assert_eq!(batch.room(3), None); + assert_eq!(batch.chains.len(), 3); + assert!(batch.invalid.is_none()); + } + + #[test] + fn a_malformed_buffer_at_the_front_is_set_aside() { + let mem = memory(); + let queue = Queue::new(8, 0x1000, 0x2000, 0x3000, &mem).unwrap(); + post(&mem, &[(0x4000, 10), (0x5000, 3776)]); + let mut batch = RxBatch::new(&mem, &queue); + assert_eq!(batch.room(0), None); + assert_eq!(batch.invalid, Some(0)); + + post(&mem, &[(0x4000, 3776), (0x5000, 10)]); + let mut batch = RxBatch::new(&mem, &queue); + assert!(batch.room(0).is_some()); + assert_eq!(batch.room(1), None); + assert!(batch.invalid.is_none()); + } + + fn connect_line(bytes: &[u8]) -> io::Result { + let (mut client, server) = UnixStream::pair().unwrap(); + client.write_all(bytes).unwrap(); + read_connect(&server) + } + + #[test] + fn connect_lines() { + assert_eq!(connect_line(b"CONNECT 1024\n").unwrap(), 1024); + assert_eq!(connect_line(b"CONNECT 4294967295\n").unwrap(), u32::MAX); + for bad in [ + &b"CONNECT 4294967296\n"[..], + b"CONNECT +1\n", + b"CONNECT \n", + b"connect 1024\n", + b"CONNECT 1024 \n", + b"CONNECT 1024", + b"CONNECT 000000000000000000000000001024\n", + ] { + assert!(connect_line(bad).is_err(), "{bad:?}"); + } + } + + #[test] + fn the_connect_line_leaves_later_bytes_for_the_guest() { + let (mut client, mut server) = UnixStream::pair().unwrap(); + client.write_all(b"CONNECT 7\nearly").unwrap(); + assert_eq!(read_connect(&server).unwrap(), 7); + let mut rest = [0; 5]; + server.read_exact(&mut rest).unwrap(); + assert_eq!(&rest, b"early"); + } + + #[test] + fn listener_clients_are_refused_before_the_driver_is_up_and_the_acceptor_stops_on_drop() { + let path = std::env::temp_dir().join(format!("shuru-vsock-test-{}", std::process::id())); + let _ = std::fs::remove_file(&path); + let listener = UnixListener::bind(&path).unwrap(); + let backend = VsockBackend::new(MAX_CONNECTIONS).unwrap(); + let control = backend.control(); + control.serve(listener).unwrap(); + let acceptor = Arc::downgrade(&control.listeners.lock().unwrap()[0]); + + let mut client = UnixStream::connect(&path).unwrap(); + client.write_all(b"CONNECT 1024\n").unwrap(); + let mut reply = Vec::new(); + assert_eq!(client.read_to_end(&mut reply).unwrap(), 0); + + // A client that never sends its line holds up nobody else. + let _silent = UnixStream::connect(&path).unwrap(); + let mut next = UnixStream::connect(&path).unwrap(); + let started = Instant::now(); + next.write_all(b"CONNECT 1024\n").unwrap(); + assert_eq!(next.read_to_end(&mut reply).unwrap(), 0); + assert!(started.elapsed() < CONNECT_TIMEOUT / 2); + std::fs::remove_file(&path).unwrap(); + + drop(control); + drop(backend); + let deadline = Instant::now() + Duration::from_secs(2); + while acceptor.upgrade().is_some() { + assert!(Instant::now() < deadline, "acceptor still running"); + std::thread::sleep(Duration::from_millis(10)); + } + } +} diff --git a/crates/shuru-linux/src/kvm/vsock/connection.rs b/crates/shuru-linux/src/kvm/vsock/connection.rs new file mode 100644 index 0000000..bb8658f --- /dev/null +++ b/crates/shuru-linux/src/kvm/vsock/connection.rs @@ -0,0 +1,1333 @@ +use std::collections::{BTreeMap, VecDeque}; +use std::io; +use std::net::Shutdown; +use std::ops::Bound; +use std::os::fd::AsRawFd; +use std::os::unix::net::UnixStream; +use std::sync::mpsc::SyncSender; +use std::time::{Duration, Instant}; + +pub const HEADER_LEN: usize = 44; +pub const BUFFER_SIZE: usize = 256 * 1024; +pub const MAX_PACKET: usize = 64 * 1024; +// One connection reads at most this much per turn before the next one goes. +const MAX_TURN: usize = 128 * 1024; +pub const MAX_CONNECTIONS: usize = 1024; +// Pending resets on top of the connection cap. +const MAX_PENDING_RESETS: usize = 256; +const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(2); +pub const HOST_CID: u64 = 2; +pub const GUEST_CID: u64 = 3; +pub const STREAM: u16 = 1; +pub const REQUEST: u16 = 1; +pub const RESPONSE: u16 = 2; +pub const RST: u16 = 3; +pub const SHUTDOWN: u16 = 4; +pub const RW: u16 = 5; +pub const CREDIT_UPDATE: u16 = 6; +pub const CREDIT_REQUEST: u16 = 7; +const RECEIVE: u32 = 1; +const SEND: u32 = 2; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Header { + pub src_cid: u64, + pub dst_cid: u64, + pub src_port: u32, + pub dst_port: u32, + pub len: u32, + pub kind: u16, + pub op: u16, + pub flags: u32, + pub buf_alloc: u32, + pub fwd_cnt: u32, +} + +impl Header { + pub fn decode(bytes: &[u8]) -> Option { + let bytes = bytes.get(..HEADER_LEN)?; + Some(Self { + src_cid: u64::from_le_bytes(bytes[0..8].try_into().ok()?), + dst_cid: u64::from_le_bytes(bytes[8..16].try_into().ok()?), + src_port: u32::from_le_bytes(bytes[16..20].try_into().ok()?), + dst_port: u32::from_le_bytes(bytes[20..24].try_into().ok()?), + len: u32::from_le_bytes(bytes[24..28].try_into().ok()?), + kind: u16::from_le_bytes(bytes[28..30].try_into().ok()?), + op: u16::from_le_bytes(bytes[30..32].try_into().ok()?), + flags: u32::from_le_bytes(bytes[32..36].try_into().ok()?), + buf_alloc: u32::from_le_bytes(bytes[36..40].try_into().ok()?), + fwd_cnt: u32::from_le_bytes(bytes[40..44].try_into().ok()?), + }) + } + + pub fn encode(self) -> [u8; HEADER_LEN] { + let mut bytes = [0; HEADER_LEN]; + bytes[0..8].copy_from_slice(&self.src_cid.to_le_bytes()); + bytes[8..16].copy_from_slice(&self.dst_cid.to_le_bytes()); + bytes[16..20].copy_from_slice(&self.src_port.to_le_bytes()); + bytes[20..24].copy_from_slice(&self.dst_port.to_le_bytes()); + bytes[24..28].copy_from_slice(&self.len.to_le_bytes()); + bytes[28..30].copy_from_slice(&self.kind.to_le_bytes()); + bytes[30..32].copy_from_slice(&self.op.to_le_bytes()); + bytes[32..36].copy_from_slice(&self.flags.to_le_bytes()); + bytes[36..40].copy_from_slice(&self.buf_alloc.to_le_bytes()); + bytes[40..44].copy_from_slice(&self.fwd_cnt.to_le_bytes()); + bytes + } +} + +#[derive(Default)] +struct Credit { + sent: u32, + peer_forwarded: u32, + peer_alloc: u32, +} + +impl Credit { + fn available(&self) -> u32 { + self.peer_alloc + .saturating_sub(self.sent.wrapping_sub(self.peer_forwarded)) + } + + fn update(&mut self, header: Header) -> bool { + // An acknowledgement cannot advance past the data already sent. + if header.fwd_cnt.wrapping_sub(self.peer_forwarded) + > self.sent.wrapping_sub(self.peer_forwarded) + { + return false; + } + self.peer_forwarded = header.fwd_cnt; + self.peer_alloc = header.buf_alloc; + true + } +} + +fn send(socket: &UnixStream, bytes: &[u8]) -> io::Result { + let n = unsafe { + libc::send( + socket.as_raw_fd(), + bytes.as_ptr().cast(), + bytes.len(), + libc::MSG_NOSIGNAL, + ) + }; + if n < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(n as usize) + } +} + +pub fn sendv(socket: &UnixStream, iov: &[libc::iovec]) -> io::Result { + let mut message: libc::msghdr = unsafe { std::mem::zeroed() }; + message.msg_iov = iov.as_ptr().cast_mut(); + message.msg_iovlen = iov.len() as _; + let n = unsafe { libc::sendmsg(socket.as_raw_fd(), &message, libc::MSG_NOSIGNAL) }; + if n < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(n as usize) + } +} + +pub fn readv(socket: &UnixStream, iov: &[libc::iovec]) -> io::Result { + let n = unsafe { libc::readv(socket.as_raw_fd(), iov.as_ptr(), iov.len() as _) }; + if n < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(n as usize) + } +} + +/// The guest rx buffers on offer in one pass, in queue order. Each holds at +/// least a header. +pub trait RxBuffers { + /// Payload bytes buffer `index` takes after its header, or `None` when + /// the guest has not posted that many. + fn room(&mut self, index: usize) -> Option; + fn put_header(&mut self, index: usize, header: Header); + /// Reads at most `max` bytes from `socket` straight into the payloads + /// of buffers `first..`, filling each to its room before the next. + fn recv(&mut self, socket: &UnixStream, first: usize, max: usize) -> io::Result; +} + +/// The payload of one guest tx packet. +pub trait Payload { + fn len(&self) -> usize; + fn is_empty(&self) -> bool { + self.len() == 0 + } + /// Sends the bytes from `offset` on to `socket` without blocking. + fn send(&self, socket: &UnixStream, offset: usize) -> io::Result; + /// Appends the bytes from `offset` on to `out`. + fn copy(&self, offset: usize, out: &mut VecDeque); +} + +impl Payload for [u8] { + fn len(&self) -> usize { + <[u8]>::len(self) + } + + fn send(&self, socket: &UnixStream, offset: usize) -> io::Result { + send(socket, &self[offset..]) + } + + fn copy(&self, offset: usize, out: &mut VecDeque) { + out.extend(&self[offset..]); + } +} + +/// Who learns that the guest accepted a host connect. +pub enum Opener { + /// A caller waiting on the channel. + Caller(SyncSender>), + /// A client of the unix listener, told with `OK \n` on its + /// socket before any guest data. + Listener, +} + +// Host to guest data stays in the host socket until the guest posts an rx +// buffer, so every packet is sized to the buffer that carries it. +pub struct Connection { + socket: UnixStream, + guest_port: u32, + host_port: u32, + opener: Option, + established: bool, + readable: bool, + writable: bool, + credit: Credit, + credit_requested: bool, + forwarded: u32, + advertised: u32, + received: u32, + incoming: VecDeque, + peer_shutdown: u32, + local_shutdown: u32, + send_request: bool, + send_shutdown: u32, + send_credit_request: bool, + send_credit_update: bool, + deadline: Option, +} + +impl Connection { + fn header(&self, op: u16, flags: u32, len: usize) -> Header { + Header { + src_cid: HOST_CID, + dst_cid: GUEST_CID, + src_port: self.host_port, + dst_port: self.guest_port, + len: len as u32, + kind: STREAM, + op, + flags, + buf_alloc: BUFFER_SIZE as u32, + fwd_cnt: self.forwarded, + } + } + + // Every header tells the guest how much we forwarded, so it doubles as + // a credit update. + fn stamp(&mut self, op: u16, flags: u32, len: usize) -> Header { + self.advertised = self.forwarded; + self.send_credit_update = false; + self.header(op, flags, len) + } + + fn control(&mut self, buffers: &mut dyn RxBuffers, index: usize, op: u16, flags: u32) -> usize { + buffers.put_header(index, self.stamp(op, flags, 0)); + 1 + } + + fn sends_data(&self) -> bool { + self.established && self.local_shutdown & SEND == 0 && self.peer_shutdown & RECEIVE == 0 + } + + fn receive(&mut self, header: Header, payload: &P) -> bool { + if !self.credit.update(header) { + return false; + } + if self.credit.available() > 0 { + self.credit_requested = false; + } + if !self.established { + if header.op != RESPONSE { + return false; + } + self.established = true; + self.deadline = None; + return match self.opener.take() { + Some(Opener::Caller(reply)) => reply.send(Ok(())).is_ok(), + // The socket is fresh, so a short write means the client is + // gone or misbehaving. + Some(Opener::Listener) => { + let greeting = format!("OK {}\n", self.host_port); + send(&self.socket, greeting.as_bytes()).is_ok_and(|n| n == greeting.len()) + } + None => false, + }; + } + match header.op { + RW => { + let window = BUFFER_SIZE + .saturating_sub(self.received.wrapping_sub(self.advertised) as usize); + if self.peer_shutdown & SEND != 0 + || self.local_shutdown & RECEIVE != 0 + || payload.len() > window + { + return false; + } + self.received = self.received.wrapping_add(payload.len() as u32); + let mut offset = 0; + // Straight from guest memory while nothing is queued ahead. + while self.writable && self.incoming.is_empty() && offset < payload.len() { + match payload.send(&self.socket, offset) { + Ok(0) => return false, + Ok(n) => { + offset += n; + self.forwarded = self.forwarded.wrapping_add(n as u32); + } + Err(e) if e.kind() == io::ErrorKind::WouldBlock => self.writable = false, + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(_) => return false, + } + } + if offset < payload.len() { + payload.copy(offset, &mut self.incoming); + } + } + CREDIT_UPDATE => {} + CREDIT_REQUEST => self.send_credit_update = true, + SHUTDOWN if header.flags != 0 && header.flags & !3 == 0 => { + self.peer_shutdown |= header.flags; + if header.flags & RECEIVE != 0 { + let _ = self.socket.shutdown(Shutdown::Read); + } + } + _ => return false, + } + true + } + + fn pump(&mut self) -> io::Result { + if self + .deadline + .is_some_and(|deadline| Instant::now() >= deadline) + { + return Ok(false); + } + if !self.established { + return Ok(true); + } + while self.writable && !self.incoming.is_empty() { + let (bytes, _) = self.incoming.as_slices(); + match send(&self.socket, bytes) { + Ok(0) => return Err(io::ErrorKind::WriteZero.into()), + Ok(n) => { + self.incoming.drain(..n); + self.forwarded = self.forwarded.wrapping_add(n as u32); + } + Err(e) if e.kind() == io::ErrorKind::WouldBlock => self.writable = false, + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + if self.incoming.is_empty() + && self.peer_shutdown & SEND != 0 + && self.local_shutdown & RECEIVE == 0 + { + self.socket.shutdown(Shutdown::Write)?; + self.local_shutdown |= RECEIVE; + } + if self.peer_shutdown == 3 && self.incoming.is_empty() { + return Ok(false); + } + // The guest stalls when its view of our free space runs out, so + // advertise forwarded bytes before that view drops below one packet. + let peer_view = + BUFFER_SIZE.saturating_sub(self.received.wrapping_sub(self.advertised) as usize); + if self.forwarded != self.advertised && peer_view < MAX_PACKET { + self.send_credit_update = true; + } + if self.local_shutdown == 3 && self.deadline.is_none() { + self.send_shutdown = 3; + self.deadline = Some(Instant::now() + HANDSHAKE_TIMEOUT); + } + Ok(true) + } + + // Returns how many buffers from `first` on now hold a packet. + fn fill(&mut self, buffers: &mut dyn RxBuffers, first: usize) -> io::Result { + if !self.established { + if !std::mem::take(&mut self.send_request) { + return Ok(0); + } + return Ok(self.control(buffers, first, REQUEST, 0)); + } + if self.send_shutdown != 0 { + let flags = std::mem::take(&mut self.send_shutdown); + return Ok(self.control(buffers, first, SHUTDOWN, flags)); + } + let used = self.fill_data(buffers, first)?; + if used > 0 { + return Ok(used); + } + if std::mem::take(&mut self.send_credit_request) { + return Ok(self.control(buffers, first, CREDIT_REQUEST, 0)); + } + if self.send_credit_update { + return Ok(self.control(buffers, first, CREDIT_UPDATE, 0)); + } + Ok(0) + } + + fn fill_data(&mut self, buffers: &mut dyn RxBuffers, first: usize) -> io::Result { + if !self.readable || !self.sends_data() || buffers.room(first).unwrap_or(0) == 0 { + return Ok(0); + } + let credit = self.credit.available() as usize; + if credit == 0 { + if !self.credit_requested { + self.credit_requested = true; + self.send_credit_request = true; + } + return Ok(0); + } + match buffers.recv(&self.socket, first, credit.min(MAX_TURN)) { + Ok(0) => { + self.readable = false; + self.local_shutdown |= SEND; + Ok(self.control(buffers, first, SHUTDOWN, SEND)) + } + Ok(n) => { + self.credit.sent = self.credit.sent.wrapping_add(n as u32); + let mut index = first; + let mut left = n; + while left > 0 { + let len = buffers.room(index).unwrap_or(0).min(left); + if len == 0 { + break; + } + let header = self.stamp(RW, 0, len); + buffers.put_header(index, header); + left -= len; + index += 1; + } + Ok(index - first) + } + Err(e) if e.kind() == io::ErrorKind::WouldBlock => { + self.readable = false; + Ok(0) + } + Err(e) if e.kind() == io::ErrorKind::Interrupted => Ok(0), + Err(e) => Err(e), + } + } +} + +impl Drop for Connection { + fn drop(&mut self) { + if let Some(Opener::Caller(reply)) = self.opener.take() { + let _ = reply.send(Err(io::Error::new( + io::ErrorKind::ConnectionRefused, + "guest connection reset", + ))); + } + let _ = self.socket.shutdown(Shutdown::Both); + } +} + +pub struct Mux { + connections: BTreeMap, + max_connections: usize, + resets: VecDeque
, + next_port: u32, + last_port: u32, +} + +impl Mux { + #[cfg(test)] + pub fn new() -> Self { + Self::with_limit(MAX_CONNECTIONS) + } + + pub fn with_limit(max_connections: usize) -> Self { + Self { + connections: BTreeMap::new(), + max_connections, + resets: VecDeque::new(), + next_port: 1024, + last_port: 0, + } + } + + pub fn clear(&mut self) { + self.connections.clear(); + self.resets.clear(); + self.last_port = 0; + } + + pub fn add(&mut self, socket: UnixStream, guest_port: u32, opener: Opener) -> io::Result { + if self.connections.len() >= self.max_connections + || self.connections.len() + self.resets.len() >= self.max_resets() + { + return Err(io::Error::new( + io::ErrorKind::OutOfMemory, + "vsock connection limit reached", + )); + } + // Never reuse a tuple during this device lifetime. + let port = self + .next_port + .checked_add(1) + .ok_or_else(|| io::Error::other("vsock ports exhausted"))?; + self.next_port = port; + socket.set_nonblocking(true)?; + self.connections.insert( + port, + Connection { + socket, + host_port: port, + guest_port, + opener: Some(opener), + established: false, + readable: true, + writable: true, + credit: Credit::default(), + credit_requested: false, + forwarded: 0, + advertised: 0, + received: 0, + incoming: VecDeque::new(), + peer_shutdown: 0, + local_shutdown: 0, + send_request: true, + send_shutdown: 0, + send_credit_request: false, + send_credit_update: false, + deadline: Some(Instant::now() + HANDSHAKE_TIMEOUT), + }, + ); + Ok(port) + } + + fn max_resets(&self) -> usize { + self.max_connections + MAX_PENDING_RESETS + } + + pub fn can_receive(&self) -> bool { + self.resets.len() + self.connections.len() < self.max_resets() + } + + fn reset(&mut self, header: Header) { + if self.resets.len() + self.connections.len() < self.max_resets() { + self.resets.push_back(header); + } + } + + fn reset_unknown(&mut self, header: Header) { + self.reset(Header { + src_cid: HOST_CID, + dst_cid: GUEST_CID, + src_port: header.dst_port, + dst_port: header.src_port, + kind: header.kind, + op: RST, + ..Header::default() + }); + } + + pub fn cancel(&mut self, port: u32) { + if let Some(connection) = self.connections.remove(&port) { + self.reset(connection.header(RST, 0, 0)); + } + } + + pub fn wake(&mut self, port: u32, readable: bool, writable: bool) { + if let Some(connection) = self.connections.get_mut(&port) { + connection.readable |= readable; + connection.writable |= writable; + } + } + + pub fn receive(&mut self, header: Header, payload: &P) { + if header.src_cid != GUEST_CID || header.dst_cid != HOST_CID { + return; + } + let valid = header.kind == STREAM + && header.len as usize == payload.len() + && (header.op == RW || payload.is_empty()) + && (header.op == SHUTDOWN || header.flags == 0); + let Some(mut connection) = self.connections.remove(&header.dst_port) else { + if header.op != RST { + self.reset_unknown(header); + } + return; + }; + if connection.guest_port != header.src_port { + self.connections.insert(header.dst_port, connection); + if header.op != RST { + self.reset_unknown(header); + } + } else if header.op == RST { + // Dropping the stream wakes both pending connects and active readers. + } else if valid && connection.receive(header, payload) { + self.connections.insert(header.dst_port, connection); + } else { + self.reset(connection.header(RST, 0, 0)); + } + } + + pub fn pump(&mut self) { + let mut closed = Vec::new(); + for (port, connection) in &mut self.connections { + if !connection.pump().unwrap_or(false) { + closed.push(*port); + } + } + for port in closed { + self.cancel(port); + } + } + + /// When the earliest handshake or close times out. + pub fn next_deadline(&self) -> Option { + self.connections.values().filter_map(|c| c.deadline).min() + } + + // Puts packets into the guest buffers on offer and returns how many it + // used, all from the front. Connections take turns so a bulk transfer + // cannot starve the others. + pub fn fill(&mut self, buffers: &mut dyn RxBuffers) -> usize { + let mut next = 0; + while buffers.room(next).is_some() { + if let Some(header) = self.resets.pop_front() { + buffers.put_header(next, header); + next += 1; + continue; + } + let used = self.fill_turn(buffers, next); + if used == 0 && self.resets.is_empty() { + break; + } + next += used; + } + next + } + + fn fill_turn(&mut self, buffers: &mut dyn RxBuffers, first: usize) -> usize { + let mut port = self.last_port; + for _ in 0..self.connections.len() { + let Some(&next) = self + .connections + .range((Bound::Excluded(port), Bound::Unbounded)) + .next() + .or_else(|| self.connections.first_key_value()) + .map(|(port, _)| port) + else { + return 0; + }; + port = next; + match self + .connections + .get_mut(&port) + .unwrap() + .fill(buffers, first) + { + Ok(0) => {} + Ok(used) => { + self.last_port = port; + return used; + } + Err(_) => { + self.cancel(port); + return 0; + } + } + } + 0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Read, Write}; + use std::sync::mpsc; + + // Linux 6.12.114 posts rx buffers of one page minus skb overhead. + const RX_BUFFER: usize = 3776; + + fn pending(mux: &mut Mux) -> (u32, UnixStream, mpsc::Receiver>) { + let (host, device) = UnixStream::pair().unwrap(); + host.set_nonblocking(true).unwrap(); + let (tx, rx) = mpsc::sync_channel(1); + let port = mux.add(device, 1024, Opener::Caller(tx)).unwrap(); + (port, host, rx) + } + + fn guest(port: u32, op: u16) -> Header { + Header { + src_cid: GUEST_CID, + dst_cid: HOST_CID, + src_port: 1024, + dst_port: port, + kind: STREAM, + op, + buf_alloc: BUFFER_SIZE as u32, + ..Header::default() + } + } + + struct Buffers { + data: Vec>, + lens: Vec, + } + + impl Buffers { + fn new(count: usize, size: usize) -> Self { + Self { + data: vec![vec![0; size]; count], + lens: vec![0; count], + } + } + + fn packets(&self, used: usize) -> Vec<(Header, Vec)> { + (0..used) + .map(|i| { + let header = Header::decode(&self.data[i]).unwrap(); + assert_eq!(header.len as usize, self.lens[i] - HEADER_LEN); + (header, self.data[i][HEADER_LEN..self.lens[i]].to_vec()) + }) + .collect() + } + } + + impl RxBuffers for Buffers { + fn room(&mut self, index: usize) -> Option { + let buffer = self.data.get(index)?; + Some((buffer.len() - HEADER_LEN).min(MAX_PACKET)) + } + + fn put_header(&mut self, index: usize, header: Header) { + self.data[index][..HEADER_LEN].copy_from_slice(&header.encode()); + self.lens[index] = HEADER_LEN + header.len as usize; + } + + fn recv(&mut self, socket: &UnixStream, first: usize, max: usize) -> io::Result { + let mut iov = Vec::new(); + let mut left = max; + for index in first.. { + let room = self.room(index).unwrap_or(0).min(left); + if room == 0 { + break; + } + iov.push(libc::iovec { + iov_base: self.data[index][HEADER_LEN..].as_mut_ptr().cast(), + iov_len: room, + }); + left -= room; + } + readv(socket, &iov) + } + } + + fn batch(mux: &mut Mux, count: usize, size: usize) -> Vec<(Header, Vec)> { + let mut buffers = Buffers::new(count, size); + let used = mux.fill(&mut buffers); + buffers.packets(used) + } + + fn next(mux: &mut Mux, size: usize) -> Option<(Header, Vec)> { + batch(mux, 1, size).pop() + } + + fn packet(mux: &mut Mux) -> Option<(Header, Vec)> { + next(mux, RX_BUFFER) + } + + fn established(mux: &mut Mux) -> (u32, UnixStream) { + let (port, host, reply) = pending(mux); + assert_eq!(packet(mux).unwrap().0.op, REQUEST); + mux.receive(guest(port, RESPONSE), &[][..]); + reply.recv().unwrap().unwrap(); + (port, host) + } + + fn guest_data(mux: &mut Mux, port: u32, data: &[u8]) { + mux.receive( + Header { + len: data.len() as u32, + ..guest(port, RW) + }, + data, + ); + } + + #[test] + fn header_wire_layout() { + let header = Header { + src_cid: 0x0807060504030201, + dst_cid: 0x100f0e0d0c0b0a09, + src_port: 0x14131211, + dst_port: 0x18171615, + len: 0x1c1b1a19, + kind: 0x1e1d, + op: 0x201f, + flags: 0x24232221, + buf_alloc: 0x28272625, + fwd_cnt: 0x2c2b2a29, + }; + let wire: Vec = (1..=44).collect(); + assert_eq!(header.encode().as_slice(), wire); + assert_eq!(Header::decode(&wire), Some(header)); + for len in 0..HEADER_LEN { + assert!(Header::decode(&wire[..len]).is_none()); + } + } + + #[test] + fn connect_waits_for_response_and_reset_refuses() { + let mut mux = Mux::new(); + let (port, _host, reply) = pending(&mut mux); + assert!(reply.try_recv().is_err()); + let (request, _) = packet(&mut mux).unwrap(); + assert_eq!(request.src_port, port); + assert_eq!(request.op, REQUEST); + assert!(packet(&mut mux).is_none()); + mux.receive(guest(port, RST), &[][..]); + assert!(reply.recv().unwrap().is_err()); + assert!(mux.connections.is_empty()); + assert!(packet(&mut mux).is_none()); + } + + #[test] + fn guest_requests_and_unknown_types_are_reset() { + let mut mux = Mux::new(); + for kind in [STREAM, 99] { + let mut header = guest(5000, REQUEST); + header.kind = kind; + mux.receive(header, &[][..]); + let (reset, _) = packet(&mut mux).unwrap(); + assert_eq!(reset.op, RST); + assert_eq!(reset.kind, kind); + assert_eq!(reset.src_port, 5000); + } + mux.receive(guest(5000, RST), &[][..]); + assert!(packet(&mut mux).is_none()); + } + + #[test] + fn independent_muxes_with_same_cid_and_ports() { + let mut first = Mux::new(); + let mut second = Mux::new(); + let (a, mut host_a) = established(&mut first); + let (b, mut host_b) = established(&mut second); + assert_eq!(a, b); + host_a.write_all(b"first").unwrap(); + host_b.write_all(b"second").unwrap(); + assert_eq!(packet(&mut first).unwrap().1, b"first"); + assert_eq!(packet(&mut second).unwrap().1, b"second"); + } + + #[test] + fn packets_are_sized_to_the_rx_buffer() { + let mut mux = Mux::new(); + let (port, host) = established(&mut mux); + let data: Vec = (0..1_000_000u32).map(|i| (i % 251) as u8).collect(); + host.set_nonblocking(false).unwrap(); + let writer = { + let data = data.clone(); + let mut host = host.try_clone().unwrap(); + std::thread::spawn(move || host.write_all(&data).unwrap()) + }; + let mut received = Vec::new(); + while received.len() < data.len() { + mux.receive( + Header { + fwd_cnt: received.len() as u32, + ..guest(port, CREDIT_UPDATE) + }, + &[][..], + ); + mux.wake(port, true, false); + while let Some((header, payload)) = packet(&mut mux) { + if header.op == RW { + assert!(HEADER_LEN + payload.len() <= RX_BUFFER); + received.extend(payload); + } + } + } + writer.join().unwrap(); + assert_eq!(received, data); + drop(host); + mux.wake(port, true, false); + let (shutdown, _) = packet(&mut mux).unwrap(); + assert_eq!((shutdown.op, shutdown.flags), (SHUTDOWN, SEND)); + } + + #[test] + fn large_buffers_carry_at_most_one_packet_of_data() { + let mut mux = Mux::new(); + let (port, host) = established(&mut mux); + host.set_nonblocking(false).unwrap(); + let writer = { + let mut host = host.try_clone().unwrap(); + std::thread::spawn(move || host.write_all(&vec![7; 3 * MAX_PACKET]).unwrap()) + }; + let mut total = 0; + while total < 3 * MAX_PACKET { + mux.wake(port, true, false); + if let Some((header, payload)) = next(&mut mux, HEADER_LEN + BUFFER_SIZE) { + assert_eq!(header.op, RW); + assert!(payload.len() <= MAX_PACKET); + total += payload.len(); + } + } + writer.join().unwrap(); + } + + #[test] + fn buffers_too_small_for_data_carry_control_only() { + let mut mux = Mux::new(); + let (_port, _host, _reply) = pending(&mut mux); + assert_eq!(next(&mut mux, HEADER_LEN).unwrap().0.op, REQUEST); + let mut mux = Mux::new(); + let (_port, mut host) = established(&mut mux); + host.write_all(b"data").unwrap(); + assert!(next(&mut mux, HEADER_LEN).is_none()); + assert_eq!(packet(&mut mux).unwrap().1, b"data"); + } + + #[test] + fn host_data_stops_at_credit_and_resumes_on_update() { + let mut mux = Mux::new(); + let (port, mut host) = established(&mut mux); + mux.receive( + Header { + buf_alloc: 3, + ..guest(port, CREDIT_UPDATE) + }, + &[][..], + ); + host.write_all(b"abcdef").unwrap(); + assert_eq!(packet(&mut mux).unwrap().1, b"abc"); + assert_eq!(packet(&mut mux).unwrap().0.op, CREDIT_REQUEST); + assert!(packet(&mut mux).is_none()); + mux.receive( + Header { + buf_alloc: 3, + fwd_cnt: 3, + ..guest(port, CREDIT_UPDATE) + }, + &[][..], + ); + assert_eq!(packet(&mut mux).unwrap().1, b"def"); + } + + #[test] + fn credit_wraparound_and_invalid_ack() { + let mut credit = Credit { + sent: 2, + peer_forwarded: u32::MAX - 1, + peer_alloc: 8, + }; + assert_eq!(credit.available(), 4); + assert!(credit.update(Header { + fwd_cnt: 1, + buf_alloc: 8, + ..Header::default() + })); + assert_eq!(credit.available(), 7); + assert!(!credit.update(Header { + fwd_cnt: 3, + buf_alloc: 8, + ..Header::default() + })); + credit.peer_alloc = 0; + assert_eq!(credit.available(), 0); + } + + #[test] + fn credit_update_is_sent_before_the_guest_runs_out() { + let mut mux = Mux::new(); + let (port, mut host) = established(&mut mux); + let chunk = vec![1; MAX_PACKET]; + for _ in 0..3 { + guest_data(&mut mux, port, &chunk); + } + mux.pump(); + let mut sink = vec![0; 3 * MAX_PACKET]; + host.read_exact(&mut sink).unwrap(); + assert!(packet(&mut mux).is_none()); + guest_data(&mut mux, port, &chunk); + mux.pump(); + let (update, _) = packet(&mut mux).unwrap(); + assert_eq!(update.op, CREDIT_UPDATE); + assert_eq!(update.fwd_cnt as usize, 4 * MAX_PACKET); + assert!(packet(&mut mux).is_none()); + } + + #[test] + fn receive_window_rejects_overrun() { + let mut mux = Mux::new(); + let (port, _host) = established(&mut mux); + mux.connections.get_mut(&port).unwrap().writable = false; + guest_data(&mut mux, port, &vec![1; BUFFER_SIZE]); + assert_eq!(mux.connections[&port].incoming.len(), BUFFER_SIZE); + guest_data(&mut mux, port, &[0]); + assert!(mux.connections.is_empty()); + assert_eq!(packet(&mut mux).unwrap().0.op, RST); + } + + #[test] + fn writes_wait_for_the_host_socket() { + let mut mux = Mux::new(); + let (port, mut host) = established(&mut mux); + let chunk = vec![2; 4096]; + while mux.connections[&port].writable { + let connection = &mux.connections[&port]; + let used = connection.received.wrapping_sub(connection.advertised) as usize; + if BUFFER_SIZE - used >= chunk.len() { + guest_data(&mut mux, port, &chunk); + } + mux.pump(); + while packet(&mut mux).is_some() {} + } + let stuck = mux.connections[&port].incoming.len(); + assert!(stuck > 0); + let mut sink = vec![0; BUFFER_SIZE]; + while host.read(&mut sink).is_ok() {} + mux.pump(); + assert_eq!(mux.connections[&port].incoming.len(), stuck); + mux.wake(port, false, true); + mux.pump(); + assert!(mux.connections[&port].incoming.len() < stuck); + } + + #[test] + fn repeated_credit_requests_get_one_update() { + let mut mux = Mux::new(); + let (port, _host) = established(&mut mux); + for _ in 0..1000 { + mux.receive(guest(port, CREDIT_REQUEST), &[][..]); + } + assert_eq!(packet(&mut mux).unwrap().0.op, CREDIT_UPDATE); + assert!(packet(&mut mux).is_none()); + } + + #[test] + fn resets_are_bounded() { + let mut mux = Mux::new(); + while mux.can_receive() { + mux.receive(guest(9999, REQUEST), &[][..]); + } + assert_eq!(mux.resets.len(), mux.max_resets()); + for _ in 0..100 { + mux.receive(guest(9999, REQUEST), &[][..]); + } + assert_eq!(mux.resets.len(), mux.max_resets()); + } + + #[test] + fn guest_half_close_drains_data_and_keeps_host_send_open() { + let mut mux = Mux::new(); + let (port, mut host) = established(&mut mux); + guest_data(&mut mux, port, b"end"); + mux.receive( + Header { + flags: SEND, + ..guest(port, SHUTDOWN) + }, + &[][..], + ); + mux.pump(); + let mut data = [0; 3]; + host.read_exact(&mut data).unwrap(); + assert_eq!(&data, b"end"); + assert_eq!(host.read(&mut data).unwrap(), 0); + host.write_all(b"reply").unwrap(); + mux.pump(); + let packets: Vec<_> = std::iter::from_fn(|| packet(&mut mux)).collect(); + assert!(packets.iter().any(|(_, payload)| payload == b"reply")); + assert!(mux.connections.contains_key(&port)); + } + + #[test] + fn host_close_delivers_pending_data_before_shutdown() { + let mut mux = Mux::new(); + let (_port, mut host) = established(&mut mux); + host.write_all(b"last words").unwrap(); + drop(host); + let (data, payload) = packet(&mut mux).unwrap(); + assert_eq!((data.op, payload.as_slice()), (RW, &b"last words"[..])); + let (shutdown, _) = packet(&mut mux).unwrap(); + assert_eq!((shutdown.op, shutdown.flags), (SHUTDOWN, SEND)); + } + + #[test] + fn host_half_close_and_guest_full_close() { + let mut mux = Mux::new(); + let (port, host) = established(&mut mux); + host.shutdown(Shutdown::Write).unwrap(); + let (shutdown, _) = packet(&mut mux).unwrap(); + assert_eq!((shutdown.op, shutdown.flags), (SHUTDOWN, SEND)); + mux.receive( + Header { + flags: 3, + ..guest(port, SHUTDOWN) + }, + &[][..], + ); + mux.pump(); + assert_eq!(packet(&mut mux).unwrap().0.op, RST); + assert!(mux.connections.is_empty()); + } + + #[test] + fn matching_half_closes_keep_the_reverse_direction_open() { + let mut mux = Mux::new(); + let (port, mut host) = established(&mut mux); + host.shutdown(Shutdown::Write).unwrap(); + packet(&mut mux).unwrap(); + mux.receive( + Header { + flags: RECEIVE, + ..guest(port, SHUTDOWN) + }, + &[][..], + ); + mux.pump(); + assert!(mux.connections[&port].deadline.is_none()); + assert!(packet(&mut mux).is_none()); + guest_data(&mut mux, port, b"reply"); + mux.pump(); + let mut bytes = [0; 5]; + host.read_exact(&mut bytes).unwrap(); + assert_eq!(&bytes, b"reply"); + } + + #[test] + fn full_close_sends_shutdown_and_waits_for_the_guest() { + let mut mux = Mux::new(); + let (port, host) = established(&mut mux); + mux.receive( + Header { + flags: SEND, + ..guest(port, SHUTDOWN) + }, + &[][..], + ); + mux.pump(); + host.shutdown(Shutdown::Write).unwrap(); + let (first, _) = packet(&mut mux).unwrap(); + assert_eq!((first.op, first.flags), (SHUTDOWN, SEND)); + mux.pump(); + let (second, _) = packet(&mut mux).unwrap(); + assert_eq!((second.op, second.flags), (SHUTDOWN, 3)); + assert!(mux.connections[&port].deadline.is_some()); + mux.connections.get_mut(&port).unwrap().deadline = Some(Instant::now()); + mux.pump(); + assert_eq!(packet(&mut mux).unwrap().0.op, RST); + } + + #[test] + fn malformed_packets_and_spoofed_tuples() { + let mut mux = Mux::new(); + let (port, _host) = established(&mut mux); + mux.receive( + Header { + src_cid: 99, + ..guest(port, RST) + }, + &[][..], + ); + mux.receive( + Header { + src_port: 99, + ..guest(port, RST) + }, + &[][..], + ); + assert!(mux.connections.contains_key(&port)); + mux.receive( + Header { + len: 100, + ..guest(port, RW) + }, + &[0][..], + ); + assert!(!mux.connections.contains_key(&port)); + assert_eq!(packet(&mut mux).unwrap().0.op, RST); + } + + #[test] + fn timeout_cancellation_and_reset_release_streams() { + let mut mux = Mux::new(); + let (port, mut host, reply) = pending(&mut mux); + mux.connections.get_mut(&port).unwrap().deadline = Some(Instant::now()); + mux.pump(); + assert!(reply.recv().unwrap().is_err()); + assert_eq!(host.read(&mut [0]).unwrap(), 0); + assert_eq!(packet(&mut mux).unwrap().0.op, RST); + let (port, mut host) = established(&mut mux); + mux.cancel(port); + assert_eq!(host.read(&mut [0]).unwrap(), 0); + } + + #[test] + fn guest_reset_drops_unsent_host_data() { + let mut mux = Mux::new(); + let (port, mut host) = established(&mut mux); + host.write_all(b"never delivered").unwrap(); + mux.receive(guest(port, RST), &[][..]); + assert!(packet(&mut mux).is_none()); + assert!(host.write_all(b"more").is_err()); + } + + #[test] + fn device_reset_does_not_reuse_ports_with_pending_connects() { + let mut mux = Mux::new(); + let (old_port, _old_host, old_reply) = pending(&mut mux); + mux.clear(); + assert!(old_reply.recv().unwrap().is_err()); + let (new_port, _new_host, _new_reply) = pending(&mut mux); + assert_ne!(old_port, new_port); + mux.cancel(old_port); + assert!(mux.connections.contains_key(&new_port)); + } + + #[test] + fn one_read_fills_consecutive_buffers() { + let mut mux = Mux::new(); + let (port, mut host) = established(&mut mux); + let data: Vec = (0..5 * RX_BUFFER as u32).map(|i| (i % 251) as u8).collect(); + host.write_all(&data).unwrap(); + let packets = batch(&mut mux, 8, RX_BUFFER); + assert!(packets.len() > 1); + let mut received: Vec = Vec::new(); + for (header, payload) in &packets { + assert_eq!((header.op, header.src_port), (RW, port)); + assert!(HEADER_LEN + payload.len() <= RX_BUFFER); + received.extend(payload); + } + assert_eq!(received.len(), MAX_TURN.min(data.len())); + assert_eq!(received, data[..received.len()]); + assert_eq!(packets.last().unwrap().0.fwd_cnt, 0); + } + + #[test] + fn a_turn_stops_at_the_guest_credit() { + let mut mux = Mux::new(); + let (port, mut host) = established(&mut mux); + mux.receive( + Header { + buf_alloc: 5000, + ..guest(port, CREDIT_UPDATE) + }, + &[][..], + ); + host.write_all(&vec![3; 3 * RX_BUFFER]).unwrap(); + let packets = batch(&mut mux, 8, RX_BUFFER); + let data: Vec = packets + .iter() + .filter(|(header, _)| header.op == RW) + .map(|(_, payload)| payload.len()) + .collect(); + assert_eq!( + data, + [RX_BUFFER - HEADER_LEN, 5000 - (RX_BUFFER - HEADER_LEN)] + ); + assert_eq!(packets.last().unwrap().0.op, CREDIT_REQUEST); + } + + #[test] + fn guest_data_goes_straight_to_a_writable_socket() { + let mut mux = Mux::new(); + let (port, mut host) = established(&mut mux); + guest_data(&mut mux, port, b"direct"); + assert!(mux.connections[&port].incoming.is_empty()); + assert_eq!(mux.connections[&port].forwarded, 6); + let mut bytes = [0; 6]; + host.read_exact(&mut bytes).unwrap(); + assert_eq!(&bytes, b"direct"); + } + + #[test] + fn connections_take_turns_within_a_batch() { + let mut mux = Mux::new(); + let (a, mut host_a) = established(&mut mux); + let (b, mut host_b) = established(&mut mux); + host_a.write_all(&vec![1; MAX_TURN + 1000]).unwrap(); + host_b.write_all(&[2; 1000]).unwrap(); + let turn = MAX_TURN.div_ceil(RX_BUFFER - HEADER_LEN); + let order: Vec = batch(&mut mux, turn + 2, RX_BUFFER) + .iter() + .map(|(header, _)| header.src_port) + .collect(); + let mut expected = vec![a; turn]; + expected.extend([b, a]); + assert_eq!(order, expected); + } + + #[test] + fn connections_take_turns() { + let mut mux = Mux::new(); + let (a, mut host_a) = established(&mut mux); + let (b, mut host_b) = established(&mut mux); + host_a.write_all(&vec![1; 4 * RX_BUFFER]).unwrap(); + host_b.write_all(&vec![2; 4 * RX_BUFFER]).unwrap(); + let order: Vec = (0..6) + .map(|_| packet(&mut mux).unwrap().0.src_port) + .collect(); + assert_eq!(order, [a, b, a, b, a, b]); + } + + #[test] + fn a_listener_client_hears_ok_before_guest_data() { + let mut mux = Mux::new(); + let (mut host, device) = UnixStream::pair().unwrap(); + let port = mux.add(device, 1024, Opener::Listener).unwrap(); + assert_eq!(packet(&mut mux).unwrap().0.op, REQUEST); + mux.receive(guest(port, RESPONSE), &[][..]); + guest_data(&mut mux, port, b"hello"); + let greeting = format!("OK {port}\n"); + let mut bytes = vec![0; greeting.len() + 5]; + host.read_exact(&mut bytes).unwrap(); + assert_eq!(bytes, [greeting.as_bytes(), b"hello"].concat()); + } + + #[test] + fn a_refused_listener_client_sees_the_socket_close() { + let mut mux = Mux::new(); + let (mut host, device) = UnixStream::pair().unwrap(); + let port = mux.add(device, 1024, Opener::Listener).unwrap(); + mux.receive(guest(port, RST), &[][..]); + assert!(mux.connections.is_empty()); + let mut bytes = Vec::new(); + assert_eq!(host.read_to_end(&mut bytes).unwrap(), 0); + } + + #[test] + fn a_listener_client_that_left_is_reset() { + let mut mux = Mux::new(); + let (host, device) = UnixStream::pair().unwrap(); + let port = mux.add(device, 1024, Opener::Listener).unwrap(); + assert_eq!(packet(&mut mux).unwrap().0.op, REQUEST); + drop(host); + mux.receive(guest(port, RESPONSE), &[][..]); + assert!(mux.connections.is_empty()); + assert_eq!(packet(&mut mux).unwrap().0.op, RST); + } + + #[test] + fn connections_past_the_limit_are_refused() { + let mut mux = Mux::with_limit(2); + let (_a, _host_a, _reply_a) = pending(&mut mux); + let (_b, _host_b, _reply_b) = pending(&mut mux); + let (_host, device) = UnixStream::pair().unwrap(); + let error = mux.add(device, 1024, Opener::Listener).unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::OutOfMemory); + } +} diff --git a/crates/shuru-linux/src/kvm/vsock/queue.rs b/crates/shuru-linux/src/kvm/vsock/queue.rs new file mode 100644 index 0000000..b03a8fb --- /dev/null +++ b/crates/shuru-linux/src/kvm/vsock/queue.rs @@ -0,0 +1,316 @@ +use std::io; +use std::sync::atomic::{fence, Ordering}; + +use vm_memory::{Bytes, GuestAddress, GuestMemory, GuestMemoryMmap}; + +use super::connection::{BUFFER_SIZE, HEADER_LEN}; + +const VRING_USED_F_NO_NOTIFY: u16 = 1; +const VRING_AVAIL_F_NO_INTERRUPT: u16 = 1; + +pub(super) fn invalid_queue() -> io::Error { + io::Error::other("invalid vsock virtqueue") +} + +pub(super) struct Queue { + size: u16, + desc_addr: u64, + avail_addr: u64, + used_addr: u64, + next_avail: u16, + used: u16, +} + +impl Queue { + pub(super) fn new( + size: u16, + desc_addr: u64, + avail_addr: u64, + used_addr: u64, + mem: &GuestMemoryMmap, + ) -> io::Result { + let count = size as usize; + if count == 0 + || count > 256 + || !count.is_power_of_two() + || !desc_addr.is_multiple_of(16) + || !avail_addr.is_multiple_of(2) + || !used_addr.is_multiple_of(4) + { + return Err(invalid_queue()); + } + for (address, len) in [ + (desc_addr, count * 16), + (avail_addr, 6 + count * 2), + (used_addr, 6 + count * 8), + ] { + if address.checked_add(len as u64).is_none() + || !mem.check_range(GuestAddress(address), len) + { + return Err(invalid_queue()); + } + } + Ok(Self { + size, + desc_addr, + avail_addr, + used_addr, + next_avail: 0, + used: 0, + }) + } + + pub(super) fn head(&self, mem: &GuestMemoryMmap) -> io::Result> { + self.head_at(mem, 0) + } + + /// The head `ahead` places after the next one, if the driver posted it. + pub(super) fn head_at(&self, mem: &GuestMemoryMmap, ahead: usize) -> io::Result> { + let q = self; + let available = u16::from_le( + mem.read_obj::(GuestAddress(q.avail_addr + 2)) + .map_err(|_| invalid_queue())?, + ); + fence(Ordering::Acquire); + let count = available.wrapping_sub(q.next_avail); + if count > q.size { + return Err(invalid_queue()); + } + if ahead >= count as usize { + return Ok(None); + } + let slot = q.next_avail.wrapping_add(ahead as u16) % q.size; + mem.read_obj::(GuestAddress(q.avail_addr + 4 + slot as u64 * 2)) + .map(u16::from_le) + .map(Some) + .map_err(|_| invalid_queue()) + } + + /// Asks the driver to kick, or not, when it posts buffers. + pub(super) fn set_notify(&self, mem: &GuestMemoryMmap, on: bool) -> io::Result<()> { + let flags: u16 = if on { 0 } else { VRING_USED_F_NO_NOTIFY }; + mem.write_obj(flags.to_le(), GuestAddress(self.used_addr)) + .map_err(|_| invalid_queue())?; + // The flag has to be visible before the ring is checked again. + fence(Ordering::SeqCst); + Ok(()) + } + + /// Whether the driver wants an interrupt for buffers already used. + pub(super) fn wants_interrupt(&self, mem: &GuestMemoryMmap) -> bool { + fence(Ordering::SeqCst); + mem.read_obj::(GuestAddress(self.avail_addr)) + .map_or(true, |flags| { + u16::from_le(flags) & VRING_AVAIL_F_NO_INTERRUPT == 0 + }) + } + + pub(super) fn descriptors( + &self, + mem: &GuestMemoryMmap, + head: u16, + writable: bool, + ) -> io::Result> { + let mut result = Vec::new(); + let mut index = head; + let mut total = 0usize; + let mut visited = [false; 256]; + loop { + if index >= self.size || visited[index as usize] { + return Err(invalid_queue()); + } + visited[index as usize] = true; + let mut bytes = [0; 16]; + mem.read_slice(&mut bytes, GuestAddress(self.desc_addr + index as u64 * 16)) + .map_err(|_| invalid_queue())?; + let address = u64::from_le_bytes(bytes[..8].try_into().unwrap()); + let len = u32::from_le_bytes(bytes[8..12].try_into().unwrap()) as usize; + let flags = u16::from_le_bytes(bytes[12..14].try_into().unwrap()); + if flags & !3 != 0 + || (flags & 2 != 0) != writable + || len > BUFFER_SIZE + HEADER_LEN + || address.checked_add(len as u64).is_none() + || !mem.check_range(GuestAddress(address), len) + { + return Err(invalid_queue()); + } + total = total.checked_add(len).ok_or_else(invalid_queue)?; + if total > BUFFER_SIZE + HEADER_LEN { + return Err(invalid_queue()); + } + result.push((address, len)); + if flags & 1 == 0 { + return Ok(result); + } + index = u16::from_le_bytes(bytes[14..16].try_into().unwrap()); + } + } + + pub(super) fn complete( + &mut self, + mem: &GuestMemoryMmap, + head: u16, + len: u32, + ) -> io::Result<()> { + let address = self.used_addr + 4 + (self.used % self.size) as u64 * 8; + mem.write_obj((head as u32).to_le(), GuestAddress(address)) + .map_err(|_| invalid_queue())?; + mem.write_obj(len.to_le(), GuestAddress(address + 4)) + .map_err(|_| invalid_queue())?; + fence(Ordering::Release); + self.used = self.used.wrapping_add(1); + mem.write_obj(self.used.to_le(), GuestAddress(self.used_addr + 2)) + .map_err(|_| invalid_queue())?; + self.next_avail = self.next_avail.wrapping_add(1); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn memory() -> GuestMemoryMmap { + GuestMemoryMmap::from_ranges(&[(GuestAddress(0), 1024 * 1024)]).unwrap() + } + + fn queue(mem: &GuestMemoryMmap) -> Queue { + Queue::new(8, 0x1000, 0x2000, 0x3000, mem).unwrap() + } + + fn descriptor( + mem: &GuestMemoryMmap, + index: u16, + address: u64, + len: u32, + flags: u16, + next: u16, + ) { + let mut bytes = Vec::new(); + bytes.extend_from_slice(&address.to_le_bytes()); + bytes.extend_from_slice(&len.to_le_bytes()); + bytes.extend_from_slice(&flags.to_le_bytes()); + bytes.extend_from_slice(&next.to_le_bytes()); + mem.write_slice(&bytes, GuestAddress(0x1000 + index as u64 * 16)) + .unwrap(); + } + + #[test] + fn queue_configuration_is_bounded_and_aligned() { + let mem = memory(); + for size in [0, 3, 257, u16::MAX] { + assert!(Queue::new(size, 0x1000, 0x2000, 0x3000, &mem).is_err()); + } + for (desc, avail, used) in [ + (1, 0x2000, 0x3000), + (0x1000, 1, 0x3000), + (0x1000, 0x2000, 1), + (u64::MAX - 15, 0x2000, 0x3000), + (0x1000, 0x2000, 1024 * 1024), + ] { + assert!(Queue::new(8, desc, avail, used, &mem).is_err()); + } + } + + #[test] + fn split_header_and_payload_chain() { + let mem = memory(); + let q = queue(&mem); + descriptor(&mem, 0, 0x4000, HEADER_LEN as u32, 3, 1); + descriptor(&mem, 1, 0x5000, 4096, 2, 0); + assert_eq!( + q.descriptors(&mem, 0, true).unwrap(), + vec![(0x4000, HEADER_LEN), (0x5000, 4096)] + ); + assert!(q.descriptors(&mem, 0, false).is_err()); + } + + #[test] + fn descriptor_cycles_flags_indices_and_ranges_are_rejected() { + let mem = memory(); + let q = queue(&mem); + for (address, len, flags, next) in [ + (0x4000, 44, 1, 0), + (0x4000, 44, 1, 8), + (0x4000, 44, 4, 0), + (0x4000, 44, 2, 0), + (u64::MAX, 44, 0, 0), + (1024 * 1024 - 10, 44, 0, 0), + (0x4000, u32::MAX, 0, 0), + ] { + descriptor(&mem, 0, address, len, flags, next); + assert!(q.descriptors(&mem, 0, false).is_err()); + } + assert!(q.descriptors(&mem, 8, false).is_err()); + descriptor(&mem, 0, 0x4000, BUFFER_SIZE as u32, 1, 1); + descriptor(&mem, 1, 0x4000, BUFFER_SIZE as u32, 0, 0); + assert!(q.descriptors(&mem, 0, false).is_err()); + } + + #[test] + fn available_and_used_indices_wrap_and_publish_lengths() { + let mem = memory(); + let mut q = queue(&mem); + assert!(q.head(&mem).unwrap().is_none()); + q.next_avail = u16::MAX; + q.used = u16::MAX; + mem.write_obj(0u16, GuestAddress(0x2002)).unwrap(); + mem.write_obj(5u16.to_le(), GuestAddress(0x2004 + 7 * 2)) + .unwrap(); + assert_eq!(q.head(&mem).unwrap(), Some(5)); + q.complete(&mem, 5, 123).unwrap(); + assert_eq!(q.used, 0); + assert_eq!(q.next_avail, 0); + assert_eq!(mem.read_obj::(GuestAddress(0x3002)).unwrap(), 0); + assert_eq!( + u32::from_le(mem.read_obj::(GuestAddress(0x3004 + 7 * 8)).unwrap()), + 5 + ); + assert_eq!( + u32::from_le(mem.read_obj::(GuestAddress(0x3008 + 7 * 8)).unwrap()), + 123 + ); + assert!(q.head(&mem).unwrap().is_none()); + mem.write_obj(9u16.to_le(), GuestAddress(0x2002)).unwrap(); + assert!(q.head(&mem).is_err()); + } + + #[test] + fn heads_ahead_and_notification_flags() { + let mem = memory(); + let q = queue(&mem); + mem.write_obj(3u16.to_le(), GuestAddress(0x2002)).unwrap(); + for (slot, head) in [4u16, 6, 1].iter().enumerate() { + mem.write_obj(head.to_le(), GuestAddress(0x2004 + slot as u64 * 2)) + .unwrap(); + } + assert_eq!(q.head_at(&mem, 0).unwrap(), Some(4)); + assert_eq!(q.head_at(&mem, 2).unwrap(), Some(1)); + assert!(q.head_at(&mem, 3).unwrap().is_none()); + q.set_notify(&mem, false).unwrap(); + assert_eq!(mem.read_obj::(GuestAddress(0x3000)).unwrap(), 1); + q.set_notify(&mem, true).unwrap(); + assert_eq!(mem.read_obj::(GuestAddress(0x3000)).unwrap(), 0); + assert!(q.wants_interrupt(&mem)); + mem.write_obj(1u16.to_le(), GuestAddress(0x2000)).unwrap(); + assert!(!q.wants_interrupt(&mem)); + } + + #[test] + fn arbitrary_descriptor_bytes_do_not_panic() { + let mem = memory(); + let q = queue(&mem); + let mut seed = 123456789u64; + for _ in 0..4096 { + let mut bytes = [0; 128]; + for byte in &mut bytes { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + *byte = seed as u8; + } + mem.write_slice(&bytes, GuestAddress(0x1000)).unwrap(); + let _ = q.descriptors(&mem, (seed % 16) as u16, seed & 16 != 0); + } + } +} diff --git a/crates/shuru-linux/src/lib.rs b/crates/shuru-linux/src/lib.rs index 0fd33da..17aa6a9 100644 --- a/crates/shuru-linux/src/lib.rs +++ b/crates/shuru-linux/src/lib.rs @@ -1,5 +1,6 @@ mod kvm; +mod anonfile; mod bootloader; mod configuration; mod directory_sharing; @@ -13,6 +14,7 @@ mod storage; pub mod terminal; mod vm; +pub use anonfile::{open_anonymous, reflink}; pub use bootloader::LinuxBootLoader; pub use configuration::VirtualMachineConfiguration; pub use directory_sharing::{SharedDirectory, VirtioFileSystemDevice}; @@ -26,4 +28,4 @@ pub use storage::{ DiskImageAttachment, DiskImageCachingMode, DiskImageSynchronizationMode, NbdAttachment, StorageAttachment, StorageDevice, VirtioBlockDevice, }; -pub use vm::{VirtualMachine, VmState}; +pub use vm::{VirtualMachine, VmDescriptors, VmState}; diff --git a/crates/shuru-linux/src/memory_balloon.rs b/crates/shuru-linux/src/memory_balloon.rs index d9ef529..0cdbf42 100644 --- a/crates/shuru-linux/src/memory_balloon.rs +++ b/crates/shuru-linux/src/memory_balloon.rs @@ -1,5 +1,6 @@ -/// Placeholder — memory balloon is not yet implemented for KVM. -/// Kept for API compatibility with shuru-darwin. +/// A virtio memory balloon. The guest reports free pages back to the host +/// on its own, and `VirtualMachine::set_memory_target` can ask it to give +/// up more. pub struct VirtioMemoryBalloonDevice; impl VirtioMemoryBalloonDevice { diff --git a/crates/shuru-linux/src/network.rs b/crates/shuru-linux/src/network.rs index 266135a..a413327 100644 --- a/crates/shuru-linux/src/network.rs +++ b/crates/shuru-linux/src/network.rs @@ -8,7 +8,9 @@ pub struct FileHandleNetworkAttachment { impl FileHandleNetworkAttachment { /// Creates a network attachment from a connected datagram socket fd. /// The fd should be one end of `socketpair(AF_UNIX, SOCK_DGRAM)`. - pub fn new(fd: RawFd) -> Self { + /// `mtu` is accepted for API parity with the Darwin implementation + /// and ignored on this platform (Linux build is a stub). + pub fn new(fd: RawFd, _mtu: i64) -> Self { FileHandleNetworkAttachment { fd } } } @@ -25,7 +27,7 @@ impl MACAddress { /// Generate a random locally-administered MAC address. pub fn random_local() -> Self { let mut bytes = [0u8; 6]; - let fd = unsafe { libc::open(b"/dev/urandom\0".as_ptr() as *const _, libc::O_RDONLY) }; + let fd = unsafe { libc::open(c"/dev/urandom".as_ptr(), libc::O_RDONLY) }; if fd >= 0 { unsafe { libc::read(fd, bytes.as_mut_ptr() as *mut libc::c_void, 6); diff --git a/crates/shuru-linux/src/terminal.rs b/crates/shuru-linux/src/terminal.rs index b13f81c..c8e9086 100644 --- a/crates/shuru-linux/src/terminal.rs +++ b/crates/shuru-linux/src/terminal.rs @@ -192,8 +192,8 @@ impl StdinRelay { return StdinEvent::Shutdown; } - for i in 0..n as usize { - match events[i].u64 { + for event in &events[..n as usize] { + match event.u64 { EV_STDIN => return StdinEvent::Ready, EV_PIPE => return StdinEvent::Shutdown, EV_SIGNAL => { diff --git a/crates/shuru-linux/src/vm.rs b/crates/shuru-linux/src/vm.rs index de893d7..12e8187 100644 --- a/crates/shuru-linux/src/vm.rs +++ b/crates/shuru-linux/src/vm.rs @@ -1,5 +1,7 @@ +use std::fs::{File, OpenOptions}; use std::net::TcpStream; -use std::os::unix::io::FromRawFd; +use std::os::fd::{AsRawFd, OwnedFd}; +use std::os::unix::net::UnixListener; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; @@ -7,7 +9,7 @@ use crossbeam_channel::{bounded, Receiver, Sender}; use crate::configuration::VirtualMachineConfiguration; use crate::error::{Result, VzError}; -use crate::kvm::{self, layout}; +use crate::kvm; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum VmState { @@ -19,59 +21,121 @@ pub enum VmState { Unknown = -1, } +/// Everything a VM needs, already open, for a process that may not open +/// files itself. There is no virtio-fs: it needs host paths. +pub struct VmDescriptors { + pub kvm: OwnedFd, + pub kernel: File, + pub initrd: Option, + pub disk: Option, + pub disk_read_only: bool, + pub serial_write: Option, + pub serial_read: Option, + pub network: Option, + pub network_mac: Option<[u8; 6]>, + pub cpu_count: usize, + pub memory_bytes: u64, + pub command_line: String, + /// The vsock connection cap, or no vsock device. + pub vsock: Option, + pub balloon: bool, +} + pub struct VirtualMachine { vm: Mutex>, - guest_cid: u64, state_tx: Sender, state_rx: Receiver, running: Arc, + // Descriptors the devices use by number. + _held: Vec, } impl VirtualMachine { - pub fn new(config: &VirtualMachineConfiguration) -> Self { + pub fn new(config: &VirtualMachineConfiguration) -> Result { let inner = config.inner.borrow(); - let (state_tx, state_rx) = bounded(1); - let running = Arc::new(AtomicBool::new(false)); + let open = |path: &str, what: &str| { + File::open(path).map_err(|e| VzError::new(format!("failed to open {}: {}", what, e))) + }; + let kernel = open(&inner.kernel_path, "kernel")?; + let initrd = inner + .initrd_path + .as_deref() + .map(|p| open(p, "initrd")) + .transpose()?; + let disk = inner + .disk_path + .as_deref() + .map(|p| { + OpenOptions::new() + .read(true) + .write(!inner.disk_read_only) + .open(p) + .map_err(|e| VzError::new(format!("failed to open disk: {}", e))) + }) + .transpose()?; let create_config = kvm::VmCreateConfig { + kvm: None, cpu_count: inner.cpu_count, memory_bytes: inner.memory_size, - kernel_path: inner.kernel_path.clone(), - initrd_path: inner.initrd_path.clone(), + kernel, + initrd, command_line: inner.command_line.clone(), serial_write_fd: inner.serial_write_fd, serial_read_fd: inner.serial_read_fd, - disk_path: inner.disk_path.clone(), + disk, disk_read_only: inner.disk_read_only, network_fd: inner.network_fd, network_mac: inner.network_mac, has_vsock: inner.has_socket, - guest_cid: layout::GUEST_CID, + vsock_connections: kvm::vsock::MAX_CONNECTIONS, mounts: inner.mounts.clone(), + has_balloon: inner.has_balloon, }; + Self::create(create_config, Vec::new()) + } - match kvm::KvmVm::create(create_config) { - Ok(kvm_vm) => { - let cid = kvm_vm.guest_cid; - VirtualMachine { - vm: Mutex::new(Some(kvm_vm)), - guest_cid: cid, - state_tx, - state_rx, - running, - } - } - Err(e) => { - eprintln!("shuru: failed to create KVM VM: {}", e); - VirtualMachine { - vm: Mutex::new(None), - guest_cid: layout::GUEST_CID, - state_tx, - state_rx, - running, - } - } - } + pub fn from_descriptors(descriptors: VmDescriptors) -> Result { + let raw = |fd: &Option| fd.as_ref().map(|fd| fd.as_raw_fd()); + let create_config = kvm::VmCreateConfig { + kvm: Some(descriptors.kvm), + cpu_count: descriptors.cpu_count, + memory_bytes: descriptors.memory_bytes, + kernel: descriptors.kernel, + initrd: descriptors.initrd, + command_line: descriptors.command_line, + serial_write_fd: raw(&descriptors.serial_write), + serial_read_fd: raw(&descriptors.serial_read), + disk: descriptors.disk, + disk_read_only: descriptors.disk_read_only, + network_fd: raw(&descriptors.network), + network_mac: descriptors.network_mac, + has_vsock: descriptors.vsock.is_some(), + vsock_connections: descriptors.vsock.unwrap_or(0), + mounts: Vec::new(), + has_balloon: descriptors.balloon, + }; + let held = [ + descriptors.serial_write, + descriptors.serial_read, + descriptors.network, + ] + .into_iter() + .flatten() + .collect(); + Self::create(create_config, held) + } + + fn create(create_config: kvm::VmCreateConfig, held: Vec) -> Result { + let (state_tx, state_rx) = bounded(1); + let kvm_vm = kvm::KvmVm::create(create_config)?; + Ok(VirtualMachine { + vm: Mutex::new(Some(kvm_vm)), + state_tx, + state_rx, + running: Arc::new(AtomicBool::new(false)), + _held: held, + }) } pub fn supported() -> bool { @@ -123,6 +187,29 @@ impl VirtualMachine { Ok(()) } + /// How much memory the guest should keep. The balloon driver in the + /// guest hands the rest to the host, and gets it back when the target + /// rises again. Fails without a balloon device. + pub fn set_memory_target(&self, bytes: u64) -> Result<()> { + let vm = self.vm.lock().unwrap(); + let control = vm + .as_ref() + .and_then(|v| v.balloon.clone()) + .ok_or_else(|| VzError::new("no memory balloon device"))?; + control.set_memory_target(bytes); + Ok(()) + } + + /// Bytes the guest has handed over in response to the memory target. + pub fn ballooned_bytes(&self) -> u64 { + self.vm + .lock() + .unwrap() + .as_ref() + .and_then(|v| v.balloon.as_ref().map(|b| b.ballooned_bytes())) + .unwrap_or(0) + } + pub fn state_channel(&self) -> Receiver { self.state_rx.clone() } @@ -146,86 +233,28 @@ impl VirtualMachine { self.can_stop() } - /// Connect to a vsock port on the guest via AF_VSOCK. - pub fn connect_to_vsock_port(&self, port: u32) -> Result { - let sock = unsafe { libc::socket(layout::AF_VSOCK, libc::SOCK_STREAM, 0) }; - if sock < 0 { - return Err(VzError::new(format!( - "failed to create vsock socket: {}", - std::io::Error::last_os_error() - ))); - } - - // Use non-blocking connect + poll to implement a 2-second timeout. - // SO_SNDTIMEO doesn't work for AF_VSOCK on some kernels. - unsafe { - let flags = libc::fcntl(sock, libc::F_GETFL); - libc::fcntl(sock, libc::F_SETFL, flags | libc::O_NONBLOCK); - } - - let addr = libc::sockaddr_vm { - svm_family: layout::AF_VSOCK as libc::sa_family_t, - svm_reserved1: 0, - svm_port: port, - svm_cid: self.guest_cid as u32, - svm_zero: [0u8; 4], - }; - - let ret = unsafe { - libc::connect( - sock, - &addr as *const libc::sockaddr_vm as *const libc::sockaddr, - std::mem::size_of::() as libc::socklen_t, - ) - }; - - if ret < 0 { - let errno = std::io::Error::last_os_error(); - if errno.raw_os_error() != Some(libc::EINPROGRESS) { - unsafe { libc::close(sock) }; - return Err(VzError::new(format!("vsock connect failed: {}", errno))); - } - - // Wait for connect to complete with 2s timeout - let mut pfd = libc::pollfd { - fd: sock, - events: libc::POLLOUT, - revents: 0, - }; - let poll_ret = unsafe { libc::poll(&mut pfd, 1, 2000) }; - if poll_ret <= 0 { - unsafe { libc::close(sock) }; - return Err(VzError::new("vsock connect timed out")); - } - - // Check if connect succeeded - let mut err: libc::c_int = 0; - let mut len: libc::socklen_t = std::mem::size_of::() as libc::socklen_t; - unsafe { - libc::getsockopt( - sock, - libc::SOL_SOCKET, - libc::SO_ERROR, - &mut err as *mut _ as *mut libc::c_void, - &mut len, - ); - } - if err != 0 { - unsafe { libc::close(sock) }; - return Err(VzError::new(format!( - "vsock connect failed: {}", - std::io::Error::from_raw_os_error(err) - ))); - } - } + fn vsock(&self) -> Result> { + self.vm + .lock() + .unwrap() + .as_ref() + .and_then(|vm| vm.vsock.clone()) + .ok_or_else(|| VzError::new("no vsock device")) + } - // Restore blocking mode for normal I/O - unsafe { - let flags = libc::fcntl(sock, libc::F_GETFL); - libc::fcntl(sock, libc::F_SETFL, flags & !libc::O_NONBLOCK); - } + pub fn connect_to_vsock_port(&self, port: u32) -> Result { + self.vsock()? + .connect(port) + .map_err(|e| VzError::new(format!("vsock connect failed: {e}"))) + } - Ok(unsafe { TcpStream::from_raw_fd(sock) }) + /// Accepts host connects on `listener` for as long as the VM lives. A + /// client writes `CONNECT \n` and reads `OK \n` + /// once the guest accepts; a refused connect closes the socket. + pub fn serve_vsock(&self, listener: UnixListener) -> Result<()> { + self.vsock()? + .serve(listener) + .map_err(|e| VzError::new(format!("vsock listener failed: {e}"))) } pub fn state(&self) -> VmState { diff --git a/crates/shuru-node/Cargo.toml b/crates/shuru-node/Cargo.toml new file mode 100644 index 0000000..1b528a6 --- /dev/null +++ b/crates/shuru-node/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "shuru-node" +version = "0.1.0" +edition = "2021" +description = "Runs shuru VMs on a Linux node, one jailed VMM per sandbox" +license = "Apache-2.0" +repository = "https://github.com/superhq-ai/shuru" +publish = false + +[[bin]] +name = "shuru-node" +path = "src/main.rs" + +[dependencies] +shuru-linux = { path = "../shuru-linux" } +shuru-proto = { path = "../shuru-proto" } +shuru-proxy = { path = "../shuru-proxy" } +anyhow = "1" +clap = { version = "4", features = ["derive"] } +getrandom = "0.3" +kvm-bindings = "0.9" +libc = "0.2" +seccompiler = "0.5" +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/crates/shuru-node/src/bin/shuru-netd/main.rs b/crates/shuru-node/src/bin/shuru-netd/main.rs new file mode 100644 index 0000000..3708d39 --- /dev/null +++ b/crates/shuru-node/src/bin/shuru-netd/main.rs @@ -0,0 +1,109 @@ +//! The network for one sandbox, run by `shuru-node` inside the jail with +//! the host network. Fd 3 is a listening control socket. The first +//! connection brings the egress policy, the secrets and the host end of +//! the VM frame socket. + +use std::collections::HashMap; +use std::os::fd::{FromRawFd, IntoRawFd, OwnedFd}; +use std::os::unix::net::{UnixListener, UnixStream}; + +use anyhow::{bail, Context, Result}; +use shuru_node::control::{self, NetdConfig, Request, Response}; +use shuru_node::seccomp; +use shuru_proxy::config::{NetworkConfig, SecretConfig}; +use shuru_proxy::ProxyConfig; + +const CONTROL_FD: i32 = 3; + +fn main() { + if let Err(e) = run() { + eprintln!("shuru-netd: {e:#}"); + std::process::exit(1); + } +} + +fn run() -> Result<()> { + unsafe { libc::prctl(libc::PR_SET_DUMPABLE, 0, 0, 0, 0) }; + let log = match std::env::args().nth(1).as_deref() { + None => false, + Some("--seccomp-log") => true, + Some(other) => bail!("unknown argument {other}"), + }; + let listener = unsafe { UnixListener::from_raw_fd(CONTROL_FD) }; + let (stream, _) = listener.accept().context("accepting the node")?; + let (request, fds) = control::receive::(&stream, 1)?.context("node left")?; + let Request::ConfigureNetwork(config) = request else { + bail!("the first request must be the network configuration"); + }; + let (_proxy, response) = match start(config, fds, log) { + Ok(started) => started, + Err(e) => { + let message = format!("{e:#}"); + let _ = control::send(&stream, &Response::Error { message }, &[]); + return Err(e); + } + }; + control::send(&stream, &response, &[])?; + + let mut stream = stream; + loop { + serve(&stream); + stream = listener.accept().context("accepting the node")?.0; + } +} + +fn start( + config: NetdConfig, + fds: Vec, + log: bool, +) -> Result<(shuru_proxy::ProxyHandle, Response)> { + let [frames]: [OwnedFd; 1] = fds + .try_into() + .map_err(|fds: Vec<_>| anyhow::anyhow!("expected 1 descriptor, got {}", fds.len()))?; + let secrets: HashMap = config + .secrets + .into_iter() + .map(|(name, secret)| { + let secret = SecretConfig { + value: Some(secret.value), + hosts: secret.hosts, + ..Default::default() + }; + (name, secret) + }) + .collect(); + let proxy = ProxyConfig { + dns_resolver: config.dns_resolver, + secrets, + network: NetworkConfig { + allow: config.allow, + }, + public_only: true, + upstream_roots: Some(config.upstream_roots.into_bytes()), + ..Default::default() + }; + seccomp::install_netd(log)?; + let handle = shuru_proxy::start(frames.into_raw_fd(), proxy)?; + let response = Response::Network { + placeholders: handle.placeholders.clone(), + ca_cert_pem: String::from_utf8(handle.ca_cert_pem.clone())?, + }; + Ok((handle, response)) +} + +fn serve(stream: &UnixStream) { + while let Ok(Some((request, _))) = control::receive::(stream, 0) { + let response = match request { + Request::Stop => { + let _ = control::send(stream, &Response::Ok, &[]); + std::process::exit(0); + } + _ => Response::Error { + message: "not a request for the network process".into(), + }, + }; + if control::send(stream, &response, &[]).is_err() { + return; + } + } +} diff --git a/crates/shuru-node/src/bin/shuru-vmm/main.rs b/crates/shuru-node/src/bin/shuru-vmm/main.rs new file mode 100644 index 0000000..717171b --- /dev/null +++ b/crates/shuru-node/src/bin/shuru-vmm/main.rs @@ -0,0 +1,134 @@ +//! One sandbox VM, run by `shuru-node` inside the jail. Fd 3 is a listening +//! control socket. The first connection brings the configuration and every +//! descriptor the VM needs; after that the VMM opens nothing. + +use std::fs::File; +use std::os::fd::{FromRawFd, OwnedFd}; +use std::os::unix::net::{UnixListener, UnixStream}; + +use anyhow::{bail, Context, Result}; +use shuru_linux::{VirtualMachine, VmDescriptors, VmState}; +use shuru_node::control::{self, Request, Response, VmmConfig}; +use shuru_node::seccomp; + +const CONTROL_FD: i32 = 3; + +fn main() { + if let Err(e) = run() { + eprintln!("shuru-vmm: {e:#}"); + std::process::exit(1); + } +} + +fn run() -> Result<()> { + unsafe { libc::prctl(libc::PR_SET_DUMPABLE, 0, 0, 0, 0) }; + let log = match std::env::args().nth(1).as_deref() { + None => false, + Some("--seccomp-log") => true, + Some(other) => bail!("unknown argument {other}"), + }; + let listener = unsafe { UnixListener::from_raw_fd(CONTROL_FD) }; + let (stream, _) = listener.accept().context("accepting the node")?; + let (request, fds) = + control::receive::(&stream, control::MAX_FDS)?.context("node left")?; + let Request::Configure(config) = request else { + bail!("the first request must be the configuration"); + }; + let vm = match start(config, fds, log) { + Ok(vm) => vm, + Err(e) => { + let message = format!("{e:#}"); + let _ = control::send(&stream, &Response::Error { message }, &[]); + return Err(e); + } + }; + control::send(&stream, &Response::Ok, &[])?; + + // A node that restarts connects again. + let mut stream = stream; + loop { + serve(&vm, &stream); + stream = listener.accept().context("accepting the node")?.0; + } +} + +fn start(config: VmmConfig, fds: Vec, log: bool) -> Result { + if fds.len() != config.fd_count() { + bail!( + "expected {} descriptors, got {}", + config.fd_count(), + fds.len() + ); + } + let mut fds = fds.into_iter(); + let mut next = || fds.next().unwrap(); + let kvm = next(); + let kernel = File::from(next()); + let initrd = config.initrd.then(|| File::from(next())); + let disk = config.disk.as_ref().map(|_| File::from(next())); + let serial_write = config.serial.then(&mut next); + let network = config.network_mac.map(|_| next()); + let vsock = UnixListener::from(next()); + + let vm = VirtualMachine::from_descriptors(VmDescriptors { + kvm, + kernel, + initrd, + disk, + disk_read_only: config.disk.as_ref().is_some_and(|disk| disk.read_only), + serial_write, + serial_read: None, + network, + network_mac: config.network_mac, + cpu_count: config.cpus, + memory_bytes: config.memory_bytes, + command_line: config.command_line, + vsock: Some(config.vsock_connections), + balloon: config.balloon, + })?; + vm.serve_vsock(vsock)?; + seccomp::install_vmm(log)?; + + // A guest power off ends the process whatever the node is doing. + let states = vm.state_channel(); + std::thread::Builder::new() + .name("shuru-vmm-state".into()) + .spawn(move || { + while let Ok(state) = states.recv() { + if matches!(state, VmState::Stopped | VmState::Error) { + std::process::exit(0); + } + } + })?; + vm.start()?; + Ok(vm) +} + +fn serve(vm: &VirtualMachine, stream: &UnixStream) { + while let Ok(Some((request, _))) = control::receive::(stream, 0) { + let response = match request { + Request::Configure(_) | Request::ConfigureNetwork(_) => Response::Error { + message: "already configured".into(), + }, + // The node shuts the guest down first when it wants a clean + // stop, so this is the power cord. + Request::Stop => { + let _ = control::send(stream, &Response::Ok, &[]); + std::process::exit(0); + } + Request::SetMemoryTarget { bytes } => match vm.set_memory_target(bytes) { + Ok(()) => Response::Ok, + Err(e) => Response::Error { + message: e.to_string(), + }, + }, + Request::Stats => Response::Stats { + running: vm.state() == VmState::Running, + ballooned_bytes: vm.ballooned_bytes(), + }, + }; + if control::send(stream, &response, &[]).is_err() { + return; + } + } +} diff --git a/crates/shuru-node/src/cgroup.rs b/crates/shuru-node/src/cgroup.rs new file mode 100644 index 0000000..ba3b696 --- /dev/null +++ b/crates/shuru-node/src/cgroup.rs @@ -0,0 +1,105 @@ +//! Per sandbox cgroups under `shuru.slice`: limits on `sbx-`, processes +//! in its children, so the envelope holds whatever the split between them. + +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +const ROOT: &str = "/sys/fs/cgroup"; +const CONTROLLERS: &str = "+cpu +memory +pids"; +const PERIOD_US: u64 = 100_000; +/// Host memory a VM may use beyond its guest RAM (RFC 0011). +pub const OVERHEAD_BYTES: u64 = 320 << 20; +const HIGH_BELOW_MAX: u64 = 64 << 20; + +pub struct Limits { + pub guest_memory_bytes: u64, + pub cpus: u64, + pub pids: u64, +} + +pub struct Sandbox { + path: PathBuf, + children: Vec, +} + +impl Sandbox { + pub fn create(id: &str, limits: &Limits) -> io::Result { + let slice = Path::new(ROOT).join("shuru.slice"); + write(&Path::new(ROOT).join("cgroup.subtree_control"), CONTROLLERS)?; + match fs::create_dir(&slice) { + Err(e) if e.kind() != io::ErrorKind::AlreadyExists => return Err(e), + _ => {} + } + write(&slice.join("cgroup.subtree_control"), CONTROLLERS)?; + + let path = slice.join(format!("sbx-{id}")); + fs::create_dir(&path)?; + let sandbox = Sandbox { + path, + children: Vec::new(), + }; + let max = limits.guest_memory_bytes + OVERHEAD_BYTES; + sandbox.set("memory.max", &max.to_string())?; + sandbox.set("memory.high", &(max - HIGH_BELOW_MAX).to_string())?; + match sandbox.set("memory.swap.max", "0") { + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + result => result?, + } + sandbox.set( + "cpu.max", + &format!("{} {PERIOD_US}", limits.cpus * PERIOD_US), + )?; + sandbox.set("pids.max", &limits.pids.to_string())?; + Ok(sandbox) + } + + fn set(&self, file: &str, value: &str) -> io::Result<()> { + write(&self.path.join(file), value) + .map_err(|e| io::Error::new(e.kind(), format!("{file}: {e}"))) + } + + /// Creates a child cgroup and opens its `cgroup.procs` for a process to + /// join. + pub fn child(&mut self, name: &str) -> io::Result { + let path = self.path.join(name); + fs::create_dir(&path)?; + self.children.push(path.clone()); + OpenOptions::new() + .write(true) + .open(path.join("cgroup.procs")) + } + + pub fn memory_peak(&self) -> Option { + fs::read_to_string(self.path.join("memory.peak")) + .ok()? + .trim() + .parse() + .ok() + } +} + +impl Drop for Sandbox { + // A cgroup goes away only once its last process is reaped, which can + // lag the exit of a pid namespace init by a moment. + fn drop(&mut self) { + for path in self.children.iter().rev().chain([&self.path]) { + for _ in 0..100 { + match fs::remove_dir(path) { + Err(e) if e.raw_os_error() == Some(libc::EBUSY) => { + std::thread::sleep(Duration::from_millis(10)) + } + _ => break, + } + } + } + } +} + +fn write(path: &Path, value: &str) -> io::Result<()> { + OpenOptions::new() + .write(true) + .open(path)? + .write_all(value.as_bytes()) +} diff --git a/crates/shuru-node/src/control.rs b/crates/shuru-node/src/control.rs new file mode 100644 index 0000000..f2a59df --- /dev/null +++ b/crates/shuru-node/src/control.rs @@ -0,0 +1,283 @@ +//! The node talks to each jailed process over one unix stream. A message is +//! a u32 little endian length and a JSON body. The configuration carries +//! the descriptors as SCM_RIGHTS on its first byte. + +use std::collections::HashMap; +use std::io::{self, Read, Write}; +use std::net::Ipv4Addr; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd}; +use std::os::unix::net::UnixStream; + +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; + +// Room for a system CA bundle. +pub const MAX_MESSAGE: usize = 1 << 20; +pub const MAX_FDS: usize = 8; + +/// The VM to build. Descriptors follow in this order, each optional one +/// present only when its field says so: `/dev/kvm`, kernel, initrd, disk, +/// serial output, network socket, vsock listener. +#[derive(Debug, Serialize, Deserialize)] +pub struct VmmConfig { + pub cpus: usize, + pub memory_bytes: u64, + pub command_line: String, + pub initrd: bool, + pub disk: Option, + pub serial: bool, + pub network_mac: Option<[u8; 6]>, + pub vsock_connections: usize, + pub balloon: bool, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Disk { + pub read_only: bool, +} + +impl VmmConfig { + pub fn fd_count(&self) -> usize { + 3 + self.initrd as usize + + self.disk.is_some() as usize + + self.serial as usize + + self.network_mac.is_some() as usize + } +} + +/// The network process for one sandbox. One descriptor follows: the host +/// end of the frame socket the VM sends Ethernet on. +#[derive(Debug, Serialize, Deserialize)] +pub struct NetdConfig { + pub dns_resolver: Ipv4Addr, + /// Domain patterns the guest may reach. Empty allows every public + /// address. + pub allow: Vec, + pub secrets: HashMap, + /// PEM roots for checking upstream servers when relaying secrets. + pub upstream_roots: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Secret { + pub value: String, + pub hosts: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "op", rename_all = "snake_case")] +pub enum Request { + Configure(VmmConfig), + ConfigureNetwork(NetdConfig), + Stop, + SetMemoryTarget { bytes: u64 }, + Stats, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "status", rename_all = "snake_case")] +pub enum Response { + Ok, + Stats { + running: bool, + ballooned_bytes: u64, + }, + /// The guest sees `placeholders` in place of secret values, and trusts + /// `ca_cert_pem` for the hosts they are relayed to. + Network { + placeholders: HashMap, + ca_cert_pem: String, + }, + Error { + message: String, + }, +} + +pub fn send(stream: &UnixStream, message: &impl Serialize, fds: &[RawFd]) -> io::Result<()> { + let body = serde_json::to_vec(message)?; + if body.len() > MAX_MESSAGE || fds.len() > MAX_FDS { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "control message too large", + )); + } + let mut bytes = (body.len() as u32).to_le_bytes().to_vec(); + bytes.extend_from_slice(&body); + + let mut iov = libc::iovec { + iov_base: bytes.as_mut_ptr().cast(), + iov_len: bytes.len(), + }; + let space = unsafe { libc::CMSG_SPACE(std::mem::size_of_val(fds) as u32) } as usize; + let mut control = vec![0u8; space]; + let mut message: libc::msghdr = unsafe { std::mem::zeroed() }; + message.msg_iov = &mut iov; + message.msg_iovlen = 1; + if !fds.is_empty() { + message.msg_control = control.as_mut_ptr().cast(); + message.msg_controllen = space as _; + unsafe { + let header = libc::CMSG_FIRSTHDR(&message); + (*header).cmsg_level = libc::SOL_SOCKET; + (*header).cmsg_type = libc::SCM_RIGHTS; + (*header).cmsg_len = libc::CMSG_LEN(std::mem::size_of_val(fds) as u32) as _; + std::ptr::copy_nonoverlapping( + fds.as_ptr(), + libc::CMSG_DATA(header).cast::(), + fds.len(), + ); + } + } + let sent = loop { + let n = unsafe { libc::sendmsg(stream.as_raw_fd(), &message, libc::MSG_NOSIGNAL) }; + if n >= 0 { + break n as usize; + } + let error = io::Error::last_os_error(); + if error.kind() != io::ErrorKind::Interrupted { + return Err(error); + } + }; + (&*stream).write_all(&bytes[sent..]) +} + +/// Reads one message, keeping at most `max_fds` descriptors sent with it. +/// Returns `None` when the peer closed between messages. +pub fn receive( + stream: &UnixStream, + max_fds: usize, +) -> io::Result)>> { + let mut length = [0u8; 4]; + let mut fds = Vec::new(); + let mut got = 0; + while got < length.len() { + let n = receive_with_fds(stream, &mut length[got..], max_fds, &mut fds)?; + if n == 0 { + if got == 0 { + return Ok(None); + } + return Err(io::ErrorKind::UnexpectedEof.into()); + } + got += n; + } + let length = u32::from_le_bytes(length) as usize; + if length > MAX_MESSAGE { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "control message too large", + )); + } + let mut body = vec![0; length]; + (&*stream).read_exact(&mut body)?; + let message = serde_json::from_slice(&body)?; + Ok(Some((message, fds))) +} + +fn receive_with_fds( + stream: &UnixStream, + buf: &mut [u8], + max_fds: usize, + fds: &mut Vec, +) -> io::Result { + let mut iov = libc::iovec { + iov_base: buf.as_mut_ptr().cast(), + iov_len: buf.len(), + }; + let space = unsafe { libc::CMSG_SPACE((max_fds * std::mem::size_of::()) as u32) }; + let mut control = vec![0u8; space as usize]; + let mut message: libc::msghdr = unsafe { std::mem::zeroed() }; + message.msg_iov = &mut iov; + message.msg_iovlen = 1; + if max_fds > 0 { + message.msg_control = control.as_mut_ptr().cast(); + message.msg_controllen = space as _; + } + let n = loop { + let n = unsafe { libc::recvmsg(stream.as_raw_fd(), &mut message, libc::MSG_CMSG_CLOEXEC) }; + if n >= 0 { + break n as usize; + } + let error = io::Error::last_os_error(); + if error.kind() != io::ErrorKind::Interrupted { + return Err(error); + } + }; + unsafe { + let mut header = libc::CMSG_FIRSTHDR(&message); + while !header.is_null() { + if (*header).cmsg_level == libc::SOL_SOCKET && (*header).cmsg_type == libc::SCM_RIGHTS { + let data = libc::CMSG_DATA(header).cast::(); + let count = ((*header).cmsg_len as usize - libc::CMSG_LEN(0) as usize) + / std::mem::size_of::(); + for i in 0..count { + fds.push(OwnedFd::from_raw_fd(data.add(i).read_unaligned())); + } + } + header = libc::CMSG_NXTHDR(&message, header); + } + } + if message.msg_flags & libc::MSG_CTRUNC != 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "unexpected descriptors on the control socket", + )); + } + Ok(n) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::File; + use std::io::Seek; + + #[test] + fn a_configuration_carries_its_descriptors() { + let (node, vmm) = UnixStream::pair().unwrap(); + let mut file = File::open("/proc/self/exe").unwrap(); + let config = VmmConfig { + cpus: 2, + memory_bytes: 1 << 30, + command_line: "quiet".into(), + initrd: false, + disk: None, + serial: false, + network_mac: None, + vsock_connections: 128, + balloon: true, + }; + let fds = [file.as_raw_fd(); 3]; + assert_eq!(config.fd_count(), fds.len()); + send(&node, &Request::Configure(config), &fds).unwrap(); + send(&node, &Request::Stop, &[]).unwrap(); + + let (request, received) = receive::(&vmm, MAX_FDS).unwrap().unwrap(); + assert!(matches!(request, Request::Configure(c) if c.vsock_connections == 128)); + assert_eq!(received.len(), 3); + file.seek(io::SeekFrom::Start(4)).unwrap(); + let mut other = File::from(received.into_iter().next().unwrap()); + assert_eq!(other.stream_position().unwrap(), 4); + + let (request, received) = receive::(&vmm, MAX_FDS).unwrap().unwrap(); + assert!(matches!(request, Request::Stop)); + assert!(received.is_empty()); + drop(node); + assert!(receive::(&vmm, MAX_FDS).unwrap().is_none()); + } + + #[test] + fn descriptors_from_a_peer_that_may_not_send_them_are_refused() { + let (vmm, node) = UnixStream::pair().unwrap(); + let file = File::open("/proc/self/exe").unwrap(); + send(&vmm, &Response::Ok, &[file.as_raw_fd()]).unwrap(); + assert!(receive::(&node, 0).is_err()); + } + + #[test] + fn oversized_messages_are_refused() { + let (a, mut b) = UnixStream::pair().unwrap(); + b.write_all(&(MAX_MESSAGE as u32 + 1).to_le_bytes()) + .unwrap(); + assert!(receive::(&a, 0).is_err()); + } +} diff --git a/crates/shuru-node/src/firewall.rs b/crates/shuru-node/src/firewall.rs new file mode 100644 index 0000000..0c3f1f2 --- /dev/null +++ b/crates/shuru-node/src/firewall.rs @@ -0,0 +1,64 @@ +//! The host firewall behind the proxy check: whatever the network +//! processes send, by their uid, reaches public IPv4 addresses only. + +use std::io::Write; +use std::process::{Command, Stdio}; + +use anyhow::{bail, Context, Result}; + +pub const NETD_UID_BASE: u32 = 200_000; +const NETD_UID_LAST: u32 = 299_999; + +const NON_PUBLIC_V4: &str = "0.0.0.0/8, 10.0.0.0/8, 100.64.0.0/10, 127.0.0.0/8, \ + 169.254.0.0/16, 172.16.0.0/12, 192.0.0.0/24, 192.0.2.0/24, 192.88.99.0/24, \ + 192.168.0.0/16, 198.18.0.0/15, 198.51.100.0/24, 203.0.113.0/24, 224.0.0.0/3"; + +fn ruleset() -> String { + let uids = format!("{NETD_UID_BASE}-{NETD_UID_LAST}"); + // Creating the table first lets the delete succeed on a fresh host; + // nft applies the whole file as one transaction. + format!( + "table inet shuru {{}} +delete table inet shuru +table inet shuru {{ + chain egress {{ + type filter hook output priority filter; policy accept; + meta skuid {uids} ip daddr {{ {NON_PUBLIC_V4} }} counter drop + meta skuid {uids} meta nfproto ipv6 counter drop + }} +}} +" + ) +} + +/// Replaces the `inet shuru` table. +pub fn install() -> Result<()> { + let mut nft = Command::new("nft") + .args(["-f", "-"]) + .stdin(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .context("running nft; is nftables installed?")?; + nft.stdin.take().unwrap().write_all(ruleset().as_bytes())?; + let output = nft.wait_with_output()?; + if !output.status.success() { + bail!( + "nft refused the shuru table: {}", + String::from_utf8_lossy(&output.stderr).trim() + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_ruleset_names_the_netd_uids_and_the_metadata_service() { + let rules = ruleset(); + assert!(rules.contains("meta skuid 200000-299999 ip daddr")); + assert!(rules.contains("169.254.0.0/16")); + assert!(rules.contains("meta nfproto ipv6 counter drop")); + } +} diff --git a/crates/shuru-node/src/jail.rs b/crates/shuru-node/src/jail.rs new file mode 100644 index 0000000..1df9c9c --- /dev/null +++ b/crates/shuru-node/src/jail.rs @@ -0,0 +1,289 @@ +//! Starts a process in the jail RFC 0008 describes: its own mount, pid, +//! ipc, uts and cgroup namespaces, a network namespace unless it is the +//! egress path, an empty read-only root, no capabilities, an unprivileged +//! uid and only the descriptors it is given. + +use std::ffi::{CStr, CString}; +use std::fs::File; +use std::io::{self, Read}; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd}; +use std::ptr; + +pub struct Jail<'a> { + pub uid: u32, + pub gid: u32, + /// `cgroup.procs` of the cgroup to run in, opened for writing. + pub cgroup_procs: &'a File, + /// An empty directory the root tmpfs is mounted on. + pub root: &'a CStr, + pub nofile: u64, + pub nproc: u64, + /// Keep the host network namespace. + pub host_network: bool, +} + +/// What the child inherits: fd 3 is `keep`, 0 and 1 are `null`, 2 is +/// `stderr`. Everything else is closed. +pub struct Handoff<'a> { + pub keep: &'a OwnedFd, + pub null: &'a File, + pub stderr: RawFd, +} + +const CGROUP: i32 = 0; +const MOUNTS: i32 = 1; +const CREDENTIALS: i32 = 2; +const LIMITS: i32 = 3; +const DESCRIPTORS: i32 = 4; +const EXEC: i32 = 5; +const STEPS: [&str; 6] = [ + "joining the cgroup", + "building the root", + "dropping privileges", + "setting limits", + "placing descriptors", + "executing the binary", +]; + +/// Executes `binary` in the jail and returns its pid. The binary must be +/// statically linked, since the root it runs in is empty, and the caller +/// must be single threaded. +pub fn spawn(jail: &Jail, binary: &File, argv: &[&CStr], handoff: &Handoff) -> io::Result { + let mut argv: Vec<*const libc::c_char> = argv.iter().map(|arg| arg.as_ptr()).collect(); + argv.push(ptr::null()); + let envp: [*const libc::c_char; 1] = [ptr::null()]; + let options = CString::new("mode=0555,size=4k").unwrap(); + + let mut pipe = [0; 2]; + if unsafe { libc::pipe2(pipe.as_mut_ptr(), libc::O_CLOEXEC) } < 0 { + return Err(io::Error::last_os_error()); + } + let (mut report_read, report_write) = + unsafe { (File::from_raw_fd(pipe[0]), OwnedFd::from_raw_fd(pipe[1])) }; + + let mut flags = libc::CLONE_NEWNS + | libc::CLONE_NEWPID + | libc::CLONE_NEWIPC + | libc::CLONE_NEWUTS + | libc::SIGCHLD; + if !jail.host_network { + flags |= libc::CLONE_NEWNET; + } + // Without a stack the child runs on a copy of ours, like fork. + let pid = unsafe { libc::syscall(libc::SYS_clone, flags as libc::c_ulong, 0, 0, 0, 0) }; + if pid < 0 { + return Err(io::Error::last_os_error()); + } + if pid == 0 { + let report = report_write.as_raw_fd(); + unsafe { + child( + jail, + binary.as_raw_fd(), + &argv, + &envp, + &options, + handoff, + report, + ) + } + } + let pid = pid as i32; + drop(report_write); + + let mut report = [0u8; 8]; + let mut read = 0; + while read < report.len() { + match report_read.read(&mut report[read..]) { + Ok(0) => break, + Ok(n) => read += n, + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + if read == 0 { + return Ok(pid); + } + let mut status = 0; + unsafe { libc::waitpid(pid, &mut status, 0) }; + let step = i32::from_le_bytes(report[..4].try_into().unwrap()); + let error = io::Error::from_raw_os_error(i32::from_le_bytes(report[4..].try_into().unwrap())); + let step = STEPS + .get(step as usize) + .copied() + .unwrap_or("an unknown step"); + Err(io::Error::new( + error.kind(), + format!("jail failed {step}: {error}"), + )) +} + +// Runs between clone and exec, so it makes system calls only: no +// allocation, no locks, and no libc wrappers that assume other threads. +unsafe fn child( + jail: &Jail, + binary: RawFd, + argv: &[*const libc::c_char], + envp: &[*const libc::c_char], + options: &CStr, + handoff: &Handoff, + report: RawFd, +) -> ! { + let mut report = report; + macro_rules! check { + ($step:expr, $call:expr) => { + if ($call as i64) < 0 { + fail(report, $step); + } + }; + } + + check!( + CGROUP, + libc::write(jail.cgroup_procs.as_raw_fd(), b"0".as_ptr().cast(), 1) + ); + // After joining, so the namespace root is the sandbox cgroup. + check!(CGROUP, libc::unshare(libc::CLONE_NEWCGROUP)); + + check!( + MOUNTS, + libc::mount( + ptr::null(), + c"/".as_ptr(), + ptr::null(), + libc::MS_REC | libc::MS_PRIVATE, + ptr::null(), + ) + ); + check!( + MOUNTS, + libc::mount( + c"tmpfs".as_ptr(), + jail.root.as_ptr(), + c"tmpfs".as_ptr(), + libc::MS_NOSUID | libc::MS_NODEV | libc::MS_NOEXEC | libc::MS_RDONLY, + options.as_ptr().cast(), + ) + ); + check!(MOUNTS, libc::chdir(jail.root.as_ptr())); + check!( + MOUNTS, + libc::syscall(libc::SYS_pivot_root, c".".as_ptr(), c".".as_ptr()) + ); + check!(MOUNTS, libc::umount2(c".".as_ptr(), libc::MNT_DETACH)); + check!(MOUNTS, libc::chdir(c"/".as_ptr())); + + // Past the last capability the kernel answers EINVAL. + for cap in 0..64 { + if libc::prctl(libc::PR_CAPBSET_DROP, cap, 0, 0, 0) < 0 { + if cap > 0 && *libc::__errno_location() == libc::EINVAL { + break; + } + fail(report, CREDENTIALS); + } + } + check!( + CREDENTIALS, + libc::syscall(libc::SYS_setgroups, 0, ptr::null::()) + ); + check!( + CREDENTIALS, + libc::syscall(libc::SYS_setresgid, jail.gid, jail.gid, jail.gid) + ); + check!( + CREDENTIALS, + libc::syscall(libc::SYS_setresuid, jail.uid, jail.uid, jail.uid) + ); + check!( + CREDENTIALS, + libc::prctl( + libc::PR_CAP_AMBIENT, + libc::PR_CAP_AMBIENT_CLEAR_ALL, + 0, + 0, + 0 + ) + ); + // Leaving uid 0 already cleared them; this makes it explicit. + let header = CapHeader { + version: LINUX_CAPABILITY_VERSION_3, + pid: 0, + }; + let data = [CapData::default(); 2]; + check!( + CREDENTIALS, + libc::syscall(libc::SYS_capset, &header, data.as_ptr()) + ); + check!( + CREDENTIALS, + libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) + ); + for (resource, limit) in [ + (libc::RLIMIT_NOFILE, jail.nofile), + (libc::RLIMIT_NPROC, jail.nproc), + (libc::RLIMIT_CORE, 0), + ] { + let limit = libc::rlimit { + rlim_cur: limit, + rlim_max: limit, + }; + check!(LIMITS, libc::setrlimit(resource, &limit)); + } + + // Lift everything above the numbers handed out before placing it. + let keep = libc::fcntl(handoff.keep.as_raw_fd(), libc::F_DUPFD_CLOEXEC, 16); + let binary = libc::fcntl(binary, libc::F_DUPFD_CLOEXEC, 16); + let null = libc::fcntl(handoff.null.as_raw_fd(), libc::F_DUPFD_CLOEXEC, 16); + let stderr = libc::fcntl(handoff.stderr, libc::F_DUPFD_CLOEXEC, 16); + let lifted = libc::fcntl(report, libc::F_DUPFD_CLOEXEC, 16); + for fd in [keep, binary, null, stderr, lifted] { + check!(DESCRIPTORS, fd); + } + report = lifted; + check!(DESCRIPTORS, libc::dup2(null, 0)); + check!(DESCRIPTORS, libc::dup2(null, 1)); + check!(DESCRIPTORS, libc::dup2(stderr, 2)); + check!(DESCRIPTORS, libc::dup2(keep, 3)); + check!(DESCRIPTORS, libc::dup3(binary, 4, libc::O_CLOEXEC)); + check!(DESCRIPTORS, libc::dup3(report, 5, libc::O_CLOEXEC)); + report = 5; + check!( + DESCRIPTORS, + libc::syscall(libc::SYS_close_range, 6, u32::MAX, 0) + ); + + libc::syscall( + libc::SYS_execveat, + 4, + c"".as_ptr(), + argv.as_ptr(), + envp.as_ptr(), + libc::AT_EMPTY_PATH, + ); + fail(report, EXEC) +} + +unsafe fn fail(report: RawFd, step: i32) -> ! { + let errno = *libc::__errno_location(); + let mut bytes = [0u8; 8]; + bytes[..4].copy_from_slice(&step.to_le_bytes()); + bytes[4..].copy_from_slice(&errno.to_le_bytes()); + libc::write(report, bytes.as_ptr().cast(), bytes.len()); + libc::_exit(127) +} + +const LINUX_CAPABILITY_VERSION_3: u32 = 0x2008_0522; + +#[repr(C)] +struct CapHeader { + version: u32, + pid: libc::c_int, +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] +struct CapData { + effective: u32, + permitted: u32, + inheritable: u32, +} diff --git a/crates/shuru-node/src/lib.rs b/crates/shuru-node/src/lib.rs new file mode 100644 index 0000000..5ed9ac5 --- /dev/null +++ b/crates/shuru-node/src/lib.rs @@ -0,0 +1,10 @@ +//! A Linux node runs each sandbox VM in its own jailed `shuru-vmm` process, +//! and its network in a jailed `shuru-netd` (RFC 0008). `shuru-node` is the +//! only privileged process: it opens every file, creates every socket and +//! hands the jailed processes descriptors. + +pub mod cgroup; +pub mod control; +pub mod firewall; +pub mod jail; +pub mod seccomp; diff --git a/crates/shuru-node/src/main.rs b/crates/shuru-node/src/main.rs new file mode 100644 index 0000000..f9e48eb --- /dev/null +++ b/crates/shuru-node/src/main.rs @@ -0,0 +1,558 @@ +//! `shuru-node run` boots one jailed VM, runs a command in it and stops it, +//! the way a node runs each sandbox. + +use std::cell::Cell; +use std::collections::HashMap; +use std::ffi::CString; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::net::Ipv4Addr; +use std::os::fd::{AsFd, AsRawFd, FromRawFd, OwnedFd}; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::DirBuilderExt; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use anyhow::{bail, Context, Result}; +use clap::{Parser, Subcommand}; +use shuru_node::cgroup::{Limits, Sandbox}; +use shuru_node::control::{self, Disk, NetdConfig, Request, Response, Secret, VmmConfig}; +use shuru_node::firewall::{self, NETD_UID_BASE}; +use shuru_node::jail::{self, Handoff, Jail}; +use shuru_proto::hello::Identity; +use shuru_proto::{frame, ExecRequest}; + +const RUN_DIR: &str = "/run/shuru"; +const VMM_UID_BASE: u32 = 100_000; +const VSOCK_CONNECTIONS: usize = 128; +const VMM_NOFILE: u64 = 512; +// 256 proxied connections, each with a guest and an upstream side. +const NETD_NOFILE: u64 = 1024; +const NPROC: u64 = 64; +const SANDBOX_PIDS: u64 = 2 * NPROC; +const GUEST_PORT: u32 = 1024; +const BOOT_TIMEOUT: Duration = Duration::from_secs(10); +const INSTALL_CA: &str = + "printf '%s' \"$SHURU_PROXY_CA\" > /usr/local/share/ca-certificates/shuru-proxy.crt \ + && update-ca-certificates --fresh > /dev/null 2>&1"; + +#[derive(Parser)] +#[command(name = "shuru-node", version)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Boot a jailed VM, run one command in it and stop the VM. + Run(RunArgs), +} + +#[derive(clap::Args)] +struct RunArgs { + #[arg(long)] + kernel: PathBuf, + #[arg(long)] + initrd: PathBuf, + /// The working disk. The guest writes to it. + #[arg(long)] + disk: PathBuf, + #[arg(long, default_value_t = 2)] + cpus: usize, + /// Guest memory in MiB. + #[arg(long, default_value_t = 2048)] + memory: u64, + /// Picks the uids and the cgroup. One VM per slot at a time. + #[arg(long, default_value_t = 0)] + slot: u32, + /// Give the guest a network through a jailed shuru-netd. It reaches + /// public addresses only. + #[arg(long)] + allow_net: bool, + /// A domain the guest may reach, wildcards allowed. Without any, every + /// public address. + #[arg(long = "allow", requires = "allow_net")] + allow: Vec, + /// NAME@HOST[,HOST]: the guest sees a placeholder in $NAME, and the + /// value of $NAME here is sent in its place to those hosts only. + #[arg(long = "secret", requires = "allow_net")] + secrets: Vec, + #[arg(long, default_value = "8.8.8.8")] + dns: Ipv4Addr, + /// Roots for checking the hosts secrets are sent to. + #[arg(long, default_value = "/etc/ssl/certs/ca-certificates.crt")] + ca_bundle: PathBuf, + /// The directory holding the statically linked shuru-vmm and + /// shuru-netd. Defaults to the one holding this binary. + #[arg(long)] + bin_dir: Option, + /// Show the guest console, guest token included. + #[arg(long)] + verbose: bool, + /// Log calls the seccomp filters forbid instead of killing. + #[arg(long)] + seccomp_log: bool, + /// Print boot timings and peak memory to stderr. + #[arg(long)] + time: bool, + #[arg(last = true, required = true)] + command: Vec, +} + +fn main() { + let Command::Run(args) = Cli::parse().command; + match run(args) { + Ok(code) => std::process::exit(code), + Err(e) => { + eprintln!("shuru-node: {e:#}"); + std::process::exit(1); + } + } +} + +/// The sandbox directory: a control socket per jailed process, the vsock +/// socket, and the empty mount point for their roots. Only root can enter +/// it. +struct RunDir(PathBuf); + +impl RunDir { + fn create(slot: u32) -> Result { + let base = Path::new(RUN_DIR); + fs::DirBuilder::new() + .mode(0o700) + .recursive(true) + .create(base)?; + let path = base.join(format!("sbx-{slot}")); + fs::DirBuilder::new() + .mode(0o700) + .create(&path) + .with_context(|| format!("slot {slot} is in use: {}", path.display()))?; + let dir = RunDir(path); + fs::DirBuilder::new().mode(0o555).create(dir.root())?; + Ok(dir) + } + + fn root(&self) -> PathBuf { + self.0.join("root") + } + + fn control(&self, name: &str) -> PathBuf { + self.0.join(format!("{name}.sock")) + } + + fn vsock(&self) -> PathBuf { + self.0.join("vsock.sock") + } +} + +impl Drop for RunDir { + fn drop(&mut self) { + for name in ["vmm", "netd"] { + let _ = fs::remove_file(self.control(name)); + } + let _ = fs::remove_file(self.vsock()); + let _ = fs::remove_dir(self.root()); + let _ = fs::remove_dir(&self.0); + } +} + +/// A jailed process and its control connection. Killed and reaped when +/// dropped. +struct Jailed { + pid: i32, + reaped: Cell, + control: UnixStream, +} + +impl Jailed { + fn exited(&self) -> bool { + let mut status = 0; + if !self.reaped.get() + && unsafe { libc::waitpid(self.pid, &mut status, libc::WNOHANG) } == self.pid + { + self.reaped.set(true); + } + self.reaped.get() + } + + fn configure(&self, request: &Request, fds: &[i32]) -> Result { + control::send(&self.control, request, fds)?; + match control::receive::(&self.control, 0)? { + Some((Response::Error { message }, _)) => bail!("{message}"), + Some((response, _)) => Ok(response), + None => bail!("exited while starting"), + } + } + + fn stop(self) { + let _ = control::send(&self.control, &Request::Stop, &[]); + } +} + +impl Drop for Jailed { + // Once reaped the pid may belong to someone else. + fn drop(&mut self) { + if self.reaped.get() { + return; + } + let mut status = 0; + unsafe { + libc::kill(self.pid, libc::SIGKILL); + libc::waitpid(self.pid, &mut status, 0); + } + } +} + +struct Spawn<'a> { + name: &'static str, + uid: u32, + nofile: u64, + host_network: bool, + dir: &'a RunDir, + sandbox: &'a mut Sandbox, + bin_dir: &'a Path, + null: &'a File, + seccomp_log: bool, +} + +fn spawn(spawn: Spawn) -> Result { + let name = format!("shuru-{}", spawn.name); + let path = spawn.bin_dir.join(&name); + let binary = File::open(&path).with_context(|| format!("{}", path.display()))?; + let listener: OwnedFd = UnixListener::bind(spawn.dir.control(spawn.name))?.into(); + let procs = spawn.sandbox.child(spawn.name)?; + let root = CString::new(spawn.dir.root().as_os_str().as_bytes())?; + let argv0 = CString::new(name.as_str())?; + let mut argv = vec![argv0.as_c_str()]; + if spawn.seccomp_log { + argv.push(c"--seccomp-log"); + } + let pid = jail::spawn( + &Jail { + uid: spawn.uid, + gid: spawn.uid, + cgroup_procs: &procs, + root: &root, + nofile: spawn.nofile, + nproc: NPROC, + host_network: spawn.host_network, + }, + &binary, + &argv, + &Handoff { + keep: &listener, + null: spawn.null, + stderr: io::stderr().as_raw_fd(), + }, + ) + .with_context(|| format!("starting {name}"))?; + let reaped = Cell::new(false); + let control = match UnixStream::connect(spawn.dir.control(spawn.name)) { + Ok(control) => control, + Err(e) => { + drop(Jailed { + pid, + reaped, + control: UnixStream::pair()?.0, + }); + return Err(e.into()); + } + }; + Ok(Jailed { + pid, + reaped, + control, + }) +} + +/// `NAME@HOST[,HOST]`, valued from the environment of this process. +fn secret(spec: &str) -> Result<(String, Secret)> { + let (name, hosts) = spec + .split_once('@') + .with_context(|| format!("--secret {spec}: expected NAME@HOST[,HOST]"))?; + let value = + std::env::var(name).with_context(|| format!("--secret {spec}: ${name} is not set"))?; + let hosts = hosts.split(',').map(str::to_string).collect(); + Ok((name.to_string(), Secret { value, hosts })) +} + +struct Network { + netd: Jailed, + vm_end: OwnedFd, + mac: [u8; 6], + placeholders: HashMap, + ca_cert_pem: String, +} + +fn start_network(args: &RunArgs, netd: Jailed) -> Result { + let secrets = args + .secrets + .iter() + .map(|spec| secret(spec)) + .collect::>()?; + let upstream_roots = fs::read_to_string(&args.ca_bundle) + .with_context(|| format!("{}", args.ca_bundle.display()))?; + let (vm_end, host_end) = shuru_proxy::create_socketpair()?; + let (vm_end, host_end) = + unsafe { (OwnedFd::from_raw_fd(vm_end), OwnedFd::from_raw_fd(host_end)) }; + let config = NetdConfig { + dns_resolver: args.dns, + allow: args.allow.clone(), + secrets, + upstream_roots, + }; + let response = netd + .configure(&Request::ConfigureNetwork(config), &[host_end.as_raw_fd()]) + .context("shuru-netd failed")?; + let Response::Network { + placeholders, + ca_cert_pem, + } = response + else { + bail!("shuru-netd answered {response:?}"); + }; + let mut mac = [0u8; 6]; + getrandom::fill(&mut mac).map_err(|e| anyhow::anyhow!("{e}"))?; + mac[0] = (mac[0] & 0xfe) | 0x02; + Ok(Network { + netd, + vm_end, + mac, + placeholders, + ca_cert_pem, + }) +} + +fn run(args: RunArgs) -> Result { + let started = Instant::now(); + if unsafe { libc::geteuid() } != 0 { + bail!("shuru-node runs as root"); + } + let bin_dir = match &args.bin_dir { + Some(dir) => dir.clone(), + None => std::env::current_exe()? + .parent() + .context("no binary directory")? + .to_path_buf(), + }; + let kvm: OwnedFd = OpenOptions::new() + .read(true) + .write(true) + .open("/dev/kvm") + .context("/dev/kvm")? + .into(); + let open = |path: &Path| File::open(path).with_context(|| format!("{}", path.display())); + let kernel = open(&args.kernel)?; + let initrd = open(&args.initrd)?; + let disk = OpenOptions::new() + .read(true) + .write(true) + .open(&args.disk) + .with_context(|| format!("{}", args.disk.display()))?; + let null = OpenOptions::new() + .read(true) + .write(true) + .open("/dev/null")?; + let serial: OwnedFd = if args.verbose { + io::stderr().as_fd().try_clone_to_owned()? + } else { + null.try_clone()?.into() + }; + if args.allow_net { + firewall::install()?; + } + + let dir = RunDir::create(args.slot)?; + let vsock_listener = UnixListener::bind(dir.vsock())?; + let mut sandbox = Sandbox::create( + &args.slot.to_string(), + &Limits { + guest_memory_bytes: args.memory << 20, + cpus: args.cpus as u64, + pids: SANDBOX_PIDS, + }, + ) + .context("creating the sandbox cgroup")?; + + let network = if args.allow_net { + let netd = spawn(Spawn { + name: "netd", + uid: NETD_UID_BASE + args.slot, + nofile: NETD_NOFILE, + host_network: true, + dir: &dir, + sandbox: &mut sandbox, + bin_dir: &bin_dir, + null: &null, + seccomp_log: args.seccomp_log, + })?; + Some(start_network(&args, netd)?) + } else { + None + }; + let networked = started.elapsed(); + + let vmm = spawn(Spawn { + name: "vmm", + uid: VMM_UID_BASE + args.slot, + nofile: VMM_NOFILE, + host_network: false, + dir: &dir, + sandbox: &mut sandbox, + bin_dir: &bin_dir, + null: &null, + seccomp_log: args.seccomp_log, + })?; + let spawned = started.elapsed(); + + let mut identity = Identity { + token: [0; 32], + instance: [0; 16], + }; + getrandom::fill(&mut identity.token).map_err(|e| anyhow::anyhow!("{e}"))?; + getrandom::fill(&mut identity.instance).map_err(|e| anyhow::anyhow!("{e}"))?; + let hex = |bytes: &[u8]| bytes.iter().map(|b| format!("{b:02x}")).collect::(); + let quiet = if args.verbose { "" } else { " quiet" }; + let command_line = format!( + "console=ttyAMA0 root=/dev/vda rw{quiet} shuru.token={} shuru.instance={}", + hex(&identity.token), + hex(&identity.instance) + ); + + let config = VmmConfig { + cpus: args.cpus, + memory_bytes: args.memory << 20, + command_line, + initrd: true, + disk: Some(Disk { read_only: false }), + serial: true, + network_mac: network.as_ref().map(|network| network.mac), + vsock_connections: VSOCK_CONNECTIONS, + balloon: true, + }; + let mut fds = vec![ + kvm.as_raw_fd(), + kernel.as_raw_fd(), + initrd.as_raw_fd(), + disk.as_raw_fd(), + serial.as_raw_fd(), + ]; + if let Some(network) = &network { + fds.push(network.vm_end.as_raw_fd()); + } + fds.push(vsock_listener.as_raw_fd()); + vmm.configure(&Request::Configure(config), &fds) + .context("shuru-vmm failed")?; + drop((kvm, kernel, initrd, disk, serial, vsock_listener)); + let ready = started.elapsed(); + + let mut env = HashMap::new(); + if let Some(network) = &network { + if !network.placeholders.is_empty() { + let guest = connect_guest(&dir.vsock(), &identity, &vmm)?; + let ca = HashMap::from([("SHURU_PROXY_CA".to_string(), network.ca_cert_pem.clone())]); + let argv = ["sh", "-c", INSTALL_CA].map(str::to_string).to_vec(); + if exec(guest, argv, ca)? != 0 { + bail!("installing the proxy CA in the guest failed"); + } + env = network.placeholders.clone(); + } + } + let guest = connect_guest(&dir.vsock(), &identity, &vmm)?; + let answered = started.elapsed(); + let code = exec(guest, args.command, env)?; + + if args.time { + let peak = sandbox + .memory_peak() + .map_or("unknown".into(), |bytes| format!("{} MiB", bytes >> 20)); + eprintln!( + "shuru-node: network {} ms, jailed {} ms, VM running {} ms, guest answered {} ms, done {} ms, peak memory {peak}", + networked.as_millis(), + spawned.as_millis(), + ready.as_millis(), + answered.as_millis(), + started.elapsed().as_millis(), + ); + } + vmm.stop(); + if let Some(network) = network { + network.netd.stop(); + } + Ok(code) +} + +/// Connects to the guest agent through the VMM listener, retrying until +/// the agent listens. +fn connect_guest(path: &Path, identity: &Identity, vmm: &Jailed) -> Result { + let deadline = Instant::now() + BOOT_TIMEOUT; + loop { + if vmm.exited() { + bail!("the VMM exited before the guest answered"); + } + if Instant::now() > deadline { + bail!("the guest did not answer within {BOOT_TIMEOUT:?}"); + } + let mut stream = UnixStream::connect(path)?; + stream.set_read_timeout(Some(Duration::from_secs(2)))?; + stream.set_write_timeout(Some(Duration::from_secs(2)))?; + stream.write_all(format!("CONNECT {GUEST_PORT}\n").as_bytes())?; + if read_line(&mut stream)?.is_some_and(|line| line.starts_with("OK ")) { + identity + .connect(&mut stream) + .context("the guest did not answer the connection hello")?; + stream.set_read_timeout(None)?; + stream.set_write_timeout(None)?; + return Ok(stream); + } + std::thread::sleep(Duration::from_millis(5)); + } +} + +/// Reads up to a newline one byte at a time, so nothing after it is lost. +fn read_line(stream: &mut UnixStream) -> io::Result> { + let mut line = Vec::new(); + let mut byte = [0]; + while line.len() < 32 { + if stream.read(&mut byte)? == 0 { + return Ok(None); + } + if byte[0] == b'\n' { + return Ok(String::from_utf8(line).ok()); + } + line.push(byte[0]); + } + Ok(None) +} + +fn exec(mut guest: UnixStream, argv: Vec, env: HashMap) -> Result { + let request = ExecRequest { + argv, + env, + tty: None, + rows: None, + cols: None, + cwd: None, + id: None, + }; + frame::send_json(&mut guest, frame::EXEC_REQ, &request)?; + let mut stdout = io::stdout().lock(); + let mut stderr = io::stderr().lock(); + loop { + match frame::read_frame(&mut guest).context("reading from the guest")? { + Some((frame::STDOUT, payload)) => stdout.write_all(&payload)?, + Some((frame::STDERR, payload)) => stderr.write_all(&payload)?, + Some((frame::EXIT, payload)) => { + stdout.flush()?; + return Ok(frame::parse_exit_code(&payload).unwrap_or(1)); + } + Some((frame::ERROR, payload)) => { + bail!("guest error: {}", String::from_utf8_lossy(&payload)) + } + Some(_) => {} + None => bail!("the guest closed the connection"), + } + } +} diff --git a/crates/shuru-node/src/seccomp.rs b/crates/shuru-node/src/seccomp.rs new file mode 100644 index 0000000..5f5161b --- /dev/null +++ b/crates/shuru-node/src/seccomp.rs @@ -0,0 +1,285 @@ +//! Seccomp filters. The VMM gets an allowlist, installed on every thread +//! once the VM is built and before the first `KVM_RUN`. The network +//! process runs tokio and two TLS libraries, so it gets a deny list: no +//! exec, no files, no tracing, no namespaces, IP sockets only. + +use std::collections::BTreeMap; + +use anyhow::{Context, Result}; +use seccompiler::{ + apply_filter_all_threads, BpfProgram, SeccompAction, SeccompCmpArgLen as Len, + SeccompCmpOp as Op, SeccompCondition as Cond, SeccompFilter, SeccompRule, +}; + +const KVMIO: u64 = 0xae; +const IOC_WRITE: u64 = 1; + +const fn io(nr: u64) -> u64 { + (KVMIO << 8) | nr +} + +const fn iow(nr: u64, size: usize) -> u64 { + (IOC_WRITE << 30) | ((size as u64) << 16) | (KVMIO << 8) | nr +} + +const KVM_RUN: u64 = io(0x80); +const KVM_IRQ_LINE: u64 = iow(0x61, size_of::()); +const KVM_IOEVENTFD: u64 = iow(0x79, size_of::()); +// An unsigned long in glibc, an int in musl. +#[allow(clippy::unnecessary_cast)] +const FIONBIO: u64 = libc::FIONBIO as u64; +#[cfg(not(all(target_env = "musl", target_arch = "aarch64")))] +const SYS_KEXEC_FILE_LOAD: libc::c_long = libc::SYS_kexec_file_load; +// The libc crate leaves it out for musl on aarch64. +#[cfg(all(target_env = "musl", target_arch = "aarch64"))] +const SYS_KEXEC_FILE_LOAD: libc::c_long = 294; + +const ALLOWED: &[libc::c_long] = &[ + libc::SYS_accept4, + libc::SYS_brk, + libc::SYS_clock_gettime, + libc::SYS_clock_nanosleep, + libc::SYS_close, + libc::SYS_epoll_ctl, + libc::SYS_epoll_pwait, + libc::SYS_eventfd2, + libc::SYS_exit, + libc::SYS_exit_group, + libc::SYS_fallocate, + libc::SYS_fcntl, + libc::SYS_fdatasync, + libc::SYS_fsync, + libc::SYS_futex, + libc::SYS_getpid, + libc::SYS_getrandom, + libc::SYS_gettid, + libc::SYS_lseek, + libc::SYS_mremap, + libc::SYS_munmap, + libc::SYS_nanosleep, + // Answered with an error by the refusals filter. + libc::SYS_openat, + libc::SYS_ppoll, + libc::SYS_pread64, + libc::SYS_pwrite64, + libc::SYS_read, + libc::SYS_readv, + libc::SYS_recvfrom, + libc::SYS_recvmsg, + libc::SYS_restart_syscall, + libc::SYS_rseq, + libc::SYS_rt_sigaction, + libc::SYS_rt_sigprocmask, + libc::SYS_rt_sigreturn, + libc::SYS_sched_getaffinity, + libc::SYS_sched_yield, + libc::SYS_sendmsg, + libc::SYS_sendto, + libc::SYS_set_robust_list, + libc::SYS_setsockopt, + libc::SYS_shutdown, + libc::SYS_sigaltstack, + libc::SYS_tgkill, + libc::SYS_write, + libc::SYS_writev, +]; + +fn rule(conditions: Vec) -> Result { + Ok(SeccompRule::new(conditions)?) +} + +fn equals(arg: u8, value: u64) -> Result { + rule(vec![Cond::new(arg, Len::Qword, Op::Eq, value)?]) +} + +fn not_executable(arg: u8) -> Result { + let exec = libc::PROT_EXEC as u64; + rule(vec![Cond::new(arg, Len::Dword, Op::MaskedEq(exec), 0)?]) +} + +fn filter( + rules: BTreeMap>, + mismatch: SeccompAction, + matched: SeccompAction, +) -> Result { + let filter = SeccompFilter::new(rules, mismatch, matched, std::env::consts::ARCH.try_into()?)?; + Ok(filter.try_into()?) +} + +fn vmm_allowlist(mismatch: SeccompAction) -> Result { + let mut rules: BTreeMap> = + ALLOWED.iter().map(|&call| (call, Vec::new())).collect(); + rules.insert( + libc::SYS_ioctl, + [KVM_RUN, KVM_IRQ_LINE, KVM_IOEVENTFD, FIONBIO] + .into_iter() + .map(|request| equals(1, request)) + .collect::>()?, + ); + rules.insert(libc::SYS_mmap, vec![not_executable(2)?]); + rules.insert(libc::SYS_mprotect, vec![not_executable(2)?]); + rules.insert( + libc::SYS_madvise, + vec![equals(2, libc::MADV_DONTNEED as u64)?], + ); + let thread = libc::CLONE_THREAD as u64; + rules.insert( + libc::SYS_clone, + vec![rule(vec![Cond::new( + 0, + Len::Qword, + Op::MaskedEq(thread), + thread, + )?])?], + ); + rules.insert(libc::SYS_prctl, vec![equals(0, libc::PR_SET_NAME as u64)?]); + filter(rules, mismatch, SeccompAction::Allow) +} + +// clone3 takes its flags in memory a filter cannot read. Refusing it makes +// the C library fall back to clone, whose flags a filter can check. +fn refuse_clone3() -> Result { + let rules = BTreeMap::from([(libc::SYS_clone3, Vec::new())]); + filter( + rules, + SeccompAction::Allow, + SeccompAction::Errno(libc::ENOSYS as u32), + ) +} + +const NETD_DENIED: &[libc::c_long] = &[ + libc::SYS_acct, + libc::SYS_add_key, + libc::SYS_bpf, + libc::SYS_capset, + libc::SYS_chroot, + libc::SYS_delete_module, + libc::SYS_execve, + libc::SYS_execveat, + libc::SYS_fanotify_init, + libc::SYS_finit_module, + libc::SYS_init_module, + libc::SYS_io_uring_enter, + libc::SYS_io_uring_register, + libc::SYS_io_uring_setup, + SYS_KEXEC_FILE_LOAD, + libc::SYS_kexec_load, + libc::SYS_keyctl, + libc::SYS_mount, + libc::SYS_name_to_handle_at, + libc::SYS_open_by_handle_at, + libc::SYS_perf_event_open, + libc::SYS_personality, + libc::SYS_pivot_root, + libc::SYS_process_vm_readv, + libc::SYS_process_vm_writev, + libc::SYS_ptrace, + libc::SYS_quotactl, + libc::SYS_reboot, + libc::SYS_request_key, + libc::SYS_setgid, + libc::SYS_setgroups, + libc::SYS_setns, + libc::SYS_setregid, + libc::SYS_setresgid, + libc::SYS_setresuid, + libc::SYS_setreuid, + libc::SYS_setuid, + libc::SYS_swapoff, + libc::SYS_swapon, + libc::SYS_umount2, + libc::SYS_unshare, + libc::SYS_userfaultfd, +]; + +fn netd_kills(matched: SeccompAction) -> Result { + let mut rules: BTreeMap> = + NETD_DENIED.iter().map(|&call| (call, Vec::new())).collect(); + let new_user = libc::CLONE_NEWUSER as u64; + rules.insert( + libc::SYS_clone, + vec![rule(vec![Cond::new( + 0, + Len::Qword, + Op::MaskedEq(new_user), + new_user, + )?])?], + ); + filter(rules, SeccompAction::Allow, matched) +} + +// The roots are empty, so opening fails anyway. glibc opens files on its +// own, such as the CPU count when malloc adds an arena, and gets an error +// here instead of a kill. +fn refusals(mut rules: BTreeMap>) -> Result { + for call in [libc::SYS_openat, libc::SYS_openat2] { + rules.insert(call, Vec::new()); + } + filter( + rules, + SeccompAction::Allow, + SeccompAction::Errno(libc::EPERM as u32), + ) +} + +fn netd_refusals() -> Result { + let ip_only = rule(vec![ + Cond::new(0, Len::Dword, Op::Ne, libc::AF_INET as u64)?, + Cond::new(0, Len::Dword, Op::Ne, libc::AF_INET6 as u64)?, + ])?; + refusals(BTreeMap::from([(libc::SYS_socket, vec![ip_only])])) +} + +fn violation(log: bool) -> SeccompAction { + if log { + SeccompAction::Log + } else { + SeccompAction::KillProcess + } +} + +fn apply(programs: &[BpfProgram]) -> Result<()> { + for program in programs { + apply_filter_all_threads(program).context("installing a seccomp filter")?; + } + Ok(()) +} + +/// With `log`, violations are logged by the kernel and allowed, to find +/// what a change needs. +pub fn install_vmm(log: bool) -> Result<()> { + apply(&[ + refuse_clone3()?, + refusals(BTreeMap::new())?, + vmm_allowlist(violation(log))?, + ]) +} + +pub fn install_netd(log: bool) -> Result<()> { + apply(&[ + refuse_clone3()?, + netd_refusals()?, + netd_kills(violation(log))?, + ]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn kvm_requests_match_the_kernel() { + assert_eq!(KVM_RUN, 0xae80); + assert_eq!(KVM_IRQ_LINE, 0x4008_ae61); + assert_eq!(KVM_IOEVENTFD, 0x4040_ae79); + } + + #[test] + fn filters_compile() { + vmm_allowlist(SeccompAction::KillProcess).unwrap(); + refuse_clone3().unwrap(); + refusals(BTreeMap::new()).unwrap(); + netd_refusals().unwrap(); + netd_kills(SeccompAction::KillProcess).unwrap(); + } +} diff --git a/crates/shuru-proto/Cargo.toml b/crates/shuru-proto/Cargo.toml index d7dc7ff..5f01ffb 100644 --- a/crates/shuru-proto/Cargo.toml +++ b/crates/shuru-proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuru-proto" -version = "0.3.3" +version = "0.6.0" edition = "2021" description = "Shared wire protocol for shuru host/guest communication" license = "Apache-2.0" @@ -9,3 +9,4 @@ repository = "https://github.com/superhq-ai/shuru" [dependencies] serde = { version = "1", features = ["derive"] } serde_json = "1" +subtle = "2.6" diff --git a/crates/shuru-proto/src/frame.rs b/crates/shuru-proto/src/frame.rs index 0e9ec85..97e2cf3 100644 --- a/crates/shuru-proto/src/frame.rs +++ b/crates/shuru-proto/src/frame.rs @@ -1,5 +1,8 @@ use std::io::{self, Read, Write}; +pub const HELLO: u8 = 0x08; +pub const HELLO_OK: u8 = 0x09; + // I/O streams pub const STDIN: u8 = 0x01; pub const STDOUT: u8 = 0x02; @@ -19,6 +22,8 @@ pub const MOUNT_REQ: u8 = 0x11; pub const MOUNT_RESP: u8 = 0x12; // File I/O +// File contents travel in frames of at most FILE_CHUNK bytes. A read ends +// with an empty READ_FILE_RESP; a write sends WriteFileRequest::len bytes. pub const READ_FILE_REQ: u8 = 0x13; pub const READ_FILE_RESP: u8 = 0x14; pub const WRITE_FILE_REQ: u8 = 0x15; @@ -53,8 +58,14 @@ pub const DISCARD_RESP: u8 = 0x4F; pub const DOWNLOAD_REQ: u8 = 0x50; pub const DOWNLOAD_PROGRESS: u8 = 0x51; +// Shutdown: host asks the guest to stop its services and sync the disk +pub const SHUTDOWN_REQ: u8 = 0x60; +pub const SHUTDOWN_RESP: u8 = 0x61; + const MAX_FRAME: u32 = 1 << 20; // 1 MB +pub const FILE_CHUNK: usize = 256 * 1024; + /// Write a binary frame: `[u32 BE length][u8 type][payload]`. /// /// Assembles the header + payload into a single buffer so that the entire @@ -98,7 +109,7 @@ pub fn read_frame(r: &mut impl Read) -> io::Result)>> { /// Serialize `msg` as JSON and send it as a typed frame. pub fn send_json(w: &mut impl Write, msg_type: u8, msg: &impl serde::Serialize) -> io::Result<()> { - let payload = serde_json::to_vec(msg).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + let payload = serde_json::to_vec(msg).map_err(io::Error::other)?; write_frame(w, msg_type, &payload) } @@ -153,3 +164,14 @@ pub fn parse_exit_code(payload: &[u8]) -> Option { payload[0], payload[1], payload[2], payload[3], ])) } + +// Disk sealing uses a lease id so a stale thaw cannot release a later seal. +pub const SEAL_REQ: u8 = 0x62; +pub const SEAL_RESP: u8 = 0x63; +pub const THAW_REQ: u8 = 0x64; +pub const THAW_RESP: u8 = 0x65; +pub const TRIM_REQ: u8 = 0x66; +pub const TRIM_RESP: u8 = 0x67; + +pub const KILL_REQ: u8 = 0x18; +pub const KILL_RESP: u8 = 0x19; diff --git a/crates/shuru-proto/src/hello.rs b/crates/shuru-proto/src/hello.rs new file mode 100644 index 0000000..d9615c4 --- /dev/null +++ b/crates/shuru-proto/src/hello.rs @@ -0,0 +1,149 @@ +use std::io::{self, Read, Write}; + +use subtle::ConstantTimeEq; + +use crate::frame; + +pub const PROTOCOL_VERSION: u32 = 3; + +#[derive(Clone, Copy)] +pub struct Identity { + pub token: [u8; 32], + pub instance: [u8; 16], +} + +fn rejected() -> io::Error { + io::Error::new( + io::ErrorKind::PermissionDenied, + "guest identity handshake failed", + ) +} + +// Fixed length reads keep unauthenticated peers from allocating frame buffers. +fn read_hello(stream: &mut impl Read, kind: u8) -> io::Result<[u8; N]> { + let mut header = [0; 5]; + stream.read_exact(&mut header)?; + if u32::from_be_bytes(header[..4].try_into().unwrap()) != N as u32 + 1 || header[4] != kind { + return Err(rejected()); + } + let mut payload = [0; N]; + stream.read_exact(&mut payload)?; + Ok(payload) +} + +impl Identity { + pub fn accept(&self, stream: &mut (impl Read + Write)) -> io::Result<()> { + let token = read_hello::<32>(stream, frame::HELLO)?; + if !bool::from(self.token.ct_eq(&token)) { + return Err(rejected()); + } + let mut reply = [0; 20]; + reply[..16].copy_from_slice(&self.instance); + reply[16..].copy_from_slice(&PROTOCOL_VERSION.to_le_bytes()); + frame::write_frame(stream, frame::HELLO_OK, &reply) + } + + pub fn connect(&self, stream: &mut (impl Read + Write)) -> io::Result<()> { + frame::write_frame(stream, frame::HELLO, &self.token)?; + let reply = read_hello::<20>(stream, frame::HELLO_OK)?; + if reply[..16] != self.instance || reply[16..] != PROTOCOL_VERSION.to_le_bytes() { + return Err(rejected()); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + struct Stream { + input: Cursor>, + output: Vec, + } + impl Read for Stream { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.input.read(buf) + } + } + impl Write for Stream { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.output.write(buf) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + fn stream(kind: u8, payload: &[u8]) -> Stream { + let mut input = Vec::new(); + frame::write_frame(&mut input, kind, payload).unwrap(); + Stream { + input: Cursor::new(input), + output: Vec::new(), + } + } + const ID: Identity = Identity { + token: [42; 32], + instance: [7; 16], + }; + + #[test] + fn hello_roundtrip() { + let mut guest = stream(frame::HELLO, &ID.token); + ID.accept(&mut guest).unwrap(); + let mut host = Stream { + input: Cursor::new(guest.output), + output: Vec::new(), + }; + ID.connect(&mut host).unwrap(); + assert_eq!( + frame::read_frame(&mut host.output.as_slice()).unwrap(), + Some((frame::HELLO, ID.token.to_vec())) + ); + } + + #[test] + fn reject_wrong_token_type_length_and_truncation() { + for index in 0..32 { + let mut token = ID.token; + token[index] ^= 1; + let mut peer = stream(frame::HELLO, &token); + assert!(ID.accept(&mut peer).is_err()); + assert!(peer.output.is_empty()); + } + for (kind, payload) in [ + (frame::EXEC_REQ, ID.token.as_slice()), + (frame::HELLO, &[0][..]), + ] { + let mut peer = stream(kind, payload); + assert!(ID.accept(&mut peer).is_err()); + assert!(peer.output.is_empty()); + } + let mut peer = stream(frame::HELLO, &ID.token); + peer.input.get_mut().pop(); + assert!(ID.accept(&mut peer).is_err()); + } + + #[test] + fn reject_old_agent_and_wrong_protocol() { + assert!(ID + .connect(&mut stream(frame::HELLO_OK, &ID.instance)) + .is_err()); + let mut reply = [0; 20]; + reply[..16].copy_from_slice(&ID.instance); + reply[16..].copy_from_slice(&(PROTOCOL_VERSION - 1).to_le_bytes()); + assert!(ID.connect(&mut stream(frame::HELLO_OK, &reply)).is_err()); + reply[16..].copy_from_slice(&PROTOCOL_VERSION.to_le_bytes()); + reply[0] ^= 1; + assert!(ID.connect(&mut stream(frame::HELLO_OK, &reply)).is_err()); + } + + #[test] + fn reject_wrong_instance_and_reply_type() { + assert!(ID.connect(&mut stream(frame::HELLO_OK, &[0; 16])).is_err()); + assert!(ID + .connect(&mut stream(frame::STDOUT, &ID.instance)) + .is_err()); + } +} diff --git a/crates/shuru-proto/src/lib.rs b/crates/shuru-proto/src/lib.rs index 5f3cf8f..26e69e4 100644 --- a/crates/shuru-proto/src/lib.rs +++ b/crates/shuru-proto/src/lib.rs @@ -7,6 +7,7 @@ use std::collections::HashMap; /// Length = size of type byte + payload (excludes the 4-byte length prefix). /// Max frame size: 1 MB. pub mod frame; +pub mod hello; // --- Exec protocol --- @@ -23,6 +24,29 @@ pub struct ExecRequest { pub cols: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub cwd: Option, + /// Names the process for a KillRequest on another connection. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, +} + +/// Kill an exec by the id it was started with. Sent on its own connection, +/// so it arrives even when the exec connection is backed up with stdin. +#[derive(Debug, Serialize, Deserialize)] +pub struct KillRequest { + pub id: u64, +} + +// --- Shutdown protocol --- + +/// Reply to SHUTDOWN_REQ, sent once every guest service has exited and the +/// disk is synced. `killed` lists services that ignored SIGTERM and were +/// killed after their stop timeout. +#[derive(Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct ShutdownResponse { + #[serde(default)] + pub stopped: Vec, + #[serde(default)] + pub killed: Vec, } // --- Port forwarding protocol --- @@ -214,6 +238,17 @@ pub mod watch_kind { pub const MODIFY: u8 = 0x02; pub const DELETE: u8 = 0x03; pub const RENAME: u8 = 0x04; + + /// The name the stdio protocol and the SDKs use for a kind. + pub fn name(kind: u8) -> Option<&'static str> { + match kind { + CREATE => Some("create"), + MODIFY => Some("modify"), + DELETE => Some("delete"), + RENAME => Some("rename"), + _ => None, + } + } } /// A filesystem watch event. Binary format: `[u8 kind][path bytes...]` @@ -269,3 +304,30 @@ mod tests { assert!(!req2.read_only); } } + +#[derive(Debug, Serialize, Deserialize)] +pub struct SealRequest { + pub id: u64, + pub stop_services: bool, + pub lease_ms: u64, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ThawRequest { + pub id: u64, +} + +/// `Sealed` carries what the guest stopped when the seal asked it to stop +/// its services. +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub enum SealResponse { + Sealed(ShutdownResponse), + Thawed, + Expired, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct TrimResponse { + pub supported: bool, + pub bytes: u64, +} diff --git a/crates/shuru-proxy/Cargo.toml b/crates/shuru-proxy/Cargo.toml index 6cc4174..e9e4109 100644 --- a/crates/shuru-proxy/Cargo.toml +++ b/crates/shuru-proxy/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuru-proxy" -version = "0.3.0" +version = "0.5.0" edition = "2021" description = "Transparent network proxy for shuru microVMs" license = "Apache-2.0" diff --git a/crates/shuru-proxy/src/config.rs b/crates/shuru-proxy/src/config.rs index ab52016..cd8aba5 100644 --- a/crates/shuru-proxy/src/config.rs +++ b/crates/shuru-proxy/src/config.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::net::Ipv4Addr; +use std::net::{IpAddr, Ipv4Addr}; use std::path::PathBuf; use std::time::Duration; @@ -29,6 +29,14 @@ pub struct ProxyConfig { /// holding the resolved shuru.json, so relative paths in a config behave /// the same wherever shuru is invoked from. None inherits the process cwd. pub config_dir: Option, + /// Refuse connections to anything but public addresses: no loopback, + /// private, link-local or shared ranges, so a guest on a shared host + /// cannot reach the host, its network or a metadata service. Turns + /// `expose_host` off. + pub public_only: bool, + /// PEM roots for verifying upstream servers when relaying secrets. + /// None reads the system store from disk. + pub upstream_roots: Option>, } impl Default for ProxyConfig { @@ -39,6 +47,8 @@ impl Default for ProxyConfig { network: NetworkConfig::default(), expose_host: Vec::new(), config_dir: None, + public_only: false, + upstream_roots: None, } } } @@ -132,6 +142,37 @@ impl ProxyConfig { } } +/// False for loopback, private, link-local, shared, multicast, documentation +/// and reserved addresses. +pub fn is_public(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(ip) => { + let [a, b, c, _] = ip.octets(); + !(a == 0 + || a == 10 + || a == 127 + || (a == 100 && (64..128).contains(&b)) + || (a == 169 && b == 254) + || (a == 172 && (16..32).contains(&b)) + || (a == 192 && b == 0 && (c == 0 || c == 2)) + || (a == 192 && b == 88 && c == 99) + || (a == 192 && b == 168) + || (a == 198 && (b == 18 || b == 19)) + || (a == 198 && b == 51 && c == 100) + || (a == 203 && b == 0 && c == 113) + || a >= 224) + } + IpAddr::V6(ip) => match ip.to_ipv4_mapped() { + Some(v4) => is_public(IpAddr::V4(v4)), + // Global unicast, minus the IANA special block and documentation. + None => { + let [a, b, ..] = ip.segments(); + a & 0xe000 == 0x2000 && !(a == 0x2001 && (b < 0x0200 || b == 0x0db8)) + } + }, + } +} + /// Simple wildcard domain matching. /// "*" matches any domain (catch-all). /// "*.example.com" matches "api.example.com" but not "example.com". @@ -212,6 +253,43 @@ mod tests { assert!(with_entries.has_allowlist()); } + #[test] + fn public_addresses() { + for public in [ + "1.1.1.1", + "8.8.8.8", + "100.128.0.1", + "172.32.0.1", + "2606:4700::1111", + "2001:4860::8888", + ] { + assert!(is_public(public.parse().unwrap()), "{public}"); + } + for private in [ + "0.0.0.0", + "10.0.0.1", + "100.64.0.1", + "127.0.0.1", + "169.254.169.254", + "172.16.0.1", + "172.31.255.255", + "192.0.0.170", + "192.168.1.1", + "198.18.0.1", + "224.0.0.1", + "255.255.255.255", + "::1", + "::", + "fe80::1", + "fd00:ec2::254", + "::ffff:169.254.169.254", + "2001:db8::1", + "ff02::1", + ] { + assert!(!is_public(private.parse().unwrap()), "{private}"); + } + } + #[test] fn default_dns_resolver_is_google_public_dns() { assert_eq!( diff --git a/crates/shuru-proxy/src/device.rs b/crates/shuru-proxy/src/device.rs index 6f7d3ed..0bcc8f3 100644 --- a/crates/shuru-proxy/src/device.rs +++ b/crates/shuru-proxy/src/device.rs @@ -1,12 +1,46 @@ use std::collections::VecDeque; +use std::io; use std::os::unix::io::RawFd; use smoltcp::phy::{self, Checksum, Device, DeviceCapabilities, Medium}; use smoltcp::time::Instant; -const MTU: usize = 1514; // 14-byte Ethernet header + 1500-byte IP payload +// 14-byte Ethernet header + 65535-byte IP payload. Must match the IP-MTU +// passed to `FileHandleNetworkAttachment::new` in `shuru-vm::sandbox.rs`. +// Jumbo frames collapse the per-frame syscall + checksum + smoltcp TCP +// processing cost by ~40× vs the 1500-byte default. +pub(crate) const MTU: usize = 65549; +// At 65 KiB per frame, this caps transient buffer usage to ~16 MiB +// (`MAX_PENDING_FRAMES × MTU`), still freed each poll cycle. Real ceiling +// on RX is the kernel SO_RCVBUF (8 MiB) set in `lib.rs::create_socketpair`, +// which itself holds ~128 super-frames before back-pressuring the guest. const MAX_PENDING_FRAMES: usize = 256; +/// Result of pulling a single frame from the host AF_UNIX socket. +enum RecvOne { + /// A frame was read into the buffer. + Frame(Vec), + /// `recv()` returned `EAGAIN`/`EWOULDBLOCK`: the socket is fully drained. + WouldBlock, + /// `recv()` returned a real error or 0-byte EOF. + Error(io::Error), +} + +/// Outcome of [`VZDevice::drain_recv`]. Drives readiness bookkeeping in the +/// event-driven poll loop: we may only `clear_ready()` on the host FD when +/// the kernel has actually told us `WouldBlock`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DrainStatus { + /// Drained until `recv()` returned `EWOULDBLOCK` — no more frames. + WouldBlock, + /// Stopped early because `pending_rx` reached `MAX_PENDING_FRAMES`. The + /// kernel buffer may still hold data; the loop must re-poll without + /// clearing FD readiness. + HitCap, + /// `recv()` returned an error (other than `EWOULDBLOCK`) or EOF. + Error, +} + /// smoltcp Device backed by a Unix datagram socketpair fd. /// /// One end of the socketpair is given to VZFileHandleNetworkDeviceAttachment @@ -28,8 +62,20 @@ impl VZDevice { } } + /// The host end of the AF_UNIX socketpair. Used by the network stack to + /// register readiness with the async runtime. + pub fn fd(&self) -> RawFd { + self.fd + } + + /// Whether any pre-read frames are still queued for smoltcp to consume. + pub fn pending_rx_empty(&self) -> bool { + self.pending_rx.is_empty() + } + /// Non-blocking read of a single frame from the socketpair. - fn recv_one_frame(&mut self) -> Option> { + /// Distinguishes "no data right now" (`WouldBlock`) from real errors. + fn recv_one_frame(&mut self) -> RecvOne { let n = unsafe { libc::recv( self.fd, @@ -38,10 +84,18 @@ impl VZDevice { libc::MSG_DONTWAIT, ) }; - if n <= 0 { - return None; + if n > 0 { + return RecvOne::Frame(self.recv_buf[..n as usize].to_vec()); + } + if n == 0 { + return RecvOne::Error(io::Error::new(io::ErrorKind::UnexpectedEof, "fd closed")); + } + let err = io::Error::last_os_error(); + // On Darwin/Linux EAGAIN == EWOULDBLOCK; match on the value once. + match err.raw_os_error() { + Some(libc::EAGAIN) => RecvOne::WouldBlock, + _ => RecvOne::Error(err), } - Some(self.recv_buf[..n as usize].to_vec()) } /// Drain all available frames from the socketpair (non-blocking). @@ -51,14 +105,19 @@ impl VZDevice { /// DNS (UDP 53). Without this filter, smoltcp auto-replies to ICMP /// echo requests for any IP (due to any_ip=true), which leaks a /// response channel even though no data actually reaches the internet. - pub fn drain_recv(&mut self) { + pub fn drain_recv(&mut self) -> DrainStatus { while self.pending_rx.len() < MAX_PENDING_FRAMES { match self.recv_one_frame() { - Some(frame) if Self::is_icmp(&frame) => continue, - Some(frame) => self.pending_rx.push_back(frame), - None => break, + RecvOne::Frame(frame) if Self::is_icmp(&frame) => continue, + RecvOne::Frame(frame) => self.pending_rx.push_back(frame), + RecvOne::WouldBlock => return DrainStatus::WouldBlock, + RecvOne::Error(e) => { + tracing::debug!("host_fd recv error: {e}"); + return DrainStatus::Error; + } } } + DrainStatus::HitCap } /// Check if a raw Ethernet frame carries an ICMP packet. @@ -120,19 +179,11 @@ impl Device for VZDevice { type TxToken<'a> = VZTxToken; fn receive(&mut self, _timestamp: Instant) -> Option<(Self::RxToken<'_>, Self::TxToken<'_>)> { - // First drain pre-read frames, then try reading directly from the - // socketpair for frames that arrived during poll. Drop ICMP in both - // paths to match the filter in drain_recv(). - loop { - let buffer = self - .pending_rx - .pop_front() - .or_else(|| self.recv_one_frame())?; - if Self::is_icmp(&buffer) { - continue; - } - return Some((VZRxToken { buffer }, VZTxToken { fd: self.fd })); - } + // Only frames pre-read by drain_recv(). The stack adds a listener for + // each SYN among them before smoltcp sees it, so a frame read here + // directly would have its SYN answered with a reset. + let buffer = self.pending_rx.pop_front()?; + Some((VZRxToken { buffer }, VZTxToken { fd: self.fd })) } fn transmit(&mut self, _timestamp: Instant) -> Option> { @@ -202,4 +253,131 @@ mod tests { assert!(!VZDevice::is_icmp(&[])); assert!(!VZDevice::is_icmp(&[0u8; 23])); // too short to read proto field } + + /// RAII socketpair: `(host_fd, peer_fd)` closed on drop. The host_fd is + /// the end given to a `VZDevice`; the peer_fd is the test driver's end. + struct Pair { + host_fd: RawFd, + peer_fd: RawFd, + } + + impl Pair { + fn new() -> Self { + let mut fds = [0i32; 2]; + let r = + unsafe { libc::socketpair(libc::AF_UNIX, libc::SOCK_DGRAM, 0, fds.as_mut_ptr()) }; + assert_eq!(r, 0, "socketpair failed"); + // Generous buffer so we can queue >MAX_PENDING_FRAMES frames. + let bufsize: libc::c_int = 16 * 1024 * 1024; + for fd in fds.iter() { + unsafe { + libc::setsockopt( + *fd, + libc::SOL_SOCKET, + libc::SO_SNDBUF, + &bufsize as *const _ as _, + std::mem::size_of::() as _, + ); + libc::setsockopt( + *fd, + libc::SOL_SOCKET, + libc::SO_RCVBUF, + &bufsize as *const _ as _, + std::mem::size_of::() as _, + ); + } + } + Pair { + host_fd: fds[1], + peer_fd: fds[0], + } + } + + fn send(&self, frame: &[u8]) { + let n = unsafe { + libc::send( + self.peer_fd, + frame.as_ptr() as *const libc::c_void, + frame.len(), + 0, + ) + }; + assert!(n > 0, "send failed: {}", std::io::Error::last_os_error()); + } + } + + impl Drop for Pair { + fn drop(&mut self) { + unsafe { + libc::close(self.host_fd); + libc::close(self.peer_fd); + } + } + } + + #[test] + fn drain_recv_returns_wouldblock_on_empty_fd() { + let pair = Pair::new(); + let mut device = VZDevice::new(pair.host_fd); + assert_eq!(device.drain_recv(), DrainStatus::WouldBlock); + assert!(device.pending_rx_empty()); + } + + #[test] + fn drain_recv_accumulates_non_icmp_frames() { + let pair = Pair::new(); + let frame = make_frame(6); // TCP + for _ in 0..3 { + pair.send(&frame); + } + let mut device = VZDevice::new(pair.host_fd); + assert_eq!(device.drain_recv(), DrainStatus::WouldBlock); + assert_eq!(device.pending_rx.len(), 3); + } + + #[test] + fn drain_recv_discards_icmp_frames() { + let pair = Pair::new(); + pair.send(&make_frame(1)); // ICMP — should be dropped + pair.send(&make_frame(6)); // TCP — should be kept + pair.send(&make_frame(1)); // ICMP — should be dropped + pair.send(&make_frame(17)); // UDP — should be kept + let mut device = VZDevice::new(pair.host_fd); + assert_eq!(device.drain_recv(), DrainStatus::WouldBlock); + assert_eq!(device.pending_rx.len(), 2); + } + + #[test] + fn drain_recv_returns_hit_cap_when_pending_rx_fills() { + let pair = Pair::new(); + let frame = make_frame(6); // TCP + // Send slightly more than MAX_PENDING_FRAMES so HitCap is unambiguous. + for _ in 0..(MAX_PENDING_FRAMES + 16) { + pair.send(&frame); + } + let mut device = VZDevice::new(pair.host_fd); + assert_eq!(device.drain_recv(), DrainStatus::HitCap); + assert_eq!(device.pending_rx.len(), MAX_PENDING_FRAMES); + // A second drain should pick up the rest and end at WouldBlock. + device.pending_rx.clear(); + let status = device.drain_recv(); + assert!(matches!( + status, + DrainStatus::WouldBlock | DrainStatus::HitCap + )); + } + + #[test] + fn receive_only_yields_inspected_frames() { + let pair = Pair::new(); + let frame = make_frame(6); + pair.send(&frame); + let mut device = VZDevice::new(pair.host_fd); + assert_eq!(device.drain_recv(), DrainStatus::WouldBlock); + pair.send(&frame); + assert!(device.receive(Instant::ZERO).is_some()); + assert!(device.receive(Instant::ZERO).is_none()); + assert_eq!(device.drain_recv(), DrainStatus::WouldBlock); + assert_eq!(device.pending_rx.len(), 1); + } } diff --git a/crates/shuru-proxy/src/lib.rs b/crates/shuru-proxy/src/lib.rs index 21e76b7..4e4a0dc 100644 --- a/crates/shuru-proxy/src/lib.rs +++ b/crates/shuru-proxy/src/lib.rs @@ -73,27 +73,55 @@ pub fn create_socketpair() -> anyhow::Result<(RawFd, RawFd)> { let host_fd = fds[1]; - // Apple recommends SO_RCVBUF >= 2x SO_SNDBUF for VZFileHandleNetworkDeviceAttachment - unsafe { - let sndbuf: libc::c_int = 1024 * 1024; - let rcvbuf: libc::c_int = 4 * 1024 * 1024; - libc::setsockopt( - host_fd, - libc::SOL_SOCKET, - libc::SO_SNDBUF, - &sndbuf as *const _ as _, - std::mem::size_of::() as _, + // Apple recommends SO_RCVBUF at least 2x SO_SNDBUF (4x preferred) for + // VZFileHandleNetworkDeviceAttachment. With the 65535-byte IP-MTU set on + // the attachment (see `shuru-vm::sandbox.rs`) each datagram can be one + // 64 KiB frame, so the receive buffer decides how many frames can be in + // flight before the guest is back-pressured. 6 MiB holds about 96 of + // them and stays under the macOS kern.ipc.maxsockbuf default of 8 MiB, + // which the kernel applies per buffer with some headroom removed; a + // request at the limit fails and leaves the AF_UNIX default of 4 KiB, + // too small to carry a single frame. The values are read back so a + // capped socket is reported instead of silently crippling the network. + let sndbuf = set_sockbuf(host_fd, libc::SO_SNDBUF, 2 * 1024 * 1024); + let rcvbuf = set_sockbuf(host_fd, libc::SO_RCVBUF, 6 * 1024 * 1024); + let frame = device::MTU as libc::c_int; + if rcvbuf < frame || sndbuf < frame { + return Err(anyhow::anyhow!( + "host socket buffers too small for a {} byte frame (SO_SNDBUF {}, SO_RCVBUF {}); raise kern.ipc.maxsockbuf", + frame, + sndbuf, + rcvbuf + )); + } + if rcvbuf < 4 * frame { + tracing::warn!( + "SO_RCVBUF is {} bytes, under four frames; guest throughput will be limited", + rcvbuf ); + } + + Ok((fds[0], fds[1])) +} + +/// Request a socket buffer size and return what the kernel actually set, +/// which may be smaller or unchanged when the request exceeds its limit. +fn set_sockbuf(fd: RawFd, opt: libc::c_int, want: libc::c_int) -> libc::c_int { + unsafe { libc::setsockopt( - host_fd, + fd, libc::SOL_SOCKET, - libc::SO_RCVBUF, - &rcvbuf as *const _ as _, + opt, + &want as *const _ as _, std::mem::size_of::() as _, ); + let mut got: libc::c_int = 0; + let mut len: libc::socklen_t = std::mem::size_of::() as _; + if libc::getsockopt(fd, libc::SOL_SOCKET, opt, &mut got as *mut _ as _, &mut len) != 0 { + return 0; + } + got } - - Ok((fds[0], fds[1])) } /// Start the proxy engine. Returns a handle that keeps it running. @@ -105,6 +133,7 @@ pub fn start(host_fd: RawFd, config: ProxyConfig) -> anyhow::Result let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let ca = CertificateAuthority::new()?; + let upstream_ssl = proxy::upstream_connector(&config)?; let ca_cert_pem = ca.ca_cert_pem(); // Generate placeholder tokens for each secret @@ -147,6 +176,7 @@ pub fn start(host_fd: RawFd, config: ProxyConfig) -> anyhow::Result ca, proxy_placeholders, proxy_allowed_ips, + upstream_ssl, ); engine.run().await; }); diff --git a/crates/shuru-proxy/src/proxy.rs b/crates/shuru-proxy/src/proxy.rs index 6434361..aaa0998 100644 --- a/crates/shuru-proxy/src/proxy.rs +++ b/crates/shuru-proxy/src/proxy.rs @@ -3,6 +3,8 @@ use std::net::SocketAddr; use std::sync::Arc; use boring::ssl::{SslConnector, SslMethod}; +use boring::x509::store::X509StoreBuilder; +use boring::x509::X509; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; use tokio::sync::mpsc; @@ -12,7 +14,7 @@ use crate::config::ProxyConfig; use crate::dns; use crate::secrets::SecretResolver; use crate::stack::{ConnectionId, StackCommand, StackEvent, TcpConnection}; -use crate::stream::ChannelStream; +use crate::stream::{ChannelStream, GuestRx, GuestTx}; use crate::substitute::RequestStream; use crate::tls::CertificateAuthority; use crate::AllowedIps; @@ -30,16 +32,35 @@ const SECRET_REFRESH_INTERVAL: std::time::Duration = std::time::Duration::from_s /// Cloudflare-protected sites accept the TLS fingerprint. The client-side /// (guest <-> proxy) uses rustls with our generated CA cert. pub struct ProxyEngine { - config: Arc, event_rx: mpsc::UnboundedReceiver, cmd_tx: mpsc::UnboundedSender, connections: HashMap>>, - secrets: Arc, - ca: Arc>, + shared: Arc, +} + +/// State every connection task reads. +struct Shared { + config: Arc, + ca: tokio::sync::Mutex, + secrets: SecretResolver, upstream_ssl: SslConnector, allowed_ips: AllowedIps, } +/// The guest side of one proxied connection. +struct Guest { + id: ConnectionId, + rx: GuestRx, + tx: GuestTx, + cmd_tx: mpsc::UnboundedSender, +} + +impl Guest { + fn close(&self) { + let _ = self.cmd_tx.send(StackCommand::Close { id: self.id }); + } +} + impl ProxyEngine { pub fn new( config: ProxyConfig, @@ -48,25 +69,22 @@ impl ProxyEngine { ca: CertificateAuthority, placeholders: HashMap, allowed_ips: AllowedIps, + upstream_ssl: SslConnector, ) -> Self { - // BoringSSL upstream connector — Chrome's TLS stack so Cloudflare - // doesn't reject our MITM connections based on JA3/JA4 fingerprint. - let mut builder = SslConnector::builder(SslMethod::tls()).expect("SslConnector"); - builder.set_alpn_protos(b"\x08http/1.1").expect("ALPN"); - let upstream_ssl = builder.build(); - let config = Arc::new(config); - let secrets = Arc::new(SecretResolver::new(config.clone(), placeholders)); + let secrets = SecretResolver::new(config.clone(), placeholders); ProxyEngine { - config, event_rx, cmd_tx, connections: HashMap::new(), - secrets, - ca: Arc::new(tokio::sync::Mutex::new(ca)), - upstream_ssl, - allowed_ips, + shared: Arc::new(Shared { + config, + ca: tokio::sync::Mutex::new(ca), + secrets, + upstream_ssl, + allowed_ips, + }), } } @@ -88,12 +106,23 @@ impl ProxyEngine { StackEvent::Closed { id } => { self.connections.remove(&id); } - StackEvent::DnsQuery { src, payload } => { + StackEvent::DnsQuery { + src, + payload, + permit, + } => { let cmd_tx = self.cmd_tx.clone(); - let config = self.config.clone(); - let allowed_ips = self.allowed_ips.clone(); + let shared = self.shared.clone(); tokio::spawn(async move { - dns::handle_dns_query(src, payload, cmd_tx, &config, &allowed_ips).await; + dns::handle_dns_query( + src, + payload, + cmd_tx, + &shared.config, + &shared.allowed_ips, + ) + .await; + drop(permit); }); } } @@ -103,46 +132,54 @@ impl ProxyEngine { fn handle_new_connection(&mut self, conn: TcpConnection) { let (data_tx, data_rx) = mpsc::unbounded_channel(); self.connections.insert(conn.id, data_tx); - - let cmd_tx = self.cmd_tx.clone(); - let config = self.config.clone(); - let ca = self.ca.clone(); - let secrets = self.secrets.clone(); - let upstream_ssl = self.upstream_ssl.clone(); - let allowed_ips = self.allowed_ips.clone(); + let guest = Guest { + id: conn.id, + rx: GuestRx::new(conn.id, data_rx, self.cmd_tx.clone()), + tx: GuestTx::new(conn.id, self.cmd_tx.clone(), conn.to_guest), + cmd_tx: self.cmd_tx.clone(), + }; + let shared = self.shared.clone(); tokio::spawn(async move { - if let Err(e) = handle_connection( - conn.id, - conn.dst, - data_rx, - cmd_tx, - &config, - ca, - secrets, - upstream_ssl, - &allowed_ips, - ) - .await - { + if let Err(e) = handle_connection(guest, conn.dst, &shared).await { debug!("connection to {} ended: {e}", conn.dst); } }); } } +/// Upstream TLS goes through BoringSSL, the TLS stack of Chrome, so Cloudflare +/// does not reject relayed connections by their JA3/JA4 fingerprint. +pub fn upstream_connector(config: &ProxyConfig) -> anyhow::Result { + let mut builder = SslConnector::builder(SslMethod::tls())?; + builder.set_alpn_protos(b"\x08http/1.1")?; + if let Some(pem) = &config.upstream_roots { + let certs = X509::stack_from_pem(pem)?; + if certs.is_empty() { + anyhow::bail!("no certificates in the upstream roots"); + } + let mut store = X509StoreBuilder::new()?; + for cert in certs { + store.add_cert(cert)?; + } + builder.set_cert_store_builder(store); + } + Ok(builder.build()) +} + /// Handle a single proxied TCP connection. async fn handle_connection( - id: ConnectionId, + mut guest: Guest, dst: SocketAddr, - mut data_rx: mpsc::UnboundedReceiver>, - cmd_tx: mpsc::UnboundedSender, - config: &ProxyConfig, - ca: Arc>, - secrets: Arc, - upstream_ssl: SslConnector, - allowed_ips: &AllowedIps, + shared: &Shared, ) -> anyhow::Result<()> { + let config = &shared.config; + if config.public_only && !crate::config::is_public(dst.ip()) { + guest.close(); + return Err(anyhow::anyhow!( + "connection to {dst} blocked: not a public address" + )); + } // Check if this is a connection to an exposed host port (host.shuru.internal). if let std::net::IpAddr::V4(ipv4) = dst.ip() { if let Some(host_port) = config.exposed_host_port(ipv4, dst.port()) { @@ -157,7 +194,7 @@ async fn handle_connection( ); let upstream = TcpStream::connect(local_dst).await?; let (mut upstream_rd, mut upstream_wr) = upstream.into_split(); - return blind_relay(id, &mut upstream_rd, &mut upstream_wr, data_rx, cmd_tx).await; + return blind_relay(guest, &mut upstream_rd, &mut upstream_wr).await; } } @@ -172,14 +209,15 @@ async fn handle_connection( if let std::net::IpAddr::V4(ipv4) = dst.ip() { const GATEWAY: std::net::Ipv4Addr = std::net::Ipv4Addr::new(10, 0, 0, 1); let is_allowed = ipv4 == GATEWAY - || allowed_ips + || shared + .allowed_ips .write() .expect("allowed_ips lock poisoned") .get(&ipv4) .is_some(); if !is_allowed { debug!("IP not in DNS-pinned set, rejecting: {dst}"); - let _ = cmd_tx.send(StackCommand::Close { id }); + guest.close(); return Err(anyhow::anyhow!( "connection to {dst} blocked: IP not resolved via allowed DNS" )); @@ -192,7 +230,8 @@ async fn handle_connection( if is_tls { // Buffer data until we have a complete TLS ClientHello record. // The ClientHello may span multiple TCP segments. - let mut tls_buf = data_rx + let mut tls_buf = guest + .rx .recv() .await .ok_or_else(|| anyhow::anyhow!("connection closed before data"))?; @@ -204,7 +243,7 @@ async fn handle_connection( if tls_buf.len() >= 5 + record_len { break; // have the complete record } - match data_rx.recv().await { + match guest.rx.recv().await { Some(chunk) => tls_buf.extend_from_slice(&chunk), None => break, // connection closed } @@ -220,14 +259,14 @@ async fn handle_connection( match &sni { Some(domain) if !config.is_domain_allowed(domain) => { debug!("SNI not in allowlist, rejecting: {domain}"); - let _ = cmd_tx.send(StackCommand::Close { id }); + guest.close(); return Err(anyhow::anyhow!( "TLS to {dst} blocked: SNI '{domain}' not in allowlist" )); } None => { debug!("no SNI in ClientHello, rejecting connection to {dst}"); - let _ = cmd_tx.send(StackCommand::Close { id }); + guest.close(); return Err(anyhow::anyhow!("TLS to {dst} blocked: no SNI")); } _ => {} @@ -237,31 +276,19 @@ async fn handle_connection( if let Some(domain) = sni { // Resolved per connection, so a rotated secret is picked up by the // next connection to the domain without the guest noticing. - let substitutions = match secrets.substitutions_for_domain(&domain).await { + let substitutions = match shared.secrets.substitutions_for_domain(&domain).await { Ok(substitutions) => substitutions, Err(e) => { // Warn rather than debug: the guest only sees a dropped // TLS connection, so this is the sole account of why. warn!("TLS to {domain} blocked: {e:#}"); - let _ = cmd_tx.send(StackCommand::Close { id }); + guest.close(); return Err(e.context(format!("TLS to {domain} blocked"))); } }; if !substitutions.is_empty() { debug!("MITM: {domain}"); - return handle_mitm( - id, - dst, - domain, - tls_buf, - data_rx, - cmd_tx, - ca, - secrets, - substitutions, - upstream_ssl, - ) - .await; + return handle_mitm(guest, dst, domain, tls_buf, substitutions, shared).await; } } @@ -273,7 +300,7 @@ async fn handle_connection( // Send the buffered TLS data upstream_wr.write_all(&tls_buf).await?; - return blind_relay(id, &mut upstream_rd, &mut upstream_wr, data_rx, cmd_tx).await; + return blind_relay(guest, &mut upstream_rd, &mut upstream_wr).await; } // Non-TLS: blind tunnel @@ -281,31 +308,28 @@ async fn handle_connection( let upstream = TcpStream::connect(dst).await?; let (mut upstream_rd, mut upstream_wr) = upstream.into_split(); - blind_relay(id, &mut upstream_rd, &mut upstream_wr, data_rx, cmd_tx).await + blind_relay(guest, &mut upstream_rd, &mut upstream_wr).await } /// Blind bidirectional relay (no inspection). async fn blind_relay( - id: ConnectionId, + guest: Guest, upstream_rd: &mut tokio::net::tcp::OwnedReadHalf, upstream_wr: &mut tokio::net::tcp::OwnedWriteHalf, - mut data_rx: mpsc::UnboundedReceiver>, - cmd_tx: mpsc::UnboundedSender, ) -> anyhow::Result<()> { - let cmd_tx_clone = cmd_tx.clone(); + let Guest { + id, + rx: mut guest_rx, + tx: guest_tx, + cmd_tx, + } = guest; let upstream_to_guest = async { let mut buf = vec![0u8; 65536]; loop { match upstream_rd.read(&mut buf).await { Ok(0) | Err(_) => break, Ok(n) => { - if cmd_tx_clone - .send(StackCommand::Send { - id, - payload: buf[..n].to_vec(), - }) - .is_err() - { + if guest_tx.send(&buf[..n]).await.is_err() { break; } } @@ -314,7 +338,7 @@ async fn blind_relay( }; let guest_to_upstream = async { - while let Some(payload) = data_rx.recv().await { + while let Some(payload) = guest_rx.recv().await { if upstream_wr.write_all(&payload).await.is_err() { break; } @@ -332,23 +356,21 @@ async fn blind_relay( /// MITM: terminate TLS on both sides, relay HTTP/1.1 with secret substitution. async fn handle_mitm( - id: ConnectionId, + guest: Guest, dst: SocketAddr, domain: String, first_chunk: Vec, - data_rx: mpsc::UnboundedReceiver>, - cmd_tx: mpsc::UnboundedSender, - ca: Arc>, - secrets: Arc, substitutions: Vec<(String, String)>, - upstream_ssl: SslConnector, + shared: &Shared, ) -> anyhow::Result<()> { let acceptor = { - let mut ca = ca.lock().await; + let mut ca = shared.ca.lock().await; ca.acceptor_for_domain(&domain)? }; - let mut guest_stream = ChannelStream::new(id, data_rx, cmd_tx.clone()); + let Guest { id, rx, tx, cmd_tx } = guest; + let secrets = &shared.secrets; + let mut guest_stream = ChannelStream::new(rx, tx); guest_stream.prepend(first_chunk); // Client-side: rustls with our generated CA cert @@ -356,9 +378,10 @@ async fn handle_mitm( // Upstream: BoringSSL — Chrome's TLS fingerprint passes Cloudflare let upstream_tcp = TcpStream::connect(dst).await?; - let upstream_tls = tokio_boring::connect(upstream_ssl.configure()?, &domain, upstream_tcp) - .await - .map_err(|e| anyhow::anyhow!("BoringSSL connect to {domain}: {e}"))?; + let upstream_tls = + tokio_boring::connect(shared.upstream_ssl.configure()?, &domain, upstream_tcp) + .await + .map_err(|e| anyhow::anyhow!("BoringSSL connect to {domain}: {e}"))?; // HTTP/1.1 text-based relay with secret substitution let (mut guest_rd, mut guest_wr) = tokio::io::split(guest_tls); @@ -507,4 +530,19 @@ mod tests { assert_eq!(extract_sni(b"GET / HTTP/1.1\r\n"), None); assert_eq!(extract_sni(&[]), None); } + + #[test] + fn upstream_roots_must_parse() { + let pem = CertificateAuthority::new().unwrap().ca_cert_pem(); + let config = ProxyConfig { + upstream_roots: Some(pem), + ..Default::default() + }; + assert!(upstream_connector(&config).is_ok()); + let config = ProxyConfig { + upstream_roots: Some(b"not a certificate".to_vec()), + ..Default::default() + }; + assert!(upstream_connector(&config).is_err()); + } } diff --git a/crates/shuru-proxy/src/stack.rs b/crates/shuru-proxy/src/stack.rs index 50aadc8..e967d77 100644 --- a/crates/shuru-proxy/src/stack.rs +++ b/crates/shuru-proxy/src/stack.rs @@ -1,6 +1,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::os::unix::io::RawFd; +use std::sync::Arc; use smoltcp::iface::{Config, Interface, SocketHandle, SocketSet}; use smoltcp::socket::tcp::{self, Socket as TcpSocket}; @@ -10,10 +11,12 @@ use smoltcp::wire::{ EthernetAddress, EthernetFrame, HardwareAddress, IpAddress, IpCidr, IpEndpoint, IpListenEndpoint, Ipv4Address, Ipv4Packet, TcpPacket, }; -use tokio::sync::mpsc; +use tokio::io::unix::AsyncFd; +use tokio::io::Interest; +use tokio::sync::{mpsc, OwnedSemaphorePermit, Semaphore}; use tracing::{debug, warn}; -use crate::device::VZDevice; +use crate::device::{DrainStatus, VZDevice}; /// Gateway IP inside the virtual network (host-side smoltcp). pub const GATEWAY_IP: Ipv4Address = Ipv4Address::new(10, 0, 0, 1); @@ -21,18 +24,50 @@ const PREFIX_LEN: u8 = 24; /// Gateway MAC address (locally administered). const GATEWAY_MAC: EthernetAddress = EthernetAddress([0x02, 0x00, 0x00, 0x00, 0x00, 0x01]); -const TCP_RX_BUF_SIZE: usize = 65536; -const TCP_TX_BUF_SIZE: usize = 65536; +// Per-connection smoltcp ring buffers, allocated on every SYN the guest +// sends. They only span the hop between guest and host, under a millisecond; +// the remote round trip is the host kernel socket. 512 KiB and 256 KiB +// measured the same single-stream throughput as 1 MiB and 512 KiB (about 2 +// Gbit/s down, 9 up on an M-series Mac), 256 KiB and 128 KiB lost a fifth. +const TCP_RX_BUF_SIZE: usize = 512 * 1024; +const TCP_TX_BUF_SIZE: usize = 256 * 1024; +// Hard ceiling on live TCP sockets, listening or connected. At the buffer +// sizes above this bounds the netstack at 384 MiB no matter what the guest +// does; SYNs beyond it are dropped and the guest retransmits. +const MAX_TCP_SOCKETS: usize = 512; + +// Bytes in flight between the stack and the engine on one connection, in +// each direction. Toward the host, the stack leaves data in smoltcp once +// this many bytes wait on the engine, so a stalled remote closes the +// window of the guest; each message also counts MESSAGE_COST, so trickled +// one-byte segments cannot queue unbounded numbers of messages. Toward the +// guest, a producer waits for permits that come back as the stack moves +// bytes into smoltcp, so a guest that stops reading stops the upstream read. +pub(crate) const TO_HOST_BUDGET: usize = 128 * 1024; +pub(crate) const MESSAGE_COST: usize = 256; +pub(crate) const TO_GUEST_BUDGET: usize = 128 * 1024; +const RECV_CHUNK: usize = 64 * 1024; +// DNS queries being resolved at once; more are dropped and the guest +// resolver retries. +const MAX_DNS_INFLIGHT: usize = 64; +// A listener whose SYN never completes a handshake is reaped after this. +const LISTEN_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); +// A socket the host closed is dropped once the guest is silent this long. +const CLOSE_TIMEOUT: smoltcp::time::Duration = smoltcp::time::Duration::from_secs(60); /// A new TCP connection from the guest, ready to be proxied. pub struct TcpConnection { pub id: ConnectionId, pub dst: SocketAddr, + /// Permits for bytes toward the guest, TO_GUEST_BUDGET in total. + pub to_guest: Arc, } -/// Opaque handle to a guest-side TCP connection inside smoltcp. +/// Opaque handle to a guest-side TCP connection inside smoltcp. smoltcp +/// reuses a handle once its socket is removed, so the serial tells a later +/// connection apart from the one a late command was meant for. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct ConnectionId(SocketHandle); +pub struct ConnectionId(SocketHandle, u64); /// Events from the network stack to the proxy engine. pub enum StackEvent { @@ -42,14 +77,22 @@ pub enum StackEvent { Data { id: ConnectionId, payload: Vec }, /// Guest closed the connection. Closed { id: ConnectionId }, - /// A DNS query arrived on UDP port 53. - DnsQuery { src: IpEndpoint, payload: Vec }, + /// A DNS query arrived on UDP port 53. The permit is held until it is + /// answered. + DnsQuery { + src: IpEndpoint, + payload: Vec, + permit: OwnedSemaphorePermit, + }, } /// Commands from the proxy engine back to the network stack. pub enum StackCommand { - /// Send data to the guest on an established connection. + /// Send data to the guest on an established connection. The sender + /// holds a permit for each byte. Send { id: ConnectionId, payload: Vec }, + /// The engine took `n` bytes of guest data off the connection. + Consumed { id: ConnectionId, n: usize }, /// Close a connection from the host side. Close { id: ConnectionId }, /// Send a DNS response back to the guest. @@ -72,6 +115,16 @@ pub struct NetworkStack { pending_send: HashMap>, /// Sockets waiting for pending_send to drain before closing. closing: HashSet, + /// Sockets the host closed, removed once smoltcp finishes with them. + draining: HashSet, + /// When each listener was created. + listen_since: HashMap, + /// Guest bytes, plus MESSAGE_COST per message, the engine has not taken. + to_host: HashMap, + to_guest: HashMap>, + serials: HashMap, + next_serial: u64, + dns_inflight: Arc, /// Reusable buffer for poll_tcp_sockets recv_slice. recv_buf: Vec, /// Reusable scratch map for inspect_pending_frames. @@ -124,80 +177,161 @@ impl NetworkStack { listening: HashMap::new(), pending_send: HashMap::new(), closing: HashSet::new(), - recv_buf: vec![0u8; TCP_RX_BUF_SIZE], + draining: HashSet::new(), + listen_since: HashMap::new(), + to_host: HashMap::new(), + to_guest: HashMap::new(), + serials: HashMap::new(), + next_serial: 0, + dns_inflight: Arc::new(Semaphore::new(MAX_DNS_INFLIGHT)), + recv_buf: vec![0u8; RECV_CHUNK], syn_scratch: HashMap::new(), event_tx, cmd_rx, } } - /// Run the poll loop. Blocks the current thread. + /// Run the poll loop. Blocks the current thread until the proxy engine + /// drops `cmd_tx` or the host FD closes. + /// + /// Internally this hosts a Tokio current-thread runtime so the loop can + /// `select!` over (a) host FD readiness via [`AsyncFd`], (b) commands on + /// `cmd_rx`, and (c) smoltcp's own timer (`poll_delay`). This replaces a + /// previous `thread::sleep(1ms)` busy-wait that bounded the smoltcp ↔ + /// guest internal RTT and capped throughput at ~1 MB/s regardless of + /// buffer sizes. pub fn run(&mut self) { - loop { - self.process_commands(); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_io() + .enable_time() + .build() + .expect("failed to build current-thread runtime for netstack"); + rt.block_on(self.run_async()); + } - // Drain all available frames and inspect for TCP SYN - self.device.drain_recv(); - self.inspect_pending_frames(); + async fn run_async(&mut self) { + let async_fd = match AsyncFd::with_interest(self.device.fd(), Interest::READABLE) { + Ok(fd) => fd, + Err(e) => { + warn!("AsyncFd registration failed for host_fd: {e}"); + return; + } + }; + loop { + // Synchronous work phase: drain everything available right now. + // (Commands are drained inside the select! arm so they share a + // single waker; this top-of-loop drain is just for frames.) + let drain_status = self.device.drain_recv(); + self.reap_sockets(); + self.inspect_pending_frames(); self.iface .poll(Self::now(), &mut self.device, &mut self.sockets); - self.poll_tcp_sockets(); self.drain_pending_sends(); self.poll_dns_socket(); - let delay = self - .iface - .poll_delay(Self::now(), &self.sockets) - .map(|d| { - let micros = d.total_micros(); - if micros == 0 { - std::time::Duration::from_millis(1) - } else { - std::time::Duration::from_micros(micros.min(50_000)) - } - }) - .unwrap_or(std::time::Duration::from_millis(1)); + let poll_delay = self.iface.poll_delay(Self::now(), &self.sockets); + + // If there is still local work, do not await — loop immediately. + // This covers: drain stopped at MAX_PENDING_FRAMES with the + // kernel buffer still non-empty, smoltcp wants an immediate + // re-poll, or pre-read frames remain queued (e.g. produced by + // smoltcp itself during the previous poll). + let immediate = matches!(drain_status, DrainStatus::HitCap) + || !self.device.pending_rx_empty() + || matches!(poll_delay, Some(d) if d.total_micros() == 0); + if immediate { + tokio::task::yield_now().await; + continue; + } - std::thread::sleep(delay); + // Convert smoltcp's poll_delay into a tokio sleep deadline. + let sleep_micros = poll_delay.map(|d| d.total_micros()); + + tokio::select! { + biased; + // (1) Frame arrived from guest. + ready = async_fd.readable() => { + match ready { + Ok(mut guard) => { + // Acknowledge readiness; the next loop iteration + // will drain via MSG_DONTWAIT. If we are wrong and + // the FD isn't really readable yet, drain_recv() + // will return WouldBlock and we re-await. + guard.clear_ready(); + } + Err(e) => { + warn!("AsyncFd readable error: {e}"); + return; + } + } + } + // (2) Command arrived from proxy engine. Drain the rest with + // try_recv() so a burst of commands costs one wake. + maybe_cmd = self.cmd_rx.recv() => { + match maybe_cmd { + Some(cmd) => { + self.handle_command(cmd); + while let Ok(cmd) = self.cmd_rx.try_recv() { + self.handle_command(cmd); + } + } + None => { + // Proxy engine dropped its cmd_tx — shut down. + return; + } + } + } + // (3) smoltcp's own timer (retransmit / delayed ACK). + _ = sleep_or_pending(sleep_micros) => {} + } } } - fn process_commands(&mut self) { - while let Ok(cmd) = self.cmd_rx.try_recv() { - match cmd { - StackCommand::Send { id, payload } => { - // Ignore sends for connections already removed by poll_tcp_sockets - if !self.connections.contains_key(&id.0) { - continue; - } - self.pending_send.entry(id.0).or_default().extend(&payload); + fn handle_command(&mut self, cmd: StackCommand) { + let id = match &cmd { + StackCommand::Send { id, .. } + | StackCommand::Consumed { id, .. } + | StackCommand::Close { id } => Some(*id), + StackCommand::DnsResponse { .. } => None, + }; + if id.is_some_and(|id| self.serials.get(&id.0) != Some(&id.1)) { + return; + } + match cmd { + StackCommand::Send { id, payload } => { + // Ignore sends for connections already removed by poll_tcp_sockets + if !self.connections.contains_key(&id.0) { + return; } - StackCommand::Close { id } => { - // Connection may have already been cleaned up by poll_tcp_sockets - if !self.connections.contains_key(&id.0) { - self.pending_send.remove(&id.0); - self.closing.remove(&id.0); - continue; - } - // Defer close until pending_send is drained so we don't - // drop data that hasn't been pushed to smoltcp yet. - if self.pending_send.get(&id.0).map_or(true, |p| p.is_empty()) { - // Nothing pending — close immediately - let socket = self.sockets.get_mut::(id.0); - socket.close(); - self.connections.remove(&id.0); - self.pending_send.remove(&id.0); - } else { - self.closing.insert(id.0); - } + self.pending_send.entry(id.0).or_default().extend(&payload); + } + StackCommand::Consumed { id, n } => { + if let Some(in_flight) = self.to_host.get_mut(&id.0) { + *in_flight = in_flight.saturating_sub(n + MESSAGE_COST); } - StackCommand::DnsResponse { dst, payload } => { - let socket = self.sockets.get_mut::(self.dns_handle); - if let Err(e) = socket.send_slice(&payload, dst) { - warn!("failed to send DNS response: {e}"); - } + } + StackCommand::Close { id } => { + // Connection may have already been cleaned up by poll_tcp_sockets + if !self.connections.contains_key(&id.0) { + return; + } + // Defer close until pending_send is drained so we don't + // drop data that has not been pushed to smoltcp yet. A guest + // that stops answering is timed out by smoltcp. + if self.pending_send.get(&id.0).is_none_or(|p| p.is_empty()) { + self.close_socket(id.0); + } else { + let socket = self.sockets.get_mut::(id.0); + socket.set_timeout(Some(CLOSE_TIMEOUT)); + self.closing.insert(id.0); + } + } + StackCommand::DnsResponse { dst, payload } => { + let socket = self.sockets.get_mut::(self.dns_handle); + if let Err(e) = socket.send_slice(&payload, dst) { + warn!("failed to send DNS response: {e}"); } } } @@ -215,6 +349,7 @@ impl NetworkStack { // Drain into a local vec to release &mut self borrow on syn_scratch let syn_entries: Vec<_> = self.syn_scratch.drain().collect(); + let mut live_sockets = self.sockets.iter().count(); for ((dst_ip, dst_port), syn_count) in syn_entries { let existing = self .listening @@ -223,6 +358,13 @@ impl NetworkStack { let needed = syn_count.saturating_sub(existing); for _ in 0..needed { + if live_sockets >= MAX_TCP_SOCKETS { + warn!( + "SYN → {}:{} dropped: {} TCP sockets live, limit {}", + dst_ip, dst_port, live_sockets, MAX_TCP_SOCKETS + ); + break; + } debug!("SYN → {}:{}, adding listener", dst_ip, dst_port); let rx_buf = tcp::SocketBuffer::new(vec![0u8; TCP_RX_BUF_SIZE]); @@ -239,6 +381,8 @@ impl NetworkStack { continue; } let handle = self.sockets.add(socket); + self.listen_since.insert(handle, std::time::Instant::now()); + live_sockets += 1; self.listening .entry((dst_ip, dst_port)) .or_default() @@ -286,6 +430,9 @@ impl NetworkStack { Ok(0) => break, Ok(n) => { pending.drain(..n); + if let Some(budget) = self.to_guest.get(&handle) { + budget.add_permits(n); + } } Err(e) => { warn!("failed to send to guest: {e}"); @@ -298,11 +445,8 @@ impl NetworkStack { } // If this socket was waiting to close and all data is now flushed, // initiate the TCP close (FIN). - if drained_empty && self.closing.remove(&handle) { - let socket = self.sockets.get_mut::(handle); - socket.close(); - self.connections.remove(&handle); - self.pending_send.remove(&handle); + if drained_empty && self.closing.contains(&handle) { + self.close_socket(handle); } } } @@ -319,14 +463,17 @@ impl NetworkStack { for handle in handles { let socket = self.sockets.get_mut::(handle); - // LISTEN → ESTABLISHED transition - if socket.is_active() && !self.connections.contains_key(&handle) { + // Handed to the proxy once the handshake completes. A socket + // closed in SYN-RECEIVED never sends its FIN. + let connected = matches!( + socket.state(), + tcp::State::Established | tcp::State::CloseWait + ); + if connected && !self.connections.contains_key(&handle) { // local_endpoint() is the destination the guest was trying to reach // (because any_ip=true, smoltcp accepted it as local) if let Some(local) = socket.local_endpoint() { - let ipv4 = match local.addr { - IpAddress::Ipv4(ip) => ip, - }; + let IpAddress::Ipv4(ipv4) = local.addr; // Remove this specific handle from the listener vec if let Some(handles) = self.listening.get_mut(&(ipv4, local.port)) { handles.retain(|h| *h != handle); @@ -338,24 +485,38 @@ impl NetworkStack { let actual_dst = SocketAddr::new(IpAddr::V4(Ipv4Addr::from(ipv4.octets())), local.port); + self.listen_since.remove(&handle); + self.next_serial += 1; + self.serials.insert(handle, self.next_serial); self.connections.insert(handle, actual_dst); + self.to_host.insert(handle, 0); + let to_guest = Arc::new(Semaphore::new(TO_GUEST_BUDGET)); + self.to_guest.insert(handle, to_guest.clone()); let _ = self.event_tx.send(StackEvent::NewConnection(TcpConnection { - id: ConnectionId(handle), + id: ConnectionId(handle, self.next_serial), dst: actual_dst, + to_guest, })); } } - // Read data from established connections - if self.connections.contains_key(&handle) && socket.can_recv() { - match socket.recv_slice(&mut self.recv_buf) { - Ok(n) if n > 0 => { + // Read data from established connections, up to the budget. + let room = self + .to_host + .get(&handle) + .map_or(0, |in_flight| TO_HOST_BUDGET.saturating_sub(*in_flight)); + if room > MESSAGE_COST && socket.can_recv() { + let max = (room - MESSAGE_COST).min(self.recv_buf.len()); + if let Ok(n) = socket.recv_slice(&mut self.recv_buf[..max]) { + if n > 0 { + if let Some(in_flight) = self.to_host.get_mut(&handle) { + *in_flight += n + MESSAGE_COST; + } let _ = self.event_tx.send(StackEvent::Data { - id: ConnectionId(handle), + id: ConnectionId(handle, self.serials[&handle]), payload: self.recv_buf[..n].to_vec(), }); } - _ => {} } } @@ -365,11 +526,9 @@ impl NetworkStack { && !socket.may_recv() && !socket.may_send() { - self.connections.remove(&handle); - self.pending_send.remove(&handle); - let _ = self.event_tx.send(StackEvent::Closed { - id: ConnectionId(handle), - }); + let id = ConnectionId(handle, self.serials[&handle]); + self.forget(handle); + let _ = self.event_tx.send(StackEvent::Closed { id }); self.sockets.remove(handle); } } @@ -380,17 +539,72 @@ impl NetworkStack { let mut buf = [0u8; 4096]; while socket.can_recv() { match socket.recv_slice(&mut buf) { - Ok((n, meta)) => { - let _ = self.event_tx.send(StackEvent::DnsQuery { - src: meta.endpoint, - payload: buf[..n].to_vec(), - }); - } + Ok((n, meta)) => match self.dns_inflight.clone().try_acquire_owned() { + Ok(permit) => { + let _ = self.event_tx.send(StackEvent::DnsQuery { + src: meta.endpoint, + payload: buf[..n].to_vec(), + permit, + }); + } + Err(_) => debug!("DNS query dropped: {MAX_DNS_INFLIGHT} in flight"), + }, Err(_) => break, } } } + /// Stop tracking a connection. Producers waiting to send to it fail. + fn forget(&mut self, handle: SocketHandle) { + self.connections.remove(&handle); + self.pending_send.remove(&handle); + self.closing.remove(&handle); + self.to_host.remove(&handle); + self.serials.remove(&handle); + if let Some(budget) = self.to_guest.remove(&handle) { + budget.close(); + } + } + + /// Start a host-side close. The socket stays in the set until smoltcp + /// finishes the close or the guest goes silent for CLOSE_TIMEOUT. + fn close_socket(&mut self, handle: SocketHandle) { + let socket = self.sockets.get_mut::(handle); + socket.set_timeout(Some(CLOSE_TIMEOUT)); + socket.close(); + self.forget(handle); + self.draining.insert(handle); + } + + /// Remove closed sockets and listeners whose handshake never came. + fn reap_sockets(&mut self) { + let sockets = &mut self.sockets; + self.draining.retain(|handle| { + let state = sockets.get::(*handle).state(); + let done = matches!(state, tcp::State::Closed | tcp::State::TimeWait); + if done { + sockets.remove(*handle); + } + !done + }); + let since = &mut self.listen_since; + self.listening.retain(|_, handles| { + handles.retain(|handle| { + let state = sockets.get::(*handle).state(); + let stale = matches!(state, tcp::State::Listen | tcp::State::SynReceived) + && since + .get(handle) + .is_some_and(|t| t.elapsed() >= LISTEN_TIMEOUT); + if stale { + since.remove(handle); + sockets.remove(*handle); + } + !stale + }); + !handles.is_empty() + }); + } + fn now() -> Instant { let ts = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -398,3 +612,14 @@ impl NetworkStack { Instant::from_micros(ts.as_micros() as i64) } } + +/// Sleep for `micros` microseconds, or never resolve if `None`. +/// Used as the third arm of `select!` so the loop wakes when smoltcp's own +/// timer fires (delayed ACK / retransmit) but stays asleep otherwise. +async fn sleep_or_pending(micros: Option) { + match micros { + Some(0) => {} // immediate; loop will continue without awaiting wakers + Some(us) => tokio::time::sleep(std::time::Duration::from_micros(us)).await, + None => std::future::pending::<()>().await, + } +} diff --git a/crates/shuru-proxy/src/stream.rs b/crates/shuru-proxy/src/stream.rs index 022e28a..11a84d8 100644 --- a/crates/shuru-proxy/src/stream.rs +++ b/crates/shuru-proxy/src/stream.rs @@ -1,36 +1,123 @@ +use std::future::Future; use std::io; use std::pin::Pin; +use std::sync::Arc; use std::task::{Context, Poll}; use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, AcquireError, OwnedSemaphorePermit, Semaphore}; use crate::stack::{ConnectionId, StackCommand}; -/// AsyncRead + AsyncWrite adapter over smoltcp channel-based data flow. -/// -/// Allows tokio-rustls (and any other async I/O) to work with data -/// arriving from the smoltcp NetworkStack via channels. -pub struct ChannelStream { +/// Largest single send toward the guest. Must not exceed TO_GUEST_BUDGET. +const SEND_CHUNK: usize = 64 * 1024; + +/// Guest data for one connection. Taking a payload returns its bytes to +/// the budget of the stack, so the stack reads more from the guest only as fast +/// as the connection consumes it. +pub struct GuestRx { id: ConnectionId, data_rx: mpsc::UnboundedReceiver>, cmd_tx: mpsc::UnboundedSender, - read_buf: Vec, - read_pos: usize, } -impl ChannelStream { +impl GuestRx { pub fn new( id: ConnectionId, data_rx: mpsc::UnboundedReceiver>, cmd_tx: mpsc::UnboundedSender, ) -> Self { - ChannelStream { + GuestRx { id, data_rx, cmd_tx, + } + } + + pub async fn recv(&mut self) -> Option> { + std::future::poll_fn(|cx| self.poll_recv(cx)).await + } + + fn poll_recv(&mut self, cx: &mut Context<'_>) -> Poll>> { + let polled = self.data_rx.poll_recv(cx); + if let Poll::Ready(Some(data)) = &polled { + let _ = self.cmd_tx.send(StackCommand::Consumed { + id: self.id, + n: data.len(), + }); + } + polled + } +} + +/// Sends toward the guest. Every byte holds a permit until the stack moves +/// it into the smoltcp socket, so a guest that stops reading makes `send` +/// wait instead of queueing. +#[derive(Clone)] +pub struct GuestTx { + id: ConnectionId, + cmd_tx: mpsc::UnboundedSender, + budget: Arc, +} + +impl GuestTx { + pub fn new( + id: ConnectionId, + cmd_tx: mpsc::UnboundedSender, + budget: Arc, + ) -> Self { + GuestTx { id, cmd_tx, budget } + } + + pub async fn send(&self, data: &[u8]) -> io::Result<()> { + for chunk in data.chunks(SEND_CHUNK) { + self.budget + .acquire_many(chunk.len() as u32) + .await + .map_err(|_| closed())? + .forget(); + self.enqueue(chunk)?; + } + Ok(()) + } + + fn enqueue(&self, chunk: &[u8]) -> io::Result<()> { + self.cmd_tx + .send(StackCommand::Send { + id: self.id, + payload: chunk.to_vec(), + }) + .map_err(|_| closed()) + } +} + +fn closed() -> io::Error { + io::Error::new(io::ErrorKind::BrokenPipe, "connection closed") +} + +type Acquire = Pin> + Send>>; + +/// AsyncRead + AsyncWrite adapter over smoltcp channel-based data flow. +/// +/// Allows tokio-rustls (and any other async I/O) to work with data +/// arriving from the smoltcp NetworkStack via channels. +pub struct ChannelStream { + rx: GuestRx, + tx: GuestTx, + read_buf: Vec, + read_pos: usize, + /// Permits being acquired for a write, and how many. + acquire: Option<(Acquire, usize)>, +} + +impl ChannelStream { + pub fn new(rx: GuestRx, tx: GuestTx) -> Self { + ChannelStream { + rx, + tx, read_buf: Vec::new(), read_pos: 0, + acquire: None, } } @@ -68,7 +155,7 @@ impl AsyncRead for ChannelStream { } // Receive from channel - match self.data_rx.poll_recv(cx) { + match self.rx.poll_recv(cx) { Poll::Ready(Some(data)) => { let n = data.len().min(buf.remaining()); buf.put_slice(&data[..n]); @@ -86,20 +173,33 @@ impl AsyncRead for ChannelStream { impl AsyncWrite for ChannelStream { fn poll_write( - self: Pin<&mut Self>, - _cx: &mut Context<'_>, + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, buf: &[u8], ) -> Poll> { - match self.cmd_tx.send(StackCommand::Send { - id: self.id, - payload: buf.to_vec(), - }) { - Ok(()) => Poll::Ready(Ok(buf.len())), - Err(_) => Poll::Ready(Err(io::Error::new( - io::ErrorKind::BrokenPipe, - "stack channel closed", - ))), + if buf.is_empty() { + return Poll::Ready(Ok(0)); + } + let this = &mut *self; + let (acquire, want) = this.acquire.get_or_insert_with(|| { + let want = buf.len().min(SEND_CHUNK); + let budget = this.tx.budget.clone(); + (Box::pin(budget.acquire_many_owned(want as u32)), want) + }); + let permit = match acquire.as_mut().poll(cx) { + Poll::Pending => return Poll::Pending, + Poll::Ready(result) => result, + }; + let want = *want; + this.acquire = None; + permit.map_err(|_| closed())?.forget(); + // The caller may retry with a shorter buffer than the permits cover. + let n = want.min(buf.len()); + if want > n { + this.tx.budget.add_permits(want - n); } + this.tx.enqueue(&buf[..n])?; + Poll::Ready(Ok(n)) } fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { @@ -107,7 +207,7 @@ impl AsyncWrite for ChannelStream { } fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { - let _ = self.cmd_tx.send(StackCommand::Close { id: self.id }); + let _ = self.tx.cmd_tx.send(StackCommand::Close { id: self.tx.id }); Poll::Ready(Ok(())) } } diff --git a/crates/shuru-proxy/src/substitute.rs b/crates/shuru-proxy/src/substitute.rs index 8261cee..f24bdb1 100644 --- a/crates/shuru-proxy/src/substitute.rs +++ b/crates/shuru-proxy/src/substitute.rs @@ -20,6 +20,8 @@ /// Cap on a single request head. A head larger than this is tunnelled rather /// than buffered without bound. const MAX_HEAD: usize = 64 * 1024; +/// Longest chunk-size or trailer line kept while looking for its end. +const MAX_CHUNK_LINE: usize = 4096; const HEAD_END: &[u8] = b"\r\n\r\n"; @@ -233,6 +235,10 @@ impl Chunked { } } None => { + // Not consuming makes the caller tunnel the rest. + if line.len() + rest.len() > MAX_CHUNK_LINE { + return (pos, false); + } line.extend_from_slice(rest); pos = data.len(); } @@ -268,6 +274,9 @@ impl Chunked { } } None => { + if line.len() + rest.len() > MAX_CHUNK_LINE { + return (pos, false); + } line.extend_from_slice(rest); pos = data.len(); } diff --git a/crates/shuru-sdk/Cargo.toml b/crates/shuru-sdk/Cargo.toml index 377a37e..3eb6493 100644 --- a/crates/shuru-sdk/Cargo.toml +++ b/crates/shuru-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuru-sdk" -version = "0.4.0" +version = "0.9.0" edition = "2021" description = "Async Rust SDK for Shuru microVMs" license = "Apache-2.0" @@ -11,12 +11,15 @@ default = [] cas = ["shuru-store", "blake3"] [dependencies] -shuru-vm = { version = "0.3.8", path = "../shuru-vm" } -shuru-proto = { version = "0.3.3", path = "../shuru-proto" } -shuru-proxy = { version = "0.3.0", path = "../shuru-proxy" } -shuru-store = { version = "0.1.1", path = "../shuru-store", optional = true } +shuru-vm = { version = "0.7.0", path = "../shuru-vm" } +shuru-proto = { version = "0.6.0", path = "../shuru-proto" } +shuru-proxy = { version = "0.5.0", path = "../shuru-proxy" } +shuru-store = { version = "0.2.0", path = "../shuru-store", optional = true } tokio = { version = "1", features = ["sync"] } anyhow = "1" blake3 = { version = "1", optional = true } libc = "0.2" tracing = "0.1" + +[dev-dependencies] +tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] } diff --git a/crates/shuru-sdk/examples/balloon.rs b/crates/shuru-sdk/examples/balloon.rs new file mode 100644 index 0000000..3539277 --- /dev/null +++ b/crates/shuru-sdk/examples/balloon.rs @@ -0,0 +1,95 @@ +//! Watch guest memory go back to the host. +//! +//! Boots a 2 GB sandbox, has the guest fill and free 1 GB, then lowers and +//! raises the memory target. After each step it prints the resident size +//! of this process, which is where guest memory lives on a Linux host, and +//! what the guest sees in /proc/meminfo. The guest reports free memory in +//! rounds two seconds apart, a sixteenth of each free list per round, so +//! a freed gigabyte drains in about ten seconds. +//! +//! SHURU_DATA_DIR=... cargo run -p shuru-sdk --example balloon + +use std::time::Duration; + +use shuru_sdk::{AsyncSandbox, SandboxConfig}; + +const MB: u64 = 1024 * 1024; + +fn host_rss_mb() -> u64 { + let out = std::process::Command::new("ps") + .args(["-o", "rss=", "-p", &std::process::id().to_string()]) + .output() + .ok(); + out.and_then(|o| String::from_utf8(o.stdout).ok()) + .and_then(|s| s.trim().parse::().ok()) + .map(|kb| kb / 1024) + .unwrap_or(0) +} + +async fn guest_meminfo(sb: &AsyncSandbox) -> String { + let out = match sb + .exec_shell("grep -E \"MemTotal|MemFree\" /proc/meminfo") + .await + { + Ok(r) => r.stdout, + Err(e) => return format!("meminfo failed: {}", e), + }; + out.lines() + .filter_map(|line| { + let mut parts = line.split_whitespace(); + let name = parts.next()?; + let kb: u64 = parts.next()?.parse().ok()?; + Some(format!("{} {} MB", name, kb / 1024)) + }) + .collect::>() + .join(" ") +} + +async fn report(sb: &AsyncSandbox, step: &str) { + println!( + "{:<40} host rss {:>5} MB guest {}", + step, + host_rss_mb(), + guest_meminfo(sb).await + ); +} + +async fn settle(sb: &AsyncSandbox, step: &str, seconds: u64) { + for s in 1..=seconds { + tokio::time::sleep(Duration::from_secs(1)).await; + report(sb, &format!("{} +{}s", step, s)).await; + } +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let config = SandboxConfig { + data_dir: std::env::var("SHURU_DATA_DIR").ok(), + memory_mb: 2048, + ..Default::default() + }; + let sb = AsyncSandbox::boot(config).await?; + report(&sb, "booted").await; + + let fill = sb + .exec_shell( + "mount -o remount,size=1536m /dev/shm && dd if=/dev/zero of=/dev/shm/fill bs=1M count=1024 status=none", + ) + .await?; + if fill.exit_code != 0 { + anyhow::bail!("filling tmpfs failed: {}", fill.stderr.trim()); + } + report(&sb, "guest holds 1 GB in tmpfs").await; + + sb.exec_shell("rm /dev/shm/fill").await?; + settle(&sb, "guest freed it", 24).await; + + sb.set_memory_target(1024 * MB).await?; + settle(&sb, "target 1 GB", 4).await; + + sb.set_memory_target(2048 * MB).await?; + settle(&sb, "target 2 GB", 2).await; + + sb.stop().await?; + Ok(()) +} diff --git a/crates/shuru-sdk/src/lib.rs b/crates/shuru-sdk/src/lib.rs index cec9f75..c4c1899 100644 --- a/crates/shuru-sdk/src/lib.rs +++ b/crates/shuru-sdk/src/lib.rs @@ -3,7 +3,7 @@ use std::io::{BufReader, Read, Write}; use std::net::TcpStream; use std::sync::Arc; -use anyhow::{bail, Context, Result}; +use anyhow::{bail, Result}; use tokio::sync::{mpsc, oneshot}; use tracing::info; @@ -12,6 +12,8 @@ pub use shuru_proto::{DirEntry, ReadDirResponse, StatResponse, WatchEvent}; pub use shuru_proxy::config::{ExposeHostMapping, NetworkConfig, ProxyConfig, SecretConfig}; pub use shuru_vm::{default_data_dir, MountConfig}; +const EVENT_QUEUE: usize = 64; + // --------------------------------------------------------------------------- // Public types // --------------------------------------------------------------------------- @@ -77,7 +79,7 @@ pub struct SandboxConfig { pub cpus: usize, /// Memory in MB. Default: 2048. pub memory_mb: u64, - /// Disk size in MB. Default: 4096. + /// Disk size in MB. Minimum and default: 4096. pub disk_size_mb: u64, /// Host → guest directory mounts (VirtioFS). pub mounts: Vec, @@ -166,7 +168,7 @@ impl ShellWriter { /// Reader half of a shell session. Receives output events asynchronously. pub struct ShellReader { - output_rx: mpsc::UnboundedReceiver, + output_rx: mpsc::Receiver, } impl ShellReader { @@ -194,7 +196,7 @@ impl ShellHandle { /// Reader half of a file watch session. Receives filesystem events asynchronously. pub struct WatchReceiver { - event_rx: mpsc::UnboundedReceiver, + event_rx: mpsc::Receiver, } impl WatchReceiver { @@ -290,10 +292,17 @@ enum SandboxCmd { port: u16, reply: oneshot::Sender>, }, + Trim { + reply: oneshot::Sender>, + }, Checkpoint { name: String, reply: oneshot::Sender>, }, + SetMemoryTarget { + bytes: u64, + reply: oneshot::Sender>, + }, Stop { reply: oneshot::Sender>, }, @@ -310,25 +319,24 @@ enum SandboxCmd { /// Virtualization framework's Objective-C objects. pub struct AsyncSandbox { cmd_tx: std::sync::mpsc::Sender, - instance_dir: String, } impl AsyncSandbox { /// Boot a new sandbox VM with the given configuration. pub async fn boot(config: SandboxConfig) -> Result { - let (ready_tx, ready_rx) = oneshot::channel::>(); + let (ready_tx, ready_rx) = oneshot::channel::>(); let (cmd_tx, cmd_rx) = std::sync::mpsc::channel(); std::thread::Builder::new() .name("shuru-vm".into()) .spawn(move || match boot_vm(config) { Ok(booted) => { - if ready_tx.send(Ok(booted.instance_dir.clone())).is_err() { + if ready_tx.send(Ok(())).is_err() { return; } run_vm_loop( booted.sandbox, - &booted.instance_dir, + booted.work_disk, &booted.data_dir, cmd_rx, booted.proxy_handle, @@ -343,12 +351,24 @@ impl AsyncSandbox { } })?; - let instance_dir = ready_rx.await??; + ready_rx.await??; - Ok(Self { - cmd_tx, - instance_dir, - }) + Ok(Self { cmd_tx }) + } + + /// How much memory the guest keeps, in bytes. The guest hands the rest + /// back to the host through the balloon device and gets it back when + /// the target rises again. On a Linux host the guest also reports free + /// pages on its own, so idle memory returns without a target. + pub async fn set_memory_target(&self, bytes: u64) -> Result<()> { + let (reply_tx, reply_rx) = oneshot::channel(); + self.cmd_tx + .send(SandboxCmd::SetMemoryTarget { + bytes, + reply: reply_tx, + }) + .map_err(|_| anyhow::anyhow!("VM thread exited"))?; + reply_rx.await? } /// Execute a command and wait for the result. @@ -405,7 +425,7 @@ impl AsyncSandbox { let writer_stream = stream.try_clone()?; let reader_stream = stream; - let (event_tx, event_rx) = mpsc::unbounded_channel(); + let (event_tx, event_rx) = mpsc::channel(EVENT_QUEUE); let reader_thread = std::thread::Builder::new() .name("shuru-shell-reader".into()) @@ -414,18 +434,18 @@ impl AsyncSandbox { loop { match shuru_proto::frame::read_frame(&mut reader) { Ok(Some((shuru_proto::frame::STDOUT, payload))) => { - if event_tx.send(ShellEvent::Output(payload)).is_err() { + if event_tx.blocking_send(ShellEvent::Output(payload)).is_err() { break; } } Ok(Some((shuru_proto::frame::EXIT, payload))) => { let code = shuru_proto::frame::parse_exit_code(&payload).unwrap_or(0); - let _ = event_tx.send(ShellEvent::Exit(code)); + let _ = event_tx.blocking_send(ShellEvent::Exit(code)); break; } Ok(Some((shuru_proto::frame::ERROR, payload))) => { let msg = String::from_utf8_lossy(&payload).to_string(); - let _ = event_tx.send(ShellEvent::Error(msg)); + let _ = event_tx.blocking_send(ShellEvent::Error(msg)); break; } Ok(Some(_)) => {} // skip unknown frame types @@ -618,8 +638,16 @@ impl AsyncSandbox { Ok((reply_rx, progress_rx)) } - /// Save the current rootfs state as a named checkpoint (CoW clone). - /// Future VMs can boot from this checkpoint via `SandboxConfig::from`. + /// Return free ext4 extents to the backing storage when discard is supported. + pub async fn trim(&self) -> Result { + let (reply, rx) = oneshot::channel(); + self.cmd_tx + .send(SandboxCmd::Trim { reply }) + .map_err(|_| anyhow::anyhow!("sandbox stopped"))?; + rx.await? + } + + /// Save a sealed disk and resume the sandbox after the copy. pub async fn checkpoint(&self, name: &str) -> Result<()> { let (reply_tx, reply_rx) = oneshot::channel(); self.cmd_tx @@ -645,7 +673,7 @@ impl AsyncSandbox { .map_err(|_| anyhow::anyhow!("VM thread exited"))?; let stream = reply_rx.await??; - let (event_tx, event_rx) = mpsc::unbounded_channel(); + let (event_tx, event_rx) = mpsc::channel(EVENT_QUEUE); let reader_thread = std::thread::Builder::new() .name("shuru-watch-reader".into()) @@ -655,7 +683,7 @@ impl AsyncSandbox { match shuru_proto::frame::read_frame(&mut reader) { Ok(Some((shuru_proto::frame::WATCH_EVENT, payload))) => { if let Some(event) = WatchEvent::decode(&payload) { - if event_tx.send(event).is_err() { + if event_tx.blocking_send(event).is_err() { break; } } @@ -678,11 +706,6 @@ impl AsyncSandbox { let _ = self.cmd_tx.send(SandboxCmd::Stop { reply: reply_tx }); reply_rx.await.unwrap_or(Ok(())) } - - /// Get the instance directory path (contains the working rootfs copy). - pub fn instance_dir(&self) -> &str { - &self.instance_dir - } } impl Drop for AsyncSandbox { @@ -690,8 +713,6 @@ impl Drop for AsyncSandbox { // Signal the VM thread to stop let (reply_tx, _) = oneshot::channel(); let _ = self.cmd_tx.send(SandboxCmd::Stop { reply: reply_tx }); - // Clean up instance directory - let _ = std::fs::remove_dir_all(&self.instance_dir); } } @@ -699,38 +720,9 @@ impl Drop for AsyncSandbox { // Internal: VM boot & command loop (runs on dedicated OS thread) // --------------------------------------------------------------------------- -use std::sync::atomic::{AtomicU64, Ordering}; - -/// Monotonic counter to ensure each SDK sandbox gets a unique instance directory, -/// even when multiple sandboxes boot concurrently in the same process. -static INSTANCE_COUNTER: AtomicU64 = AtomicU64::new(0); - -#[cfg(target_os = "macos")] -extern "C" { - fn clonefile(src: *const libc::c_char, dst: *const libc::c_char, flags: u32) -> libc::c_int; -} - -#[cfg(target_os = "macos")] -fn clone_file_cow(src: &str, dst: &str) -> Result<()> { - let c_src = std::ffi::CString::new(src).context("invalid source path")?; - let c_dst = std::ffi::CString::new(dst).context("invalid destination path")?; - let ret = unsafe { clonefile(c_src.as_ptr(), c_dst.as_ptr(), 0) }; - if ret != 0 { - let err = std::io::Error::last_os_error(); - bail!("clonefile({} -> {}) failed: {}", src, dst, err); - } - Ok(()) -} - -#[cfg(target_os = "linux")] -fn clone_file_cow(src: &str, dst: &str) -> Result<()> { - std::fs::copy(src, dst).with_context(|| format!("failed to copy {} -> {}", src, dst))?; - Ok(()) -} - struct BootedVm { sandbox: shuru_vm::Sandbox, - instance_dir: String, + work_disk: shuru_vm::WorkDisk, data_dir: String, proxy_handle: Option, fwd_handle: Option, @@ -746,7 +738,8 @@ fn boot_vm(config: SandboxConfig) -> Result { // Resolve asset paths let kernel_path = format!("{}/Image", data_dir); let rootfs_path = format!("{}/rootfs.ext4", data_dir); - let initrd_path_str = format!("{}/initramfs.cpio.gz", data_dir); + shuru_vm::validate_disk_size(config.disk_size_mb)?; + let initrd_path_str = format!("{}/initramfs.cpio", data_dir); if !std::path::Path::new(&kernel_path).exists() { bail!( @@ -805,37 +798,29 @@ fn boot_vm(config: SandboxConfig) -> Result { #[cfg(not(feature = "cas"))] let use_nbd = false; - // Create per-instance working copy. - // Use PID + atomic counter so concurrent boots in the same process - // each get their own directory (avoids remove_dir_all racing). - let seq = INSTANCE_COUNTER.fetch_add(1, Ordering::Relaxed); - let instance_dir = format!("{}/instances/sdk-{}-{}", data_dir, std::process::id(), seq,); - std::fs::create_dir_all(&instance_dir)?; - let work_rootfs = format!("{}/rootfs.ext4", instance_dir); - - if use_nbd { - // CAS mode: NBD server handles I/O, just need a placeholder file - std::fs::File::create(&work_rootfs)?; - } else { - // Direct mode: CoW clone the source rootfs - clone_file_cow(&source, &work_rootfs)?; - } - - // Extend disk to requested size (only meaningful for direct mode) - if !use_nbd { - let f = std::fs::OpenOptions::new().write(true).open(&work_rootfs)?; - let target = config.disk_size_mb * 1024 * 1024; - let current = f.metadata()?.len(); - if target > current { - f.set_len(target)?; + if config.from.is_none() && !use_nbd { + if let Err(e) = shuru_vm::WorkDisk::prepare_base(&source) { + tracing::warn!("{:#}", e); } - drop(f); } + // A CAS run is served over NBD, so its disk is only a placeholder. + let work_disk = if use_nbd { + shuru_vm::WorkDisk::empty(&data_dir)? + } else { + shuru_vm::WorkDisk::clone_from(&source, &data_dir, config.disk_size_mb)? + }; + // Start CAS NBD server if needed #[cfg(feature = "cas")] let nbd_handle = if use_nbd { - let socket_path = format!("{}/nbd.sock", instance_dir); + std::fs::create_dir_all(format!("{}/cas", data_dir))?; + let socket_path = format!( + "{}/cas/nbd-{}-{}.sock", + data_dir, + std::process::id(), + work_disk.as_raw_fd() + ); let cas_dir_str = match &config.storage { StorageMode::Cas { cas_dir } => cas_dir .clone() @@ -863,15 +848,19 @@ fn boot_vm(config: SandboxConfig) -> Result { let initrd_path = if std::path::Path::new(&initrd_path_str).exists() { Some(initrd_path_str) } else { - None + bail!("required initramfs not found at {}", initrd_path_str); }; // Set up proxy networking if enabled let (vm_fd, proxy_handle) = if config.allow_net { - let mut proxy_config = ProxyConfig::default(); - proxy_config.secrets = config.secrets; - proxy_config.network.allow = config.allowed_hosts; - proxy_config.expose_host = config.expose_host; + let proxy_config = ProxyConfig { + secrets: config.secrets, + network: NetworkConfig { + allow: config.allowed_hosts, + }, + expose_host: config.expose_host, + ..Default::default() + }; let (vm_fd, host_fd) = shuru_proxy::create_socketpair()?; let handle = shuru_proxy::start(host_fd, proxy_config)?; @@ -883,7 +872,7 @@ fn boot_vm(config: SandboxConfig) -> Result { // Build the VM let mut builder = shuru_vm::Sandbox::builder() .kernel(&kernel_path) - .rootfs(&work_rootfs) + .rootfs(work_disk.attach_path()) .cpus(config.cpus) .memory_mb(config.memory_mb) .console(false); // No serial console in SDK mode @@ -937,7 +926,7 @@ fn boot_vm(config: SandboxConfig) -> Result { Ok(BootedVm { sandbox, - instance_dir, + work_disk, data_dir, proxy_handle, fwd_handle, @@ -949,7 +938,7 @@ fn boot_vm(config: SandboxConfig) -> Result { fn run_vm_loop( sandbox: shuru_vm::Sandbox, - instance_dir: &str, + work_disk: shuru_vm::WorkDisk, data_dir: &str, cmd_rx: std::sync::mpsc::Receiver, proxy_handle: Option, @@ -1090,6 +1079,12 @@ fn run_vm_loop( let _ = reply.send(sb.connect_forward(port)); }); } + SandboxCmd::SetMemoryTarget { bytes, reply } => { + let _ = reply.send(sandbox.set_memory_target(bytes)); + } + SandboxCmd::Trim { reply } => { + let _ = reply.send(sandbox.trim()); + } SandboxCmd::Checkpoint { name, reply } => { let result = (|| -> Result<()> { shuru_vm::validate_checkpoint_name(&name).map_err(|e| anyhow::anyhow!(e))?; @@ -1100,21 +1095,18 @@ fn run_vm_loop( #[cfg(feature = "cas")] if let Some(ref handle) = nbd_handle { let index_path = format!("{}/{}.idx", checkpoints_dir, name); - if std::path::Path::new(&index_path).exists() { - std::fs::remove_file(&index_path)?; - } - handle.save_checkpoint(&index_path)?; + sandbox.save_disk(&index_path, shuru_vm::AfterSave::Thaw, |p| { + handle.save_checkpoint(p) + })?; info!("checkpoint '{}' saved (CAS)", name); return Ok(()); } // Direct path: save as .ext4 via CoW clone let checkpoint_path = format!("{}/{}.ext4", checkpoints_dir, name); - if std::path::Path::new(&checkpoint_path).exists() { - std::fs::remove_file(&checkpoint_path)?; - } - let work_rootfs = format!("{}/rootfs.ext4", instance_dir); - clone_file_cow(&work_rootfs, &checkpoint_path)?; + sandbox.save_disk(&checkpoint_path, shuru_vm::AfterSave::Thaw, |p| { + work_disk.save_to(p) + })?; info!("checkpoint '{}' saved", name); Ok(()) })(); @@ -1136,15 +1128,10 @@ fn exec_command( argv: &[String], env: &HashMap, ) -> Result { - let mut stdout_buf = Vec::new(); - let mut stderr_buf = Vec::new(); - let argv_refs: Vec<&str> = argv.iter().map(|s| s.as_str()).collect(); - - let exit_code = sandbox.exec_with_env(&argv_refs, env, &mut stdout_buf, &mut stderr_buf)?; - + let output = sandbox.exec_buffered(argv, env)?; Ok(ExecResult { - stdout: String::from_utf8_lossy(&stdout_buf).to_string(), - stderr: String::from_utf8_lossy(&stderr_buf).to_string(), - exit_code, + stdout: String::from_utf8_lossy(&output.stdout).into_owned(), + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + exit_code: output.exit_code, }) } diff --git a/crates/shuru-store/Cargo.toml b/crates/shuru-store/Cargo.toml index 38a8a14..7861af8 100644 --- a/crates/shuru-store/Cargo.toml +++ b/crates/shuru-store/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuru-store" -version = "0.1.3" +version = "0.2.0" edition = "2021" description = "NBD-backed storage layer for shuru microVMs" license = "Apache-2.0" diff --git a/crates/shuru-store/src/cas.rs b/crates/shuru-store/src/cas.rs index 24fac1c..3a543da 100644 --- a/crates/shuru-store/src/cas.rs +++ b/crates/shuru-store/src/cas.rs @@ -40,15 +40,29 @@ impl ChunkStore for LocalChunkStore { let hash = blake3::hash(data); let hex = hash.to_hex().to_string(); let path = self.chunk_path(&hex); - match fs::OpenOptions::new().write(true).create_new(true).open(&path) { - Ok(mut f) => { - use std::io::Write; - f.write_all(data) - .with_context(|| format!("failed to write chunk {}", hex))?; - } - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} - Err(e) => return Err(e).with_context(|| format!("failed to create chunk {}", hex)), + if path.exists() { + return Ok(hex); + } + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let seq = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let partial = self + .chunks_dir + .join(format!(".partial-{}-{seq}", std::process::id())); + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&partial)?; + let result = (|| -> Result<()> { + file.write_all(data)?; + file.sync_all()?; + fs::rename(&partial, &path)?; + fs::File::open(&self.chunks_dir)?.sync_all()?; + Ok(()) + })(); + if result.is_err() { + let _ = fs::remove_file(&partial); } + result?; Ok(hex) } @@ -77,7 +91,7 @@ const ZERO_CHUNK_HASH: &str = "ZERO"; impl ChunkIndex { pub fn new(disk_size: u64) -> Self { - let num_chunks = ((disk_size + CHUNK_SIZE as u64 - 1) / CHUNK_SIZE as u64) as usize; + let num_chunks = disk_size.div_ceil(CHUNK_SIZE as u64) as usize; ChunkIndex { hashes: vec![ZERO_CHUNK_HASH.to_string(); num_chunks], disk_size, @@ -109,8 +123,8 @@ impl ChunkIndex { if let Some(parent) = Path::new(path).parent() { fs::create_dir_all(parent)?; } - let mut f = fs::File::create(path) - .with_context(|| format!("failed to create index: {}", path))?; + let mut f = + fs::File::create(path).with_context(|| format!("failed to create index: {}", path))?; // Header: disk_size, num_chunks, parent_path, fallback_path f.write_all(&self.disk_size.to_le_bytes())?; f.write_all(&(self.hashes.len() as u64).to_le_bytes())?; @@ -131,19 +145,22 @@ impl ChunkIndex { /// Load index from a file. pub fn load(path: &str) -> Result { - let mut f = fs::File::open(path) - .with_context(|| format!("failed to open index: {}", path))?; + let mut f = + fs::File::open(path).with_context(|| format!("failed to open index: {}", path))?; let mut buf8 = [0u8; 8]; f.read_exact(&mut buf8)?; let disk_size = u64::from_le_bytes(buf8); f.read_exact(&mut buf8)?; let num_chunks = u64::from_le_bytes(buf8) as usize; - let expected_chunks = ((disk_size + CHUNK_SIZE as u64 - 1) / CHUNK_SIZE as u64) as usize; + let expected_chunks = disk_size.div_ceil(CHUNK_SIZE as u64) as usize; anyhow::ensure!( num_chunks == expected_chunks, "index {}: chunk count {} does not match disk_size {} (expected {})", - path, num_chunks, disk_size, expected_chunks, + path, + num_chunks, + disk_size, + expected_chunks, ); // Parent path @@ -177,7 +194,12 @@ impl ChunkIndex { hashes.push(String::from_utf8(hash_bytes)?); } - Ok(ChunkIndex { hashes, disk_size, parent_path, fallback_path }) + Ok(ChunkIndex { + hashes, + disk_size, + parent_path, + fallback_path, + }) } /// Validate that the given backing store size does not exceed disk_size. @@ -186,7 +208,9 @@ impl ChunkIndex { anyhow::ensure!( backend_size <= self.disk_size, "{} size ({}) exceeds index disk_size ({})", - label, backend_size, self.disk_size, + label, + backend_size, + self.disk_size, ); Ok(()) } @@ -201,8 +225,6 @@ pub struct CasBackend { parents: RwLock>, /// Optional flat file for lazy ingestion at the bottom of the chain. fallback: Option, - /// The index path we booted from (becomes the parent when saving a checkpoint). - pub source_index_path: Option, } impl CasBackend { @@ -215,11 +237,14 @@ impl CasBackend { dirty: RwLock::new(HashMap::new()), parents: RwLock::new(parents), fallback: None, - source_index_path: None, } } - pub fn with_fallback(store: Box, index: ChunkIndex, fallback: crate::backend::FlatFileBackend) -> Self { + pub fn with_fallback( + store: Box, + index: ChunkIndex, + fallback: crate::backend::FlatFileBackend, + ) -> Self { let parents = Self::load_parent_chain(&index); CasBackend { store, @@ -227,7 +252,6 @@ impl CasBackend { dirty: RwLock::new(HashMap::new()), parents: RwLock::new(parents), fallback: Some(fallback), - source_index_path: None, } } @@ -258,7 +282,7 @@ impl CasBackend { pub fn set_disk_size(&mut self, new_size: u64) { let mut index = self.index.write().unwrap(); if new_size > index.disk_size() { - let new_num_chunks = ((new_size + CHUNK_SIZE as u64 - 1) / CHUNK_SIZE as u64) as usize; + let new_num_chunks = new_size.div_ceil(CHUNK_SIZE as u64) as usize; while index.num_chunks() < new_num_chunks { index.hashes.push(ZERO_CHUNK_HASH.to_string()); } @@ -327,29 +351,50 @@ impl CasBackend { let mut dirty = self.dirty.write().unwrap(); let mut index = self.index.write().unwrap(); - for (chunk_idx, data) in dirty.drain() { - let hash = self.store.put(&data).map_err(|e| { - std::io::Error::new(std::io::ErrorKind::Other, e.to_string()) - })?; + for (&chunk_idx, data) in dirty.iter() { + let hash = self + .store + .put(data) + .map_err(|e| std::io::Error::other(e.to_string()))?; index.set_hash(chunk_idx, hash); } + dirty.clear(); Ok(()) } - /// Save the current index as a checkpoint. Only writes the delta — ZERO entries - /// resolve through the parent chain at read time. Instant regardless of disk size. + /// Save a standalone index so replacing a checkpoint cannot create a parent cycle. + /// The caller holds the guest seal throughout this operation. pub fn save_index(&self, path: &str) -> Result<()> { self.flush().map_err(|e| anyhow::anyhow!(e))?; - let mut index = self.index.write().unwrap(); - index.parent_path = self.source_index_path.clone(); - // Propagate fallback path so the chain can always resolve to the flat file - if index.fallback_path.is_none() { - if let Some(ref fb) = self.fallback { - index.fallback_path = Some(fb.path().to_string()); + let index = self.index.read().unwrap(); + let parents = self.parents.read().unwrap(); + let mut saved = ChunkIndex::new(index.disk_size); + for (i, hash) in index.hashes.iter().enumerate() { + saved.hashes[i] = if hash != ZERO_CHUNK_HASH { + hash.clone() + } else { + parents + .iter() + .filter_map(|p| p.get_hash(i)) + .find(|h| *h != ZERO_CHUNK_HASH) + .unwrap_or(ZERO_CHUNK_HASH) + .to_string() + }; + } + drop(parents); + drop(index); + if let Some(fallback) = &self.fallback { + for (i, hash) in saved.hashes.iter_mut().enumerate() { + if *hash == ZERO_CHUNK_HASH && (i as u64) * (CHUNK_SIZE as u64) < fallback.size() { + let data = self.read_chunk(i)?; + if data.iter().any(|b| *b != 0) { + *hash = self.store.put(&data)?; + } + } } } - index.save(path) + saved.save(path) } fn read_chunk(&self, chunk_idx: usize) -> std::io::Result> { @@ -361,7 +406,10 @@ impl CasBackend { // 2. Check current index let hash = { let index = self.index.read().unwrap(); - index.get_hash(chunk_idx).unwrap_or(ZERO_CHUNK_HASH).to_string() + index + .get_hash(chunk_idx) + .unwrap_or(ZERO_CHUNK_HASH) + .to_string() }; if hash != ZERO_CHUNK_HASH { return self.fetch_chunk(&hash); @@ -381,16 +429,22 @@ impl CasBackend { if offset < fb.size() { let read_len = CHUNK_SIZE.min((fb.size() - offset) as usize); let mut buf = vec![0u8; read_len]; - fb.read(offset, &mut buf)?; + if fb.read(offset, &mut buf)? != read_len { + return Err(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "short read from CAS base image", + )); + } if buf.iter().all(|&b| b == 0) { return Ok(vec![0u8; CHUNK_SIZE]); } // Cache in chunk store for future reads - let new_hash = self.store.put(&buf).map_err(|e| { - std::io::Error::new(std::io::ErrorKind::Other, e.to_string()) - })?; + let new_hash = self + .store + .put(&buf) + .map_err(|e| std::io::Error::other(e.to_string()))?; self.index.write().unwrap().set_hash(chunk_idx, new_hash); return Ok(buf); } @@ -407,7 +461,7 @@ impl CasBackend { tracing::warn!("chunk {} not found in store, returning zeros", hash); Ok(vec![0u8; CHUNK_SIZE]) } - Err(e) => Err(std::io::Error::new(std::io::ErrorKind::Other, e.to_string())), + Err(e) => Err(std::io::Error::other(e.to_string())), } } } @@ -519,9 +573,13 @@ mod tests { let index = ChunkIndex::new(4 * 1024 * 1024); // 4MB virtual disk // flat file == virtual disk: ok - assert!(index.check_size_against_backend(4 * 1024 * 1024, "test").is_ok()); + assert!(index + .check_size_against_backend(4 * 1024 * 1024, "test") + .is_ok()); // flat file < virtual disk: ok (sparse expansion — the normal case) - assert!(index.check_size_against_backend(1024 * 1024, "test").is_ok()); + assert!(index + .check_size_against_backend(1024 * 1024, "test") + .is_ok()); // flat file > virtual disk: corrupt — must fail match index.check_size_against_backend(8 * 1024 * 1024, "test") { Err(e) => assert!(e.to_string().contains("exceeds")), @@ -529,3 +587,55 @@ mod tests { } } } + +#[cfg(test)] +mod checkpoint_tests { + use super::*; + + #[test] + fn saved_checkpoint_does_not_depend_on_mutable_base_file() { + let dir = tempfile::tempdir().unwrap(); + let flat = dir.path().join("base.ext4"); + fs::write(&flat, b"unread base data").unwrap(); + let store = || Box::new(LocalChunkStore::open(dir.path().to_str().unwrap()).unwrap()); + let backend = CasBackend::with_fallback( + store(), + ChunkIndex::new(CHUNK_SIZE as u64), + crate::backend::FlatFileBackend::open(flat.to_str().unwrap()).unwrap(), + ); + let path = dir.path().join("saved.idx"); + backend.save_index(path.to_str().unwrap()).unwrap(); + fs::write(&flat, b"replaced image!").unwrap(); + let saved = ChunkIndex::load(path.to_str().unwrap()).unwrap(); + assert!(saved.fallback_path.is_none()); + let restored = CasBackend::new(store(), saved); + let mut data = [0; 16]; + restored.read(0, &mut data).unwrap(); + assert_eq!(&data, b"unread base data"); + } + + #[test] + fn replacing_a_parent_keeps_inherited_chunks() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("disk.idx"); + let path = path.to_str().unwrap(); + let store = || Box::new(LocalChunkStore::open(dir.path().to_str().unwrap()).unwrap()); + let base = CasBackend::new(store(), ChunkIndex::new(2 * CHUNK_SIZE as u64)); + base.write(0, b"base").unwrap(); + base.save_index(path).unwrap(); + let mut delta = ChunkIndex::new(2 * CHUNK_SIZE as u64); + delta.parent_path = Some(path.to_string()); + let backend = CasBackend::new(store(), delta); + backend.write(CHUNK_SIZE as u64, b"child").unwrap(); + backend.save_index(path).unwrap(); + let saved = ChunkIndex::load(path).unwrap(); + assert!(saved.parent_path.is_none()); + let restored = CasBackend::new(store(), saved); + let mut head = [0; 4]; + restored.read(0, &mut head).unwrap(); + assert_eq!(&head, b"base"); + let mut tail = [0; 5]; + restored.read(CHUNK_SIZE as u64, &mut tail).unwrap(); + assert_eq!(&tail, b"child"); + } +} diff --git a/crates/shuru-store/src/lib.rs b/crates/shuru-store/src/lib.rs index cf37264..20c52fd 100644 --- a/crates/shuru-store/src/lib.rs +++ b/crates/shuru-store/src/lib.rs @@ -32,7 +32,7 @@ impl NbdHandle { format!("nbd+unix:///export?socket={}", self.socket_path) } - /// Save the current disk state as a checkpoint index. Only works with CAS backend. + /// Write a CAS index to the private staging path supplied by Sandbox::save_disk. pub fn save_checkpoint(&self, index_path: &str) -> Result<()> { let backend = self .cas_backend @@ -136,22 +136,24 @@ pub fn start_cas_nbd_server( ) -> Result { let store: Box = Box::new(LocalChunkStore::open(cas_dir)?); - let (index, fallback, source_idx) = if Path::new(index_path).exists() { + let (index, fallback) = if Path::new(index_path).exists() { info!("loading CAS index from {}", index_path); let idx = ChunkIndex::load(index_path)?; // If the index has a fallback_path, open it and validate disk_size let fb = idx .fallback_path .as_ref() - .and_then(|p| FlatFileBackend::open(p).ok()); + .map(|p| FlatFileBackend::open(p).with_context(|| format!("open CAS base image {}", p))) + .transpose()?; if let Some(ref fb) = fb { anyhow::ensure!( fb.size() <= idx.disk_size(), "fallback file size ({}) exceeds index disk_size ({}); index may be corrupt", - fb.size(), idx.disk_size(), + fb.size(), + idx.disk_size(), ); } - (idx, fb, Some(index_path.to_string())) + (idx, fb) } else { // No index yet — create empty index with fallback to flat file for lazy ingestion let fb = FlatFileBackend::open(rootfs_path).with_context(|| { @@ -159,7 +161,7 @@ pub fn start_cas_nbd_server( })?; let disk_size = fb.size(); info!("CAS: lazy mode, {} MB rootfs", disk_size / (1024 * 1024)); - (ChunkIndex::new(disk_size), Some(fb), None) + (ChunkIndex::new(disk_size), Some(fb)) }; let mut backend = if let Some(fb) = fallback { @@ -167,7 +169,10 @@ pub fn start_cas_nbd_server( } else { CasBackend::new(store, index) }; - backend.source_index_path = source_idx; + anyhow::ensure!( + disk_size >= backend.size(), + "disk size is smaller than the source checkpoint" + ); if disk_size > 0 { backend.set_disk_size(disk_size); } diff --git a/crates/shuru-vm/Cargo.toml b/crates/shuru-vm/Cargo.toml index 4fa7f69..1dd331a 100644 --- a/crates/shuru-vm/Cargo.toml +++ b/crates/shuru-vm/Cargo.toml @@ -1,21 +1,22 @@ [package] name = "shuru-vm" -version = "0.3.8" +version = "0.7.0" edition = "2021" description = "High-level VM management for shuru" license = "Apache-2.0" repository = "https://github.com/superhq-ai/shuru" [dependencies] -shuru-proto = { version = "0.3.3", path = "../shuru-proto" } +shuru-proto = { version = "0.6.0", path = "../shuru-proto" } anyhow = "1" crossbeam-channel = "0.5" serde = { version = "1", features = ["derive"] } serde_json = "1" tracing = "0.1" socket2 = "0.5" +getrandom = "0.3" [target.'cfg(target_os = "macos")'.dependencies] -shuru-darwin = { version = "0.1.1", path = "../shuru-darwin" } +shuru-darwin = { version = "0.2.3", path = "../shuru-darwin" } [target.'cfg(target_os = "linux")'.dependencies] -shuru-linux = { version = "0.2.0", path = "../shuru-linux" } +shuru-linux = { version = "0.4.1", path = "../shuru-linux" } diff --git a/crates/shuru-vm/src/checkpoint.rs b/crates/shuru-vm/src/checkpoint.rs new file mode 100644 index 0000000..e547739 --- /dev/null +++ b/crates/shuru-vm/src/checkpoint.rs @@ -0,0 +1,126 @@ +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; + +struct Partial(PathBuf); +impl Drop for Partial { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +pub fn check_format(dest: &str) -> Result<()> { + let path = Path::new(dest); + let other = match path.extension().and_then(|s| s.to_str()) { + Some("ext4") => path.with_extension("idx"), + Some("idx") => path.with_extension("ext4"), + _ => return Ok(()), + }; + anyhow::ensure!( + !other.exists(), + "checkpoint uses another storage format; choose a new name" + ); + Ok(()) +} + +pub(crate) fn publish( + dest: &str, + copy: impl FnOnce(&str) -> Result<()>, + finish: impl FnOnce() -> Result<()>, +) -> Result<()> { + let staging = PathBuf::from(format!("{dest}.partial")); + match std::fs::create_dir(&staging) { + Err(e) if e.kind() != std::io::ErrorKind::AlreadyExists => { + return Err(e).context("reserve checkpoint staging directory"); + } + _ => {} + } + // The lock tells a save in progress from one that died and left its + // staging directory behind. + let lock = std::fs::File::open(&staging)?; + if lock.try_lock().is_err() { + anyhow::bail!("another save of this checkpoint is in progress"); + } + let guard = Partial(staging); + let image = guard.0.join("disk"); + match std::fs::remove_file(&image) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => return Err(e.into()), + _ => {} + } + copy(image.to_str().context("checkpoint path is not UTF-8")?)?; + std::fs::File::open(&image)?.sync_all()?; + finish()?; + std::fs::rename(&image, dest).context("publish sealed checkpoint")?; + let parent = Path::new(dest) + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + std::fs::File::open(parent)?.sync_all()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use anyhow::bail; + + #[test] + fn failed_copy_or_expired_lease_preserves_destination() { + let dir = std::env::temp_dir().join(format!("shuru-publish-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let dest = dir.join("checkpoint"); + let dest = dest.to_str().unwrap(); + std::fs::write(dest, b"previous").unwrap(); + assert!(publish( + dest, + |p| { + std::fs::write(p, b"partial")?; + bail!("copy failed") + }, + || Ok(()) + ) + .is_err()); + assert_eq!(std::fs::read(dest).unwrap(), b"previous"); + assert!(publish( + dest, + |p| { + std::fs::write(p, b"new")?; + Ok(()) + }, + || { bail!("Expired") } + ) + .is_err()); + assert_eq!(std::fs::read(dest).unwrap(), b"previous"); + publish( + dest, + |p| { + std::fs::write(p, b"new")?; + Ok(()) + }, + || Ok(()), + ) + .unwrap(); + assert_eq!(std::fs::read(dest).unwrap(), b"new"); + + let staging = format!("{dest}.partial"); + std::fs::create_dir(&staging).unwrap(); + std::fs::write(format!("{staging}/disk"), b"left by a crash").unwrap(); + let held = std::fs::File::open(&staging).unwrap(); + held.lock().unwrap(); + assert!(publish(dest, |_| panic!("must not copy"), || Ok(())).is_err()); + assert!(Path::new(&staging).is_dir()); + drop(held); + publish( + dest, + |p| { + std::fs::write(p, b"after crash")?; + Ok(()) + }, + || Ok(()), + ) + .unwrap(); + assert_eq!(std::fs::read(dest).unwrap(), b"after crash"); + assert!(!Path::new(&staging).exists()); + std::fs::remove_dir_all(dir).unwrap(); + } +} diff --git a/crates/shuru-vm/src/disk_tests.rs b/crates/shuru-vm/src/disk_tests.rs new file mode 100644 index 0000000..65fbe49 --- /dev/null +++ b/crates/shuru-vm/src/disk_tests.rs @@ -0,0 +1,216 @@ +use super::*; +use crate::WorkDisk; +use std::path::{Path, PathBuf}; + +struct Fixture { + sandbox: Sandbox, + disk: WorkDisk, + dir: PathBuf, +} + +impl Fixture { + fn boot(source: Option<&str>, size: u64) -> Self { + let data = std::env::var("SHURU_FAST_BOOT_TEST_DATA") + .expect("set SHURU_FAST_BOOT_TEST_DATA to an isolated directory with built OS assets"); + let data = std::fs::canonicalize(data).unwrap(); + assert!(data.starts_with("/tmp") || data.starts_with("/private/tmp")); + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let dir = data.join(format!( + "test-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + std::fs::create_dir(&dir).unwrap(); + let base = data.join("rootfs.ext4"); + let disk = WorkDisk::clone_from( + source.unwrap_or(base.to_str().unwrap()), + dir.to_str().unwrap(), + size, + ) + .unwrap(); + let sandbox = Sandbox::builder() + .kernel(data.join("Image").to_str().unwrap()) + .initrd(data.join("initramfs.cpio").to_str().unwrap()) + .rootfs(disk.attach_path()) + .console(false) + .verbose(true) + .mount(MountConfig { + host_path: data.to_str().unwrap().into(), + guest_path: "/evidence".into(), + read_only: true, + }) + .build() + .unwrap(); + sandbox.start().unwrap(); + Self { sandbox, disk, dir } + } + + fn exec(&self, command: &str) -> String { + let mut out = Vec::new(); + let mut err = Vec::new(); + let code = self + .sandbox + .exec(&["sh", "-c", command], &mut out, &mut err) + .unwrap(); + assert_eq!(code, 0, "{}", String::from_utf8_lossy(&err)); + String::from_utf8(out).unwrap() + } + + fn dest(&self) -> String { + self.dir.join("saved.ext4").to_str().unwrap().to_string() + } +} + +impl Drop for Fixture { + fn drop(&mut self) { + let _ = self.sandbox.kill(); + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +#[test] +#[ignore = "requires built isolated OS assets and a native hypervisor"] +fn boot_grow_seal_and_boot_saved_disk() { + for size in [4096, 8192, 65536] { + let first = Fixture::boot(None, size); + assert!(first + .exec("readlink /proc/1/exe") + .contains("/init (deleted)")); + first.exec("test ! -e /usr/bin/shuru-init; echo marker > /marker; echo stale-agent > /usr/bin/shuru-init"); + assert_eq!( + first + .exec("blockdev --getsize64 /dev/vda") + .trim() + .parse::() + .unwrap(), + size * 1024 * 1024 + ); + let dest = first.dest(); + first + .sandbox + .save_disk(&dest, AfterSave::PowerOff, |p| first.disk.save_to(p)) + .unwrap(); + let second = Fixture::boot(Some(&dest), size); + assert_eq!(second.exec("cat /marker").trim(), "marker"); + assert!(second + .exec("readlink /proc/1/exe") + .contains("/init (deleted)")); + let relative = Path::new(&dest) + .strip_prefix(first.dir.parent().unwrap()) + .unwrap(); + second.exec(&format!( + "set -- $(od -An -tu4 -j1120 -N4 /evidence/{}); test $(( $1 & 4 )) -eq 0", + relative.display() + )); + } +} + +#[test] +#[ignore = "requires built isolated OS assets and a native hypervisor"] +fn running_checkpoint_resumes_writers_and_trim_reclaims_space() { + use std::os::unix::fs::MetadataExt; + let vm = Fixture::boot(None, 4096); + vm.exec("dd if=/dev/urandom of=/large bs=1M count=256 status=none; sync"); + let allocated = std::fs::metadata(vm.disk.attach_path()).unwrap().blocks(); + vm.exec("rm /large; sync"); + let report = vm.sandbox.trim().unwrap(); + let trimmed = std::fs::metadata(vm.disk.attach_path()).unwrap().blocks(); + eprintln!( + "trim supported={} submitted={} allocated_before={} allocated_after={}", + report.supported, + report.bytes, + allocated * 512, + trimmed * 512 + ); + assert!(report.supported); + assert!(trimmed < allocated); + vm.exec("nohup sh -c \"while :; do echo write >> /writer; sleep 0.01; done\" /run/writer.log 2>&1 &"); + vm.sandbox + .save_disk(&vm.dest(), AfterSave::Thaw, |p| vm.disk.save_to(p)) + .unwrap(); + vm.exec("echo resumed > /resumed; test -s /writer"); + let restored = Fixture::boot(Some(&vm.dest()), 4096); + restored.exec("test -s /writer; test ! -e /resumed"); +} + +#[test] +#[ignore = "requires built isolated OS assets and a native hypervisor"] +fn expired_seal_and_wrong_id_cannot_publish() { + let vm = Fixture::boot(None, 4096); + vm.exec("true"); + let request = shuru_proto::SealRequest { + id: 71, + stop_services: false, + lease_ms: 1000, + }; + let response: shuru_proto::SealResponse = vm + .sandbox + .disk_request(frame::SEAL_REQ, &serde_json::to_vec(&request).unwrap()) + .unwrap(); + assert_eq!( + response, + shuru_proto::SealResponse::Sealed(Default::default()) + ); + assert!(vm + .sandbox + .disk_request::( + frame::THAW_REQ, + &serde_json::to_vec(&shuru_proto::ThawRequest { id: 72 }).unwrap() + ) + .is_err()); + std::thread::sleep(Duration::from_millis(1200)); + vm.exec("echo watchdog-thawed > /after-expiry"); + let response: shuru_proto::SealResponse = vm + .sandbox + .disk_request( + frame::THAW_REQ, + &serde_json::to_vec(&shuru_proto::ThawRequest { id: 71 }).unwrap(), + ) + .unwrap(); + assert_eq!(response, shuru_proto::SealResponse::Expired); + let dest = vm.dest(); + std::fs::write(&dest, b"previous checkpoint").unwrap(); + assert!(vm + .sandbox + .save_disk(&dest, AfterSave::Thaw, |_| bail!("injected clone failure")) + .is_err()); + assert_eq!(std::fs::read(&dest).unwrap(), b"previous checkpoint"); + vm.exec("echo thawed-after-failure > /after-failure"); + vm.sandbox.kill().unwrap(); + assert!(vm + .sandbox + .save_disk(&dest, AfterSave::PowerOff, |_| panic!( + "stopped guest must not copy" + )) + .is_err()); + assert_eq!(std::fs::read(&dest).unwrap(), b"previous checkpoint"); +} + +#[test] +#[ignore = "requires built isolated OS assets and a native hypervisor"] +fn legacy_dirty_disk_replays_and_ext4_errors_refuse_save_and_boot() { + let old = Fixture::boot(None, 4096); + old.exec("echo old-checkpoint > /legacy; sync"); + old.sandbox.kill().unwrap(); + // Deliberately manufacture a pre-seal checkpoint for compatibility testing. + old.disk.save_to(&old.dest()).unwrap(); + let restored = Fixture::boot(Some(&old.dest()), 4096); + assert_eq!(restored.exec("cat /legacy").trim(), "old-checkpoint"); + // 6.12 shuts the filesystem down on error, reads included, so nothing + // from the disk can run after the trigger. + restored.exec("echo injected-test-error > /sys/fs/ext4/vda/trigger_fs_error"); + std::thread::sleep(std::time::Duration::from_secs(1)); + let dest = restored.dest(); + std::fs::write(&dest, b"previous checkpoint").unwrap(); + assert!(restored + .sandbox + .save_disk(&dest, AfterSave::Thaw, |p| restored.disk.save_to(p)) + .is_err()); + assert_eq!(std::fs::read(&dest).unwrap(), b"previous checkpoint"); + restored.sandbox.kill().unwrap(); + let damaged = restored.dir.join("damaged.ext4"); + // This fixture intentionally carries the error flag into a boot attempt. + restored.disk.save_to(damaged.to_str().unwrap()).unwrap(); + let refused = Fixture::boot(Some(damaged.to_str().unwrap()), 4096); + assert!(refused.sandbox.wait_ready().is_err()); +} diff --git a/crates/shuru-vm/src/lib.rs b/crates/shuru-vm/src/lib.rs index b3bd872..0033519 100644 --- a/crates/shuru-vm/src/lib.rs +++ b/crates/shuru-vm/src/lib.rs @@ -1,12 +1,19 @@ #![forbid(unsafe_code)] +mod checkpoint; +pub use checkpoint::check_format as check_checkpoint_format; mod sandbox; +mod serial; +mod workdisk; -pub use sandbox::{MountConfig, PortForwardHandle, Sandbox, VmConfigBuilder}; +pub use sandbox::{ + AfterSave, ExecOutput, MountConfig, PortForwardHandle, Sandbox, VmConfigBuilder, MAX_BUFFERED, +}; pub use shuru_proto::{ frame, ExecRequest, ForwardRequest, ForwardResponse, MountRequest, MountResponse, PortMapping, ReadFileRequest, WriteFileRequest, WriteFileResponse, VSOCK_PORT, VSOCK_PORT_FORWARD, }; +pub use workdisk::WorkDisk; // Re-exports from platform-specific backend for advanced/escape-hatch use #[cfg(target_os = "macos")] @@ -34,7 +41,39 @@ pub fn validate_checkpoint_name(name: &str) -> Result<(), String> { Ok(()) } +/// Where assets, checkpoints and instances live. `SHURU_DATA_DIR` overrides +/// the default so a build or a test run can use its own kernel, rootfs and +/// checkpoints without touching the installed ones. pub fn default_data_dir() -> String { + if let Ok(dir) = std::env::var("SHURU_DATA_DIR") { + if !dir.is_empty() { + return dir; + } + } let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()); format!("{}/.local/share/shuru", home) } + +pub const MIN_DISK_MB: u64 = 4096; + +pub fn validate_disk_size(size_mb: u64) -> anyhow::Result { + if size_mb < MIN_DISK_MB { + anyhow::bail!("disk size must be at least {} MB", MIN_DISK_MB); + } + size_mb + .checked_mul(1024 * 1024) + .filter(|n| *n <= i64::MAX as u64) + .ok_or_else(|| anyhow::anyhow!("disk size exceeds the supported byte range")) +} + +#[cfg(test)] +mod disk_size_tests { + use super::*; + + #[test] + fn disk_sizes_have_a_minimum_and_do_not_overflow() { + assert!(validate_disk_size(4095).is_err()); + assert!(validate_disk_size(u64::MAX).is_err()); + assert_eq!(validate_disk_size(4096).unwrap(), 4 * 1024 * 1024 * 1024); + } +} diff --git a/crates/shuru-vm/src/sandbox.rs b/crates/shuru-vm/src/sandbox.rs index f018cf2..ce7c2af 100644 --- a/crates/shuru-vm/src/sandbox.rs +++ b/crates/shuru-vm/src/sandbox.rs @@ -23,11 +23,49 @@ use shuru_linux::terminal; #[cfg(target_os = "linux")] use shuru_linux::*; +/// Most bytes a buffered call (`read_file`, `exec_buffered`) holds on the +/// host, so a guest cannot grow host memory by producing more. +pub const MAX_BUFFERED: usize = 64 * 1024 * 1024; + +fn too_large() -> String { + format!( + "{} MiB, the limit for buffered calls; stream it instead", + MAX_BUFFERED >> 20 + ) +} + +pub struct ExecOutput { + pub stdout: Vec, + pub stderr: Vec, + pub exit_code: i32, +} + +#[derive(Default)] +struct Capped(Vec); + +impl Write for Capped { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + if self.0.len() + buf.len() > MAX_BUFFERED { + return Err(std::io::Error::other(format!( + "output is larger than {}", + too_large() + ))); + } + self.0.extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + use shuru_proto::{ frame, ChmodRequest, CopyRequest, DiscardRequest, ExecRequest, ForwardRequest, ForwardResponse, FsOkResponse, MkdirRequest, MountRequest, MountResponse, PortMapping, ReadDirRequest, - ReadDirResponse, ReadFileRequest, RemoveRequest, RenameRequest, StatRequest, StatResponse, - WatchRequest, WriteFileRequest, WriteFileResponse, VSOCK_PORT, VSOCK_PORT_FORWARD, + ReadDirResponse, ReadFileRequest, RemoveRequest, RenameRequest, ShutdownResponse, StatRequest, + StatResponse, WatchRequest, WriteFileRequest, WriteFileResponse, VSOCK_PORT, + VSOCK_PORT_FORWARD, }; // --- Mount types --- @@ -137,32 +175,48 @@ impl VmConfigBuilder { } let boot_loader = LinuxBootLoader::new_with_kernel(&kernel_path); - if let Some(ref initrd) = self.initrd { - boot_loader.set_initrd(initrd); - } + boot_loader.set_initrd( + self.initrd + .as_ref() + .context("image initramfs is required")?, + ); + let mut identity = shuru_proto::hello::Identity { + token: [0; 32], + instance: [0; 16], + }; + getrandom::fill(&mut identity.token) + .map_err(|_| anyhow::anyhow!("failed to generate guest token"))?; + getrandom::fill(&mut identity.instance) + .map_err(|_| anyhow::anyhow!("failed to generate guest instance"))?; + let hex = |bytes: &[u8]| bytes.iter().map(|b| format!("{b:02x}")).collect::(); + let token = hex(&identity.token); let cmdline = if self.verbose { - "console=hvc0 root=/dev/vda rw init=/usr/bin/shuru-init" + "console=hvc0 root=/dev/vda rw" } else { - "console=hvc0 root=/dev/vda rw init=/usr/bin/shuru-init quiet" + "console=hvc0 root=/dev/vda rw quiet" }; - boot_loader.set_command_line(cmdline); + boot_loader.set_command_line(&format!( + "{cmdline} shuru.token={token} shuru.instance={}", + hex(&identity.instance) + )); let memory_bytes = self.memory_mb * 1024 * 1024; let config = VirtualMachineConfiguration::new(&boot_loader, self.cpus, memory_bytes); - let dev_null; // keep the File alive so the fd stays valid + let serial_output: std::os::fd::OwnedFd = if self.console || self.verbose { + crate::serial::output(token, self.console)?.into() + } else { + std::fs::OpenOptions::new() + .write(true) + .open("/dev/null")? + .into() + }; + let output_fd = serial_output.as_raw_fd(); let serial_attachment = if self.console { - FileHandleSerialAttachment::new( - std::io::stdin().as_raw_fd(), - std::io::stdout().as_raw_fd(), - ) - } else if self.verbose { - FileHandleSerialAttachment::new_write_only(std::io::stderr().as_raw_fd()) + FileHandleSerialAttachment::new(std::io::stdin().as_raw_fd(), output_fd) } else { - dev_null = std::fs::File::open("/dev/null") - .map_err(|e| anyhow::anyhow!("failed to open /dev/null: {}", e))?; - FileHandleSerialAttachment::new_write_only(dev_null.as_raw_fd()) + FileHandleSerialAttachment::new_write_only(output_fd) }; let serial = VirtioConsoleSerialPort::new_with_attachment(&serial_attachment); config.set_serial_ports(&[serial]); @@ -186,7 +240,13 @@ impl VmConfigBuilder { config.set_storage_devices(&[&block_device]); if let Some(fd) = self.network_fd { - let net_attachment = FileHandleNetworkAttachment::new(fd); + // IP-MTU 65535: hands smoltcp/host one ~64 KiB super-frame per + // syscall instead of segmenting at 1500 B. Must match + // `shuru_proxy::device::MTU` (Ethernet frame size = 14 + this). + // The associated socket buffers are sized in + // `shuru_proxy::create_socketpair`. + const NET_IP_MTU: i64 = 65535; + let net_attachment = FileHandleNetworkAttachment::new(fd, NET_IP_MTU); let net_device = VirtioNetworkDevice::new_with_attachment(&net_attachment); net_device.set_mac_address(&MACAddress::random_local()); config.set_network_devices(&[net_device]); @@ -215,23 +275,40 @@ impl VmConfigBuilder { config.set_socket_devices(&[socket_device]); config.set_entropy_devices(&[VirtioEntropyDevice::new()]); + config.set_memory_balloon_devices(&[VirtioMemoryBalloonDevice::new()]); config .validate() .map_err(|e| anyhow::anyhow!("VM configuration invalid: {}", e))?; + #[cfg(target_os = "linux")] + let machine = VirtualMachine::new(&config)?; + #[cfg(target_os = "macos")] + let machine = VirtualMachine::new(&config); Ok(Sandbox { - vm: Arc::new(VirtualMachine::new(&config)), + vm: Arc::new(GuestVm { + machine, + identity, + _serial_output: serial_output, + }), mounts: Mutex::new(mount_requests), + disk_operation: Mutex::new(()), }) } } // --- Sandbox --- +#[derive(Clone, Copy, PartialEq)] +pub enum AfterSave { + Thaw, + PowerOff, +} + pub struct Sandbox { - vm: Arc, + vm: Arc, mounts: Mutex>, + disk_operation: Mutex<()>, } impl Sandbox { @@ -245,12 +322,140 @@ impl Sandbox { .map_err(|e| anyhow::anyhow!("Failed to start VM: {}", e)) } + fn disk_request(&self, kind: u8, payload: &[u8]) -> Result { + let mut stream = self.connect_vsock()?; + stream.set_read_timeout(Some(SHUTDOWN_TIMEOUT))?; + stream.set_write_timeout(Some(SHUTDOWN_TIMEOUT))?; + frame::write_frame(&mut stream, kind, payload)?; + match frame::read_frame(&mut stream)? { + Some((k, data)) if k == kind + 1 => Ok(serde_json::from_slice(&data)?), + Some((frame::ERROR, data)) => bail!( + "guest disk operation failed: {}", + String::from_utf8_lossy(&data) + ), + _ => bail!("guest did not acknowledge disk operation"), + } + } + + pub fn trim(&self) -> Result { + let _guard = self.disk_operation.lock().unwrap(); + self.disk_request(frame::TRIM_REQ, &[]) + } + + /// Save a sealed disk through a file clone or a CAS index writer. + /// The writer must create the supplied path and finish before returning. + /// Returns the services the guest stopped, which only a PowerOff save + /// asks for. + pub fn save_disk( + &self, + dest: &str, + after: AfterSave, + copy: impl FnOnce(&str) -> Result<()>, + ) -> Result { + let _guard = self.disk_operation.lock().unwrap(); + crate::checkpoint::check_format(dest)?; + let mut bytes = [0; 8]; + getrandom::fill(&mut bytes).map_err(|_| anyhow::anyhow!("generate seal id"))?; + let id = u64::from_ne_bytes(bytes); + let req = shuru_proto::SealRequest { + id, + stop_services: after == AfterSave::PowerOff, + lease_ms: 300_000, + }; + let sealed = self + .disk_request(frame::SEAL_REQ, &serde_json::to_vec(&req)?) + .and_then(|response| match response { + shuru_proto::SealResponse::Sealed(services) => Ok(services), + _ => Err(anyhow::anyhow!("guest did not seal root")), + }); + let services = match sealed { + Ok(services) => services, + Err(e) => { + if after == AfterSave::PowerOff { + let _ = self.kill(); + } + return Err(e); + } + }; + let thaw = || -> Result<()> { + let resp: shuru_proto::SealResponse = self.disk_request( + frame::THAW_REQ, + &serde_json::to_vec(&shuru_proto::ThawRequest { id })?, + )?; + if resp != shuru_proto::SealResponse::Thawed { + bail!("seal expired before checkpoint copy completed"); + } + Ok(()) + }; + let result = crate::checkpoint::publish(dest, copy, || { + thaw()?; + if after == AfterSave::PowerOff { + self.kill()?; + } + Ok(()) + }); + if result.is_err() { + let _ = thaw(); + if after == AfterSave::PowerOff { + let _ = self.kill(); + } + } + result.map(|()| services) + } + + /// Ask the guest to stop its services and sync the disk (RFC 0004). + /// Returns what the guest stopped, or an error if it did not answer + /// within the shutdown timeout. + pub fn shutdown(&self) -> Result { + let stream = self.connect_vsock()?; + let _ = stream.set_read_timeout(Some(SHUTDOWN_TIMEOUT)); + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(shutdown_handshake(stream)); + }); + match rx.recv_timeout(SHUTDOWN_TIMEOUT + Duration::from_secs(5)) { + Ok(result) => result, + Err(_) => bail!( + "guest did not answer the shutdown request within {}s", + SHUTDOWN_TIMEOUT.as_secs() + ), + } + } + + /// Graceful stop: shutdown handshake, then power off. Falls back to a + /// plain power off when the guest does not answer. pub fn stop(&self) -> Result<()> { + // Only a running guest can answer; a VM that already died would + // just make the vsock connect retry until it gives up. + if matches!(self.vm.state(), VmState::Running) { + if let Err(e) = self.shutdown() { + tracing::warn!("guest shutdown failed, powering off: {}", e); + } + } + self.kill() + } + + /// Power off without asking the guest. Services get no SIGTERM and the + /// disk is not synced. + pub fn kill(&self) -> Result<()> { + if matches!(self.vm.state(), VmState::Stopped) { + return Ok(()); + } self.vm .stop() .map_err(|e| anyhow::anyhow!("Failed to stop VM: {}", e)) } + /// How much memory the guest keeps, in bytes. The guest hands the rest + /// back to the host through the balloon device and gets it back when + /// the target rises again. On a Linux host the guest also reports free + /// pages on its own, so idle memory returns without a target. + pub fn set_memory_target(&self, bytes: u64) -> Result<()> { + self.vm + .set_memory_target(bytes) + .map_err(|e| anyhow::anyhow!("Failed to set memory target: {}", e)) + } + /// Block until the guest control server accepts vsock connections. pub fn wait_ready(&self) -> Result<()> { let stream = self.connect_vsock()?; @@ -292,6 +497,23 @@ impl Sandbox { Ok(()) } + /// Run a command and hold its output in memory, up to MAX_BUFFERED per + /// stream. Output past that fails the call; stream it with `open_exec`. + pub fn exec_buffered( + &self, + argv: &[impl AsRef], + env: &HashMap, + ) -> Result { + let mut stdout = Capped::default(); + let mut stderr = Capped::default(); + let exit_code = self.exec_with_env(argv, env, &mut stdout, &mut stderr)?; + Ok(ExecOutput { + stdout: stdout.0, + stderr: stderr.0, + exit_code, + }) + } + /// Run a command non-interactively over vsock, streaming output to the /// provided writers. Returns the guest process exit code. pub fn exec( @@ -323,6 +545,7 @@ impl Sandbox { rows: None, cols: None, cwd: None, + id: None, }; frame::send_json(&mut writer, frame::EXEC_REQ, &req)?; @@ -366,18 +589,27 @@ impl Sandbox { }; frame::send_json(&mut writer, frame::READ_FILE_REQ, &req)?; - match frame::read_frame(&mut reader).context("reading read_file response")? { - Some((frame::READ_FILE_RESP, payload)) => Ok(payload), - Some((frame::ERROR, payload)) => { - bail!("{}", String::from_utf8_lossy(&payload)); - } - Some((other, _)) => { - bail!( - "unexpected frame type 0x{:02x} in read_file response", - other - ); + let mut data = Vec::new(); + loop { + match frame::read_frame(&mut reader).context("reading read_file response")? { + Some((frame::READ_FILE_RESP, chunk)) if chunk.is_empty() => return Ok(data), + Some((frame::READ_FILE_RESP, chunk)) => { + if data.len() + chunk.len() > MAX_BUFFERED { + bail!("{path} is larger than {}", too_large()); + } + data.extend_from_slice(&chunk); + } + Some((frame::ERROR, payload)) => { + bail!("{}", String::from_utf8_lossy(&payload)); + } + Some((other, _)) => { + bail!( + "unexpected frame type 0x{:02x} in read_file response", + other + ); + } + None => bail!("guest closed connection during read_file"), } - None => bail!("guest closed connection during read_file"), } } @@ -393,7 +625,9 @@ impl Sandbox { len: content.len() as u64, }; frame::send_json(&mut writer, frame::WRITE_FILE_REQ, &req)?; - frame::write_frame(&mut writer, frame::WRITE_FILE_DATA, content)?; + for chunk in content.chunks(frame::FILE_CHUNK) { + frame::write_frame(&mut writer, frame::WRITE_FILE_DATA, chunk)?; + } let (_msg_type, payload) = frame::read_frame(&mut reader) .context("reading write_file response")? @@ -609,18 +843,22 @@ impl Sandbox { /// Open a vsock connection for streaming exec. Returns the raw stream /// after sending mounts + ExecRequest. Caller manages I/O (reads /// STDOUT/STDERR/EXIT frames, writes STDIN/KILL frames). + /// Returns the stream and an id that `kill_exec` takes. pub fn open_exec( &self, argv: &[impl AsRef], env: &HashMap, cwd: Option<&str>, - ) -> Result { + ) -> Result<(TcpStream, u64)> { let stream = self.connect_vsock()?; let mut writer = stream.try_clone()?; let mut reader = stream.try_clone()?; self.send_mount_requests(&mut writer, &mut reader)?; + let mut bytes = [0; 8]; + getrandom::fill(&mut bytes).map_err(|_| anyhow::anyhow!("generate exec id"))?; + let id = u64::from_ne_bytes(bytes); let req = ExecRequest { argv: argv.iter().map(|s| s.as_ref().to_string()).collect(), env: env.clone(), @@ -628,10 +866,28 @@ impl Sandbox { rows: None, cols: None, cwd: cwd.map(|s| s.to_string()), + id: Some(id), }; frame::send_json(&mut writer, frame::EXEC_REQ, &req)?; - Ok(stream) + Ok((stream, id)) + } + + /// SIGKILL the process group of an exec from `open_exec`. Works when its + /// own connection is stalled, for example behind stdin it never reads. + pub fn kill_exec(&self, id: u64) -> Result<()> { + let mut stream = self.connect_vsock()?; + stream.set_read_timeout(Some(Duration::from_secs(5)))?; + stream.set_write_timeout(Some(Duration::from_secs(5)))?; + frame::send_json( + &mut stream, + frame::KILL_REQ, + &shuru_proto::KillRequest { id }, + )?; + match frame::read_frame(&mut stream)? { + Some((frame::KILL_RESP, _)) => Ok(()), + _ => bail!("guest did not acknowledge kill"), + } } /// Open a vsock connection for an interactive shell with PTY support. @@ -669,6 +925,7 @@ impl Sandbox { rows: Some(rows), cols: Some(cols), cwd: cwd.map(|s| s.to_string()), + id: None, }; frame::send_json(&mut writer, frame::EXEC_REQ, &req)?; @@ -716,6 +973,7 @@ impl Sandbox { rows: Some(rows), cols: Some(cols), cwd: None, + id: None, }; frame::send_json(&mut writer, frame::EXEC_REQ, &req)?; @@ -890,8 +1148,31 @@ impl Sandbox { } fn connect_vsock(&self) -> Result { - let state_rx = self.vm.state_channel(); - for attempt in 1..=1000 { + self.vm.connect_to_vsock_port(VSOCK_PORT) + } +} + +struct GuestVm { + machine: VirtualMachine, + identity: shuru_proto::hello::Identity, + _serial_output: std::os::fd::OwnedFd, +} + +impl std::ops::Deref for GuestVm { + type Target = VirtualMachine; + fn deref(&self) -> &Self::Target { + &self.machine + } +} + +impl GuestVm { + fn connect_to_vsock_port(&self, port: u32) -> Result { + let state_rx = self.machine.state_channel(); + let attempts = if port == VSOCK_PORT { 1000 } else { 1 }; + for attempt in 1..=attempts { + if matches!(self.machine.state(), VmState::Stopped | VmState::Error) { + bail!("VM stopped before guest connection"); + } // Check if VM died (e.g. guest mount failure -> reboot POWER_OFF) if let Ok(state) = state_rx.try_recv() { match state { @@ -902,13 +1183,22 @@ impl Sandbox { _ => {} } } - match self.vm.connect_to_vsock_port(VSOCK_PORT) { - Ok(s) => { + match self.machine.connect_to_vsock_port(port) { + Ok(mut s) => { let _ = s.set_nodelay(true); + let timeout = Some(Duration::from_secs(2)); + s.set_read_timeout(timeout)?; + s.set_write_timeout(timeout)?; + self.identity.connect(&mut s).context( + "the guest did not answer the connection hello; \ + its image is likely older than this shuru, run `shuru upgrade`", + )?; + s.set_read_timeout(None)?; + s.set_write_timeout(None)?; return Ok(s); } Err(e) => { - if attempt == 1000 { + if attempt == attempts { bail!( "Failed to connect to guest after {} attempts: {}", attempt, @@ -924,6 +1214,31 @@ impl Sandbox { } } +/// Upper bound on a graceful shutdown. Covers the slowest service stop +/// timeout an image can reasonably declare, plus the sync. +const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(90); + +fn shutdown_handshake(stream: TcpStream) -> Result { + let mut writer = stream.try_clone()?; + let mut reader = stream; + frame::write_frame(&mut writer, frame::SHUTDOWN_REQ, &[])?; + loop { + match frame::read_frame(&mut reader).context("waiting for guest shutdown")? { + Some((frame::SHUTDOWN_RESP, payload)) => { + return serde_json::from_slice(&payload).context("parsing shutdown response"); + } + Some((frame::ERROR, payload)) => { + bail!( + "guest refused shutdown: {}", + String::from_utf8_lossy(&payload) + ) + } + Some(_) => {} + None => bail!("guest closed the connection before answering shutdown"), + } + } +} + // --- Port forwarding --- /// Handle returned by `start_port_forwarding`. Signals all listener threads @@ -942,11 +1257,7 @@ impl Drop for PortForwardHandle { } } -fn handle_forward_connection( - tcp_stream: TcpStream, - vm: &VirtualMachine, - guest_port: u16, -) -> Result<()> { +fn handle_forward_connection(tcp_stream: TcpStream, vm: &GuestVm, guest_port: u16) -> Result<()> { let vsock_stream = open_forward_stream(vm, guest_port)?; // Bidirectional relay between TCP and vsock relay(tcp_stream, vsock_stream); @@ -956,7 +1267,7 @@ fn handle_forward_connection( /// Open a vsock forward channel to a TCP port inside the guest and complete /// the forward handshake, returning the connected stream. The stream is a raw /// bidirectional pipe to `127.0.0.1:guest_port` inside the guest. -fn open_forward_stream(vm: &VirtualMachine, guest_port: u16) -> Result { +fn open_forward_stream(vm: &GuestVm, guest_port: u16) -> Result { let mut vsock_stream = vm .connect_to_vsock_port(VSOCK_PORT_FORWARD) .map_err(|e| anyhow::anyhow!("vsock connect for port forward: {}", e))?; @@ -1000,3 +1311,7 @@ fn relay(a: TcpStream, b: TcpStream) { let _ = t1.join(); let _ = t2.join(); } + +#[cfg(test)] +#[path = "disk_tests.rs"] +mod disk_tests; diff --git a/crates/shuru-vm/src/serial.rs b/crates/shuru-vm/src/serial.rs new file mode 100644 index 0000000..d269dff --- /dev/null +++ b/crates/shuru-vm/src/serial.rs @@ -0,0 +1,81 @@ +use std::io::{self, Read, Write}; +use std::os::unix::net::UnixStream; + +// Kernel boot messages include the command line, even before the agent starts. +pub fn output(token: String, console: bool) -> io::Result { + let (mut reader, writer) = UnixStream::pair()?; + std::thread::Builder::new() + .name("shuru-console".into()) + .spawn(move || { + let mut output: Box = if console { + Box::new(io::stdout()) + } else { + Box::new(io::stderr()) + }; + let _ = redact(&mut reader, &mut output, token.as_bytes()); + })?; + Ok(writer) +} + +fn redact(reader: &mut impl Read, writer: &mut impl Write, secret: &[u8]) -> io::Result<()> { + let mut pending = Vec::new(); + let mut buf = [0; 4096]; + loop { + let n = reader.read(&mut buf)?; + if n == 0 { + writer.write_all(&pending)?; + return writer.flush(); + } + let mut output = Vec::with_capacity(n); + for byte in &buf[..n] { + pending.push(*byte); + while !secret.starts_with(&pending) { + output.push(pending[0]); + pending.remove(0); + } + if pending == secret { + output.extend_from_slice(b"[redacted]"); + pending.clear(); + } + } + writer.write_all(&output)?; + writer.flush()?; + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn redacts_across_reads() { + let input = b"cmdline shuru.token=abcdabcd next abcdabcd end"; + for chunk in 1..=input.len() { + struct Chunks { + bytes: Vec, + chunk: usize, + } + impl Read for Chunks { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let n = self.chunk.min(self.bytes.len()).min(buf.len()); + buf[..n].copy_from_slice(&self.bytes[..n]); + self.bytes.drain(..n); + Ok(n) + } + } + let mut output = Vec::new(); + redact( + &mut Chunks { + bytes: input.to_vec(), + chunk, + }, + &mut output, + b"abcdabcd", + ) + .unwrap(); + assert_eq!( + output, + b"cmdline shuru.token=[redacted] next [redacted] end" + ); + } + } +} diff --git a/crates/shuru-vm/src/workdisk.rs b/crates/shuru-vm/src/workdisk.rs new file mode 100644 index 0000000..a3ec78b --- /dev/null +++ b/crates/shuru-vm/src/workdisk.rs @@ -0,0 +1,342 @@ +//! The working disk of one VM run: a copy-on-write clone of a base image +//! that exists only as long as the process holding it. +//! +//! The clone is anonymous. On Linux it is created with `O_TMPFILE` and +//! never has a name; on macOS the framework needs a path, so the clone gets +//! a name for the two syscalls between creation and open, then is unlinked +//! and attached through `/dev/fd`. Either way the kernel frees the blocks +//! when the last descriptor closes, however the process ends, so nothing +//! has to be cleaned up afterwards. Same idiom as `tmpfile(3)` and QEMU +//! `-snapshot`. +//! +//! Cloning is a reflink where the filesystem has one (APFS, XFS with +//! reflink=1, btrfs), so a boot costs no disk and no copy time. Filesystems +//! without reflinks fall back to a full copy, with a note the first time so +//! an operator knows why boots are slow and disks fill up. + +use std::fs::File; +use std::os::unix::fs::FileExt; +use std::os::unix::io::{AsRawFd, RawFd}; + +use anyhow::{bail, Context, Result}; + +pub struct WorkDisk { + file: File, +} + +impl WorkDisk { + /// Clone `base` into an anonymous file on the filesystem holding `dir`, + /// grown to `size_mb` if that is larger than the base. + pub fn clone_from(base: &str, dir: &str, size_mb: u64) -> Result { + let target = crate::validate_disk_size(size_mb)?; + let disk = Self::clone_anonymous(base, dir)?; + let current = disk.file.metadata()?.len(); + if target < current { + bail!( + "disk size {}MB is smaller than the base image ({}MB)", + size_mb, + current / 1024 / 1024 + ); + } + if target > current { + disk.file.set_len(target)?; + } + Ok(disk) + } + + /// Lay a base image out for cheap clones, once. On APFS, growing a + /// clone or writing into its new space costs time in proportion to the + /// extent count of its source, and an image produced by build tools has + /// thousands of extents: half a second to two seconds on every boot. A + /// sequential copy already at the minimum disk size leaves clones + /// nothing to grow. A base at least that size is taken as prepared. + pub fn prepare_base(path: &str) -> Result<()> { + let target = crate::validate_disk_size(crate::MIN_DISK_MB)?; + if std::fs::metadata(path)?.len() >= target { + return Ok(()); + } + let tmp = format!("{}.prepare-{}", path, random_hex()); + let written = rewrite_sequential(path, &tmp, target) + .and_then(|()| std::fs::rename(&tmp, path).map_err(Into::into)); + if written.is_err() { + let _ = std::fs::remove_file(&tmp); + } + written.with_context(|| format!("preparing {}", path)) + } + + /// An empty anonymous file, for a VM whose disk is served some other + /// way and only needs a placeholder to attach. + pub fn empty(dir: &str) -> Result { + Ok(Self { + file: anonymous_file(dir)?, + }) + } + + /// The path the VM opens. Valid only in this process and only while + /// this value lives. + pub fn attach_path(&self) -> String { + #[cfg(target_os = "macos")] + { + format!("/dev/fd/{}", self.file.as_raw_fd()) + } + #[cfg(not(target_os = "macos"))] + { + format!("/proc/self/fd/{}", self.file.as_raw_fd()) + } + } + + pub fn as_raw_fd(&self) -> RawFd { + self.file.as_raw_fd() + } + + /// Clone the current contents to `dest`, which must not exist. Instant + /// on a reflink filesystem, a copy elsewhere. + pub fn save_to(&self, dest: &str) -> Result<()> { + #[cfg(target_os = "macos")] + { + shuru_darwin::clone_fd(self.file.as_raw_fd(), dest) + .with_context(|| format!("clone to {} failed", dest)) + } + #[cfg(not(target_os = "macos"))] + { + let out = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(dest) + .with_context(|| format!("creating {}", dest))?; + if shuru_linux::reflink(&out, &self.file)? { + return Ok(()); + } + copy_contents(&self.file, &out).with_context(|| format!("copying to {}", dest)) + } + } + + #[cfg(target_os = "macos")] + fn clone_anonymous(base: &str, dir: &str) -> Result { + let name = format!("{}/.work-{}", dir, random_hex()); + shuru_darwin::clone_path(base, &name) + .with_context(|| format!("clonefile({} -> {}) failed", base, name))?; + let opened = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(&name); + let _ = std::fs::remove_file(&name); + Ok(Self { file: opened? }) + } + + #[cfg(not(target_os = "macos"))] + fn clone_anonymous(base: &str, dir: &str) -> Result { + use std::sync::Once; + static COPY_NOTE: Once = Once::new(); + + let file = anonymous_file(dir)?; + let src = File::open(base).with_context(|| format!("opening {}", base))?; + if !shuru_linux::reflink(&file, &src)? { + COPY_NOTE.call_once(|| { + eprintln!( + "shuru: note: {} does not support reflinks, copying disk images instead; put the data directory on XFS or btrfs for instant clones", + dir + ); + }); + copy_contents(&src, &file).with_context(|| format!("copying {}", base))?; + } + Ok(Self { file }) + } +} + +/// A file with no name on the filesystem holding `dir`. +fn anonymous_file(dir: &str) -> Result { + #[cfg(not(target_os = "macos"))] + if let Some(file) = shuru_linux::open_anonymous(dir)? { + return Ok(file); + } + let name = format!("{}/.work-{}", dir, random_hex()); + let opened = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create_new(true) + .open(&name); + let _ = std::fs::remove_file(&name); + Ok(opened?) +} + +/// A plain read and write loop: `io::copy` may reflink on Linux, which +/// would carry the source extents over. +fn rewrite_sequential(src: &str, dst: &str, len: u64) -> Result<()> { + use std::io::{Read, Write}; + let mut input = File::open(src)?; + let mut output = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(dst)?; + let mut buf = vec![0u8; 8 << 20]; + loop { + let n = input.read(&mut buf)?; + if n == 0 { + break; + } + output.write_all(&buf[..n])?; + } + output.set_len(len)?; + output.sync_all()?; + Ok(()) +} + +/// Copy by offset so neither descriptor's position is touched. +#[cfg(not(target_os = "macos"))] +fn copy_contents(src: &File, dst: &File) -> Result<()> { + let len = src.metadata()?.len(); + let mut buf = vec![0u8; 1 << 20]; + let mut off = 0u64; + while off < len { + let n = src.read_at(&mut buf, off)?; + if n == 0 { + bail!("unexpected end of disk during checkpoint copy"); + } + if buf[..n].iter().any(|b| *b != 0) { + dst.write_all_at(&buf[..n], off)?; + } + off += n as u64; + } + dst.set_len(len)?; + Ok(()) +} + +fn random_hex() -> String { + let mut bytes = [0u8; 8]; + if let Ok(f) = File::open("/dev/urandom") { + let _ = f.read_exact_at(&mut bytes, 0); + } + let pid = std::process::id() as u64; + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos() as u64) + .unwrap_or(0); + format!( + "{:016x}", + u64::from_le_bytes(bytes) ^ pid.rotate_left(32) ^ nanos + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn temp_dir() -> String { + let dir = std::env::temp_dir().join(format!("shuru-workdisk-{}", random_hex())); + std::fs::create_dir_all(&dir).unwrap(); + dir.to_string_lossy().into_owned() + } + + fn names_in(dir: &str) -> Vec { + std::fs::read_dir(dir) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect() + } + + #[test] + fn a_clone_has_no_name_and_the_base_content() { + let dir = temp_dir(); + let base = format!("{}/base.img", dir); + std::fs::File::create(&base) + .unwrap() + .write_all(b"base image bytes") + .unwrap(); + + let disk = WorkDisk::clone_from(&base, &dir, 4096).unwrap(); + assert_eq!(names_in(&dir), vec!["base.img".to_string()]); + + let mut head = [0u8; 16]; + disk.file.read_exact_at(&mut head, 0).unwrap(); + assert_eq!(&head, b"base image bytes"); + assert_eq!(disk.file.metadata().unwrap().len(), 4096 * 1024 * 1024); + assert!(disk.attach_path().ends_with(&disk.as_raw_fd().to_string())); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn writes_go_to_the_clone_not_the_base() { + let dir = temp_dir(); + let base = format!("{}/base.img", dir); + std::fs::File::create(&base) + .unwrap() + .write_all(b"original") + .unwrap(); + + let disk = WorkDisk::clone_from(&base, &dir, 4096).unwrap(); + disk.file.write_all_at(b"changed!", 0).unwrap(); + assert_eq!(std::fs::read(&base).unwrap(), b"original"); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn save_to_captures_the_current_contents_and_refuses_to_overwrite() { + let dir = temp_dir(); + let base = format!("{}/base.img", dir); + std::fs::File::create(&base) + .unwrap() + .write_all(&vec![7u8; 4096]) + .unwrap(); + + let disk = WorkDisk::clone_from(&base, &dir, 4096).unwrap(); + disk.file.write_all_at(b"saved state", 100).unwrap(); + let dest = format!("{}/snap.img", dir); + disk.save_to(&dest).unwrap(); + + let file = File::open(&dest).unwrap(); + assert_eq!(file.metadata().unwrap().len(), 4096 * 1024 * 1024); + let mut saved = [0; 4096]; + file.read_exact_at(&mut saved, 0).unwrap(); + assert_eq!(&saved[100..111], b"saved state"); + assert_eq!(saved[0], 7); + assert!(disk.save_to(&dest).is_err()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn prepare_base_grows_the_base_once_and_keeps_its_contents() { + let dir = temp_dir(); + let base = format!("{}/base.img", dir); + std::fs::File::create(&base) + .unwrap() + .write_all(b"base image bytes") + .unwrap(); + + WorkDisk::prepare_base(&base).unwrap(); + let file = File::open(&base).unwrap(); + assert_eq!(file.metadata().unwrap().len(), 4096 * 1024 * 1024); + let mut head = [0u8; 16]; + file.read_exact_at(&mut head, 0).unwrap(); + assert_eq!(&head, b"base image bytes"); + assert_eq!(names_in(&dir), vec!["base.img".to_string()]); + + let inode = std::os::unix::fs::MetadataExt::ino(&file.metadata().unwrap()); + WorkDisk::prepare_base(&base).unwrap(); + let again = std::fs::metadata(&base).unwrap(); + assert_eq!(std::os::unix::fs::MetadataExt::ino(&again), inode); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn a_disk_smaller_than_the_base_is_refused() { + let dir = temp_dir(); + let base = format!("{}/base.img", dir); + std::fs::File::create(&base) + .unwrap() + .set_len(4097 * 1024 * 1024) + .unwrap(); + assert!(WorkDisk::clone_from(&base, &dir, 4096).is_err()); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn an_empty_disk_has_no_name_either() { + let dir = temp_dir(); + let disk = WorkDisk::empty(&dir).unwrap(); + assert!(names_in(&dir).is_empty()); + assert_eq!(disk.file.metadata().unwrap().len(), 0); + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/docs/rfcs/0003-docker-on-shuru-evidence.log b/docs/rfcs/0003-docker-on-shuru-evidence.log new file mode 100644 index 0000000..f846f15 --- /dev/null +++ b/docs/rfcs/0003-docker-on-shuru-evidence.log @@ -0,0 +1,179 @@ +running 9 tests +test container_cannot_reach_host_loopback_services ... $ shuru run --from docker +--- stdout --- +GATEWAY_BLOCKED +HOSTNAME_BLOCKED + +--- stderr --- +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... + +$ shuru run --from docker --allow-net +--- stdout --- +GATEWAY_BLOCKED +HOSTNAME_BLOCKED + +--- stderr --- +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... + +ok +test container_cannot_write_to_host_through_default_mount ... $ shuru run --from docker --mount /.tmplVg3uU:/workspace +--- stdout --- +from-host +WROTE_IN_GUEST +HOST_FS_ABSENT + +--- stderr --- +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... + +ok +test container_egress_follows_host_allowlist ... $ shuru run --from docker --allow-net --allow-host example.com +--- stdout --- +Example Domain +ALLOWED_OK +DOMAIN_BLOCKED +IP_BLOCKED +DNS_BLOCKED + +--- stderr --- +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... + +ok +test container_has_no_egress_without_allow_net ... $ shuru run --from docker +--- stdout --- +guest-eth0=Device "eth0" does not exist. +EGRESS_BLOCKED + +IP_BLOCKED + +--- stderr --- +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +wget: bad address 'example.com' +wget: can't connect to remote host (93.184.215.14): Network unreachable + +ok +test dockerd_runs_pinned_image_as_non_root_with_default_isolation ... $ shuru run --from docker +--- stdout --- +cgroupfs cgroup-v2 storage=overlayfs kernel=6.12.114 security=[name=seccomp,profile=builtin name=cgroupns name=no-new-privileges] +docker=29.9.2 containerd=v2.4.2 runc=1.5.2 +digest=nginxinc/nginx-unprivileged@sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7 user=101 +uid=101 +nginx-master-Uid: 101 101 101 101 +privileged=false +seccomp=[] +no-new-privs=NoNewPrivs: 1 +seccomp-mode=Seccomp: 2 + +--- stderr --- +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... + +ok +test http_through_port_forward_reaches_container ... $ shuru run --from docker -p 62055:8080 +--- stdout --- +READY +STOPPED + +--- stderr --- +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +shuru: forwarding 127.0.0.1:62055 -> guest:8080 + +$ GET http://127.0.0.1:62055/ +HTTP/1.1 200 OK +Server: nginx/1.31.7 +Date: Mon, 26 Oct 2026 01:52:48 GMT +Content-Type: text/html +Content-Length: 18 +Last-Modified: Mon, 26 Oct 2026 01:52:48 GMT +Connection: close +ETag: "6adeb270-12" +Accept-Ranges: bytes + +shuru-docker-3781 + +ok +test named_volume_persists_across_container_restart ... $ shuru run --from docker +--- stdout --- +after-container-restart=persisted-3781 +after-new-container=persisted-3781 + +--- stderr --- +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... + +ok +test named_volume_persists_across_vm_stop_start_via_checkpoint ... $ shuru checkpoint create shuru-test-docker-data-3781 --verbose --from docker +--- stdout --- + +--- stderr --- +shuru: creating working copy... +shuru: kernel=/Image +shuru: rootfs=/instances/3954/rootfs.ext4 (work copy) +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +shuru: VM created and validated successfully +shuru: VM started, waiting for guest... +random: crng init done +virtio_blk virtio1: 1/0/0 default/read/poll queues +virtio_blk virtio1: [vda] 8388608 512-byte logical blocks (4.29 GB/4.00 GiB) +rtc-pl031 20050000.pl031: registered as rtc0 +rtc-pl031 20050000.pl031: setting system clock to 2026-10-26T01:52:55 UTC (1792979575) +NET: Registered PF_INET6 protocol family +Segment Routing with IPv6 +In-situ OAM (IOAM) with IPv6 +sit: IPv6, IPv4 and MPLS over IPv4 tunneling driver +NET: Registered PF_PACKET protocol family +Bridge firewalling registered +NET: Registered PF_VSOCK protocol family +clk: Disabling unused clocks +Freeing unused kernel memory: 896K +Run /init as init process +EXT4-fs (vda): mounted filesystem 728de991-e6e3-4e1a-873d-bc29aeff5c07 r/w with ordered data mode. Quota mode: disabled. +shuru-guest[ 197.7ms]: starting as PID 1 +shuru-guest[ 198.0ms]: filesystems mounted +shuru-guest: no network device (sandbox mode) +shuru-guest[ 198.1ms]: networking ready +shuru-guest[ 201.2ms]: 1 service(s) starting +shuru-guest[ 201.2ms]: vsock listening on port 1024 +shuru-guest[ 201.3ms]: port forward listener on port 1025 +shuru-guest: accepted vsock connection +shuru-guest: service docker ready +docker0: port 1(vethc9efcd6) entered blocking state +docker0: port 1(vethc9efcd6) entered disabled state +vethc9efcd6: entered allmulticast mode +vethc9efcd6: entered promiscuous mode +eth0: renamed from vethd1f34a1 +docker0: port 1(vethc9efcd6) entered blocking state +docker0: port 1(vethc9efcd6) entered forwarding state +docker0: port 1(vethc9efcd6) entered disabled state +vethd1f34a1: renamed from eth0 +docker0: port 1(vethc9efcd6) entered disabled state +vethc9efcd6 (unregistering): left allmulticast mode +vethc9efcd6 (unregistering): left promiscuous mode +docker0: port 1(vethc9efcd6) entered disabled state +shuru-guest: accepted vsock connection +shuru-guest: services stopped in 1.0s (stopped: ["docker"], killed: []) +shuru: stopped services: docker +shuru: saving checkpoint 'shuru-test-docker-data-3781'... +shuru: checkpoint 'shuru-test-docker-data-3781' saved + +$ shuru run --from shuru-test-docker-data-3781 +--- stdout --- +after-vm-restart=vm-persisted-3781 + +--- stderr --- +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... + +ok +test resource_limits_apply_inside_containers ... $ shuru run --from docker +--- stdout --- +memory.max=67108864 +cpu.max=50000 100000 +pids.max=20 +oom-exit=137 +sh: can't fork: Resource temporarily unavailable + +--- stderr --- +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +WARNING: Your kernel does not support swap limit capabilities or the cgroup is not mounted. Memory limited without swap. +WARNING: Your kernel does not support swap limit capabilities or the cgroup is not mounted. Memory limited without swap. + +ok + +test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 78.79s diff --git a/docs/rfcs/0003-docker-on-shuru.md b/docs/rfcs/0003-docker-on-shuru.md new file mode 100644 index 0000000..80e6b8a --- /dev/null +++ b/docs/rfcs/0003-docker-on-shuru.md @@ -0,0 +1,339 @@ +# RFC 0003: Docker Engine inside a Shuru VM + +- Status: Implemented +- Date: 2026-10-26 +- Scope: kernel/shuru_defconfig, shuru-guest init, shuru-vm data dir, scripts, shuru-cli integration tests + +## Summary + +Run an unmodified Docker Engine inside a Shuru microVM and start one small, +maintained, non-root HTTP server image pinned by digest: + + nginxinc/nginx-unprivileged:1.31.7-alpine-slim + @sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7 + +Docker is not baked into the base rootfs. It is installed into a checkpoint +with the existing `shuru checkpoint create` flow, pinned to exact package +versions from the Docker apt repository, with the image pre-pulled. The guest +kernel gains the options a container runtime needs, and the guest init mounts +cgroup v2 and the tmpfs directories any container runtime assumes. + +Nothing about the sandbox boundary changes. Containers sit inside the VM, the +VM sits inside the existing proxy, port forwarding, and mount policy. Docker +isolation inside the VM stays at its defaults: seccomp on, no privileged +containers, no host socket, no host networking. + +## Motivation + +Agents increasingly need to run a service exactly as it ships, which means a +container image. Today that requires either translating a Dockerfile into apt +commands by hand or giving the agent the host Docker socket, which is the +opposite of what Shuru exists for. A VM that can run dockerd gives the agent +the real tool while the host keeps the same boundary it has for everything +else Shuru runs. + +## Non-goals + +- Docker Desktop style integration (host CLI talking to a VM daemon). The + daemon socket stays inside the guest. +- Rootless dockerd. The guest already runs everything as root inside a VM; + rootless would add complexity without changing the host boundary. +- Swarm, IPVS, overlay networks, VXLAN. Not enabled in the kernel. +- GPU or device passthrough. +- Nested virtualization. + +## Design + +### Kernel + +`kernel/shuru_defconfig` grows a "Container runtime support" section. The +required set is what moby's `contrib/check-config.sh` lists, minus IPVS: + +- cgroup v2 controllers runc uses: cpu with bandwidth (`CFS_BANDWIDTH`), + cpuset, io (`BLK_CGROUP`), memory and pids were already on. +- `CGROUP_BPF` and `BPF_SYSCALL`: on cgroup v2, device access rules are an + eBPF program. Without it runc refuses to start a non-userns container. + `BPF_UNPRIV_DEFAULT_OFF` keeps bpf() root-only inside the guest. +- `SECCOMP_FILTER`: the default Docker seccomp profile. +- `KEYS`: runc joins a session keyring per container. +- `VETH`, `BRIDGE_NETFILTER`, and the xt matches and targets dockerd installs + through iptables-nft (`addrtype`, `conntrack`, `comment`, `mark`, DNAT, + MASQUERADE, raw and mangle tables, plus the ip6 equivalents). `NFT_COMPAT` + is what lets iptables-nft load xt extensions at all. +- `EXT4_FS_SECURITY` for overlay2 security xattrs. +- `IKCONFIG_PROC` so `check-config.sh` can audit the running guest. + +The kernel version default moves from 6.12.17 to 6.12.114, the current 6.12 +LTS release. Both build scripts honour `SHURU_DATA_DIR` so a build can land +somewhere other than the installed assets. `scripts/build-kernel-hf.sh` builds +the same defconfig on a Hugging Face CPU job for hosts without Docker or disk. + +### Guest init + +`mount_filesystems` in shuru-guest additionally mounts: + +- `tmpfs` on `/run` (with `/run/lock`). Pid files and sockets no longer leak + into checkpoints, which otherwise leaves dockerd refusing to start from a + checkpoint because of a stale `/run/docker.pid`. +- `tmpfs` on `/dev/shm`, mode 1777. +- `cgroup2` on `/sys/fs/cgroup` with `nsdelegate,memory_recursiveprot`, and + every available controller enabled in the root `cgroup.subtree_control`. + +These are what a normal init does and they apply to every sandbox, not only +Docker ones. The behaviour change is that `/run` is no longer persisted into +checkpoints. + +### Docker checkpoint + +The `docker` recipe in `superhq-ai/recipes` (RFC 0005; originally a script +in this repository) creates the checkpoint by running, inside a VM with a +strict host allowlist: + +1. Enable the trixie-security and trixie-updates archives (debootstrap only + lists main) and `apt-get upgrade`. +2. Add the Docker apt repository with its signing key verified against a + pinned SHA-256, then install `docker-ce`, `docker-ce-cli` and + `containerd.io` at exact pinned versions. +3. Write `/etc/docker/daemon.json` with `no-new-privileges: true`. Everything + else stays at Docker defaults (iptables on, userland proxy on, seccomp + default profile, overlay2, json-file logs). +4. Declare dockerd as a guest service in `/etc/shuru/services/docker.json` + (RFC 0004), with `docker info` as the readiness check and a 30 second + stop timeout. The guest init starts it at boot and stops it before power + off. An earlier revision of this RFC used a `with-dockerd` wrapper + script for the same job; RFC 0004 replaced it. +5. Boot from that intermediate checkpoint and `docker pull` the pinned image + so the final checkpoint runs it without network. + +The checkpoint is the unit users reuse: `shuru run --from docker -p 8080:8080 +-- docker run ...`. + +### Data dir isolation + +`default_data_dir()` in shuru-vm reads `SHURU_DATA_DIR` before falling back +to `~/.local/share/shuru`. This is what lets the whole verification run with +its own kernel, initramfs, checkpoints and instances without touching the +installed assets, and what the integration tests use. + +### How traffic flows + +- Host `127.0.0.1:P` to container: existing vsock forward to guest + `127.0.0.1:P`, answered by docker-proxy, which relays to the container. +- Container egress: container to docker0, MASQUERADE in the guest, eth0, the + smoltcp proxy on the host. The proxy sees ordinary guest TCP and DNS, so the + domain allowlist and DNS pinning apply unchanged. Without `--allow-net` + there is no eth0 and containers have no egress at all. +- Container DNS: the default bridge copies the guest `/etc/resolv.conf`, + which points at 10.0.0.1, the proxy. A container that sets `--dns 8.8.8.8` + sends UDP to an address the proxy does not serve, so it gets nothing. + +## Security + +What stays the same: the host boundary. Containers run as processes of a +guest kernel inside the VM. The only paths out are the ones Shuru already +exposes to any guest process: the proxy (policy applied), explicit port +forwards (host dials in, never out), and explicit mounts (read-only overlay +by default). + +What this adds inside the guest: bpf() and keyctl() syscalls, seccomp, and +netfilter surface, all reachable only by root in the guest. The container to +guest-root boundary is Docker's own and is left at its defaults. The guest +root to host boundary is unchanged. + +Residual risks and limits: + +- A container escape to guest root gains what any sandbox command already + has. That is the same as today. +- The host disk only resets on exit. A long lived VM running a malicious + container has the same egress policy as a malicious `curl` in the guest. +- `--privileged`, `--cap-add`, and `-v /var/run/docker.sock` remain possible + for the guest user, as in any Docker install. They do not cross the VM + boundary, but the test suite never uses them and the docs say not to. +- Vulnerability posture is a snapshot. The RFC appendix records what was + scanned and what was found on the date of the run. + +## Testing + +`crates/shuru-cli/tests/docker.rs`, `#[ignore]` like the mount tests, boots +real VMs from the Docker checkpoint and checks: + +1. dockerd starts, reports cgroup v2, seccomp, overlay2; the pinned image runs + and `id -u` inside it is 101. +2. HTTP through `-p`: a page served by the container is fetched from the host. +3. Egress is blocked without `--allow-net`. +4. With `--allow-net --allow-host example.com`, the container fetches + example.com but a non-listed domain, a raw IP, and a custom DNS server all + fail. +5. The container cannot reach a listener on the host loopback. +6. A container bind-mounting a read-only host mount cannot write to the host. +7. `--memory`, `--cpus`, `--pids-limit` are reflected in the container cgroup + and a memory hog is OOM-killed. +8. A named volume survives container stop/start and a dockerd restart, and + survives a clean VM stop/start via checkpoint. + +Evidence from the real run is in the appendix. + +## Rollout + +The base OS image for the next release carries the new kernel and init, so +the checkpoint script works with stock assets. Until then, `SHURU_KERNEL` and +`SHURU_INITRD` point at a locally built pair. No behaviour changes for users +who never create the checkpoint, apart from `/run` being tmpfs. + +## Appendix A: Evidence + +Recorded on 2026-10-26 on a MacBook with an Apple M-series CPU, macOS 14.4.1, +8 GB RAM. All VM output below comes from real boots of the worktree CLI with +`SHURU_DATA_DIR=~/.local/share/shuru-docker-dev`, which holds its own kernel, +initramfs and checkpoint. The installed `~/.local/share/shuru` assets were not +modified. + +### Kernel + +Built with `scripts/build-kernel-hf.sh` on a Hugging Face `cpu-upgrade` job +(job `6ade955e1e112550ce7f01dc`, `hf jobs ssh --dry-run` printed the ssh +target). Source tarball verified against kernel.org `sha256sums.asc`. + + Image-6.12.114 sha256 bd0550e93ab88a9977af6c5f4ba231aeeff143cd874980a99b247ca0e2a0e633 9320960 bytes + +Guest boot on the new kernel: + + 6.12.114 + tmpfs on /run type tmpfs (rw,relatime,size=65536k,mode=755) + tmpfs on /dev/shm type tmpfs (rw,relatime,size=262144k) + cgroup2 on /sys/fs/cgroup type cgroup2 (rw,relatime,nsdelegate,memory_recursiveprot) + subtree_control: cpuset cpu io memory pids + +moby `contrib/check-config.sh` run inside the guest against `/proc/config.gz`: +every "Generally Necessary" item enabled except `NETFILTER_XT_MATCH_IPVS` +(swarm routing mesh only). Optional items left out are swarm overlay, IPVS, +SELinux, AppArmor, btrfs, macvlan, ipvlan, ftp conntrack helpers. + +### Checkpoint + +The Docker checkpoint build ran inside a VM allowed to reach only +deb.debian.org, security.debian.org, download.docker.com, `*.docker.io`, +`*.docker.com`, `*.cloudflarestorage.com`: + + Docker version 29.9.2, build a82e528 + containerd containerd v2.4.2 b7a25261e029d68978bbf290df0a62fc819b3c51 + runc version 1.5.2 + Digest: sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7 + nginxinc/nginx-unprivileged sha256:a8cb0f33... a8cb0f33d51f 6 days ago 30.3MB + +Image config read from the registry before pulling: `User: 101`, +`ExposedPorts: 8080/tcp`, arm64 manifest +`sha256:d99276c6bdeded8a96168938a0f1dfa158c492410019d0d3766884d0919d1b45`. + +### Runtime checks from the integration suite + +Final run with the `with-dockerd` wrapper, each test booting one VM from +the rebuilt checkpoint: + + test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 125.51s + +Rerun after RFC 0004 moved dockerd to a guest service, same nine tests: + + test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 78.79s + +Raw output of the rerun is in `docs/rfcs/0003-docker-on-shuru-evidence.log`. +Excerpts, identical between the two runs: + + cgroupfs cgroup-v2 storage=overlayfs kernel=6.12.114 security=[name=seccomp,profile=builtin name=cgroupns name=no-new-privileges] + docker=29.9.2 containerd=v2.4.2 runc=1.5.2 + digest=nginxinc/nginx-unprivileged@sha256:a8cb0f33... user=101 + uid=101 + nginx-master-Uid: 101 101 101 101 + privileged=false + seccomp=[] + no-new-privs=NoNewPrivs: 1 + seccomp-mode=Seccomp: 2 + +HTTP through `-p`: a page written in the guest and served by the container +came back on the host port with `HTTP/1.1 200` and `Server: nginx`; the VM +shut down after the container logged the probe request. + +Egress without `--allow-net`: + + guest-eth0=Device "eth0" does not exist + EGRESS_BLOCKED + IP_BLOCKED + +Egress with `--allow-net --allow-host example.com`: + + Example Domain + ALLOWED_OK + DOMAIN_BLOCKED (www.iana.org, DNS refused by the proxy) + IP_BLOCKED (1.1.1.1, not DNS-pinned) + DNS_BLOCKED (--dns 8.8.8.8, UDP not relayed) + +Host loopback listener on a random port, with and without `--allow-net`: +`GATEWAY_BLOCKED`, `HOSTNAME_BLOCKED`, and the host listener accepted no +connection. + +Read-only host mount bind-mounted into a container: the container read the +host file, wrote `escaped.txt` into the guest overlay, and the file never +appeared on the host. `-v /:/vmroot` shows no `/Users`, the VM root only. + +Resource limits: + + memory.max=67108864 + cpu.max=50000 100000 + pids.max=20 + oom-exit=137 (32 MB limit, 200 MB allocation) + (fork beyond --pids-limit 5 fails, "forked-all" never printed) + +Named volume: data written through `docker exec` survived `docker stop` and +`docker start`, survived stopping and restarting dockerd, and survived a +clean VM stop and start through `shuru checkpoint create --from docker`. + +### Vulnerability posture + +Image, Trivy 0.76 with the 2026-10-25 database, `linux/arm64`: + + alpine 3.24.2 Total: 3 (UNKNOWN: 2, MEDIUM: 1, HIGH: 0, CRITICAL: 0) + musl 1.2.6-r2 -> fixed in 1.2.6-r3 (CVE-2026-107891, no severity yet) + zlib 1.3.2-r0 -> fixed in 1.3.2-r1 (CVE-2026-85091, medium, heap overflow) + +Both fixes landed in Alpine after the 2026-10-19 image build. There is no +newer `nginx-unprivileged` build yet. Residual: a medium zlib issue in a +process that runs as uid 101 under seccomp inside a VM. Update the digest in +the recipe when the image rebuilds. + +Guest Debian layer, `debsecan --suite trixie --only-fixed` inside the VM: +36 fixable package entries (openssl, curl, expat, pcre2, brotli, xz) before +enabling the security archive, 0 after. 294 entries remain open with no +Debian fix available; that is the normal trixie backlog of low and +unimportant issues and is not actionable from this repo. + +Runtime components, GitHub advisory database queried for the exact +versions, with a control query against runc 1.1.11 returning 5 advisories: + + containerd/containerd/v2 2.4.2 0 advisories (latest release, 2026-10-13) + moby 29.9.2 0 advisories (latest release, 2026-10-20) + opencontainers/runc 1.5.2 0 advisories (latest release, 2026-09-25) + +Kernel 6.12.114 is the current 6.12 LTS release at build time. + +Not scanned: the proxy and CLI Rust dependencies (unchanged by this RFC) and +the kernel against per-CVE data (LTS point release is the mitigation). + +### Reproduction + + # kernel (HF job) and initramfs into an isolated data dir + export SHURU_DATA_DIR=~/.local/share/shuru-docker-dev + cargo build -p shuru-guest --target aarch64-unknown-linux-musl --release + ./scripts/build-kernel-hf.sh # or ./scripts/build-kernel.sh with Docker + ln -s ~/.local/share/shuru/rootfs.ext4 $SHURU_DATA_DIR/rootfs.ext4 + ./scripts/prepare-rootfs.sh # only builds the initramfs here + + # CLI: sign a copy, cargo test relinks target/debug/shuru + cargo build -p shuru-cli + cp target/debug/shuru /tmp/shuru-signed + codesign --entitlements shuru.entitlements --force -s - /tmp/shuru-signed + + export SHURU_KERNEL=$SHURU_DATA_DIR/Image SHURU_INITRD=$SHURU_DATA_DIR/initramfs.cpio.gz + export SHURU_ROOTFS=~/.local/share/shuru/rootfs.ext4 + /tmp/shuru-signed checkpoint create docker --recipe docker + SHURU_BIN=/tmp/shuru-signed SHURU_TEST_NET=1 \ + cargo test -p shuru-cli --test docker -- --ignored --test-threads=1 diff --git a/docs/rfcs/0004-guest-services-evidence.log b/docs/rfcs/0004-guest-services-evidence.log new file mode 100644 index 0000000..fcdacdd --- /dev/null +++ b/docs/rfcs/0004-guest-services-evidence.log @@ -0,0 +1,265 @@ + +running 4 tests +test service_ignoring_sigterm_is_killed_after_its_stop_timeout ... $ shuru checkpoint create shuru-test-svc-kill-87504 --verbose +--- stdout --- + +--- stderr --- +shuru: creating working copy... +shuru: kernel=/Image +shuru: rootfs=/instances/87505/rootfs.ext4 (work copy) +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +shuru: VM created and validated successfully +shuru: VM started, waiting for guest... +random: crng init done +virtio_blk virtio1: 1/0/0 default/read/poll queues +virtio_blk virtio1: [vda] 8388608 512-byte logical blocks (4.29 GB/4.00 GiB) +rtc-pl031 20050000.pl031: registered as rtc0 +rtc-pl031 20050000.pl031: setting system clock to 2026-10-26T00:57:55 UTC (1792976275) +NET: Registered PF_INET6 protocol family +Segment Routing with IPv6 +In-situ OAM (IOAM) with IPv6 +sit: IPv6, IPv4 and MPLS over IPv4 tunneling driver +NET: Registered PF_PACKET protocol family +Bridge firewalling registered +NET: Registered PF_VSOCK protocol family +clk: Disabling unused clocks +Freeing unused kernel memory: 896K +Run /init as init process +EXT4-fs (vda): mounted filesystem 728de991-e6e3-4e1a-873d-bc29aeff5c07 r/w with ordered data mode. Quota mode: disabled. +shuru-guest[ 147.2ms]: starting as PID 1 +shuru-guest[ 147.5ms]: filesystems mounted +shuru-guest: no network device (sandbox mode) +shuru-guest[ 147.7ms]: networking ready +shuru-guest[ 147.8ms]: vsock listening on port 1024 +shuru-guest[ 147.8ms]: port forward listener on port 1025 +shuru-guest: accepted vsock connection +shuru-guest: accepted vsock connection +shuru: saving checkpoint 'shuru-test-svc-kill-87504'... +shuru: checkpoint 'shuru-test-svc-kill-87504' saved + +$ shuru run --from shuru-test-svc-kill-87504 +--- stdout --- +RAN + +--- stderr --- +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +shuru: services ignored SIGTERM and were killed: stubborn + +ok +test service_that_fails_blocks_the_command_with_its_log ... $ shuru checkpoint create shuru-test-svc-fail-87504 --verbose +--- stdout --- + +--- stderr --- +shuru: creating working copy... +shuru: kernel=/Image +shuru: rootfs=/instances/87512/rootfs.ext4 (work copy) +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +shuru: VM created and validated successfully +shuru: VM started, waiting for guest... +random: crng init done +virtio_blk virtio1: 1/0/0 default/read/poll queues +virtio_blk virtio1: [vda] 8388608 512-byte logical blocks (4.29 GB/4.00 GiB) +rtc-pl031 20050000.pl031: registered as rtc0 +rtc-pl031 20050000.pl031: setting system clock to 2026-10-26T00:57:58 UTC (1792976278) +NET: Registered PF_INET6 protocol family +Segment Routing with IPv6 +In-situ OAM (IOAM) with IPv6 +sit: IPv6, IPv4 and MPLS over IPv4 tunneling driver +NET: Registered PF_PACKET protocol family +Bridge firewalling registered +NET: Registered PF_VSOCK protocol family +clk: Disabling unused clocks +Freeing unused kernel memory: 896K +Run /init as init process +EXT4-fs (vda): mounted filesystem 728de991-e6e3-4e1a-873d-bc29aeff5c07 r/w with ordered data mode. Quota mode: disabled. +shuru-guest[ 134.2ms]: starting as PID 1 +shuru-guest[ 134.6ms]: filesystems mounted +shuru-guest: no network device (sandbox mode) +shuru-guest[ 134.7ms]: networking ready +shuru-guest[ 134.8ms]: vsock listening on port 1024 +shuru-guest[ 134.8ms]: port forward listener on port 1025 +shuru-guest: accepted vsock connection +shuru-guest: accepted vsock connection +shuru: saving checkpoint 'shuru-test-svc-fail-87504'... +shuru: checkpoint 'shuru-test-svc-fail-87504' saved + +$ shuru run --from shuru-test-svc-fail-87504 +--- stdout --- + +--- stderr --- +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +guest error: service bad exited with status 3 +--- /run/shuru/log/bad.log --- +booting bad service +giving up + +ok +test service_with_delayed_ready_check_gates_the_command ... $ shuru checkpoint create shuru-test-svc-gate-87504 --verbose +--- stdout --- + +--- stderr --- +shuru: creating working copy... +shuru: kernel=/Image +shuru: rootfs=/instances/87519/rootfs.ext4 (work copy) +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +shuru: VM created and validated successfully +shuru: VM started, waiting for guest... +random: crng init done +virtio_blk virtio1: 1/0/0 default/read/poll queues +virtio_blk virtio1: [vda] 8388608 512-byte logical blocks (4.29 GB/4.00 GiB) +rtc-pl031 20050000.pl031: registered as rtc0 +rtc-pl031 20050000.pl031: setting system clock to 2026-10-26T00:57:59 UTC (1792976279) +NET: Registered PF_INET6 protocol family +Segment Routing with IPv6 +In-situ OAM (IOAM) with IPv6 +sit: IPv6, IPv4 and MPLS over IPv4 tunneling driver +NET: Registered PF_PACKET protocol family +Bridge firewalling registered +NET: Registered PF_VSOCK protocol family +clk: Disabling unused clocks +Freeing unused kernel memory: 896K +Run /init as init process +EXT4-fs (vda): mounted filesystem 728de991-e6e3-4e1a-873d-bc29aeff5c07 r/w with ordered data mode. Quota mode: disabled. +shuru-guest[ 134.4ms]: starting as PID 1 +shuru-guest[ 134.8ms]: filesystems mounted +shuru-guest: no network device (sandbox mode) +shuru-guest[ 134.9ms]: networking ready +shuru-guest[ 135.0ms]: vsock listening on port 1024 +shuru-guest[ 135.0ms]: port forward listener on port 1025 +shuru-guest: accepted vsock connection +shuru-guest: accepted vsock connection +shuru: saving checkpoint 'shuru-test-svc-gate-87504'... +shuru: checkpoint 'shuru-test-svc-gate-87504' saved + +$ shuru run --from shuru-test-svc-gate-87504 --verbose +--- stdout --- +GATED_OK +fast-log=fast-started +slow-alive=2 + +--- stderr --- +shuru: creating working copy... +shuru: kernel=/Image +shuru: rootfs=/instances/87521/rootfs.ext4 (work copy) +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +shuru: VM created and validated successfully +shuru: VM started, waiting for guest... +random: crng init done +virtio_blk virtio1: 1/0/0 default/read/poll queues +virtio_blk virtio1: [vda] 8388608 512-byte logical blocks (4.29 GB/4.00 GiB) +rtc-pl031 20050000.pl031: registered as rtc0 +rtc-pl031 20050000.pl031: setting system clock to 2026-10-26T00:58:00 UTC (1792976280) +NET: Registered PF_INET6 protocol family +Segment Routing with IPv6 +In-situ OAM (IOAM) with IPv6 +sit: IPv6, IPv4 and MPLS over IPv4 tunneling driver +NET: Registered PF_PACKET protocol family +Bridge firewalling registered +NET: Registered PF_VSOCK protocol family +clk: Disabling unused clocks +Freeing unused kernel memory: 896K +Run /init as init process +EXT4-fs (vda): mounted filesystem 728de991-e6e3-4e1a-873d-bc29aeff5c07 r/w with ordered data mode. Quota mode: disabled. +shuru-guest[ 130.3ms]: starting as PID 1 +shuru-guest[ 130.7ms]: filesystems mounted +shuru-guest: no network device (sandbox mode) +shuru-guest[ 130.8ms]: networking ready +shuru-guest[ 133.4ms]: 2 service(s) starting +shuru-guest[ 133.4ms]: vsock listening on port 1024 +shuru-guest[ 133.4ms]: port forward listener on port 1025 +shuru-guest: accepted vsock connection +shuru-guest: service slow ready +shuru-guest: accepted vsock connection +shuru-guest: services stopped in 0.1s (stopped: ["fast", "slow"], killed: []) +shuru: stopped services: fast, slow + +ok +test shutdown_delivers_sigterm_and_syncs_before_the_checkpoint_is_cloned ... $ shuru checkpoint create shuru-test-svc-term-87504 --verbose +--- stdout --- + +--- stderr --- +shuru: creating working copy... +shuru: kernel=/Image +shuru: rootfs=/instances/87526/rootfs.ext4 (work copy) +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +shuru: VM created and validated successfully +shuru: VM started, waiting for guest... +random: crng init done +virtio_blk virtio1: 1/0/0 default/read/poll queues +virtio_blk virtio1: [vda] 8388608 512-byte logical blocks (4.29 GB/4.00 GiB) +rtc-pl031 20050000.pl031: registered as rtc0 +rtc-pl031 20050000.pl031: setting system clock to 2026-10-26T00:58:02 UTC (1792976282) +NET: Registered PF_INET6 protocol family +Segment Routing with IPv6 +In-situ OAM (IOAM) with IPv6 +sit: IPv6, IPv4 and MPLS over IPv4 tunneling driver +NET: Registered PF_PACKET protocol family +Bridge firewalling registered +NET: Registered PF_VSOCK protocol family +clk: Disabling unused clocks +Freeing unused kernel memory: 896K +Run /init as init process +EXT4-fs (vda): mounted filesystem 728de991-e6e3-4e1a-873d-bc29aeff5c07 r/w with ordered data mode. Quota mode: disabled. +shuru-guest[ 136.4ms]: starting as PID 1 +shuru-guest[ 136.8ms]: filesystems mounted +shuru-guest: no network device (sandbox mode) +shuru-guest[ 136.8ms]: networking ready +shuru-guest[ 136.9ms]: vsock listening on port 1024 +shuru-guest[ 137.0ms]: port forward listener on port 1025 +shuru-guest: accepted vsock connection +shuru-guest: accepted vsock connection +shuru: saving checkpoint 'shuru-test-svc-term-87504'... +shuru: checkpoint 'shuru-test-svc-term-87504' saved + +$ shuru checkpoint create shuru-test-svc-term-derived-87504 --verbose --from shuru-test-svc-term-87504 +--- stdout --- +RAN_IN_BASE + +--- stderr --- +shuru: creating working copy... +shuru: kernel=/Image +shuru: rootfs=/instances/87528/rootfs.ext4 (work copy) +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +shuru: VM created and validated successfully +shuru: VM started, waiting for guest... +random: crng init done +virtio_blk virtio1: 1/0/0 default/read/poll queues +virtio_blk virtio1: [vda] 8388608 512-byte logical blocks (4.29 GB/4.00 GiB) +rtc-pl031 20050000.pl031: registered as rtc0 +rtc-pl031 20050000.pl031: setting system clock to 2026-10-26T00:58:02 UTC (1792976282) +NET: Registered PF_INET6 protocol family +Segment Routing with IPv6 +In-situ OAM (IOAM) with IPv6 +sit: IPv6, IPv4 and MPLS over IPv4 tunneling driver +NET: Registered PF_PACKET protocol family +Bridge firewalling registered +NET: Registered PF_VSOCK protocol family +clk: Disabling unused clocks +Freeing unused kernel memory: 896K +Run /init as init process +EXT4-fs (vda): mounted filesystem 728de991-e6e3-4e1a-873d-bc29aeff5c07 r/w with ordered data mode. Quota mode: disabled. +shuru-guest[ 130.2ms]: starting as PID 1 +shuru-guest[ 130.5ms]: filesystems mounted +shuru-guest: no network device (sandbox mode) +shuru-guest[ 130.6ms]: networking ready +shuru-guest[ 133.3ms]: 1 service(s) starting +shuru-guest[ 133.4ms]: vsock listening on port 1024 +shuru-guest[ 133.4ms]: port forward listener on port 1025 +shuru-guest: accepted vsock connection +shuru-guest: accepted vsock connection +shuru-guest: services stopped in 1.0s (stopped: ["trap"], killed: []) +shuru: stopped services: trap +shuru: saving checkpoint 'shuru-test-svc-term-derived-87504'... +shuru: checkpoint 'shuru-test-svc-term-derived-87504' saved + +$ shuru run --from shuru-test-svc-term-derived-87504 +--- stdout --- +got-term + +--- stderr --- +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... + +ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 9.49s + diff --git a/docs/rfcs/0004-guest-services.md b/docs/rfcs/0004-guest-services.md new file mode 100644 index 0000000..e2ff366 --- /dev/null +++ b/docs/rfcs/0004-guest-services.md @@ -0,0 +1,258 @@ +# RFC 0004: Guest services + +- Status: Implemented +- Date: 2026-10-26 +- Scope: shuru-guest, shuru-proto, shuru-vm, shuru-cli, Docker checkpoint script +- Depends on: RFC 0003 + +## Summary + +Let an image declare long running processes that the guest starts at boot, +waits on before running the user command, and stops cleanly before the VM +goes away. A service is one JSON file under `/etc/shuru/services/` in the +rootfs or checkpoint. Init owns the lifecycle. The host asks the guest to +shut down over vsock before it stops the VM, and the guest answers only +after every service has exited and the disk is synced. + +With this in place `shuru run --from docker -- docker run ...` just works. +The `with-dockerd` wrapper from RFC 0003 goes away. + +## Motivation + +The guest has no init system. PID 1 is shuru-init, which mounts +filesystems, brings up networking and serves the vsock control protocol. +Nothing starts daemons and nothing stops them. RFC 0003 worked around this +with a shell wrapper that started dockerd around each command, which has +four problems: + +- Every daemon needs its own copy of the wrapper and users must remember to + type it. `shuru run --from docker -- docker ps` fails with a socket error. +- Shutdown runs only on the happy path. An interrupted `shuru run`, a stop + from the SDK, or a host side error leaves the daemon without SIGTERM and + the disk without a sync. For a checkpoint build that means a cloned disk + with half written state. +- The host already hard stops the VM. There is no cooperative shutdown at + all, so even the `sync` before a checkpoint clone is a separate exec that + can be skipped by any error between the command and the stop. +- Startup cost is paid per command instead of per boot. + +Other sandboxes solve this at the platform level. E2B templates take a +start command and a ready command, and the ready command is retried until +it exits 0 before the sandbox is handed over. Sprites declare services that +start on boot and expose a lifecycle state. Shuru should offer the same +contract: declare once in the image, the runtime handles start, readiness +and stop. + +## Non-goals + +- Restart on failure. A service that exits is reported, not restarted. +- Dependencies between services. All services start in parallel. +- Per service users or cgroups. Services run as root in the root cgroup, + the same as the user command today. +- Services that depend on host mounts. Mounts arrive over vsock after boot, + services start before the first vsock connection. +- Changing the on disk format of checkpoints. + +## Design + +### Service definition + +A service is a JSON file at `/etc/shuru/services/.json`. The name is +the file stem and must match `[a-z0-9][a-z0-9-]*`. + +```json +{ + "exec": ["dockerd"], + "ready": ["docker", "info"], + "ready_timeout_secs": 60, + "stop_timeout_secs": 30, + "env": {"DOCKER_RAMDISK": "1"}, + "cwd": "/" +} +``` + +| field | required | default | meaning | +|---|---|---|---| +| `exec` | yes | | argv of the process to run. Resolved on the init PATH. | +| `ready` | no | none | argv retried every 250 ms until it exits 0. Without it the service is ready as soon as it is spawned. | +| `ready_timeout_secs` | no | 30 | give up on `ready` after this long. | +| `stop_timeout_secs` | no | 15 | time between SIGTERM and SIGKILL at shutdown. | +| `env` | no | `{}` | extra environment for `exec` and `ready`. | +| `cwd` | no | `/` | working directory for `exec`. | + +Files that fail to parse are reported on the console and skipped. They do +not block the boot or the user command, because a broken file in an image +should be debuggable with `shuru run`. + +### Lifecycle in the guest + +After networking is up and before the vsock listeners are created, init +reads the services directory and spawns every service. Each gets its own +process group and its stdout and stderr appended to +`/run/shuru/log/.log`. The log lives on tmpfs so it never lands in a +checkpoint. + +Each service has one of four states: starting, ready, failed or stopped. A +thread per service runs the ready check until it passes or the timeout is +hit. The zombie reaper already in init records the exit of any service pid, +so a daemon that dies at any point moves to failed with its exit status. + +Before spawning a user command, in both the piped and the tty exec paths, +init waits until no service is still starting. If every service is ready +the command runs as before. If any service is failed, init answers the exec +with an ERROR frame that names the service, the reason, and the last lines +of its log, and does not run the command. File, mount, watch and port +forward requests are never gated, so the log of a failed service can always +be read from the host. + +### Shutdown handshake + +Two new frames on the control connection: + +- `SHUTDOWN_REQ` (0x60), empty payload. +- `SHUTDOWN_RESP` (0x61), JSON `ShutdownResponse { stopped: [name], killed: [name] }`. + +On `SHUTDOWN_REQ` the guest sends SIGTERM to every running service, waits +up to each one's `stop_timeout_secs` for it to exit, sends SIGKILL to the +process group of any that remain, calls `sync`, and then replies. Services +are stopped in parallel so the wall clock cost is the slowest service, not +the sum. Once a shutdown has started, new exec requests are refused. + +On the host, `Sandbox::stop()` becomes a graceful stop. It opens a control +connection, sends `SHUTDOWN_REQ`, waits for the reply with a read timeout, +and then performs the hard stop it did before. Anything that goes wrong on +the way, including an old guest that does not know the frame, falls through +to the hard stop so a stop can never hang. `Sandbox::kill()` is the old +behaviour for callers that want it. + +Because the handshake includes `sync`, the `sync_before_stop` flag in the +CLI and the separate `sync` exec go away. Checkpoint creation and plain runs +share one path, and the SDK gets the same shutdown through its existing +`stop()` and `Drop`. + +### Docker checkpoint + +The Docker recipe in `superhq-ai/recipes` (RFC 0005) installs Docker and +declares `/etc/shuru/services/docker.json` in its first step. Its second +step boots from that result, where the service model has already started +dockerd, and runs `docker pull` for the pinned image. The build itself is +the first consumer of the feature. + +The `daemon.json` from RFC 0003 is unchanged. The `with-dockerd` helper is +no longer installed. + +### Traffic and isolation + +Nothing in this RFC changes networking or mounts. Services run inside the +VM with the same proxy, allowlist and mount rules as the user command. + +## Security + +- Services come from the image. Anything that can write to + `/etc/shuru/services/` in a checkpoint could already run arbitrary code + in that checkpoint, so this adds no new trust boundary. +- Service logs are on tmpfs under `/run`, which is size capped at 64 MB. + A very chatty service fills its log and then gets write errors, it does + not fill the disk or the checkpoint. +- Shutdown is bounded. The guest waits at most `stop_timeout_secs` per + service and the host waits at most 90 seconds for the reply before it + hard stops. A hung service cannot hold a VM open. +- Readiness failure fails closed for exec. A command never runs against a + half started service. + +Residual risks: + +- SIGKILL after the stop timeout can still leave a service with unflushed + state. The sync that follows only covers what reached the page cache. +- The host hard stop on timeout is the same crash semantics as before. The + improvement is that it is now the exception rather than the rule. + +## Testing + +`crates/shuru-cli/tests/services.rs` builds throwaway checkpoints from tiny +shell services and checks, against a real VM: + +- a service with a delayed ready check gates the user command, a second + service without a ready check is up immediately, and both are stopped + at shutdown with their logs under `/run/shuru/log` +- a service that exits before its ready check passes fails the exec with + an error naming it and the tail of its log, and the command never runs +- SIGTERM reaches a service at shutdown and the disk is synced before the + checkpoint is cloned, proven by a marker written from the TERM trap and + read back from a derived checkpoint +- a service that ignores SIGTERM is killed after its stop timeout and the + CLI reports it + +`crates/shuru-cli/tests/docker.rs` drops every `with-dockerd` and runs +plain `docker` commands. The nine RFC 0003 tests pass unchanged otherwise. + +## Rollout + +The guest and host change together in one release. An old host against a +new guest never sends `SHUTDOWN_REQ` and behaves as today. An old guest +skips unknown frames without answering, so a new host against it waits out +the shutdown timeout and then hard stops. Mixed versions degrade to the old +behaviour at the cost of a slow stop, and the asset version check keeps the +two in step for stock installs. + +## Appendix A: Evidence + +Captured on 2026-10-26 against real VMs on an M-series Mac with 8 GB RAM, +kernel 6.12.114 (sha256 `bd0550e9...`), isolated data dir, codesigned debug +CLI. Full `--nocapture` output is in +`docs/rfcs/0004-guest-services-evidence.log`; the Docker suite rerun is in +`docs/rfcs/0003-docker-on-shuru-evidence.log`. + +Service suite, four tests, one run: + +``` +test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 9.49s +``` + +Lines that show each behaviour, from that log: + +``` +shuru-guest: service slow ready +GATED_OK +fast-log=fast-started +slow-alive=2 +shuru-guest: services stopped in 0.1s (stopped: ["fast", "slow"], killed: []) +shuru: stopped services: fast, slow + +guest error: service bad exited with status 3 +--- /run/shuru/log/bad.log --- +booting bad service +giving up + +shuru-guest: services stopped in 1.0s (stopped: ["trap"], killed: []) +shuru: stopped services: trap +got-term + +shuru: services ignored SIGTERM and were killed: stubborn +``` + +Docker checkpoint build from the docker recipe (RFC 0005). The service +file is written in step one, so dockerd is already up when step two pulls: + +``` +shuru: building recipe docker from /docker@dcbbf2c (2 steps) +shuru: step 1/2: install.sh +shuru: checkpoint 'docker-step1' saved +shuru: step 2/2: pull.sh +Digest: sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7 +shuru: checkpoint 'docker' saved +shuru: checkpoint 'docker-step1' deleted +real 2m3.869s +``` + +RFC 0003 Docker suite, nine tests, against that checkpoint with no +`with-dockerd` anywhere: + +``` +test result: ok. 9 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 78.79s +``` + +The same nine tests took 125.51s with the wrapper, because every script +paid a daemon start and stop. The checkpoint persistence test additionally +asserts `shuru: stopped services: docker` on the create side, which is the +handshake running before the clone. diff --git a/docs/rfcs/0005-recipes-evidence.log b/docs/rfcs/0005-recipes-evidence.log new file mode 100644 index 0000000..b5f00be --- /dev/null +++ b/docs/rfcs/0005-recipes-evidence.log @@ -0,0 +1,799 @@ +=== recipes suite (local recipes and the gated docker recipe) + +running 6 tests +test create_builds_under_a_chosen_name_and_refuses_a_command ... $ shuru checkpoint create shuru-test-recipe-named-4199 --recipe /.tmpjsyqiE -- echo no +--- stdout --- + +--- stderr --- +Error: --recipe builds from the recipe steps; drop the command + +$ shuru checkpoint create shuru-test-recipe-named-4199 --recipe /.tmpjsyqiE +--- stdout --- + +--- stderr --- +shuru: building recipe irrelevant from /.tmpjsyqiE@5cf1c28 (1 step) +shuru: step 1/1: build.sh +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +shuru: saving checkpoint 'shuru-test-recipe-named-4199'... +shuru: checkpoint 'shuru-test-recipe-named-4199' saved + +$ shuru run --from shuru-test-recipe-named-4199 -- cat /opt/marker +--- stdout --- +built-by-recipe + +--- stderr --- +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... + +$ shuru checkpoint create shuru-test-recipe-named-4199 --recipe /.tmpjsyqiE +--- stdout --- + +--- stderr --- +Error: checkpoint shuru-test-recipe-named-4199 already exists, delete it first + +ok +test docker_recipe_builds_and_runs_a_container ... $ shuru checkpoint create shuru-test-recipe-docker-4199 --recipe /docker +--- stdout --- +==> Upgrading Debian base (main, security and updates archives) +Preconfiguring packages ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9863 files and directories currently installed.) +Preparing to unpack .../libc6_2.41-12+deb13u4_arm64.deb ... +Unpacking libc6:arm64 (2.41-12+deb13u4) over (2.41-12+deb13u3) ... +Setting up libc6:arm64 (2.41-12+deb13u4) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9863 files and directories currently installed.) +Preparing to unpack .../base-files_13.8+deb13u7_arm64.deb ... +Unpacking base-files (13.8+deb13u7) over (13.8+deb13u6) ... +Setting up base-files (13.8+deb13u7) ... +Installing new version of config file /etc/debian_version ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../bash_5.2.37-2+b10_arm64.deb ... +update-alternatives: warning: alternative /usr/share/man/man7/bash-builtins.7.gz (part of link group builtins.7.gz) doesn't exist; removing from list of alternatives +update-alternatives: warning: /etc/alternatives/builtins.7.gz is dangling; it will be updated with best choice +Unpacking bash (5.2.37-2+b10) over (5.2.37-2+b9) ... +Setting up bash (5.2.37-2+b10) ... +update-alternatives: error: alternative path /usr/share/man/man7/bash-builtins.7.gz doesn't exist +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../bsdutils_1%3a2.41.5-0+deb13u1_arm64.deb ... +Unpacking bsdutils (1:2.41.5-0+deb13u1) over (1:2.41-5) ... +Setting up bsdutils (1:2.41.5-0+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../gzip_1.13-1+deb13u1_arm64.deb ... +Unpacking gzip (1.13-1+deb13u1) over (1.13-1) ... +Setting up gzip (1.13-1+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../libperl5.40_5.40.1-6+deb13u1_arm64.deb ... +Unpacking libperl5.40:arm64 (5.40.1-6+deb13u1) over (5.40.1-6) ... +Preparing to unpack .../perl_5.40.1-6+deb13u1_arm64.deb ... +Unpacking perl (5.40.1-6+deb13u1) over (5.40.1-6) ... +Preparing to unpack .../perl-base_5.40.1-6+deb13u1_arm64.deb ... +Unpacking perl-base (5.40.1-6+deb13u1) over (5.40.1-6) ... +Setting up perl-base (5.40.1-6+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../perl-modules-5.40_5.40.1-6+deb13u1_all.deb ... +Unpacking perl-modules-5.40 (5.40.1-6+deb13u1) over (5.40.1-6) ... +Preparing to unpack .../liblastlog2-2_2.41.5-0+deb13u1_arm64.deb ... +Unpacking liblastlog2-2:arm64 (2.41.5-0+deb13u1) over (2.41-5) ... +Setting up liblastlog2-2:arm64 (2.41.5-0+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../libuuid1_2.41.5-0+deb13u1_arm64.deb ... +Unpacking libuuid1:arm64 (2.41.5-0+deb13u1) over (2.41-5) ... +Setting up libuuid1:arm64 (2.41.5-0+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../libblkid1_2.41.5-0+deb13u1_arm64.deb ... +Unpacking libblkid1:arm64 (2.41.5-0+deb13u1) over (2.41-5) ... +Setting up libblkid1:arm64 (2.41.5-0+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../libmount1_2.41.5-0+deb13u1_arm64.deb ... +Unpacking libmount1:arm64 (2.41.5-0+deb13u1) over (2.41-5) ... +Setting up libmount1:arm64 (2.41.5-0+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../libsmartcols1_2.41.5-0+deb13u1_arm64.deb ... +Unpacking libsmartcols1:arm64 (2.41.5-0+deb13u1) over (2.41-5) ... +Setting up libsmartcols1:arm64 (2.41.5-0+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../util-linux_2.41.5-0+deb13u1_arm64.deb ... +Unpacking util-linux (2.41.5-0+deb13u1) over (2.41-5) ... +Setting up util-linux (2.41.5-0+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9877 files and directories currently installed.) +Preparing to unpack .../mount_2.41.5-0+deb13u1_arm64.deb ... +Unpacking mount (2.41.5-0+deb13u1) over (2.41-5) ... +Preparing to unpack .../libsqlite3-0_3.46.1-7+deb13u2_arm64.deb ... +Unpacking libsqlite3-0:arm64 (3.46.1-7+deb13u2) over (3.46.1-7+deb13u1) ... +Setting up libsqlite3-0:arm64 (3.46.1-7+deb13u2) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9877 files and directories currently installed.) +Preparing to unpack .../libc-bin_2.41-12+deb13u4_arm64.deb ... +Unpacking libc-bin (2.41-12+deb13u4) over (2.41-12+deb13u3) ... +Setting up libc-bin (2.41-12+deb13u4) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9877 files and directories currently installed.) +Preparing to unpack .../libaudit-common_1%3a4.0.2-2+deb13u1_all.deb ... +Unpacking libaudit-common (1:4.0.2-2+deb13u1) over (1:4.0.2-2) ... +Setting up libaudit-common (1:4.0.2-2+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9877 files and directories currently installed.) +Preparing to unpack .../libaudit1_1%3a4.0.2-2+deb13u1_arm64.deb ... +Unpacking libaudit1:arm64 (1:4.0.2-2+deb13u1) over (1:4.0.2-2+b2) ... +Setting up libaudit1:arm64 (1:4.0.2-2+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9876 files and directories currently installed.) +Preparing to unpack .../login_1%3a4.16.0-2+really2.41.5-0+deb13u1_arm64.deb ... +Unpacking login (1:4.16.0-2+really2.41.5-0+deb13u1) over (1:4.16.0-2+really2.41-5) ... +Preparing to unpack .../libexpat1_2.9.0-1~deb13u1_arm64.deb ... +Unpacking libexpat1:arm64 (2.9.0-1~deb13u1) over (2.7.1-2) ... +Preparing to unpack .../libcap2_1%3a2.75-10+deb13u1+b3_arm64.deb ... +Unpacking libcap2:arm64 (1:2.75-10+deb13u1+b3) over (1:2.75-10+deb13u1+b1) ... +Setting up libcap2:arm64 (1:2.75-10+deb13u1+b3) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9878 files and directories currently installed.) +Preparing to unpack .../liblzma5_5.8.1-1+deb13u2_arm64.deb ... +Unpacking liblzma5:arm64 (5.8.1-1+deb13u2) over (5.8.1-1+deb13u1) ... +Setting up liblzma5:arm64 (5.8.1-1+deb13u2) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9878 files and directories currently installed.) +Preparing to unpack .../libpcre2-8-0_10.46-1~deb13u3_arm64.deb ... +Unpacking libpcre2-8-0:arm64 (10.46-1~deb13u3) over (10.46-1~deb13u1) ... +Setting up libpcre2-8-0:arm64 (10.46-1~deb13u3) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9878 files and directories currently installed.) +Preparing to unpack .../openssl-provider-legacy_3.5.7-1~deb13u3_arm64.deb ... +Unpacking openssl-provider-legacy (3.5.7-1~deb13u3) over (3.5.6-1~deb13u2) ... +Setting up openssl-provider-legacy (3.5.7-1~deb13u3) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9878 files and directories currently installed.) +Preparing to unpack .../libssl3t64_3.5.7-1~deb13u3_arm64.deb ... +Unpacking libssl3t64:arm64 (3.5.7-1~deb13u3) over (3.5.6-1~deb13u2) ... +Setting up libssl3t64:arm64 (3.5.7-1~deb13u3) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9878 files and directories currently installed.) +Preparing to unpack .../00-tzdata_2026e-0+deb13u1_all.deb ... +Unpacking tzdata (2026e-0+deb13u1) over (2026b-0+deb13u1) ... +Preparing to unpack .../01-openssl_3.5.7-1~deb13u3_arm64.deb ... +Unpacking openssl (3.5.7-1~deb13u3) over (3.5.6-1~deb13u2) ... +Preparing to unpack .../02-ca-certificates_20250419+deb13u1_all.deb ... +Unpacking ca-certificates (20250419+deb13u1) over (20250419) ... +Preparing to unpack .../03-xz-utils_5.8.1-1+deb13u2_arm64.deb ... +Unpacking xz-utils (5.8.1-1+deb13u2) over (5.8.1-1+deb13u1) ... +Preparing to unpack .../04-libbrotli1_1.1.0-2+deb13u1_arm64.deb ... +Unpacking libbrotli1:arm64 (1.1.0-2+deb13u1) over (1.1.0-2+b7) ... +Preparing to unpack .../05-libssh2-1t64_1.11.1-1+deb13u2_arm64.deb ... +Unpacking libssh2-1t64:arm64 (1.11.1-1+deb13u2) over (1.11.1-1+deb13u1) ... +Preparing to unpack .../06-curl_8.14.1-2+deb13u7_arm64.deb ... +Unpacking curl (8.14.1-2+deb13u7) over (8.14.1-2+deb13u4) ... +Preparing to unpack .../07-libcurl4t64_8.14.1-2+deb13u7_arm64.deb ... +Unpacking libcurl4t64:arm64 (8.14.1-2+deb13u7) over (8.14.1-2+deb13u4) ... +Preparing to unpack .../08-jq_1.7.1-6+deb13u4_arm64.deb ... +Unpacking jq (1.7.1-6+deb13u4) over (1.7.1-6+deb13u2) ... +Preparing to unpack .../09-libjq1_1.7.1-6+deb13u4_arm64.deb ... +Unpacking libjq1:arm64 (1.7.1-6+deb13u4) over (1.7.1-6+deb13u2) ... +Preparing to unpack .../10-libcap2-bin_1%3a2.75-10+deb13u1+b3_arm64.deb ... +Unpacking libcap2-bin (1:2.75-10+deb13u1+b3) over (1:2.75-10+deb13u1+b1) ... +Preparing to unpack .../11-libcom-err2_1.47.2-3+b12_arm64.deb ... +Unpacking libcom-err2:arm64 (1.47.2-3+b12) over (1.47.2-3+b11) ... +Preparing to unpack .../12-libcurl3t64-gnutls_8.14.1-2+deb13u7_arm64.deb ... +Unpacking libcurl3t64-gnutls:arm64 (8.14.1-2+deb13u7) over (8.14.1-2+deb13u4) ... +Setting up libexpat1:arm64 (2.9.0-1~deb13u1) ... +Setting up libjq1:arm64 (1.7.1-6+deb13u4) ... +Setting up libbrotli1:arm64 (1.1.0-2+deb13u1) ... +Setting up libcom-err2:arm64 (1.47.2-3+b12) ... +Setting up tzdata (2026e-0+deb13u1) ... + +Current default time zone: 'Etc/UTC' +Local time is now: Mon Oct 26 01:54:54 UTC 2026. +Universal Time is now: Mon Oct 26 01:54:54 UTC 2026. +Run 'dpkg-reconfigure tzdata' if you wish to change it. + +Setting up libcap2-bin (1:2.75-10+deb13u1+b3) ... +Setting up xz-utils (5.8.1-1+deb13u2) ... +Setting up mount (2.41.5-0+deb13u1) ... +Setting up perl-modules-5.40 (5.40.1-6+deb13u1) ... +Setting up jq (1.7.1-6+deb13u4) ... +Setting up libssh2-1t64:arm64 (1.11.1-1+deb13u2) ... +Setting up openssl (3.5.7-1~deb13u3) ... +Setting up login (1:4.16.0-2+really2.41.5-0+deb13u1) ... +Setting up libcurl4t64:arm64 (8.14.1-2+deb13u7) ... +Setting up libcurl3t64-gnutls:arm64 (8.14.1-2+deb13u7) ... +Setting up libperl5.40:arm64 (5.40.1-6+deb13u1) ... +Setting up ca-certificates (20250419+deb13u1) ... +Updating certificates in /etc/ssl/certs... +rehash: warning: skipping ca-certificates.crt, it does not contain exactly one certificate or CRL +0 added, 2 removed; done. +Setting up perl (5.40.1-6+deb13u1) ... +Setting up curl (8.14.1-2+deb13u7) ... +Processing triggers for debianutils (5.23.2) ... +Processing triggers for libc-bin (2.41-12+deb13u4) ... +Processing triggers for ca-certificates (20250419+deb13u1) ... +Updating certificates in /etc/ssl/certs... +0 added, 0 removed; done. +Running hooks in /etc/ca-certificates/update.d... +done. +==> Adding Docker apt repository (key pinned by SHA-256) +/etc/apt/keyrings/docker.asc: OK +==> Installing docker-ce 5:29.9.2-1~debian.13~trixie, containerd.io 2.4.2-3~debian.13~trixie +Selecting previously unselected package containerd.io. +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9875 files and directories currently installed.) +Preparing to unpack .../00-containerd.io_2.4.2-3~debian.13~trixie_arm64.deb ... +Unpacking containerd.io (2.4.2-3~debian.13~trixie) ... +Selecting previously unselected package docker-ce-cli. +Preparing to unpack .../01-docker-ce-cli_5%3a29.9.2-1~debian.13~trixie_arm64.deb ... +Unpacking docker-ce-cli (5:29.9.2-1~debian.13~trixie) ... +Selecting previously unselected package libip4tc2:arm64. +Preparing to unpack .../02-libip4tc2_1.8.11-2_arm64.deb ... +Unpacking libip4tc2:arm64 (1.8.11-2) ... +Selecting previously unselected package libip6tc2:arm64. +Preparing to unpack .../03-libip6tc2_1.8.11-2_arm64.deb ... +Unpacking libip6tc2:arm64 (1.8.11-2) ... +Selecting previously unselected package netbase. +Preparing to unpack .../04-netbase_6.5_all.deb ... +Unpacking netbase (6.5) ... +Selecting previously unselected package libnfnetlink0:arm64. +Preparing to unpack .../05-libnfnetlink0_1.0.2-3_arm64.deb ... +Unpacking libnfnetlink0:arm64 (1.0.2-3) ... +Selecting previously unselected package libnetfilter-conntrack3:arm64. +Preparing to unpack .../06-libnetfilter-conntrack3_1.1.0-1_arm64.deb ... +Unpacking libnetfilter-conntrack3:arm64 (1.1.0-1) ... +Selecting previously unselected package libnftnl11:arm64. +Preparing to unpack .../07-libnftnl11_1.2.9-1_arm64.deb ... +Unpacking libnftnl11:arm64 (1.2.9-1) ... +Selecting previously unselected package iptables. +Preparing to unpack .../08-iptables_1.8.11-2_arm64.deb ... +Unpacking iptables (1.8.11-2) ... +Selecting previously unselected package libjansson4:arm64. +Preparing to unpack .../09-libjansson4_2.14-2+b3_arm64.deb ... +Unpacking libjansson4:arm64 (2.14-2+b3) ... +Selecting previously unselected package libnftables1:arm64. +Preparing to unpack .../10-libnftables1_1.1.3-1_arm64.deb ... +Unpacking libnftables1:arm64 (1.1.3-1) ... +Selecting previously unselected package nftables. +Preparing to unpack .../11-nftables_1.1.3-1_arm64.deb ... +Unpacking nftables (1.1.3-1) ... +Selecting previously unselected package docker-ce. +Preparing to unpack .../12-docker-ce_5%3a29.9.2-1~debian.13~trixie_arm64.deb ... +Unpacking docker-ce (5:29.9.2-1~debian.13~trixie) ... +Setting up libip4tc2:arm64 (1.8.11-2) ... +Setting up libip6tc2:arm64 (1.8.11-2) ... +Setting up libnftnl11:arm64 (1.2.9-1) ... +Setting up libjansson4:arm64 (2.14-2+b3) ... +Setting up containerd.io (2.4.2-3~debian.13~trixie) ... +Setting up docker-ce-cli (5:29.9.2-1~debian.13~trixie) ... +Setting up libnfnetlink0:arm64 (1.0.2-3) ... +Setting up netbase (6.5) ... +Setting up libnftables1:arm64 (1.1.3-1) ... +Setting up nftables (1.1.3-1) ... +Setting up libnetfilter-conntrack3:arm64 (1.1.0-1) ... +Setting up iptables (1.8.11-2) ... +update-alternatives: using /usr/sbin/iptables-legacy to provide /usr/sbin/iptables (iptables) in auto mode +update-alternatives: using /usr/sbin/ip6tables-legacy to provide /usr/sbin/ip6tables (ip6tables) in auto mode +update-alternatives: using /usr/sbin/iptables-nft to provide /usr/sbin/iptables (iptables) in auto mode +update-alternatives: using /usr/sbin/ip6tables-nft to provide /usr/sbin/ip6tables (ip6tables) in auto mode +update-alternatives: using /usr/sbin/arptables-nft to provide /usr/sbin/arptables (arptables) in auto mode +update-alternatives: using /usr/sbin/ebtables-nft to provide /usr/sbin/ebtables (ebtables) in auto mode +Setting up docker-ce (5:29.9.2-1~debian.13~trixie) ... +invoke-rc.d: could not determine current runlevel +invoke-rc.d: WARNING: No init system and policy-rc.d missing! Defaulting to block. +Processing triggers for libc-bin (2.41-12+deb13u4) ... +==> Installed versions +Docker version 29.9.2, build a82e528 +containerd containerd v2.4.2 b7a25261e029d68978bbf290df0a62fc819b3c51 +runc version 1.5.2 +==> Pulling nginxinc/nginx-unprivileged:1.31.7-alpine-slim@sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7 +docker.io/nginxinc/nginx-unprivileged@sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7: Pulling from nginxinc/nginx-unprivileged +4b40f23576bc: Pulling fs layer +a9986cd6f37d: Pulling fs layer +d74d0cecaf1a: Pulling fs layer +115972ae778a: Pulling fs layer +98f56d1336e8: Pulling fs layer +581a829fa84a: Pulling fs layer +4bc482c4f270: Pulling fs layer +2fe8f2a34241: Pulling fs layer +44136fa355b3: Download complete +115972ae778a: Download complete +2fe8f2a34241: Download complete +98f56d1336e8: Download complete +4b40f23576bc: Download complete +581a829fa84a: Download complete +4bc482c4f270: Download complete +e437ba1f09a3: Download complete +d74d0cecaf1a: Download complete +a9986cd6f37d: Download complete +a9986cd6f37d: Pull complete +115972ae778a: Pull complete +2fe8f2a34241: Pull complete +98f56d1336e8: Pull complete +4b40f23576bc: Pull complete +581a829fa84a: Pull complete +4bc482c4f270: Pull complete +d74d0cecaf1a: Pull complete +Digest: sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7 +Status: Downloaded newer image for nginxinc/nginx-unprivileged@sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7 +docker.io/nginxinc/nginx-unprivileged:1.31.7-alpine-slim@sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7 +REPOSITORY TAG DIGEST IMAGE ID CREATED SIZE +nginxinc/nginx-unprivileged 1.31.7-alpine-slim sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7 a8cb0f33d51f 7 days ago 30.3MB + +--- stderr --- +shuru: building recipe docker from /docker@1b30f3d (2 steps) +shuru: step 1/2: install.sh +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +shuru: saving checkpoint 'shuru-test-recipe-docker-4199-step1'... +shuru: checkpoint 'shuru-test-recipe-docker-4199-step1' saved +shuru: step 2/2: pull.sh +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +shuru: saving checkpoint 'shuru-test-recipe-docker-4199'... +shuru: checkpoint 'shuru-test-recipe-docker-4199' saved +shuru: checkpoint 'shuru-test-recipe-docker-4199-step1' deleted + +$ shuru run --from shuru-test-recipe-docker-4199 -- sh -c docker info --format '{{.ServerVersion}}' && docker run --rm nginxinc/nginx-unprivileged:1.31.7-alpine-slim id -u +--- stdout --- +29.9.2 +101 + +--- stderr --- +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... + +ok +test failing_step_keeps_no_final_checkpoint ... $ shuru run --recipe /.tmpOh1uVw -- true +--- stdout --- +fine + +--- stderr --- +shuru: building recipe shuru-test-recipe-fail-4199 from /.tmpOh1uVw@6590f2b (2 steps) +shuru: step 1/2: ok.sh +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +shuru: saving checkpoint 'shuru-test-recipe-fail-4199-step1'... +shuru: checkpoint 'shuru-test-recipe-fail-4199-step1' saved +shuru: step 2/2: bad.sh +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +boom +shuru: saving checkpoint 'shuru-test-recipe-fail-4199'... +shuru: checkpoint 'shuru-test-recipe-fail-4199' saved +shuru: checkpoint 'shuru-test-recipe-fail-4199' deleted +shuru: intermediate checkpoint shuru-test-recipe-fail-4199-step1 kept for debugging; delete it with `shuru checkpoint delete shuru-test-recipe-fail-4199-step1` +Error: recipe step bad.sh exited with status 7 + +$ shuru checkpoint list +--- stdout --- +NAME SIZE CREATED RECIPE +docker 1.3 GB 4m ago /docker@dcbbf2c +shuru-test-recipe-fail-4199-step1 1008 MB just now + +--- stderr --- + +ok +test run_builds_once_reuses_and_rebuilds_when_the_recipe_changes ... $ shuru run --recipe /.tmpEZbp9s -- sh -c cat /opt/recipe/log; test -f /run/hello-ran && echo RUN_SAW_SERVICE +--- stdout --- +step1 +service-file-present +step2 +step2-saw-service +RUN_SAW_SERVICE + +--- stderr --- +shuru: building recipe shuru-test-recipe-4199 from /.tmpEZbp9s@48ff0ed (2 steps) +shuru: step 1/2: first.sh +shuru: booting VM (2cpus, 2048MB RAM, 2048MB disk)... +shuru: saving checkpoint 'shuru-test-recipe-4199-step1'... +shuru: checkpoint 'shuru-test-recipe-4199-step1' saved +shuru: step 2/2: second.sh +shuru: booting VM (2cpus, 2048MB RAM, 2048MB disk)... +shuru: saving checkpoint 'shuru-test-recipe-4199'... +shuru: checkpoint 'shuru-test-recipe-4199' saved +shuru: checkpoint 'shuru-test-recipe-4199-step1' deleted +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... + +$ shuru run --recipe /.tmpEZbp9s -- sh -c cat /opt/recipe/log; test -f /run/hello-ran && echo RUN_SAW_SERVICE +--- stdout --- +step1 +service-file-present +step2 +step2-saw-service +RUN_SAW_SERVICE + +--- stderr --- +shuru: using checkpoint shuru-test-recipe-4199 (/.tmpEZbp9s@48ff0ed) +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... + +$ shuru checkpoint list +--- stdout --- +NAME SIZE CREATED RECIPE +docker 1.3 GB 4m ago /docker@dcbbf2c +shuru-test-recipe-4199 1008 MB just now /.tmpEZbp9s@48ff0ed + +--- stderr --- + +$ shuru run --recipe /.tmpEZbp9s -- sh -c cat /opt/recipe/log; test -f /run/hello-ran && echo RUN_SAW_SERVICE +--- stdout --- +step1 +service-file-present +step2 +step2-saw-service +step2-v2 +RUN_SAW_SERVICE + +--- stderr --- +shuru: recipe changed (/.tmpEZbp9s@48ff0ed -> /.tmpEZbp9s@ae89e3d), rebuilding checkpoint shuru-test-recipe-4199 +shuru: checkpoint 'shuru-test-recipe-4199' deleted +shuru: building recipe shuru-test-recipe-4199 from /.tmpEZbp9s@ae89e3d (2 steps) +shuru: step 1/2: first.sh +shuru: booting VM (2cpus, 2048MB RAM, 2048MB disk)... +shuru: saving checkpoint 'shuru-test-recipe-4199-step1'... +shuru: checkpoint 'shuru-test-recipe-4199-step1' saved +shuru: step 2/2: second.sh +shuru: booting VM (2cpus, 2048MB RAM, 2048MB disk)... +shuru: saving checkpoint 'shuru-test-recipe-4199'... +shuru: checkpoint 'shuru-test-recipe-4199' saved +shuru: checkpoint 'shuru-test-recipe-4199-step1' deleted +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... + +$ shuru checkpoint list +--- stdout --- +NAME SIZE CREATED RECIPE +docker 1.3 GB 4m ago /docker@dcbbf2c +shuru-test-recipe-4199 1008 MB just now /.tmpEZbp9s@ae89e3d + +--- stderr --- + +ok +test show_prints_metadata_and_steps_without_booting ... $ shuru recipe show /.tmp0W2qfk +--- stdout --- +name: show-test +description: two steps and a service +source: /.tmp0W2qfk@d559800 +network: offline during build +services: hello +steps: first.sh (138 bytes), second.sh (98 bytes) + +--- /etc/shuru/services/hello.json --- +{ + "exec": [ + "sh", + "-c", + "touch /run/hello-ran; exec sleep infinity" + ], + "ready": [ + "test", + "-f", + "/run/hello-ran" + ] +} + +--- first.sh --- +mkdir -p /opt/recipe +echo step1 >> /opt/recipe/log +test -f /etc/shuru/services/hello.json && echo service-file-present >> /opt/recipe/log + +--- second.sh --- +echo step2 >> /opt/recipe/log +test -f /run/hello-ran && echo step2-saw-service >> /opt/recipe/log + +--- stderr --- + +ok +test show_rejects_bad_recipes ... $ shuru recipe show /.tmpkH5s9O +--- stdout --- + +--- stderr --- +Error: /.tmpkH5s9O: recipe name "Bad Name" must match [a-z0-9][a-z0-9._-]* + +$ shuru recipe show /.tmpqu4jAZ +--- stdout --- + +--- stderr --- +Error: reading step second.sh in /.tmpqu4jAZ + +Caused by: + No such file or directory (os error 2) + +$ shuru recipe show /.tmpk4H0o8 +--- stdout --- + +--- stderr --- +Error: /.tmpk4H0o8: recipe needs shuru 99.0.0 or newer, this is 0.8.0. Run `shuru upgrade`. + +ok + +test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 130.08s + +recipes-exit=0 +=== shuru recipe list (github.com/superhq-ai/recipes) +NAME DESCRIPTION +docker Docker Engine 29.9 with dockerd as a guest service and nginx-unprivileged pre-pulled + +Source: github.com/superhq-ai/recipes. Build one with `shuru run --recipe NAME` or `shuru checkpoint create NAME --recipe NAME`. +list-exit=0 +=== shuru run --recipe docker (first use rebuilds: existing checkpoint came from a local path) +shuru: recipe changed (/docker@dcbbf2c -> superhq-ai/recipes/docker@2a5c726), rebuilding checkpoint docker +shuru: checkpoint 'docker' deleted +shuru: building recipe docker from superhq-ai/recipes/docker@2a5c726 (2 steps) +shuru: step 1/2: install.sh +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +==> Upgrading Debian base (main, security and updates archives) +Preconfiguring packages ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9863 files and directories currently installed.) +Preparing to unpack .../libc6_2.41-12+deb13u4_arm64.deb ... +Unpacking libc6:arm64 (2.41-12+deb13u4) over (2.41-12+deb13u3) ... +Setting up libc6:arm64 (2.41-12+deb13u4) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9863 files and directories currently installed.) +Preparing to unpack .../base-files_13.8+deb13u7_arm64.deb ... +Unpacking base-files (13.8+deb13u7) over (13.8+deb13u6) ... +Setting up base-files (13.8+deb13u7) ... +Installing new version of config file /etc/debian_version ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../bash_5.2.37-2+b10_arm64.deb ... +update-alternatives: warning: alternative /usr/share/man/man7/bash-builtins.7.gz (part of link group builtins.7.gz) doesn't exist; removing from list of alternatives +update-alternatives: warning: /etc/alternatives/builtins.7.gz is dangling; it will be updated with best choice +Unpacking bash (5.2.37-2+b10) over (5.2.37-2+b9) ... +Setting up bash (5.2.37-2+b10) ... +update-alternatives: error: alternative path /usr/share/man/man7/bash-builtins.7.gz doesn't exist +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../bsdutils_1%3a2.41.5-0+deb13u1_arm64.deb ... +Unpacking bsdutils (1:2.41.5-0+deb13u1) over (1:2.41-5) ... +Setting up bsdutils (1:2.41.5-0+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../gzip_1.13-1+deb13u1_arm64.deb ... +Unpacking gzip (1.13-1+deb13u1) over (1.13-1) ... +Setting up gzip (1.13-1+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../libperl5.40_5.40.1-6+deb13u1_arm64.deb ... +Unpacking libperl5.40:arm64 (5.40.1-6+deb13u1) over (5.40.1-6) ... +Preparing to unpack .../perl_5.40.1-6+deb13u1_arm64.deb ... +Unpacking perl (5.40.1-6+deb13u1) over (5.40.1-6) ... +Preparing to unpack .../perl-base_5.40.1-6+deb13u1_arm64.deb ... +Unpacking perl-base (5.40.1-6+deb13u1) over (5.40.1-6) ... +Setting up perl-base (5.40.1-6+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../perl-modules-5.40_5.40.1-6+deb13u1_all.deb ... +Unpacking perl-modules-5.40 (5.40.1-6+deb13u1) over (5.40.1-6) ... +Preparing to unpack .../liblastlog2-2_2.41.5-0+deb13u1_arm64.deb ... +Unpacking liblastlog2-2:arm64 (2.41.5-0+deb13u1) over (2.41-5) ... +Setting up liblastlog2-2:arm64 (2.41.5-0+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../libuuid1_2.41.5-0+deb13u1_arm64.deb ... +Unpacking libuuid1:arm64 (2.41.5-0+deb13u1) over (2.41-5) ... +Setting up libuuid1:arm64 (2.41.5-0+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../libblkid1_2.41.5-0+deb13u1_arm64.deb ... +Unpacking libblkid1:arm64 (2.41.5-0+deb13u1) over (2.41-5) ... +Setting up libblkid1:arm64 (2.41.5-0+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../libmount1_2.41.5-0+deb13u1_arm64.deb ... +Unpacking libmount1:arm64 (2.41.5-0+deb13u1) over (2.41-5) ... +Setting up libmount1:arm64 (2.41.5-0+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../libsmartcols1_2.41.5-0+deb13u1_arm64.deb ... +Unpacking libsmartcols1:arm64 (2.41.5-0+deb13u1) over (2.41-5) ... +Setting up libsmartcols1:arm64 (2.41.5-0+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9872 files and directories currently installed.) +Preparing to unpack .../util-linux_2.41.5-0+deb13u1_arm64.deb ... +Unpacking util-linux (2.41.5-0+deb13u1) over (2.41-5) ... +Setting up util-linux (2.41.5-0+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9877 files and directories currently installed.) +Preparing to unpack .../mount_2.41.5-0+deb13u1_arm64.deb ... +Unpacking mount (2.41.5-0+deb13u1) over (2.41-5) ... +Preparing to unpack .../libsqlite3-0_3.46.1-7+deb13u2_arm64.deb ... +Unpacking libsqlite3-0:arm64 (3.46.1-7+deb13u2) over (3.46.1-7+deb13u1) ... +Setting up libsqlite3-0:arm64 (3.46.1-7+deb13u2) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9877 files and directories currently installed.) +Preparing to unpack .../libc-bin_2.41-12+deb13u4_arm64.deb ... +Unpacking libc-bin (2.41-12+deb13u4) over (2.41-12+deb13u3) ... +Setting up libc-bin (2.41-12+deb13u4) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9877 files and directories currently installed.) +Preparing to unpack .../libaudit-common_1%3a4.0.2-2+deb13u1_all.deb ... +Unpacking libaudit-common (1:4.0.2-2+deb13u1) over (1:4.0.2-2) ... +Setting up libaudit-common (1:4.0.2-2+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9877 files and directories currently installed.) +Preparing to unpack .../libaudit1_1%3a4.0.2-2+deb13u1_arm64.deb ... +Unpacking libaudit1:arm64 (1:4.0.2-2+deb13u1) over (1:4.0.2-2+b2) ... +Setting up libaudit1:arm64 (1:4.0.2-2+deb13u1) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9876 files and directories currently installed.) +Preparing to unpack .../login_1%3a4.16.0-2+really2.41.5-0+deb13u1_arm64.deb ... +Unpacking login (1:4.16.0-2+really2.41.5-0+deb13u1) over (1:4.16.0-2+really2.41-5) ... +Preparing to unpack .../libexpat1_2.9.0-1~deb13u1_arm64.deb ... +Unpacking libexpat1:arm64 (2.9.0-1~deb13u1) over (2.7.1-2) ... +Preparing to unpack .../libcap2_1%3a2.75-10+deb13u1+b3_arm64.deb ... +Unpacking libcap2:arm64 (1:2.75-10+deb13u1+b3) over (1:2.75-10+deb13u1+b1) ... +Setting up libcap2:arm64 (1:2.75-10+deb13u1+b3) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9878 files and directories currently installed.) +Preparing to unpack .../liblzma5_5.8.1-1+deb13u2_arm64.deb ... +Unpacking liblzma5:arm64 (5.8.1-1+deb13u2) over (5.8.1-1+deb13u1) ... +Setting up liblzma5:arm64 (5.8.1-1+deb13u2) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9878 files and directories currently installed.) +Preparing to unpack .../libpcre2-8-0_10.46-1~deb13u3_arm64.deb ... +Unpacking libpcre2-8-0:arm64 (10.46-1~deb13u3) over (10.46-1~deb13u1) ... +Setting up libpcre2-8-0:arm64 (10.46-1~deb13u3) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9878 files and directories currently installed.) +Preparing to unpack .../openssl-provider-legacy_3.5.7-1~deb13u3_arm64.deb ... +Unpacking openssl-provider-legacy (3.5.7-1~deb13u3) over (3.5.6-1~deb13u2) ... +Setting up openssl-provider-legacy (3.5.7-1~deb13u3) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9878 files and directories currently installed.) +Preparing to unpack .../libssl3t64_3.5.7-1~deb13u3_arm64.deb ... +Unpacking libssl3t64:arm64 (3.5.7-1~deb13u3) over (3.5.6-1~deb13u2) ... +Setting up libssl3t64:arm64 (3.5.7-1~deb13u3) ... +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9878 files and directories currently installed.) +Preparing to unpack .../00-tzdata_2026e-0+deb13u1_all.deb ... +Unpacking tzdata (2026e-0+deb13u1) over (2026b-0+deb13u1) ... +Preparing to unpack .../01-openssl_3.5.7-1~deb13u3_arm64.deb ... +Unpacking openssl (3.5.7-1~deb13u3) over (3.5.6-1~deb13u2) ... +Preparing to unpack .../02-ca-certificates_20250419+deb13u1_all.deb ... +Unpacking ca-certificates (20250419+deb13u1) over (20250419) ... +Preparing to unpack .../03-xz-utils_5.8.1-1+deb13u2_arm64.deb ... +Unpacking xz-utils (5.8.1-1+deb13u2) over (5.8.1-1+deb13u1) ... +Preparing to unpack .../04-libbrotli1_1.1.0-2+deb13u1_arm64.deb ... +Unpacking libbrotli1:arm64 (1.1.0-2+deb13u1) over (1.1.0-2+b7) ... +Preparing to unpack .../05-libssh2-1t64_1.11.1-1+deb13u2_arm64.deb ... +Unpacking libssh2-1t64:arm64 (1.11.1-1+deb13u2) over (1.11.1-1+deb13u1) ... +Preparing to unpack .../06-curl_8.14.1-2+deb13u7_arm64.deb ... +Unpacking curl (8.14.1-2+deb13u7) over (8.14.1-2+deb13u4) ... +Preparing to unpack .../07-libcurl4t64_8.14.1-2+deb13u7_arm64.deb ... +Unpacking libcurl4t64:arm64 (8.14.1-2+deb13u7) over (8.14.1-2+deb13u4) ... +Preparing to unpack .../08-jq_1.7.1-6+deb13u4_arm64.deb ... +Unpacking jq (1.7.1-6+deb13u4) over (1.7.1-6+deb13u2) ... +Preparing to unpack .../09-libjq1_1.7.1-6+deb13u4_arm64.deb ... +Unpacking libjq1:arm64 (1.7.1-6+deb13u4) over (1.7.1-6+deb13u2) ... +Preparing to unpack .../10-libcap2-bin_1%3a2.75-10+deb13u1+b3_arm64.deb ... +Unpacking libcap2-bin (1:2.75-10+deb13u1+b3) over (1:2.75-10+deb13u1+b1) ... +Preparing to unpack .../11-libcom-err2_1.47.2-3+b12_arm64.deb ... +Unpacking libcom-err2:arm64 (1.47.2-3+b12) over (1.47.2-3+b11) ... +Preparing to unpack .../12-libcurl3t64-gnutls_8.14.1-2+deb13u7_arm64.deb ... +Unpacking libcurl3t64-gnutls:arm64 (8.14.1-2+deb13u7) over (8.14.1-2+deb13u4) ... +Setting up libexpat1:arm64 (2.9.0-1~deb13u1) ... +Setting up libjq1:arm64 (1.7.1-6+deb13u4) ... +Setting up libbrotli1:arm64 (1.1.0-2+deb13u1) ... +Setting up libcom-err2:arm64 (1.47.2-3+b12) ... +Setting up tzdata (2026e-0+deb13u1) ... + +Current default time zone: 'Etc/UTC' +Local time is now: Mon Oct 26 01:57:04 UTC 2026. +Universal Time is now: Mon Oct 26 01:57:04 UTC 2026. +Run 'dpkg-reconfigure tzdata' if you wish to change it. + +Setting up libcap2-bin (1:2.75-10+deb13u1+b3) ... +Setting up xz-utils (5.8.1-1+deb13u2) ... +Setting up mount (2.41.5-0+deb13u1) ... +Setting up perl-modules-5.40 (5.40.1-6+deb13u1) ... +Setting up jq (1.7.1-6+deb13u4) ... +Setting up libssh2-1t64:arm64 (1.11.1-1+deb13u2) ... +Setting up openssl (3.5.7-1~deb13u3) ... +Setting up login (1:4.16.0-2+really2.41.5-0+deb13u1) ... +Setting up libcurl4t64:arm64 (8.14.1-2+deb13u7) ... +Setting up libcurl3t64-gnutls:arm64 (8.14.1-2+deb13u7) ... +Setting up libperl5.40:arm64 (5.40.1-6+deb13u1) ... +Setting up ca-certificates (20250419+deb13u1) ... +Updating certificates in /etc/ssl/certs... +rehash: warning: skipping ca-certificates.crt, it does not contain exactly one certificate or CRL +0 added, 2 removed; done. +Setting up perl (5.40.1-6+deb13u1) ... +Setting up curl (8.14.1-2+deb13u7) ... +Processing triggers for debianutils (5.23.2) ... +Processing triggers for libc-bin (2.41-12+deb13u4) ... +Processing triggers for ca-certificates (20250419+deb13u1) ... +Updating certificates in /etc/ssl/certs... +0 added, 0 removed; done. +Running hooks in /etc/ca-certificates/update.d... +done. +==> Adding Docker apt repository (key pinned by SHA-256) +/etc/apt/keyrings/docker.asc: OK +==> Installing docker-ce 5:29.9.2-1~debian.13~trixie, containerd.io 2.4.2-3~debian.13~trixie +Selecting previously unselected package containerd.io. +(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 9875 files and directories currently installed.) +Preparing to unpack .../00-containerd.io_2.4.2-3~debian.13~trixie_arm64.deb ... +Unpacking containerd.io (2.4.2-3~debian.13~trixie) ... +Selecting previously unselected package docker-ce-cli. +Preparing to unpack .../01-docker-ce-cli_5%3a29.9.2-1~debian.13~trixie_arm64.deb ... +Unpacking docker-ce-cli (5:29.9.2-1~debian.13~trixie) ... +Selecting previously unselected package libip4tc2:arm64. +Preparing to unpack .../02-libip4tc2_1.8.11-2_arm64.deb ... +Unpacking libip4tc2:arm64 (1.8.11-2) ... +Selecting previously unselected package libip6tc2:arm64. +Preparing to unpack .../03-libip6tc2_1.8.11-2_arm64.deb ... +Unpacking libip6tc2:arm64 (1.8.11-2) ... +Selecting previously unselected package netbase. +Preparing to unpack .../04-netbase_6.5_all.deb ... +Unpacking netbase (6.5) ... +Selecting previously unselected package libnfnetlink0:arm64. +Preparing to unpack .../05-libnfnetlink0_1.0.2-3_arm64.deb ... +Unpacking libnfnetlink0:arm64 (1.0.2-3) ... +Selecting previously unselected package libnetfilter-conntrack3:arm64. +Preparing to unpack .../06-libnetfilter-conntrack3_1.1.0-1_arm64.deb ... +Unpacking libnetfilter-conntrack3:arm64 (1.1.0-1) ... +Selecting previously unselected package libnftnl11:arm64. +Preparing to unpack .../07-libnftnl11_1.2.9-1_arm64.deb ... +Unpacking libnftnl11:arm64 (1.2.9-1) ... +Selecting previously unselected package iptables. +Preparing to unpack .../08-iptables_1.8.11-2_arm64.deb ... +Unpacking iptables (1.8.11-2) ... +Selecting previously unselected package libjansson4:arm64. +Preparing to unpack .../09-libjansson4_2.14-2+b3_arm64.deb ... +Unpacking libjansson4:arm64 (2.14-2+b3) ... +Selecting previously unselected package libnftables1:arm64. +Preparing to unpack .../10-libnftables1_1.1.3-1_arm64.deb ... +Unpacking libnftables1:arm64 (1.1.3-1) ... +Selecting previously unselected package nftables. +Preparing to unpack .../11-nftables_1.1.3-1_arm64.deb ... +Unpacking nftables (1.1.3-1) ... +Selecting previously unselected package docker-ce. +Preparing to unpack .../12-docker-ce_5%3a29.9.2-1~debian.13~trixie_arm64.deb ... +Unpacking docker-ce (5:29.9.2-1~debian.13~trixie) ... +Setting up libip4tc2:arm64 (1.8.11-2) ... +Setting up libip6tc2:arm64 (1.8.11-2) ... +Setting up libnftnl11:arm64 (1.2.9-1) ... +Setting up libjansson4:arm64 (2.14-2+b3) ... +Setting up containerd.io (2.4.2-3~debian.13~trixie) ... +Setting up docker-ce-cli (5:29.9.2-1~debian.13~trixie) ... +Setting up libnfnetlink0:arm64 (1.0.2-3) ... +Setting up netbase (6.5) ... +Setting up libnftables1:arm64 (1.1.3-1) ... +Setting up nftables (1.1.3-1) ... +Setting up libnetfilter-conntrack3:arm64 (1.1.0-1) ... +Setting up iptables (1.8.11-2) ... +update-alternatives: using /usr/sbin/iptables-legacy to provide /usr/sbin/iptables (iptables) in auto mode +update-alternatives: using /usr/sbin/ip6tables-legacy to provide /usr/sbin/ip6tables (ip6tables) in auto mode +update-alternatives: using /usr/sbin/iptables-nft to provide /usr/sbin/iptables (iptables) in auto mode +update-alternatives: using /usr/sbin/ip6tables-nft to provide /usr/sbin/ip6tables (ip6tables) in auto mode +update-alternatives: using /usr/sbin/arptables-nft to provide /usr/sbin/arptables (arptables) in auto mode +update-alternatives: using /usr/sbin/ebtables-nft to provide /usr/sbin/ebtables (ebtables) in auto mode +Setting up docker-ce (5:29.9.2-1~debian.13~trixie) ... +invoke-rc.d: could not determine current runlevel +invoke-rc.d: WARNING: No init system and policy-rc.d missing! Defaulting to block. +Processing triggers for libc-bin (2.41-12+deb13u4) ... +==> Installed versions +Docker version 29.9.2, build a82e528 +containerd containerd v2.4.2 b7a25261e029d68978bbf290df0a62fc819b3c51 +runc version 1.5.2 +shuru: saving checkpoint 'docker-step1'... +shuru: checkpoint 'docker-step1' saved +shuru: step 2/2: pull.sh +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +==> Pulling nginxinc/nginx-unprivileged:1.31.7-alpine-slim@sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7 +docker.io/nginxinc/nginx-unprivileged@sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7: Pulling from nginxinc/nginx-unprivileged +4b40f23576bc: Pulling fs layer +115972ae778a: Pulling fs layer +98f56d1336e8: Pulling fs layer +d74d0cecaf1a: Pulling fs layer +581a829fa84a: Pulling fs layer +a9986cd6f37d: Pulling fs layer +2fe8f2a34241: Pulling fs layer +4bc482c4f270: Pulling fs layer +44136fa355b3: Download complete +115972ae778a: Download complete +98f56d1336e8: Download complete +4bc482c4f270: Download complete +581a829fa84a: Download complete +2fe8f2a34241: Download complete +4b40f23576bc: Download complete +e437ba1f09a3: Download complete +a9986cd6f37d: Download complete +a9986cd6f37d: Pull complete +d74d0cecaf1a: Download complete +115972ae778a: Pull complete +98f56d1336e8: Pull complete +4bc482c4f270: Pull complete +581a829fa84a: Pull complete +2fe8f2a34241: Pull complete +4b40f23576bc: Pull complete +d74d0cecaf1a: Pull complete +Digest: sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7 +Status: Downloaded newer image for nginxinc/nginx-unprivileged@sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7 +docker.io/nginxinc/nginx-unprivileged:1.31.7-alpine-slim@sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7 +REPOSITORY TAG DIGEST IMAGE ID CREATED SIZE +nginxinc/nginx-unprivileged 1.31.7-alpine-slim sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7 a8cb0f33d51f 7 days ago 30.3MB +shuru: saving checkpoint 'docker'... +shuru: checkpoint 'docker' saved +shuru: checkpoint 'docker-step1' deleted +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +29.9.2 +101 + +real 2m3.632s +user 0m20.139s +sys 0m3.259s +run-recipe-exit=0 +=== shuru run --recipe docker (second use reuses the checkpoint) +shuru: using checkpoint docker (superhq-ai/recipes/docker@2a5c726) +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +REPOSITORY TAG DIGEST IMAGE ID CREATED SIZE +nginxinc/nginx-unprivileged 1.31.7-alpine-slim sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7 a8cb0f33d51f 7 days ago 30.3MB + +real 0m3.910s +user 0m0.068s +sys 0m0.017s +run-recipe-reuse-exit=0 +=== checkpoint list and sidecar +NAME SIZE CREATED RECIPE +docker 1.3 GB just now superhq-ai/recipes/docker@2a5c726 +{ + "recipe": { + "name": "docker", + "source": "superhq-ai/recipes/docker", + "revision": "2a5c726ee5fa4812b5315fa4b8b81650118b5305" + } +} diff --git a/docs/rfcs/0005-recipes.md b/docs/rfcs/0005-recipes.md new file mode 100644 index 0000000..63f0ab9 --- /dev/null +++ b/docs/rfcs/0005-recipes.md @@ -0,0 +1,299 @@ +# RFC 0005: Recipes + +- Status: Implemented +- Date: 2026-10-26 +- Scope: shuru-cli, the superhq-ai/recipes repository +- Depends on: RFC 0004 + +## Summary + +A recipe is a small directory, `recipe.json` plus one or more shell steps, +that describes how to turn the base image or an existing checkpoint into a +new checkpoint. Recipes live in git. The official catalog is +`superhq-ai/recipes` on GitHub, any repository can hold more, and a local +directory works for development. The CLI resolves a recipe to an exact +commit, builds it through the ordinary checkpoint flow, and records where +the checkpoint came from. + +``` +shuru recipe list +shuru recipe show docker +shuru checkpoint create docker --recipe docker +shuru run --recipe docker -p 8080:8080 -- docker run --rm -p 8080:8080 nginx +``` + +The last form builds the checkpoint on first use and reuses it afterwards. + +## Motivation + +RFC 0003 shipped Docker support as a shell script in the repository. The +installed CLI knows nothing about it, so a Homebrew user has to clone the +repo to get a Docker checkpoint. The same is true of every "install X into +a sandbox" recipe people write: it is a one-off script in some other +project, with no versioning, no provenance on the checkpoint it produced, +and no way to share it short of copy and paste. + +The pieces already exist. `shuru checkpoint create` is the build step, RFC +0004 services are the runtime contract, and git is the distribution +channel every user already has. A recipe is the glue, kept deliberately +small. + +## Non-goals + +- Parameters or templating inside recipes. A recipe is fixed; variants are + separate recipes. +- Dependencies between recipes beyond `from` naming a checkpoint. +- Signing beyond the git commit SHA. Authenticity rests on GitHub and the + repository owner. +- A registry service. The catalog is a JSON file in a git repository. +- Prebuilt checkpoint downloads. Possible later, orthogonal to this. + +## Design + +### Recipe layout + +``` +docker/ + recipe.json + install.sh + pull.sh +``` + +`recipe.json`: + +```json +{ + "name": "docker", + "description": "Docker Engine with dockerd as a guest service", + "min_shuru_version": "0.8.0", + "from": null, + "disk_size": 4096, + "memory": 2048, + "network": {"allow": ["deb.debian.org", "download.docker.com"]}, + "services": { + "docker": {"exec": ["dockerd"], "ready": ["docker", "info"]} + }, + "steps": ["install.sh", "pull.sh"] +} +``` + +| field | required | meaning | +|---|---|---| +| `name` | yes | checkpoint name used by `shuru run --recipe`. `[a-z0-9][a-z0-9._-]*`. | +| `description` | no | one line for `recipe list` and `recipe show`. | +| `min_shuru_version` | no | refuse to build on an older CLI. | +| `from` | no | existing checkpoint to start from instead of the base image. | +| `cpus`, `memory`, `disk_size` | no | VM size for the build. CLI flags override. | +| `network.allow` | no | hosts the build may reach. Absent means the build runs offline. An empty list means unrestricted. | +| `services` | no | RFC 0004 service files, written before the first step runs. | +| `steps` | no | scripts run in order, default `["build.sh"]`. | + +Each step runs in its own boot. The first boots from `from` or the base +image, every later step boots from the checkpoint the previous step saved. +Services are written during the first step, so from the second step on +they are running. This is why the Docker recipe has two steps: the first +installs the engine and declares the service, the second pulls an image +with dockerd already up. Intermediate checkpoints are deleted when the +build succeeds and kept, with a message, when it fails. + +A step script is sent to the guest as a base64 payload the same way the +RFC 0003 script was, so it can use any quoting it likes. It runs with +`sh`, from `/`, as root, and its exit status fails the build. + +### References + +| form | resolves to | +|---|---| +| `docker` | `superhq-ai/recipes`, directory `docker`, default branch | +| `owner/repo` | that repository root | +| `owner/repo/path/to/dir` | a directory in that repository | +| any of the above `@ref` | a branch, tag or commit | +| `./dir`, `/abs/dir`, `../dir`, `~/dir` | a local directory | + +A bare name is never looked up in the working directory, even when +`./NAME/recipe.json` exists. Otherwise a cloned repository could shadow an +official recipe, inherit its trust and run its own steps without the +prompt. The CLI prints a note pointing at `./NAME` in that case. + +A GitHub reference is resolved to a commit SHA through the GitHub API +before anything is fetched, and the files are then read at that SHA from +`raw.githubusercontent.com`. The SHA is printed, is what the checkpoint +records, and is what `--recipe docker@` pins. `GITHUB_TOKEN` or +`GH_TOKEN`, when set, is sent to api.github.com only, for private +repositories and rate limits. + +A local recipe gets a content hash in place of a SHA, so editing a step +counts as a new revision. + +### Trust + +A recipe runs inside the VM, under the network allowlist it declares, so +it can shape the checkpoint but cannot touch the host. It still decides +what every later sandbox built from it contains. The CLI therefore prints +the source, the commit and the allowlist before every build, and for +anything that is not the official catalog or a local path it asks for +confirmation on a terminal, or requires `--yes` without one. + +### Checkpoint provenance + +A checkpoint built from a recipe gets a sidecar +`checkpoints/.json`: + +```json +{"recipe": {"name": "docker", "source": "superhq-ai/recipes/docker", "revision": ""}} +``` + +`shuru checkpoint list` shows it. `shuru checkpoint delete` removes it +with the image. + +### `shuru run --recipe` + +1. Resolve the reference and confirm trust. +2. Look for a checkpoint named after the recipe, or after `--checkpoint + NAME` when given. The flag keeps two recipes that share a `name`, such + as a fork and the official one, from rebuilding over each other. + - Present with a matching revision: reuse it. + - Present with a different revision: say so, delete it, rebuild. + - Present without a sidecar: refuse, since it is not ours to replace. + - Absent: build. +3. Run the command from that checkpoint with the flags the user gave. + +`shuru checkpoint update NAME --recipe REF` forces a rebuild at the current +revision. It builds under `NAME.update` and swaps the files in only when +every step succeeds, so the old checkpoint survives a failed build. + +The build and the run are separate phases with separate network policy. +The build uses the recipe allowlist. The run uses whatever `--allow-net` +and `--allow-host` the user passed, which by default is nothing. + +### Catalog + +`shuru recipe list` reads `index.json` from the default branch of +`superhq-ai/recipes`, a flat array of `{name, description}` that the +repository validates in CI against its directories. The file is cached in +the data directory for a day; `--refresh` bypasses the cache. +`shuru recipe show ` prints the metadata and every step script, so +the contents can be read before they are built. + +### The recipes repository + +`superhq-ai/recipes` holds `index.json`, one directory per recipe, a +validation script, and a workflow that runs it. The Docker recipe moves +there from `scripts/prepare-docker-checkpoint.sh`, which is removed from +this repository. Pinned versions, the apt key hash and the image digest +stay in the recipe. + +## Security + +- Builds run with the recipe allowlist, never the project `shuru.json`, so + a project config cannot leak secrets into a recipe build and a recipe + cannot widen a project policy. +- The checkpoint records the exact commit. Two users building the same + reference at the same SHA run the same script. +- Third-party recipes require an explicit yes. Official and local do not, + because the official catalog is reviewed in its own repository and a + local directory is code the user already owns. +- A recipe cannot name a service with a path, and service names and + recipe names are validated before anything is written. + +Residual risks: + +- GitHub is trusted for both the SHA lookup and the file contents. A + compromised GitHub account that owns a recipe can change what builds, + which is the same exposure as any git dependency. +- Reuse is keyed on the recipe revision, not on the base image. A new + Shuru release with a new rootfs does not rebuild existing recipe + checkpoints. `min_shuru_version` catches the case where the recipe + needs the newer kernel. + +## Testing + +`crates/shuru-cli/tests/recipes.rs` uses local recipes built on the fly: + +- `recipe show` prints metadata and steps without booting anything +- a two step recipe with a service: step one output, step two output, + and proof the service was running during step two and during the run +- `run --recipe` builds once, reuses on the next run, and rebuilds after + the step script changes, with the sidecar reflecting each revision +- `checkpoint create NAME --recipe` under a chosen name +- with network, the real Docker recipe from a checkout of + `superhq-ai/recipes` builds and runs a container as uid 101 + +## Rollout + +The Docker checkpoint for the RFC 0003 test suite is now built with +`shuru checkpoint create docker --recipe docker` against the official +catalog, or from a checkout while a recipe is being changed. The README and +the website show that form first and `shuru run --recipe docker` as the +one-liner. + +## Appendix A: Evidence + +Captured on 2026-10-26 against real VMs on an M-series Mac, kernel 6.12.114, +isolated data dir, codesigned debug CLI 0.8.0, with the official catalog at +superhq-ai/recipes commit 2a5c726. Full `--nocapture` output is in +`docs/rfcs/0005-recipes-evidence.log`. + +Recipes suite, six tests in one run (two parse-only, three on throwaway +local recipes, one that builds the docker recipe for real): + +``` +test result: ok. 6 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 130.08s +``` + +Lines that show each behaviour, from that log: + +``` +shuru: building recipe shuru-test-recipe-4199 from /.tmpEZbp9s@48ff0ed (2 steps) +shuru: step 1/2: first.sh +shuru: step 2/2: second.sh +step1 +service-file-present +step2 +step2-saw-service +RUN_SAW_SERVICE +shuru: using checkpoint shuru-test-recipe-4199 (/.tmpEZbp9s@48ff0ed) +shuru: recipe changed (/.tmpEZbp9s@48ff0ed -> /.tmpEZbp9s@ae89e3d), rebuilding checkpoint shuru-test-recipe-4199 + +Error: --recipe builds from the recipe steps; drop the command +Error: checkpoint shuru-test-recipe-named-4199 already exists, delete it first + +shuru: intermediate checkpoint shuru-test-recipe-fail-4199-step1 kept for debugging; delete it with `shuru checkpoint delete shuru-test-recipe-fail-4199-step1` +Error: recipe step bad.sh exited with status 7 + +shuru: step 2/2: pull.sh +Digest: sha256:a8cb0f33d51fa107c96fd2bd3a71aaf564ec131e9d2bdf8700944f0ed8b170d7 +29.9.2 +101 +``` + +The official path, resolved from GitHub. The data dir held a `docker` +checkpoint built from a local checkout, so the first `shuru run --recipe +docker` saw a different source and rebuilt; the second reused it: + +``` +$ shuru recipe list --refresh +NAME DESCRIPTION +docker Docker Engine 29.9 with dockerd as a guest service and nginx-unprivileged pre-pulled + +$ shuru run --recipe docker -- sh -c "docker info --format '{{.ServerVersion}}'; docker run --rm nginxinc/nginx-unprivileged:1.31.7-alpine-slim id -u" +shuru: recipe changed (/docker@dcbbf2c -> superhq-ai/recipes/docker@2a5c726), rebuilding checkpoint docker +shuru: step 1/2: install.sh +shuru: step 2/2: pull.sh +shuru: checkpoint 'docker' saved +29.9.2 +101 +real 2m3.632s + +$ shuru run --recipe docker -- docker image ls --digests +shuru: using checkpoint docker (superhq-ai/recipes/docker@2a5c726) +real 0m3.910s + +$ shuru checkpoint list +NAME SIZE CREATED RECIPE +docker 1.3 GB just now superhq-ai/recipes/docker@2a5c726 +``` + +The RFC 0003 Docker suite was rerun against a checkpoint built by the +recipe and passed nine of nine in 78.79s; its log is +`docs/rfcs/0003-docker-on-shuru-evidence.log`. diff --git a/docs/rfcs/0006-memory-balloon-evidence.log b/docs/rfcs/0006-memory-balloon-evidence.log new file mode 100644 index 0000000..78e89be --- /dev/null +++ b/docs/rfcs/0006-memory-balloon-evidence.log @@ -0,0 +1,93 @@ +=== Linux host: a1.metal, Ubuntu 24.04, kernel 7.0.0-1014-aws, KVM, 2 GB guest +=== shuru 0.9.3 built in an arm64 container from this branch; OS image 0.9.2 + +=== host +7.0.0-1014-aws +16 + total used free shared buff/cache available +Mem: 32011 699 29759 1 1885 31311 +4096 +crw-rw---- 1 root kvm 10, 232 Oct 27 00:37 /dev/kvm +/xfs.img on /mnt/xfs type xfs (rw,relatime,inode64,logbufs=8,logbsize=32k,noquota) +shuru 0.9.3 +=== guest view of the device +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +virtio0 device=0x0002 driver=virtio_blk +virtio1 device=0x0013 driver=vmw_vsock_virtio_transport +virtio2 device=0x0005 driver=virtio_balloon +balloon features: 0010010000000000000000000000000010000000000000000000000000000000 + deflate_on_oom(2)=1 reporting(5)=1 version_1(32)=1 +page_reporting_order=10 +=== balloon example +booted host rss 23 MB guest MemTotal: 1998 MB MemFree: 1968 MB +guest holds 1 GB in tmpfs host rss 1109 MB guest MemTotal: 1998 MB MemFree: 933 MB +guest freed it +1s host rss 1109 MB guest MemTotal: 1998 MB MemFree: 1975 MB +guest freed it +2s host rss 855 MB guest MemTotal: 1998 MB MemFree: 1975 MB +guest freed it +3s host rss 855 MB guest MemTotal: 1998 MB MemFree: 1975 MB +guest freed it +4s host rss 599 MB guest MemTotal: 1998 MB MemFree: 1975 MB +guest freed it +5s host rss 599 MB guest MemTotal: 1998 MB MemFree: 1975 MB +guest freed it +6s host rss 535 MB guest MemTotal: 1998 MB MemFree: 1975 MB +guest freed it +7s host rss 343 MB guest MemTotal: 1998 MB MemFree: 1975 MB +guest freed it +8s host rss 323 MB guest MemTotal: 1998 MB MemFree: 1975 MB +guest freed it +9s host rss 87 MB guest MemTotal: 1998 MB MemFree: 1975 MB +guest freed it +10s host rss 87 MB guest MemTotal: 1998 MB MemFree: 1975 MB +guest freed it +11s host rss 87 MB guest MemTotal: 1998 MB MemFree: 1975 MB +guest freed it +12s host rss 87 MB guest MemTotal: 1998 MB MemFree: 1975 MB +guest freed it +13s host rss 87 MB guest MemTotal: 1998 MB MemFree: 1974 MB +guest freed it +14s host rss 87 MB guest MemTotal: 1998 MB MemFree: 1974 MB +guest freed it +15s host rss 87 MB guest MemTotal: 1998 MB MemFree: 1975 MB +guest freed it +16s host rss 87 MB guest MemTotal: 1998 MB MemFree: 1974 MB +guest freed it +17s host rss 87 MB guest MemTotal: 1998 MB MemFree: 1974 MB +guest freed it +18s host rss 87 MB guest MemTotal: 1998 MB MemFree: 1974 MB +guest freed it +19s host rss 87 MB guest MemTotal: 1998 MB MemFree: 1974 MB +guest freed it +20s host rss 87 MB guest MemTotal: 1998 MB MemFree: 1974 MB +guest freed it +21s host rss 87 MB guest MemTotal: 1998 MB MemFree: 1974 MB +guest freed it +22s host rss 87 MB guest MemTotal: 1998 MB MemFree: 1974 MB +guest freed it +23s host rss 87 MB guest MemTotal: 1998 MB MemFree: 1974 MB +guest freed it +24s host rss 87 MB guest MemTotal: 1998 MB MemFree: 1974 MB +target 1 GB +1s host rss 84 MB guest MemTotal: 1998 MB MemFree: 951 MB +target 1 GB +2s host rss 84 MB guest MemTotal: 1998 MB MemFree: 951 MB +target 1 GB +3s host rss 84 MB guest MemTotal: 1998 MB MemFree: 951 MB +target 1 GB +4s host rss 84 MB guest MemTotal: 1998 MB MemFree: 951 MB +target 2 GB +1s host rss 84 MB guest MemTotal: 1998 MB MemFree: 1887 MB +target 2 GB +2s host rss 84 MB guest MemTotal: 1998 MB MemFree: 1894 MB +wall=38.99s +=== guest reporting worker after a run (should be idle, not stuck in the report call) +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +stacks-checked +=== done + +=== Before the queue layout followed the negotiated features: the guest reporting worker +=== stuck in the report call and the balloon device (virtio2, IRQ 50) with a single +=== interrupt, no memory returned + +--- interrupts + 13: 1 0 GICv3 50 Level virtio2 + 14: 509 0 GICv3 48 Level virtio0 + 15: 77 0 GICv3 49 Level virtio1 +--- stacks mentioning balloon or reporting +pid 39 kworker/1:1+events_freezable +[<0>] virtballoon_free_page_report+0xa8/0xf8 +[<0>] page_reporting_process+0x268/0x380 +[<0>] process_scheduled_works+0x188/0x230 +[<0>] worker_thread+0x144/0x1c8 +[<0>] kthread+0xc8/0xd8 +[<0>] ret_from_fork+0x10/0x20 + +=== macOS host: Apple Silicon, Virtualization.framework, 2 GB guest +=== the framework balloon has no free page reporting; host rss is this process only + +booted host rss 10 MB guest MemTotal: 1998 MB MemFree: 1965 MB +guest holds 1 GB in tmpfs host rss 10 MB guest MemTotal: 1998 MB MemFree: 1705 MB +guest freed it +1s host rss 11 MB guest MemTotal: 1998 MB MemFree: 1968 MB +guest freed it +2s host rss 11 MB guest MemTotal: 1998 MB MemFree: 1968 MB +guest freed it +3s host rss 11 MB guest MemTotal: 1998 MB MemFree: 1969 MB +guest freed it +4s host rss 11 MB guest MemTotal: 1998 MB MemFree: 1969 MB +guest freed it +5s host rss 11 MB guest MemTotal: 1998 MB MemFree: 1970 MB +guest freed it +6s host rss 10 MB guest MemTotal: 1998 MB MemFree: 1970 MB +target 1 GB +1s host rss 10 MB guest MemTotal: 1998 MB MemFree: 947 MB +target 1 GB +2s host rss 10 MB guest MemTotal: 1998 MB MemFree: 947 MB +target 1 GB +3s host rss 9 MB guest MemTotal: 1998 MB MemFree: 947 MB +target 1 GB +4s host rss 10 MB guest MemTotal: 1998 MB MemFree: 947 MB +target 2 GB +1s host rss 10 MB guest MemTotal: 1998 MB MemFree: 1890 MB +target 2 GB +2s host rss 10 MB guest MemTotal: 1998 MB MemFree: 1894 MB diff --git a/docs/rfcs/0006-memory-balloon.md b/docs/rfcs/0006-memory-balloon.md new file mode 100644 index 0000000..928f3db --- /dev/null +++ b/docs/rfcs/0006-memory-balloon.md @@ -0,0 +1,138 @@ +# RFC 0006: Memory balloon + +- Status: Implemented +- Date: 2026-10-27 +- Scope: shuru-linux, shuru-darwin, shuru-vm, shuru-sdk +- Depends on: nothing + +## Summary + +Every VM gets a virtio memory balloon. On a Linux host the guest kernel +reports pages it has freed and the host discards them, so an idle sandbox +costs the host what the guest is actually using rather than what it has +touched since boot. On both hosts a caller can set a memory target and the +guest gives up or takes back the difference. + +```rust +let sb = AsyncSandbox::boot(config).await?; +sb.set_memory_target(1024 * 1024 * 1024).await?; +``` + +Nothing changes for `shuru run`. The device is attached, the guest driver +binds it, and free memory flows back on its own. + +## Motivation + +A hosted fleet of sandboxes is sold on memory. Each VM is configured with +2 GB, and before this change the host paid for every page the guest ever +touched: a build that used 1.5 GB and finished left 1.5 GB resident in the +host process until the VM stopped, even though the guest had freed it. +Overcommit is the whole business, and overcommit without a way to get +memory back is a crash waiting for a busy hour. + +The two standard answers are the virtio balloon and free page reporting, +and every serious hypervisor ships both. Firecracker, Cloud Hypervisor and +QEMU offer `VIRTIO_BALLOON_F_REPORTING`; Apple offers the traditional +balloon through `VZVirtioTraditionalMemoryBalloonDevice`. The guest kernel +already had `CONFIG_VIRTIO_BALLOON=y`, which pulls in page reporting, so +the only missing piece was the device on the host side. + +## Design + +### The device + +`shuru-linux` gains a virtio-mmio balloon backend (device ID 5) that offers +three features: + +- `VIRTIO_F_VERSION_1`. +- `VIRTIO_BALLOON_F_REPORTING`. The guest hands over runs of free pages of + `page_reporting_order` and above (order 10, 4 MB, on this kernel) a + couple of seconds after they free up. The host answers each run with + `madvise(MADV_DONTNEED)` on the matching host range, which drops the + pages from the process. Reporting happens in rounds two seconds apart, a + sixteenth of each free list per round, so a freed gigabyte drains in + under ten seconds. +- `VIRTIO_BALLOON_F_DEFLATE_ON_OOM`. A guest under memory pressure may + take ballooned pages back without asking, which is the safe default for + a host that set a target and walked away. + +The inflate queue carries arrays of 4 KiB page numbers; consecutive pages +are discarded as one run, and only whole host pages go, since a 4 KiB +guest page on a 64 KiB host page may share that page with memory still in +use. The deflate queue is acknowledged and nothing else: a discarded page +reads as zero when the guest touches it again, so giving memory back costs +one page fault per page and no host work. + +Queue indices follow the negotiated features. The spec lists stats, +free page hints and reporting as queues 2, 3 and 4, but each exists only +when its feature is negotiated, and a queue that does not exist takes no +index. The Linux driver, Cloud Hypervisor, crosvm and QEMU all number the +optional queues this way, in spec order, skipping the absent ones. The +backend builds its layout from the features the driver accepted at +`FEATURES_OK`, so with only reporting on offer the reporting queue is +index 2, and if stats are added later it moves to 3 on its own. The first +build hardcoded index 4, the guest kicked index 2, and its reporting +worker waited forever for an acknowledgement that never came (visible as +a kworker parked in `virtballoon_free_page_report`). + +### The target + +`BalloonControl` is the host handle. `set_memory_target(bytes)` turns the +difference between the configured size and the target into a page count, +writes it to the device config and raises a config change interrupt; the +guest driver allocates that many pages and sends them up the inflate +queue. The guest writes back how many pages it has actually given up, +which `ballooned_bytes` reads. + +On macOS the same call reaches `VZVirtioTraditionalMemoryBalloonDevice` +through `targetVirtualMachineMemorySize`, on the VM queue as the framework +requires. The framework offers no free page reporting, so on a Mac only +the target moves memory. That is fine: a Mac is one user running a few +sandboxes, and the hosted fleet is Linux. + +### Where it lives + +- `shuru-linux` 0.3.2: `kvm/balloon.rs`, `VirtualMachine::set_memory_target` + and `ballooned_bytes`. The device is attached when the configuration + carries a `VirtioMemoryBalloonDevice`. +- `shuru-darwin` 0.2.2: `VirtualMachine::set_memory_target`. +- `shuru-vm` 0.5.2: attaches the balloon on both platforms and exposes + `Sandbox::set_memory_target`. +- `shuru-sdk` 0.7.1: `AsyncSandbox::set_memory_target`, and an example + (`cargo run -p shuru-sdk --example balloon`) that prints host and guest + memory at each step. + +## What was measured + +a1.metal, Ubuntu 24.04 host kernel 7.0.0-1014-aws, 2 GB guest, resident +size of the host process: + +| step | host resident | guest MemFree | +|---|---|---| +| booted | 23 MB | 1968 MB | +| guest fills 1 GB of tmpfs | 1109 MB | 933 MB | +| 2 s after the guest frees it | 855 MB | 1975 MB | +| 5 s | 599 MB | 1975 MB | +| 7 s | 343 MB | 1975 MB | +| 9 s | 87 MB | 1975 MB | +| target 1 GB, 1 s later | 84 MB | 951 MB | +| target 2 GB, 1 s later | 84 MB | 1887 MB | + +The 1 GB came back without anyone asking, about 256 MB per two second +round. +Setting the target afterwards found nothing resident left to take, which +is the point: reporting had already done the work. On macOS the target +moved guest MemFree from 1970 MB to 947 MB and back to 1894 MB; the host +side is not visible from the process there. + +The guest reporting worker is idle after a run and the balloon device +shows the three negotiated feature bits, see the evidence log. + +## Not in this RFC + +- `VIRTIO_BALLOON_F_STATS_VQ`, guest memory statistics for the host. A + hosted scheduler will want them; they are a small addition on the same + device. +- Memory hotplug (`virtio-mem`). The balloon moves memory inside a fixed + size; growing a VM past its configured size is a different device. +- A `shuru run` flag. Nobody sets a target by hand on a laptop. diff --git a/docs/rfcs/0008-node-runtime-evidence.log b/docs/rfcs/0008-node-runtime-evidence.log new file mode 100644 index 0000000..3811f73 --- /dev/null +++ b/docs/rfcs/0008-node-runtime-evidence.log @@ -0,0 +1,65 @@ +=== Linux host: a1.metal, Ubuntu 24.04, kernel 7.0.0-1014-aws, KVM +=== shuru 0.9.3 (released), OS image 0.9.2, data dir on XFS +=== VM A writes a marker into its own /tmp and stays up; VM B, started 10 s later, +=== reads /tmp/who. Each VM has its own working disk, so B should never see the marker. + +shuru 0.9.3 +=== VM A: writes a marker, stays up 40 s +=== VM B: reads /tmp/who +b-exit=0 +--- VM processes while A is still up +2234 /home/ubuntu/shuru run -- sh -c echo marker-from-VM-A > /tmp/who; echo A-kernel-boot-id $(cat /proc/sys/kernel/random/boot_id); sleep 40; echo A-exiting +a-exit=0 +--- a.log +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +A-kernel-boot-id 4cc534ea-e08a-4207-a79b-c7b9685f731a +A-exiting +--- b.log +shuru: booting VM (2cpus, 2048MB RAM, 4096MB disk)... +marker-from-VM-A +B-kernel-boot-id 4cc534ea-e08a-4207-a79b-c7b9685f731a +B-done +--- vsock CID errors +none logged +=== done + +=== drivers/vhost/vsock.c, linux-6.12.y: a second VM asking for CID 3 gets EADDRINUSE. +=== shuru-linux would log that and carry on. B logged nothing, which fits the host side +=== connecting to CID 3, owned by VM A, and finishing before B set up its own vsock device. + +static int vhost_vsock_set_cid(struct vhost_vsock *vsock, u64 guest_cid) +{ + struct vhost_vsock *other; + + /* Refuse reserved CIDs */ + if (guest_cid <= VMADDR_CID_HOST || + guest_cid == U32_MAX) + return -EINVAL; + + /* 64-bit CIDs are not yet supported */ + if (guest_cid > U32_MAX) + return -EINVAL; + + /* Refuse if CID is assigned to the guest->host transport (i.e. nested + * VM), to make the loopback work. + */ + if (vsock_find_cid(guest_cid)) + return -EADDRINUSE; + + /* Refuse if CID is already in use */ + mutex_lock(&vhost_vsock_mutex); + other = vhost_vsock_get(guest_cid); + if (other && other != vsock) { + mutex_unlock(&vhost_vsock_mutex); + return -EADDRINUSE; + } + + if (vsock->guest_cid) + hash_del_rcu(&vsock->hash); + + vsock->guest_cid = guest_cid; + hash_add_rcu(vhost_vsock_hash, &vsock->hash, vsock->guest_cid); + mutex_unlock(&vhost_vsock_mutex); + + return 0; +} diff --git a/docs/rfcs/0008-node-runtime.md b/docs/rfcs/0008-node-runtime.md new file mode 100644 index 0000000..d34e45a --- /dev/null +++ b/docs/rfcs/0008-node-runtime.md @@ -0,0 +1,342 @@ +# RFC 0008: Linux node runtime + +- Status: Draft. The vsock device and the connection hello shipped in 0.10.0 (PR #56); host memory bounds in 0.11.0 (RFC 0011); the jail, `shuru-vmm` and `shuru-netd` in PR #60, driven by a one-shot `shuru-node run` until the agent exists. +- Date: 2026-10-27, updated 2026-10-28 +- Scope: shuru-linux, shuru-proto, shuru-guest, shuru-proxy, new `shuru-vmm`, `shuru-netd` and `shuru-node` binaries +- Depends on: nothing +- Required by: hosted Shuru, RFC 0009 (Linux VM snapshots) + +## Summary + +Every VM on a Linux host runs in its own process, under its own user, in +its own namespaces and cgroup, behind a seccomp filter, and holding nothing +but the file descriptors it was handed. The host reaches the guest through +a vsock device implemented inside that process and exposed as a unix +socket only the node agent can open, and every connection proves which VM +it reached. The network proxy, which holds real secret values, runs in a +separate process from the VM. + +The immediate reason is a bug: on Linux, a second sandbox started while +another is running executes its commands inside the first VM. The larger +reason is that a hosted node runs strangers next to each other, and today +the only wall between them is the hardware virtualization boundary. + +## Motivation + +### Two sandboxes on one Linux host share one guest + +Reproduced on a1.metal with the released 0.9.3 CLI (evidence log): VM A +writes a marker to its own `/tmp` and stays up; a second `shuru run` +started ten seconds later prints that marker and the kernel boot ID of VM A. +VM B never ran anything. Nothing was logged. + +The cause is that every VM asks vhost-vsock for guest CID 3. CIDs are +global to the host kernel, the kernel refuses the duplicate, and the host +side, which connects to CID 3 within milliseconds, reaches whichever VM +owns it. The guest protocol carries no identity, so nothing notices. + +Handing out unique CIDs would stop the collision and leave the design +problem in place: + +- Any process on the host that can create an `AF_VSOCK` socket can connect + to any guest agent port, and the agent executes what it is sent as root. + vsock ignores network namespaces, so jailing processes does not contain + this. +- vhost-vsock keeps its state in the host kernel, where a snapshot cannot + reach it (RFC 0009). +- Every local Linux user needs the `vhost_vsock` module loaded and access + to `/dev/vhost-vsock`. + +macOS is not affected. Virtualization.framework gives every VM its own +vsock device and the host reaches it through that VM object only. + +### The VM runs inside the caller + +The SDK and the CLI run the VMM on threads in their own process: vCPU +loops, virtio device emulation, the virtio-fs server and the network +proxy. A guest that finds a bug in descriptor parsing lands in a process +with the caller user, the caller files, every other VM that process +started, and, through the proxy, the real values of every secret. There +are no limits on memory, CPU, processes or disk bandwidth beyond what the +guest was configured with. + +On a laptop that process belongs to the person who started the sandbox, +and the in-process model is acceptable. On a hosted node it is not: one +tenant escaping device emulation would reach every other tenant on the +machine. + +## Design + +### Process layout on a node + +``` +shuru-node (root, one per host) + | control channel to the hosted control plane + | data API: exec, files, ports + | + +-- per sandbox, uid 100000 + slot, cgroup shuru.slice/sbx- + | + +-- shuru-vmm namespaces: mnt pid net ipc uts cgroup + | seccomp: kill on anything outside the allowlist + | fds in: /dev/kvm, disk, kernel, initrd, memory file, + | net socketpair end, vsock listener, control listener + | + +-- shuru-netd uid 200000 + slot, own cgroup + namespaces: mnt pid ipc uts cgroup, host network + seccomp: deny list, IP sockets only, no exec, no open + fds in: the other net socketpair end, egress policy, secrets +``` + +`shuru-node` is the only privileged process. It creates every resource a +VM needs, opens every file, and passes descriptors to two unprivileged +children per sandbox. + +### shuru-vmm + +A thin binary over the `shuru-linux` library. It receives its +configuration as one message on its control socket, builds the VM from the +descriptors it was given, and never opens a path or creates a socket after +that. Control operations on the same socket: + +- `start`, `stop` +- `pause`, `resume` +- `snapshot`, `restore` (RFC 0009) +- `set_memory_target`, `stats` (vCPU time, guest memory resident, balloon + state, virtio queue counters) + +It exits when the guest powers off or when its control peer asks. + +The control socket is a listening unix socket in a directory owned by the +agent, so a restarted agent reconnects to running VMs instead of losing +them. VMMs run in their own systemd scope and are not killed with the +agent. + +The `shuru` CLI and SDK on a local Linux machine keep calling the library +in process, as they do now. The library is the same in both places; only +the process wrapper and the jail differ. + +### The jail + +For each sandbox the agent: + +1. Allocates a slot. The slot gives the VMM uid and gid `100000 + slot` + and the proxy uid and gid `200000 + slot`. A slot is not reused until + both processes have exited and the cgroup is gone. +2. Creates the cgroup `shuru.slice/sbx-`, with the VMM and the proxy + in child cgroups below it, and on the parent: + - `memory.max` = guest memory + 320 MiB, `memory.high` 64 MiB below + that, and `memory.swap.max` = 0, so an overrun throttles and then + kills this sandbox only. RFC 0011 measured the allowance on a1.metal: + a 1024 MiB guest filling its RAM while 400 connections flooded it + peaked at 1122 and 1239 MB in a 1280 MB envelope. 320 MiB covers 256 + proxy sockets (192 MiB of smoltcp buffers and 64 MiB of relay + windows), 128 vsock connections (32 MiB) and the base process. Those + numbers were taken with the VMM and the proxy in one process; the + split between the two child cgroups is measured when they separate, + and the parent limit holds either way. + - `cpu.max` = vCPU count times the period, and a `cpu.weight` by plan. + - `pids.max` sized to the vCPU and device threads. + - `io.max` on the node data device, by plan. +3. Clones the VMM child into new mount, pid, network, ipc and uts + namespaces. The child joins its cgroup, unshares the cgroup namespace, + pivots into an empty read-only tmpfs root, drops every capability from + all sets including the bounding set, sets rlimits (`NOFILE`, `NPROC`, + `CORE` = 0), switches to the slot uid, sets `no_new_privs`, closes every + descriptor not on the hand-off list and executes the VMM binary from a + descriptor with `execveat`, so the root stays empty. The binaries are + static PIE: rustc links crt-static aarch64 executables without PIE, so + `scripts/static-pie-cc` relinks them with `-static-pie`. +4. The VMM installs its seccomp filter on all threads (`TSYNC`) after + building the VM and before the first `KVM_RUN`. The filter is an + allowlist built with the rust-vmm `seccompiler` crate: `ioctl` limited to + the KVM requests in use, `read`, `write`, `epoll`, `eventfd`, `futex`, + `mmap` and `mprotect` without `PROT_EXEC`, `madvise` with + `MADV_DONTNEED` only, `clone` with `CLONE_THREAD` only, `accept4` on the + handed-over listeners, signal and exit calls. `clone3` gets `ENOSYS`, so + the C library falls back to `clone`, whose flags a filter can read. + `openat` gets `EPERM`: glibc opens `/sys/devices/system/cpu/online` on + its own when a thread adds a malloc arena, and the root is empty anyway. + `socket`, `connect`, `execve`, `ptrace` and everything else kill the + process. `--seccomp-log` runs the filters in log mode to find missing + calls. +5. Starts the proxy child the same way with its own uid and cgroup, but in + the host network namespace, because it is the egress path. It runs + tokio and two TLS libraries, so its filter is a deny list: `execve`, + namespaces, `mount`, `ptrace`, `bpf`, `perf_event_open`, `io_uring`, + module and key calls and uid changes kill it, `openat` and sockets + other than IPv4 and IPv6 get `EPERM`. + +User namespaces are not used. The agent is root on a machine dedicated to +this, real uids are simpler to reason about, and unprivileged user +namespaces widen the host kernel surface a guest escape could reach. + +### Guest channel: vsock inside the VMM + +`shuru-linux` gains a virtio-vsock device (device ID 19) implemented in the +VMM, replacing vhost-vsock everywhere, including local Linux runs. + +- The device speaks the standard virtio-vsock packet protocol on its rx, + tx and event queues: connection requests, responses, resets, shutdowns, + data and credit updates, with per-connection credit flow control. The + guest driver is the one the kernel already has; nothing changes in the + guest image. +- The guest CID stays 3. It now names this VM inside this device only, so + there is no host-wide namespace to collide in. +- As shipped in 0.10.0, the host end of each connection is a socketpair + inside the process that runs the VM, which is all the CLI and SDK need. + The receive path is pull based: host data stays in the host socket + until the guest posts rx buffers, then one `readv` fills as many as the + guest credit allows. +- Under `shuru-vmm` the agent is a different process, so the VMM also + takes a listening unix socket. A client connects, writes + `CONNECT \n`, and gets `OK \n` once the guest + accepted, after which the stream carries bytes in both directions. This + is the convention Firecracker uses, so existing tooling understands it. + Each client reads its line on its own thread, with at most 16 waiting, + so a client that never writes holds up nobody. +- Guest-initiated connections to the host are refused with a reset. + Nothing uses them today, and a guest has no business opening one. +- Connections are capped per VM: 1024 locally, 128 on a hosted node, + given to `shuru-vmm` with its configuration. The proxy takes its socket + cap the same way: 512 locally, 256 hosted (RFC 0011). +- On restore from a snapshot the device sends a transport reset event, the + guest drops its connections, and the host reconnects (RFC 0009). + +On the node only the agent can open the socket: it lives in an +agent-owned directory with mode 0700. On a laptop it lives in the run +directory of the process that started the VM. + +### Guest identity on every connection + +Every connection to the guest agent opens with a hello: + +1. The host sends `HELLO { token }`, 32 random bytes generated by the host + for this VM and delivered on the kernel command line. +2. The guest checks the token in constant time, closes the connection on a + mismatch, and replies `HELLO_OK { instance_id }`. +3. The host checks the instance ID against the VM it meant to reach and + drops the connection otherwise. + +The token does not defend against the guest; the guest belongs to the +sandbox owner and can read its own command line. It defends against the +host reaching the wrong guest, which is exactly the failure above, and +against any host process other than the one that started the VM. After a +snapshot restore the host sends a new token and instance ID over the +first connection, so clones of one snapshot never share either (RFC 0009). + +This shipped in `shuru-proto` as protocol version 2 (0.10.0). The guest +image and the host binary come from the same release and are already +locked together by the `VERSION` file, so there is no mixed-version window +to support. macOS uses the same hello for uniformity. Serial output +redacts the token, and a guest booted without one stops instead of +serving. The rotation after restore is not built yet; it lands with RFC +0009. + +### Network + +The proxy (`shuru-proxy`) moves out of the VMM into `shuru-netd`, one per +sandbox. The VMM holds one end of the frame socketpair and never sees a +secret value. The proxy holds the other end, the egress policy and the +secrets bound to this sandbox only. + +Egress is enforced twice. The proxy refuses every destination that is +not a public address (`public_only`, which also turns off host exposure), +so a guest cannot reach the instance metadata service, the agent, or +other nodes. The host firewall (nftables, table `inet shuru`) drops the +same IPv4 ranges and all IPv6 from the proxy uid range (`meta skuid`), +which also gives per-sandbox byte counters and rate limits without any +proxy code. + +The agent sends the proxy its configuration with the frame socket: +the domain allowlist, the DNS resolver, secret values with their hosts, +and the root certificates for checking upstream TLS, so the proxy reads +nothing from disk and no secret passes through an environment or a +command. The proxy answers with the placeholders and its CA; the agent +installs the CA in the guest and puts the placeholders in the command +environment. + +On a laptop the proxy stays in process; the secrets it holds belong to +the person running it. + +### shuru-node + +The agent is the only component with root: + +- slots, cgroups, jails, uids, firewall rules; +- working disks (the descriptor-held clones from 0.9.2, on XFS) and + snapshot files (RFC 0009); +- the control channel to the hosted control plane; +- the data API that SDK clients reach (exec, PTY, files, watch, port + connections), served by opening the VM vsock socket and speaking the + existing guest protocol. + +It never parses guest-controlled data with privileges beyond what the +data API needs, and its own data API endpoint is authenticated with +tokens from the hosted control plane. + +## What will be measured + +Measured on a1.metal for PR #56, against 0.9.3 with vhost-vsock on the +same host: + +- The reproduction in the evidence log fails: two concurrent VMs each see + only their own marker and boot ID, six VMs at once give six boot IDs. +- Exec round trip p50 1.8 ms against 1.6 ms (target: within 1 ms of vhost). +- Guest to host 434 to 451 MB/s against 117 to 191 MB/s; host to guest 315 + to 360 MB/s against 353 to 366 MB/s (target: at least 200 MB/s). + +Measured for RFC 0011: host overhead per VM is 6 to 11 MB outside guest +RAM in every pressure scenario except 400 stalled connections (about 200 +MB), and a sandbox in the cgroup envelope above survives its worst case. + +Measured on a1.metal for PR #60, every filter in kill mode: + +- Boot to exec ready: `shuru-node run -- true` 372 to 416 ms against 414 + to 427 ms for the CLI; the jail itself costs 6 to 17 ms. The proxy adds + about 30 ms, and installing its CA in the guest about 1.7 s when a + secret is bound. +- Exec, disk writes, 64 MiB over vsock, https through `shuru-netd` and + stop with no kill; log mode logs nothing. +- From `/proc`: both processes on their slot uid with every capability + set empty, `NoNewPrivs`, the filters in place, an empty root, and own + namespaces except user (and network for the proxy). +- The guest reaches public addresses; the metadata service, the host + private address and the gateway are refused, and so is a name outside + the allowlist. A secret goes out real to its bound host and as the + placeholder elsewhere. The firewall drops the same addresses for a + proxy uid outside any jail. +- 1.5 GiB written in a 2 GiB guest peaks at 1621 MiB in the sandbox + cgroup, so guest memory is charged to it. +- 100 MB download: 23 MB/s through the proxy, 40 MB/s on the host. + +Still to measure, before hosted ships: + +- The full SDK test suite against a jailed VM. +- Density: idle 256 MB sandboxes on one host until memory runs out, with + free page reporting on (RFC 0006). + +## Alternatives considered + +- **Unique CIDs with vhost-vsock.** Fixes the collision, leaves every + guest reachable from every host process and leaves vsock state outside + any snapshot. Rejected. +- **Firecracker as the hosted VMM.** It has a jailer, a seccomp allowlist, + userspace vsock and snapshots today, and years of production use. It + would also mean two VMMs: ours for local Linux and macOS parity, and + Firecracker for hosted, each with its own device set, boot path and + failure modes against one guest image. The pieces this RFC adds are + bounded: one virtio device, one jail routine, one filter. Keeping one + VMM is the recommendation, with the caveat that ours is younger and + unaudited; a fuzzing harness for the virtio queue and descriptor + parsing is part of this work. +- **gVisor or containers instead of VMs.** A different isolation model + from the one Shuru is built on. Out of scope. + +## Not in this RFC + +- The hosted control plane, the public API and the node data API wire + format. +- Snapshots (RFC 0009). +- virtio-fs on hosted nodes. Hosted sandboxes have no host directory to + share; the device is not attached there. +- Changes to macOS beyond the protocol hello. diff --git a/docs/rfcs/0009-linux-vm-snapshots.md b/docs/rfcs/0009-linux-vm-snapshots.md new file mode 100644 index 0000000..393c693 --- /dev/null +++ b/docs/rfcs/0009-linux-vm-snapshots.md @@ -0,0 +1,283 @@ +# RFC 0009: Linux VM snapshots + +- Status: Draft +- Date: 2026-10-27, updated 2026-10-28 +- Scope: shuru-linux, shuru-guest, shuru-proto, guest kernel config, `shuru-vmm`, `shuru-node` +- Depends on: RFC 0008 (node runtime) +- Required by: hosted Shuru + +## Summary + +A running VM on a Linux host can be saved to a snapshot: guest memory, +vCPU and interrupt controller state, device state and the working disk, +captured at one instant. A snapshot restores into a new VM by mapping the +memory file instead of reading it, so restore cost depends on the size of +the device state, not the size of guest memory, and pages load when the +guest touches them. Many sandboxes can restore from one snapshot at once +and share its unchanged pages in the host page cache. + +One mechanism serves three uses in hosted Shuru: templates (set a sandbox up +once, start every new one from that point), pause and resume of idle +sandboxes, and forks of a running sandbox. + +## Motivation + +### The macOS attempt lost to a cold boot, and why + +RFC 0001 saved and restored machine state on macOS. A cold boot took 0.44 +to 0.49 s; restoring took about 0.63 s. Two things made restore lose: + +- Apple restore reads and decrypts the whole state file before the guest + runs. The cost grows with the state, and it is not under our control. +- The snapshot was taken at the first vsock handshake, the earliest point + a guest can answer. It held nothing a cold boot does not reach in 0.3 s, + so there was nothing for restore to skip. + +The snapshot paid full price and skipped almost no work. + +### What is different on Linux + +- **Restore can be lazy.** We own the VMM. Mapping a memory file with + `MAP_PRIVATE` makes restore a handful of system calls; the kernel pages + memory in from the page cache on first touch and copies a page only when + the guest writes it. Firecracker restores this way, and its published + restore times are in milliseconds. +- **Cold boot is slower here.** On a1.metal, `shuru run -- true` takes + 412 to 431 ms with the 0.11.0 release, down from 1.25 to 1.77 s in the + 0.9.2 cycle, and the guest reaches the command 160 to 170 ms after the + kernel starts. The same command takes 140 to 160 ms on an M-series Mac. + There is more to skip than on a Mac, though far less than when this RFC + was first drafted. +- **The real win is warm state.** A template snapshot is taken after the + setup a user cares about: packages installed, an interpreter with its + imports loaded, a dev server listening, a browser started. A cold boot + cannot skip that work at any speed. This is the case the macOS attempt + never measured. + +So this RFC makes two claims to test, not assume: restoring a bare base +snapshot beats a cold boot on Linux, and restoring a warm template beats +cold boot plus setup by a wide margin. + +## Design + +### What a snapshot is + +A directory on the node data volume (XFS): + +``` +snap-/ + manifest.json format version, release and guest protocol version, + CPU fingerprint, guest image hashes, memory size, + vCPU count, device list, parent snapshot + state.bin vCPU, interrupt controller, timer and device state, + versioned binary encoding + memory guest memory, a sparse file the size of guest RAM + disk the working disk, a reflink clone +``` + +`memory` and `disk` are reflink clones where possible, so a snapshot of a +VM that was itself restored from a snapshot shares every unchanged extent +with its parent on disk. + +### Guest memory + +Guest memory today is anonymous memory from `vm-memory`. Under this RFC it +comes from one of two places: + +- **Cold boot:** a `memfd` of guest RAM size, mapped shared. Saving writes + only the resident ranges of the memfd to `memory`, using `SEEK_DATA` and + `SEEK_HOLE` to skip what was never touched. Free page reporting (RFC + 0006) keeps the resident set close to what the guest actually uses, so a + 2 GB guest using 300 MB writes about 300 MB. +- **Restore:** `memory` mapped `MAP_PRIVATE`. Unwritten pages stay in the + page cache, shared by every VM restored from the same file. A guest + write gives that VM a private copy of one page. + +Saving a VM that was restored from a snapshot writes only what it changed. +Pages the guest wrote are anonymous copies; pages it did not write still +map the parent file. `/proc/self/pagemap` tells the two apart, so the +snapshot is a reflink clone of the parent `memory` with the changed pages +written over it. Pausing a sandbox that has been idle since its last +resume costs almost nothing. + +Free page reporting has to follow the backing. On a shared memfd, +`MADV_DONTNEED` only unmaps and the page stays in the file, so the balloon +backend switches to `MADV_REMOVE`, which punches the hole. On a restored +VM, `MADV_DONTNEED` on the private mapping drops the private copy and the +page reverts to the file content, which stays in the page cache shared and +reclaimable. The guest does not rely on the content of a reported page, +because Shuru does not offer page poisoning (RFC 0006). + +### vCPU and interrupt state (arm64) + +Per vCPU, with the VM paused: + +- every register in `KVM_GET_REG_LIST`, read with `KVM_GET_ONE_REG`: core, + floating point and SIMD, system registers including `MPIDR_EL1`, and + the timer registers; +- `KVM_GET_MP_STATE`, so vCPUs the guest powered off through PSCI stay off; +- `KVM_GET_VCPU_EVENTS`, for a pending SError. + +The GICv3 through its device attributes: distributor registers, +redistributor registers per vCPU, CPU interface system registers and line +level information. Shuru does not create an ITS, so there are no pending +tables in guest memory to flush. + +Restore order follows the kernel requirements: create vCPUs, initialise +them, restore `MPIDR_EL1`, create and initialise the GIC, restore its +state, then restore the rest of each vCPU. + +### Time + +The guest virtual counter must not jump forward by the time a snapshot sat +on disk, or every timer fires at once. On restore the VMM sets the counter +offset (`KVM_ARM_SET_COUNTER_OFFSET`, host kernel 6.4 and later) so the +guest counter continues from where it stopped. The guest then learns the +real time from the host over the first connection after restore and steps +`CLOCK_REALTIME`, which also fixes the stale clock RFC 0001 corrected with +an exec of `date`. + +### Clones must not share randomness + +Every VM restored from one snapshot starts with identical kernel random +state. Two sandboxes from one template would generate the same keys. + +The guest kernel gets a VM generation ID device (`CONFIG_VMGENID`, device +tree compatible `microsoft,vmgenid`, supported by Linux since 6.10). The +VMM writes a new random generation ID into the device on every restore and +raises its interrupt. The kernel reseeds its random number generator +before anything else in the guest runs again, and announces the change to +userspace with a uevent. + +Process state is a different matter. A program that seeded its own +generator before the snapshot keeps that seed in every clone. Shuru cannot +fix that from outside. The documentation will say so plainly: take +templates before starting processes that hold keys, or restart them after +restore. The guest agent also rotates the per-boot token and instance ID +from RFC 0008 on the first connection after restore. + +### Devices + +The virtio-mmio transport saves features, status, interrupt status and +each queue (size, ready, ring addresses, last available index). Backends +add their own state: + +| device | saved | on restore | +|---|---|---| +| block | nothing beyond the queues | attaches the snapshot `disk` clone | +| net | nothing | a new proxy (RFC 0008); open guest TCP connections to the outside are reset | +| balloon | target and actual pages | keeps the target; reporting continues | +| vsock | nothing | sends a transport reset event, the guest drops connections, the host reconnects | +| console, rng | nothing | resumes | +| virtio-fs | refuses to snapshot | hosted nodes do not attach it; a local VM with mounts cannot be snapshotted | + +### Taking a snapshot + +1. Stop the vCPUs: signal every vCPU thread out of `KVM_RUN` and wait. +2. Stop device threads from pulling new work (net rx, vsock). +3. Save vCPU, GIC and device state. +4. Write memory: the resident ranges for a cold-booted VM, the changed + pages over a reflink of the parent for a restored one. +5. Reflink the working disk descriptor into `disk`. +6. `fsync` the files, write `manifest.json` last, rename the staging + directory into place. +7. Resume the vCPUs, or stop the VM when the snapshot is for a pause. + +Memory and disk come from the same paused instant, and dirty guest page +cache is part of memory, so the guest does not need to `sync` first. This +is why a snapshot skips the freeze that disk-only checkpoints take (RFC +0010): a checkpoint keeps the disk and drops memory, so the filesystem +must be consistent on its own; a snapshot keeps both. + +### Restoring + +1. Check the manifest: format version, release, and the CPU fingerprint + (`MIDR_EL1` and the ID registers KVM exposes). Nodes in one pool run + identical hardware. A snapshot this host cannot resume is not + attempted; it falls back as described under Snapshots across releases. +2. Map `memory` private, reflink `disk` into a new working disk. +3. Create the VM and vCPUs, restore GIC and vCPU state, set the counter + offset, restore devices, write a new VM generation ID. +4. Start the vCPUs. Connect over vsock, rotate the token, send the time. + +### Snapshots across releases + +A snapshot holds the guest kernel and agent of the release that took it, +running, in `memory`, and its device state in the encoding of that VMM. +A newer host restoring it does not boot the new kernel; it resumes the old +one, so it has to read the old device state and talk to the old agent. +Refusing every snapshot from an earlier release would make each upgrade +break every template and every paused sandbox. Instead: + +- The manifest records the release and the guest protocol version. + `state.bin` is versioned per section, and the VMM reads the encodings of + the last few releases (four to start) and refuses older ones. +- The host keeps speaking the guest protocol versions of those releases. + The hello reply gains the guest protocol version, and the host uses the + matching framing on that connection. Until now a protocol change could + drop older agents, as protocol 3 did in 0.11.0. From the first release + with snapshots, a change keeps the older versions in the window + answerable. +- A device added in a later release is absent from an older snapshot; + the restored VM runs without it until its next cold boot. A device whose + state encoding changed reads the old encoding. +- Past the window, or on a CPU fingerprint mismatch, a resume becomes a + cold boot of the snapshot `disk` with the current kernel and image. + Files survive; processes and memory do not. The host reports it as a + reboot with the reason, never as a successful restore. + +Disks need none of this. A working disk is a block level clone, so +replacing the base image never changes a disk cloned from it, and an old +disk boots with the current kernel and agent because the agent lives in +the initramfs (RFC 0010). + +### Snapshots that leave the node + +A snapshot is a self-contained directory: the manifest names everything +restore checks, and `memory` and `disk` are plain files. Restore needs +them on local XFS. Where a snapshot is kept when it is not on the host, +and how it gets back, is up to whoever runs the host and is outside this +RFC. + +Fetching pages on demand through `userfaultfd` would let a restore start +before the whole `memory` file has arrived; it is left for later because +the local version is the one to measure first. + +### Prefetching the working set + +Pages loaded on first touch cost a page fault each, and E2B documents that +templates restore faster than live snapshots because they prefetch. For +templates, the agent records which pages a restored VM touches in its +first second (`mincore` on the mapping), stores that list beside the +snapshot, and asks the kernel to read those ranges ahead +(`MADV_WILLNEED`) at the next restore. It is an optimisation and lands +only if the measurements show page faults matter. + +## What will be measured + +On a1.metal, 2 vCPU and 2 GB guests, against the 412 to 431 ms cold boot +of 0.11.0: + +| case | measured as | target | +|---|---|---| +| restore of a base snapshot taken at first handshake | restore call to first exec reply | under 100 ms | +| restore of a warm template (Python with numpy imported, a Node dev server listening, a headless browser started) | restore call to the warm work answering | under 150 ms, against cold boot plus setup | +| 50 concurrent restores from one snapshot | wall time, host memory used | shared pages counted once | +| pause of a sandbox restored from a template and idle since | pause call to VM gone, bytes written | under 100 ms, changed pages only | +| pause of a cold-booted 2 GB guest using 300 MB | bytes written, time | about 300 MB | +| restore of a snapshot taken by the previous release | restore call to first exec reply | same as a same-release restore | +| resume of a snapshot past the window | what the caller sees | a reported reboot from its disk, files intact | + +Every number goes in the evidence log next to the cold boot figures from +the same host and build. If restore of the base snapshot does not beat a +cold boot, the RFC says so and templates carry the feature alone. + +## Not in this RFC + +- macOS. The framework restore stays what it is; local runs keep cold + boots. Revisit only if Apple changes how restore loads memory. +- Snapshots of a VM with virtio-fs mounts. +- Live migration between nodes. +- Storing snapshots off the host and moving them between hosts. +- On-demand page fetching (`userfaultfd`). +- Snapshot compression and encryption at rest. diff --git a/docs/rfcs/0010-fast-boot.md b/docs/rfcs/0010-fast-boot.md new file mode 100644 index 0000000..ff665fc --- /dev/null +++ b/docs/rfcs/0010-fast-boot.md @@ -0,0 +1,564 @@ +# RFC 0010: Fast boot and sealed disks + +- Status: Implemented +- Date: 2026-10-27 +- Scope: shuru-guest, shuru-proto, shuru-linux, shuru-vm, shuru-sdk, shuru-cli, `scripts/prepare-rootfs.sh`, `kernel/shuru_defconfig`, OS image +- Depends on: nothing +- Bears on: hosted Shuru, RFC 0009 (Linux VM snapshots) + +## Summary + +Two changes that need each other. + +**Sealed disks.** Every saved disk (checkpoint, recipe build, fork of a +running sandbox) is taken from a guest that froze its root filesystem +first. One guest call, one host function, used by every path that saves a +disk. A saved disk has its ext4 journal flushed and `needs_recovery` cleared. +Orphan cleanup can still be needed for open, unlinked files. It contains +completed filesystem writes, not application buffers or process state. +A guest that cannot seal leaves no checkpoint. + +**Fast boot.** The kernel handles journal replay; ordinary boots need no +userspace fsck or offline resize. A small error gate refuses known damage, +and the kernel still validates the filesystem as it mounts. The initramfs +shrinks to one uncompressed executable, shuru-guest installed as `/init`, +which mounts the disk, grows the filesystem when the disk is bigger, +sets up the network and becomes the agent. Busybox, e2fsck, resize2fs and +the shell script go. + +The earlier measurements suggest about 350 ms of removable work from the +0.65 s Linux boot. That is a target, not a measured result of this change. +The new mount, error checks and resize costs must be measured on both hosts. + +## Motivation + +### Where a boot goes today + +a1.metal, 2 vCPU, 2 GB guest, 0.10.0 OS image, host page cache warm, +seven runs: + +| stage | time | +|---|---| +| VM created to guest kernel start, plus connect and hello | about 210 ms, not yet broken down | +| kernel start to `Run /init` | 142 to 146 ms, of which about 105 ms unpacks the 3.8 MB gzip initramfs | +| `/init` to `EXT4-fs (vda): mounted` | 247 to 261 ms: `e2fsck -p` and `resize2fs` on `/dev/vda` | +| root mounted to the agent answering | 35 to 45 ms | +| VM created to first exec answered | 644 to 664 ms | + +A cold host cache adds about 100 ms. `shuru run -- true` takes 0.70 to +0.75 s end to end, against 0.44 to 0.49 s on a Mac. + +### Why fsck and resize run at all + +The image is a 1 GB ext4 made by a default `mkfs.ext4`: it has a journal, +`metadata_csum`, `resize_inode` and `64bit`. + +- **resize2fs** came with configurable disk sizes (8503e0e). The default + disk is 4096 MB and `WorkDisk::clone_from` extends the cloned file + without touching the filesystem in it, so every run from the base grows + the same 1 GB to 4 GB again. +- **e2fsck** came with checkpoint resizing (8f9db5d). A checkpoint is + saved from a guest that was powered off without unmounting its root, so + its journal still holds work (`needs_recovery` is set; the `docker` + checkpoint on this machine has it). Offline resize2fs will not touch a + filesystem in that state, or one not checked since it was last mounted, + so e2fsck ran first to replay the journal and stamp the check. + +Neither does anything for the kernel, which replays a journal on its own +when it mounts. They exist only because resizing was done offline. + +### Saved disks are not taken the same way twice + +There are three ways to save a disk today and each has different +guarantees: + +| path | what happens before the clone | result | +|---|---|---| +| `shuru checkpoint create` and `update` | services stopped, `sync`, power off. If the guest does not answer in 90 s it is powered off anyway and the clone still happens | journal not flushed; after a timeout, writes still in the guest page cache are missing | +| `run --stdio` checkpoint (TypeScript SDK) | `sync` run as an exec while everything keeps running, then the clone, then stop | writes between the `sync` and the clone race the clone | +| `AsyncSandbox::checkpoint` (Rust SDK) | nothing; the VM keeps running | whatever had reached the disk; the existing checkpoint is deleted before the new one is written | + +A reflink can capture a block-level crash state. A streaming copy of a +changing disk can mix states from different times; journaling does not +make that safe. Even a crash-consistent clone can omit writes still in +the guest cache. + +### What the initramfs still does that matters + +Every boot it copies its own shuru-init over `/usr/bin/shuru-init` on the +disk. That is how a checkpoint built by an older image gets the agent that +matches the host, which the connection hello depends on. Booting the disk +directly would run the agent the checkpoint was saved with. So the +initramfs stays as the way the agent reaches the guest, and everything +else leaves it. + +## What is supported + +**Supported** + +- **Ephemeral runs.** The disk is discarded at exit. Nothing is sealed. +- **Checkpoint at stop.** `shuru checkpoint create` and `update`, recipe + builds, the TypeScript SDK. The guest services of RFC 0004 (daemons an + image declares, such as dockerd, supervised by the agent inside the + guest) get their SIGTERM and stop timeout, then the seal, the clone and + the power off. Other processes are not signalled; the seal captures + what they had written. +- **Checkpoint of a running sandbox.** The Rust SDK today, and forks of a + running sandbox in hosted Shuru. Seal, clone, thaw; the sandbox carries on. + Services are not stopped. What they had written is on the disk and what + they held in memory is not, the same contract as a cloud disk snapshot: + a database recovers as it would after a power cut. +- **Forks at the same size or larger.** The guest grows the filesystem + once, at boot, and every disk saved afterwards keeps the new size. +- **Large free extents offered for discard,** while a sandbox runs and + before every save. Reclamation depends on the backing storage. +- **Checkpoints from before this change.** The kernel replays their + journal when it mounts them. No code for it; the first seal after that + makes them clean. + +**Not supported** + +- **Saving from a guest that did not seal.** Hung, panicked, killed, + powered itself off, timed out: no checkpoint, an error saying which, and + on `update` the old checkpoint untouched. A disk of unknown state is not + saved under a name. +- **Shrinking a disk, or a disk smaller than 4096 MB.** See Sizes below. +- **Memory or process state in a checkpoint.** RFC 0009. +- **Repairing a damaged filesystem at boot.** The filesystem is set to + `errors=remount-ro`, so damage the kernel finds stops writes and the run + fails loudly instead of spreading it. A model where damage is normal is + the wrong model. On 6.12 the option shuts the filesystem down: reads + fail as well as writes, so no program on the root runs again. The agent + lives in memory and still answers, refusing a seal with the error. Read + access after an error arrives with `EXT4_FLAGS_EMERGENCY_RO` in 6.16. +- **A running disk that survives a host crash.** The durable state of a + sandbox is its last saved disk. The journal would make a crashed disk + consistent, so this can be offered later without a new design; it is + not promised now. + +## Design + +### Seal and thaw + +`shuru-proto` gains `Seal` and `Thaw` here, and `Trim` (below). + +- `Seal { id: u64, stop_services: bool, lease_ms: u64 }`. The lease is + between 1000 and 300000 ms. A second active seal is refused. Stop the + declared services if requested, check root health, call `syncfs` on the + root descriptor, trim, then `FIFREEZE`. Check health again after freezing + and require `needs_recovery` to be clear before answering `Sealed`. +- `Thaw { id: u64 }`. Release only that seal with `FITHAW`. Answer `Thawed` + if the copy finished within its lease, otherwise `Expired`. + +The guest starts the monotonic lease after freeze returns. An independent +thread thaws at the deadline and retains the expired state for the next +matching Thaw. The host must receive `Thawed` before publication. A lost +reply, wrong id, timeout or `Expired` discards the private copy. A lease +cannot cancel a blocked freeze ioctl: the host times out, saves nothing, +and a freeze that eventually completes still gets its lease watchdog. + +Freeze waits for ordinary writers, shared writable page faults and +internal filesystem writers, then syncs and flushes the journal. Reads +that need atime updates can also wait. The control and watchdog paths must +not write the root, touch writable root mappings, or wait for locks held +by blocked writers. Runtime state and service logs live on `/run` tmpfs; +the executable remains on initramfs. Existing request threads and guest +processes can block in the kernel until thaw without blocking control. + +This freezes the ext4 superblock, including overlay upper files on it. +It does not freeze an independent filesystem on a loop-backed file, and +raw block writes bypass the filesystem. Freezing an outer filesystem before +syncing an inner loop filesystem can deadlock. The shipped kernel disables +loop devices and swap. Custom kernels with these dependencies need their +own quiescing protocol; they are not covered by this seal contract. +Dockerd overlay2 data is covered, but daemon memory and writes to other +filesystems are not. + +Before mount the guest reads only the fixed ext4 superblock on `/dev/vda` +and rejects `s_state & EXT4_ERROR_FS`. Mount read-only with journal replay +enabled, check again, then remount read-write with `errors=remount-ro`. +This is an error gate, not an integrity scan. At seal, reject a read-only +root, the persisted error flag, an increase from the boot value of +`/sys/fs/ext4/vda/errors_count`, or a failed sync/freeze. A historical error +count alone is not an error flag. The kernel still validates checksums and +mount features. No host code reads ext4 metadata. + +While sealed the agent refuses requests that would write to the root, and +it writes nothing there itself (its state is on the `/run` tmpfs). +Writes to other mounts (tmpfs, virtiofs) are unaffected and are not part +of the disk anyway. + +Freezing, not a read-only remount, because a remount fails with `EBUSY` +while any process holds a file open for writing. That would make +checkpoints of a running sandbox impossible and checkpoints at stop +depend on killing every stray process first. Freeze can still block on +I/O or filesystem dependencies. A host timeout +is a failed save, never permission to copy an unsealed disk. + +### One host function + +`shuru-vm` gets `Sandbox::save_disk(dest, after, copy)` with +`AfterSave::Thaw` or `AfterSave::PowerOff`. The copy callback writes either +the work disk clone or the CAS index: + +1. Serialize saves for this sandbox and seal with a fresh id. A stop save + asks the guest to stop its declared services. +2. Reserve `dest.partial` as the staging directory and hold an exclusive + `flock` on it. A held lock means another save of `dest` is running, and + this one fails. A directory with no lock holder was left by a save that + died, and is reused. Clone into its new `disk` file, by reflink or sparse + copy, or write the CAS index. Sync the staged file. +3. Thaw and require the matching `Thawed` answer. A failure removes only + the staging directory owned by this operation and leaves `dest` intact. +4. For PowerOff, stop the VM. Then atomically rename the staged disk onto + `dest` and sync its parent directory. Remove the empty staging directory. + +Both paths validate by thawing before publication. On the stop path this +briefly allows writes to the original disk before power off; the completed +clone is independent and stays sealed. Keeping the guest frozen through +power off would require another acknowledgement protocol to prove that its +lease had not already expired. Renaming before validation is incorrect. +A directory sync failure after rename reports uncertain durability; it +cannot restore the old name. Interrupted staging directories are left for +explicit operator cleanup, never silently reused. + +The CLI checkpoint commands, recipes, stdio, Rust SDK and CAS all use this +function. No running disk can be saved by a caller skipping the seal. CAS +indexes merge inherited hashes and ingest remaining flat-base chunks when +saving, so neither a mutable parent name nor a replaced base image changes +a saved child. The first save can therefore read the remaining base image. +CAS chunks are published atomically before the index. Unsupported NBD trim +remains disabled. NBD attachment is currently available only on macOS; +the existing Linux backend rejects it. A name cannot switch between `.idx` +and `.ext4` in place; use a +new name, so an older alternate file cannot silently shadow the new disk. + +The agent comes from the image initramfs. HELLO_OK now carries the 16-byte +instance id followed by protocol version 3 as a little-endian u32. The host +rejects the old 16-byte reply or a different version before sending work. +An explicitly supplied old initramfs cannot silently bypass this contract. + +### The initramfs is the agent + +The image ships `initramfs.cpio`: an uncompressed newc archive holding +`/init`, the static musl shuru-guest binary (2.3 MB), and the directories +it mounts on. Unpacking an uncompressed archive is a copy. + +shuru-guest starts as `/init` and runs a first stage when it finds itself +on the initramfs: + +1. Require PID 1 on ramfs or tmpfs. Mount devtmpfs on `/dev`, perform + the error gate above and mount `/dev/vda` on `/newroot`. +2. Grow the filesystem only if the device is larger (below). +3. Unmount the initramfs `/dev` and delete `/init` and the initial `/dev` + contents while the old root is still reachable. Never traverse the + `/newroot` mount during cleanup. +4. `chdir("/newroot")`, move `.` onto `/` with `MS_MOVE`, `chroot(".")`, + `chdir("/")`. No exec is needed: musl is static and PID 1 continues in + the same process. Close old-root directory and device descriptors. +5. Mount proc, sysfs, devtmpfs and runtime tmpfs, then carry on as the agent. + +Unlinking does not free the executable while mappings and the executable +reference hold its inode. Account for the binary backing pages for the +life of the VM; the old 2.3 MB size is only a measurement of the old build. +Deleting after chroot would instead address the new disk. + +There is no exec and no copy onto the disk. The agent never lives on the +guest writable disk, and the image stops installing `/usr/bin/shuru-init`. +Copies inside old checkpoints stay as dead files. + +`CONFIG_BLK_DEV_INITRD` and `CONFIG_DEVTMPFS` are already set, and an +initramfs is always run as `/init`, so `init=/usr/bin/shuru-init` is +dropped from the command line. + +### Kernel tick + +The guest kernel moves from `CONFIG_HZ_100` to `CONFIG_HZ_1000`. A virtio +PCI reset writes 0 to `device_status` and must wait until it reads back 0 +(virtio 1.2, PCI transport); `vp_reset` in +`drivers/virtio/virtio_pci_modern.c` polls with `msleep(1)`. Apple devices +finish a reset asynchronously, so each device sleeps once, and at 100 Hz +`msleep(1)` is two ticks, 10 to 20 ms. On a Mac the four devices took +about 100 ms of every boot this way. + +Measured on an M-series Mac, the 0.8.0 CLI and image, 6.12.114 built from +this defconfig with only `CONFIG_HZ` changed, median of eight warm boots: + +| | 100 Hz | 1000 Hz | +|---|---|---| +| `shuru run -- true` wall | 300 ms | 214 ms | +| guest uptime at the command | 150 ms | 70 ms | +| `hvc0` enabled (end of device probe) | 113 ms | 25 ms | +| `Run /init` | 118 ms | 41 ms | + +The guest is tickless when idle (`CONFIG_NO_HZ_IDLE`), so an idle vCPU +takes no ticks either way; a busy one takes 1000 timer interrupts a second +instead of 100. Linux hosts use virtio-mmio, whose reset path differs, so +the effect there is measured, not assumed. + +### Growing the filesystem + +Online resize is one ioctl, and the kernel does the work on the mounted +filesystem: + +1. `BLKGETSIZE64` on `/dev/vda` for the device size. +2. `statfs` for the block size and the guest-read superblock for the total + block count. `statfs.f_blocks` excludes overhead and cannot replace the + latter. Honor the `64bit` feature when reading the high count. +3. Skip resize when counts match. Otherwise use `EXT4_IOC_RESIZE_FS` with + a pointer to a u64 block count, then verify the resulting total. + +Equal counts return success from `ext4_resize_fs`, but the ioctl still +reads the final device block and flushes the journal. Avoiding it matters. +Growth requires CAP_SYS_RESOURCE, a writable primary superblock, no ext4 +error state and a readable last block. `sparse_super2` is unsupported. +The usual journal, metadata_csum, resize_inode, 64bit and flex_bg features +support growth; reserved GDT exhaustion can convert resize_inode to meta_bg. +64bit does not remove the 32-bit inode count limit. The last partial group +can be rounded down if too small for metadata; verification refuses an +incomplete requested growth rather than silently reporting the wrong size. + +Growth adds group tables, bitmaps and backup metadata and flushes the +journal. With 4 KiB blocks and normal 32768-block groups, 4 GiB to 8 GiB +adds 32 groups; to 64 GiB adds 480. These are not constant-time +operations and no latency estimate is claimed until measured. + +The root is mounted with `noinit_itable`. Without it the lazyinit thread +writes zeros over the inode tables of every new group after a grow, 48 MB +for 1 GB to 4 GB, competing with the first commands of the run. Those +tables sit in space the disk file never wrote, which reads as zero +already, and the checksummed group descriptors record which inodes are +unused. The filesystem keeps its new +size in later checkpoints. Online growth needs no fsck timestamp. + +It runs before the agent listens, so the first request always sees the +size the caller asked for. Growing happens once per size change, never on +a plain boot. A failure stops the boot with the kernel error on the +console, because a disk smaller than asked for is a wrong disk. + +Growth happens in the guest kernel. The host never parses a filesystem a +guest wrote. + +### Sizes live with the disk + +A disk is its size. A run or fork asking for more extends the file, as +now, and the guest grows the filesystem once; a disk saved afterwards +keeps that size. A request smaller than its source disk is rejected, including CAS. + +4096 MB becomes the smallest disk. A configuration asking for less is +refused with a message naming the minimum. + +The base image still ships as a 1 GB filesystem, and the host prepares the +file once: `WorkDisk::prepare_base` rewrites it sequentially into a new +file, extends that file to 4096 MB and renames it over the base. The +installer runs it after unpacking, and a boot from the base runs it when +the base is still short, which covers SDK users and bases built locally. +A base at least 4096 MB long is taken as prepared, so every later boot +pays one `stat`. It never applies to `--rootfs` or to a checkpoint. + +The file layout is the reason. On APFS, extending a clone, or writing +into the space beyond its source, costs time in proportion to the extent +count of the source file. Measured on an M-series Mac, extending a clone +of the 1 GB base to 4 GB: + +| source | extend | +|---|---| +| base built by `prepare-rootfs.sh` | 550 to 1480 ms | +| base unpacked from the 0.8.0 release | 26 to 250 ms | +| the same bytes written sequentially | 12 to 90 ms | +| sequential and already 4 GB | nothing to extend; clone 0.2 ms | + +This was the largest single cost of a base boot on the Mac, larger than +the guest kernel. With the base prepared, the guest grows its filesystem +online in about 6 ms and nothing else is paid per boot. The filesystem is +not grown on the host, since the host does not run resize tools on images. +A fixed 4096 MB image built that way would also not need preparing, but +the file a tar unpacks is laid out by the unpacker, so the host would +prepare it anyway. + +A disk never shrinks. Its size is a ceiling, not an allocation: a 16 GB +disk does not allocate all 16 GB up front. Metadata, unreclaimed extents +and filesystem allocation units add to the live file payload. Shrinking would mean an offline +resize, either on the host (parsing a guest filesystem) or in a guest +stage with the root unmounted, with a full check first and a failure when +the data does not fit, all to lower a capacity limit. What +does cost the host is space the guest freed and the host still holds, +which the next section fixes. + +### Freed space goes back to the host + +Today a file deleted in the guest keeps its blocks on the host for the +life of the disk, and in every checkpoint saved from it. A sandbox that +builds and deletes 20 GB holds 20 GB. + +- **Linux host.** The virtio-blk backend offers `VIRTIO_BLK_F_DISCARD` and + `VIRTIO_BLK_F_WRITE_ZEROES`. A discard, or a write zeroes with the unmap + flag, becomes `fallocate(FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE)` on + the disk file; a write zeroes without it becomes `FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE`. A failed host operation + fails the request; write zeroes must never acknowledge nonzero data. + Ranges and segment counts come from the guest, so each is checked + against the disk size and the advertised limits before it reaches the + host. Bad bounds fail with `VIRTIO_BLK_S_IOERR`; unsupported flag bits + use `VIRTIO_BLK_S_UNSUPP`, as required by the spec. On a + reflinked disk a punched range drops this disk reference to the shared + extents and leaves the base and other clones alone. +- **macOS host.** The framework already offers discard to the guest + (measured: `discard_max_bytes` 4 GiB); nothing in the guest used it. +- **Guest.** Batched trim, not the `discard` mount option. With `discard`, + ext4 sends one discard per freed extent when the journal commits and + holds those blocks until the discard finishes. A build that churns + small files turns into a stream of small hole punches: commit latency + in the guest, a host disk file split into many small extents (slower + I/O and slower clones afterwards), and holes punched in blocks that are + allocated again a moment later. `FITRIM` walks the free space instead + and sends large merged ranges, skips ranges below a minimum length, and + skips fully trimmed groups until newly freed blocks clear that state + or a smaller minlen requires another pass. This tracking is in memory + and resets across mounts. Use minlen 1 MiB; the kernel can raise it to + device granularity. Trim byte counts are submitted free ranges, not + proof of reclaimed host allocation. +- **When.** Two triggers, both decided by the host. Seal runs `FITRIM` on + `/` before `FIFREEZE`, including checkpoints from before this change. + Small free extents, concurrent frees after trim and storage that ignores + discard can still occupy host blocks. Unsupported trim is an explicit + nonfatal result; other ioctl failures fail seal. And a `Trim` request lets the host + ask a running sandbox to give space back: the SDK exposes it, and the + node runtime sends it under disk pressure. There is no timer in the + guest. Policy stays on the host, which is the + side that knows when space matters. +- **CAS storage.** The NBD export advertises trim only if `shuru-store` + handles `NBD_CMD_TRIM`; until it does, discards there are not offered + and the disk behaves as it does today. + +The Linux configuration layout follows virtio 1.2 section 5.2.4: capacity +at byte 0, discard limits at 36 and 40, alignment at 44, zero limits at 48 +and 52, and `write_zeroes_may_unmap` at 56. Advertise 2097152 sectors per +range, 32 ranges, discard alignment 8 sectors and zero-unmap support. Read +only disks advertise neither destructive feature. Requests 11 and 13 use +a 16-byte header, packed 16-byte little-endian ranges and a writable status +byte. The header sector is zero. Range bit 0 permits unmap only for zero; +all discard flags and other zero flag bits must be zero. + +Bound descriptor traversal by the queue size, reject cycles and indirect +chains, validate direction and guest memory spans, cap data segment sizes, +and copy all range records before validating all of them. Check counts, +capacity subtraction and signed fallocate byte offsets before any mutation. +Unsupported host filesystems may refuse fallocate. A real I/O failure can +leave earlier validated ranges completed, as the spec permits. + +The backend does not offer `VIRTIO_BLK_F_FLUSH`, as before. A write +completes once it is in the host page cache, which survives the guest and +the VMM process and is what a checkpoint clone reads; only a host crash +loses it, and a running disk does not promise to survive one (see What is +supported). Syncing to stable storage on every guest flush would put a host +`fdatasync` on every journal commit and every guest `fsync`, the hot path of +package installs and git, to buy a guarantee nothing relies on. This is +the default of Firecracker (`Unsafe` cache type) and QEMU `cache=unsafe`. +Durability starts at publication: the checkpoint writer syncs the new file +and its directory. + +### The image + +`prepare-rootfs.sh` gains `-e remount-ro` on `mkfs.ext4`, and the image is +unmounted cleanly before it is packed, so the base is sealed by +construction. The image stays 1024 MB. The rootfs no longer installs +shuru-init. + +### Network + +shuru-guest already brings up `lo` and checks for `eth0`. It gains the +rest of what the initramfs did when `eth0` exists: `SIOCSIFADDR` +10.0.0.2, `SIOCSIFNETMASK` 255.255.255.0, `SIOCADDRT` default route via +10.0.0.1, and `/etc/resolv.conf` pointing at 10.0.0.1. With no `eth0` it +skips all of it. + +## Security + +- No host filesystem parser or performance telemetry endpoint. Discard + and zero extend the existing virtio-blk request surface with bounded + standard operations. The guest reads filesystem metadata; the host + handles only block ranges and file clones. +- A guest that lies about sealing hurts only its own checkpoint, which + only its own kernel reads later. +- Network settings in the guest were never a boundary; the proxy, the + allowlist and secret substitution on the host are. +- The agent is read from the image on every boot, never from the guest + writable disk. + +## Compatibility + +- The OS image changes, so this ships as a new OS release; the CLI already + refuses an image whose VERSION does not match. +- Old checkpoints boot: the agent comes from the new initramfs, the kernel + replays their journal, and the equal-size resize ioctl is skipped, and growth happens once. +- macOS boots the same initramfs through `VZLinuxBootLoader`. + +## What will be measured + +On a1.metal and an M-series Mac, each in the evidence log: + +| case | measure | expect | +|---|---|---| +| `shuru run -- true` | wall time | targets: a1.metal under 0.40 s; Mac under 0.20 s | +| 100 Hz against 1000 Hz on a1.metal | guest uptime at `Run /init` | the virtio-mmio effect of the tick | +| same | guest uptime at `Run /init`, root mounted, first exec | `/init` under 50 ms | +| VM created to guest kernel start | host side timestamps | the 210 ms split into kernel load, memory setup, disk clone, connect | +| boot that grows 4 GB to 8 GB, and to 64 GB | time in stage one | paid once per size change | +| seal during a sustained write (`fio`) | seal time; the clone booted | superblock clean, no journal replay, file contents match up to the seal | +| checkpoint at stop and of a running sandbox | both paths through `save_disk` | the same clean result on both hosts | +| guest hung before the seal | `checkpoint update` | error, old checkpoint bytes unchanged | +| write 2 GB, delete it, `Trim` | host allocated blocks of the disk file, before and after | substantial reclamation of large free extents, both hosts | +| a small file churning build (`npm install`, a kernel build) with `discard` mounted, against `Trim` after it | guest wall time, host extent count of the disk file, hole punch calls | confirms batched trim over the mount option | +| `FITRIM` in a seal, fresh disk and a pre-change checkpoint | seal time | | +| discard ranges outside the disk, too many segments | request status | `IOERR`, file untouched | +| a pre-change checkpoint | boot | kernel replays the journal; next save is clean | +| seal with no thaw before the deadline | guest state, next `Thaw` | guest writable again, `Expired`, no checkpoint | +| disk with the errors flag set | boot, seal | both refused with the message | +| first boot after install, and later boots, Mac and a1.metal | `prepare_base` time, clone time | a few seconds once, then under 1 ms | + +## Alternatives considered + +- **Read-only remount instead of freeze.** Fails with `EBUSY` on any open + writer, so it needs every process killed first and cannot serve a + running sandbox. +- **Keep e2fsck, skip only resize.** Every checkpoint boot pays a check of + a filesystem the journal already made consistent. +- **No initramfs, boot the disk.** Checkpoints would keep the agent they + were saved with and fail the hello after an upgrade. +- **resize2fs on the host when a disk is cloned.** The host would parse + filesystems that guests wrote. +- **Kernel `ip=`.** Needs `CONFIG_IP_PNP`; the host must leave it off when + no network device is attached or the kernel waits for one, and boot can + wait on link state each VMM reports differently. Both failures are + stalls that show only in dmesg. +- **Snapshots to skip the boot.** RFC 0009. With a cold boot near 0.3 s a + bare snapshot has little to skip; templates of warm state are what is + left for it. + +## Source checks + +The review used Linux v6.12 sources. The build script pins 6.12.114; +repeat runtime checks on that exact built kernel before accepting timing +or compatibility results. The stable tag was not retrievable in this review. + +- `fs/super.c`, `freeze_super` and `thaw_super_locked`: freeze ordering and + the inability to cancel an ioctl by expiring a userspace deadline. +- `fs/ext4/super.c`, `ext4_freeze`, `ext4_unfreeze`, `ext4_handle_error` + and `ext4_clear_journal_err`: recovery flag and filesystem errors. +- `fs/ext4/ioctl.c`, `__ext4_ioctl`: resize journal flush and trim support; + `fs/ext4/resize.c`, `ext4_resize_begin`, `ext4_resize_fs` and + `setup_new_flex_group_blocks`: limits and metadata work. +- `fs/ext4/mballoc.c`, `ext4_trim_fs`, `ext4_trim_all_free` and + `ext4_try_to_trim_range`: minlen and per-mount trimmed group state. +- `fs/ext4/sysfs.c`, `errors_count`; `fs/ext4/ext4.h`, `ext4_super_block`: + error count versus state flag and fixed superblock fields. +- `fs/overlayfs/util.c`, `ovl_start_write`; `fs/overlayfs/file.c`, + `ovl_write_iter`: upper filesystem writes participate in freeze. +- BusyBox `util-linux/switch_root.c`, `switch_root_main`: old-root cleanup + precedes MS_MOVE and chroot. The static agent omits the final exec. +- [Virtio 1.2 sections 5.2.3 to 5.2.6.2](https://docs.oasis-open.org/virtio/virtio/v1.2/virtio-v1.2.html): + block features, configuration layout, ranges, statuses and flush. + +Source files are available under +[Linux v6.12](https://github.com/torvalds/linux/tree/v6.12) and +[BusyBox switch_root](https://github.com/mirror/busybox/blob/master/util-linux/switch_root.c). diff --git a/docs/rfcs/0011-host-memory-bounds.md b/docs/rfcs/0011-host-memory-bounds.md new file mode 100644 index 0000000..3d5d938 --- /dev/null +++ b/docs/rfcs/0011-host-memory-bounds.md @@ -0,0 +1,207 @@ +# RFC 0011: Host memory bounds + +- Status: Implemented +- Date: 2026-10-28 +- Scope: shuru-proxy, shuru-linux, shuru-vm, shuru-cli, shuru-sdk +- Depends on: nothing +- Bears on: hosted Shuru, RFC 0008 (node runtime) + +## Summary + +`--memory` sets how much RAM the guest gets. It says nothing about the host +processes that serve the guest: the VMM device code, the network proxy, the +CLI stdio server and the SDK. Each of them copies guest data into its own +heap, and before this change several of them kept copying for as long as +the guest kept producing. A guest that wrote to a socket nobody read, or +printed to a stdout nobody consumed, grew the host process by gigabytes in +seconds. + +This RFC states the rule those processes follow from now on, records what +was changed to meet it, lists the ceilings that remain per VM, and sets out +the cgroup envelope that hosted nodes put around each sandbox. + +## The rule + +What a guest does never changes how much host memory serves it. Three +mechanisms keep that true: + +1. **Streams apply backpressure.** Data the consumer has not taken stays in + the guest. The host holds a fixed window per stream and stops reading + from the guest when the window is full; the guest then blocks on its own + socket, pipe or vsock buffer, inside its own RAM. +2. **Buffered calls have a size limit.** A call that returns everything at + once (exec output, file read) fails past a fixed size and names the + streaming alternative, as Node does with `maxBuffer`. +3. **Tables have a count limit.** Anything the guest can make the host + allocate per item (sockets, connections, open files, inodes, DNS queries + in flight) has a cap, and hitting it fails that item, not the process. + +A fourth rule sits next to these: guest input never panics a host process. +Release builds use `panic = "abort"`, so a guest-triggered panic in the VMM +takes the whole sandbox down with it. + +## What changed + +| Path | Before | After | +|---|---|---| +| Proxy, guest to upstream | every read from smoltcp queued for the engine | 128 KiB in flight per connection; the stack reads from the guest only as the engine consumes | +| Proxy, upstream to guest | every upstream read queued for the stack | 128 KiB of permits per connection; the upstream reader waits for the guest | +| Proxy, closed sockets | removed only on the guest side close path | host closes drain with a 60 s timeout, then are removed; unanswered listeners go after 5 s | +| Proxy, stale commands | a reused socket handle could receive the data of a dead connection | connection ids carry a serial; stale commands are dropped | +| Proxy, DNS | a task per query, unbounded | 64 queries in flight, the rest dropped | +| Proxy, chunked bodies | a chunk size line could grow without end | lines past 4 KiB switch the connection to a plain tunnel | +| CLI stdio, spawn and watch events | unbounded channel to the writer thread | 64 events; a client that stops reading stalls the guest | +| CLI stdio and Rust SDK, exec | all output held | 64 MiB per stream, then the call fails | +| CLI stdio and Rust SDK, read_file | the whole file held | 64 MiB, then the call fails | +| Rust SDK, shell and watch events | unbounded channels | 64 events, reader threads block | +| Linux VMM, virtqueues | queue size, ring addresses and chains taken from the guest as given | size within the device maximum, rings inside guest memory, avail distance at most the size, chains checked for loops, range and flags | +| Linux VMM, net TX | a frame buffer sized by the guest chain | one fixed buffer of the largest valid frame; longer chains are dropped | +| Linux VMM, balloon | inflate length from the guest | 256 page numbers per descriptor | +| Linux VMM, virtio-fs | reads and writes sized by the guest, unbounded handle and inode tables, names read until a NUL | 128 KiB per read or write, 512 open files, 262144 inodes, names stop at PATH_MAX | + +The proxy also fixes a race found while testing the above: smoltcp could +read a guest SYN straight from the socketpair while it polled, skipping the +pass that adds a listener for it, and answered with a reset. A loop of short +connections failed somewhere between 150 and 475 connections in. smoltcp +now only sees frames that were inspected. + +## Measured + +macOS, release build, `footprint` physical memory of the `shuru` process. +The old build is the commit before these changes. + +| Scenario | Old build | New build | +|---|---|---| +| Host floods a socket the guest never reads | 1.3 GB after 3 s, still growing | 6 MB flat | +| Guest floods a socket the host never reads | not run | 8 MB flat | +| 600 short connections in a row | refused after 150 to 475 | 600 of 600, four runs | +| Guest `cat /dev/zero` over stdio spawn, client never reads | 2 GB after 5 s | 5 MB flat | +| stdio exec printing 2 GiB | holds it all | fails at 64 MiB with an error, server keeps serving | +| stdio read_file of a 1 GiB file | holds it all | fails at 64 MiB with an error | + +Linux on an a1.metal (Graviton, KVM), release build, new build only. +Overhead is the resident memory of the `shuru` process outside the guest +RAM mapping, read from `/proc/PID/smaps`, peak over the run. + +| Scenario | Host overhead | +|---|---| +| Idle guest | 6 MB | +| Host floods a socket the guest never reads | 7 MB | +| Guest floods a socket the host never reads | 7 MB | +| 600 short connections in a row | 8 to 11 MB, 600 of 600 | +| 400 connections the host floods, guest never reads | 199 MB | +| Guest `cat /dev/zero` over stdio spawn, client never reads | 6 MB | +| stdio exec printing 2 GiB, then read_file of 1 GiB | 33 to 45 MB, both fail at 64 MiB, server keeps serving | +| Kill a process that stopped reading 8 MiB of stdin | 11 MB, gone after 2.1 s | +| Client hangs up on a running exec | 30 to 38 MB, nothing left running | + +The 400 connection row is the cost of a stalled connection at the current +sizes: about 0.5 MB each, the smoltcp send buffer and the proxy window full +toward a guest that does not read. + +## Ceilings that remain per VM + +These are fixed, independent of guest behaviour, and some are large. + +| Item | Ceiling | +|---|---| +| smoltcp sockets | 512 sockets of 512 KiB receive plus 256 KiB send, 384 MiB if every one is full | +| Proxy windows | 128 KiB each way per connection, 128 MiB across 512 sockets | +| Linux vsock | 1024 connections of 256 KiB, 256 MiB | +| virtio-fs | 512 open files, 262144 inodes | +| Buffered calls | 64 MiB per stream per call; the stdio server runs one at a time, SDK callers choose their own concurrency | + +The smoltcp and vsock numbers are worst cases that need hundreds of +connections each holding full windows. They are fine on a laptop. On a +hosted node they set the per-sandbox overhead budget, so the node runtime +lowers them (see below). + +The smoltcp buffers were halved from 1 MiB and 512 KiB. They span only the +hop between guest and host, under a millisecond; the remote round trip is +the host kernel socket to the upstream. On an M-series Mac a single stream +measured 1.8 to 2.0 Gbit/s down and 8.8 up at the new sizes against 2.0 to +2.1 and 7 to 9 before; halving again to 256 KiB and 128 KiB lost a fifth. + +## Stdin and kill + +Stdin for a spawned process used to be fire and forget. When the process +stopped reading, stdin queued in the stdio server without limit, and the +KILL frame that shares the exec connection sat behind it: the guest agent +was blocked writing to the full stdin pipe and never read the next frame. + +- `input` is now a request, answered once the data is written to the exec + connection. The TypeScript SDK keeps up to 1 MiB in flight per process + and queues the rest in the calling process, the way a Node stream + buffers. The stdio server refuses more than 4 MiB of unanswered stdin + per process. Writes are split into frames under the 1 MiB frame limit. +- A kill sends SIGTERM in band as before. If the process has not exited + two seconds later, the host sends a `KILL_REQ` naming the exec id on a + connection of its own, and the guest SIGKILLs the process group. The + guest forgets an id after the child exits but before it is reaped, so a + late kill cannot reach a reused pid. +- The guest also SIGKILLs the process group when the host hangs up on a + piped exec, so a client that dies leaves nothing running. On macOS this + does not fire while host data is still queued for the guest, which is why + the kill needs its own connection. The watch stops through a pipe once + the child exits and before it is reaped, so it neither delays the exit + frame nor reaches a reused pid. + +## Hosted: the cgroup envelope + +Bounded buffers make the host overhead of a sandbox small and predictable. +They do not make it zero, and a bug in any one of them would reopen the +problem. Hosted nodes therefore put each sandbox in its own cgroup v2, +the same way the Firecracker jailer does: + +- One cgroup per sandbox holds the VMM process, its device threads, the + proxy and anything else spawned for that sandbox. +- `memory.max` is the guest RAM plus a fixed host allowance. Guest RAM is + anonymous memory mapped by the VMM, so it is charged to the same cgroup, + and pages the balloon returns (RFC 0006) are uncharged. +- `memory.swap.max` is 0, so a sandbox cannot spill into node swap. +- `memory.high` sits a little under `memory.max`, so a sandbox near its + limit is throttled and reclaimed before the OOM killer runs. +- `pids.max` caps threads, since device threads and per-connection tasks + are the other thing the guest can make the host create. +- When the OOM killer does run, it kills that sandbox and nothing else. The + node runtime reports it as an OOM exit, not a crash. + +This was run on an a1.metal with a systemd scope standing in for the node +runtime: `MemoryMax=1280M`, `MemoryHigh=1216M`, `MemorySwapMax=0`, guest +RAM 1024 MiB, so a 256 MiB allowance. The guest touched 900 MiB of its RAM, +then opened 400 connections the host flooded and never read them. Over +two runs the cgroup peaked at 1122 MB and 1239 MB. The second run crossed +`memory.high` and was throttled 790 times; neither run saw a `max` or +`oom` event, and both sandboxes exited normally. Guest RAM is charged to +the scope, as expected, so 256 MiB of allowance is already tight for a +sandbox that fills its RAM and stalls 400 connections. + +The host allowance is sized from the ceilings above, and at the current sizes +the sum is too large to grant: 384 MiB of smoltcp buffers, 128 MiB of proxy +windows and 256 MiB of vsock buffers. The measured idle overhead is 6 MB +and a busy sandbox stays near 10 MB; only stalled connections cost real +memory, about 0.5 MB each. Hosted builds therefore lower the counts rather +than the buffer sizes, which would cost throughput: + +| Item | Hosted limit | Worst case | +|---|---|---| +| smoltcp sockets | 256 | 192 MiB buffers, 64 MiB windows | +| Linux vsock connections | 128 | 32 MiB | + +That gives an allowance of 320 MiB per sandbox: 288 MiB of ceilings plus +32 MiB for code, stacks and the TLS state of the proxy. A guest that goes +past it is throttled at `memory.high` and then killed alone, which is the +outcome the envelope exists for. RFC 0008 carries the final numbers. + +macOS has no cgroups. Local sandboxes rely on the bounds alone, which is +the right trade for a laptop where the user owns both sides. + +## Testing + +- Proxy unit tests cover receive only yielding inspected frames. +- Linux unit tests cover rejected queues, chain loops, out of range + descriptors, foreign flags, oversized net TX chains, virtio-fs name and + handle caps. +- The pressure scenarios above run against a real VM with servers on the + host reached through `--expose-host`. +- The TS SDK integration suite and the CLI real VM suite pass unchanged. diff --git a/examples/browser-use.py b/examples/browser-use.py index 6a3c950..b27c2ef 100644 --- a/examples/browser-use.py +++ b/examples/browser-use.py @@ -1,4 +1,8 @@ -"""Run a Browser Use agent against Chromium isolated inside Shuru.""" +"""Run a Browser Use agent against Chromium isolated inside Shuru. + +The browser recipe runs headless Chromium as a guest service, so the VM is +ready when the command starts and the only thing to do is keep it alive. +""" import asyncio import os @@ -19,7 +23,9 @@ def free_port() -> int: def wait_for_cdp(url: str, process: subprocess.Popen[bytes]) -> None: - deadline = time.monotonic() + 15 + # The first run builds the browser checkpoint from the recipe, which + # takes a few minutes; later runs answer within seconds. + deadline = time.monotonic() + 600 while time.monotonic() < deadline: if process.poll() is not None: raise RuntimeError(f"Shuru exited with code {process.returncode}") @@ -58,22 +64,16 @@ def main() -> None: [ "shuru", "run", - "--from", - os.getenv("SHURU_BROWSER_CHECKPOINT", "browser-use"), + "--recipe", + os.getenv("SHURU_BROWSER_RECIPE", "browser"), "--allow-net", "--port", f"{host_port}:9222", "--memory", "4096", "--", - "/usr/bin/chromium", - "--headless", - "--no-sandbox", - "--disable-dev-shm-usage", - "--remote-debugging-address=0.0.0.0", - "--remote-debugging-port=9222", - "--user-data-dir=/tmp/browser-use-profile", - "about:blank", + "sleep", + "infinity", ] ) diff --git a/justfile b/justfile index 099f626..0c4d937 100644 --- a/justfile +++ b/justfile @@ -35,6 +35,12 @@ shell: # Full setup from scratch: rootfs + build setup: prepare-rootfs build +# Build shuru-node, shuru-vmm and shuru-netd as static PIE, on aarch64 Linux +build-node: + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER={{ justfile_directory() }}/scripts/static-pie-cc \ + RUSTFLAGS="-C target-feature=+crt-static -C relocation-model=pie" \ + cargo build --release -p shuru-node --target aarch64-unknown-linux-gnu + # Check all crates compile (host targets only) check: cargo check --workspace diff --git a/kernel/shuru_defconfig b/kernel/shuru_defconfig index 5c0da8e..022cd74 100644 --- a/kernel/shuru_defconfig +++ b/kernel/shuru_defconfig @@ -18,7 +18,7 @@ CONFIG_POSIX_MQUEUE=y CONFIG_NO_HZ_IDLE=y CONFIG_HIGH_RES_TIMERS=y CONFIG_PREEMPT_VOLUNTARY=y -CONFIG_HZ_100=y +CONFIG_HZ_1000=y CONFIG_SYSCTL=y CONFIG_MULTIUSER=y CONFIG_FHANDLE=y @@ -127,6 +127,9 @@ CONFIG_TTY=y CONFIG_VT=y CONFIG_SERIAL_EARLYCON=y CONFIG_HVC_DRIVER=y +# The Linux backend emulates a PL011 and boots with console=ttyAMA0 +CONFIG_SERIAL_AMBA_PL011=y +CONFIG_SERIAL_AMBA_PL011_CONSOLE=y CONFIG_PRINTK=y # PTY support (for shell/TTY exec) @@ -190,3 +193,51 @@ CONFIG_RTC_DRV_PL031=y # CONFIG_MEDIA_SUPPORT is not set # CONFIG_POWER_SUPPLY is not set # CONFIG_MD is not set + +# Container runtime support (Docker, containerd, runc on cgroup v2) +# cgroup v2 controllers runc expects: cpu (with bandwidth), cpuset, io, memory, pids +CONFIG_CPUSETS=y +CONFIG_FAIR_GROUP_SCHED=y +CONFIG_CFS_BANDWIDTH=y +CONFIG_BLK_CGROUP=y +CONFIG_BLK_DEV_THROTTLING=y +# runc joins a session keyring per container +CONFIG_KEYS=y +# cgroup v2 device access control is an eBPF program attached to the cgroup. +# Unprivileged bpf() stays off so containers cannot reach the verifier. +CONFIG_BPF=y +CONFIG_BPF_SYSCALL=y +CONFIG_CGROUP_BPF=y +CONFIG_BPF_UNPRIV_DEFAULT_OFF=y +# Default Docker seccomp profile +CONFIG_SECCOMP=y +CONFIG_SECCOMP_FILTER=y +# Bridge networking: veth pairs on docker0, bridge traffic through iptables +CONFIG_VETH=y +CONFIG_BRIDGE_NETFILTER=y +# iptables-nft rules installed by dockerd (xt matches run through nft compat) +CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=y +CONFIG_NETFILTER_XT_MATCH_COMMENT=y +CONFIG_NETFILTER_XT_MARK=y +CONFIG_NETFILTER_XT_NAT=y +CONFIG_NETFILTER_XT_TARGET_MASQUERADE=y +CONFIG_NFT_COMPAT=y +CONFIG_NFT_MASQ=y +CONFIG_NFT_FIB_IPV4=y +CONFIG_NFT_FIB_IPV6=y +CONFIG_NFT_REJECT=y +CONFIG_IP_NF_MANGLE=y +CONFIG_IP_NF_RAW=y +CONFIG_IP_NF_TARGET_REJECT=y +CONFIG_IP6_NF_IPTABLES=y +CONFIG_IP6_NF_FILTER=y +CONFIG_IP6_NF_MANGLE=y +CONFIG_IP6_NF_NAT=y +CONFIG_IP6_NF_RAW=y +CONFIG_IP6_NF_TARGET_MASQUERADE=y +CONFIG_IP6_NF_TARGET_REJECT=y +# overlay2 stores security xattrs on the backing ext4 +CONFIG_EXT4_FS_SECURITY=y +# Expose the running config so moby check-config.sh can audit the guest +CONFIG_IKCONFIG=y +CONFIG_IKCONFIG_PROC=y diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 92b59a6..4e71967 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -77,7 +77,7 @@ const sb = await Sandbox.start({ | `from` | `string` | Checkpoint name to start from | | `cpus` | `number` | Number of vCPUs | | `memory` | `number` | Memory in MB | -| `diskSize` | `number` | Disk size in MB | +| `diskSize` | `number` | Disk size in MB, minimum and default 4096 | | `allowNet` | `boolean` | Enable network access | | `ports` | `string[]` | Port forwards (`"host:guest"`) | | `mounts` | `Record` | Directory mounts (`{ hostPath: guestPath }`) | @@ -106,7 +106,7 @@ proc.on("stdout", (data: Buffer) => { /* real-time chunks */ }); proc.on("stderr", (data: Buffer) => { /* real-time chunks */ }); proc.on("exit", (code: number) => { /* process exited */ }); -proc.write("input to stdin\n"); // write to stdin +await proc.write("input to stdin\n"); // resolves once it reaches the VM await proc.kill(); // send SIGTERM const exitCode = await proc.exited; // await completion console.log(proc.pid); // process ID diff --git a/packages/sdk/package.json b/packages/sdk/package.json index f3f800a..53a02aa 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@superhq/shuru", - "version": "0.4.2", + "version": "0.5.0", "type": "module", "main": "src/index.ts", "types": "src/index.ts", diff --git a/packages/sdk/src/process-handle.ts b/packages/sdk/src/process-handle.ts index 85e4c10..daf0921 100644 --- a/packages/sdk/src/process-handle.ts +++ b/packages/sdk/src/process-handle.ts @@ -1,5 +1,8 @@ import type { ShuruProcess } from "./process"; +const STDIN_CHUNK = 256 * 1024; +const STDIN_WINDOW = 1024 * 1024; + type EventMap = { stdout: (data: Buffer) => void; stderr: (data: Buffer) => void; @@ -15,6 +18,12 @@ export class SandboxProcess { stderr: ((data: Buffer) => void)[]; exit: ((code: number) => void)[]; } = { stdout: [], stderr: [], exit: [] }; + private stdinQueue: { + chunk: Buffer; + resolve: () => void; + reject: (err: Error) => void; + }[] = []; + private stdinInFlight = 0; constructor(proc: ShuruProcess, pid: string) { this.proc = proc; @@ -41,11 +50,51 @@ export class SandboxProcess { return this; } - write(data: Buffer | string): void { - this.proc.sendNotification("input", { - pid: this.pid, - data: Buffer.from(data).toString("base64"), - }); + /** + * Write to the process stdin. Resolves once the data reaches the VM. + * Writes stay in order; past STDIN_WINDOW in flight they wait here, in + * this process, instead of piling up in the sandbox host. + */ + write(data: Buffer | string): Promise { + const buf = Buffer.from(data); + const parts: Promise[] = []; + for (let at = 0; at < buf.length; at += STDIN_CHUNK) { + const chunk = buf.subarray(at, at + STDIN_CHUNK); + parts.push( + new Promise((resolve, reject) => { + this.stdinQueue.push({ chunk, resolve, reject }); + }), + ); + } + this.pumpStdin(); + const done = Promise.all(parts).then(() => undefined); + // A caller that does not await should not get an unhandled rejection. + done.catch(() => {}); + return done; + } + + private pumpStdin(): void { + while (this.stdinQueue.length > 0) { + const next = this.stdinQueue[0]; + if ( + this.stdinInFlight > 0 && + this.stdinInFlight + next.chunk.length > STDIN_WINDOW + ) { + return; + } + this.stdinQueue.shift(); + this.stdinInFlight += next.chunk.length; + this.proc + .send("input", { + pid: this.pid, + data: next.chunk.toString("base64"), + }) + .then(() => next.resolve(), next.reject) + .finally(() => { + this.stdinInFlight -= next.chunk.length; + this.pumpStdin(); + }); + } } async kill(): Promise { diff --git a/packages/sdk/src/process.ts b/packages/sdk/src/process.ts index 0ca367b..cb622b5 100644 --- a/packages/sdk/src/process.ts +++ b/packages/sdk/src/process.ts @@ -111,20 +111,24 @@ export class ShuruProcess { const reader = this.proc.stdout.getReader(); const decoder = new TextDecoder(); - let remainder = ""; + // A file read arrives as one line of tens of megabytes, so only the + // newest chunk is searched for the newline. + let parts: string[] = []; try { while (true) { const { done, value } = await reader.read(); if (done) break; - remainder += decoder.decode(value, { stream: true }); - - while (true) { - const newlineIdx = remainder.indexOf("\n"); - if (newlineIdx === -1) break; - const line = remainder.slice(0, newlineIdx); - remainder = remainder.slice(newlineIdx + 1); + const text = decoder.decode(value, { stream: true }); + let start = 0; + let newlineIdx = text.indexOf("\n"); + while (newlineIdx !== -1) { + parts.push(text.slice(start, newlineIdx)); + const line = parts.join(""); + parts = []; + start = newlineIdx + 1; + newlineIdx = text.indexOf("\n", start); if (!line) continue; @@ -135,6 +139,7 @@ export class ShuruProcess { // malformed line } } + if (start < text.length) parts.push(text.slice(start)); } } catch { // stream closed diff --git a/packages/sdk/src/sandbox.ts b/packages/sdk/src/sandbox.ts index 2f8bc0e..a1db90b 100644 --- a/packages/sdk/src/sandbox.ts +++ b/packages/sdk/src/sandbox.ts @@ -20,6 +20,7 @@ const Method = { READ_FILE: "read_file", WRITE_FILE: "write_file", CHECKPOINT: "checkpoint", + TRIM: "trim", WATCH: "watch", MKDIR: "mkdir", READ_DIR: "read_dir", @@ -177,6 +178,11 @@ export class Sandbox { } } + async trim(): Promise<{ supported: boolean; bytes: number }> { + const resp = await this.proc.send(Method.TRIM, {}); + return resp.result as { supported: boolean; bytes: number }; + } + async checkpoint(name: string): Promise { await this.proc.send(Method.CHECKPOINT, { name }); this.stopped = true; @@ -197,7 +203,12 @@ export function buildArgs(bin: string, opts: StartOptions): string[] { if (opts.from) args.push("--from", opts.from); if (opts.cpus) args.push("--cpus", String(opts.cpus)); if (opts.memory) args.push("--memory", String(opts.memory)); - if (opts.diskSize) args.push("--disk-size", String(opts.diskSize)); + if (opts.diskSize !== undefined) { + if (!Number.isSafeInteger(opts.diskSize) || opts.diskSize < 4096) { + throw new Error("disk size must be an integer of at least 4096 MB"); + } + args.push("--disk-size", String(opts.diskSize)); + } if (opts.allowNet) args.push("--allow-net"); if (opts.allowHostWrites) args.push("--allow-host-writes"); diff --git a/packages/sdk/src/types.ts b/packages/sdk/src/types.ts index f77c60b..2625a39 100644 --- a/packages/sdk/src/types.ts +++ b/packages/sdk/src/types.ts @@ -14,6 +14,7 @@ export interface StartOptions { from?: string; cpus?: number; memory?: number; + /** Disk size in MB. Minimum and default: 4096. */ diskSize?: number; allowNet?: boolean; /** Allow :rw mounts to write to host filesystem. Default: false. */ diff --git a/packages/sdk/test/integration/integration.test.ts b/packages/sdk/test/integration/integration.test.ts index 515e7c3..48a2379 100644 --- a/packages/sdk/test/integration/integration.test.ts +++ b/packages/sdk/test/integration/integration.test.ts @@ -55,6 +55,49 @@ describe.if(canRun)("sandbox lifecycle", () => { expect(new TextDecoder().decode(read)).toBe(content); }, 30_000); + test("files larger than one frame roundtrip", async () => { + const content = new Uint8Array(3 * 1024 * 1024 + 7); + for (let i = 0; i < content.length; i++) content[i] = (i * 31) & 0xff; + await sb.writeFile("/tmp/sdk-big.bin", content); + const size = await sb.exec("stat -c %s /tmp/sdk-big.bin"); + expect(size.stdout.trim()).toBe(String(content.length)); + const read = await sb.readFile("/tmp/sdk-big.bin"); + expect(Buffer.from(read).equals(Buffer.from(content))).toBe(true); + }, 30_000); + + test("failed write reports the guest error", async () => { + await expect( + sb.writeFile("/proc/nope/file", new Uint8Array(2 * 1024 * 1024)), + ).rejects.toThrow("No such file or directory"); + const result = await sb.exec("echo alive"); + expect(result.stdout.trim()).toBe("alive"); + }, 30_000); + + test("stdin larger than a frame reaches a spawned process", async () => { + const size = 3 * 1024 * 1024 + 5; + const proc = await sb.spawn(`head -c ${size} | wc -c`); + let out = ""; + proc.on("stdout", (d) => { + out += d.toString(); + }); + await proc.write(Buffer.alloc(size, 0x61)); + expect(await proc.exited).toBe(0); + expect(out.trim()).toBe(String(size)); + }, 30_000); + + test("kill reaches a process that stopped reading stdin", async () => { + const proc = await sb.spawn("sleep 600"); + const writes = proc.write(Buffer.alloc(8 * 1024 * 1024)); + await new Promise((r) => setTimeout(r, 1000)); + const start = Date.now(); + await proc.kill(); + await proc.exited; + expect(Date.now() - start).toBeLessThan(10_000); + await expect(writes).rejects.toThrow(); + const result = await sb.exec("pgrep -x sleep || echo none"); + expect(result.stdout.trim()).toBe("none"); + }, 30_000); + test("multiple sequential execs", async () => { await sb.exec("echo a > /tmp/seq.txt"); await sb.exec("echo b >> /tmp/seq.txt"); @@ -63,10 +106,10 @@ describe.if(canRun)("sandbox lifecycle", () => { }, 30_000); test("checkpoint creates disk snapshot", async () => { - const cpPath = resolve( - process.env.HOME ?? "/tmp", - ".local/share/shuru/checkpoints/sdk-integration-test.ext4", - ); + const dataDir = + process.env.SHURU_DATA_DIR ?? + resolve(process.env.HOME ?? "/tmp", ".local/share/shuru"); + const cpPath = resolve(dataDir, "checkpoints/sdk-integration-test.ext4"); try { unlinkSync(cpPath); diff --git a/packages/sdk/test/unit/filesystem.test.ts b/packages/sdk/test/unit/filesystem.test.ts index e25d6fd..a1aa6e4 100644 --- a/packages/sdk/test/unit/filesystem.test.ts +++ b/packages/sdk/test/unit/filesystem.test.ts @@ -183,3 +183,13 @@ describe("exists", () => { expect(result).toBe(false); }); }); + +test("trim reports support and leaves the sandbox running", async () => { + const sb = await Sandbox.start({ shuruBin: MOCK_BIN }); + try { + expect(await sb.trim()).toEqual({ supported: true, bytes: 1048576 }); + await sb.mkdir("/tmp/after-trim"); + } finally { + await sb.stop(); + } +}); diff --git a/packages/sdk/test/unit/mock-shuru.ts b/packages/sdk/test/unit/mock-shuru.ts index 7a2337e..03d21ab 100644 --- a/packages/sdk/test/unit/mock-shuru.ts +++ b/packages/sdk/test/unit/mock-shuru.ts @@ -185,7 +185,10 @@ async function handleMessage(msg: { } case "input": { - // In mock, we just ignore input data + // In mock, the data is dropped; requests still get an answer + if (msg.id !== undefined) { + writeLine({ jsonrpc: "2.0", id: msg.id, result: {} }); + } break; } @@ -311,6 +314,10 @@ async function handleMessage(msg: { break; } + case "trim": { + writeLine({ jsonrpc: "2.0", id: msg.id, result: { supported: true, bytes: 1048576 } }); + break; + } case "checkpoint": { writeLine({ jsonrpc: "2.0", id: msg.id, result: {} }); process.exit(0); diff --git a/packages/sdk/test/unit/unit.test.ts b/packages/sdk/test/unit/unit.test.ts index c1476c4..1325c28 100644 --- a/packages/sdk/test/unit/unit.test.ts +++ b/packages/sdk/test/unit/unit.test.ts @@ -41,6 +41,12 @@ describe("buildArgs", () => { ]); }); + test("invalid disk sizes", () => { + for (const diskSize of [0, 1024, 4095, 4096.5, Number.NaN, Number.MAX_SAFE_INTEGER + 1]) { + expect(() => buildArgs("shuru", { diskSize })).toThrow(); + } + }); + test("disk size", () => { const args = buildArgs("shuru", { diskSize: 8192 }); expect(args).toEqual(["shuru", "run", "--stdio", "--disk-size", "8192"]); diff --git a/scripts/build-kernel-hf.sh b/scripts/build-kernel-hf.sh new file mode 100755 index 0000000..3b24990 --- /dev/null +++ b/scripts/build-kernel-hf.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# Build the Shuru guest kernel on a Hugging Face Job instead of local Docker. +# +# Same defconfig and kernel version as build-kernel.sh, cross-compiled on an +# x86_64 CPU job. The job uploads the Image and the resolved .config to a +# private dataset repo, and this script downloads them into the data dir. +# Nothing is written to the data dir until the download at the end, so the +# installed kernel is untouched until then. Point SHURU_DATA_DIR somewhere +# else to keep the result fully isolated. +# +# Requires: hf CLI logged in (hf auth whoami), a billable HF account. +set -euo pipefail + +KERNEL_VERSION="${KERNEL_VERSION:-6.12.114}" +KERNEL_MAJOR="${KERNEL_VERSION%%.*}" +DATA_DIR="${SHURU_DATA_DIR:-${HOME}/.local/share/shuru}" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_DIR="$(dirname "$SCRIPT_DIR")" +DEFCONFIG="${REPO_DIR}/kernel/shuru_defconfig" +HF_FLAVOR="${HF_FLAVOR:-cpu-upgrade}" +HF_USER="$(hf auth whoami --format json | python3 -I -c 'import json,sys; print(json.load(sys.stdin)["user"])')" +HF_REPO="${HF_REPO:-${HF_USER}/shuru-kernel-build}" +JOB_NAME="shuru-kernel-${KERNEL_VERSION//./-}" + +if [ ! -f "$DEFCONFIG" ]; then + echo "ERROR: Defconfig not found at ${DEFCONFIG}" + exit 1 +fi + +export HF_TOKEN="${HF_TOKEN:-$(hf auth token)}" +DEFCONFIG_B64="$(base64 < "$DEFCONFIG" | tr -d '\n')" + +echo "==> Building kernel ${KERNEL_VERSION} on Hugging Face Jobs (${HF_FLAVOR})" +echo " artifacts: https://huggingface.co/datasets/${HF_REPO} (private)" + +hf repos create "$HF_REPO" --type dataset --private --exist-ok > /dev/null + +read -r -d '' JOB_SCRIPT << 'JOBEOF' || true +set -euxo pipefail +export DEBIAN_FRONTEND=noninteractive +apt-get update -qq > /dev/null +apt-get install -y -qq build-essential bc flex bison libelf-dev libssl-dev \ + gcc-aarch64-linux-gnu curl xz-utils ca-certificates python3 python3-venv > /dev/null +curl -LsSf https://hf.co/cli/install.sh | bash > /dev/null +export PATH="$HOME/.local/bin:$PATH" + +cd /tmp +curl -sSLO "https://cdn.kernel.org/pub/linux/kernel/v${KERNEL_MAJOR}.x/linux-${KERNEL_VERSION}.tar.xz" +curl -sSL "https://cdn.kernel.org/pub/linux/kernel/v${KERNEL_MAJOR}.x/sha256sums.asc" \ + | grep " linux-${KERNEL_VERSION}.tar.xz$" | sha256sum -c - +tar xf "linux-${KERNEL_VERSION}.tar.xz" +cd "linux-${KERNEL_VERSION}" + +echo "$DEFCONFIG_B64" | base64 -d > arch/arm64/configs/shuru_defconfig +make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- shuru_defconfig +make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- -j"$(nproc)" Image 2>&1 | tail -3 + +cp arch/arm64/boot/Image "/tmp/Image-${KERNEL_VERSION}" +cp .config "/tmp/config-${KERNEL_VERSION}" +(cd /tmp && sha256sum "Image-${KERNEL_VERSION}" | tee "Image-${KERNEL_VERSION}.sha256") +hf upload "$HF_REPO" "/tmp/Image-${KERNEL_VERSION}" "Image-${KERNEL_VERSION}" --type dataset > /dev/null +hf upload "$HF_REPO" "/tmp/Image-${KERNEL_VERSION}.sha256" "Image-${KERNEL_VERSION}.sha256" --type dataset > /dev/null +hf upload "$HF_REPO" "/tmp/config-${KERNEL_VERSION}" "config-${KERNEL_VERSION}" --type dataset > /dev/null +echo "KERNEL_BUILD_DONE" +JOBEOF + +JOB_ID="$(hf jobs run debian:trixie-slim bash -c "$JOB_SCRIPT" \ + --ssh --detach --flavor "$HF_FLAVOR" --timeout 2h \ + --name "$JOB_NAME" \ + --secrets HF_TOKEN \ + --env "KERNEL_VERSION=${KERNEL_VERSION}" \ + --env "KERNEL_MAJOR=${KERNEL_MAJOR}" \ + --env "HF_REPO=${HF_REPO}" \ + --env "DEFCONFIG_B64=${DEFCONFIG_B64}" \ + --format quiet)" +echo "==> Job ${JOB_ID} launched" + +echo "==> SSH access check (dry run)" +hf jobs ssh "$JOB_ID" --dry-run || true + +echo "==> Waiting for the build (10 to 20 minutes on ${HF_FLAVOR})..." +hf jobs wait "$JOB_ID" --timeout 2h +hf jobs logs "$JOB_ID" | tail -5 | grep -q KERNEL_BUILD_DONE + +mkdir -p "$DATA_DIR" +DL_DIR="$(mktemp -d)" +hf download "$HF_REPO" "Image-${KERNEL_VERSION}" "Image-${KERNEL_VERSION}.sha256" "config-${KERNEL_VERSION}" \ + --type dataset --local-dir "$DL_DIR" > /dev/null +(cd "$DL_DIR" && sha256sum -c "Image-${KERNEL_VERSION}.sha256") +cp "${DL_DIR}/Image-${KERNEL_VERSION}" "${DATA_DIR}/Image" +cp "${DL_DIR}/config-${KERNEL_VERSION}" "${DATA_DIR}/Image.config" +rm -rf "$DL_DIR" + +echo "==> Kernel ready at ${DATA_DIR}/Image ($(du -h "${DATA_DIR}/Image" | cut -f1))" +echo " Resolved config saved to ${DATA_DIR}/Image.config" diff --git a/scripts/build-kernel.sh b/scripts/build-kernel.sh index f038979..0b61706 100755 --- a/scripts/build-kernel.sh +++ b/scripts/build-kernel.sh @@ -1,9 +1,9 @@ #!/bin/bash set -euo pipefail -KERNEL_VERSION="${KERNEL_VERSION:-6.12.17}" +KERNEL_VERSION="${KERNEL_VERSION:-6.12.114}" KERNEL_MAJOR="${KERNEL_VERSION%%.*}" -DATA_DIR="${HOME}/.local/share/shuru" +DATA_DIR="${SHURU_DATA_DIR:-${HOME}/.local/share/shuru}" SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_DIR="$(dirname "$SCRIPT_DIR")" DEFCONFIG="${REPO_DIR}/kernel/shuru_defconfig" diff --git a/scripts/prepare-rootfs.sh b/scripts/prepare-rootfs.sh index 122073c..9bff3be 100755 --- a/scripts/prepare-rootfs.sh +++ b/scripts/prepare-rootfs.sh @@ -2,10 +2,10 @@ set -euo pipefail DEBIAN_RELEASE="trixie" -DATA_DIR="${HOME}/.local/share/shuru" +DATA_DIR="${SHURU_DATA_DIR:-${HOME}/.local/share/shuru}" ROOTFS_IMG="${DATA_DIR}/rootfs.ext4" KERNEL_PATH="${DATA_DIR}/Image" -INITRAMFS_PATH="${DATA_DIR}/initramfs.cpio.gz" +INITRAMFS_PATH="${DATA_DIR}/initramfs.cpio" GUEST_BINARY="target/aarch64-unknown-linux-musl/release/shuru-guest" ROOTFS_SIZE_MB=1024 @@ -48,45 +48,17 @@ if [ ! -f "$INITRAMFS_PATH" ]; then debian:${DEBIAN_RELEASE}-slim /bin/sh -c ' set -e apt-get update -qq > /dev/null 2>&1 - apt-get install -y -qq busybox-static e2fsprogs pax-utils cpio > /dev/null 2>&1 - - mkdir -p /initramfs/bin /initramfs/sbin /initramfs/usr/sbin - mkdir -p /initramfs/proc /initramfs/dev /initramfs/newroot - - cp /bin/busybox /initramfs/bin/busybox - mkdir -p /initramfs/etc - for cmd in sh mount umount switch_root cp chmod echo ifconfig route cat; do - ln -sf busybox "/initramfs/bin/${cmd}" - done - - lddtree -l /sbin/e2fsck /usr/sbin/resize2fs | sort -u \ - | cpio --quiet -pmdL /initramfs - - cp /tmp/shuru-init /initramfs/bin/shuru-init - chmod 755 /initramfs/bin/shuru-init - - cat > /initramfs/init << '\''INITEOF'\'' -#!/bin/sh -mount -t proc none /proc -mount -t devtmpfs none /dev -/sbin/e2fsck -p /dev/vda > /dev/null 2>&1 || true -/usr/sbin/resize2fs /dev/vda > /dev/null 2>&1 || true -mount -t ext4 /dev/vda /newroot -cp /bin/shuru-init /newroot/usr/bin/shuru-init -chmod 755 /newroot/usr/bin/shuru-init -if ifconfig eth0 up 2>/dev/null; then - ifconfig eth0 10.0.0.2 netmask 255.255.255.0 up - route add default gw 10.0.0.1 - echo "nameserver 10.0.0.1" > /newroot/etc/resolv.conf -fi -umount /proc -exec switch_root /newroot /usr/bin/shuru-init -INITEOF + apt-get install -y -qq cpio > /dev/null 2>&1 + mkdir -p /initramfs/dev /initramfs/newroot + # The agent is PID 1 and needs stdio before main runs. Without a + # console the kernel can drive, as under KVM, these give it one. + mknod -m 600 /initramfs/dev/console c 5 1 + mknod -m 666 /initramfs/dev/null c 1 3 + cp /tmp/shuru-init /initramfs/init chmod 755 /initramfs/init - cd /initramfs - find . | cpio -o -H newc 2>/dev/null | gzip > /output/initramfs.cpio.gz - echo "==> Initramfs created: $(du -h /output/initramfs.cpio.gz | cut -f1)" + find . | cpio -o -H newc 2>/dev/null > /output/initramfs.cpio + echo "==> Initramfs created: $(du -h /output/initramfs.cpio | cut -f1)" ' echo " Initramfs saved to ${INITRAMFS_PATH}" else @@ -109,13 +81,12 @@ if [[ "$(uname)" == "Darwin" ]]; then --platform linux/arm64/v8 \ -e DEBIAN_RELEASE="${DEBIAN_RELEASE}" \ -v "${ROOTFS_IMG}:/rootfs.ext4" \ - -v "${GUEST_BINARY}:/tmp/shuru-guest:ro" \ debian:${DEBIAN_RELEASE}-slim /bin/sh -c ' set -e apt-get update -qq apt-get install -y -qq debootstrap e2fsprogs > /dev/null 2>&1 - mkfs.ext4 -F -E lazy_itable_init=0 /rootfs.ext4 + mkfs.ext4 -F -e remount-ro -E lazy_itable_init=0 /rootfs.ext4 mkdir -p /mnt/rootfs mount -o loop /rootfs.ext4 /mnt/rootfs @@ -142,9 +113,6 @@ DPKGEOF chroot /mnt/rootfs apt-get clean rm -rf /mnt/rootfs/var/lib/apt/lists/* - cp /tmp/shuru-guest /mnt/rootfs/usr/bin/shuru-init - chmod 755 /mnt/rootfs/usr/bin/shuru-init - mkdir -p /mnt/rootfs/proc /mnt/rootfs/sys /mnt/rootfs/dev /mnt/rootfs/tmp /mnt/rootfs/run echo "shuru" > /mnt/rootfs/etc/hostname echo "nameserver 8.8.8.8" > /mnt/rootfs/etc/resolv.conf @@ -160,7 +128,7 @@ else sudo apt-get update && sudo apt-get install -y $MISSING_PKGS fi - mkfs.ext4 -F -E lazy_itable_init=0 "$ROOTFS_IMG" + mkfs.ext4 -F -e remount-ro -E lazy_itable_init=0 "$ROOTFS_IMG" MOUNT_DIR=$(mktemp -d) sudo mount -o loop "$ROOTFS_IMG" "$MOUNT_DIR" @@ -187,9 +155,6 @@ DPKGEOF sudo chroot "$MOUNT_DIR" apt-get clean sudo rm -rf "${MOUNT_DIR}/var/lib/apt/lists/"* - sudo cp "$GUEST_BINARY" "${MOUNT_DIR}/usr/bin/shuru-init" - sudo chmod 755 "${MOUNT_DIR}/usr/bin/shuru-init" - sudo mkdir -p "${MOUNT_DIR}/proc" "${MOUNT_DIR}/sys" "${MOUNT_DIR}/dev" "${MOUNT_DIR}/tmp" "${MOUNT_DIR}/run" echo "shuru" | sudo tee "${MOUNT_DIR}/etc/hostname" > /dev/null echo "nameserver 8.8.8.8" | sudo tee "${MOUNT_DIR}/etc/resolv.conf" > /dev/null diff --git a/scripts/static-pie-cc b/scripts/static-pie-cc new file mode 100755 index 0000000..9f53c54 --- /dev/null +++ b/scripts/static-pie-cc @@ -0,0 +1,14 @@ +#!/bin/sh +# Linker for the jailed node binaries. rustc links crt-static aarch64 +# Linux executables with -static -no-pie, since its target specs do not +# enable static-pie, and gcc then starts them from the non-relocating +# crt1.o. Dropping both flags for -static-pie gives rcrt1.o, so the binary +# relocates itself and loads at a random address. +for arg do + shift + case $arg in + -static | -no-pie) continue ;; + esac + set -- "$@" "$arg" +done +exec "${CC:-cc}" -static-pie "$@" diff --git a/skills/shuru/SKILL.md b/skills/shuru/SKILL.md index 7361412..9f62cde 100644 --- a/skills/shuru/SKILL.md +++ b/skills/shuru/SKILL.md @@ -1,15 +1,15 @@ --- name: shuru -description: Run commands in an isolated Linux microVM sandbox using the shuru CLI. Use when the user asks to execute untrusted code, install packages safely, test in a clean environment, or needs Linux-specific tooling on macOS. +description: Run commands in an isolated Linux microVM sandbox using the shuru CLI. Use when the user asks to execute untrusted code, install packages safely, test in a clean environment, run Docker containers or a headless browser inside a VM, or needs Linux-specific tooling on macOS. --- # Sandboxed Execution with Shuru -Shuru boots an ephemeral Linux microVM (Debian, ARM64) on macOS. Each `shuru run` gets a fresh disk clone - all changes are discarded on exit. Use it whenever you need to run commands in isolation from the host. +Shuru boots an ephemeral Linux microVM (Debian, ARM64) on macOS, with experimental support on Linux ARM64 hosts. Each `shuru run` gets a fresh disk clone and all changes are discarded on exit. Use it whenever you need to run commands in isolation from the host. ## Core Workflow -The pattern is: **run in sandbox, mount to share files, checkpoint to persist state**. +The pattern is: **run in sandbox, mount to share files, checkpoint to persist state, recipe for a ready-made environment**. ```bash # 1. Run a command in a fresh VM @@ -24,6 +24,10 @@ shuru run --allow-net -- sh -c 'apt-get install -y curl && curl https://example. # 4. If setup is expensive, save a checkpoint and reuse it shuru checkpoint create node-env --allow-net -- apt-get install -y nodejs npm shuru run --from node-env --mount .:/workspace -- node /workspace/app.js + +# 5. For Docker or a browser, use a recipe from the official catalog +shuru run --recipe docker -- docker run --rm hello-world +shuru run --recipe browser -p 9222:9222 -- sleep infinity ``` ## Command Chaining @@ -60,6 +64,9 @@ shuru run --mount ./src:/src --mount ./data:/data -- ls /src /data # From a checkpoint shuru run --from myenv -- npm test + +# From a recipe: builds the checkpoint on first use, reuses it after +shuru run --recipe docker -- docker ps ``` ### Checkpoints @@ -71,14 +78,47 @@ shuru checkpoint create [flags] [-- command...] # Stack: create from an existing checkpoint shuru checkpoint create with-deps --from base-env --allow-net -- npm install -# List all checkpoints (shows actual disk usage) +# Update: rebuild in place; the old checkpoint stays until the new build succeeds +shuru checkpoint update base-env --allow-net -- apt-get install -y build-essential git + +# Layer onto the existing checkpoint instead of starting over +shuru checkpoint update base-env --from base-env --allow-net -- apt-get install -y jq + +# Build a checkpoint from a recipe +shuru checkpoint create docker --recipe docker + +# List all checkpoints (shows actual disk usage and recipe source) shuru checkpoint list # Delete shuru checkpoint delete ``` -Checkpoint names must be unique - delete the old one before re-creating with the same name. +`create` refuses an existing name. Use `update` to replace a checkpoint; it builds under a staging name and swaps in only when the command exits 0, so a failed update leaves the old one intact. + +### Recipes + +A recipe is a versioned description of a checkpoint: a `recipe.json` plus shell steps, fetched from GitHub at an exact commit or read from a local directory. The official catalog is [superhq-ai/recipes](https://github.com/superhq-ai/recipes) and currently ships `docker` and `browser`. + +```bash +shuru recipe list # the official catalog +shuru recipe show docker # metadata, services and every step, without booting +shuru run --recipe docker -- docker ps # build on first use, reuse after +shuru checkpoint create docker --recipe docker # build explicitly under that name +shuru run --recipe docker --checkpoint ci -- ... # pick the checkpoint name +shuru checkpoint update docker --recipe docker # force a rebuild at the current revision +shuru run --recipe owner/repo/path@v1 --yes -- ... # third-party, pinned to a tag +shuru run --recipe ./my-recipe -- ... # local directory while developing one +``` + +Rules that matter when scripting: + +- A bare name such as `docker` always means the official catalog. A local directory must be written as a path (`./dir`, `../dir`, `/abs`, `~/dir`). +- Official and local recipes build without prompting. Third-party GitHub recipes ask for confirmation, so pass `--yes` in non-interactive runs. +- References resolve to a commit SHA that is recorded next to the checkpoint. A changed recipe rebuilds on the next `run --recipe`; an unchanged one boots the existing checkpoint in seconds. +- Builds use the network allowlist declared by the recipe, never the project `shuru.json`. The first build of `docker` or `browser` takes a few minutes. + +See [references/recipes.md](references/recipes.md) for the `recipe.json` format and guest services. ### Other Commands @@ -89,9 +129,6 @@ shuru init --force # re-download even if up to date # Upgrade CLI + OS image shuru upgrade - -# Clean up leftover data from crashed VMs -shuru prune ``` ## Common Patterns @@ -108,12 +145,47 @@ shuru checkpoint create python-dev --allow-net -- sh -c 'apt-get install -y pyth shuru run --from python-dev --mount .:/workspace -- sh -c 'cd /workspace && pytest' ``` +### Docker Inside the VM + +The `docker` recipe installs Docker Engine, runs `dockerd` as a guest service, and pre-pulls `nginxinc/nginx-unprivileged:1.31.7-alpine-slim`. The command starts only after `docker info` answers. + +```bash +# Build once (a few minutes), then every run boots in seconds +shuru checkpoint create docker --recipe docker + +# Serve the pre-pulled image on host port 8080; no --allow-net needed +shuru run --from docker -p 8080:8080 -- \ + docker run --rm -p 8080:8080 nginxinc/nginx-unprivileged:1.31.7-alpine-slim + +# Pulling other images needs network +shuru run --from docker --allow-net -- docker run --rm python:3.12-alpine python3 -c 'print(1)' + +# Build an image from a mounted project +shuru run --from docker --mount .:/app -- sh -c 'cd /app && docker build -t app . && docker run --rm app' +``` + +Containers get the same boundary as any guest process: no egress without `--allow-net`, the host allowlist applies, and host ports only open through `-p`. Keep Docker at its defaults inside the VM; do not use `--privileged` or mount the Docker socket. + +### Headless Browser (CDP) + +The `browser` recipe runs Chromium headless as an unprivileged user with the DevTools protocol on port 9222 inside the VM. Forward that port and point any CDP client (Playwright, Puppeteer, browser-use) at `http://127.0.0.1:9222`. + +```bash +# Keep the VM alive while a client on the host drives the browser +shuru run --recipe browser -p 9222:9222 -- sleep infinity + +# In another terminal +curl -s http://127.0.0.1:9222/json/version +``` + +Chromium stays inside the microVM; only its CDP port reaches the host. Add `--allow-net` when the browser must load real websites, and `--allow-host` to restrict which ones. The DevTools HTTP endpoint ignores HTTP/1.0 requests, so use a normal HTTP/1.1 client. + ### Testing Untrusted Code Run untrusted scripts with no network access and no host filesystem access: ```bash -# Fully isolated — no --allow-net, no --mount +# Fully isolated: no --allow-net, no --mount shuru run -- sh -c 'echo "malicious script here" && rm -rf / 2>/dev/null; echo "host is safe"' ``` @@ -176,19 +248,21 @@ Place `shuru.json` in the project root to avoid repeating flags: } ``` -CLI flags override config values. When `secrets` are configured, the guest receives placeholder tokens and the proxy substitutes real values on HTTPS requests to allowed hosts. See [references/config.md](references/config.md) for all fields. +CLI flags override config values. When `secrets` are configured, the guest receives placeholder tokens and the proxy substitutes real values on HTTPS requests to allowed hosts. A secret can also be minted by a `command` and refreshed before it expires. See [references/config.md](references/config.md) for all fields. ## Important Constraints -- **Networking is off by default.** You must pass `--allow-net` to install packages or make HTTP requests. +- **Networking is off by default.** You must pass `--allow-net` to install packages or make HTTP requests. Port forwarding with `-p` works without it. - **The guest is Debian Linux (aarch64).** Use `apt-get install` for packages. - **Ephemeral by default.** Everything is discarded on exit unless you checkpoint. - **Mounts are read-only by default.** Guest writes go to a tmpfs overlay and are discarded on exit. Use `:rw` suffix + `--allow-host-writes` to write to the host. -- **macOS only** (Apple Silicon). Uses Apple Virtualization.framework. +- **No systemd in the guest.** Long-running daemons are declared as guest services (see references/recipes.md); `systemctl` does not exist. +- **Apple Silicon macOS**, plus experimental Linux ARM64 hosts with `/dev/kvm`. - **Default resources:** 2 CPUs, 2048 MB RAM, 4096 MB disk. Override with `--cpus`, `--memory`, `--disk-size`. ## Deep-Dive Documentation -- [references/checkpoints.md](references/checkpoints.md) — checkpoint lifecycle, stacking, disk usage -- [references/config.md](references/config.md) — shuru.json fields and resolution order -- [references/networking.md](references/networking.md) — allow-net, port forwarding, proxy behavior +- [references/checkpoints.md](references/checkpoints.md): checkpoint lifecycle, update, stacking, disk usage +- [references/recipes.md](references/recipes.md): recipe.json format, references, trust, guest services +- [references/config.md](references/config.md): shuru.json fields and resolution order +- [references/networking.md](references/networking.md): allow-net, port forwarding, proxy behavior diff --git a/skills/shuru/references/checkpoints.md b/skills/shuru/references/checkpoints.md index 10cca76..38c1b47 100644 --- a/skills/shuru/references/checkpoints.md +++ b/skills/shuru/references/checkpoints.md @@ -12,6 +12,35 @@ This boots a VM, runs the command (or drops to `/bin/sh` if none given), and sav If `--from` is specified, the VM boots from that checkpoint instead of the base image. +`create` refuses a name that already exists. Use `update` to replace it. + +## Updating a Checkpoint + +```bash +shuru checkpoint update [flags] [-- command...] +``` + +`update` takes the same arguments as `create` and rebuilds the checkpoint in place, or creates it when missing. The new disk is built under a staging name and swapped in only when the command exits 0, so a failed update leaves the old checkpoint untouched and prints that it is unchanged. + +```bash +# Rebuild from the base image +shuru checkpoint update base --allow-net -- apt-get install -y build-essential git make + +# Layer the command onto the existing checkpoint +shuru checkpoint update base --from base --allow-net -- apt-get install -y jq + +# Rebuild a recipe checkpoint at the current recipe revision +shuru checkpoint update docker --recipe docker +``` + +## Building from a Recipe + +```bash +shuru checkpoint create --recipe +``` + +Runs the recipe steps instead of a command, each step in a fresh boot from the previous one. A recipe checkpoint records its source and revision in a sidecar file, and `checkpoint list` shows where it came from. `shuru run --recipe ` builds the same checkpoint on first use under the recipe name (or `--checkpoint `), and rebuilds it when the recipe changes. See [recipes.md](recipes.md). + ## Stacking Checkpoints can be layered. Each layer only stores its diff from the parent: @@ -30,17 +59,21 @@ shuru checkpoint create deps --from node --allow-net --mount .:/app -- sh -c 'cd shuru run --from [flags] [-- command...] ``` -The VM gets a fresh clone of the checkpoint — changes during the run are discarded on exit. +The VM gets a fresh clone of the checkpoint. Changes during the run are discarded on exit. + +If the checkpoint declares guest services (Docker, the browser), they start at boot and the command runs only once their readiness checks pass. They are stopped and the disk synced before the VM powers off. ## Lifecycle -- `checkpoint create` — save disk state -- `checkpoint list` — show all checkpoints with size and age -- `checkpoint delete ` — permanently remove a checkpoint -- Names must be unique. Delete before re-creating with the same name. +- `checkpoint create`: save disk state under a new name +- `checkpoint update`: replace an existing checkpoint, keeping the old one until the new build succeeds +- `checkpoint list`: show all checkpoints with size, age and recipe source +- `checkpoint delete `: permanently remove a checkpoint ## Disk Usage -Checkpoints use APFS clonefile. A fresh checkpoint from a 512MB base image might only use 10-50MB of actual disk space depending on what changed. Use `checkpoint list` to see real usage. +Checkpoints use APFS clonefile. A fresh checkpoint from the base image might only use 10-50MB of actual disk space depending on what changed. Use `checkpoint list` to see real usage. + +`/run`, `/dev/shm` and `/sys/fs/cgroup` are tmpfs inside the guest and are never part of a checkpoint. -If you're running low on disk, delete unused checkpoints with `checkpoint delete`. +If you are running low on disk, delete unused checkpoints with `checkpoint delete`. diff --git a/skills/shuru/references/config.md b/skills/shuru/references/config.md index c67ec70..e912e5d 100644 --- a/skills/shuru/references/config.md +++ b/skills/shuru/references/config.md @@ -46,7 +46,7 @@ Secrets let the guest use API keys without exposing the real values. The guest r - `from`: host environment variable containing the real value - `hosts`: domains where the proxy will substitute the placeholder with the real value -The guest sees `$API_KEY=shuru_tok_...`. The real secret never enters the VM. +The guest sees `$API_KEY=shuru_tok_...`. The real secret is never placed in the VM. Bind a secret only to hosts you trust with it: responses are not rewritten, so a bound host that reflects the request shows the real value to the guest. Substitution applies to the request head: the request line and any header value, whatever the header is named. A custom `X-My-Vendor-Token` works with diff --git a/skills/shuru/references/recipes.md b/skills/shuru/references/recipes.md new file mode 100644 index 0000000..892c19b --- /dev/null +++ b/skills/shuru/references/recipes.md @@ -0,0 +1,132 @@ +# Recipes and Guest Services + +A recipe is a directory with a `recipe.json` and shell steps. Shuru builds it into a checkpoint by running each step in a fresh boot from the previous step, then records the recipe source and revision next to the checkpoint. + +## Commands + +```bash +shuru recipe list # official catalog (github.com/superhq-ai/recipes) +shuru recipe show # metadata, services and every step, without booting +shuru run --recipe [-- command] # build on first use, reuse after, rebuild when the recipe changes +shuru run --recipe --checkpoint -- ... # choose the checkpoint name +shuru checkpoint create --recipe # build explicitly +shuru checkpoint update --recipe # force a rebuild at the current revision +``` + +`--recipe` replaces the command on `checkpoint create` and `update`; passing both is an error. + +## References + +| form | resolves to | +|------|-------------| +| `docker` | the official catalog, directory `docker`, default branch | +| `owner/repo` | that repository root | +| `owner/repo/path/to/dir` | a directory in that repository | +| any of the above `@ref` | a branch, tag or commit | +| `./dir`, `../dir`, `/abs/dir`, `~/dir` | a local directory | + +A bare name never means a local directory, even when one of that name exists in the working directory. Write `./name` for the local one. + +GitHub references resolve to a commit SHA through the GitHub API. Local recipes get a content hash. The resolved revision is printed during the build and stored in `.json` next to the checkpoint, which is what `run --recipe` compares to decide between reuse and rebuild. + +## Trust + +- Official catalog and local paths build without prompting. +- Third-party GitHub recipes print their source and ask for confirmation on a TTY. In scripts and agents pass `--yes`. +- `run --recipe` refuses to replace a checkpoint of the same name that was not built from a recipe. Delete it or pick another name with `--checkpoint`. + +## recipe.json + +```json +{ + "name": "docker", + "description": "Docker Engine 29.9 with dockerd as a guest service and nginx-unprivileged pre-pulled", + "min_shuru_version": "0.8.0", + "disk_size": 4096, + "memory": 2048, + "network": { + "allow": ["deb.debian.org", "security.debian.org", "download.docker.com", "*.docker.io"] + }, + "services": { + "docker": { + "exec": ["dockerd"], + "ready": ["docker", "info"], + "ready_timeout_secs": 60, + "stop_timeout_secs": 30 + } + }, + "steps": ["install.sh", "pull.sh"] +} +``` + +| field | required | meaning | +|-------|----------|---------| +| `name` | yes | default checkpoint name. `[a-z0-9][a-z0-9._-]*` | +| `description` | no | one line for `recipe list` and `recipe show` | +| `min_shuru_version` | no | refuse to build on an older CLI | +| `from` | no | existing checkpoint to start from instead of the base image | +| `cpus`, `memory`, `disk_size` | no | VM size for the build; CLI flags override | +| `network.allow` | no | hosts the build may reach. Absent means the build runs offline. An empty list means unrestricted | +| `services` | no | guest service files, written before the first step runs | +| `steps` | no | scripts run in order, default `["build.sh"]` | + +Builds use the recipe allowlist, never the project `shuru.json`. Every step runs as root in `/bin/sh` from a fresh boot of the previous step, so services declared in `recipe.json` are already running from step 2 onwards. + +## Guest services + +Any image can declare long-running processes in `/etc/shuru/services/.json`. The guest init starts them at boot, waits for the readiness check before running the command, and sends SIGTERM before the VM powers off. There is no systemd in the guest; this is the mechanism for daemons. + +```json +{ + "exec": ["dockerd"], + "ready": ["docker", "info"], + "ready_timeout_secs": 60, + "stop_timeout_secs": 30, + "env": {"HOME": "/root"}, + "cwd": "/" +} +``` + +| field | meaning | +|-------|---------| +| `exec` | argv of the daemon, never passed to a shell | +| `ready` | argv polled until it exits 0; the command waits for it | +| `ready_timeout_secs` | give up and fail the run if `ready` never passes | +| `stop_timeout_secs` | wait this long after SIGTERM before SIGKILL | +| `env` | extra environment for the daemon | +| `cwd` | working directory for the daemon | + +A recipe writes these files for you through its `services` block. A hand-built checkpoint can write them directly during `checkpoint create`. + +## Official recipes + +### docker + +Docker Engine (pinned docker-ce and containerd) with `dockerd` as a guest service and `nginxinc/nginx-unprivileged:1.31.7-alpine-slim` pre-pulled, so `docker run` of that image works offline. Containers reach the internet only through the Shuru proxy with `--allow-net`, host ports only through `-p`. Keep Docker at its defaults inside the VM. + +```bash +shuru run --recipe docker -p 8080:8080 -- \ + docker run --rm -p 8080:8080 nginxinc/nginx-unprivileged:1.31.7-alpine-slim +``` + +### browser + +Chromium from Debian, run headless as the unprivileged `browser` user with the DevTools protocol on port 9222 inside the VM. The service is ready when `/json/version` answers. + +```bash +shuru run --recipe browser -p 9222:9222 -- sleep infinity +curl -s http://127.0.0.1:9222/json/version +``` + +Point Playwright, Puppeteer or browser-use at `http://127.0.0.1:9222`. The DevTools HTTP endpoint ignores HTTP/1.0 requests, so use an HTTP/1.1 client. Add `--allow-net` for real websites and `--allow-host` to restrict them. Use `--memory 4096` for heavy pages. + +## Writing a recipe + +``` +my-recipe/ + recipe.json + install.sh + check.sh +``` + +Pin everything a step installs (package versions, signing keys, image digests) so two people building the recipe get the same checkpoint. Test locally with `shuru run --recipe ./my-recipe -- ...`, and share it from any GitHub repository as `owner/repo/my-recipe@tag`. diff --git a/www/src/components/landing/CliSection.astro b/www/src/components/landing/CliSection.astro index 3cc9ea7..96ecb0e 100644 --- a/www/src/components/landing/CliSection.astro +++ b/www/src/components/landing/CliSection.astro @@ -33,6 +33,11 @@ const tabs = [ title: "Port forwarding", description: "Expose guest ports to the host. Works without network access.", }, + { + id: "docker", + title: "Recipes & Docker", + description: "Build checkpoints from versioned recipes. The docker recipe runs containers in the sandbox under the same network rules.", + }, ]; --- @@ -109,6 +114,29 @@ const tabs = [

# No --allow-net needed. Tunneled over vsock.

+ +
+

$ shuru checkpoint create docker --recipe docker

+

shuru: building recipe docker from superhq-ai/recipes/docker@2a5c726 (2 steps)

+

shuru: checkpoint 'docker' saved

+
+

$ shuru run --from docker -p 8080:8080 -- docker run --rm -p 8080:8080 nginxinc/nginx-unprivileged:1.31.7-alpine-slim

+

shuru: forwarding 127.0.0.1:8080 -> guest:8080

+

# Or skip the create: shuru run --recipe docker builds it on first use.

+
+

$ curl -sI http://127.0.0.1:8080/ | head -1

+

HTTP/1.1 200 OK

+
+

$ shuru run --from docker -- cat /etc/shuru/services/docker.json

+

{"{"}

+

"exec": ["dockerd"],

+

"ready": ["docker", "info"],

+

"ready_timeout_secs": 60,

+

"stop_timeout_secs": 30

+

{"}"}

+
+

# Services start at boot, the command waits until they are ready, and they get SIGTERM before power off.

+
diff --git a/www/src/components/landing/Footer.astro b/www/src/components/landing/Footer.astro index f500cc6..a53f788 100644 --- a/www/src/components/landing/Footer.astro +++ b/www/src/components/landing/Footer.astro @@ -9,7 +9,7 @@ Linux Blog diff --git a/www/src/components/landing/HeroSection.astro b/www/src/components/landing/HeroSection.astro index fbb589b..2954470 100644 --- a/www/src/components/landing/HeroSection.astro +++ b/www/src/components/landing/HeroSection.astro @@ -1,6 +1,4 @@ -
- - +

Local-First Sandboxes @@ -32,14 +30,3 @@

- - diff --git a/www/src/components/landing/Nav.astro b/www/src/components/landing/Nav.astro index b1af14f..3996d38 100644 --- a/www/src/components/landing/Nav.astro +++ b/www/src/components/landing/Nav.astro @@ -1,9 +1,9 @@