From 9ef95ce071d61b9cfa0175ab62132e87e72dc4a8 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Sun, 17 May 2026 09:24:51 +0200 Subject: [PATCH 1/8] chore(devenv): add nix development shell Signed-off-by: Evan Lezar --- .gitignore | 11 +++ CONTRIBUTING.md | 24 +++++- architecture/build.md | 5 ++ crates/openshell-driver-vm/README.md | 2 + devenv.lock | 65 +++++++++++++++ devenv.nix | 114 +++++++++++++++++++++++++++ devenv.yaml | 20 +++++ tasks/scripts/vm/build-libkrun.sh | 43 +++++++++- 8 files changed, 281 insertions(+), 3 deletions(-) create mode 100644 devenv.lock create mode 100644 devenv.nix create mode 100644 devenv.yaml diff --git a/.gitignore b/.gitignore index 24a77fce2..969f5bef6 100644 --- a/.gitignore +++ b/.gitignore @@ -221,3 +221,14 @@ rfc.md # Markdown/mermaid lint tooling deps scripts/lint-mermaid/node_modules/ + +# Devenv +.devenv* +devenv.local.nix +devenv.local.yaml + +# direnv +.direnv + +# pre-commit +.pre-commit-config.yaml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9dddac69a..fe137ce4c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,7 +96,23 @@ New issues opened by users without `write`, `maintain`, or `admin` repository pe ## Prerequisites -Install [mise](https://mise.jdx.dev/). This is used to set up the development environment. +If you use Nix, enter the checked-in devenv shell first. It provides `mise`, +Z3, `pkg-config`, VM build tools, the Docker CLI, and local workflow CLIs +without using `apt` or `dnf`: + +```bash +devenv shell +mise trust +mise install --locked +``` + +The Docker daemon/machine still has to be configured on the host. Podman is not +installed by the devenv shell; if you use the Podman driver, install it on the +host and configure rootless UID/GID mapping (`newuidmap`, `newgidmap`, +`/etc/subuid`, and `/etc/subgid`). + +Without Nix/devenv, install [mise](https://mise.jdx.dev/). This is used to set +up the development environment. ```bash # Install mise (macOS/Linux) @@ -127,7 +143,11 @@ Project requirements: ### Z3 installation -The `openshell-prover` crate links against the system Z3 library via pkg-config. +The Nix/devenv shell provides Z3 and `pkg-config` for local builds. If you are +not using Nix/devenv, install Z3 through your platform package manager. + +The `openshell-prover` crate links against the system Z3 library via +`pkg-config`. ```bash # macOS diff --git a/architecture/build.md b/architecture/build.md index 8a4212aa7..5f31fc477 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -60,6 +60,11 @@ the macOS user's shared home directory. Local image work should use `mise` tasks rather than direct Docker commands so the same staging and tagging assumptions are used locally and in CI. +Local development can run inside the optional Nix/devenv shell. `devenv.nix` +provides host-native prerequisites such as Z3, `pkg-config`, libclang, e2fsprogs, +and container client CLIs; `mise` remains the task runner and installs the +version-pinned language/toolchain entries from `mise.toml`. + ## CI and E2E Required checks run on GitHub Actions. E2E and GPU workflows use NVIDIA diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 8da0b96a4..67bf87bc3 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -229,6 +229,8 @@ Each table is created atomically via `nft -f` on VM start and torn down atomical ## Prerequisites - macOS on Apple Silicon, or Linux on aarch64/x86_64 with KVM +- On Linux, `devenv shell` provides the native build prerequisites without + installing distro packages with `apt` or `dnf` - Rust toolchain - e2fsprogs (`mke2fs` or `mkfs.ext4`, plus `debugfs`) for root and overlay disk image creation and QEMU environment injection - Guest-supervisor cross-compile toolchain (needed on macOS, and on Linux when host arch ≠ guest arch): diff --git a/devenv.lock b/devenv.lock new file mode 100644 index 000000000..118e2fd09 --- /dev/null +++ b/devenv.lock @@ -0,0 +1,65 @@ +{ + "nodes": { + "devenv": { + "locked": { + "dir": "src/modules", + "lastModified": 1778893748, + "narHash": "sha256-VqVecb+w+gCD/hNAUI+3xcJEI4aHThCoq8jr0yBKODU=", + "owner": "cachix", + "repo": "devenv", + "rev": "768261db727bdd30d6a70af0dc9938a79830e515", + "type": "github" + }, + "original": { + "dir": "src/modules", + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "nixpkgs": { + "inputs": { + "nixpkgs-src": "nixpkgs-src" + }, + "locked": { + "lastModified": 1778507786, + "narHash": "sha256-HzSQCKMsMr8r55LwM1JuzIOB+8bzk0FEv6sItKvsfoY=", + "owner": "cachix", + "repo": "devenv-nixpkgs", + "rev": "8f24a228a782e24576b155d1e39f0d914b380691", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "rolling", + "repo": "devenv-nixpkgs", + "type": "github" + } + }, + "nixpkgs-src": { + "flake": false, + "locked": { + "lastModified": 1778274207, + "narHash": "sha256-I4puXmX1iovcCHZlRmztO3vW0mAbbRvq4F8wgIMQ1MM=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "b3da656039dc7a6240f27b2ef8cc6a3ef3bccae7", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "devenv": "devenv", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} \ No newline at end of file diff --git a/devenv.nix b/devenv.nix new file mode 100644 index 000000000..60450ec6e --- /dev/null +++ b/devenv.nix @@ -0,0 +1,114 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +{ pkgs, lib, ... }: + +let + z3Dev = lib.getDev pkgs.z3; + z3Lib = lib.getLib pkgs.z3; + opensslDev = lib.getDev pkgs.openssl; + opensslLib = lib.getLib pkgs.openssl; + elfutilsDev = lib.getDev pkgs.elfutils; + elfutilsLib = lib.getLib pkgs.elfutils; + libcapNgDev = lib.getDev pkgs.libcap_ng; + libcapNgLib = lib.getLib pkgs.libcap_ng; + libclangLib = pkgs.llvmPackages.libclang.lib; + + pkgConfigInputs = [ + z3Dev + opensslDev + elfutilsDev + libcapNgDev + ]; + + nativeLibs = [ + z3Lib + opensslLib + elfutilsLib + libcapNgLib + libclangLib + ]; + + pythonWithPyelftools = pkgs.python3.withPackages (ps: [ + ps.pyelftools + ]); +in +{ + packages = with pkgs; [ + # Project task runner. mise installs the version-pinned Rust, Python, Node, + # Kubernetes, documentation, and SBOM tools from mise.toml. + mise + + # Core source-control and scripting tools. + cacert + coreutils + curl + file + findutils + gawk + git + gnugrep + gnused + gnutar + gzip + jq + patch + perl + xz + zstd + + # Native build prerequisites used by Rust crates and VM runtime builds. + bc + bison + cmake + cpio + e2fsprogs + elfutils + flex + gcc + gnumake + libcap_ng + llvmPackages.libclang + openssl + pkg-config + z3 + + # Local workflow CLIs. The daemon or machine still needs host setup. + # Podman is intentionally not installed here: Linux rootless Podman also + # needs host uidmap helpers and /etc/subuid + /etc/subgid entries, so use a + # host-installed Podman when that driver is required. + docker-client + gh + ] ++ [ + pythonWithPyelftools + ]; + + env.PKG_CONFIG_PATH = lib.concatStringsSep ":" [ + (lib.makeSearchPathOutput "dev" "lib/pkgconfig" pkgConfigInputs) + (lib.makeSearchPathOutput "dev" "share/pkgconfig" pkgConfigInputs) + ]; + env.LD_LIBRARY_PATH = lib.makeLibraryPath nativeLibs; + env.LIBRARY_PATH = lib.makeLibraryPath nativeLibs; + + env.LIBCLANG_PATH = "${libclangLib}/lib"; + env.OPENSHELL_LIBKRUNFW_PYTHON = "${pythonWithPyelftools}/bin/python3"; + env.OPENSHELL_SKIP_SYSTEM_DEPS = "1"; + + env.Z3_SYS_Z3_HEADER = "${z3Dev}/include/z3.h"; + env.Z3_LIBRARY_PATH_OVERRIDE = "${z3Lib}/lib"; + + enterShell = '' + echo "OpenShell devenv ready. Run 'mise trust' once, then 'mise install --locked'." + ''; + + enterTest = '' + pkg-config --exists z3 + test -f "$Z3_SYS_Z3_HEADER" + test -d "$Z3_LIBRARY_PATH_OVERRIDE" + test -e "$LIBCLANG_PATH/libclang.so" + "$OPENSHELL_LIBKRUNFW_PYTHON" -c 'from elftools.elf.elffile import ELFFile' + test -x "$(command -v mise)" + test -x "$(command -v mke2fs)" + test -x "$(command -v docker)" + ''; +} diff --git a/devenv.yaml b/devenv.yaml new file mode 100644 index 000000000..813b4016a --- /dev/null +++ b/devenv.yaml @@ -0,0 +1,20 @@ +# yaml-language-server: $schema=https://devenv.sh/devenv.schema.json +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +inputs: + nixpkgs: + url: github:cachix/devenv-nixpkgs/rolling + +# If you're using non-OSS software, you can set allow_unfree to true. +# allow_unfree: true + +# If you're not willing to allow unsupported packages: +# allow_unsupported_system: false + +# If you're willing to use a package that's vulnerable +# permitted_insecure_packages: +# - "openssl-1.1.1w" + +# If you have more than one devenv you can merge them +#imports: +# - ./backend diff --git a/tasks/scripts/vm/build-libkrun.sh b/tasks/scripts/vm/build-libkrun.sh index 580f6bf4e..fd1036b26 100755 --- a/tasks/scripts/vm/build-libkrun.sh +++ b/tasks/scripts/vm/build-libkrun.sh @@ -20,7 +20,8 @@ # Usage: # ./build-libkrun.sh # -# The script will install missing dependencies on Debian/Ubuntu and Fedora. +# The script will install missing dependencies on Debian/Ubuntu and Fedora unless +# OPENSHELL_SKIP_SYSTEM_DEPS=1 or the script runs inside a Nix/devenv shell. set -euo pipefail @@ -61,8 +62,37 @@ if [ "$(id -u)" -ne 0 ]; then SUDO="sudo" fi +deps_provided_by_active_shell() { + [ "${OPENSHELL_SKIP_SYSTEM_DEPS:-}" = "1" ] || [ -n "${DEVENV_ROOT:-}" ] || [ -n "${IN_NIX_SHELL:-}" ] +} + +check_shell_deps() { + local missing=() + local cmd + + for cmd in make git gcc flex bison bc curl cpio zstd jq pkg-config python3; do + if ! command -v "$cmd" &>/dev/null; then + missing+=("$cmd") + fi + done + + if [ "${#missing[@]}" -gt 0 ]; then + echo "ERROR: missing build tools from the active shell: ${missing[*]}" >&2 + echo " Nix/devenv users: run 'devenv shell' from the repository root." >&2 + echo " Non-Nix users: unset OPENSHELL_SKIP_SYSTEM_DEPS to allow apt/dnf setup." >&2 + exit 1 + fi + + echo " Build tools found in active shell; skipping apt/dnf setup" +} + install_deps() { echo "==> Checking/installing build dependencies..." + + if deps_provided_by_active_shell; then + check_shell_deps + return 0 + fi if command -v apt-get &>/dev/null; then # Debian/Ubuntu @@ -101,7 +131,16 @@ install_deps # project venv, or other early PATH entry often shadows /usr/bin/python3 and does # not ship pyelftools even when python3-pyelftools is installed for the distro. ensure_python3_with_pyelftools_for_libkrunfw() { + local configured_python="${OPENSHELL_LIBKRUNFW_PYTHON:-}" echo " Checking Python 3 + pyelftools (libkrunfw bin2cbundle.py)..." + if [ -n "$configured_python" ]; then + if [ -x "$configured_python" ] && "$configured_python" -c 'from elftools.elf.elffile import ELFFile' 2>/dev/null; then + export PATH="$(dirname "$configured_python"):${PATH}" + echo " OK (${configured_python} from OPENSHELL_LIBKRUNFW_PYTHON)" + return 0 + fi + echo " Warning: OPENSHELL_LIBKRUNFW_PYTHON='${configured_python}' does not provide pyelftools" >&2 + fi if python3 -c 'from elftools.elf.elffile import ELFFile' 2>/dev/null; then echo " OK ($(command -v python3))" return 0 @@ -115,6 +154,8 @@ ensure_python3_with_pyelftools_for_libkrunfw() { echo " Install: Debian/Ubuntu: sudo apt-get install -y python3-pyelftools" >&2 echo " Fedora/RHEL: sudo dnf install -y python3-pyelftools" >&2 echo " pip: python3 -m pip install --user pyelftools" >&2 + echo " Nix/devenv: run 'devenv shell' from the repository root" >&2 + echo " Or set OPENSHELL_LIBKRUNFW_PYTHON to a python3 binary with pyelftools." >&2 echo " If the package is installed but this still fails, PATH may point at another python3 (mise, venv)." >&2 echo " Try: PATH=/usr/bin:\$PATH mise run vm:setup" >&2 exit 1 From d021df0ec67b44d0eb4176f65ef8c7cda3948fba Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Sun, 17 May 2026 10:35:20 +0200 Subject: [PATCH 2/8] fix(devenv): add zig musl toolchain wrappers Signed-off-by: Evan Lezar --- devenv.nix | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/devenv.nix b/devenv.nix index 60450ec6e..6524d6a3e 100644 --- a/devenv.nix +++ b/devenv.nix @@ -32,6 +32,26 @@ let pythonWithPyelftools = pkgs.python3.withPackages (ps: [ ps.pyelftools ]); + + zigMuslWrapper = name: tool: target: + pkgs.writeShellScriptBin name '' + set -euo pipefail + + args=() + for arg in "$@"; do + case "$arg" in + --target=*) ;; + *) args+=("$arg") ;; + esac + done + + exec zig ${tool} --target=${target} "''${args[@]}" + ''; + + aarch64MuslCc = zigMuslWrapper "openshell-zig-aarch64-linux-musl-cc" "cc" "aarch64-linux-musl"; + aarch64MuslCxx = zigMuslWrapper "openshell-zig-aarch64-linux-musl-cxx" "c++" "aarch64-linux-musl"; + x86_64MuslCc = zigMuslWrapper "openshell-zig-x86_64-linux-musl-cc" "cc" "x86_64-linux-musl"; + x86_64MuslCxx = zigMuslWrapper "openshell-zig-x86_64-linux-musl-cxx" "c++" "x86_64-linux-musl"; in { packages = with pkgs; [ @@ -97,6 +117,18 @@ in env.Z3_SYS_Z3_HEADER = "${z3Dev}/include/z3.h"; env.Z3_LIBRARY_PATH_OVERRIDE = "${z3Lib}/lib"; + # Match CI's Zig-backed musl toolchain so native Nix shells do not leak + # glibc-built C objects into static supervisor/CLI links. + env.CC_aarch64_unknown_linux_musl = "${aarch64MuslCc}/bin/openshell-zig-aarch64-linux-musl-cc"; + env.CXX_aarch64_unknown_linux_musl = "${aarch64MuslCxx}/bin/openshell-zig-aarch64-linux-musl-cxx"; + env.CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER = "${aarch64MuslCc}/bin/openshell-zig-aarch64-linux-musl-cc"; + env.CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS = "-Clink-self-contained=no"; + + env.CC_x86_64_unknown_linux_musl = "${x86_64MuslCc}/bin/openshell-zig-x86_64-linux-musl-cc"; + env.CXX_x86_64_unknown_linux_musl = "${x86_64MuslCxx}/bin/openshell-zig-x86_64-linux-musl-cxx"; + env.CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER = "${x86_64MuslCc}/bin/openshell-zig-x86_64-linux-musl-cc"; + env.CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS = "-Clink-self-contained=no"; + enterShell = '' echo "OpenShell devenv ready. Run 'mise trust' once, then 'mise install --locked'." ''; @@ -110,5 +142,9 @@ in test -x "$(command -v mise)" test -x "$(command -v mke2fs)" test -x "$(command -v docker)" + test -x "$CC_aarch64_unknown_linux_musl" + test -x "$CXX_aarch64_unknown_linux_musl" + test -x "$CC_x86_64_unknown_linux_musl" + test -x "$CXX_x86_64_unknown_linux_musl" ''; } From 61bcce39d17a79f9ed272aefc189c973292168c9 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 19 May 2026 10:22:39 +0200 Subject: [PATCH 3/8] fix(build): configure musl toolchain wrappers Signed-off-by: Evan Lezar --- architecture/build.md | 11 +- tasks/python.toml | 11 +- tasks/scripts/stage-prebuilt-binaries.sh | 124 +++++++++++++++++++++++ 3 files changed, 141 insertions(+), 5 deletions(-) diff --git a/architecture/build.md b/architecture/build.md index 5f31fc477..80f4d9449 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -33,10 +33,13 @@ Binary staging is driven by `tasks/scripts/stage-prebuilt-binaries.sh`, which runs `cargo build` natively on a matching host or `cargo zigbuild` when cross-compiling. Local Docker image tasks infer the target architecture from `DOCKER_PLATFORM` when set, otherwise from the container engine host metadata -with the kernel architecture as the fallback. CI invokes the same staging step -via the `rust-native-build.yml` workflow (per-architecture, per-component) and -uploads the result as an artifact that the image build job downloads back into -the staging directory before running Buildx. +with the kernel architecture as the fallback. Static musl supervisor builds use +target C/C++ linker settings from the environment when present, and otherwise +create Zig-backed wrappers so native Nix shells do not link musl Rust artifacts +with the host glibc compiler. CI invokes the same staging step via the +`rust-native-build.yml` workflow (per-architecture, per-component) and uploads +the result as an artifact that the image build job downloads back into the +staging directory before running Buildx. Runtime layout: diff --git a/tasks/python.toml b/tasks/python.toml index b95d96671..70944b837 100644 --- a/tasks/python.toml +++ b/tasks/python.toml @@ -12,7 +12,16 @@ hide = true ["build:python:wheel"] description = "Build Python wheel with CLI binary (native)" depends = ["python:proto"] -run = "uv run maturin build --release" +run = ''' +#!/usr/bin/env bash +set -euo pipefail + +# maturin/auditwheel rewrites the packaged binary's dynamic-library entries when +# it repairs a wheel. Relink the CLI before each native wheel build so a cached +# target/release/openshell binary does not point at a prior wheel-local lib name. +rm -f target/release/openshell target/release/openshell.d target/release/deps/openshell-* +uv run maturin build --release +''' hide = true ["python:build"] diff --git a/tasks/scripts/stage-prebuilt-binaries.sh b/tasks/scripts/stage-prebuilt-binaries.sh index 7c20d53a0..059a2bc37 100755 --- a/tasks/scripts/stage-prebuilt-binaries.sh +++ b/tasks/scripts/stage-prebuilt-binaries.sh @@ -52,6 +52,129 @@ host_os() { uname -s } +target_env_suffix() { + echo "${1//-/_}" | tr '[:lower:]' '[:upper:]' +} + +zig_target() { + case "$1" in + amd64) echo "x86_64-linux-musl" ;; + arm64) echo "aarch64-linux-musl" ;; + *) + echo "unsupported architecture for Zig musl wrapper: $1" >&2 + exit 1 + ;; + esac +} + +find_zig() { + if command -v zig >/dev/null 2>&1; then + command -v zig + return + fi + + if command -v mise >/dev/null 2>&1; then + mise which zig 2>/dev/null || true + fi +} + +zig_musl_wrapper() { + local name=$1 + local tool=$2 + local target=$3 + local zig_bin + local wrapper_dir + local wrapper + + zig_bin="$(find_zig)" + if [[ -z "$zig_bin" ]]; then + echo "Error: building musl binaries requires Zig when no target C linker is configured." >&2 + echo "Run 'mise install --locked' or set the CARGO_TARGET_*_LINKER and CC_* env vars for the musl target." >&2 + exit 1 + fi + + wrapper_dir="${ROOT}/target/toolchain-wrappers" + wrapper="${wrapper_dir}/${name}" + mkdir -p "$wrapper_dir" + cat >"$wrapper" </dev/null 2>&1 || true + configure_musl_toolchain "$arch" "$target" args=( --release From 407ccb72ffbe80fccaef60307c1467e9679bb5fa Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 19 May 2026 09:20:54 +0200 Subject: [PATCH 4/8] fix(devenv): support Darwin shells Signed-off-by: Evan Lezar --- CONTRIBUTING.md | 7 ++- architecture/build.md | 9 ++- crates/openshell-driver-vm/README.md | 20 +++++-- devenv.nix | 82 +++++++++++++++++----------- 4 files changed, 76 insertions(+), 42 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fe137ce4c..2c1807ad2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -97,8 +97,8 @@ New issues opened by users without `write`, `maintain`, or `admin` repository pe ## Prerequisites If you use Nix, enter the checked-in devenv shell first. It provides `mise`, -Z3, `pkg-config`, VM build tools, the Docker CLI, and local workflow CLIs -without using `apt` or `dnf`: +Z3, `pkg-config`, platform-specific VM build tools, the Docker CLI, and local +workflow CLIs without using host package managers for those tools: ```bash devenv shell @@ -106,6 +106,9 @@ mise trust mise install --locked ``` +On macOS, install Xcode Command Line Tools separately; the devenv shell uses the +host Apple compiler so C++ builds can see the Apple SDK and libc++ headers. + The Docker daemon/machine still has to be configured on the host. Podman is not installed by the devenv shell; if you use the Podman driver, install it on the host and configure rootless UID/GID mapping (`newuidmap`, `newgidmap`, diff --git a/architecture/build.md b/architecture/build.md index 80f4d9449..b5b0acee4 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -64,9 +64,12 @@ Local image work should use `mise` tasks rather than direct Docker commands so the same staging and tagging assumptions are used locally and in CI. Local development can run inside the optional Nix/devenv shell. `devenv.nix` -provides host-native prerequisites such as Z3, `pkg-config`, libclang, e2fsprogs, -and container client CLIs; `mise` remains the task runner and installs the -version-pinned language/toolchain entries from `mise.toml`. +provides host-native prerequisites such as Z3, `pkg-config`, libclang, +e2fsprogs, container client CLIs, and platform-specific VM build dependencies +(`elfutils`/`libcap-ng` on Linux, `lld`/`dtc` on macOS); `mise` remains the task +runner and installs the version-pinned language/toolchain entries from +`mise.toml`. On macOS the shell uses the host Xcode/Command Line Tools compiler +so C++ probes build against the Apple SDK and libc++ headers. ## CI and E2E diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 67bf87bc3..eb5a584e5 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -229,13 +229,21 @@ Each table is created atomically via `nft -f` on VM start and torn down atomical ## Prerequisites - macOS on Apple Silicon, or Linux on aarch64/x86_64 with KVM -- On Linux, `devenv shell` provides the native build prerequisites without - installing distro packages with `apt` or `dnf` +- On macOS and Linux, `devenv shell` provides the native build prerequisites + without installing them through Homebrew, `apt`, or `dnf` +- Xcode Command Line Tools on macOS; the devenv shell uses the host Apple + compiler for SDK and libc++ support - Rust toolchain -- e2fsprogs (`mke2fs` or `mkfs.ext4`, plus `debugfs`) for root and overlay disk image creation and QEMU environment injection -- Guest-supervisor cross-compile toolchain (needed on macOS, and on Linux when host arch ≠ guest arch): - - Matching rustup target: `rustup target add aarch64-unknown-linux-gnu` (or `x86_64-unknown-linux-gnu` for an amd64 guest) - - `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `vm:supervisor` uses `cargo zigbuild` to cross-compile the in-VM `openshell-sandbox` supervisor binary. +- e2fsprogs (`mke2fs` or `mkfs.ext4`, plus `debugfs`) for root and overlay disk + image creation and QEMU environment injection +- Guest-supervisor cross-compile toolchain (needed on macOS, and on Linux when + host arch differs from guest arch): + - Matching rustup target: `rustup target add aarch64-unknown-linux-gnu` (or + `x86_64-unknown-linux-gnu` for an amd64 guest) + - `mise install --locked` installs Zig and `cargo-zigbuild` where needed. + Non-mise setups can install `cargo-zigbuild` and Zig through the host + package manager. `vm:supervisor` uses `cargo zigbuild` to cross-compile the + in-VM `openshell-sandbox` supervisor binary. - [mise](https://mise.jdx.dev/) task runner - Docker or Podman socket on the local CLI/gateway host when using `openshell sandbox create --from ./Dockerfile` or `--from ./dir`; the CLI diff --git a/devenv.nix b/devenv.nix index 6524d6a3e..c66dd74f1 100644 --- a/devenv.nix +++ b/devenv.nix @@ -4,6 +4,9 @@ { pkgs, lib, ... }: let + isLinux = pkgs.stdenv.isLinux; + isDarwin = pkgs.stdenv.isDarwin; + z3Dev = lib.getDev pkgs.z3; z3Lib = lib.getLib pkgs.z3; opensslDev = lib.getDev pkgs.openssl; @@ -17,6 +20,7 @@ let pkgConfigInputs = [ z3Dev opensslDev + ] ++ lib.optionals isLinux [ elfutilsDev libcapNgDev ]; @@ -24,11 +28,14 @@ let nativeLibs = [ z3Lib opensslLib + libclangLib + ] ++ lib.optionals isLinux [ elfutilsLib libcapNgLib - libclangLib ]; + libclangSharedLibrary = if isDarwin then "libclang.dylib" else "libclang.so"; + pythonWithPyelftools = pkgs.python3.withPackages (ps: [ ps.pyelftools ]); @@ -83,12 +90,8 @@ in cmake cpio e2fsprogs - elfutils flex - gcc gnumake - libcap_ng - llvmPackages.libclang openssl pkg-config z3 @@ -99,35 +102,52 @@ in # host-installed Podman when that driver is required. docker-client gh + ] ++ lib.optionals isLinux [ + # Linux-only VM runtime build dependencies. + elfutils + gcc + libcap_ng + llvmPackages.libclang + ] ++ lib.optionals isDarwin [ + # macOS VM runtime build dependencies that are otherwise documented as + # Homebrew prerequisites. Darwin uses the host Xcode/CLT compiler so C++ + # probes can see the Apple SDK and libc++ headers. + dtc + llvmPackages.lld ] ++ [ pythonWithPyelftools ]; - env.PKG_CONFIG_PATH = lib.concatStringsSep ":" [ - (lib.makeSearchPathOutput "dev" "lib/pkgconfig" pkgConfigInputs) - (lib.makeSearchPathOutput "dev" "share/pkgconfig" pkgConfigInputs) - ]; - env.LD_LIBRARY_PATH = lib.makeLibraryPath nativeLibs; - env.LIBRARY_PATH = lib.makeLibraryPath nativeLibs; - - env.LIBCLANG_PATH = "${libclangLib}/lib"; - env.OPENSHELL_LIBKRUNFW_PYTHON = "${pythonWithPyelftools}/bin/python3"; - env.OPENSHELL_SKIP_SYSTEM_DEPS = "1"; - - env.Z3_SYS_Z3_HEADER = "${z3Dev}/include/z3.h"; - env.Z3_LIBRARY_PATH_OVERRIDE = "${z3Lib}/lib"; - - # Match CI's Zig-backed musl toolchain so native Nix shells do not leak - # glibc-built C objects into static supervisor/CLI links. - env.CC_aarch64_unknown_linux_musl = "${aarch64MuslCc}/bin/openshell-zig-aarch64-linux-musl-cc"; - env.CXX_aarch64_unknown_linux_musl = "${aarch64MuslCxx}/bin/openshell-zig-aarch64-linux-musl-cxx"; - env.CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER = "${aarch64MuslCc}/bin/openshell-zig-aarch64-linux-musl-cc"; - env.CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS = "-Clink-self-contained=no"; - - env.CC_x86_64_unknown_linux_musl = "${x86_64MuslCc}/bin/openshell-zig-x86_64-linux-musl-cc"; - env.CXX_x86_64_unknown_linux_musl = "${x86_64MuslCxx}/bin/openshell-zig-x86_64-linux-musl-cxx"; - env.CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER = "${x86_64MuslCc}/bin/openshell-zig-x86_64-linux-musl-cc"; - env.CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS = "-Clink-self-contained=no"; + env = { + PKG_CONFIG_PATH = lib.concatStringsSep ":" [ + (lib.makeSearchPathOutput "dev" "lib/pkgconfig" pkgConfigInputs) + (lib.makeSearchPathOutput "dev" "share/pkgconfig" pkgConfigInputs) + ]; + LIBRARY_PATH = lib.makeLibraryPath nativeLibs; + + LIBCLANG_PATH = "${libclangLib}/lib"; + OPENSHELL_LIBKRUNFW_PYTHON = "${pythonWithPyelftools}/bin/python3"; + OPENSHELL_SKIP_SYSTEM_DEPS = "1"; + + Z3_SYS_Z3_HEADER = "${z3Dev}/include/z3.h"; + Z3_LIBRARY_PATH_OVERRIDE = "${z3Lib}/lib"; + + # Match CI's Zig-backed musl toolchain so native Nix shells do not leak + # glibc-built C objects into static supervisor/CLI links. + CC_aarch64_unknown_linux_musl = "${aarch64MuslCc}/bin/openshell-zig-aarch64-linux-musl-cc"; + CXX_aarch64_unknown_linux_musl = "${aarch64MuslCxx}/bin/openshell-zig-aarch64-linux-musl-cxx"; + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER = "${aarch64MuslCc}/bin/openshell-zig-aarch64-linux-musl-cc"; + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS = "-Clink-self-contained=no"; + + CC_x86_64_unknown_linux_musl = "${x86_64MuslCc}/bin/openshell-zig-x86_64-linux-musl-cc"; + CXX_x86_64_unknown_linux_musl = "${x86_64MuslCxx}/bin/openshell-zig-x86_64-linux-musl-cxx"; + CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER = "${x86_64MuslCc}/bin/openshell-zig-x86_64-linux-musl-cc"; + CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS = "-Clink-self-contained=no"; + } // lib.optionalAttrs isLinux { + LD_LIBRARY_PATH = lib.makeLibraryPath nativeLibs; + } // lib.optionalAttrs isDarwin { + DYLD_FALLBACK_LIBRARY_PATH = lib.makeLibraryPath nativeLibs; + }; enterShell = '' echo "OpenShell devenv ready. Run 'mise trust' once, then 'mise install --locked'." @@ -137,7 +157,7 @@ in pkg-config --exists z3 test -f "$Z3_SYS_Z3_HEADER" test -d "$Z3_LIBRARY_PATH_OVERRIDE" - test -e "$LIBCLANG_PATH/libclang.so" + test -e "$LIBCLANG_PATH/${libclangSharedLibrary}" "$OPENSHELL_LIBKRUNFW_PYTHON" -c 'from elftools.elf.elffile import ELFFile' test -x "$(command -v mise)" test -x "$(command -v mke2fs)" From dea9c60e3c61719f00fa7d57339093abcaa2731d Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 19 May 2026 10:22:32 +0200 Subject: [PATCH 5/8] fix(devenv): raise shell open file limit Signed-off-by: Evan Lezar --- architecture/build.md | 4 +++- devenv.nix | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/architecture/build.md b/architecture/build.md index b5b0acee4..12f4c949d 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -69,7 +69,9 @@ e2fsprogs, container client CLIs, and platform-specific VM build dependencies (`elfutils`/`libcap-ng` on Linux, `lld`/`dtc` on macOS); `mise` remains the task runner and installs the version-pinned language/toolchain entries from `mise.toml`. On macOS the shell uses the host Xcode/Command Line Tools compiler -so C++ probes build against the Apple SDK and libc++ headers. +so C++ probes build against the Apple SDK and libc++ headers. The shell also +raises the soft open-file limit when possible so parallel Cargo and `sccache` +workloads do not inherit macOS's low interactive default. ## CI and E2E diff --git a/devenv.nix b/devenv.nix index c66dd74f1..07f9cb646 100644 --- a/devenv.nix +++ b/devenv.nix @@ -150,6 +150,21 @@ in }; enterShell = '' + desired_open_files=8192 + current_open_files="$(ulimit -Sn 2>/dev/null || ulimit -n 2>/dev/null || echo 0)" + + case "$current_open_files" in + ""|*[!0-9]*) + ;; + *) + if [ "$current_open_files" -lt "$desired_open_files" ]; then + ulimit -Sn "$desired_open_files" 2>/dev/null \ + || ulimit -n "$desired_open_files" 2>/dev/null \ + || echo "Warning: could not raise open file limit to $desired_open_files" + fi + ;; + esac + echo "OpenShell devenv ready. Run 'mise trust' once, then 'mise install --locked'." ''; From 44dd3481c3ff598afd942f10c5f7e15f0f0843a7 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 19 May 2026 10:36:02 +0200 Subject: [PATCH 6/8] chore(devenv): gate mise trust hint Signed-off-by: Evan Lezar --- devenv.nix | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/devenv.nix b/devenv.nix index 07f9cb646..015370720 100644 --- a/devenv.nix +++ b/devenv.nix @@ -165,7 +165,12 @@ in ;; esac - echo "OpenShell devenv ready. Run 'mise trust' once, then 'mise install --locked'." + mise_trust_status="$(mise trust --show 2>/dev/null || true)" + if printf '%s\n' "$mise_trust_status" | grep -q ': untrusted$'; then + echo "OpenShell devenv ready. Run 'mise trust' once, then 'mise install --locked'." + else + echo "OpenShell devenv ready. Run 'mise install --locked'." + fi ''; enterTest = '' From 0aa8e5ee57bcf2359c08d48a7dd19bac1acf3acb Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 19 May 2026 10:39:11 +0200 Subject: [PATCH 7/8] chore(devenv): gate mise install hint Signed-off-by: Evan Lezar --- devenv.nix | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/devenv.nix b/devenv.nix index 015370720..753508a6f 100644 --- a/devenv.nix +++ b/devenv.nix @@ -167,9 +167,14 @@ in mise_trust_status="$(mise trust --show 2>/dev/null || true)" if printf '%s\n' "$mise_trust_status" | grep -q ': untrusted$'; then - echo "OpenShell devenv ready. Run 'mise trust' once, then 'mise install --locked'." + echo "OpenShell devenv ready. Run 'mise trust' once." else - echo "OpenShell devenv ready. Run 'mise install --locked'." + mise_missing_tools="$(mise ls --current --missing --no-header 2>/dev/null || true)" + if [ -n "$mise_missing_tools" ]; then + echo "OpenShell devenv ready. Run 'mise install --locked'." + else + echo "OpenShell devenv ready." + fi fi ''; From 5e1e71e5fe84f844eb60c758d3e6e865b55d0be6 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 19 May 2026 11:31:49 +0200 Subject: [PATCH 8/8] docs(devenv): clarify Darwin compiler setup Signed-off-by: Evan Lezar --- CONTRIBUTING.md | 5 +++-- architecture/build.md | 4 ++-- crates/openshell-driver-vm/README.md | 4 ++-- devenv.nix | 4 ++-- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2c1807ad2..f0a7789ed 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,8 +106,9 @@ mise trust mise install --locked ``` -On macOS, install Xcode Command Line Tools separately; the devenv shell uses the -host Apple compiler so C++ builds can see the Apple SDK and libc++ headers. +On macOS, keep Xcode Command Line Tools installed for host developer tools. The +devenv shell configures Nix's Darwin clang wrapper and Apple SDK paths so C++ +builds can see SDK and libc++ headers. The Docker daemon/machine still has to be configured on the host. Podman is not installed by the devenv shell; if you use the Podman driver, install it on the diff --git a/architecture/build.md b/architecture/build.md index 12f4c949d..b52845631 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -68,8 +68,8 @@ provides host-native prerequisites such as Z3, `pkg-config`, libclang, e2fsprogs, container client CLIs, and platform-specific VM build dependencies (`elfutils`/`libcap-ng` on Linux, `lld`/`dtc` on macOS); `mise` remains the task runner and installs the version-pinned language/toolchain entries from -`mise.toml`. On macOS the shell uses the host Xcode/Command Line Tools compiler -so C++ probes build against the Apple SDK and libc++ headers. The shell also +`mise.toml`. On macOS the shell configures Nix's Darwin clang wrapper and Apple +SDK paths so C++ probes build against SDK and libc++ headers. The shell also raises the soft open-file limit when possible so parallel Cargo and `sccache` workloads do not inherit macOS's low interactive default. diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index eb5a584e5..4ba080385 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -231,8 +231,8 @@ Each table is created atomically via `nft -f` on VM start and torn down atomical - macOS on Apple Silicon, or Linux on aarch64/x86_64 with KVM - On macOS and Linux, `devenv shell` provides the native build prerequisites without installing them through Homebrew, `apt`, or `dnf` -- Xcode Command Line Tools on macOS; the devenv shell uses the host Apple - compiler for SDK and libc++ support +- Xcode Command Line Tools on macOS for host developer tools; the devenv shell + configures Nix's Darwin clang wrapper and Apple SDK paths for C++ builds - Rust toolchain - e2fsprogs (`mke2fs` or `mkfs.ext4`, plus `debugfs`) for root and overlay disk image creation and QEMU environment injection diff --git a/devenv.nix b/devenv.nix index 753508a6f..06ff47fc2 100644 --- a/devenv.nix +++ b/devenv.nix @@ -110,8 +110,8 @@ in llvmPackages.libclang ] ++ lib.optionals isDarwin [ # macOS VM runtime build dependencies that are otherwise documented as - # Homebrew prerequisites. Darwin uses the host Xcode/CLT compiler so C++ - # probes can see the Apple SDK and libc++ headers. + # Homebrew prerequisites. Darwin builds use Nix's clang wrapper and Apple + # SDK paths so C++ probes can see SDK and libc++ headers. dtc llvmPackages.lld ] ++ [