Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
355 changes: 354 additions & 1 deletion .github/workflows/main.yml

Large diffs are not rendered by default.

54 changes: 53 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ SRCS := \
core/vdso.c \
core/shim-globals.c \
core/bootstrap.c \
core/launch.c \
core/rosetta.c \
core/sysroot.c \
runtime/thread.c \
Expand Down Expand Up @@ -100,7 +101,7 @@ endef
.PHONY: all elfuse
.PHONY: gen-syscall-dispatch check-syscall-dispatch

all: elfuse
all: elfuse elfuse-container

## Regenerate build/dispatch.h from src/syscall/dispatch.tbl
gen-syscall-dispatch:
Expand All @@ -125,6 +126,57 @@ elfuse: $(ELFUSE_BIN)
$(ELFUSE_BIN): $(OBJS) | $(BUILD_DIR)
$(call link-and-sign,$@,$(OBJS))

# OCI container CLI (Go). Pure Go, no HVF entitlement or codesigning required,
# so it also builds under Linux for spec-conformance / interop CI. The version
# is stamped from the same VERSION string the C binary uses.
CONTAINER_BIN := $(BUILD_DIR)/elfuse-container
CONTAINER_SRCS := $(shell find cmd/elfuse-container -type f -name '*.go' 2>/dev/null)

.PHONY: elfuse-container
elfuse-container: $(CONTAINER_BIN)

# OCI image-layout conformance + cross-tool interop. Pulls fixtures into a
# throwaway store and asserts the on-disk layout is spec-shaped and readable by
# crane/skopeo/umoci (whichever are installed locally; all are required in CI).
# Pure Go + jq; no HVF, runs on Linux. Requires network to pull fixtures.
.PHONY: oci-interop
oci-interop: $(CONTAINER_BIN)
$(Q)scripts/oci-interop.sh

# Go unit tests for the OCI container CLI (offline). Set ELFUSE_OCI_NETTEST=1
# to also exercise the network pull round-trip conformance test.
.PHONY: oci-test
oci-test:
$(Q)$(GO) test ./cmd/elfuse-container/

# gofmt + go vet gate for the OCI container CLI. `go vet` is run for both GOOS
# values so the darwin-only sparsebundle files are checked from Linux CI and the
# non-darwin stubs are checked from a macOS host. The darwin pass pins
# GOARCH=arm64 (the only supported darwin target, and what CI checks) so an
# amd64 Linux host does not silently vet darwin/amd64 instead. oci-lint
# bundles both so a local run matches the CI gate.
.PHONY: oci-vet oci-fmt-check oci-lint
oci-vet:
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
$(Q)$(GO) vet ./cmd/elfuse-container/
$(Q)GOOS=darwin GOARCH=arm64 $(GO) vet ./cmd/elfuse-container/
$(Q)GOOS=linux $(GO) vet ./cmd/elfuse-container/

oci-fmt-check:
$(Q)out="$$(gofmt -l cmd/elfuse-container)"; \
if [ -n "$$out" ]; then \
echo "gofmt needs to run on:"; echo "$$out"; exit 1; \
fi

oci-lint: oci-fmt-check oci-vet

# rm -f first: `go build -o` follows an existing symlink at the output path,
# so a stale build/elfuse-container symlink would clobber build/elfuse.
$(CONTAINER_BIN): go.mod $(CONTAINER_SRCS) | $(BUILD_DIR)
@echo " GO $@"
$(Q)rm -f $@
$(Q)cd $(CURDIR) && $(GO) build -ldflags "-X main.version=$(VERSION)" \
-o $@ ./cmd/elfuse-container

# Native test binaries (macOS, Hypervisor.framework)

## Build the multi-vCPU HVF validation test (native macOS binary)
Expand Down
34 changes: 32 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ boot-time overhead those tools impose.
- Xcode Command Line Tools, `clang`, `codesign`, and GNU `make`
- GNU `objcopy` or `llvm-objcopy`
- Hypervisor entitlement: `com.apple.security.hypervisor`
- Go, for building the `elfuse-container` OCI CLI

To build only (`make elfuse`) without running tests, just the
Xcode Command Line Tools and `objcopy` (`brew install binutils`) suffice.
Expand Down Expand Up @@ -101,18 +102,44 @@ state.
The build signs `build/elfuse` before use. Override the signing identity with
`SIGN_IDENTITY="Developer ID ..."` when needed.

## OCI Images

OCI images are handled by `elfuse-container`, a Go companion binary that owns
the whole image lifecycle and invokes `elfuse` purely as the runtime. This
uses OCI image packaging only; it is not a Docker-compatible container runtime
and does not add namespaces, cgroups, port mapping, or a daemon.

```sh
make elfuse elfuse-container

build/elfuse-container pull alpine:3
build/elfuse-container run alpine:3 /bin/sh -c 'echo hello from elfuse'
```

Images are stored under `$ELFUSE_OCI_STORE`, or `~/.local/share/elfuse/oci`
by default. On macOS, `run` uses a case-sensitive APFS sparsebundle and a
per-run copy-on-write rootfs clone so normal APFS case folding does not
corrupt Linux filenames.

See [docs/usage.md](docs/usage.md#oci-images) for commands and flags, and
[docs/oci-design.md](docs/oci-design.md) for the implementation model,
including exactly which OCI features are and are not implemented.

## Documentation

- [docs/usage.md](docs/usage.md): command-line options, x86_64 via
Rosetta, dynamic linking via `--sysroot`, and attaching `gdb` /
`lldb` to the built-in stub.
Rosetta, dynamic linking via `--sysroot`, OCI images, and attaching
`gdb` / `lldb` to the built-in stub.
- [docs/testing.md](docs/testing.md): build prerequisites, the
`make check` flow, the QEMU and Rosetta cross-check matrices, and
fixture handling.
- [docs/internals.md](docs/internals.md): canonical technical
reference -- runtime lifecycle, HVF constraints, EL1 shim and HVC
protocol, page-table splitting, syscall translation tables, threads
/ futex, fork / clone IPC, signals, ptrace, and the GDB stub.
- [docs/oci-design.md](docs/oci-design.md): how elfuse-container, the image
store, layer unpacker, sparsebundle run path, and lifecycle commands
fit into elfuse.

## Build And Validation

Expand All @@ -123,6 +150,7 @@ make elfuse # build and codesign build/elfuse
make check # quick unit suite + BusyBox applet smoke
make test-gdbstub # debugger integration
make test-matrix # cross-check elfuse against QEMU on the same corpus
make oci-test # elfuse-container unit and conformance tests
make lint # clang-tidy
```

Expand All @@ -141,6 +169,8 @@ do.
- Linux kernel features that have no user-space-syscall analog:
namespaces, cgroups, kernel modules, eBPF, `io_uring`, KVM, perf
events.
- Docker-compatible container runtime features such as port mapping,
detached containers, `docker exec`, image build/push, and daemon APIs.
- Intel Macs. Apple Silicon only (M1 and later).
- Hosting a VM from inside a guest. The guest cannot use HVF or KVM.
- One guest process tree per `elfuse` host process. HVF allows one VM
Expand Down
128 changes: 128 additions & 0 deletions cmd/elfuse-container/bundlelock.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
package main

import (
"errors"
"fmt"
"os"
"path/filepath"
"syscall"
)

// Per-digest bundle locks.
//
// A case-sensitive sparsebundle bundle (<store>/cs/<algo>/<hex>/) is shared
// mutable state: concurrent `run`s of one digest share its attached volume,
// while prune --cache and rmi --force want to detach and remove it. Liveness
// is decided by advisory flocks, not by pids or directory scans -- a held
// lock proves a live holder regardless of pid reuse, and a free lock proves
// the holder is gone regardless of what the directory contains.
//
// Two lock files live in the bundle directory, deliberately OUTSIDE the
// mounted volume: hdiutil detach -force revokes descriptors inside the
// volume, which would silently drop a lock held there, and the locks must be
// probeable while the volume is not attached at all.
//
// - attach.lock: exclusive, serializes bundle lifecycle transitions. A run
// holds it (blocking) across provisioning -- stale-mount recovery,
// hdiutil create/attach -- and the last-one-out detach; a sweep holds it
// (non-blocking) for its whole reap-detach-remove sequence.
// - run.lock: every live run holds it shared from before the volume is
// attached until the guest exits (the process exiting releases it, so a
// killed run cannot leak liveness). Anyone holding it exclusively has
// proven there are zero live runs: sweeps take it non-blocking (busy =>
// skip the bundle), and provision probes it to tell a stale leftover
// mount from one that is live.
//
// Lock ordering: attach.lock is always acquired before run.lock is taken
// exclusively. That makes the EX->SH downgrade in provision safe (flock
// downgrades by release-and-reacquire, but no EX taker can slip in without
// attach.lock, which the downgrader holds) and rules out lock-order cycles
// with the store-level .lock, which prune/rmi already hold around the sweep
// while runs never take bundle locks under the store lock.

// errCacheBusy reports that a bundle lock is held by a live run (or an
// in-flight provision), so the caller must not detach or remove the bundle.
var errCacheBusy = errors.New("in use by a live run")

func attachLockPath(bundle string) string { return filepath.Join(bundle, "attach.lock") }
func runLockPath(bundle string) string { return filepath.Join(bundle, "run.lock") }

// flockFile is an open file holding (or having held) an advisory flock.
type flockFile struct {
f *os.File
}

// acquireFlock opens path (creating it if absent) and takes the flock mode
// `how` (syscall.LOCK_SH or LOCK_EX, optionally |LOCK_NB). A non-blocking
// request that loses returns errCacheBusy (wrapped with the path).
//
// A sweeper removes the whole bundle directory -- lock files included --
// while holding both locks. A racing acquirer may then have opened the path
// just before the unlink and be holding a lock on an orphaned inode no later
// process can observe. Guard against that: after locking, verify the path
// still resolves to the locked inode; otherwise retry against the recreated
// file. The retry count is a defense bound, not a correctness knob -- one
// retry per concurrent unlink is the worst case.
func acquireFlock(path string, how int) (*flockFile, error) {
for range 16 {
f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644)
if err != nil {
return nil, err
}
if err := flockRetryIntr(int(f.Fd()), how); err != nil {
f.Close()
if errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) {
return nil, fmt.Errorf("%s: %w", path, errCacheBusy)
}
return nil, fmt.Errorf("lock %s: %w", path, err)
}
var pathSt, fdSt syscall.Stat_t
if err := syscall.Stat(path, &pathSt); err != nil {
f.Close()
if errors.Is(err, syscall.ENOENT) {
continue // unlinked under us; retry on the recreated file
}
return nil, fmt.Errorf("lock %s: %w", path, err)
}
if err := syscall.Fstat(int(f.Fd()), &fdSt); err != nil {
f.Close()
return nil, fmt.Errorf("lock %s: %w", path, err)
}
if pathSt.Dev == fdSt.Dev && pathSt.Ino == fdSt.Ino {
return &flockFile{f: f}, nil
}
f.Close() // path now names a different file; lock that one instead
}
return nil, fmt.Errorf("lock %s: persistent unlink race", path)
}

// flockRetryIntr issues flock, retrying on EINTR (a blocking acquisition may
// be interrupted by the signal forwarding the run wrapper installs).
func flockRetryIntr(fd, how int) error {
for {
err := syscall.Flock(fd, how)
if !errors.Is(err, syscall.EINTR) {
return err
}
}
}

// Downgrade converts a held exclusive lock to shared. flock implements this
// as release-then-reacquire, so it is race-free only while the caller holds
// attach.lock: every exclusive taker of run.lock acquires attach.lock first,
// so none can slip into the gap.
func (l *flockFile) Downgrade() error {
return flockRetryIntr(int(l.f.Fd()), syscall.LOCK_SH)
}

// Close releases the lock and closes the file. Safe on nil and after a prior
// Close.
func (l *flockFile) Close() error {
if l == nil || l.f == nil {
return nil
}
_ = syscall.Flock(int(l.f.Fd()), syscall.LOCK_UN)
err := l.f.Close()
l.f = nil
return err
}
119 changes: 119 additions & 0 deletions cmd/elfuse-container/bundlelock_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
package main

import (
"errors"
"os"
"path/filepath"
"syscall"
"testing"
)

func TestAcquireFlockSharedCoexists(t *testing.T) {
path := filepath.Join(t.TempDir(), "run.lock")
a, err := acquireFlock(path, syscall.LOCK_SH)
if err != nil {
t.Fatal(err)
}
defer a.Close()
b, err := acquireFlock(path, syscall.LOCK_SH|syscall.LOCK_NB)
if err != nil {
t.Fatalf("second shared lock: %v, want success", err)
}
defer b.Close()
}

func TestAcquireFlockExclusiveBlockedIsCacheBusy(t *testing.T) {
path := filepath.Join(t.TempDir(), "run.lock")
a, err := acquireFlock(path, syscall.LOCK_SH)
if err != nil {
t.Fatal(err)
}
defer a.Close()
_, err = acquireFlock(path, syscall.LOCK_EX|syscall.LOCK_NB)
if !errors.Is(err, errCacheBusy) {
t.Fatalf("exclusive over shared err = %v, want errCacheBusy", err)
}

// Releasing the shared lock frees the exclusive probe.
if err := a.Close(); err != nil {
t.Fatal(err)
}
b, err := acquireFlock(path, syscall.LOCK_EX|syscall.LOCK_NB)
if err != nil {
t.Fatalf("exclusive after release: %v, want success", err)
}
defer b.Close()
}

func TestFlockDowngradeAdmitsSharedBlocksExclusive(t *testing.T) {
path := filepath.Join(t.TempDir(), "run.lock")
a, err := acquireFlock(path, syscall.LOCK_EX)
if err != nil {
t.Fatal(err)
}
defer a.Close()
if _, err := acquireFlock(path, syscall.LOCK_SH|syscall.LOCK_NB); !errors.Is(err, errCacheBusy) {
t.Fatalf("shared over exclusive err = %v, want errCacheBusy", err)
}
if err := a.Downgrade(); err != nil {
t.Fatalf("Downgrade: %v", err)
}
b, err := acquireFlock(path, syscall.LOCK_SH|syscall.LOCK_NB)
if err != nil {
t.Fatalf("shared after downgrade: %v, want success", err)
}
defer b.Close()
if _, err := acquireFlock(path, syscall.LOCK_EX|syscall.LOCK_NB); !errors.Is(err, errCacheBusy) {
t.Fatalf("exclusive after downgrade err = %v, want errCacheBusy", err)
}
}

// TestAcquireFlockUnlinkRace pins the verify-retry: when a sweeper unlinks
// the lock file while another process still holds a lock on the orphaned
// inode, a fresh acquire must land on the recreated file -- not block on or
// share fate with the orphan.
func TestAcquireFlockUnlinkRace(t *testing.T) {
path := filepath.Join(t.TempDir(), "run.lock")
orphan, err := acquireFlock(path, syscall.LOCK_EX)
if err != nil {
t.Fatal(err)
}
defer orphan.Close()
// Simulate the sweeper's RemoveAll of the bundle: the path is gone while
// the orphan's lock is still held on the old inode.
if err := os.Remove(path); err != nil {
t.Fatal(err)
}
fresh, err := acquireFlock(path, syscall.LOCK_EX|syscall.LOCK_NB)
if err != nil {
t.Fatalf("acquire after unlink: %v, want success on recreated file", err)
}
defer fresh.Close()
}

func TestFlockCloseIdempotentAndNilSafe(t *testing.T) {
path := filepath.Join(t.TempDir(), "run.lock")
a, err := acquireFlock(path, syscall.LOCK_EX)
if err != nil {
t.Fatal(err)
}
if err := a.Close(); err != nil {
t.Fatal(err)
}
if err := a.Close(); err != nil {
t.Fatalf("second Close: %v, want nil", err)
}
var nilLock *flockFile
if err := nilLock.Close(); err != nil {
t.Fatalf("nil Close: %v, want nil", err)
}
}

func TestBundleLockPaths(t *testing.T) {
if got := attachLockPath("/store/cs/sha256/ab"); got != "/store/cs/sha256/ab/attach.lock" {
t.Fatalf("attachLockPath = %q", got)
}
if got := runLockPath("/store/cs/sha256/ab"); got != "/store/cs/sha256/ab/run.lock" {
t.Fatalf("runLockPath = %q", got)
}
}
Loading
Loading