diff --git a/Makefile b/Makefile index 9a0ed40f..abd57059 100644 --- a/Makefile +++ b/Makefile @@ -197,6 +197,33 @@ $(BUILD_DIR)/test-pthread: tests/test-pthread.c | $(BUILD_DIR) @echo " CROSS $< (with -lpthread)" $(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread +# bench-mmap has a multi-threaded mmap_lock-contention section; needs -lpthread. +$(BUILD_DIR)/bench-mmap: tests/bench-mmap.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread + +# test-mmap-lazy races concurrent first touch from several threads. +$(BUILD_DIR)/test-mmap-lazy: tests/test-mmap-lazy.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread + +.PHONY: test-mmap-lazy +test-mmap-lazy: $(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-lazy + @$(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-lazy + @sh tests/test-mmap-dirty-stats.sh $(ELFUSE_BIN) \ + $(BUILD_DIR)/test-mmap-lazy + +# EL1 consumer-mmap integration/stress test. +$(BUILD_DIR)/test-mmap-fastpath: tests/test-mmap-fastpath.c | $(BUILD_DIR) + @echo " CROSS $< (with -lpthread)" + $(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< -lpthread + +.PHONY: test-mmap-fastpath +test-mmap-fastpath: $(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-fastpath + @$(ELFUSE_BIN) $(BUILD_DIR)/test-mmap-fastpath + @sh tests/test-mmap-fastpath-stats.sh $(ELFUSE_BIN) \ + $(BUILD_DIR)/test-mmap-fastpath + # test-thread-churn creates >64 threads to force thread-table slot reuse. $(BUILD_DIR)/test-thread-churn: tests/test-thread-churn.c | $(BUILD_DIR) @echo " CROSS $< (with -lpthread)" diff --git a/docs/internals.md b/docs/internals.md index fa285303..d25a78bb 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -226,6 +226,27 @@ Splitting is triggered by: block needs splitting, splits it if so, then updates the affected L3 page entries. Whole-block permission changes are done in place without splitting. +For a private anonymous mmap-arena block, the EL1 shim specializes a one-page +`PROT_READ` ↔ `PROT_READ|PROT_WRITE` transition. Arena refill prepares the +upper TTBR0 chain, a per-vCPU batch of 32 L3 table pages, and a clean-backing +bitmap for a rolling window of 32 two-megabyte blocks. A slow-path miss moves +that window to the current allocation area. If an undrained mmap publication +proves that one anonymous RW mapping owns a complete clean block, EL1 may +populate a prepared L3 page and install the first table descriptor itself. A +previously retired, empty L3 table is reused in place. Existing valid L3 leaves +need only an AP[2:1] update. + +The shim issues `TLBI RVAE1IS` for first materialization or `TLBI VAE1IS` for +an existing leaf and returns without an HVC exit. A tagged entry in the mmap +publication ring defers region metadata plus host-only PTE/dirty-index +reconciliation to the next drain. The producer-active/PT-gate handshake +prevents a host page-table writer from racing the EL1 update. Dirty backing, +but still-lazy target pages use a metadata-only tag and keep their PTE invalid; +their eventual fault drains the new protection before materializing data. +Partial ownership, an already-consumed publication, an exhausted table cache, +executable or `PROT_NONE` transitions, multi-page ranges, and non-arena mappings +retain the host path. + `mmap` itself uses a gap-finding allocator that walks the sorted region array to find free address space. `PROT_EXEC` requests go to the RX region (`MMAP_RX_BASE = 0x10000000`); other requests go to the RW region @@ -426,6 +447,14 @@ goes above the structured area, never below. Post-push masking ### `mmap` Notes +Private anonymous mappings are lazy at 2 MiB materialization granularity. A +host-side hierarchical bitmap records which low-VA 2 MiB blocks contain any +valid TTBR0 PTE, independently of the dirty-block bitmap. `munmap` and recycled +fast-path arenas use this index to visit only materialized blocks, so untouched +multi-GiB reservations have length-independent teardown. When every mapping in +a per-vCPU arena has been released and the index confirms that no PTE remains, +the arena cursor rewinds in place instead of taking a refill HVC. + Aligned file-backed `MAP_SHARED` (fixed or non-fixed) installs a real host `mmap(MAP_FIXED|MAP_SHARED, fd)` overlay onto the guest slab so the kernel page cache keeps the mapping coherent with the file (and diff --git a/docs/usage.md b/docs/usage.md index bef8aca2..7e41f093 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -29,6 +29,16 @@ only bounds a single `hv_vcpu_run()` iteration before the host regains control, which is what allows host-side timers and signals to be observed promptly. Setting `--timeout 0` disables this watchdog for long-running CPU-bound guests. +### mmap call fast path + +The aarch64 EL1 consumer fast path is enabled by default for +`mmap(NULL, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, ...)` up to +32 GiB per request. +Set `ELFUSE_MMAP_FASTPATH=0` to disable it. Unsupported mmap shapes, exhausted +arenas, and full consumption rings fall back to the normal host syscall path. +Verbose tracing, the syscall histogram, GDB, and Rosetta keep mmap on the host +path so observability and translated-guest behavior are unchanged. + ## Common Launch Patterns Run a statically linked guest binary: diff --git a/src/core/bootstrap.c b/src/core/bootstrap.c index 70bddfd5..b69c8f71 100644 --- a/src/core/bootstrap.c +++ b/src/core/bootstrap.c @@ -36,6 +36,7 @@ #include "syscall/signal.h" #include "debug/log.h" +#include "core/mmap-fastpath.h" /* Worst case: 7 fixed regions (shim, shim-data, vDSO, brk, stack, mmap RX, mmap * RW) plus up to ELF_MAX_SEGMENTS for both the executable and the interpreter. @@ -136,6 +137,26 @@ static void register_runtime_regions(guest_t *g, size_t shim_bin_len) guest_invalidate_ptes(g, 0, 0x1000); } +/* ELF/shim/stack bytes are populated directly in the slab before their page + * tables become live. Mark their semantic backing regardless of requested + * permissions: a read-only file segment is still nonzero and must be scrubbed + * if a later MAP_FIXED lazy-anonymous mapping reuses the same slab block. + * Synthetic page-table coverage for unallocated mmap space has no semantic + * region and therefore remains clean. + */ +static void mark_registered_backing_dirty(guest_t *g) +{ + for (int i = 0; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->end <= r->start) + continue; + uint64_t len = r->end - r->start; + if (r->gpa_base > g->guest_size || len > g->guest_size - r->gpa_base) + continue; + guest_dirty_mark_range(g, r->gpa_base, r->gpa_base + len); + } +} + int guest_bootstrap_probe_elf(const char *elf_path, elf_info_t *info) { memset(info, 0, sizeof(*info)); @@ -289,7 +310,14 @@ static bool build_boot_regions(mem_region_t *regions, * to the vDSO page when splitting the block; otherwise vdso_build cannot * write into it through guest_ptr. */ - if (!append_boot_region(regions, nregions, g->shim_base, + /* EL1 fast munmap walks and atomically clears the live TTBR0 tree. Give + * the page-table pool an identity VA visible only to EL1; EL0 remains + * unable to inspect or corrupt descriptors, and every guest syscall still + * rejects the encompassing infrastructure range. + */ + if (!append_boot_region(regions, nregions, g->pt_pool_base, g->pt_pool_end, + MEM_PERM_RW_EL1_ONLY) || + !append_boot_region(regions, nregions, g->shim_base, g->shim_base + shim_bin_len, MEM_PERM_RX) || /* shim_data is EL1-only: the guest must not directly read or write the * identity cache, attention flag, urandom bitmap, or ring, any of which @@ -547,6 +575,7 @@ int guest_bootstrap_prepare(guest_t *g, } register_runtime_regions(g, shim_bin_len); + mark_registered_backing_dirty(g); startup_trace_step("register_regions", t0); log_debug("TTBR0=0x%llx, IPA base=0x%llx", (unsigned long long) boot->ttbr0, @@ -732,6 +761,13 @@ int guest_bootstrap_create_vcpu(guest_t *g, */ shim_globals_set_singleton(g); + /* Publish the main vCPU's first arena only after shim_globals_init has + * cleared every recycled control slot. Verbose tracing keeps all shim + * syscall fast paths on HVC so the trace remains complete. + */ + if (!verbose) + mmap_fastpath_prepare_vcpu(g, current_thread); + /* CNTKCTL_EL1.EL0VCTEN | EL0PCTEN: allow EL0 to read {CNTVCT,CNTPCT}_EL0. * Required by the vDSO clock_gettime fast path (and is the default on * native Linux), without which the guest gets 0 back from MRS. @@ -843,6 +879,7 @@ int guest_bootstrap_rosetta_post_reset(guest_t *g, g->rosetta_guest_base - g->rosetta_va_base, ROSETTA_PATH); register_runtime_regions(g, shim_bin_len); + mark_registered_backing_dirty(g); int rosetta_argc = 0; const char **rosetta_argv = NULL; diff --git a/src/core/guest.c b/src/core/guest.c index 02250cea..7cd93bab 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -40,13 +40,15 @@ #include #include "core/guest.h" +#include "core/mmap-fastpath.h" #include "core/startup-trace.h" #include "debug/log.h" #include "utils.h" -#include "runtime/futex.h" /* futex_interrupt_request */ -#include "runtime/thread.h" /* thread_destroy_all_vcpus */ -#include "syscall/poll.h" /* wakeup_pipe_signal */ -#include "syscall/proc.h" /* proc_request_exit_group */ +#include "runtime/futex.h" /* futex_interrupt_request */ +#include "runtime/thread.h" /* thread_destroy_all_vcpus */ +#include "syscall/internal.h" /* mmap_lock (lazy fault-in) */ +#include "syscall/poll.h" /* wakeup_pipe_signal */ +#include "syscall/proc.h" /* proc_request_exit_group */ /* Per-vCPU pending TLBI request. Zero-initialized in every host pthread by * virtue of TLS default-zeroing, which maps to TLBI_NONE. @@ -86,6 +88,7 @@ static void guest_region_clear(guest_t *g); /* Forward declaration (defined in the page table section below) */ static int desc_to_perms(uint64_t desc); +static uint64_t *find_l2_entry(guest_t *g, uint64_t va); /* Page table pool allocator. */ @@ -168,11 +171,13 @@ static int compute_infra_layout(guest_t *g) return 0; } -/* Allocate a zeroed 4KiB page from the page table pool. - * Returns GPA of the page, or 0 on pool exhaustion. Acquires pt_lock - * internally. Caller typically holds mmap_lock. +/* Allocate a 4KiB page from the page table pool. Returns its GPA, or 0 on + * exhaustion. Acquires pt_lock internally; caller typically holds mmap_lock. + * Most callers need a zero-filled page because they publish only some entries. + * split_l2_block instead overwrites all 512 entries before publication and can + * skip the redundant clear. */ -static uint64_t pt_alloc_page(guest_t *g) +static uint64_t pt_alloc_page_impl(guest_t *g, bool clear) { pthread_mutex_lock(&pt_lock); if (g->pt_pool_next + PAGE_SIZE > g->pt_pool_end) { @@ -199,10 +204,50 @@ static uint64_t pt_alloc_page(guest_t *g) pt_pool_warned = true; } - /* Zero the page while still holding the lock so no other thread can observe - * a partially-zeroed page table page. - */ - memset((uint8_t *) g->host_base + gpa, 0, PAGE_SIZE); + if (clear) + memset((uint8_t *) g->host_base + gpa, 0, PAGE_SIZE); + pthread_mutex_unlock(&pt_lock); + return gpa; +} + +static uint64_t pt_alloc_page(guest_t *g) +{ + return pt_alloc_page_impl(g, true); +} + +static uint64_t pt_alloc_page_uninitialized(guest_t *g) +{ + return pt_alloc_page_impl(g, false); +} + +uint64_t guest_reserve_pt_pages_uninitialized(guest_t *g, unsigned count) +{ + if (!g || count == 0) + return 0; + uint64_t bytes = (uint64_t) count * PAGE_SIZE; + + pthread_mutex_lock(&pt_lock); + if (g->pt_pool_next > g->pt_pool_end || + bytes > g->pt_pool_end - g->pt_pool_next) { + log_error( + "guest: page table pool exhausted reserving %u pages " + "(used %llu / %llu bytes)", + count, (unsigned long long) (g->pt_pool_next - g->pt_pool_base), + (unsigned long long) (g->pt_pool_end - g->pt_pool_base)); + pthread_mutex_unlock(&pt_lock); + return 0; + } + uint64_t gpa = g->pt_pool_next; + g->pt_pool_next += bytes; + + uint64_t used = g->pt_pool_next - g->pt_pool_base; + uint64_t total = g->pt_pool_end - g->pt_pool_base; + if (!pt_pool_warned && used > (total * 4 / 5)) { + log_debug("guest: page table pool at %llu%% (%llu / %llu bytes)", + (unsigned long long) (used * 100 / total), + (unsigned long long) used, (unsigned long long) total); + pt_pool_warned = true; + } pthread_mutex_unlock(&pt_lock); return gpa; } @@ -252,6 +297,49 @@ static inline void pte_store_release(uint64_t *entry, uint64_t desc) __atomic_store_n(entry, desc, __ATOMIC_RELEASE); } +/* Low-VA TTBR0 occupancy index. All mutators run under mmap_lock (or during + * single-threaded bootstrap), so plain bitmap operations are sufficient. The + * page-table descriptors themselves remain release-published for lock-free + * guest walkers; this host-only index is never consulted by a vCPU. + */ +static inline bool guest_pte_present_index(uint64_t va, uint64_t *block_out) +{ + if (va >= GUEST_PTE_PRESENT_LIMIT) + return false; + *block_out = va / BLOCK_2MIB; + return true; +} + +static inline void guest_pte_present_set(guest_t *g, uint64_t va) +{ + uint64_t block; + if (!guest_pte_present_index(va, &block)) + return; + uint64_t word = block >> 6; + g->pte_present_blocks[word] |= 1ULL << (block & 63); + g->pte_present_summary[word >> 6] |= 1ULL << (word & 63); +} + +static inline void guest_pte_present_clear(guest_t *g, uint64_t va) +{ + uint64_t block; + if (!guest_pte_present_index(va, &block)) + return; + uint64_t word = block >> 6; + g->pte_present_blocks[word] &= ~(1ULL << (block & 63)); + if (g->pte_present_blocks[word] == 0) + g->pte_present_summary[word >> 6] &= ~(1ULL << (word & 63)); +} + +static inline bool guest_pte_present_test(const guest_t *g, uint64_t va) +{ + uint64_t block; + if (!guest_pte_present_index(va, &block)) + return false; + return (g->pte_present_blocks[block >> 6] & + (1ULL << (block & 63))) != 0; +} + /* Public API */ /* FEAT_TLBIRANGE probe -- runs exactly once via pthread_once. ARMv8.4 @@ -474,6 +562,7 @@ int guest_init(guest_t *g, uint64_t size, uint32_t ipa_bits) */ g->segments[0] = (hvf_segment_t) {.ipa = GUEST_IPA_BASE, .len = size}; g->n_segments = 1; + pthread_cond_init(&g->materialize_cond, NULL); return 0; } @@ -586,6 +675,7 @@ int guest_init_from_shm(guest_t *g, */ g->segments[0] = (hvf_segment_t) {.ipa = GUEST_IPA_BASE, .len = size}; g->n_segments = 1; + pthread_cond_init(&g->materialize_cond, NULL); log_debug( "guest: CoW fork: mapped %llu GiB from shm " @@ -707,6 +797,7 @@ void guest_destroy(guest_t *g) close(g->shm_fd); g->shm_fd = -1; } + pthread_cond_destroy(&g->materialize_cond); } /* Check whether a candidate IPA range [gpa, gpa+size) overlaps the primary @@ -1131,9 +1222,11 @@ int guest_map_va_range(guest_t *g, * guest_split_block instead. Skip silently to mirror upstream's * sys_mmap_high_va "reuse existing GPA" behavior. */ + guest_pte_present_set(g, va); continue; } pte_store_release(&l2[l2_idx], make_block_desc(cur_gpa, perms)); + guest_pte_present_set(g, va); if (!bcast) { if (va < changed_lo) changed_lo = va; @@ -1441,11 +1534,11 @@ static uint64_t gva_contiguous_avail(const guest_t *g, * (MEM_PERM_R/W/X bitmask). The walk continues across adjacent L2/L3 entries * until a mapping, permission, or physical-contiguity break is found. */ -static void *gva_resolve_perm(const guest_t *g, - uint64_t gva, - uint64_t *avail, - int required_perms, - uint64_t avail_limit) +static void *gva_resolve_perm_walk(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms, + uint64_t avail_limit) { /* Always walk page tables to enforce permissions. The guest slab is * identity-mapped (GVA == GPA == offset), but L2 block descriptors carry @@ -1491,14 +1584,114 @@ static void *gva_resolve_perm(const guest_t *g, return NULL; } +/* Host-access fault-in for lazy (deferred-PTE) regions. + * + * A syscall may target guest memory the guest itself has never touched: a + * fresh calloc()-style arena handed straight to read(2), a futex word inside + * an untouched mapping, an iovec into a new heap chunk. The guest-fault path + * (guest_materialize_lazy via the EL1 shim) never runs for those, so the + * page-table walk in gva_resolve_perm_walk fails even though the access is + * legal. Materialize the touched blocks here, then let the caller re-walk. + * + * Locking: takes mmap_lock; callers of the resolve API that already hold + * mmap_lock must use the _nofault variants. Do not call while holding any + * lock that ranks after mmap_lock in the ordering (syscall/internal.h); + * callers that resolve under such locks (futex bucket paths) pre-fault at + * their lock-free entry points so the hook never engages there. + * + * TLBI: guest_materialize_lazy accumulates TLBI requests in the calling + * thread's per-vCPU slot. On a vCPU thread the syscall epilogue emits them. + * On non-vCPU threads the request is lost, which is self-healing: a vCPU + * that still holds a stale negative TLB entry re-faults, and the + * already-valid early-return in guest_materialize_lazy re-issues a page + * TLBI for it without re-zeroing. + * + * Returns 0 if at least one block in [gva, gva+len) is now materialized (or + * already was), -1 if the range intersects no materializable lazy region. + */ +int guest_lazy_faultin_locked(const guest_t *cg, uint64_t gva, uint64_t len) +{ + /* The lazy machinery mutates page tables; the const on the resolve API + * reflects the pure-walk fast path, not this slow path. + */ + guest_t *g = (guest_t *) (uintptr_t) cg; + + if (gva >= g->guest_size) + return -1; /* High-VA / non-identity ranges are never lazy. */ + if (len == 0) + len = 1; + uint64_t end = (len > g->guest_size - gva) ? g->guest_size : gva + len; + + int rc = -1; + for (uint64_t b = gva & ~(uint64_t) (BLOCK_2MIB - 1); b < end; + b += BLOCK_2MIB) { + uint64_t probe = (b > gva) ? b : gva; + if (guest_materialize_lazy(g, probe) == 0) + rc = 0; + } + return rc; +} + +static int gva_lazy_faultin(const guest_t *cg, + uint64_t gva, + uint64_t len, + int required_perms) +{ + (void) required_perms; /* Region prot gates the retry walk, not this. */ + + int rc; + mmap_lock_acquire((guest_t *) (uintptr_t) cg); + rc = guest_lazy_faultin_locked(cg, gva, len); + mmap_lock_release(); + return rc; +} + +int guest_lazy_faultin(const guest_t *g, uint64_t gva, uint64_t len) +{ + return gva_lazy_faultin(g, gva, len, MEM_PERM_R); +} + +static void *gva_resolve_perm(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms, + uint64_t avail_limit, + bool allow_faultin) +{ + void *ptr = + gva_resolve_perm_walk(g, gva, avail, required_perms, avail_limit); + if (!allow_faultin) + return ptr; + + /* Window the fault-in to what the caller actually needs. Length-less + * resolves (guest_ptr / guest_ptr_avail) materialize a single block; + * their callers iterate and re-enter here per chunk. + */ + uint64_t want = (avail_limit == UINT64_MAX) ? 1 : avail_limit; + if (!ptr) { + if (gva_lazy_faultin(g, gva, want, required_perms) < 0) + return NULL; + return gva_resolve_perm_walk(g, gva, avail, required_perms, + avail_limit); + } + if (avail && *avail < want && gva <= UINT64_MAX - *avail && + gva_lazy_faultin(g, gva + *avail, want - *avail, required_perms) == 0) { + void *again = + gva_resolve_perm_walk(g, gva, avail, required_perms, avail_limit); + if (again) + ptr = again; + } + return ptr; +} + void *guest_ptr(const guest_t *g, uint64_t gva) { - return gva_resolve_perm(g, gva, NULL, MEM_PERM_R, UINT64_MAX); + return gva_resolve_perm(g, gva, NULL, MEM_PERM_R, UINT64_MAX, true); } void *guest_ptr_w(const guest_t *g, uint64_t gva) { - return gva_resolve_perm(g, gva, NULL, MEM_PERM_W, UINT64_MAX); + return gva_resolve_perm(g, gva, NULL, MEM_PERM_W, UINT64_MAX, true); } void *guest_ptr_avail(const guest_t *g, @@ -1506,7 +1699,19 @@ void *guest_ptr_avail(const guest_t *g, uint64_t *avail, int required_perms) { - return gva_resolve_perm(g, gva, avail, required_perms, UINT64_MAX); + return gva_resolve_perm(g, gva, avail, required_perms, UINT64_MAX, true); +} + +/* Pure page-table walk without lazy fault-in. For callers that already hold + * mmap_lock (e.g. the stale-TLB re-walk in the EL0 fault handler, which runs + * after guest_materialize_lazy has already been consulted). + */ +void *guest_ptr_avail_nofault(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms) +{ + return gva_resolve_perm(g, gva, avail, required_perms, UINT64_MAX, false); } void *guest_ptr_bound(const guest_t *g, @@ -1515,7 +1720,7 @@ void *guest_ptr_bound(const guest_t *g, int required_perms, uint64_t len_limit) { - return gva_resolve_perm(g, gva, avail, required_perms, len_limit); + return gva_resolve_perm(g, gva, avail, required_perms, len_limit, true); } static inline int guest_copy(const guest_t *g, @@ -1523,7 +1728,8 @@ static inline int guest_copy(const guest_t *g, void *dst, const void *src, size_t len, - int required_perms) + int required_perms, + bool allow_faultin) { if (len == 0) return 0; @@ -1537,7 +1743,7 @@ static inline int guest_copy(const guest_t *g, while (copied < len) { uint64_t avail; void *ptr = gva_resolve_perm(g, gva + copied, &avail, required_perms, - (uint64_t) (len - copied)); + (uint64_t) (len - copied), allow_faultin); if (!ptr) return -1; size_t chunk = len - copied; @@ -1554,7 +1760,7 @@ static inline int guest_copy(const guest_t *g, int guest_read(const guest_t *g, uint64_t gva, void *dst, size_t len) { - return guest_copy(g, gva, dst, NULL, len, MEM_PERM_R); + return guest_copy(g, gva, dst, NULL, len, MEM_PERM_R, true); } int guest_read_small(const guest_t *g, uint64_t gva, void *dst, size_t len) @@ -1571,7 +1777,12 @@ int guest_read_small(const guest_t *g, uint64_t gva, void *dst, size_t len) int guest_write(guest_t *g, uint64_t gva, const void *src, size_t len) { - return guest_copy(g, gva, NULL, src, len, MEM_PERM_W); + return guest_copy(g, gva, NULL, src, len, MEM_PERM_W, true); +} + +int guest_write_nofault(guest_t *g, uint64_t gva, const void *src, size_t len) +{ + return guest_copy(g, gva, NULL, src, len, MEM_PERM_W, false); } int guest_write_small(guest_t *g, uint64_t gva, const void *src, size_t len) @@ -1597,7 +1808,7 @@ int guest_read_str(const guest_t *g, uint64_t gva, char *dst, size_t max) break; uint64_t avail; void *ptr = gva_resolve_perm(g, gva + copied, &avail, MEM_PERM_R, - (uint64_t) (limit - copied)); + (uint64_t) (limit - copied), true); if (!ptr) break; @@ -1668,6 +1879,7 @@ void guest_reset(guest_t *g) if (gpa > g->guest_size || len > g->guest_size - gpa) continue; /* backing lies outside the primary slab */ memset((uint8_t *) g->host_base + gpa, 0, len); + guest_dirty_clear_zeroed_range(g, gpa, gpa + len); } /* Zero page table pool (not tracked in region array) */ @@ -1703,7 +1915,11 @@ void guest_reset(guest_t *g) g->mmap_rx_end = MMAP_RX_INITIAL_END; g->mmap_rw_gap_hint = 0; g->mmap_rx_gap_hint = 0; + atomic_store_explicit(&g->mmap_fastpath_active_slots, 0, + memory_order_relaxed); g->ttbr0 = 0; + memset(g->pte_present_blocks, 0, sizeof(g->pte_present_blocks)); + memset(g->pte_present_summary, 0, sizeof(g->pte_present_summary)); tlbi_request_clear(); g->elf_load_min = ELF_DEFAULT_BASE; @@ -2573,6 +2789,9 @@ uint64_t guest_build_page_tables(guest_t *g, const mem_region_t *regions, int n) { uint64_t base = g->ipa_base; + memset(g->pte_present_blocks, 0, sizeof(g->pte_present_blocks)); + memset(g->pte_present_summary, 0, sizeof(g->pte_present_summary)); + /* Allocate L0 table */ uint64_t l0_gpa = pt_alloc_page(g); if (!l0_gpa) @@ -2668,6 +2887,8 @@ uint64_t guest_build_page_tables(guest_t *g, const mem_region_t *regions, int n) * to the primary-buffer GPA where the bytes actually are. */ l2[l2_idx] = make_block_desc(output_ipa, block_perms); + if (lookup_addr >= base) + guest_pte_present_set(g, lookup_addr - base); } } @@ -2681,6 +2902,11 @@ uint64_t guest_build_page_tables(guest_t *g, const mem_region_t *regions, int n) if (!finalize_block_perms(g, regions, n)) return 0; + for (int r = 0; r < n; r++) { + if (regions[r].perms & MEM_PERM_W) + guest_dirty_mark_range(g, regions[r].gpa_start, regions[r].gpa_end); + } + guest_pt_gen_bump(g); return ttbr0; } @@ -2798,9 +3024,12 @@ int guest_extend_page_tables(guest_t *g, * an explicit PT_VALID test so the intent survives a future * descriptor-bit renumbering. */ - if (l2[l2_idx] & PT_VALID) + if (l2[l2_idx] & PT_VALID) { + guest_pte_present_set(g, addr); continue; + } pte_store_release(&l2[l2_idx], make_block_desc(ipa, perms)); + guest_pte_present_set(g, addr); if (!bcast) { if (addr < changed_lo) changed_lo = addr; @@ -2816,10 +3045,172 @@ int guest_extend_page_tables(guest_t *g, return 0; } +int guest_prepare_l2_tables(guest_t *g, uint64_t start, uint64_t end) +{ + if (!g || !g->ttbr0 || start >= end || end > g->guest_size || + end > UINT64_MAX - (BLOCK_2MIB - 1)) + return -1; + + uint64_t base = g->ipa_base; + uint64_t *l0 = pt_at(g, g->ttbr0 - base); + uint64_t addr_start = ALIGN_2MIB_DOWN(start); + uint64_t addr_end = ALIGN_2MIB_UP(end); + + for (uint64_t addr = addr_start; addr < addr_end;) { + uint64_t ipa = base + addr; + unsigned l0_idx = (unsigned) (ipa / (512ULL * BLOCK_1GIB)); + if (l0_idx >= 512) + return -1; + + if (!(pte_load_acquire(&l0[l0_idx]) & PT_VALID)) { + uint64_t l1_gpa = pt_alloc_page(g); + if (!l1_gpa) + return -1; + pte_store_release(&l0[l0_idx], + (base + l1_gpa) | PT_VALID | PT_TABLE); + } + uint64_t l0e = pte_load_acquire(&l0[l0_idx]); + if ((l0e & 3) != (PT_VALID | PT_TABLE)) + return -1; + uint64_t *l1 = pt_at(g, (l0e & 0xFFFFFFFFF000ULL) - base); + unsigned l1_idx = + (unsigned) ((ipa % (512ULL * BLOCK_1GIB)) / BLOCK_1GIB); + + if (!(pte_load_acquire(&l1[l1_idx]) & PT_VALID)) { + uint64_t l2_gpa = pt_alloc_page(g); + if (!l2_gpa) + return -1; + pte_store_release(&l1[l1_idx], + (base + l2_gpa) | PT_VALID | PT_TABLE); + } + uint64_t l1e = pte_load_acquire(&l1[l1_idx]); + if ((l1e & 3) != (PT_VALID | PT_TABLE)) + return -1; + + uint64_t next = ALIGN_UP(addr + 1, BLOCK_1GIB); + if (next <= addr || next > addr_end) + next = addr_end; + addr = next; + } + return 0; +} + +static uint64_t guest_va_next_page_table_block(const guest_t *g, + uint64_t va, + uint64_t end) +{ + if (!g || !g->ttbr0) + return end; + uint64_t base = g->ipa_base; + uint64_t *l0 = pt_at(g, g->ttbr0 - base); + if (!l0) + return end; + + va &= ~(uint64_t) (BLOCK_2MIB - 1); + while (va < end) { + uint64_t ipa = base + va; + unsigned l0_idx = (unsigned) (ipa / (512ULL * BLOCK_1GIB)); + if (l0_idx >= 512) + return end; + if (!(l0[l0_idx] & PT_VALID)) { + uint64_t next_ipa = (uint64_t) (l0_idx + 1) * 512ULL * BLOCK_1GIB; + if (next_ipa <= ipa || next_ipa - base <= va) + return end; /* wrap guard */ + va = next_ipa - base; + continue; + } + uint64_t *l1 = pt_at(g, (l0[l0_idx] & 0xFFFFFFFFF000ULL) - base); + if (!l1) + return end; + unsigned l1_idx = + (unsigned) ((ipa % (512ULL * BLOCK_1GIB)) / BLOCK_1GIB); + if (!(l1[l1_idx] & PT_VALID)) { + uint64_t next_ipa = (ipa / BLOCK_1GIB + 1) * BLOCK_1GIB; + if (next_ipa <= ipa || next_ipa - base <= va) + return end; + va = next_ipa - base; + continue; + } + return va; + } + return end; +} + +static uint64_t guest_va_next_indexed_block(const guest_t *g, + uint64_t va, + uint64_t end) +{ + if (va >= end || va >= GUEST_PTE_PRESENT_LIMIT) + return end; + if (end > GUEST_PTE_PRESENT_LIMIT) + end = GUEST_PTE_PRESENT_LIMIT; + + uint64_t first_block = va / BLOCK_2MIB; + uint64_t last_block = (end - 1) / BLOCK_2MIB; + uint64_t first_word = first_block >> 6; + uint64_t last_word = last_block >> 6; + uint64_t bits = g->pte_present_blocks[first_word] & + (~0ULL << (first_block & 63)); + if (first_word == last_word && (last_block & 63) != 63) + bits &= (1ULL << ((last_block & 63) + 1)) - 1; + if (bits) + return (first_word * 64 + (uint64_t) __builtin_ctzll(bits)) * + BLOCK_2MIB; + + uint64_t word = first_word + 1; + while (word <= last_word) { + uint64_t summary_word = word >> 6; + uint64_t summary_last = last_word >> 6; + uint64_t summary = g->pte_present_summary[summary_word] & + (~0ULL << (word & 63)); + if (summary_word == summary_last && (last_word & 63) != 63) + summary &= (1ULL << ((last_word & 63) + 1)) - 1; + if (summary) { + uint64_t present_word = summary_word * 64 + + (uint64_t) __builtin_ctzll(summary); + uint64_t present = g->pte_present_blocks[present_word]; + if (present_word == last_word && (last_block & 63) != 63) + present &= (1ULL << ((last_block & 63) + 1)) - 1; + if (present) + return (present_word * 64 + + (uint64_t) __builtin_ctzll(present)) * BLOCK_2MIB; + } + if (summary_word == summary_last) + break; + word = (summary_word + 1) * 64; + } + return end; +} + +uint64_t guest_va_next_present_block(const guest_t *g, + uint64_t va, + uint64_t end) +{ + if (!g || va >= end) + return end; + + if (va < GUEST_PTE_PRESENT_LIMIT) { + uint64_t low_end = end < GUEST_PTE_PRESENT_LIMIT + ? end + : GUEST_PTE_PRESENT_LIMIT; + uint64_t next = guest_va_next_indexed_block(g, va, low_end); + if (next < low_end || end <= GUEST_PTE_PRESENT_LIMIT) + return next; + va = GUEST_PTE_PRESENT_LIMIT; + } + + /* Non-identity/high-VA mappings live outside the compact low-VA index. + * Preserve the page-table walker for those uncommon ranges. + */ + return guest_va_next_page_table_block(g, va, end); +} + bool guest_va_block_mapped(const guest_t *g, uint64_t va) { if (!g || !g->ttbr0 || (va & (BLOCK_2MIB - 1))) return false; + if (va < GUEST_PTE_PRESENT_LIMIT) + return guest_pte_present_test(g, va); uint64_t base = g->ipa_base; uint64_t *l0 = pt_at(g, g->ttbr0 - base); @@ -2848,6 +3239,71 @@ bool guest_va_block_mapped(const guest_t *g, uint64_t va) return (l2[l2_idx] & PT_VALID) != 0; } +void guest_rebuild_pte_present(guest_t *g) +{ + if (!g) + return; + memset(g->pte_present_blocks, 0, sizeof(g->pte_present_blocks)); + memset(g->pte_present_summary, 0, sizeof(g->pte_present_summary)); + if (!g->ttbr0) + return; + + uint64_t limit = g->guest_size; + if (limit > GUEST_PTE_PRESENT_LIMIT) + limit = GUEST_PTE_PRESENT_LIMIT; + for (uint64_t va = 0; va < limit; va += BLOCK_2MIB) { + uint64_t *l2_entry = find_l2_entry(g, va); + if (!l2_entry || !(*l2_entry & PT_VALID)) + continue; + if ((*l2_entry & 3) == 1) { + guest_pte_present_set(g, va); + continue; + } + uint64_t l3_ipa = *l2_entry & 0xFFFFFFFFF000ULL; + uint64_t *l3 = pt_at(g, l3_ipa - g->ipa_base); + if (!l3) + continue; + for (unsigned i = 0; i < BLOCK_2MIB / PAGE_SIZE; i++) { + if (l3[i] & PT_VALID) { + guest_pte_present_set(g, va); + break; + } + } + } +} + +void guest_retire_ptes_committed(guest_t *g, uint64_t start, uint64_t end) +{ + if (!g || end <= start) + return; + uint64_t block = ALIGN_2MIB_DOWN(start); + while (block < end) { + bool present = false; + uint64_t *l2_entry = find_l2_entry(g, block); + if (l2_entry && (*l2_entry & PT_VALID)) { + if ((*l2_entry & 3) == 1) { + present = true; + } else { + uint64_t l3_ipa = *l2_entry & 0xFFFFFFFFF000ULL; + uint64_t *l3 = pt_at(g, l3_ipa - g->ipa_base); + if (l3) { + for (unsigned i = 0; i < BLOCK_2MIB / PAGE_SIZE; i++) { + if (pte_load_acquire(&l3[i]) & PT_VALID) { + present = true; + break; + } + } + } + } + } + if (present) + guest_pte_present_set(g, block); + else + guest_pte_present_clear(g, block); + block += BLOCK_2MIB; + } +} + /* L3 page table splitting. */ /* L3 page descriptor: bits[1:0]=11 = valid page at level 3. This is distinct @@ -2971,7 +3427,7 @@ static int split_l2_block(guest_t *g, uint64_t *l2_entry) int old_perms = desc_to_perms(*l2_entry); - uint64_t l3_gpa = pt_alloc_page(g); + uint64_t l3_gpa = pt_alloc_page_uninitialized(g); if (!l3_gpa) return -1; uint64_t *l3 = pt_at(g, l3_gpa); @@ -3004,6 +3460,7 @@ int guest_split_block(guest_t *g, uint64_t block_gpa) int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) { uint64_t base = g->ipa_base; + bool any_changed = false; /* Page-align the range. The ALIGN_UP step on end could wrap to 0 for inputs * within PAGE_SIZE-1 of UINT64_MAX, silently turning the invalidation into @@ -3018,10 +3475,21 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) return 0; for (uint64_t addr = start; addr < end;) { + uint64_t indexed_block = ALIGN_2MIB_DOWN(addr); + if (indexed_block < GUEST_PTE_PRESENT_LIMIT && + !guest_pte_present_test(g, indexed_block)) { + addr = guest_va_next_present_block( + g, indexed_block + BLOCK_2MIB, end); + continue; + } uint64_t *l2_entry = find_l2_entry(g, addr); if (!l2_entry) { - /* No L2 entry (already unmapped); skip this 2MiB block */ - addr = ALIGN_2MIB_UP(addr + 1); + /* No L2 table: nothing to invalidate in this block. The low-VA + * occupancy index jumps directly to the next block containing a + * valid PTE; high VA retains the page-table hierarchy fallback. + */ + guest_pte_present_clear(g, indexed_block); + addr = guest_va_next_present_block(g, ALIGN_2MIB_UP(addr + 1), end); continue; } @@ -3030,6 +3498,7 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) /* Not mapped at all: skip */ if (!(*l2_entry & 1)) { + guest_pte_present_clear(g, block_start); addr = block_end; continue; } @@ -3043,6 +3512,8 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) * broadcast. */ pte_store_release(l2_entry, 0); + guest_pte_present_clear(g, block_start); + any_changed = true; tlbi_request_range(base + block_start, base + block_end); addr = block_end; continue; @@ -3068,12 +3539,14 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) uint64_t page_end = (end < block_end) ? end : block_end; uint64_t changed_lo = UINT64_MAX, changed_hi = 0; bool bcast = tlbi_request_is_broadcast(); + bool block_changed = false; for (uint64_t pa = page_start; pa < page_end; pa += PAGE_SIZE) { unsigned l3_idx = (unsigned) (((base + pa) % BLOCK_2MIB) / PAGE_SIZE); if (l3[l3_idx] != 0) { pte_store_release(&l3[l3_idx], 0); /* Invalid descriptor */ + block_changed = true; if (!bcast) { if (pa < changed_lo) changed_lo = pa; @@ -3085,10 +3558,27 @@ int guest_invalidate_ptes(guest_t *g, uint64_t start, uint64_t end) if (!bcast && changed_hi > changed_lo) tlbi_request_range(base + changed_lo, base + changed_hi); + if (block_changed) + any_changed = true; + + bool block_present = false; + if (page_start > block_start || page_end < block_end) { + for (unsigned i = 0; i < BLOCK_2MIB / PAGE_SIZE; i++) { + if (l3[i] & PT_VALID) { + block_present = true; + break; + } + } + } + if (block_present) + guest_pte_present_set(g, block_start); + else + guest_pte_present_clear(g, block_start); addr = page_end; } - guest_pt_gen_bump(g); + if (any_changed) + guest_pt_gen_bump(g); return 0; } @@ -3164,6 +3654,7 @@ int guest_update_perms(guest_t *g, uint64_t start, uint64_t end, int perms) pte_store_release(l2_entry, make_block_desc(ipa, perms)); tlbi_request_range(base + block_start, base + block_end); } + guest_pte_present_set(g, block_start); addr = block_end; continue; } @@ -3237,9 +3728,12 @@ int guest_update_perms(guest_t *g, uint64_t start, uint64_t end, int perms) if (!bcast && changed_hi > changed_lo) tlbi_request_range(base + changed_lo, base + changed_hi); + guest_pte_present_set(g, block_start); addr = page_end; } + if (perms & MEM_PERM_W) + guest_dirty_mark_range(g, start, end); guest_pt_gen_bump(g); return 0; } @@ -3317,19 +3811,136 @@ int guest_install_va_pages(guest_t *g, changed_hi = v + PAGE_SIZE; } } + guest_pte_present_set(g, v); } if (!bcast && changed_hi > changed_lo) tlbi_request_range(changed_lo, changed_hi); + if (perms & MEM_PERM_W) + guest_dirty_mark_range(g, gpa, gpa + length); guest_pt_gen_bump(g); return 0; } -/* Lazy page materialization for MAP_NORESERVE. */ +/* Lazy page materialization for deferred-PTE (private anonymous / + * MAP_NORESERVE) regions. + */ -int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) +bool guest_block_may_be_dirty(const guest_t *g, uint64_t block_start) { - /* Find the noreserve region containing this offset */ + if (!g || block_start >= g->guest_size) + return true; + uint64_t block = block_start / BLOCK_2MIB; + return (g->dirty_blocks[block >> 6] & (1ULL << (block & 63))) != 0; +} + +void guest_dirty_mark_range(guest_t *g, uint64_t start, uint64_t end) +{ + if (!g || end <= start || start >= g->guest_size) + return; + if (end > g->guest_size) + end = g->guest_size; + uint64_t first = start / BLOCK_2MIB; + uint64_t last = (end - 1) / BLOCK_2MIB; + for (uint64_t block = first; block <= last; block++) + g->dirty_blocks[block >> 6] |= 1ULL << (block & 63); +} + +void guest_dirty_clear_zeroed_range(guest_t *g, uint64_t start, uint64_t end) +{ + if (!g || end <= start || start >= g->guest_size) + return; + if (end > g->guest_size) + end = g->guest_size; + uint64_t first = ALIGN_2MIB_UP(start); + uint64_t last = ALIGN_2MIB_DOWN(end); + for (uint64_t addr = first; addr < last; addr += BLOCK_2MIB) { + uint64_t block = addr / BLOCK_2MIB; + g->dirty_blocks[block >> 6] &= ~(1ULL << (block & 63)); + } +} + +static bool materialize_claim_overlaps(const guest_materialize_claim_t *claim, + uint64_t start, + uint64_t end) +{ + return claim->active && start < claim->end && end > claim->start; +} + +void guest_materialize_wait_range_locked(guest_t *g, + uint64_t start, + uint64_t end) +{ + if (!g || end <= start) + return; + for (;;) { + bool overlap = false; + for (int i = 0; i < GUEST_MATERIALIZE_CLAIMS; i++) { + if (materialize_claim_overlaps(&g->materialize_claims[i], start, + end)) { + overlap = true; + break; + } + } + if (!overlap) + return; + mmap_lock_cond_wait(g, &g->materialize_cond); + } +} + +void guest_materialize_wait_all_locked(guest_t *g) +{ + guest_materialize_wait_range_locked(g, 0, UINT64_MAX); +} + +static int materialize_claim_alloc_locked(guest_t *g, + uint64_t start, + uint64_t end) +{ + guest_materialize_wait_range_locked(g, start, end); + for (int i = 0; i < GUEST_MATERIALIZE_CLAIMS; i++) { + guest_materialize_claim_t *claim = &g->materialize_claims[i]; + if (!claim->active) { + claim->start = start; + claim->end = end; + claim->active = true; + return i; + } + } + return -1; +} + +static void materialize_claim_release_locked(guest_t *g, int slot) +{ + if (slot < 0) + return; + g->materialize_claims[slot].active = false; + pthread_cond_broadcast(&g->materialize_cond); +} + +/* Whether the 4KiB page containing va has a valid stage-1 descriptor. Callers + * must hold mmap_lock; used to detect blocks a concurrent fault already + * materialized. + */ +bool guest_va_pte_valid(guest_t *g, uint64_t va) +{ + uint64_t *l2_entry = find_l2_entry(g, va); + if (!l2_entry || !(*l2_entry & PT_VALID)) + return false; + if ((*l2_entry & 3) == 1) + return true; /* 2MiB block descriptor */ + uint64_t *l3 = pt_at(g, (*l2_entry & 0xFFFFFFFFF000ULL) - g->ipa_base); + if (!l3) + return false; + unsigned l3_idx = + (unsigned) (((g->ipa_base + va) % BLOCK_2MIB) / PAGE_SIZE); + return (l3[l3_idx] & PT_VALID) != 0; +} + +static int guest_materialize_lazy_one(guest_t *g, uint64_t fault_offset) +{ +retry:; + /* Find the lazy region containing this offset */ const guest_region_t *region = NULL; for (int i = 0; i < g->nregions; i++) { if (g->regions[i].start <= fault_offset && @@ -3340,7 +3951,36 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) } if (!region) - return -1; /* Not a noreserve region */ + return -1; /* Not a lazy region */ + + /* PROT_NONE is a reservation, not a mapping: a fault inside it is a + * genuine SIGSEGV, never a materialization request. Without this check a + * PROT_NONE|MAP_NORESERVE region would be silently granted read + * permission by the perms fallback below. + */ + if (region->prot == LINUX_PROT_NONE) + return -1; + + uint64_t block_start = fault_offset & ~(BLOCK_2MIB - 1); + uint64_t block_end = block_start + BLOCK_2MIB; + if (block_end > g->guest_size) + block_end = g->guest_size; + + /* Already materialized: another thread (concurrent guest fault or a + * host-side fault-in on a syscall path) completed this block while this + * vCPU was queued on mmap_lock, and the guest may have written real data + * through the new PTEs since. Running the memset below again would wipe + * those writes. The fault that got us here is then either a stale + * negative TLB entry or an in-flight retry. Invalidate the whole + * materialization block so this path follows the same one-RVAE-per-block + * contract as a newly installed block. + */ + if (guest_va_pte_valid(g, fault_offset)) { + mmap_fastpath_note_materialized_locked(g, block_start, block_end); + g->materialize_stats[GUEST_MATERIALIZE_ALREADY_VALID]++; + tlbi_request_range(g->ipa_base + block_start, g->ipa_base + block_end); + return 0; + } /* Materialize one 2MiB block containing the fault address. This is the * smallest granule that guest_extend_page_tables works with. For the common @@ -3348,12 +3988,17 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) * trade-off: it avoids over-committing the large reservation while keeping * the fault rate manageable. */ - uint64_t block_start = fault_offset & ~(BLOCK_2MIB - 1); - uint64_t block_end = block_start + BLOCK_2MIB; - - /* Clamp to guest size */ - if (block_end > g->guest_size) - block_end = g->guest_size; + /* A sibling may be zeroing another window in this block without the lock. + * Wait before inspecting regions/PTEs, then restart because a mutator that + * was itself waiting may have changed the region layout first. + */ + for (int i = 0; i < GUEST_MATERIALIZE_CLAIMS; i++) { + if (materialize_claim_overlaps(&g->materialize_claims[i], block_start, + block_end)) { + guest_materialize_wait_range_locked(g, block_start, block_end); + goto retry; + } + } uint64_t materialize_start = (block_start > region->start) ? block_start : region->start; @@ -3374,18 +4019,70 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) if (perms == 0) perms = MEM_PERM_R; /* At minimum readable */ + /* Zero the window BEFORE any PTE becomes valid. The moment a descriptor + * is published, sibling vCPUs with no stale TLB entry can write through + * it without ever faulting; zeroing afterwards (the historical order) + * would wipe such a write. The slab is host memory, so zeroing needs no + * PTEs, and every writer that could touch the window first goes through + * mmap_lock (guest faults and host-side fault-in alike), so nothing can + * write between this memset and the descriptor stores below. Skip pages + * that are already valid: they belong to a previously materialized + * neighbor in the same block and may hold live data. + */ + int claim_slot = -1; + bool dirty = guest_block_may_be_dirty(g, block_start); + if (dirty) { + uint64_t zero_pages[8] = {0}; + bool any_valid = false; + for (uint64_t pg = materialize_start; pg < materialize_end; + pg += PAGE_SIZE) { + unsigned page = (unsigned) ((pg - block_start) / PAGE_SIZE); + if (guest_va_pte_valid(g, pg)) + any_valid = true; + else + zero_pages[page >> 6] |= 1ULL << (page & 63); + } + + claim_slot = materialize_claim_alloc_locked(g, block_start, block_end); + if (claim_slot >= 0) + mmap_lock_drop_keep_gate(); + for (unsigned page = 0; page < 512;) { + if (!(zero_pages[page >> 6] & (1ULL << (page & 63)))) { + page++; + continue; + } + unsigned first = page; + do { + page++; + } while (page < 512 && + (zero_pages[page >> 6] & (1ULL << (page & 63)))); + memset((uint8_t *) g->host_base + block_start + + (uint64_t) first * PAGE_SIZE, + 0, (uint64_t) (page - first) * PAGE_SIZE); + } + if (claim_slot >= 0) + mmap_lock_reacquire_with_gate(g); + if (!any_valid && materialize_start == block_start && + materialize_end == block_end) + guest_dirty_clear_zeroed_range(g, block_start, block_end); + } + /* Create page table entries. guest_extend_page_tables creates L2 block * descriptors but skips existing table descriptors (L2->L3 splits). * guest_update_perms handles the L3 case: if guest_invalidate_ptes * previously split the block and invalidated the L3 entries, update_perms * recreates them with correct perms. */ - if (guest_extend_page_tables(g, block_start, block_end, perms) < 0) + if (guest_extend_page_tables(g, block_start, block_end, perms) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } if (partial_block) { - if (guest_split_block(g, block_start) < 0) + if (guest_split_block(g, block_start) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } /* If this block had no page-table entry before the lazy fault, * guest_extend_page_tables() necessarily created a full 2MiB block. @@ -3395,25 +4092,101 @@ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) */ if (!had_mapping) { if (block_start < materialize_start && - guest_invalidate_ptes(g, block_start, materialize_start) < 0) + guest_invalidate_ptes(g, block_start, materialize_start) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } if (materialize_end < block_end && - guest_invalidate_ptes(g, materialize_end, block_end) < 0) + guest_invalidate_ptes(g, materialize_end, block_end) < 0) { + materialize_claim_release_locked(g, claim_slot); return -1; + } } } guest_update_perms(g, materialize_start, materialize_end, perms); - - /* Zero the materialized memory. Only zero within the region boundaries to - * avoid clobbering adjacent data. - */ - if (materialize_end > materialize_start) - memset((uint8_t *) g->host_base + materialize_start, 0, - materialize_end - materialize_start); - - /* The page-table helpers above already requested the matching TLBI; no - * additional flush is needed here. + /* One 2MiB materialization gets one block-sized RVAE1IS. This is cheaper + * than per-page invalidation and covers every negative entry that may have + * been cached for the block while its descriptors were invalid. */ + tlbi_request_range(g->ipa_base + block_start, g->ipa_base + block_end); + g->materialize_stats[dirty ? GUEST_MATERIALIZE_DIRTY_MEMSET + : GUEST_MATERIALIZE_CLEAN_SKIP]++; + g->materialize_stats[GUEST_MATERIALIZE_WINDOW_BYTES] += + materialize_end - materialize_start; + mmap_fastpath_note_materialized_locked(g, materialize_start, + materialize_end); + materialize_claim_release_locked(g, claim_slot); return 0; } + +int guest_materialize_lazy(guest_t *g, uint64_t fault_offset) +{ + return guest_materialize_lazy_one(g, fault_offset); +} + +int guest_materialize_lazy_fault(guest_t *g, uint64_t fault_offset) +{ + typedef struct { + guest_t *guest; + uint64_t next_block; + unsigned streak; + } fault_around_state_t; + static _Thread_local fault_around_state_t state; + + uint64_t block = ALIGN_2MIB_DOWN(fault_offset); + if (state.guest == g && block == state.next_block) { + if (state.streak < 4) + state.streak++; + } else { + state.guest = g; + state.streak = 0; + } + + const guest_region_t *region = guest_region_find(g, fault_offset); + if (!region || !region->noreserve || region->prot == LINUX_PROT_NONE) + return -1; + + unsigned blocks = 1U << state.streak; + if (blocks > 16) + blocks = 16; + uint64_t region_last = ALIGN_2MIB_UP(region->end); + if (region_last > g->guest_size) + region_last = g->guest_size; + + for (unsigned i = 0; i < blocks; i++) { + uint64_t ahead = block + (uint64_t) i * BLOCK_2MIB; + if (ahead >= region_last) + break; + if (guest_block_may_be_dirty(g, ahead) && blocks > 4) { + blocks = 4; + break; + } + } + + int result = -1; + uint64_t last = block; + for (unsigned i = 0; i < blocks; i++) { + uint64_t ahead = block + (uint64_t) i * BLOCK_2MIB; + if (ahead >= region_last) + break; + /* The current block must probe the actual FAR. A fast-path mmap can + * extend an already materialized region within the same 2MiB block; + * probing the region/block start would see an older valid page and + * return without installing the newly faulted page. Ahead blocks have + * no FAR, so use their first address inside the region. + */ + uint64_t probe = (i == 0) + ? fault_offset + : (ahead < region->start ? region->start : ahead); + int rc = guest_materialize_lazy_one(g, probe); + if (i == 0) + result = rc; + if (rc < 0) + break; + last = ahead; + } + if (result == 0) + state.next_block = last + BLOCK_2MIB; + return result; +} diff --git a/src/core/guest.h b/src/core/guest.h index 78c6200f..1c2c5ca2 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include #include @@ -234,7 +235,11 @@ typedef struct { uint64_t offset; /* File offset (for /proc/self/maps display) */ int backing_fd; /* Duplicated host fd for file-backed mappings, or -1 */ bool shared; /* MAP_SHARED (writes should propagate) */ - bool noreserve; /* MAP_NORESERVE: PTEs deferred until fault */ + bool noreserve; /* Lazy region: PTEs and zeroing deferred until first + * touch. Set for MAP_NORESERVE and for all private + * anonymous mappings (sys_mmap folds the latter into + * the tracked MAP_NORESERVE bit; not guest visible). + */ bool backing_ro; /* MAP_SHARED region whose backing_fd was opened * without write access, so its Linux max_prot is * capped to PROT_READ. sys_mprotect must reject any @@ -266,6 +271,8 @@ typedef struct { * TLBI_BROADCAST -> X8 = 1 (TLBI VMALLE1IS, broadest) * TLBI_RANGE -> X8 = 3, X9 = start VA, X10 = page count * (TLBI VAE1IS loop preserves unrelated TLB entries) + * TLBI_RANGE_LARGE -> X8 = 4, X9 = encoded range operand + * (single TLBI RVAE1IS) * X8 = 2 is reserved for the execve drop-frame marker the shim handles * separately; it is never produced by the accumulator. */ @@ -287,14 +294,12 @@ typedef enum { */ #define TLBI_SELECTIVE_MAX_PAGES 16 -/* Cap single-shot TLBI RVAE1IS at this many 4 KiB pages. With SCALE=0 the - * RVAE1IS operand encoding covers (NUM+1)*2 pages with NUM in [0..31], so a - * single instruction reaches 64 pages == 256 KiB. Beyond that the host would - * need SCALE=1 (NUM*64 step), which over-invalidates for the typical - * dynamic-linker RELRO / glibc-bring-up storm sizes seen in practice; stay at - * SCALE=0 for now and broadcast above 64 pages. +/* Cap single-shot TLBI RVAE1IS at this many 4 KiB pages. SCALE=0 covers up to + * 64 pages, SCALE=1 up to 2048 pages, and SCALE=2 much larger ranges. 32768 + * pages (128 MiB) covers the lazy fault-around window with one instruction + * while still fitting tlbi_request_t.pages in uint16_t. */ -#define TLBI_RVAE_MAX_PAGES 64 +#define TLBI_RVAE_MAX_PAGES 32768 /* TLBI RVAE1IS operand bit-field constants. Per ARM ARM DDI 0487J.a D8.7.6 the * operand layout is: @@ -310,28 +315,37 @@ typedef enum { */ #define RVAE_OPERAND_BADDR_MASK ((1ULL << 37) - 1) #define RVAE_OPERAND_NUM_SHIFT 39 +#define RVAE_OPERAND_SCALE_SHIFT 44 #define RVAE_OPERAND_TG_4KB (1ULL << 46) /* Pure encoder: build the TLBI RVAE1IS Xt operand from a 4 KiB-aligned VA and a - * page count in the SCALE=0 range (1..TLBI_RVAE_MAX_PAGES). Lives in the header - * as static inline so tlbi_request_emit_to_vcpu and any future caller - * (host-side unit tests included) compile to the same expression. NUM = - * ceil(pages / 2) - 1 over-invalidates odd page counts by exactly one page, - * which is a perf-only side effect (the extra invalidation evicts a neighbour - * TLB entry that the guest's next access reloads). pages < 2 is clamped to 2 - * because SCALE=0 NUM=0 means 2 pages -- the encoder cannot represent a single - * page through RVAE1IS; single-page callers go through the per-page VAE1IS path - * instead, but the clamp keeps the encoder total in any pathological input. + * page count in the supported SCALE=0..2 range. Lives in the header so the + * emit path and host unit tests use the same expression. Each NUM step covers + * 2^(5*SCALE+1) pages; the normalizer aligns and widens callers to that unit. + * pages < 2 is clamped to 2 because SCALE=0 NUM=0 is the smallest encodable + * range. Single-page callers normally use VAE1IS instead. */ +static inline uint64_t tlbi_rvae_unit_pages(uint64_t pages) +{ + if (pages <= 64) + return 2; /* SCALE=0 */ + if (pages <= 2048) + return 64; /* SCALE=1 */ + return 2048; /* SCALE=2 */ +} + static inline uint64_t tlbi_rvae1is_operand(uint64_t start_va, uint16_t pages) { if (pages < 2) pages = 2; + uint64_t scale = pages <= 64 ? 0 : (pages <= 2048 ? 1 : 2); + uint64_t unit_pages = 1ULL << (5 * scale + 1); uint64_t baddr = (start_va >> 12) & RVAE_OPERAND_BADDR_MASK; - uint64_t num = ((pages + 1) / 2) - 1; + uint64_t num = ((pages + unit_pages - 1) / unit_pages) - 1; if (num > 31) num = 31; - return baddr | (num << RVAE_OPERAND_NUM_SHIFT) | RVAE_OPERAND_TG_4KB; + return baddr | (num << RVAE_OPERAND_NUM_SHIFT) | + (scale << RVAE_OPERAND_SCALE_SHIFT) | RVAE_OPERAND_TG_4KB; } /* Runtime feature flag: TRUE when the host PE implements FEAT_TLBIRANGE @@ -346,8 +360,8 @@ typedef struct { * visible to EL0, so the shim must IC IALLU * after the TLBI sequence. 0 = data-only * change, skip the I-cache invalidation. */ - uint16_t pages; /* Page count when kind == TLBI_RANGE (1..MAX) */ - uint64_t start; /* Page-aligned VA when kind == TLBI_RANGE */ + uint16_t pages; /* Page count for either range kind (1..MAX) */ + uint64_t start; /* Page-aligned VA for either range kind */ } tlbi_request_t; /* Layout contract: 16 bytes (1+1+2+4 padding+8). Documents the padding and pins * the TLS slot size so future field additions surface as a build break rather @@ -399,6 +413,45 @@ typedef struct { uint64_t next; /* Bump offset; (next + BLOCK_2MIB) > size means full */ } guest_overflow_t; +/* One conservative "may contain nonzero bytes" bit per 2 MiB primary-slab + * block. The largest supported slab is 1 TiB, so this costs 64 KiB per guest. + * All access is serialized by mmap_lock. + */ +#define GUEST_DIRTY_BLOCKS_MAX ((1ULL << 40) / BLOCK_2MIB) +#define GUEST_DIRTY_WORDS (GUEST_DIRTY_BLOCKS_MAX / 64) + +/* Host-side occupancy index for low-VA TTBR0 mappings. One bit per 2 MiB + * block records whether that block contains at least one valid L2/L3 PTE; a + * second-level bitmap records which occupancy words are non-zero. This lets + * huge lazy mmap/munmap ranges skip untouched address space without walking + * one page-table slot per GiB. The index is separate from dirty_blocks: a + * read-only mapping can have valid PTEs without dirty backing bytes. + */ +#define GUEST_PTE_PRESENT_BLOCKS_MAX GUEST_DIRTY_BLOCKS_MAX +#define GUEST_PTE_PRESENT_WORDS (GUEST_PTE_PRESENT_BLOCKS_MAX / 64) +#define GUEST_PTE_PRESENT_SUMMARY_WORDS (GUEST_PTE_PRESENT_WORDS / 64) +#define GUEST_PTE_PRESENT_LIMIT \ + (GUEST_PTE_PRESENT_BLOCKS_MAX * BLOCK_2MIB) + +enum { + GUEST_MATERIALIZE_CLEAN_SKIP = 0, + GUEST_MATERIALIZE_DIRTY_MEMSET, + GUEST_MATERIALIZE_ALREADY_VALID, + GUEST_MATERIALIZE_WINDOW_BYTES, + GUEST_MATERIALIZE_STATS_N, +}; + +/* Dirty-block zeroing claims. A claim makes its block's invalid PTE window + * stable while the expensive host memset runs without mmap_lock. Waiters use + * one guest-wide condition variable; the fixed table bounds host allocation + * and is ample for the vCPU limit. + */ +#define GUEST_MATERIALIZE_CLAIMS 64 +typedef struct { + uint64_t start, end; + bool active; +} guest_materialize_claim_t; + /* Guest state. */ typedef struct { void *host_base; /* Host pointer to allocated guest memory */ @@ -522,6 +575,20 @@ typedef struct { */ _Atomic uint64_t pt_gen; + /* Host-only index of shim mmap controls that are currently enabled. A bit + * is published only after the control descriptor is complete and cleared + * only while mmap_lock has closed the EL1 producer gate. This lets the + * common one-vCPU path avoid touching 63 cold control pages per drain. + */ + _Atomic uint64_t mmap_fastpath_active_slots; + + uint64_t pte_present_blocks[GUEST_PTE_PRESENT_WORDS]; + uint64_t pte_present_summary[GUEST_PTE_PRESENT_SUMMARY_WORDS]; + uint64_t dirty_blocks[GUEST_DIRTY_WORDS]; + uint64_t materialize_stats[GUEST_MATERIALIZE_STATS_N]; + guest_materialize_claim_t materialize_claims[GUEST_MATERIALIZE_CLAIMS]; + pthread_cond_t materialize_cond; + /* Optional HVC 6 embedder extension hook. * * Native AArch64 guests reach this through HVC 6. When the build enables @@ -643,8 +710,8 @@ static inline void tlbi_request_emit_to_vcpu(hv_vcpu_t vcpu) hv_vcpu_set_reg(vcpu, HV_REG_X11, cpu_tlbi_req.icache_flush ? 1 : 0); break; case TLBI_RANGE_LARGE: { - /* Single-shot TLBI RVAE1IS for ranges in (16..64] pages. The operand - * format and the SCALE=0 / TG=01 / ASID=0 assumptions are documented at + /* Single-shot TLBI RVAE1IS for ranges above the selective cap. The + * SCALE/NUM format and TG=01 assumption are documented at * tlbi_rvae1is_operand above. ASID stays 0 because the shim runs * single-ASID (TCR_EL1.A1=0, TTBR0 ASID=0; rosetta does not allocate a * separate ASID). If a future change introduces non-zero ASIDs, the @@ -666,6 +733,31 @@ static inline void tlbi_request_emit_to_vcpu(hv_vcpu_t vcpu) tlbi_request_clear(); } +/* RVAE1IS requires BaseADDR alignment to its SCALE granule. Widen a requested + * interval to that granule, repeating if widening crosses a SCALE threshold. + * Over-invalidation is architecturally harmless and preserves unrelated TLB + * entries far better than a VMALLE1IS broadcast. + */ +static inline bool tlbi_rvae_normalize(uint64_t *start, uint64_t *end) +{ + for (int pass = 0; pass < 3; pass++) { + uint64_t pages = (*end - *start) >> 12; + if (pages <= TLBI_SELECTIVE_MAX_PAGES) + return true; + uint64_t unit_pages = tlbi_rvae_unit_pages(pages); + uint64_t unit_bytes = unit_pages << 12; + uint64_t s = *start & ~(unit_bytes - 1); + if (*end > UINT64_MAX - (unit_bytes - 1)) + return false; + uint64_t e = (*end + unit_bytes - 1) & ~(unit_bytes - 1); + *start = s; + *end = e; + if (((e - s) >> 12) <= unit_pages * 32) + return ((e - s) >> 12) <= TLBI_RVAE_MAX_PAGES; + } + return false; +} + static inline void tlbi_request_range(uint64_t start, uint64_t end) { if (cpu_tlbi_req.kind == TLBI_BROADCAST) @@ -693,6 +785,13 @@ static inline void tlbi_request_range(uint64_t start, uint64_t end) */ uint64_t large_cap = g_tlbi_range_supported ? TLBI_RVAE_MAX_PAGES : TLBI_SELECTIVE_MAX_PAGES; + if (g_tlbi_range_supported && n > TLBI_SELECTIVE_MAX_PAGES) { + if (!tlbi_rvae_normalize(&s, &e)) { + tlbi_request_broadcast(); + return; + } + n = (e - s) >> 12; + } if (n > large_cap) { tlbi_request_broadcast(); return; @@ -722,6 +821,13 @@ static inline void tlbi_request_range(uint64_t start, uint64_t end) uint64_t us = s < es ? s : es; uint64_t ue = e > ee ? e : ee; uint64_t un = (ue - us) >> 12; + if (g_tlbi_range_supported && un > TLBI_SELECTIVE_MAX_PAGES) { + if (!tlbi_rvae_normalize(&us, &ue)) { + tlbi_request_broadcast(); + return; + } + un = (ue - us) >> 12; + } if (un > large_cap) { tlbi_request_broadcast(); return; @@ -951,11 +1057,22 @@ int guest_install_va_pages(guest_t *g, uint64_t gpa, int perms); -/* Query whether a 2 MiB TTBR0 VA block already has a leaf mapping. - * Returns true only for a present L2 block descriptor. +/* Query whether a 2 MiB TTBR0 VA block already has any L2 entry, either a + * block descriptor or an L3 table descriptor. */ bool guest_va_block_mapped(const guest_t *g, uint64_t va); +/* Rebuild the low-VA PTE occupancy index from TTBR0. Used after fork restores + * page-table pages into a freshly initialized guest_t. + */ +void guest_rebuild_pte_present(guest_t *g); + +/* Reconcile the host-only 2 MiB occupancy index after EL1 has invalidated + * descriptors in [start,end). This observes PTEs only; it never writes a + * descriptor or requests another TLBI. Caller holds mmap_lock. + */ +void guest_retire_ptes_committed(guest_t *g, uint64_t start, uint64_t end); + /* Returns true when the VA range [va, va+size) overlaps the user-VA kbuf alias * window [KBUF_USER_VA, KBUF_USER_VA+KBUF_SIZE). Callers that install TTBR0 * mappings (the future rosetta_finalize, sys_mmap MAP_FIXED touching this @@ -978,6 +1095,38 @@ static inline bool guest_kbuf_user_va_overlap(uint64_t va, uint64_t size) */ void *guest_ptr(const guest_t *g, uint64_t gva); +/* Like guest_ptr_avail but never triggers lazy fault-in. For callers that + * already hold mmap_lock. + */ +void *guest_ptr_avail_nofault(const guest_t *g, + uint64_t gva, + uint64_t *avail, + int required_perms); + +/* Materialize any lazy (deferred-PTE) blocks intersecting [gva, gva+len) so + * later resolves under locks that rank after mmap_lock (futex buckets) do + * not have to. Takes mmap_lock; call only from lock-free context. Returns 0 + * if anything was (or already is) materialized, -1 otherwise; callers that + * merely pre-fault can ignore the result. + */ +int guest_lazy_faultin(const guest_t *g, uint64_t gva, uint64_t len); + +/* Same as guest_lazy_faultin for callers that already hold mmap_lock (e.g. + * SC_LOCKED syscall handlers about to guest_read/guest_write a lazy range: + * the resolve-time hook would self-deadlock re-acquiring mmap_lock, so they + * must materialize up front through this variant). + */ +int guest_lazy_faultin_locked(const guest_t *g, uint64_t gva, uint64_t len); + +/* Smallest 2MiB-block-aligned va' in [va, end) containing at least one valid + * TTBR0 PTE, or end if none. Low VA uses the host-side hierarchical occupancy + * bitmap; non-identity/high VA falls back to the page-table hierarchy. Locking: + * callers MUST hold mmap_lock. + */ +uint64_t guest_va_next_present_block(const guest_t *g, + uint64_t va, + uint64_t end); + /* Get a host pointer for a guest virtual address (write access). * Returns NULL if gva is out of bounds or not writable. */ @@ -1023,6 +1172,12 @@ int guest_read_small(const guest_t *g, uint64_t gva, void *dst, size_t len); */ int guest_write(guest_t *g, uint64_t gva, const void *src, size_t len); +/* Same copy without lazy materialization. Callers that already hold mmap_lock + * can use this after guest_lazy_faultin_locked(); an invalid destination then + * fails instead of recursively trying to acquire mmap_lock. + */ +int guest_write_nofault(guest_t *g, uint64_t gva, const void *src, size_t len); + /* Optimized host-to-guest copy for small fixed-size outputs. Uses a direct * guest pointer when the full range is contiguous and writable, otherwise falls * back to guest_write() for boundary-crossing safety. @@ -1061,6 +1216,19 @@ int guest_extend_page_tables(guest_t *g, uint64_t end, int perms); +/* Prepare only the upper TTBR0 chain needed to reach L2 entries in + * [start,end); all L2 leaf descriptors remain untouched/invalid. This lets + * the host provision an arena for a later EL1-owned L3 install without ever + * transiently mapping unallocated user memory. + */ +int guest_prepare_l2_tables(guest_t *g, uint64_t start, uint64_t end); + +/* Reserve a contiguous run of uninitialized 4 KiB pages from the page-table + * pool. The caller/EL1 producer must overwrite every entry before publishing + * any page as a table descriptor. Returns the first GPA, or 0 on exhaustion. + */ +uint64_t guest_reserve_pt_pages_uninitialized(guest_t *g, unsigned count); + /* Split a 2MiB block descriptor into 512 x 4KiB L3 page descriptors. block_gpa * must be within a currently-mapped 2MiB block. The block's permissions are * inherited by all 512 page entries. If the block is already split (L2 entry is @@ -1245,12 +1413,34 @@ bool guest_region_range_has_ro_shared_backing(const guest_t *g, uint64_t start, uint64_t end); -/* Try to materialize a lazy (MAP_NORESERVE) page at the given offset. Called - * from the data/instruction abort handler when the faulting address falls - * within a noreserve region. Creates page table entries for one 2MiB block - * containing the fault address, zeros the memory, and clears the noreserve flag - * for the materialized sub-range. - * Returns 0 on success (caller should TLBI and retry), -1 if the offset is not - * in a noreserve region. +/* Try to materialize a lazy (deferred-PTE: private anonymous or + * MAP_NORESERVE) page at the given offset. Called from the data/instruction + * abort handler when the faulting address falls within a lazy region, and + * from the host-access fault-in path when a syscall targets a lazy range the + * guest has not touched yet. Creates page table entries for one 2MiB block + * containing the fault address. A slab block known to be clean skips zeroing; + * a possibly-dirty block zeros invalid pages before publishing them. A block + * that is already valid returns success without re-zeroing + * (concurrent-fault idempotence). + * Returns 0 on success (caller should TLBI and retry), -1 if the offset is + * not in a lazy region or the region is PROT_NONE. + * Locking: callers MUST hold mmap_lock. */ int guest_materialize_lazy(guest_t *g, uint64_t fault_offset); + +/* Guest-fault variant with per-vCPU sequential fault-around. */ +int guest_materialize_lazy_fault(guest_t *g, uint64_t fault_offset); + +/* Dirty-map and in-flight-claim helpers. Callers hold mmap_lock. */ +bool guest_block_may_be_dirty(const guest_t *g, uint64_t block_start); +void guest_dirty_mark_range(guest_t *g, uint64_t start, uint64_t end); +void guest_dirty_clear_zeroed_range(guest_t *g, uint64_t start, uint64_t end); +void guest_materialize_wait_range_locked(guest_t *g, + uint64_t start, + uint64_t end); +void guest_materialize_wait_all_locked(guest_t *g); + +/* Whether the 4KiB page containing va has a valid stage-1 descriptor. + * Locking: callers MUST hold mmap_lock. + */ +bool guest_va_pte_valid(guest_t *g, uint64_t va); diff --git a/src/core/mmap-fastpath.h b/src/core/mmap-fastpath.h new file mode 100644 index 00000000..df1178d2 --- /dev/null +++ b/src/core/mmap-fastpath.h @@ -0,0 +1,232 @@ +/* + * Per-vCPU EL1 anonymous-mmap consumer rings. + * + * The host produces arenas, consumes mmap/munmap publications, and returns + * metadata-committed holes through a reverse SPSC ring. EL1 first-fit consumes + * those private extents before bump-allocating fresh VA. Control blocks live + * in the EL1-only shim-data mapping and are selected from SP_EL1's per-thread + * stack slot, so no guest-visible register ABI is consumed. + */ + +#pragma once + +#include +#include +#include +#include + +#include "core/guest.h" + +typedef struct thread_entry thread_entry_t; + +#define SHIM_MMAP_CONTROL_BASE 0x20000u +#define SHIM_MMAP_CONTROL_STRIDE 0x1000u +#define SHIM_MMAP_RING_SIZE 32u +#define SHIM_MMAP_CTRL_ENABLED 0x1u +#define SHIM_MMAP_CTRL_TLBIRANGE 0x2u +/* Host mprotect changed region/PTE shape inside this arena. mmap remains safe, + * and EL1 mprotect validates L3 directly, but fast munmap falls back so its + * cleanup is not deferred across the changed metadata. + */ +#define SHIM_MMAP_CTRL_NO_FAST_MUNMAP 0x4u + +/* Host page-table writers set this gate before changing an arena descriptor or + * a stage-1 PTE. Each EL1 producer announces itself in its private control + * before rechecking the gate. This is a writer-vs-per-vCPU-reader handshake, + * not an allocator lock: fast munmap producers never write a shared cache + * line or wait for one another. + */ +#define SHIM_MMAP_PT_GATE_OFF 0x1160u + +#define SHIM_MUNMAP_RETIRE_RING_SIZE 32u +#define SHIM_MUNMAP_RETIRE_OFF 0x400u +#define SHIM_MUNMAP_RETIRE_BYTES_SOFT (256ULL * 1024 * 1024) +#define SHIM_MUNMAP_RETIRE_F_ARENA_SLOT_MASK 0x3fu +#define SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT 6u +#define SHIM_MUNMAP_RETIRE_F_CHARGE_MASK 0xffffffc0u + +#define SHIM_MMAP_REUSE_RING_SIZE 32u +#define SHIM_MMAP_REUSE_OFF 0x720u + +/* The mmap publication ring also carries completed EL1 single-page mprotect + * metadata updates. The tag lives outside the Linux PROT_* namespace. + */ +#define SHIM_MMAP_ENTRY_MPROTECT (1ULL << 63) +/* The mprotect also installed the first L3 table for a clean, fully-owned + * 2 MiB lazy block. The host must reconcile its PTE/dirty indexes when the + * deferred metadata entry drains. + */ +#define SHIM_MMAP_ENTRY_MPROTECT_SPLIT (1ULL << 62) +/* Permission metadata changed while the target PTE remained lazy/invalid. + * The next fault drains the entry before consulting regions[]. + */ +#define SHIM_MMAP_ENTRY_MPROTECT_LAZY (1ULL << 61) + +/* A cache batch covers a rolling 32-block (64 MiB) arena window. Unused pages + * stay attached to the vCPU across arena generations, so refills do not leak a + * batch each time the VA reservation rolls over. + */ +#define SHIM_MMAP_L3_CACHE_PAGES 32u +#define SHIM_MMAP_SPLIT_CLEAN_BLOCKS 32u + +#define MMAP_FAST_ARENA_MIN (64ULL * 1024 * 1024) +#define MMAP_FAST_ARENA_MAX (32ULL * 1024 * 1024 * 1024) +#define MMAP_FAST_ARENA_TARGET_ENTRIES 32u + +enum { + SHIM_MMAP_COUNTER_SHAPE_MISS = 0, + SHIM_MMAP_COUNTER_CAPACITY_MISS, + SHIM_MMAP_COUNTER_RING_FULL, + SHIM_MMAP_COUNTER_GENERATION_STALE, + SHIM_MMAP_COUNTER_ATTENTION, + SHIM_MMAP_COUNTER_HIT, + SHIM_MMAP_COUNTERS_N, +}; + +typedef struct { + uint64_t addr; + uint64_t len; + uint64_t prot; +} shim_mmap_entry_t; + +typedef struct munmap_retire_entry { + uint64_t addr; + uint64_t length; + uint32_t arena_generation; + uint32_t flags; +} munmap_retire_entry_t; + +typedef struct munmap_retire_ring { + _Atomic uint32_t head; /* host consumer */ + _Atomic uint32_t tail; /* EL1 producer */ + _Atomic uint64_t produced_bytes; /* EL1 producer, monotonic */ + _Atomic uint64_t consumed_bytes; /* host consumer, monotonic */ + _Atomic uint32_t producer_active; + /* Advisory; never forces the producer to exit. */ + _Atomic uint32_t cleanup_requested; + munmap_retire_entry_t entries[SHIM_MUNMAP_RETIRE_RING_SIZE]; +} munmap_retire_ring_t; + +typedef struct mmap_reuse_entry { + uint64_t addr; + uint64_t length; +} mmap_reuse_entry_t; + +/* Host producer -> EL1 consumer. Entries transfer committed, quarantined VA + * extents back to their owning arena only after semantic metadata and PTE + * retirement have completed. EL1 may shorten an acquired entry in place; + * the host does not touch its slot again until the release-published head has + * advanced past it. */ +typedef struct mmap_reuse_ring { + _Atomic uint32_t head; /* EL1 consumer */ + _Atomic uint32_t tail; /* host producer */ + _Atomic uint64_t published; + _Atomic uint64_t dropped; + _Atomic uint64_t hits; /* EL1 producer */ + mmap_reuse_entry_t entries[SHIM_MMAP_REUSE_RING_SIZE]; +} mmap_reuse_ring_t; + +typedef struct { + _Atomic uint32_t generation; /* host publish word */ + _Atomic uint32_t consumer_generation; /* EL1 generation ack */ + _Atomic uint32_t flags; /* host-owned enable bits */ + _Atomic uint32_t head; /* host consumer cursor */ + _Atomic uint32_t tail; /* EL1 producer cursor */ + uint32_t _pad0; + _Atomic uint64_t arena_base; + _Atomic uint64_t arena_limit; + _Atomic uint64_t cursor; /* EL1 bump cursor */ + uint64_t next_arena_size; /* most recently selected generation size */ + uint64_t max_len_seen; /* outgoing-generation maximum request length */ + shim_mmap_entry_t ring[SHIM_MMAP_RING_SIZE]; + _Atomic uint64_t counters[SHIM_MMAP_COUNTERS_N]; + uint64_t refill_count; + uint64_t recycle_count; + uint64_t peak_arena_size; + _Atomic uint32_t materialized_generation; + uint32_t _pad1; + _Atomic uint64_t materialized_start; + _Atomic uint64_t materialized_end; + uint8_t _pad2[SHIM_MUNMAP_RETIRE_OFF - 0x3a0]; + munmap_retire_ring_t retire; + mmap_reuse_ring_t reuse; + _Atomic uint64_t l3_cache_next; /* EL1-owned bump cursor (GPA) */ + uint64_t l3_cache_end; /* host-published exclusive end */ + uint64_t split_clean_base; /* 2 MiB-aligned arena window */ + uint64_t split_clean_bitmap; /* one clean-backing bit per block */ + uint64_t split_prep_miss; + uint64_t split_owner_miss; + uint64_t split_hit; + uint64_t lazy_mprotect_hit; +} shim_mmap_control_t; + +_Static_assert(offsetof(shim_mmap_control_t, retire) == SHIM_MUNMAP_RETIRE_OFF, + "shim.S retire-ring offset ABI"); +_Static_assert(offsetof(shim_mmap_control_t, reuse) == SHIM_MMAP_REUSE_OFF, + "shim.S reuse-ring offset ABI"); +_Static_assert(sizeof(shim_mmap_control_t) <= SHIM_MMAP_CONTROL_STRIDE, + "mmap control exceeds per-vCPU stride"); + +/* Called while initializing a vCPU, before it can enter guest code. */ +void mmap_fastpath_prepare_vcpu(guest_t *g, thread_entry_t *t); + +/* Drain every per-vCPU SPSC ring. Caller holds mmap_lock. */ +void mmap_fastpath_drain_locked(guest_t *g); + +/* Opportunistically drain publications and retirements at a natural VM exit. + * Safe before every exit handler; it acquires mmap_lock internally. + */ +void mmap_fastpath_drain_vmexit(guest_t *g); + +/* True when the stopped current vCPU was interrupted in the middle of its EL1 + * producer critical section. A cancellation exit must resume it before host + * drain can close the PT gate. + */ +bool mmap_fastpath_current_producer_active(const guest_t *g); + +/* Mark every arena intersecting a successfully materialized lazy range. Caller + * holds mmap_lock. EL1 may skip its PTE walk only while this marker differs + * from the arena's current generation. + */ +void mmap_fastpath_note_materialized_locked(guest_t *g, + uint64_t start, + uint64_t end); + +/* Refill the current vCPU after an eligible mmap slow-path. request_len is + * page-rounded; requests above MMAP_FAST_ARENA_MAX leave the arena untouched. + * Caller holds mmap_lock. + */ +void mmap_fastpath_refill_current_locked(guest_t *g, uint64_t request_len); + +/* Fulfil an eligible mmap that reached HVC (capacity/ring/generation fallback) + * directly from the current vCPU's refilled arena. Metadata is committed by + * the host immediately, so no mmap publication entry is needed. Caller holds + * mmap_lock. + */ +bool mmap_fastpath_allocate_current_locked(guest_t *g, + uint64_t request_len, + uint64_t *addr_out); +bool mmap_fastpath_allocate_current_publication_only(guest_t *g, + uint64_t request_len, + uint64_t *addr_out); + +/* Give an explicit slow-path hint precedence over this stopped vCPU's + * unconsumed arena tail. Caller holds mmap_lock. + */ +void mmap_fastpath_release_current_hint_locked(guest_t *g, + uint64_t addr, + uint64_t length); + +/* Revoke all arenas while sibling vCPUs are quiesced. Caller holds mmap_lock. + */ +void mmap_fastpath_revoke_all_locked(guest_t *g, bool shrink_high_water); + +/* Disable the feature before first guest entry (debugger/observability). */ +void mmap_fastpath_disable(guest_t *g); + +/* Advance *start past an EL1 arena reservation that overlaps length bytes. */ +void mmap_fastpath_skip_reserved(const guest_t *g, + uint64_t *start, + uint64_t length, + uint64_t align, + uint64_t max_addr); diff --git a/src/core/shim-globals.c b/src/core/shim-globals.c index d5fd53d7..eef3374b 100644 --- a/src/core/shim-globals.c +++ b/src/core/shim-globals.c @@ -10,6 +10,7 @@ */ #include +#include #include #include #include @@ -18,6 +19,7 @@ #include "hvutil.h" #include "core/guest.h" +#include "core/mmap-fastpath.h" #include "core/shim-globals.h" #include "core/vdso.h" #include "debug/log.h" @@ -95,6 +97,96 @@ _Static_assert(SHIM_GLOBALS_SIZE <= BLOCK_2MIB, _Static_assert(SHIM_COUNTERS_OFF + SHIM_COUNTERS_N * 8 <= SHIM_IDENTITY_OFF_PGID, "counter array must not overlap the PGID slot"); +_Static_assert(SHIM_MMAP_CONTROL_BASE == 0x20000, + "shim.S mmap fast path hard-codes control base 0x20000"); +_Static_assert(SHIM_MMAP_CONTROL_STRIDE == 0x1000, + "shim.S mmap fast path hard-codes control stride 0x1000"); +_Static_assert(SHIM_MMAP_RING_SIZE == 32, + "shim.S mmap fast path hard-codes 32 ring entries"); +_Static_assert(offsetof(shim_mmap_control_t, generation) == 0, + "shim.S mmap generation offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, consumer_generation) == 4, + "shim.S mmap consumer-generation offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, flags) == 8, + "shim.S mmap flags offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, head) == 12, + "shim.S mmap head offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, tail) == 16, + "shim.S mmap tail offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, arena_base) == 24, + "shim.S mmap arena-base offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, arena_limit) == 32, + "shim.S mmap arena-limit offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, cursor) == 40, + "shim.S mmap cursor offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, next_arena_size) == 48, + "mmap next-arena-size offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, max_len_seen) == 56, + "mmap max-len-seen offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, ring) == 64, + "shim.S mmap ring offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, counters) == 0x340, + "shim.S mmap counter offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, materialized_generation) == 0x388, + "shim.S mmap materialized-generation offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, materialized_start) == 0x390 && + offsetof(shim_mmap_control_t, materialized_end) == 0x398, + "shim.S mmap materialized bounds offset drift"); +_Static_assert(offsetof(shim_mmap_control_t, retire) == 0x400, + "shim.S munmap retire offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, produced_bytes) == 8, + "shim.S munmap produced-byte offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, consumed_bytes) == 16, + "shim.S munmap consumed-byte offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, producer_active) == 24, + "shim.S munmap active offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, cleanup_requested) == 28, + "shim.S munmap cleanup-request offset drift"); +_Static_assert(offsetof(munmap_retire_ring_t, entries) == 32, + "shim.S munmap entries offset drift"); +_Static_assert(SHIM_MUNMAP_RETIRE_RING_SIZE == 32 && MAX_THREADS == 64, + "shim.S munmap ring/arena scan constants drift"); +_Static_assert(SHIM_MUNMAP_RETIRE_BYTES_SOFT == 0x10000000ULL, + "shim.S munmap byte advisory threshold drift"); +_Static_assert(SHIM_MUNMAP_RETIRE_F_ARENA_SLOT_MASK == 0x3f && + SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT == 6, + "shim.S munmap retire flag encoding drift"); +_Static_assert(offsetof(shim_mmap_control_t, reuse) == 0x720, + "shim.S mmap reuse offset drift"); +_Static_assert(offsetof(mmap_reuse_ring_t, published) == 8 && + offsetof(mmap_reuse_ring_t, dropped) == 16 && + offsetof(mmap_reuse_ring_t, hits) == 24 && + offsetof(mmap_reuse_ring_t, entries) == 32, + "shim.S mmap reuse-ring layout drift"); +_Static_assert(SHIM_MMAP_REUSE_RING_SIZE == 32, + "shim.S mmap reuse-ring size drift"); +_Static_assert(offsetof(shim_mmap_control_t, l3_cache_next) == 0x940 && + offsetof(shim_mmap_control_t, l3_cache_end) == 0x948 && + offsetof(shim_mmap_control_t, split_clean_base) == 0x950 && + offsetof(shim_mmap_control_t, split_clean_bitmap) == 0x958, + "shim.S mmap L3-cache layout drift"); +_Static_assert(offsetof(shim_mmap_control_t, split_prep_miss) == 0x960 && + offsetof(shim_mmap_control_t, split_owner_miss) == 0x968 && + offsetof(shim_mmap_control_t, split_hit) == 0x970 && + offsetof(shim_mmap_control_t, lazy_mprotect_hit) == 0x978, + "shim.S mmap L3 diagnostic layout drift"); +_Static_assert(SHIM_MMAP_L3_CACHE_PAGES == 32 && + SHIM_MMAP_SPLIT_CLEAN_BLOCKS == 32, + "shim.S mmap L3-cache constants drift"); +_Static_assert((MMAP_FAST_ARENA_MAX >> 12) <= + (SHIM_MUNMAP_RETIRE_F_CHARGE_MASK >> + SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT), + "munmap retire flags cannot encode maximum arena charge"); +_Static_assert(SHIM_MMAP_PT_GATE_OFF >= SHIM_GLOBALS_SIZE && + SHIM_MMAP_PT_GATE_OFF + sizeof(uint32_t) <= + SHIM_MMAP_CONTROL_BASE, + "host PT gate overlaps shim globals or mmap controls"); +_Static_assert(sizeof(shim_mmap_control_t) <= SHIM_MMAP_CONTROL_STRIDE, + "per-vCPU mmap control exceeds its shim-data stride"); +_Static_assert(SHIM_MMAP_CONTROL_BASE + + MAX_THREADS * SHIM_MMAP_CONTROL_STRIDE <= + BLOCK_2MIB - MAX_THREADS * 4096, + "mmap controls overlap per-vCPU EL1 stacks"); static uint8_t *cache_base(const guest_t *g) { @@ -125,7 +217,15 @@ static void urandom_ring_unlock(uint32_t *lock_p) void shim_globals_init(guest_t *g) { - memset(cache_base(g), 0, SHIM_GLOBALS_SIZE); + /* mmap controls occupy a separate low shim-data range. Init/exec/fork + * child all call this while no sibling can execute, so clearing the whole + * control array also prevents a recycled SP_EL1 slot from inheriting an + * arena published to its previous owner. + */ + memset(cache_base(g), 0, + SHIM_MMAP_CONTROL_BASE + MAX_THREADS * SHIM_MMAP_CONTROL_STRIDE); + atomic_store_explicit(&g->mmap_fastpath_active_slots, 0, + memory_order_relaxed); } void shim_globals_publish_pid(guest_t *g, int64_t pid, int64_t ppid) @@ -435,11 +535,10 @@ static const char *const counter_names[SHIM_COUNTERS_N] = { [SHIM_COUNTER_URANDOM_HIT] = "URANDOM_HIT", [SHIM_COUNTER_GETRANDOM_HIT] = "GETRANDOM_HIT", [SHIM_COUNTER_PGSID_HIT] = "PGSID_HIT", - /* Slots 12..15 (SHIM_COUNTERS_N == 16) are intentionally unnamed; the dump - * prints "(reserved)" so they appear in the output when non-zero, which - * would flag an out-of-band increment. Bind a name here when a future EL1 - * service claims one of these slots. - */ + [SHIM_COUNTER_FAULT_MATERIALIZE] = "FAULT_MATERIALIZE", + [SHIM_COUNTER_FAULT_TLBI_VAE] = "FAULT_TLBI_VAE", + [SHIM_COUNTER_FAULT_TLBI_RVAE] = "FAULT_TLBI_RVAE", + [SHIM_COUNTER_FAULT_TLBI_BCAST] = "FAULT_TLBI_BCAST", }; uint64_t shim_globals_counter_get(const guest_t *g, unsigned slot) @@ -452,6 +551,15 @@ uint64_t shim_globals_counter_get(const guest_t *g, unsigned slot) return __atomic_load_n(slot_p, __ATOMIC_RELAXED); } +void shim_globals_counter_inc(guest_t *g, unsigned slot) +{ + if (!shim_globals_stats_enabled() || slot >= SHIM_COUNTERS_N) + return; + uint8_t *page = (uint8_t *) g->host_base + g->shim_data_base; + uint64_t *slot_p = (uint64_t *) (page + SHIM_COUNTERS_OFF) + slot; + __atomic_fetch_add(slot_p, 1, __ATOMIC_RELAXED); +} + void shim_globals_counters_dump(const guest_t *g) { fprintf(stderr, "shim-stats (pid=%lld)\n", (long long) proc_get_pid()); @@ -463,6 +571,108 @@ void shim_globals_counters_dump(const guest_t *g) fprintf(stderr, " %-20s %llu\n", name ? name : "(reserved)", (unsigned long long) v); } + + static const char *const mmap_counter_names[SHIM_MMAP_COUNTERS_N] = { + [SHIM_MMAP_COUNTER_SHAPE_MISS] = "MMAP_SHAPE_MISS", + [SHIM_MMAP_COUNTER_CAPACITY_MISS] = "MMAP_CAPACITY_MISS", + [SHIM_MMAP_COUNTER_RING_FULL] = "MMAP_RING_FULL", + [SHIM_MMAP_COUNTER_GENERATION_STALE] = "MMAP_GENERATION_STALE", + [SHIM_MMAP_COUNTER_ATTENTION] = "MMAP_ATTENTION", + [SHIM_MMAP_COUNTER_HIT] = "MMAP_HIT", + }; + uint64_t mmap_counters[SHIM_MMAP_COUNTERS_N] = {0}; + uint64_t refill_count = 0, recycle_count = 0; + uint64_t reuse_published = 0, reuse_dropped = 0, reuse_hits = 0; + uint64_t reuse_pending = 0; + uint64_t current_max = 0, peak_max = 0; + uint64_t l3_cache_available = 0, split_clean_blocks = 0; + uint64_t split_prep_miss = 0, split_owner_miss = 0, split_hit = 0; + uint64_t lazy_mprotect_hit = 0; + const uint8_t *shim_data = + (const uint8_t *) g->host_base + g->shim_data_base; + for (int slot = 0; slot < MAX_THREADS; slot++) { + const shim_mmap_control_t *c = + (const shim_mmap_control_t *) (shim_data + SHIM_MMAP_CONTROL_BASE + + (uint64_t) slot * + SHIM_MMAP_CONTROL_STRIDE); + for (unsigned i = 0; i < SHIM_MMAP_COUNTERS_N; i++) + mmap_counters[i] += + atomic_load_explicit(&c->counters[i], memory_order_relaxed); + refill_count += c->refill_count; + recycle_count += c->recycle_count; + reuse_published += + atomic_load_explicit(&c->reuse.published, memory_order_relaxed); + reuse_dropped += + atomic_load_explicit(&c->reuse.dropped, memory_order_relaxed); + reuse_hits += + atomic_load_explicit(&c->reuse.hits, memory_order_relaxed); + uint32_t reuse_head = + atomic_load_explicit(&c->reuse.head, memory_order_relaxed); + uint32_t reuse_tail = + atomic_load_explicit(&c->reuse.tail, memory_order_relaxed); + reuse_pending += (uint32_t) (reuse_tail - reuse_head); + if (c->next_arena_size > current_max) + current_max = c->next_arena_size; + if (c->peak_arena_size > peak_max) + peak_max = c->peak_arena_size; + uint64_t l3_next = + atomic_load_explicit(&c->l3_cache_next, memory_order_relaxed); + if (l3_next < c->l3_cache_end) + l3_cache_available += (c->l3_cache_end - l3_next) / 4096; + split_clean_blocks += + (uint64_t) __builtin_popcountll(c->split_clean_bitmap); + split_prep_miss += c->split_prep_miss; + split_owner_miss += c->split_owner_miss; + split_hit += c->split_hit; + lazy_mprotect_hit += c->lazy_mprotect_hit; + } + for (unsigned i = 0; i < SHIM_MMAP_COUNTERS_N; i++) + fprintf(stderr, " %-20s %llu\n", mmap_counter_names[i], + (unsigned long long) mmap_counters[i]); + fprintf(stderr, " %-20s %llu\n", "MMAP_REFILL", + (unsigned long long) refill_count); + fprintf(stderr, " %-20s %llu\n", "MMAP_RECYCLE", + (unsigned long long) recycle_count); + fprintf(stderr, " %-20s %llu\n", "MMAP_REUSE_PUBLISH", + (unsigned long long) reuse_published); + fprintf(stderr, " %-20s %llu\n", "MMAP_REUSE_DROP", + (unsigned long long) reuse_dropped); + fprintf(stderr, " %-20s %llu\n", "MMAP_REUSE_HIT", + (unsigned long long) reuse_hits); + fprintf(stderr, " %-20s %llu\n", "MMAP_REUSE_PENDING", + (unsigned long long) reuse_pending); + fprintf(stderr, " %-20s %llu\n", "MMAP_ARENA_CURRENT", + (unsigned long long) current_max); + fprintf(stderr, " %-20s %llu\n", "MMAP_ARENA_PEAK", + (unsigned long long) peak_max); + fprintf(stderr, " %-20s %llu\n", "MMAP_L3_AVAILABLE", + (unsigned long long) l3_cache_available); + fprintf(stderr, " %-20s %llu\n", "MMAP_SPLIT_CLEAN", + (unsigned long long) split_clean_blocks); + fprintf(stderr, " %-20s %llu\n", "MPROT_SPLIT_PREP_MISS", + (unsigned long long) split_prep_miss); + fprintf(stderr, " %-20s %llu\n", "MPROT_SPLIT_OWNER_MISS", + (unsigned long long) split_owner_miss); + fprintf(stderr, " %-20s %llu\n", "MPROT_SPLIT_HIT", + (unsigned long long) split_hit); + fprintf(stderr, " %-20s %llu\n", "MPROT_LAZY_HIT", + (unsigned long long) lazy_mprotect_hit); + uint64_t high_water = + g->mmap_next > MMAP_BASE ? g->mmap_next - MMAP_BASE : 0; + fprintf(stderr, " %-20s %llu\n", "MMAP_HIGH_WATER", + (unsigned long long) high_water); + fprintf(stderr, " %-20s %llu\n", "FAULT_CLEAN_SKIP", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_CLEAN_SKIP]); + fprintf(stderr, " %-20s %llu\n", "FAULT_DIRTY_MEMSET", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_DIRTY_MEMSET]); + fprintf(stderr, " %-20s %llu\n", "FAULT_ALREADY_VALID", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_ALREADY_VALID]); + fprintf(stderr, " %-20s %llu\n", "FAULT_WINDOW_BYTES", + (unsigned long long) + g->materialize_stats[GUEST_MATERIALIZE_WINDOW_BYTES]); } static pthread_once_t stats_once = PTHREAD_ONCE_INIT; diff --git a/src/core/shim-globals.h b/src/core/shim-globals.h index 14cb24f1..049775d4 100644 --- a/src/core/shim-globals.h +++ b/src/core/shim-globals.h @@ -165,7 +165,7 @@ * not in urandom bitmap, len zero, len over inline cap, ring fill below * request, ring wrap, EL0 buffer probe failure). Slots 8..11 record fast-path * hits so bail rates can be computed against a hit denominator. Slots 12..15 - * are reserved. + * attribute lazy-fault materializations and the TLBI wire mode they emit. * * The shim hardcodes the byte offset of each slot; the static_asserts in * shim-globals.c keep the C-side macros and the assembly in sync. @@ -185,6 +185,10 @@ #define SHIM_COUNTER_URANDOM_HIT 9 #define SHIM_COUNTER_GETRANDOM_HIT 10 #define SHIM_COUNTER_PGSID_HIT 11 +#define SHIM_COUNTER_FAULT_MATERIALIZE 12 +#define SHIM_COUNTER_FAULT_TLBI_VAE 13 +#define SHIM_COUNTER_FAULT_TLBI_RVAE 14 +#define SHIM_COUNTER_FAULT_TLBI_BCAST 15 /* Extended identity slots: pgid and sid. * @@ -390,6 +394,7 @@ void shim_globals_refill_urandom_ring(guest_t *g); * when ELFUSE_SHIM_STATS is set. */ uint64_t shim_globals_counter_get(const guest_t *g, unsigned slot); +void shim_globals_counter_inc(guest_t *g, unsigned slot); void shim_globals_counters_dump(const guest_t *g); /* ELFUSE_SHIM_STATS env-var gate (idempotent / cached). When enabled the exit diff --git a/src/core/shim.S b/src/core/shim.S index 4c6a1b86..73a390b5 100644 --- a/src/core/shim.S +++ b/src/core/shim.S @@ -26,7 +26,7 @@ * 4 = single-shot TLBI RVAE1IS (FEAT_TLBIRANGE); * X9 carries the pre-encoded RVAE1IS operand * (baddr | NUM<<39 | SCALE<<44 | TTL<<37 | - * ASID<<48; SCALE=0, TTL=0, ASID=0 today) + * ASID<<48; TTL=0 and ASID=0) * X11 carries the I-cache hint for X8 in {1, 3, 4}: * 1 = IC IALLU after the TLBI sequence (new * executable content visible to EL0), 0 = skip the @@ -251,6 +251,38 @@ .Lctr_skip_\@: .endm +/* Per-vCPU mmap counters live after the 32 x 24-byte publication ring in the + * mmap control block. Each vCPU is the sole writer of its own slots. Keep the + * shared stats gate so normal fast paths do not touch diagnostic cache lines. + */ +.equ SHIM_MMAP_COUNTERS_OFF, 0x340 +.equ MC_SHAPE_MISS, 0 +.equ MC_CAPACITY_MISS, 1 +.equ MC_RING_FULL, 2 +.equ MC_GENERATION_STALE, 3 +.equ MC_ATTENTION, 4 +.equ MC_HIT, 5 + +.macro MMAP_COUNTER_INC slot + mrs x29, tpidr_el1 + ldrb w30, [x29, #SHIM_STATS_EN_OFF] + cbz w30, .Lmmap_ctr_skip_\@ + ldr x29, [x12, #(SHIM_MMAP_COUNTERS_OFF + 8 * \slot)] + add x29, x29, #1 + str x29, [x12, #(SHIM_MMAP_COUNTERS_OFF + 8 * \slot)] +.Lmmap_ctr_skip_\@: +.endm + +.macro MPROTECT_COUNTER_INC byte_off + mrs x29, tpidr_el1 + ldrb w30, [x29, #SHIM_STATS_EN_OFF] + cbz w30, .Lmprotect_ctr_skip_\@ + ldr x29, [x21, #\byte_off] + add x29, x29, #1 + str x29, [x21, #\byte_off] +.Lmprotect_ctr_skip_\@: +.endm + /* BAD_VEC: vector-table entry that reports an unexpected exception. * Each table slot is 128 bytes; the leading .align 7 places this entry at the * next 128-byte boundary. @@ -440,6 +472,1093 @@ svc_handler: b.eq getsid_fast cmp x10, #278 /* SYS_getrandom? */ b.eq getrandom_fast + cmp x10, #215 /* SYS_munmap? */ + b.eq munmap_anon_fast + cmp x10, #222 /* SYS_mmap? */ + b.eq mmap_anon_fast + cmp x10, #226 /* SYS_mprotect? */ + b.eq mprotect_page_fast + b handle_svc_0 + +/* mprotect_page_fast: change one L3 page between R and RW without exiting to + * the host. Besides an existing L3 leaf, EL1 may consume a host-prepared table + * page to split an existing L2 block or materialize a clean 2 MiB lazy block. + * The latter is allowed only while an undrained mmap publication proves that + * this vCPU owns the complete block; this prevents neighboring arena holes + * from becoming accessible. PTE + broadcast TLBI complete synchronously here; + * a tagged mmap-ring entry defers host region/index reconciliation. + */ +mprotect_page_fast: + mrs x12, tpidr_el1 + ldar w13, [x12] /* attention / tracing gate */ + cbnz w13, handle_svc_0 + + add x20, sp, #0xfff + and x20, x20, #~0xfff /* this vCPU's stack top */ + add x21, x12, #0x200, lsl #12 + sub x21, x21, x20 + add x21, x12, x21 + add x21, x21, #0x20, lsl #12 /* current mmap control */ + + ldr x14, [sp, #0] /* addr */ + tst x14, #0xfff + b.ne handle_svc_0 + ldr x15, [sp, #8] /* Linux accepts len 1..4096 */ + cbz x15, handle_svc_0 + cmp x15, #0x1000 + b.hi handle_svc_0 + ldr x16, [sp, #16] /* prot */ + cmp x16, #1 /* PROT_READ */ + b.eq 1f + cmp x16, #3 /* PROT_READ|PROT_WRITE */ + b.ne handle_svc_0 + +1: /* Join the host-writer gate before validating the control or PTE. */ + add x20, x12, #0x1, lsl #12 + add x20, x20, #0x160 /* SHIM_MMAP_PT_GATE_OFF */ + ldar w22, [x20] + cbnz w22, handle_svc_0 + add x24, x21, #0x400 + add x24, x24, #24 /* retire.producer_active */ + mov w22, #1 + stlr w22, [x24] + ldar w22, [x20] + cbnz w22, mprotect_active_bail + + ldar w22, [x21] /* generation */ + ldr w23, [x21, #4] /* consumer generation */ + cmp w22, w23 + b.ne mprotect_active_bail + ldr w23, [x21, #8] /* control flags */ + tbz w23, #0, mprotect_active_bail + + ldr x25, [x21, #24] /* arena base */ + cmp x14, x25 + b.lo mprotect_active_bail + ldr x25, [x21, #40] /* consumed cursor */ + add x17, x14, #0x1000 + cmp x17, x25 + b.hi mprotect_active_bail + + /* Reserve the same SPSC publication ring used by fast mmap. */ + add x23, x21, #12 /* host head */ + ldar w22, [x23] + ldr w23, [x21, #16] /* producer tail */ + sub w25, w23, w22 + cmp w25, #32 + b.hs mprotect_active_bail + + /* Walk TTBR0. Arena refill has prepared the upper chain for the first + * clean window, while older/out-of-window mappings may still bail here. + */ + mrs x25, ttbr0_el1 + ubfx x25, x25, #12, #36 + lsl x25, x25, #12 + ubfx x26, x14, #39, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + and x28, x27, #3 + cmp x28, #3 + b.ne mprotect_split_prep_bail + ubfx x25, x27, #12, #36 + lsl x25, x25, #12 + ubfx x26, x14, #30, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + and x28, x27, #3 + cmp x28, #3 + b.ne mprotect_split_prep_bail + ubfx x25, x27, #12, #36 + lsl x25, x25, #12 + ubfx x26, x14, #21, #9 + add x26, x25, x26, lsl #3 /* target L2 entry */ + ldar x27, [x26] + and x28, x27, #3 + cmp x28, #3 + b.eq mprotect_existing_l3 + cmp x28, #1 + b.eq mprotect_install_l3_block + cbnz x28, mprotect_active_bail + mov w20, #0 /* needs a prepared table page */ + +mprotect_validate_lazy_block: + /* An invalid L2 slot can be materialized only when the host's snapshot + * says the identity backing is clean and a still-pending mmap publication + * covers this whole 2 MiB block. + */ + lsr x11, x14, #21 + lsl x11, x11, #21 /* block start */ + mov x17, x11 /* retain exact block for bookkeeping */ + ldr x12, [x21, #0x950] /* clean-window base */ + cmp x11, x12 + b.lo mprotect_split_prep_bail + sub x13, x11, x12 + lsr x13, x13, #21 /* clean-window bit */ + cmp x13, #32 + b.hs mprotect_split_prep_bail + ldr x12, [x21, #0x958] /* clean bitmap */ + lsrv x12, x12, x13 + and w9, w12, #1 /* full-block materialization is safe */ + + mov w10, w22 /* host head snapshot */ +mprotect_find_full_block_mapping: + cmp w10, w23 + b.eq mprotect_split_owner_bail + and w12, w10, #31 + add x12, x12, x12, lsl #1 + add x13, x21, #64 + add x12, x13, x12, lsl #3 + ldr x13, [x12, #16] /* mmap prot / entry tag */ + tbnz x13, #63, mprotect_find_full_block_next + cmp x13, #3 + b.ne mprotect_find_full_block_next + ldr x13, [x12, #0] /* mapping start */ + cmp x14, x13 + b.lo mprotect_find_full_block_next + ldr x12, [x12, #8] /* mapping length */ + adds x12, x13, x12 + b.cs mprotect_active_bail + add x15, x14, #0x1000 + cmp x12, x15 + b.lo mprotect_find_full_block_next + cbz w9, mprotect_publish_lazy_only + cmp x11, x13 + b.lo mprotect_find_full_block_next + add x13, x11, #0x200000 + cmp x12, x13 + b.hs mprotect_install_l3_lazy +mprotect_find_full_block_next: + add w10, w10, #1 + b mprotect_find_full_block_mapping + +mprotect_install_l3_block: + mov w18, #0 /* split preserves materialization */ + mov x10, x27 + orr x10, x10, #2 /* L2 block -> L3 page type */ + b mprotect_install_l3_common + +mprotect_install_l3_lazy: + mov w18, #1 /* first lazy materialization */ + mov x10, #0x767 /* AF|ISH|NS|ATTR1|RW_EL0|PAGE */ + movk x10, #0x60, lsl #48 /* UXN|PXN */ + orr x10, x10, x11 /* first identity page */ + cbnz w20, mprotect_fill_existing_empty_l3 + +mprotect_install_l3_common: + ldr x25, [x21, #0x940] /* prepared L3 page */ + ldr x29, [x21, #0x948] + cmp x25, x29 + b.hs mprotect_split_prep_bail + add x29, x25, #0x1000 + str x29, [x21, #0x940] /* sole EL1 consumer */ + MPROTECT_COUNTER_INC 0x970 + mov x19, x26 /* preserve target L2 entry */ + mov x12, x25 /* L3 fill cursor */ + mov x13, #512 +mprotect_fill_l3: + str x10, [x12], #8 + add x10, x10, #0x1000 + subs x13, x13, #1 + b.ne mprotect_fill_l3 + + /* Tighten the requested page before publishing the table. */ + ubfx x28, x14, #12, #9 + add x26, x25, x28, lsl #3 + ldr x27, [x26] + bic x27, x27, #0xc0 + cmp x16, #3 + mov x28, #0xc0 /* AP=11: EL0 read-only */ + mov x29, #0x40 /* AP=01: EL0 read-write */ + csel x28, x29, x28, eq + orr x27, x27, x28 + str x27, [x26] + orr x27, x25, #3 /* L2 table descriptor */ + stlr x27, [x19] + + b mprotect_lazy_l3_installed + +mprotect_fill_existing_empty_l3: + /* A previous materialize/retire cycle may leave an empty L3 table in the + * L2 slot. Prove every entry is zero before reusing it. The target page is + * installed with its final AP first so a sibling can never observe a + * transient RW permission while this mprotect requests read-only. + */ + mov x12, x25 + mov x13, #512 +1: ldr x27, [x12], #8 + cbnz x27, mprotect_split_prep_bail + subs x13, x13, #1 + b.ne 1b + + ubfx x28, x14, #12, #9 /* target page index */ + lsl x29, x28, #12 + add x27, x10, x29 /* target page descriptor */ + bic x27, x27, #0xc0 + cmp x16, #3 + mov x29, #0xc0 + mov x19, #0x40 + csel x29, x19, x29, eq + orr x27, x27, x29 + add x26, x25, x28, lsl #3 + stlr x27, [x26] + + mov x12, x25 + mov x13, #0 /* current page index */ +2: cmp x13, x28 + b.eq 3f + str x10, [x12] +3: add x12, x12, #8 + add x10, x10, #0x1000 + add x13, x13, #1 + cmp x13, #512 + b.lo 2b + MPROTECT_COUNTER_INC 0x970 + +mprotect_lazy_l3_installed: + + cbz w18, mprotect_pte_changed + + /* Make the materialized envelope visible before a following fast munmap, + * and consume the clean proof so it cannot authorize a second install. + */ + ldr w22, [x21] + ldr w12, [x21, #0x388] + add x13, x11, #0x200000 + cmp w12, w22 + b.ne 1f + ldr x10, [x21, #0x390] + ldr x12, [x21, #0x398] + cmp x10, x11 + csel x11, x10, x11, lo + cmp x12, x13 + csel x13, x12, x13, hi +1: str x11, [x21, #0x390] + str x13, [x21, #0x398] + add x10, x21, #0x388 + stlr w22, [x10] + ldr x10, [x21, #0x950] + sub x10, x17, x10 + lsr x10, x10, #21 + mov x12, #1 + lsl x12, x12, x10 + ldr x13, [x21, #0x958] + bic x13, x13, x12 + str x13, [x21, #0x958] + b mprotect_pte_changed + +mprotect_existing_l3: + ubfx x25, x27, #12, #36 + lsl x25, x25, #12 + ubfx x28, x14, #12, #9 + add x26, x25, x28, lsl #3 /* target L3 entry */ + ldar x27, [x26] + tbnz x27, #0, 1f + mov w20, #1 /* reuse this published empty table */ + b mprotect_validate_lazy_block +1: + mov w18, #0 + + /* Preserve output address and attributes; replace only AP[2:1]. */ + bic x27, x27, #0xc0 + cmp x16, #3 + mov x28, #0xc0 /* AP=11: EL0 read-only */ + mov x29, #0x40 /* AP=01: EL0 read-write */ + csel x28, x29, x28, eq + orr x27, x27, x28 + stlr x27, [x26] + +mprotect_pte_changed: + dsb ishst + cbnz w18, mprotect_tlbi_lazy_block + ubfx x25, x14, #12, #44 + tlbi vae1is, x25 + b mprotect_tlbi_done +mprotect_tlbi_lazy_block: + ldr w25, [x21, #8] + tbz w25, #1, mprotect_tlbi_full + lsr x25, x14, #21 + lsl x25, x25, #9 /* block VA >> 12 */ + mov x27, #7 /* 8 x 256 KiB scale units */ + lsl x27, x27, #39 /* NUM */ + orr x25, x25, x27 + mov x27, #1 + lsl x27, x27, #44 /* SCALE=1 */ + orr x25, x25, x27 + mov x27, #1 + lsl x27, x27, #46 /* TG=01 (4 KiB) */ + orr x25, x25, x27 + tlbi rvae1is, x25 + b mprotect_tlbi_done +mprotect_tlbi_full: + tlbi vmalle1is +mprotect_tlbi_done: + dsb ish + isb + +mprotect_publish_entry: + /* Tag the publication as metadata-only mprotect. Host drains it before + * consulting regions[] on any later VM exit. + */ + and w22, w23, #31 + add x22, x22, x22, lsl #1 + add x25, x21, #64 + add x25, x25, x22, lsl #3 + str x14, [x25, #0] + mov x22, #0x1000 + str x22, [x25, #8] + mov x22, #1 + lsl x22, x22, #63 + orr x22, x22, x16 + cmp w18, #1 + b.ne 1f + mov x26, #1 + lsl x26, x26, #62 + orr x22, x22, x26 +1: cmp w18, #2 + b.ne 2f + mov x26, #1 + lsl x26, x26, #61 + orr x22, x22, x26 +2: + str x22, [x25, #16] + add w23, w23, #1 + add x25, x21, #16 + stlr w23, [x25] + + stlr wzr, [x24] /* producer_active = 0 */ + mov x0, #0 + b svc_restore_eret + +mprotect_publish_lazy_only: + /* The target belongs to a pending anonymous mapping but its backing is not + * proven clean. Keep the PTE invalid and defer only the VMA permission + * update; a later fault drains this entry before materializing the page. + */ + MPROTECT_COUNTER_INC 0x978 + mov w18, #2 + b mprotect_publish_entry + +mprotect_active_bail: + stlr wzr, [x24] + b handle_svc_0 + +mprotect_split_prep_bail: + MPROTECT_COUNTER_INC 0x960 + b mprotect_active_bail + +mprotect_split_owner_bail: + MPROTECT_COUNTER_INC 0x968 + b mprotect_active_bail + +/* mmap_anon_fast: consume a host-prepared, per-vCPU VA arena for the exact + * anonymous RW shape used by allocators: + * + * mmap(NULL, len, PROT_READ|PROT_WRITE, + * MAP_PRIVATE|MAP_ANONYMOUS[|MAP_NORESERVE], fd, off) + * + * The control is selected without another system register. SP_EL1 stack tops + * are shim_data_end - slot*4KiB; controls are shim_data+0x20000 + slot*4KiB, + * so shim_data_end - ALIGN_UP(sp,4KiB) is the control-array displacement. + * Each vCPU is the sole producer of its mmap publication ring and cursor, and + * the sole consumer of its host-committed reuse ring. The host acquire-drains + * all publications whenever it takes mmap_lock. + */ +mmap_anon_fast: + mrs x12, tpidr_el1 /* shared shim-data base */ + ldar w13, [x12] /* tracing/signal attention gate */ + + add x14, sp, #0xfff + and x14, x14, #~0xfff /* this vCPU's stack top */ + add x15, x12, #0x200, lsl #12 /* shim_data + 2MiB */ + sub x15, x15, x14 /* slot * 4KiB */ + add x12, x12, x15 + add x12, x12, #0x20, lsl #12 /* control base + 0x20000 */ + + cbnz w13, mmap_attention_bail + + ldr x14, [sp, #0] /* addr */ + cbnz x14, mmap_shape_bail + ldr x16, [sp, #16] /* prot */ + cmp x16, #3 /* PROT_READ|PROT_WRITE */ + b.ne mmap_shape_bail + ldr x17, [sp, #24] /* flags */ + bic x18, x17, #0x4000 /* tolerate MAP_NORESERVE */ + cmp x18, #0x22 /* MAP_PRIVATE|MAP_ANONYMOUS */ + b.ne mmap_shape_bail + + ldr x15, [sp, #8] /* len */ + cbz x15, mmap_shape_bail + adds x15, x15, #0xfff + b.cs mmap_shape_bail /* PAGE_ALIGN_UP overflow */ + and x15, x15, #~0xfff + + /* Join the same host-writer handshake as fast munmap. Arena revoke, + * refill, and reuse publication all run with the gate closed; announcing + * this per-vCPU producer prevents the host from invalidating a descriptor + * between our generation check and mmap publication. */ + mrs x20, tpidr_el1 + add x20, x20, #0x1, lsl #12 + add x20, x20, #0x160 /* SHIM_MMAP_PT_GATE_OFF */ + ldar w22, [x20] + cbnz w22, mmap_gate_bail + add x24, x12, #0x400 + add x24, x24, #24 /* retire.producer_active */ + mov w22, #1 + stlr w22, [x24] + ldar w22, [x20] + cbnz w22, mmap_active_capacity_bail + + ldar w13, [x12] /* descriptor generation */ + ldr w14, [x12, #4] /* consumer generation */ + cmp w13, w14 + b.ne mmap_generation_bail + ldr w13, [x12, #8] /* SHIM_MMAP_CTRL_ENABLED */ + tbz w13, #0, mmap_capacity_bail + + /* Reserve publication capacity before mutating either the bump cursor or + * a host-committed reuse extent. */ + add x23, x12, #12 /* &mmap publication head */ + ldar w22, [x23] + ldr w23, [x12, #16] /* producer-owned tail */ + sub w24, w23, w22 + cmp w24, #32 /* SHIM_MMAP_RING_SIZE */ + b.hs mmap_ring_full_bail + + /* Prefer committed holes returned by the host. The host release-publishes + * immutable entries through reuse.tail; after acquiring tail, this EL1 + * consumer owns every pending slot and may shorten it in place. First-fit + * handles mixed allocation sizes. Large requests retain the bump path's + * 2 MiB start-alignment promise; an unaligned extent is simply skipped. */ + add x26, x12, #0x720 /* host -> EL1 reuse ring */ + ldr w27, [x26, #0] /* consumer-owned head */ + add x25, x26, #4 + ldar w28, [x25] /* host-published tail */ + mov w29, w27 +mmap_reuse_scan: + cmp w29, w28 + b.eq mmap_reuse_miss + and w30, w29, #31 + add x25, x26, #32 /* reuse.entries */ + add x25, x25, x30, lsl #4 + ldr x19, [x25, #0] /* extent address */ + ldr x24, [x25, #8] /* extent length; zero = consumed */ + cbz x24, mmap_reuse_next + cmp x24, x15 + b.lo mmap_reuse_next + mov x20, #0x200000 + cmp x15, x20 + b.lo mmap_reuse_fit + sub x20, x20, #1 + tst x19, x20 + b.ne mmap_reuse_next +mmap_reuse_fit: + adds x21, x19, x15 + b.cs mmap_reuse_next /* corrupt extent cannot be used */ + sub x24, x24, x15 + str x21, [x25, #0] /* retain suffix in this slot */ + str x24, [x25, #8] /* zero marks full consumption */ + ldr x20, [x26, #24] /* per-vCPU reuse hits */ + add x20, x20, #1 + str x20, [x26, #24] + mov w28, #0 /* address came from reuse ring */ + b mmap_addr_ready +mmap_reuse_next: + add w29, w29, #1 + b mmap_reuse_scan + +mmap_reuse_miss: + ldr x19, [x12, #40] /* private bump cursor */ + mov x20, #0x200000 + cmp x15, x20 + b.lo 1f + sub x21, x20, #1 + adds x19, x19, x21 + b.cs mmap_capacity_bail + bic x19, x19, x21 /* >=2MiB: 2MiB-align */ +1: ldr x20, [x12, #32] /* arena limit */ + adds x21, x19, x15 /* new cursor */ + b.cs mmap_capacity_bail + cmp x21, x20 + b.hi mmap_capacity_bail + mov w28, #1 /* bump cursor must advance */ + +mmap_addr_ready: + and w24, w23, #31 + add x24, x24, x24, lsl #1 /* index * 3 */ + add x25, x12, #64 /* ring base */ + add x25, x25, x24, lsl #3 /* index * 24 */ + str x19, [x25, #0] + str x15, [x25, #8] + str x16, [x25, #16] + cbz w28, 2f + str x21, [x12, #40] /* publish bump cursor before entry */ +2: + add w23, w23, #1 + add x25, x12, #16 + stlr w23, [x25] /* release-publish ring entry */ + + /* Only after the mmap publication is visible may fully consumed reuse + * slots be released to the host producer. Skip zero tombstones at the + * head; first-fit may have consumed a later slot, which will be reclaimed + * when all earlier entries have drained. */ + add x26, x12, #0x720 + ldr w27, [x26, #0] + add x25, x26, #4 + ldar w29, [x25] + mov w30, w27 +3: cmp w30, w29 + b.eq 5f + and w24, w30, #31 + add x25, x26, #32 + add x25, x25, x24, lsl #4 + ldr x24, [x25, #8] + cbnz x24, 5f + add w30, w30, #1 + b 3b +5: cmp w30, w27 + b.eq 6f + stlr w30, [x26] +6: + add x24, x12, #0x400 + add x24, x24, #24 + stlr wzr, [x24] /* producer_active = 0 */ + mov x0, x19 + MMAP_COUNTER_INC MC_HIT + b svc_restore_eret + +mmap_shape_bail: + MMAP_COUNTER_INC MC_SHAPE_MISS + b handle_svc_0 + +mmap_capacity_bail: + add x24, x12, #0x400 + add x24, x24, #24 + stlr wzr, [x24] + MMAP_COUNTER_INC MC_CAPACITY_MISS + b handle_svc_0 + +mmap_ring_full_bail: + add x24, x12, #0x400 + add x24, x24, #24 + stlr wzr, [x24] + MMAP_COUNTER_INC MC_RING_FULL + b handle_svc_0 + +mmap_active_capacity_bail: + add x24, x12, #0x400 + add x24, x24, #24 + stlr wzr, [x24] +mmap_gate_bail: + MMAP_COUNTER_INC MC_CAPACITY_MISS + b handle_svc_0 + +mmap_attention_bail: + MMAP_COUNTER_INC MC_ATTENTION + b attn_bail + +mmap_generation_bail: + /* A revocation deliberately leaves the consumer generation stale. Ack + * only after the acquire load above, then make this syscall take HVC; the + * host will either leave the control disabled or publish a fresh arena. + */ + MMAP_COUNTER_INC MC_GENERATION_STALE + str w13, [x12, #4] + add x24, x12, #0x400 + add x24, x24, #24 + stlr wzr, [x24] + b handle_svc_0 + +/* munmap_anon_fast: invalidate an anonymous arena range synchronously at EL1, + * then release-publish a metadata retirement to this vCPU's SPSC ring. The + * host consumes it at a later natural VM exit. No shared allocator cursor is + * modified here. + * + * The two-pass page-table walk is deliberate. Pass one proves every partial + * 2 MiB block already has an L3 table, so pass two cannot discover a need to + * split after it has cleared an earlier descriptor. Whole L2 blocks and L3 + * leaves are the only descriptors this path writes. + */ +munmap_anon_fast: + mrs x12, tpidr_el1 /* shim-data base */ + ldar w13, [x12] /* attention gate */ + + add x20, sp, #0xfff + and x20, x20, #~0xfff /* this vCPU's stack top */ + add x21, x12, #0x200, lsl #12 + sub x21, x21, x20 + add x21, x12, x21 + add x21, x21, #0x20, lsl #12 /* current mmap control */ + + cbnz w13, munmap_pre_bail + ldr x14, [sp, #0] /* addr */ + ldr x15, [sp, #8] /* length */ + cbz x15, munmap_pre_bail + tst x14, #0xfff + b.ne munmap_pre_bail + tst x15, #0xfff + b.ne munmap_pre_bail + adds x16, x14, x15 /* exclusive end */ + b.cs munmap_pre_bail + + add x17, x21, #0x400 /* private retire ring */ + add x20, x17, #0 + ldar w20, [x20] /* host head */ + ldr w19, [x17, #4] /* producer tail */ + sub w22, w19, w20 + cmp w22, #31 /* near-full => batched HVC drain */ + b.hs munmap_pre_bail + ldr x18, [x17, #8] /* producer byte sequence */ + + /* Join the per-vCPU side of the host-writer handshake. The second gate + * read closes the check-vs-announce race: if host closed it in between, + * withdraw without touching a PTE. + */ + mrs x20, tpidr_el1 + add x20, x20, #0x1, lsl #12 + add x20, x20, #0x160 /* SHIM_MMAP_PT_GATE_OFF */ + ldar w22, [x20] + cbnz w22, munmap_pre_bail + mov w22, #1 + add x25, x17, #24 + stlr w22, [x25] /* producer_active = 1 */ + ldar w22, [x20] + cbnz w22, munmap_active_bail + + /* Locate the one arena generation containing the whole range. Scanning + * 64 cache-cold descriptors is only the cross-vCPU case; the owner's + * control normally matches early. generation is re-read after bounds so + * a descriptor publication can never be observed torn. + */ + mrs x25, tpidr_el1 + add x25, x25, #0x20, lsl #12 /* control[0] */ + mov w24, #0 /* arena slot */ +1: add x26, x25, #8 + ldar w23, [x26] /* arena flags */ + tbz w23, #0, 2f + tbnz w23, #2, 2f /* host mprotect => slow munmap */ + ldar w22, [x25] /* generation snapshot */ + ldr x26, [x25, #24] /* arena base */ + cmp x14, x26 + b.lo 2f + add x27, x25, #40 + ldar x27, [x27] /* consumed cursor */ + cmp x16, x27 + b.hi 2f + ldar w28, [x25] + cmp w22, w28 + b.eq munmap_arena_found +2: add w24, w24, #1 + add x25, x25, #0x1000 + cmp w24, #64 + b.lo 1b + b munmap_active_bail + +munmap_arena_found: + /* Reserve and fill, but do not advance tail until PTE+TLBI completion. */ + and w20, w19, #31 + add x20, x20, x20, lsl #1 /* index * 3 */ + add x26, x17, #32 /* retire.entries */ + add x26, x26, x20, lsl #3 /* index * 24 */ + str x14, [x26, #0] + str x15, [x26, #8] + str w22, [x26, #16] + str w24, [x26, #20] + + /* Refill invalidates every stale descriptor before publishing a new + * generation. Until the host records a lazy materialization in that + * generation, the entire arena is known PTE-empty and no walk/TLBI is + * needed. The host PT gate makes the generation marker stable here. + */ + add x20, x25, #0x388 + ldar w20, [x20] + cmp w20, w22 + b.ne munmap_publish + ldr x11, [x25, #0x390] /* materialized lower bound */ + ldr x12, [x25, #0x398] /* materialized upper bound */ + cmp x11, x14 + csel x11, x11, x14, hi /* max(addr, materialized_start) */ + cmp x12, x16 + csel x12, x12, x16, lo /* min(end, materialized_end) */ + cmp x11, x12 + b.hs munmap_publish + + /* Common large-anonymous case: the materialized envelope is 2 MiB + * aligned and consists only of L2 block descriptors (or holes retired by + * a sibling). Walk L0 once, then L1 once per 1 GiB window and scan the L2 + * entries linearly. The generic path below remains the fallback for any + * L3 table or unusual upper-level shape. + */ + mov w10, #0 /* generic mutation by default */ + orr x20, x11, x12 + tst x20, #0x1fffff + b.ne munmap_validate_generic + sub x20, x12, #1 + eor x20, x20, x11 + lsr x20, x20, #39 /* must stay in one L0 slot */ + cbnz x20, munmap_validate_generic + + mov x13, #0 + mrs x25, ttbr0_el1 + ubfx x25, x25, #12, #36 + lsl x25, x25, #12 /* L0 table */ + ubfx x26, x11, #39, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + and x28, x27, #3 + cbz x28, munmap_validate_l2_fast_done + cmp x28, #3 + b.ne munmap_validate_generic + ubfx x25, x27, #12, #36 + lsl x25, x25, #12 /* L1 table */ + + mov x20, x11 +munmap_validate_l2_fast_window: + ubfx x26, x20, #30, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + and x28, x27, #3 + lsr x29, x20, #30 + add x29, x29, #1 + lsl x30, x29, #30 /* next 1 GiB boundary */ + cmp x30, x12 + csel x30, x30, x12, lo /* this window's end */ + cbz x28, munmap_validate_l2_fast_next_window + cmp x28, #3 + b.ne munmap_validate_generic + ubfx x26, x27, #12, #36 + lsl x26, x26, #12 /* L2 table */ + ubfx x29, x20, #21, #9 + add x26, x26, x29, lsl #3 +munmap_validate_l2_fast_entry: + ldr x27, [x26] + and x28, x27, #3 + cbz x28, 1f + cmp x28, #1 /* L2 block, never L3 table */ + b.ne munmap_validate_generic + add x13, x13, #512 +1: add x26, x26, #8 + add x20, x20, #0x200000 + cmp x20, x30 + b.lo munmap_validate_l2_fast_entry + b 2f +munmap_validate_l2_fast_next_window: + mov x20, x30 +2: cmp x20, x12 + b.lo munmap_validate_l2_fast_window +munmap_validate_l2_fast_done: + mov w10, #1 + b munmap_clear_begin + + /* Validation pass. x20 is the current VA; x13 counts valid target pages. + * Pending bytes are charged by materialized backing rather than virtual + * length. Crossing the soft threshold records cleanup_requested but never + * forces this producer to HVC; another natural exit will consume the ring. + */ +munmap_validate_generic: + mov w10, #0 + mov x13, #0 + mov x20, x11 +munmap_validate_block: + cmp x20, x12 + b.hs munmap_clear_begin + mrs x25, ttbr0_el1 + ubfx x25, x25, #12, #36 + lsl x25, x25, #12 + ubfx x26, x20, #39, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + and x28, x27, #3 + cbz x28, munmap_validate_absent + cmp x28, #3 + b.ne munmap_active_bail + ubfx x25, x27, #12, #36 + lsl x25, x25, #12 + ubfx x26, x20, #30, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + and x28, x27, #3 + cbz x28, munmap_validate_absent + cmp x28, #3 + b.ne munmap_active_bail + ubfx x25, x27, #12, #36 + lsl x25, x25, #12 + ubfx x26, x20, #21, #9 + add x26, x25, x26, lsl #3 /* &L2[index] */ + ldar x27, [x26] + and x28, x27, #3 + cbz x28, munmap_validate_absent + cmp x28, #3 + b.eq munmap_validate_l3 + cmp x28, #1 + b.ne munmap_active_bail + /* An L2 block can only be cleared as a whole. */ + lsr x29, x20, #21 + lsl x29, x29, #21 /* block start */ + cmp x14, x29 + b.hi munmap_active_bail + add x30, x29, #0x200000 + cmp x16, x30 + b.lo munmap_active_bail + add x13, x13, #512 + b munmap_validate_advance + +munmap_validate_l3: + ubfx x25, x27, #12, #36 /* L3 table VA */ + lsl x25, x25, #12 + lsr x29, x20, #21 + lsl x29, x29, #21 + add x30, x29, #0x200000 + cmp x30, x12 + csel x30, x30, x12, lo +1: ubfx x26, x20, #12, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + tbz x27, #0, 2f + add x13, x13, #1 +2: add x20, x20, #0x1000 + cmp x20, x30 + b.lo 1b + b munmap_validate_block + +munmap_validate_absent: + lsr x29, x20, #21 + add x29, x29, #1 + lsl x20, x29, #21 + b munmap_validate_block +munmap_validate_advance: + mov x20, x30 + b munmap_validate_block + + /* Mutation pass. Host writers are gated; sibling fast munmaps can only + * change the same descriptors to zero, so repeated invalidation is safe. + */ +munmap_clear_begin: + /* Encode the charged 4 KiB page count in the retire flags so the host's + * single-writer consumed sequence advances by exactly the same amount. + */ + and w20, w19, #31 + add x20, x20, x20, lsl #1 + add x26, x17, #32 + add x26, x26, x20, lsl #3 + lsl w29, w13, #6 + orr w29, w29, w24 + str w29, [x26, #20] + cbz x13, munmap_publish + + add x20, x17, #16 + ldar x20, [x20] /* host-consumed byte sequence */ + sub x29, x18, x20 /* pending materialized bytes */ + lsl x30, x13, #12 /* this retirement's charge */ + adds x29, x29, x30 + b.cs munmap_active_bail + adds x18, x18, x30 /* next producer byte sequence */ + b.cs munmap_active_bail + mov x20, #0x10000000 /* 256 MiB soft advisory */ + cmp x29, x20 + b.ls 1f + mov w20, #1 + add x25, x17, #28 + stlr w20, [x25] /* never makes this vCPU exit */ +1: + + cbnz w10, munmap_clear_l2_fast_begin + mov x20, x11 +munmap_clear_block: + cmp x20, x12 + b.hs munmap_tlbi + mrs x25, ttbr0_el1 + ubfx x25, x25, #12, #36 + lsl x25, x25, #12 + ubfx x26, x20, #39, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + and x28, x27, #3 + cmp x28, #3 + b.ne munmap_clear_next_block + ubfx x25, x27, #12, #36 + lsl x25, x25, #12 + ubfx x26, x20, #30, #9 + add x26, x25, x26, lsl #3 + ldar x27, [x26] + and x28, x27, #3 + cmp x28, #3 + b.ne munmap_clear_next_block + ubfx x25, x27, #12, #36 + lsl x25, x25, #12 + ubfx x26, x20, #21, #9 + add x26, x25, x26, lsl #3 /* &L2[index] */ + ldar x27, [x26] + and x28, x27, #3 + cbz x28, munmap_clear_next_block + cmp x28, #1 + b.eq munmap_clear_l2 + cmp x28, #3 + b.ne munmap_active_bail /* validation invariant */ + + ubfx x25, x27, #12, #36 /* L3 table VA */ + lsl x25, x25, #12 + lsr x29, x20, #21 + lsl x29, x29, #21 /* block start */ + add x30, x29, #0x200000 + cmp x30, x12 + csel x30, x30, x12, lo /* page-loop end */ +munmap_clear_l3_page: + ubfx x26, x20, #12, #9 + add x26, x25, x26, lsl #3 + stlr xzr, [x26] + add x20, x20, #0x1000 + cmp x20, x30 + b.lo munmap_clear_l3_page + b munmap_clear_block + +munmap_clear_l2: + stlr xzr, [x26] +munmap_clear_next_block: + lsr x29, x20, #21 + add x29, x29, #1 + lsl x20, x29, #21 + b munmap_clear_block + +munmap_clear_l2_fast_begin: + /* Validation proved one L0 table and L2-block-only leaves. Re-walk L1 once + * per 1 GiB window, then use ordinary aligned 64-bit stores. Those stores + * are atomic; the DSB ISHST below supplies the required publication order, + * so a release store on every descriptor is unnecessary. + */ + mrs x25, ttbr0_el1 + ubfx x25, x25, #12, #36 + lsl x25, x25, #12 + ubfx x26, x11, #39, #9 + add x26, x25, x26, lsl #3 + ldr x27, [x26] + ubfx x25, x27, #12, #36 + lsl x25, x25, #12 /* L1 table */ + mov x20, x11 +munmap_clear_l2_fast_window: + ubfx x26, x20, #30, #9 + add x26, x25, x26, lsl #3 + ldr x27, [x26] + and x28, x27, #3 + lsr x29, x20, #30 + add x29, x29, #1 + lsl x30, x29, #30 + cmp x30, x12 + csel x30, x30, x12, lo + cbz x28, munmap_clear_l2_fast_next_window + ubfx x26, x27, #12, #36 + lsl x26, x26, #12 + ubfx x29, x20, #21, #9 + add x26, x26, x29, lsl #3 +munmap_clear_l2_fast_entry: + /* Validation plus the closed host gate proves this slot is a block or + * already zero. A sibling producer can only move it toward zero, so an + * unconditional aligned store is safe and avoids a second load/branch. + */ + str xzr, [x26] + add x26, x26, #8 + add x20, x20, #0x200000 + cmp x20, x30 + b.lo munmap_clear_l2_fast_entry + b 2f +munmap_clear_l2_fast_next_window: + mov x20, x30 +2: cmp x20, x12 + b.lo munmap_clear_l2_fast_window + b munmap_tlbi + +munmap_tlbi: + /* Descriptor stores -> broadcast invalidation -> completed visibility, + * before the retire tail release makes metadata cleanup eligible. + */ + dsb ishst + sub x20, x12, x11 + lsr x20, x20, #12 /* changed-page envelope */ + cmp x20, #8 + b.hi munmap_tlbi_large + ubfx x25, x11, #12, #44 +3: tlbi vae1is, x25 + add x25, x25, #1 + subs x20, x20, #1 + b.ne 3b + b munmap_tlbi_done + +munmap_tlbi_large: + /* Encode SCALE=0..3 RVAE1IS batches directly from the changed envelope. + * Each instruction covers up to 32 scale units: 64 pages at SCALE=0, + * 2048 at SCALE=1, 65536 at SCALE=2, and 2097152 (8 GiB) at SCALE=3. + * Widening to the scale-unit boundary only invalidates neighboring TLB + * entries; it cannot change mappings. Ranges larger than one instruction + * are emitted as adjacent batches while retaining a single completion DSB. + */ + tbz w23, #1, munmap_tlbi_full + cmp x20, #64 + b.hi 1f + mov x27, #13 /* 2 pages = 8 KiB */ + mov x28, #0 /* SCALE=0 */ + b munmap_tlbi_rvae_normalize +1: cmp x20, #0x800 + b.hi 2f + mov x27, #18 /* 64 pages = 256 KiB */ + mov x28, #1 /* SCALE=1 */ + b munmap_tlbi_rvae_normalize +2: mov x29, #0x10000 + cmp x20, x29 + b.hi 3f + mov x27, #23 /* 2048 pages = 8 MiB */ + mov x28, #2 /* SCALE=2 */ + b munmap_tlbi_rvae_normalize +3: mov x27, #28 /* 65536 pages = 256 MiB */ + mov x28, #3 /* SCALE=3 */ + +munmap_tlbi_rvae_normalize: + mov x29, #1 + lsl x29, x29, x27 + sub x29, x29, #1 /* scale-unit byte mask */ + bic x25, x11, x29 /* widened start */ + adds x26, x12, x29 + b.cs munmap_tlbi_full + bic x26, x26, x29 /* widened end */ +munmap_tlbi_rvae_batch: + sub x20, x26, x25 + lsrv x20, x20, x27 /* remaining scale units */ + mov x30, #32 + cmp x20, x30 + csel x20, x20, x30, lo /* this instruction's units */ + ubfx x29, x25, #12, #37 /* BaseADDR */ + sub x30, x20, #1 /* NUM */ + lsl x30, x30, #39 + orr x29, x29, x30 + lsl x30, x28, #44 /* SCALE */ + orr x29, x29, x30 + mov x30, #1 + lsl x30, x30, #46 /* TG=01 (4 KiB) */ + orr x29, x29, x30 + tlbi rvae1is, x29 + lslv x20, x20, x27 + add x25, x25, x20 + cmp x25, x26 + b.lo munmap_tlbi_rvae_batch + b munmap_tlbi_done +munmap_tlbi_full: + tlbi vmalle1is +munmap_tlbi_done: + dsb ish + isb + +munmap_publish: + str x18, [x17, #8] /* publish producer byte sequence */ + add w19, w19, #1 + add x20, x17, #4 + stlr w19, [x20] /* publish retire record */ + add x20, x17, #24 + stlr wzr, [x20] /* leave writer handshake */ + mov x0, #0 + b svc_restore_eret + +munmap_active_bail: + add x20, x17, #24 + stlr wzr, [x20] +munmap_pre_bail: b handle_svc_0 identity_class_fast: @@ -1203,6 +2322,7 @@ handle_el0_fault: .Lel0_fault_tlbi_full: /* Broadcast TLB + conditional I-cache flush. X11=0 skips IC IALLU. */ + dsb ishst tlbi vmalle1is dsb ish cbz x11, .Lel0_fault_full_no_ic @@ -1223,6 +2343,7 @@ handle_el0_fault: mov x13, x11 ubfx x11, x9, #12, #44 mov x12, x10 + dsb ishst 4: tlbi vae1is, x11 add x11, x11, #1 subs x12, x12, #1 @@ -1237,7 +2358,9 @@ handle_el0_fault: .Lel0_fault_tlbi_rvae: /* Single-shot TLBI RVAE1IS (FEAT_TLBIRANGE). X9 carries the pre-encoded - * operand (baddr | NUM<<39 | TG=01<<46); X11 the I-cache hint. */ + * operand (baddr | NUM<<39 | SCALE<<44 | TG=01<<46); X11 the I-cache + * hint. */ + dsb ishst tlbi rvae1is, x9 dsb ish cbz x11, .Lel0_fault_rvae_no_ic @@ -1281,6 +2404,7 @@ tlbi_restore_eret: * leak VA bits into the TTL [47:44] or ASID [63:48] operand fields. */ ubfx x0, x0, #12, #44 + dsb ishst tlbi vae1is, x0 dsb ish ic iallu @@ -1356,6 +2480,7 @@ handle_svc_0: * 1 = broadcast TLBI VMALLE1IS * 2 = execve replaced register state (drop frame + flush) * 3 = selective TLBI VAE1IS over X10 pages starting at X9 + * 4 = single-shot TLBI RVAE1IS with encoded operand in X9 * 5. Resume vCPU (execution continues below) */ hvc #5 @@ -1384,6 +2509,7 @@ tlbi_full: * include exec), so the shim must IC IALLU; zero means a data-only * PT change and the I-cache invalidation is skipped. */ + dsb ishst tlbi vmalle1is dsb ish cbz x11, .Ltlbi_full_skip_ic @@ -1419,6 +2545,7 @@ tlbi_selective: * leak VA bits into the TTL [47:44] or ASID [63:48] operand fields. */ ubfx x11, x9, #12, #44 /* x11 = VA[55:12] (current page operand) */ mov x12, x10 /* x12 = remaining page counter */ + dsb ishst 3: tlbi vae1is, x11 add x11, x11, #1 /* next page (operand is in 4 KiB units) */ subs x12, x12, #1 @@ -1434,11 +2561,11 @@ tlbi_selective: tlbi_range_large: /* Single-shot TLBI RVAE1IS (FEAT_TLBIRANGE, ARMv8.4+). The host has * encoded the full operand in X9: baddr (VA >> 12), TTL=0, NUM in bits - * [43:39], SCALE=0, ASID=0. One instruction covers up to 64 pages, - * avoiding the broadcast TLBI VMALLE1IS that the prior selective cap - * forced for 17..64-page ranges. X11 carries the I-cache hint as in - * tlbi_full / tlbi_selective. + * [43:39], SCALE in bits [45:44], ASID=0. One instruction covers up to + * TLBI_RVAE_MAX_PAGES. X11 carries the I-cache hint as in tlbi_full / + * tlbi_selective. */ + dsb ishst tlbi rvae1is, x9 dsb ish cbz x11, .Ltlbi_rvae_skip_ic diff --git a/src/main.c b/src/main.c index 62fac804..9127e598 100644 --- a/src/main.c +++ b/src/main.c @@ -33,6 +33,7 @@ #include "core/rosetta.h" #include "core/shim-globals.h" #include "core/sysroot.h" +#include "core/mmap-fastpath.h" #include "runtime/forkipc.h" #include "runtime/futex.h" /* futex_interrupt_request */ @@ -654,6 +655,14 @@ int main(int argc, char **argv) return 1; } + /* Tracing/debuggers require every mmap to reach host dispatch so syscall + * observation and region state advance in lockstep. This also prevents a + * trace-gated shim from leaving invisible arena reservations that perturb + * the host slow path's address-hint behavior. + */ + if (verbose || gdb_port > 0) + mmap_fastpath_disable(&g); + /* GDB setup must happen before the first run so entry-stop and hardware * breakpoints can affect the initial vCPU. */ diff --git a/src/runtime/fork-state.c b/src/runtime/fork-state.c index b5b6bc2b..d9ca7a3e 100644 --- a/src/runtime/fork-state.c +++ b/src/runtime/fork-state.c @@ -181,9 +181,9 @@ int fork_ipc_send_memory_regions(int ipc_sock, const guest_t *g, bool use_shm) #define MAX_USED_REGIONS 16 used_region_t used[MAX_USED_REGIONS]; unsigned int shim_sz = proc_get_shim_size(); - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire((guest_t *) (uintptr_t) g); int nregions = guest_get_used_regions(g, shim_sz, used, MAX_USED_REGIONS); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); uint32_t num_regions = (uint32_t) nregions; if (fork_ipc_write_all(ipc_sock, &num_regions, sizeof(num_regions)) < 0) @@ -695,6 +695,7 @@ int fork_ipc_send_process_state(int ipc_sock, const guest_region_t *regions_snapshot, uint32_t num_guest_regions, bool regions_tracker_stale_snapshot, + const uint64_t *dirty_blocks_snapshot, const guest_region_t *preannounced_snapshot, uint32_t num_preannounced) { @@ -747,6 +748,9 @@ int fork_ipc_send_process_state(int ipc_sock, fork_ipc_write_all(ipc_sock, regions_snapshot, num_guest_regions * sizeof(guest_region_t)) < 0) return -1; + if (fork_ipc_write_all(ipc_sock, dirty_blocks_snapshot, + GUEST_DIRTY_WORDS * sizeof(uint64_t)) < 0) + return -1; if (fork_ipc_write_all(ipc_sock, &num_preannounced, sizeof(num_preannounced)) < 0) @@ -958,6 +962,12 @@ int fork_ipc_recv_process_state(int ipc_fd, guest_t *g, signal_state_t *sig) g->regions_tracker_stale = (regions_tracker_stale != 0) || (num_guest_regions > recv_regions); + if (fork_ipc_read_all(ipc_fd, g->dirty_blocks, + GUEST_DIRTY_WORDS * sizeof(uint64_t)) < 0) { + log_error("fork-child: failed to read dirty block bitmap"); + return -1; + } + uint32_t num_preannounced = 0; if (fork_ipc_read_all(ipc_fd, &num_preannounced, sizeof(num_preannounced)) < 0) { diff --git a/src/runtime/fork-state.h b/src/runtime/fork-state.h index 87a59555..572b4d40 100644 --- a/src/runtime/fork-state.h +++ b/src/runtime/fork-state.h @@ -19,7 +19,7 @@ /* Fork IPC protocol identity. Bump this whenever the header layout or ordered * fork payload changes incompatibly. */ -#define FORK_IPC_PROTOCOL_MAGIC 0x454C464CU /* "ELFL" */ +#define FORK_IPC_PROTOCOL_MAGIC 0x454C464DU /* "ELFM" */ #define IPC_MAGIC_HEADER FORK_IPC_PROTOCOL_MAGIC #define IPC_MAGIC_SENTINEL 0x454C4F4BU /* "ELOK" */ @@ -127,6 +127,7 @@ int fork_ipc_send_process_state(int ipc_sock, const guest_region_t *regions_snapshot, uint32_t num_guest_regions, bool regions_tracker_stale_snapshot, + const uint64_t *dirty_blocks_snapshot, const guest_region_t *preannounced_snapshot, uint32_t num_preannounced); int fork_ipc_recv_process_state(int ipc_fd, guest_t *g, signal_state_t *sig); diff --git a/src/runtime/forkipc.c b/src/runtime/forkipc.c index d00acf70..1e395b55 100644 --- a/src/runtime/forkipc.c +++ b/src/runtime/forkipc.c @@ -32,6 +32,7 @@ #include "utils.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "runtime/forkipc.h" #include "runtime/fork-state.h" @@ -259,6 +260,7 @@ int fork_child_main(int ipc_fd, guest_destroy(&g); return 1; } + guest_rebuild_pte_present(&g); if (fork_ipc_recv_fd_table(ipc_fd, &g) < 0) { log_error("fork-child: failed to receive fd table"); @@ -438,6 +440,11 @@ int fork_child_main(int ipc_fd, */ shim_globals_rebuild_urandom_bitmap(); + if (!verbose) + mmap_fastpath_prepare_vcpu(&g, current_thread); + else + mmap_fastpath_disable(&g); + /* Now that current_thread is set, apply signal state. This must happen * after thread_register_main() so the per-thread blocked mask and altstack * are properly restored to the thread entry. @@ -498,7 +505,7 @@ typedef struct { vcpu_simd_state_t simd_state; } thread_create_args_t; -static void resolve_clone_stack_range(const guest_t *g, +static void resolve_clone_stack_range(guest_t *g, uint64_t child_stack, uint64_t *start_out, uint64_t *end_out) @@ -514,14 +521,22 @@ static void resolve_clone_stack_range(const guest_t *g, if (sp_off == 0 || sp_off > g->guest_size) return; + /* EL1 consumer-mmap entries are drained into regions[] under mmap_lock. A + * sibling worker can drain and merge the array while clone resolves its new + * stack, so take the same lock and copy the bounds before releasing it. + */ + mmap_lock_acquire(g); const guest_region_t *r = guest_region_find(g, sp_off - 1); - if (!r) + if (!r) { + mmap_lock_release(); return; + } if (start_out) *start_out = r->start; if (end_out) *end_out = r->end; + mmap_lock_release(); } /* Forward declaration: worker entry runs after sys_clone_thread */ @@ -692,11 +707,10 @@ static int64_t sys_clone_thread(hv_vcpu_t parent_vcpu, return -LINUX_EFAULT; } - /* Create the host pthread (joinable; the main thread joins all live - * workers via thread_join_workers before guest teardown, and a worker - * that exits on its own is joined when its table slot is reused by - * thread_alloc). Threads clean up their TID address via - * CLONE_CHILD_CLEARTID + futex wake. + /* Create the host pthread (joinable; the main thread joins all live workers + * via thread_join_workers before guest teardown, and a worker that exits on + * its own is joined when its table slot is reused by thread_alloc). Threads + * clean up their TID address via CLONE_CHILD_CLEARTID + futex wake. */ pthread_t host_thread; pthread_attr_t attr; @@ -846,6 +860,13 @@ static void *thread_create_and_run(void *arg) WORKER_HV(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_TTBR0_EL1, tca->ttbr0)); WORKER_HV(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_CPACR_EL1, tca->cpacr)); + /* CNTKCTL_EL1.EL0VCTEN|EL0PCTEN: worker vCPUs need EL0 counter access too, + * or the vDSO clock_gettime fast path reads 0 from CNTVCT_EL0 and every + * non-main thread silently drops to the SVC clock. Same constant bootstrap + * sets on the primary vCPU; not thread-specific. + */ + WORKER_HV(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_CNTKCTL_EL1, 0x3ULL)); + /* All worker vCPUs in the process share the same shim_globals base (one VM * per process); a fresh TPIDR_EL1 set is still required because HVF created * this vCPU empty. CONTEXTIDR_EL1 holds the per-thread tid that the gettid @@ -947,6 +968,8 @@ startup_ok:; */ thread_fork_barrier_check(); + mmap_fastpath_prepare_vcpu(g, t); + log_debug("thread tid=%lld starting on vCPU", (long long) t->guest_tid); vcpu_run_loop(vcpu, vexit, g, verbose, 0); @@ -962,10 +985,12 @@ startup_ok:; * how pthread_join works in musl: the joining thread does FUTEX_WAIT on * this address until it becomes 0. * - * Drain any deferred munmap of this thread's stack before waking the - * joiner: the parent may reuse the freed VA as soon as it returns from - * pthread_join, and reuse must not race with the deferred unmap. + * Drain any deferred munmap before publishing clear_child_tid. A joiner may + * observe the zero without ever sleeping in FUTEX_WAIT, then reuse the + * freed VA immediately; ordering only the wake after cleanup leaves a + * window where MAP_FIXED_NOREPLACE still sees the old stack VMA. */ + mem_cleanup_deferred_stack_unmaps(g, t); bool wake_ctid = false; if (t->clear_child_tid != 0) { uint32_t zero = 0; @@ -980,7 +1005,6 @@ startup_ok:; (unsigned long long) t->clear_child_tid); } } - mem_cleanup_deferred_stack_unmaps(g, t); if (wake_ctid) futex_wake_one(g, t->clear_child_tid); @@ -1194,6 +1218,10 @@ static void *vm_clone_thread_run(void *arg) HV_CHECK(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_TCR_EL1, tca->tcr)); HV_CHECK(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_TTBR0_EL1, tca->ttbr0)); HV_CHECK(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_CPACR_EL1, tca->cpacr)); + /* EL0 counter access for the vDSO clock fast path; see + * thread_create_and_run. + */ + HV_CHECK(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_CNTKCTL_EL1, 0x3ULL)); HV_CHECK(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_SCTLR_EL1, tca->sctlr)); HV_CHECK(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_SP_EL1, tca->sp_el1)); HV_CHECK(hv_vcpu_set_sys_reg(vcpu, HV_SYS_REG_SP_EL0, tca->child_stack)); @@ -1230,15 +1258,17 @@ static void *vm_clone_thread_run(void *arg) /* Set per-thread TLS pointer and enter worker run loop */ current_thread = t; thread_fork_barrier_check(); + mmap_fastpath_prepare_vcpu(g, t); log_debug("vm_clone tid=%lld starting on vCPU", (long long) t->guest_tid); int exit_code = vcpu_run_loop(vcpu, vexit, g, verbose, 0); - /* CLONE_CHILD_CLEARTID cleanup. Same ordering as thread_entry: drain - * deferred stack munmaps before waking the joiner so the parent does not - * reuse the VA before it is released. + /* CLONE_CHILD_CLEARTID cleanup. Same ordering as thread_entry: the zero + * itself, not just the futex wake, releases a joiner, so publish it only + * after the deferred stack mapping is gone. */ + mem_cleanup_deferred_stack_unmaps(g, t); bool wake_ctid = false; if (t->clear_child_tid != 0) { uint32_t zero = 0; @@ -1253,7 +1283,6 @@ static void *vm_clone_thread_run(void *arg) (unsigned long long) t->clear_child_tid); } } - mem_cleanup_deferred_stack_unmaps(g, t); if (wake_ctid) futex_wake_one(g, t->clear_child_tid); @@ -1288,9 +1317,9 @@ static void *vm_clone_thread_run(void *arg) * harmless no-op. */ thread_interrupt_all(); - /* thread_interrupt_all only reaches threads inside hv_vcpu_run. - * Peers parked on fork_cond or another slot's ptrace_cond/resume_cond - * see neither the vCPU kick nor this exit, so broadcast separately. + /* thread_interrupt_all only reaches threads inside hv_vcpu_run. Peers + * parked on fork_cond or another slot's ptrace_cond/resume_cond see + * neither the vCPU kick nor this exit, so broadcast separately. */ thread_wake_exit_waiters(); } @@ -1368,6 +1397,17 @@ int64_t sys_clone(hv_vcpu_t vcpu, if ((flags & ~(uint64_t) 0xff) & LINUX_CLONE3_NS_FLAGS) return -LINUX_EINVAL; + /* Once an anonymous arena allocation becomes a live thread stack, munmap + * must pass through thread_collect_and_defer_stack_ranges(). Revoke arena + * generations before publishing the stack to the thread table so no EL1 + * fast munmap can bypass that lifetime rule. + */ + if (child_stack != 0) { + mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, false); + mmap_lock_release(); + } + /* CLONE_THREAD: create a new thread in the same VM (not a new process) */ if (flags & LINUX_CLONE_THREAD) { return sys_clone_thread(vcpu, g, flags, child_stack, stack_map_start, @@ -1534,6 +1574,7 @@ int64_t sys_clone(hv_vcpu_t vcpu, mmap_fork_anon_shared_txn_t *anon_shared_txn = NULL; guest_region_t *regions_snapshot = NULL; + uint64_t *dirty_blocks_snapshot = NULL; guest_region_t preannounced_snapshot[GUEST_MAX_PREANNOUNCED]; /* APFS clone fd for the CoW snapshot sent to the child. Declared up front * so early goto fail_snapshot exits do not read an uninitialized local. @@ -1749,6 +1790,10 @@ int64_t sys_clone(hv_vcpu_t vcpu, } memcpy(regions_snapshot, g->regions, snap_sz); } + dirty_blocks_snapshot = malloc(sizeof(g->dirty_blocks)); + if (!dirty_blocks_snapshot) + goto fail_snapshot; + memcpy(dirty_blocks_snapshot, g->dirty_blocks, sizeof(g->dirty_blocks)); int npreannounced_snapshot = g->npreannounced; if (npreannounced_snapshot > 0) { memcpy(preannounced_snapshot, g->preannounced, @@ -1773,8 +1818,8 @@ int64_t sys_clone(hv_vcpu_t vcpu, uint32_t num_preannounced = (uint32_t) npreannounced_snapshot; if (fork_ipc_send_process_state( ipc_sock, regions_snapshot, num_guest_regions, - regions_tracker_stale_snapshot, preannounced_snapshot, - num_preannounced) < 0) { + regions_tracker_stale_snapshot, dirty_blocks_snapshot, + preannounced_snapshot, num_preannounced) < 0) { log_error("clone: failed to send process state"); goto fail_snapshot; } @@ -1837,12 +1882,14 @@ int64_t sys_clone(hv_vcpu_t vcpu, child_host_pid); free(regions_snapshot); + free(dirty_blocks_snapshot); if (snapshot_shm_fd >= 0) close(snapshot_shm_fd); return child_guest_pid; fail_snapshot: free(regions_snapshot); + free(dirty_blocks_snapshot); if (snapshot_shm_fd >= 0) close(snapshot_shm_fd); /* Roll back the in-place anon-shared overlay conversion while siblings are diff --git a/src/runtime/futex.c b/src/runtime/futex.c index 826d1a1f..871af561 100644 --- a/src/runtime/futex.c +++ b/src/runtime/futex.c @@ -1493,6 +1493,16 @@ int64_t sys_futex(guest_t *g, { int cmd = op & FUTEX_CMD_MASK; + /* Pre-fault lazy mappings before any bucket lock is taken. The word + * resolves below run under per-bucket locks, which rank after mmap_lock; + * materializing there would invert the lock order. A futex word in a + * mapping the guest never touched reads as zero, matching Linux. + */ + guest_lazy_faultin(g, uaddr, sizeof(uint32_t)); + if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE || + cmd == FUTEX_WAKE_OP) + guest_lazy_faultin(g, uaddr2, sizeof(uint32_t)); + switch (cmd) { case FUTEX_WAIT: #if ELFUSE_HAVE_OS_SYNC_WAIT_ON_ADDRESS @@ -1734,6 +1744,11 @@ int64_t sys_futex_waitv(guest_t *g, */ if (!futex_uaddr_is_aligned(elts[i].uaddr)) return -LINUX_EINVAL; + /* Pre-fault lazy mappings: the word resolves below run with every + * bucket lock held, where materializing would invert the lock order + * against mmap_lock. + */ + guest_lazy_faultin(g, elts[i].uaddr, sizeof(uint32_t)); } waitv_shared_t shared; diff --git a/src/syscall/exec.c b/src/syscall/exec.c index d694ba74..a31a0976 100644 --- a/src/syscall/exec.c +++ b/src/syscall/exec.c @@ -668,6 +668,14 @@ int64_t sys_execve(hv_vcpu_t vcpu, return err; } + /* Input copying above may fault in argv/env strings from lazy anonymous + * mappings, so it must run without mmap_lock held. Serialize only after + * all recoverable validation is complete and immediately before replacing + * the guest address space. From this point every failure is fatal and both + * successful return paths release the lock explicitly. + */ + mmap_lock_acquire(g); + /* Point of no return. guest_reset() zeroes all guest memory. The old * process image is gone. All validation that can fail gracefully MUST * happen above this line. Failures below are unrecoverable; elfuse exits @@ -883,6 +891,7 @@ int64_t sys_execve(hv_vcpu_t vcpu, free(temp_str); exec_cleanup_inputs(argv, envp, argv_buf, envp_buf, path_host_buf, path_host_temp, interp_host_buf, interp_host_temp); + mmap_lock_release(); return SYSCALL_EXEC_HAPPENED; } @@ -1187,6 +1196,7 @@ int64_t sys_execve(hv_vcpu_t vcpu, exec_cleanup_inputs(argv, envp, argv_buf, envp_buf, path_host_buf, path_host_temp, interp_host_buf, interp_host_temp); + mmap_lock_release(); return SYSCALL_EXEC_HAPPENED; too_many_regions: diff --git a/src/syscall/internal.h b/src/syscall/internal.h index 35882e9c..6f598c7f 100644 --- a/src/syscall/internal.h +++ b/src/syscall/internal.h @@ -50,6 +50,18 @@ typedef int host_fd_t; extern pthread_mutex_t mmap_lock; /* Lock order: 1, mmap/brk + page tables */ extern pthread_mutex_t fd_lock; /* Lock order: 3, FD table */ +/* The only supported mmap_lock entry/exit API. Acquire drains the per-vCPU + * EL1 mmap rings before any caller can inspect semantic region state. + */ +void mmap_lock_acquire(guest_t *g); +void mmap_lock_release(void); +void mmap_lock_cond_wait(guest_t *g, pthread_cond_t *cond); +/* Temporarily drop mmap_lock while retaining the host PT-gate reference, then + * reacquire without taking a second reference. Used only by lazy zeroing. + */ +void mmap_lock_drop_keep_gate(void); +void mmap_lock_reacquire_with_gate(guest_t *g); + /* FD table (defined in syscall/fdtable.c). */ extern fd_entry_t fd_table[FD_TABLE_SIZE]; diff --git a/src/syscall/mem.c b/src/syscall/mem.c index a5477c5e..be6e4054 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -19,10 +19,13 @@ #include #include #include +#include #include "debug/log.h" +#include "debug/syscall-hist.h" #include "utils.h" +#include "core/mmap-fastpath.h" #include "runtime/thread.h" #include "syscall/abi.h" #include "syscall/fuse.h" @@ -35,6 +38,1402 @@ */ pthread_mutex_t mmap_lock = PTHREAD_MUTEX_INITIALIZER; /* Lock order: 1 */ +static pthread_once_t mmap_fastpath_env_once = PTHREAD_ONCE_INIT; +static bool mmap_fastpath_env_enabled; +static _Atomic bool mmap_fastpath_forced_off; + +static uint64_t find_free_gap_inner(const guest_t *g, + uint64_t length, + uint64_t min_addr, + uint64_t max_addr, + uint64_t align); +static bool mmap_fastpath_rewind_control_if_clean_locked( + guest_t *g, shim_mmap_control_t *c); + +/* A host-VA replacement has a fixed cost: sibling quiesce plus an exact HVF + * stage-2 unmap/remap. Native measurements on Apple M4 leave a comfortable + * margin over memset once at least 32 MiB of materialized backing can be + * discarded at once. This is independent of EL1's pending-byte soft advisory: + * crossing that threshold never makes the munmap producer exit. + */ +#define MUNMAP_HVF_REPLACE_THRESHOLD (32ULL * 1024 * 1024) + +static int hvf_replace_slab_zero_range_quiesced(guest_t *g, + uint64_t ipa, + uint64_t len); + +static void mmap_fastpath_read_env(void) +{ + const char *v = getenv("ELFUSE_MMAP_FASTPATH"); + mmap_fastpath_env_enabled = + !v || (strcmp(v, "0") != 0 && strcmp(v, "false") != 0); +} + +static bool mmap_fastpath_available(const guest_t *g) +{ + pthread_once(&mmap_fastpath_env_once, mmap_fastpath_read_env); + return mmap_fastpath_env_enabled && !g->is_rosetta && + !atomic_load_explicit(&mmap_fastpath_forced_off, + memory_order_acquire) && + !syscall_hist_enabled(); +} + +static shim_mmap_control_t *mmap_fastpath_control(const guest_t *g, int slot) +{ + if (!g || !g->host_base || slot < 0 || slot >= MAX_THREADS) + return NULL; + return (shim_mmap_control_t *) ((uint8_t *) g->host_base + + g->shim_data_base + SHIM_MMAP_CONTROL_BASE + + (uint64_t) slot * SHIM_MMAP_CONTROL_STRIDE); +} + +static uint64_t mmap_fastpath_active_slots(const guest_t *g) +{ + return g ? atomic_load_explicit(&g->mmap_fastpath_active_slots, + memory_order_acquire) + : 0; +} + +static int mmap_fastpath_pop_slot(uint64_t *slots) +{ + int slot = __builtin_ctzll(*slots); + *slots &= *slots - 1; + return slot; +} + +static bool mmap_fastpath_provision_l3_cache_locked(guest_t *g, + shim_mmap_control_t *c) +{ + uint64_t next = + atomic_load_explicit(&c->l3_cache_next, memory_order_relaxed); + uint64_t end = c->l3_cache_end; + if (next < end) + return true; + if (end == UINT64_MAX) + return false; + + uint64_t first = + guest_reserve_pt_pages_uninitialized(g, SHIM_MMAP_L3_CACHE_PAGES); + if (!first) { + atomic_store_explicit(&c->l3_cache_next, UINT64_MAX, + memory_order_relaxed); + c->l3_cache_end = UINT64_MAX; + return false; + } + c->l3_cache_end = + first + (uint64_t) SHIM_MMAP_L3_CACHE_PAGES * GUEST_PAGE_SIZE; + atomic_store_explicit(&c->l3_cache_next, first, memory_order_release); + return true; +} + +static void mmap_fastpath_prepare_split_window_locked(guest_t *g, + shim_mmap_control_t *c, + uint64_t base, + uint64_t limit) +{ + c->split_clean_base = base; + c->split_clean_bitmap = 0; + if (base >= limit || (base & (BLOCK_2MIB - 1)) || + !mmap_fastpath_provision_l3_cache_locked(g, c)) + return; + + uint64_t window_end = limit; + uint64_t max_window = (uint64_t) SHIM_MMAP_SPLIT_CLEAN_BLOCKS * BLOCK_2MIB; + if (window_end - base > max_window) + window_end = base + max_window; + if (guest_prepare_l2_tables(g, base, window_end) < 0) + return; + + uint64_t clean = 0; + unsigned bit = 0; + for (uint64_t block = base; block < window_end; + block += BLOCK_2MIB, bit++) { + if (!guest_block_may_be_dirty(g, block)) + clean |= BIT64(bit); + } + c->split_clean_bitmap = clean; +} + +static void mmap_fastpath_reuse_reset(shim_mmap_control_t *c) +{ + if (!c) + return; + atomic_store_explicit(&c->reuse.head, 0, memory_order_relaxed); + atomic_store_explicit(&c->reuse.tail, 0, memory_order_relaxed); +} + +/* Publish a semantically removed, PTE-retired range back to its owning arena. + * mmap_lock serializes host producers; EL1 is the sole consumer. A full ring + * is only a performance miss: the range remains unavailable to EL1 until a + * later arena refill lets the ordinary host gap allocator recover it. */ +static void mmap_fastpath_publish_reuse_locked(shim_mmap_control_t *c, + uint32_t generation, + uint64_t start, + uint64_t end) +{ + if (!c || end <= start || + !(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED) || + atomic_load_explicit(&c->generation, memory_order_acquire) != + generation) + return; + + uint64_t base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (start < base || end > limit) + return; + + uint32_t head = + atomic_load_explicit(&c->reuse.head, memory_order_acquire); + uint32_t tail = + atomic_load_explicit(&c->reuse.tail, memory_order_acquire); + if ((uint32_t) (tail - head) > SHIM_MMAP_REUSE_RING_SIZE) { + log_fatal("mmap reuse: corrupt ring (head=%u tail=%u)", head, tail); + return; + } + + /* mmap_fastpath_host_gate_close() has stopped every EL1 allocator before + * drain reaches here, so the host may compact the consumer-owned pending + * set in place. Fold tombstones and adjacent extents on every publication; + * otherwise mixed-size prefix splits would fill a FIFO with tiny suffixes + * even though the arena had ample aggregate free space. */ + mmap_reuse_entry_t extents[SHIM_MMAP_REUSE_RING_SIZE + 1]; + unsigned count = 0; + bool touches_existing = false; + for (uint32_t seq = head; seq != tail; seq++) { + mmap_reuse_entry_t e = + c->reuse.entries[seq & (SHIM_MMAP_REUSE_RING_SIZE - 1)]; + if (!e.length) + continue; + if (e.addr < base || e.addr > limit || e.length > limit - e.addr) { + log_fatal("mmap reuse: invalid pending extent"); + continue; + } + uint64_t e_end = e.addr + e.length; + if (end >= e.addr && start <= e_end) + touches_existing = true; + extents[count++] = e; + } + if (count == SHIM_MMAP_REUSE_RING_SIZE && !touches_existing) { + atomic_fetch_add_explicit(&c->reuse.dropped, 1, + memory_order_relaxed); + return; + } + extents[count++] = + (mmap_reuse_entry_t) {.addr = start, .length = end - start}; + + for (unsigned i = 1; i < count; i++) { + mmap_reuse_entry_t key = extents[i]; + unsigned j = i; + while (j > 0 && extents[j - 1].addr > key.addr) { + extents[j] = extents[j - 1]; + j--; + } + extents[j] = key; + } + + unsigned merged = 0; + for (unsigned i = 0; i < count; i++) { + uint64_t e_end = extents[i].addr + extents[i].length; + if (merged != 0) { + mmap_reuse_entry_t *last = &extents[merged - 1]; + uint64_t last_end = last->addr + last->length; + if (extents[i].addr <= last_end) { + if (e_end > last_end) + last->length = e_end - last->addr; + continue; + } + } + extents[merged++] = extents[i]; + } + if (merged > SHIM_MMAP_REUSE_RING_SIZE) { + atomic_fetch_add_explicit(&c->reuse.dropped, 1, + memory_order_relaxed); + return; + } + + for (unsigned i = 0; i < merged; i++) + c->reuse.entries[i] = extents[i]; + for (unsigned i = merged; i < SHIM_MMAP_REUSE_RING_SIZE; i++) + c->reuse.entries[i].length = 0; + atomic_store_explicit(&c->reuse.head, 0, memory_order_relaxed); + atomic_fetch_add_explicit(&c->reuse.published, 1, memory_order_relaxed); + atomic_store_explicit(&c->reuse.tail, merged, memory_order_release); +} + +static _Atomic uint32_t *mmap_fastpath_pt_gate(const guest_t *g) +{ + if (!g || !g->host_base) + return NULL; + return (_Atomic uint32_t *) ((uint8_t *) g->host_base + g->shim_data_base + + SHIM_MMAP_PT_GATE_OFF); +} + +/* mmap_lock serializes host writers. The gate extends that exclusion to EL1 + * fast munmap without making the per-vCPU producers contend with each other: + * after publishing gate=closed, wait for each producer's private active word. + */ +static void mmap_fastpath_host_gate_close(guest_t *g) +{ + _Atomic uint32_t *gate = mmap_fastpath_pt_gate(g); + if (!gate) + return; + uint32_t previous = + atomic_fetch_add_explicit(gate, 1, memory_order_acq_rel); + if (previous != 0) + return; + uint64_t slots = mmap_fastpath_active_slots(g); + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + while (atomic_load_explicit(&c->retire.producer_active, + memory_order_acquire) != 0) + sched_yield(); + } +} + +static void mmap_fastpath_host_gate_open(guest_t *g) +{ + _Atomic uint32_t *gate = mmap_fastpath_pt_gate(g); + if (gate) { + uint32_t count = atomic_load_explicit(gate, memory_order_relaxed); + while (count != 0 && !atomic_compare_exchange_weak_explicit( + gate, &count, count - 1, memory_order_release, + memory_order_relaxed)) { + } + /* exec resets the entire shim-data page while holding mmap_lock, + * including this implementation-only counter. Seeing zero here is + * therefore an already-open gate, not an underflow. + */ + } +} + +static _Thread_local guest_t *mmap_lock_guest; + +static void mmap_fastpath_disable_control(guest_t *g, int slot) +{ + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (!c) + return; + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_relaxed) + 1; + if (generation == 0) + generation = 1; + atomic_store_explicit(&c->flags, 0, memory_order_relaxed); + atomic_store_explicit(&c->arena_base, 0, memory_order_relaxed); + atomic_store_explicit(&c->arena_limit, 0, memory_order_relaxed); + atomic_store_explicit(&c->cursor, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_start, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, 0, + memory_order_relaxed); + c->split_clean_base = 0; + c->split_clean_bitmap = 0; + mmap_fastpath_reuse_reset(c); + c->next_arena_size = MMAP_FAST_ARENA_MIN; + c->max_len_seen = 0; + atomic_store_explicit(&c->generation, generation, memory_order_release); + atomic_fetch_and_explicit(&g->mmap_fastpath_active_slots, ~BIT64(slot), + memory_order_release); +} + +static void mmap_fastpath_mark_mprotect_locked(guest_t *g, + uint64_t start, + uint64_t end) +{ + if (!g || end <= start) + return; + uint64_t slots = mmap_fastpath_active_slots(g); + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + uint64_t base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (start < limit && end > base) { + /* A long-lived adaptive arena can move its cursor many GiB beyond + * the 32-block window installed at refill. The current mprotect + * has already fallen back under the closed PT gate, so use that + * natural exit to roll the prepared window forward. Subsequent + * nearby allocator permission changes can then stay in EL1. + */ + uint64_t block = ALIGN_DOWN(start, BLOCK_2MIB); + uint64_t split_base = c->split_clean_base; + uint64_t split_span = + (uint64_t) SHIM_MMAP_SPLIT_CLEAN_BLOCKS * BLOCK_2MIB; + if (block < split_base || block - split_base >= split_span) + mmap_fastpath_prepare_split_window_locked(g, c, block, limit); + atomic_fetch_or_explicit(&c->flags, SHIM_MMAP_CTRL_NO_FAST_MUNMAP, + memory_order_release); + } + } +} + +static void mmap_fastpath_drain_publications_locked(guest_t *g, uint64_t slots) +{ + if (!g || !g->host_base) + return; + + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + uint32_t head = atomic_load_explicit(&c->head, memory_order_relaxed); + uint32_t tail = atomic_load_explicit(&c->tail, memory_order_acquire); + if ((uint32_t) (tail - head) > SHIM_MMAP_RING_SIZE) { + log_fatal( + "mmap fast path: corrupt ring in vCPU slot %d " + "(head=%u tail=%u)", + slot, head, tail); + } + + uint64_t arena_base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t arena_limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + while (head != tail) { + const shim_mmap_entry_t *e = + &c->ring[head & (SHIM_MMAP_RING_SIZE - 1)]; + uint64_t addr = e->addr; + uint64_t len = e->len; + if (e->prot & SHIM_MMAP_ENTRY_MPROTECT) { + bool installed_l3 = + (e->prot & SHIM_MMAP_ENTRY_MPROTECT_SPLIT) != 0; + bool stayed_lazy = + (e->prot & SHIM_MMAP_ENTRY_MPROTECT_LAZY) != 0; + uint64_t prot = e->prot & ~(SHIM_MMAP_ENTRY_MPROTECT | + SHIM_MMAP_ENTRY_MPROTECT_SPLIT | + SHIM_MMAP_ENTRY_MPROTECT_LAZY); + if ((addr & (GUEST_PAGE_SIZE - 1)) || len != GUEST_PAGE_SIZE || + addr < arena_base || addr > arena_limit || + len > arena_limit - addr || + (prot != LINUX_PROT_READ && + prot != (LINUX_PROT_READ | LINUX_PROT_WRITE))) { + log_fatal( + "mprotect fast path: invalid entry in vCPU slot %d " + "(addr=0x%llx len=0x%llx prot=0x%llx)", + slot, (unsigned long long) addr, + (unsigned long long) len, (unsigned long long) prot); + } + guest_region_set_prot(g, addr, addr + len, (int) prot); + if (installed_l3) { + uint64_t block = ALIGN_DOWN(addr, BLOCK_2MIB); + guest_retire_ptes_committed(g, block, block + BLOCK_2MIB); + mmap_fastpath_note_materialized_locked(g, block, + block + BLOCK_2MIB); + /* The newly published table exposes the rest of the + * full-block anonymous mapping as RW. Conservatively mark + * all of its backing dirty before any later retirement. + */ + guest_dirty_mark_range(g, block, block + BLOCK_2MIB); + } else if (!stayed_lazy && (prot & LINUX_PROT_WRITE)) { + guest_dirty_mark_range(g, addr, addr + len); + } + guest_pt_gen_bump(g); + head++; + continue; + } + if ((addr & (GUEST_PAGE_SIZE - 1)) || !len || + (len & (GUEST_PAGE_SIZE - 1)) || addr < arena_base || + addr > arena_limit || len > arena_limit - addr || + e->prot != (LINUX_PROT_READ | LINUX_PROT_WRITE)) { + log_fatal( + "mmap fast path: invalid entry in vCPU slot %d " + "(addr=0x%llx len=0x%llx arena=0x%llx..0x%llx)", + slot, (unsigned long long) addr, (unsigned long long) len, + (unsigned long long) arena_base, + (unsigned long long) arena_limit); + } + if (guest_region_add_ex(g, addr, addr + len, + LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | + LINUX_MAP_NORESERVE, + 0, NULL, -1) < 0) { + /* EL1 already returned this address to the guest. Continuing + * without semantic metadata would turn first touch into a false + * SIGSEGV, so fail closed on the violated provisioning + * invariant instead of silently corrupting process state. + */ + log_fatal( + "mmap fast path: region metadata exhausted while " + "draining vCPU slot %d", + slot); + } + if (len > c->max_len_seen) + c->max_len_seen = len; + head++; + } + atomic_store_explicit(&c->head, head, memory_order_release); + } +} + +/* Return true only when every semantic mapping of this backing range is part + * of the VA retirement being committed. Fast mmap arenas are identity mapped + * today, but retaining this check prevents a future high-VA alias from having + * its backing replaced underneath a still-live PTE. + */ +static bool munmap_retire_backing_is_exclusive(const guest_t *g, + uint64_t backing_start, + uint64_t backing_end, + uint64_t retire_start, + uint64_t retire_end) +{ + for (int i = 0; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + uint64_t rlen = r->end - r->start; + if (r->gpa_base > UINT64_MAX - rlen) + return false; + uint64_t r_gpa_end = r->gpa_base + rlen; + uint64_t lo = backing_start > r->gpa_base ? backing_start + : r->gpa_base; + uint64_t hi = backing_end < r_gpa_end ? backing_end : r_gpa_end; + if (hi <= lo) + continue; + + uint64_t mapped_start = r->start + (lo - r->gpa_base); + uint64_t mapped_end = mapped_start + (hi - lo); + if (mapped_start < retire_start || mapped_end > retire_end) + return false; + } + return true; +} + +static void munmap_retire_commit_locked(guest_t *g, + const munmap_retire_entry_t *e, + uint64_t backing_start, + uint64_t backing_end, + uint64_t charged_bytes) +{ + uint64_t start = e->addr; + uint64_t end = start + e->length; + + /* Publication drain ran first, so every mapping causally preceding this + * retirement is now represented in regions[]. A non-anonymous overlay in + * an arena indicates a missing revocation and must fail closed: EL1 has + * already invalidated the PTEs, so silently retaining such metadata would + * permit a later fault path to recreate them. + */ + for (int i = guest_region_first_end_above(g, start); i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= end) + break; + if (r->end <= start) + continue; + if (!(r->flags & LINUX_MAP_ANONYMOUS) || + (r->flags & LINUX_MAP_SHARED) || r->backing_fd >= 0 || + r->overlay_active) { + log_fatal( + "munmap retire: non-fast mapping in arena " + "[0x%llx-0x%llx)", + (unsigned long long) start, (unsigned long long) end); + } + } + + guest_materialize_wait_range_locked(g, start, end); + + /* The PTE occupancy evidence was consumed by EL1, so use the conservative + * dirty bitmap to avoid touching huge never-materialized reservations. + * Full, contiguous dirty runs at or above 32 MiB are cheaper to discard by + * replacing their zero-fill backing while their HVF stage-2 segments are + * detached. For shorter or partial runs, retain the dirty bits and let a + * future lazy materialization zero only the backing that is actually + * reused. In particular, do not charge an unrelated VM exit (often the + * next mapping's first fault) with an eager memset of the retired range. + * The replacement mapping cannot observe stale bytes: EL1 has already + * invalidated the old descriptors, and guest_materialize_lazy_one() zeros + * dirty backing before publishing any new descriptor. + * + * charged_bytes is the EL1 producer's page-accurate materialized count, so + * a sparse virtual retirement cannot trigger replacement merely because + * its address span is large. + */ + bool allow_replace = charged_bytes >= MUNMAP_HVF_REPLACE_THRESHOLD; + bool siblings_quiesced = false; + if (allow_replace) { + uint64_t block = ALIGN_DOWN(backing_start, BLOCK_2MIB); + while (block < backing_end) { + uint64_t block_end = block + BLOCK_2MIB; + if (!guest_block_may_be_dirty(g, block)) { + block = block_end; + continue; + } + uint64_t lo = backing_start > block ? backing_start : block; + uint64_t hi = + backing_end < block_end ? backing_end : block_end; + if (lo != block || hi != block_end) { + block = block_end; + continue; + } + + uint64_t run_end = block_end; + while (run_end < backing_end && + run_end <= UINT64_MAX - BLOCK_2MIB && + guest_block_may_be_dirty(g, run_end)) + run_end += BLOCK_2MIB; + + if (run_end - block >= MUNMAP_HVF_REPLACE_THRESHOLD && + munmap_retire_backing_is_exclusive( + g, block, run_end, start, end)) { + if (!siblings_quiesced) { + thread_quiesce_siblings(); + siblings_quiesced = true; + } + if (hvf_replace_slab_zero_range_quiesced( + g, block, run_end - block) == 0) { + guest_dirty_clear_zeroed_range(g, block, run_end); + block = run_end; + continue; + } + } + block = block_end; + } + } + if (siblings_quiesced) + thread_resume_siblings(); + + guest_region_remove(g, start, end); + if (backing_end > backing_start) + guest_retire_ptes_committed(g, backing_start, backing_end); + if (start < g->mmap_rw_gap_hint) + g->mmap_rw_gap_hint = start; + if (start < g->mmap_rx_gap_hint) + g->mmap_rx_gap_hint = start; +} + +void mmap_fastpath_drain_locked(guest_t *g) +{ + if (!g || !g->host_base) + return; + + /* Acquire-snapshot every retirement tail before consuming any mmap + * publication. This is the cross-vCPU causal ordering required for + * "A mmap; publish pointer; B munmap": the acquire observes B's retire, + * then publication drain establishes A's semantic region before removal. + */ + uint32_t retire_tails[MAX_THREADS]; + uint64_t active_slots = mmap_fastpath_active_slots(g); + uint64_t slots = active_slots; + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + retire_tails[slot] = + atomic_load_explicit(&c->retire.tail, memory_order_acquire); + } + + mmap_fastpath_drain_publications_locked(g, active_slots); + + bool retired_any = false; + slots = active_slots; + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); + shim_mmap_control_t *producer = mmap_fastpath_control(g, slot); + uint32_t head = + atomic_load_explicit(&producer->retire.head, memory_order_relaxed); + uint32_t tail = retire_tails[slot]; + if ((uint32_t) (tail - head) > SHIM_MUNMAP_RETIRE_RING_SIZE) { + log_fatal( + "munmap retire: corrupt ring in vCPU slot %d " + "(head=%u tail=%u)", + slot, head, tail); + } + + while (head != tail) { + const munmap_retire_entry_t *e = + &producer->retire + .entries[head & (SHIM_MUNMAP_RETIRE_RING_SIZE - 1)]; + uint32_t arena_slot = + e->flags & SHIM_MUNMAP_RETIRE_F_ARENA_SLOT_MASK; + uint64_t charged_pages = + (e->flags & SHIM_MUNMAP_RETIRE_F_CHARGE_MASK) >> + SHIM_MUNMAP_RETIRE_F_CHARGE_SHIFT; + uint64_t charged_bytes = charged_pages * GUEST_PAGE_SIZE; + if (arena_slot >= MAX_THREADS || !e->length || + (e->addr & (GUEST_PAGE_SIZE - 1)) || + (e->length & (GUEST_PAGE_SIZE - 1)) || + e->addr > UINT64_MAX - e->length || + charged_bytes > e->length) { + log_fatal("munmap retire: invalid entry in vCPU slot %d", slot); + } + + shim_mmap_control_t *arena = + mmap_fastpath_control(g, (int) arena_slot); + uint32_t generation = + atomic_load_explicit(&arena->generation, memory_order_acquire); + uint64_t base = + atomic_load_explicit(&arena->arena_base, memory_order_relaxed); + uint64_t cursor = + atomic_load_explicit(&arena->cursor, memory_order_relaxed); + uint64_t end = e->addr + e->length; + if (generation != e->arena_generation || e->addr < base || + end > cursor) { + log_fatal( + "munmap retire: stale arena generation/range " + "(producer=%d arena=%u gen=%u/%u)", + slot, arena_slot, e->arena_generation, generation); + } + + uint64_t backing_start = 0, backing_end = 0; + if (charged_bytes != 0) { + backing_start = atomic_load_explicit( + &arena->materialized_start, memory_order_relaxed); + backing_end = atomic_load_explicit( + &arena->materialized_end, memory_order_relaxed); + if (backing_start < e->addr) + backing_start = e->addr; + if (backing_end > end) + backing_end = end; + if (backing_end <= backing_start) + log_fatal( + "munmap retire: charged entry has no materialized " + "bounds (producer=%d arena=%u range=0x%llx..0x%llx " + "marker=0x%llx..0x%llx charged=0x%llx)", + slot, arena_slot, (unsigned long long) e->addr, + (unsigned long long) end, + (unsigned long long) atomic_load_explicit( + &arena->materialized_start, memory_order_relaxed), + (unsigned long long) atomic_load_explicit( + &arena->materialized_end, memory_order_relaxed), + (unsigned long long) charged_bytes); + } + + /* Capture only bytes that are still semantically mapped. Linux + * permits munmap across holes (and repeated munmap of a hole), so + * publishing the raw retire range would double-free VA. Adjacent + * anonymous regions are coalesced to preserve reuse-ring capacity. + * More than one ring's worth is safely left to the host gap + * allocator after a later arena rollover. */ + mmap_reuse_entry_t reusable[SHIM_MMAP_REUSE_RING_SIZE]; + unsigned reusable_count = 0; + for (int i = guest_region_first_end_above(g, e->addr); + i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= end) + break; + if (r->end <= e->addr || !r->noreserve || + !(r->flags & LINUX_MAP_ANONYMOUS) || + (r->flags & LINUX_MAP_SHARED) || r->backing_fd >= 0 || + r->overlay_active) + continue; + uint64_t lo = r->start > e->addr ? r->start : e->addr; + uint64_t hi = r->end < end ? r->end : end; + if (reusable_count != 0 && + reusable[reusable_count - 1].addr + + reusable[reusable_count - 1].length == + lo) { + reusable[reusable_count - 1].length += hi - lo; + } else if (reusable_count < SHIM_MMAP_REUSE_RING_SIZE) { + reusable[reusable_count++] = + (mmap_reuse_entry_t) {.addr = lo, + .length = hi - lo}; + } + } + + munmap_retire_commit_locked(g, e, backing_start, backing_end, + charged_bytes); + for (unsigned i = 0; i < reusable_count; i++) { + mmap_fastpath_publish_reuse_locked( + arena, generation, reusable[i].addr, + reusable[i].addr + reusable[i].length); + } + uint64_t consumed = atomic_load_explicit( + &producer->retire.consumed_bytes, memory_order_relaxed); + atomic_store_explicit(&producer->retire.consumed_bytes, + consumed + charged_bytes, + memory_order_release); + head++; + retired_any = true; + } + atomic_store_explicit(&producer->retire.head, head, + memory_order_release); + /* The PT gate is closed while draining, so no producer can race this + * acknowledgement. Ring fullness remains the only hard per-vCPU + * backpressure; byte pressure is deliberately advisory. + */ + atomic_store_explicit(&producer->retire.cleanup_requested, 0, + memory_order_release); + } + + /* A stopped owner whose whole arena retired can collapse all published + * sub-extents back into its bump cursor. Like envelope reset, this must + * wait for the complete snapshot: overlapping retire records from sibling + * producers may otherwise observe a prematurely reset arena. */ + if (current_thread && current_thread->sp_el1_slot >= 0) + mmap_fastpath_rewind_control_if_clean_locked( + g, mmap_fastpath_control(g, current_thread->sp_el1_slot)); + + /* EL1 may publish several charged retirements before this drain. Their + * PTEs are all already invalid, so clearing an arena's materialized + * envelope after the first commit would make later entries in the same + * snapshot lose their backing bounds. Restore the PTE-empty proof only + * after every snapshotted retirement has consumed the old envelope. */ + slots = active_slots; + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); + shim_mmap_control_t *arena = mmap_fastpath_control(g, slot); + if (!(atomic_load_explicit(&arena->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + continue; + uint64_t base = + atomic_load_explicit(&arena->arena_base, memory_order_relaxed); + uint64_t cursor = + atomic_load_explicit(&arena->cursor, memory_order_relaxed); + if (base < cursor && + guest_va_next_present_block(g, base, cursor) >= cursor) { + atomic_store_explicit(&arena->materialized_start, 0, + memory_order_relaxed); + atomic_store_explicit(&arena->materialized_end, 0, + memory_order_relaxed); + atomic_store_explicit(&arena->materialized_generation, 0, + memory_order_release); + } + } + + /* EL1 PTE stores cannot update guest_t's host-only cache generation. One + * bump per batch invalidates every host GVA translation cache after all + * retirement entries have committed. + */ + if (retired_any) + guest_pt_gen_bump(g); + + /* Replenish only exhausted per-vCPU batches. The gate is closed and every + * active producer is quiescent, so EL1 cannot race the cursor publication. + */ + slots = active_slots; + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); + mmap_fastpath_provision_l3_cache_locked(g, + mmap_fastpath_control(g, slot)); + } +} + +void mmap_lock_acquire(guest_t *g) +{ + pthread_mutex_lock(&mmap_lock); + mmap_fastpath_host_gate_close(g); + mmap_lock_guest = g; + mmap_fastpath_drain_locked(g); +} + +void mmap_lock_release(void) +{ + mmap_fastpath_host_gate_open(mmap_lock_guest); + mmap_lock_guest = NULL; + pthread_mutex_unlock(&mmap_lock); +} + +void mmap_lock_cond_wait(guest_t *g, pthread_cond_t *cond) +{ + mmap_fastpath_host_gate_open(g); + mmap_lock_guest = NULL; + pthread_cond_wait(cond, &mmap_lock); + /* pthread_cond_wait reacquires mmap_lock directly, so preserve the + * drain-before-region-read invariant of mmap_lock_acquire(). + */ + mmap_fastpath_host_gate_close(g); + mmap_lock_guest = g; + mmap_fastpath_drain_locked(g); +} + +void mmap_lock_drop_keep_gate(void) +{ + /* Dirty lazy-materialization drops mmap_lock around a potentially large + * memset. Retain this thread's gate reference so EL1 cannot retire the + * invalid PTE window and let the materializer recreate it afterwards. + * Another host thread may temporarily acquire mmap_lock; the refcounted + * gate remains closed until this owner finishes the materialization. + */ + mmap_lock_guest = NULL; + pthread_mutex_unlock(&mmap_lock); +} + +void mmap_lock_reacquire_with_gate(guest_t *g) +{ + pthread_mutex_lock(&mmap_lock); + mmap_lock_guest = g; + /* EL1 mmap publication does not need the PT gate and may have progressed + * during the memset, so refresh semantic metadata before resuming. + */ + mmap_fastpath_drain_locked(g); +} + +void mmap_fastpath_drain_vmexit(guest_t *g) +{ + mmap_lock_acquire(g); + mmap_lock_release(); +} + +bool mmap_fastpath_current_producer_active(const guest_t *g) +{ + if (!g || !current_thread || current_thread->sp_el1_slot < 0) + return false; + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + return c && atomic_load_explicit(&c->retire.producer_active, + memory_order_acquire) != 0; +} + +void mmap_fastpath_note_materialized_locked(guest_t *g, + uint64_t start, + uint64_t end) +{ + if (!g || end <= start) + return; + uint64_t slots = mmap_fastpath_active_slots(g); + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + continue; + uint64_t base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (start >= limit || end <= base) + continue; + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_relaxed); + uint64_t lo = start > base ? start : base; + uint64_t hi = end < limit ? end : limit; + uint32_t materialized = atomic_load_explicit( + &c->materialized_generation, memory_order_relaxed); + if (materialized == generation) { + uint64_t old_lo = atomic_load_explicit( + &c->materialized_start, memory_order_relaxed); + uint64_t old_hi = atomic_load_explicit( + &c->materialized_end, memory_order_relaxed); + if (old_lo < lo) + lo = old_lo; + if (old_hi > hi) + hi = old_hi; + } + atomic_store_explicit(&c->materialized_start, lo, + memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, hi, + memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, generation, + memory_order_release); + /* Fast-path arenas are allocated from disjoint VA ranges. Once this + * block has updated its owner, no later vCPU control can overlap it; + * avoid another 63 control-page probes on the single-vCPU hot path. */ + break; + } +} + +static bool mmap_fastpath_request_fits(uint64_t cursor, + uint64_t limit, + uint64_t len) +{ + if (!len) + return cursor < limit; + uint64_t start = cursor; + if (len >= BLOCK_2MIB) { + if (start > UINT64_MAX - (BLOCK_2MIB - 1)) + return false; + start = ALIGN_UP(start, BLOCK_2MIB); + } + return start <= limit && len <= limit - start; +} + +static uint64_t mmap_fastpath_pow2_clamped(uint64_t value) +{ + if (value <= MMAP_FAST_ARENA_MIN) + return MMAP_FAST_ARENA_MIN; + if (value >= MMAP_FAST_ARENA_MAX) + return MMAP_FAST_ARENA_MAX; + value--; + value |= value >> 1; + value |= value >> 2; + value |= value >> 4; + value |= value >> 8; + value |= value >> 16; + value |= value >> 32; + return value + 1; +} + +static uint64_t mmap_fastpath_arena_size(uint64_t max_len_seen, + uint64_t request_len) +{ + uint64_t adaptive = MMAP_FAST_ARENA_MIN; + if (max_len_seen) { + const uint64_t target_entries = MMAP_FAST_ARENA_TARGET_ENTRIES; + uint64_t target = max_len_seen > MMAP_FAST_ARENA_MAX / target_entries + ? MMAP_FAST_ARENA_MAX + : max_len_seen * target_entries; + adaptive = mmap_fastpath_pow2_clamped(target); + } + + uint64_t covering = MMAP_FAST_ARENA_MIN; + if (request_len) { + uint64_t target = request_len > MMAP_FAST_ARENA_MAX / 2 + ? MMAP_FAST_ARENA_MAX + : request_len * 2; + covering = mmap_fastpath_pow2_clamped(target); + } + return adaptive > covering ? adaptive : covering; +} + +static void mmap_fastpath_refill_thread_locked(guest_t *g, + thread_entry_t *t, + uint64_t request_len) +{ + if (!t || t->sp_el1_slot < 0) + return; + int slot = t->sp_el1_slot; + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (!c) + return; + if (!mmap_fastpath_available(g)) { + mmap_fastpath_disable_control(g, slot); + return; + } + + /* Giant requests are deliberately slow-path-only. Do not abandon a useful + * small-request arena or poison its adaptive history. + */ + if (request_len > MMAP_FAST_ARENA_MAX) + return; + + if (request_len > c->max_len_seen) + c->max_len_seen = request_len; + + uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t arena_base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + uint32_t flags = atomic_load_explicit(&c->flags, memory_order_relaxed); + uint64_t arena_size = + mmap_fastpath_arena_size(c->max_len_seen, request_len); + if ((flags & SHIM_MMAP_CTRL_ENABLED) && + mmap_fastpath_request_fits(cursor, limit, request_len)) { + /* A capacity miss enters HVC, whose drain can rewind a completely + * retired arena before this refill check. Retaining that undersized + * arena merely because one more request fits makes the same miss recur + * every few operations and turns mmap latency into a periodic sawtooth. + * Grow an empty arena to the adaptive target; never relocate one that + * still contains allocations served by the current generation. + */ + bool empty = cursor == arena_base; + bool target_sized = arena_base <= limit && + limit - arena_base >= arena_size; + if (!empty || target_sized) + return; + } + + /* The owner is parked in HVC. Make the stranded tail immediately recyclable + * before the gap scan; mappings already served from the prefix were drained + * into regions[] on mmap_lock acquisition. + */ + if (flags & SHIM_MMAP_CTRL_ENABLED) + atomic_store_explicit(&c->cursor, limit, memory_order_relaxed); + + /* Prefer a real hole below the current high-water mark. Active sibling + * arena tails are excluded by mmap_fastpath_skip_reserved inside the gap + * allocator. Only grow mmap_next when no recyclable hole fits. + */ + uint64_t high = g->mmap_next; + if (high > g->mmap_limit) + high = g->mmap_limit; + uint64_t base = UINT64_MAX; + bool recycled = false; + if (high > MMAP_BASE) { + base = find_free_gap_inner(g, arena_size, MMAP_BASE, high, BLOCK_2MIB); + recycled = base != UINT64_MAX; + } + + if (!recycled) { + if (g->mmap_next > UINT64_MAX - (BLOCK_2MIB - 1)) { + mmap_fastpath_disable_control(g, slot); + return; + } + base = ALIGN_UP(g->mmap_next, BLOCK_2MIB); + if (base > g->mmap_limit || arena_size > g->mmap_limit - base) { + mmap_fastpath_disable_control(g, slot); + return; + } + } + uint64_t new_limit = base + arena_size; + + /* Carve VA only. Clearing stale descriptors once here makes every later + * bump allocation PTE-free without putting page-table work in EL1. Fresh + * bump-tail arenas beyond mmap_end cannot contain stale descriptors. + */ + if (recycled || base < g->mmap_end) { + if (guest_invalidate_ptes(g, base, new_limit) < 0) { + mmap_fastpath_disable_control(g, slot); + return; + } + } + if (!recycled) { + g->mmap_next = new_limit; + if (g->mmap_rw_gap_hint < new_limit) + g->mmap_rw_gap_hint = new_limit; + } + + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_relaxed) + 1; + if (generation == 0) + generation = 1; + atomic_store_explicit(&c->arena_base, base, memory_order_relaxed); + atomic_store_explicit(&c->arena_limit, new_limit, memory_order_relaxed); + atomic_store_explicit(&c->cursor, base, memory_order_relaxed); + atomic_store_explicit(&c->materialized_start, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, 0, memory_order_relaxed); + mmap_fastpath_prepare_split_window_locked(g, c, base, new_limit); + mmap_fastpath_reuse_reset(c); + c->next_arena_size = arena_size; + c->max_len_seen = 0; + c->refill_count++; + if (recycled) + c->recycle_count++; + if (arena_size > c->peak_arena_size) + c->peak_arena_size = arena_size; + uint32_t control_flags = SHIM_MMAP_CTRL_ENABLED; + if (g_tlbi_range_supported) + control_flags |= SHIM_MMAP_CTRL_TLBIRANGE; + atomic_store_explicit(&c->flags, control_flags, memory_order_relaxed); + /* This vCPU is stopped in HVC (or has never run), so host may acknowledge + * the freshly published descriptor on its behalf. Revocation deliberately + * does not do this, making an in-flight stale generation bail once. + */ + atomic_store_explicit(&c->consumer_generation, generation, + memory_order_relaxed); + atomic_store_explicit(&c->generation, generation, memory_order_release); + atomic_fetch_or_explicit(&g->mmap_fastpath_active_slots, BIT64(slot), + memory_order_release); +} + +void mmap_fastpath_refill_current_locked(guest_t *g, uint64_t request_len) +{ + mmap_fastpath_refill_thread_locked(g, current_thread, request_len); +} + +bool mmap_fastpath_allocate_current_locked(guest_t *g, + uint64_t request_len, + uint64_t *addr_out) +{ + if (!addr_out || !request_len || !mmap_fastpath_available(g) || + !current_thread || current_thread->sp_el1_slot < 0) + return false; + + mmap_fastpath_refill_thread_locked(g, current_thread, request_len); + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + if (!c || !(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + return false; + + uint64_t start = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (request_len >= BLOCK_2MIB) { + if (start > UINT64_MAX - (BLOCK_2MIB - 1)) + return false; + start = ALIGN_UP(start, BLOCK_2MIB); + } + if (start > limit || request_len > limit - start) + return false; + uint64_t end = start + request_len; + + if (guest_region_add_ex(g, start, end, + LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | + LINUX_MAP_NORESERVE, + 0, NULL, -1) < 0) + return false; + + atomic_store_explicit(&c->cursor, end, memory_order_release); + if (end > g->mmap_end) + g->mmap_end = end; + *addr_out = start; + return true; +} + +/* Service mmap publication-ring backpressure without turning it into a + * global PT-gate rendezvous. The calling vCPU is stopped in HVC, so the host + * may advance only that vCPU's existing bump cursor after draining its prior + * publications. Sibling EL1 allocators own disjoint arenas and may continue + * publishing concurrently. + * + * This path deliberately refuses every operation that could require a host + * writer transaction: a pending retirement, a closed gate, arena refill or + * generation change all fall back to mmap_lock_acquire(). The acquire + * snapshots of every retire tail preserve the usual drain-before-metadata + * ordering for causally prior munmaps. */ +bool mmap_fastpath_allocate_current_publication_only(guest_t *g, + uint64_t request_len, + uint64_t *addr_out) +{ + if (!g || !addr_out || !request_len || !mmap_fastpath_available(g) || + !current_thread || current_thread->sp_el1_slot < 0) + return false; + + pthread_mutex_lock(&mmap_lock); + _Atomic uint32_t *gate = mmap_fastpath_pt_gate(g); + if (!gate || atomic_load_explicit(gate, memory_order_acquire) != 0) + goto miss; + + uint64_t slots = mmap_fastpath_active_slots(g); + uint64_t pending_slots = slots; + while (pending_slots) { + int slot = mmap_fastpath_pop_slot(&pending_slots); + shim_mmap_control_t *producer = mmap_fastpath_control(g, slot); + uint32_t head = atomic_load_explicit(&producer->retire.head, + memory_order_relaxed); + uint32_t tail = atomic_load_explicit(&producer->retire.tail, + memory_order_acquire); + if (head != tail) + goto miss; + } + + mmap_fastpath_drain_publications_locked(g, slots); + + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + uint32_t generation = + atomic_load_explicit(&c->generation, memory_order_acquire); + if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED) || + atomic_load_explicit(&c->consumer_generation, memory_order_relaxed) != + generation) + goto miss; + + uint64_t start = atomic_load_explicit(&c->cursor, memory_order_relaxed); + uint64_t base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (start == base) { + uint64_t target = + mmap_fastpath_arena_size(c->max_len_seen, request_len); + if (base > limit || limit - base < target) + goto miss; + } + if (request_len >= BLOCK_2MIB) { + if (start > UINT64_MAX - (BLOCK_2MIB - 1)) + goto miss; + start = ALIGN_UP(start, BLOCK_2MIB); + } + if (start > limit || request_len > limit - start) + goto miss; + uint64_t end = start + request_len; + + if (guest_region_add_ex(g, start, end, + LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS | + LINUX_MAP_NORESERVE, + 0, NULL, -1) < 0) + goto miss; + + atomic_store_explicit(&c->cursor, end, memory_order_release); + if (request_len > c->max_len_seen) + c->max_len_seen = request_len; + if (end > g->mmap_end) + g->mmap_end = end; + *addr_out = start; + pthread_mutex_unlock(&mmap_lock); + return true; + +miss: + pthread_mutex_unlock(&mmap_lock); + return false; +} + +void mmap_fastpath_release_current_hint_locked(guest_t *g, + uint64_t addr, + uint64_t length) +{ + if (!current_thread || current_thread->sp_el1_slot < 0 || !length || + addr > UINT64_MAX - length) + return; + shim_mmap_control_t *c = + mmap_fastpath_control(g, current_thread->sp_el1_slot); + if (!c || !(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + return; + uint64_t base = + atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (base >= limit || addr >= limit || addr + length <= base) + return; + + /* The owner is stopped in the HVC that reached sys_mmap, so it cannot race + * this descriptor update. Revoke its bump tail and committed free extents: + * the explicit hint must take precedence over every EL1-reserved hole, and + * the post-syscall refill will provision a new non-overlapping arena. + */ + mmap_fastpath_disable_control(g, current_thread->sp_el1_slot); +} + +/* Reuse a fully released arena in place. mmap_lock acquisition has drained + * this vCPU's publication ring, and sys_munmap has removed the last semantic + * region before calling here. The PTE occupancy index is the final guard: an + * arena is rewound only when no live metadata and no valid descriptor remain. + * The owner is stopped in HVC, so resetting its private bump cursor cannot + * race EL1. Keeping base/limit/generation unchanged avoids a host refill on + * the next mmap -- especially important when one 32 GiB request consumes the + * entire maximum-sized arena. + */ +static bool mmap_fastpath_rewind_control_if_clean_locked( + guest_t *g, shim_mmap_control_t *c) +{ + if (!g || !c) + return false; + if (!(atomic_load_explicit(&c->flags, memory_order_relaxed) & + SHIM_MMAP_CTRL_ENABLED)) + return false; + + uint64_t base = atomic_load_explicit(&c->arena_base, memory_order_relaxed); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + uint64_t cursor = atomic_load_explicit(&c->cursor, memory_order_relaxed); + if (base >= limit || cursor <= base) + return false; + + for (int i = 0; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= limit) + break; + if (r->end > base) + return false; + } + if (guest_va_next_present_block(g, base, limit) < limit) + return false; + + /* The stopped owner can safely discard pending sub-extents because the + * whole arena is becoming one bump-allocatable extent again. */ + mmap_fastpath_reuse_reset(c); + atomic_store_explicit(&c->cursor, base, memory_order_relaxed); + atomic_store_explicit(&c->materialized_start, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_end, 0, memory_order_relaxed); + atomic_store_explicit(&c->materialized_generation, 0, memory_order_relaxed); + /* Host retirement has now removed every live region/PTE and zeroed dirty + * backing. Refresh the EL1 clean proof for the rewound generation and + * allow fast munmap again; the PTE-shape concern that set the bit no longer + * survives this full-arena reset. + */ + mmap_fastpath_prepare_split_window_locked(g, c, base, limit); + atomic_fetch_and_explicit(&c->flags, ~SHIM_MMAP_CTRL_NO_FAST_MUNMAP, + memory_order_release); + c->recycle_count++; + return true; +} + +static void mmap_fastpath_rewind_current_if_clean_locked(guest_t *g) +{ + if (!g || !current_thread || current_thread->sp_el1_slot < 0) + return; + mmap_fastpath_rewind_control_if_clean_locked( + g, mmap_fastpath_control(g, current_thread->sp_el1_slot)); +} + +void mmap_fastpath_prepare_vcpu(guest_t *g, thread_entry_t *t) +{ + mmap_lock_acquire(g); + mmap_fastpath_refill_thread_locked(g, t, 0); + mmap_lock_release(); +} + +void mmap_fastpath_revoke_all_locked(guest_t *g, bool shrink_high_water) +{ + mmap_fastpath_drain_locked(g); + uint64_t slots = mmap_fastpath_active_slots(g); + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); + mmap_fastpath_disable_control(g, slot); + } + + if (!shrink_high_water) + return; + uint64_t high = MMAP_BASE; + for (int i = 0; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= MMAP_BASE && r->start < g->mmap_limit && r->end > high) + high = r->end; + } + g->mmap_next = high; + if (g->mmap_rw_gap_hint > high) + g->mmap_rw_gap_hint = high; +} + +void mmap_fastpath_disable(guest_t *g) +{ + atomic_store_explicit(&mmap_fastpath_forced_off, true, + memory_order_release); + mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, true); + mmap_lock_release(); +} + +void mmap_fastpath_skip_reserved(const guest_t *g, + uint64_t *start, + uint64_t length, + uint64_t align, + uint64_t max_addr) +{ + if (!g || !start || !length) + return; + bool advanced; + do { + advanced = false; + if (*start > max_addr || length > max_addr - *start) + return; + uint64_t end = *start + length; + uint64_t slots = mmap_fastpath_active_slots(g); + while (slots) { + int slot = mmap_fastpath_pop_slot(&slots); + shim_mmap_control_t *c = mmap_fastpath_control(g, slot); + if (!(atomic_load_explicit(&c->flags, memory_order_acquire) & + SHIM_MMAP_CTRL_ENABLED)) + continue; + uint64_t cursor = + atomic_load_explicit(&c->cursor, memory_order_acquire); + uint64_t limit = + atomic_load_explicit(&c->arena_limit, memory_order_relaxed); + if (cursor < limit && *start < limit && end > cursor) { + *start = ALIGN_UP(limit, align); + advanced = true; + break; + } + + /* mmap_lock acquisition closed the producer gate and drained mmap + * publications before a host gap scan reaches here. Pending reuse + * entries are therefore stable: reserve them precisely instead of + * hoarding the whole arena after a vCPU becomes idle. */ + uint32_t reuse_head = + atomic_load_explicit(&c->reuse.head, memory_order_acquire); + uint32_t reuse_tail = + atomic_load_explicit(&c->reuse.tail, memory_order_acquire); + if ((uint32_t) (reuse_tail - reuse_head) > + SHIM_MMAP_REUSE_RING_SIZE) { + *start = ALIGN_UP(limit, align); + advanced = true; + break; + } + for (uint32_t seq = reuse_head; seq != reuse_tail; seq++) { + const mmap_reuse_entry_t *e = + &c->reuse + .entries[seq & (SHIM_MMAP_REUSE_RING_SIZE - 1)]; + if (!e->length || e->addr > UINT64_MAX - e->length) + continue; + uint64_t reuse_end = e->addr + e->length; + if (*start < reuse_end && end > e->addr) { + *start = ALIGN_UP(reuse_end, align); + advanced = true; + break; + } + } + if (advanced) + break; + } + } while (advanced); +} + /* Host kernel page size (16 KiB on Apple Silicon, typically 4 KiB on Intel * macOS). MAP_FIXED requires addr/length/offset multiples of this, so an * overlay onto a guest 4 KiB-aligned IPA is only applicable when the IPA @@ -103,6 +1502,11 @@ static int restore_snapshot_page_tables(guest_t *g, static int restore_region_snapshots(guest_t *g, region_snapshot_t *snaps, int n); +static int read_file_range_to_guest(guest_t *g, + uint64_t gpa, + int fd, + uint64_t file_off, + uint64_t len); static int region_count_after_removes(const guest_t *g, const remove_range_t *ranges, @@ -298,6 +1702,7 @@ static uint64_t find_free_gap_inner(const guest_t *g, * allocation patterns. */ uint64_t gap_start = ALIGN_UP(min_addr, align); + mmap_fastpath_skip_reserved(g, &gap_start, length, align, max_addr); /* Skip the prefix of regions entirely below gap_start in O(log n). After a * successful allocation the gap hint advances near or past the existing @@ -306,6 +1711,7 @@ static uint64_t find_free_gap_inner(const guest_t *g, */ for (int i = guest_region_first_end_above(g, gap_start); i < g->nregions; i++) { + mmap_fastpath_skip_reserved(g, &gap_start, length, align, max_addr); /* A region can still slip below gap_start after the ALIGN_UP advance * below skips past a smaller adjacent region; keep the cheap guard. */ @@ -335,6 +1741,7 @@ static uint64_t find_free_gap_inner(const guest_t *g, } /* Check trailing space after all regions */ + mmap_fastpath_skip_reserved(g, &gap_start, length, align, max_addr); if (gap_start <= max_addr && length <= max_addr - gap_start) return gap_start; return UINT64_MAX; /* No suitable gap found */ @@ -714,6 +2121,7 @@ static int64_t sys_mmap_high_va(guest_t *g, if (!host) goto fail; memset(host, 0, BLOCK_2MIB); + guest_dirty_clear_zeroed_range(g, gpa, gpa + BLOCK_2MIB); /* Detect freshness BEFORE guest_map_va_range so the decision is not * confused by a prior high-VA mmap into the same 2 MiB block. A fresh @@ -787,27 +2195,11 @@ static int64_t sys_mmap_high_va(guest_t *g, } else if (prot != LINUX_PROT_NONE) { memset(map_host, 0, length); replaced_bytes_dirty = replacing_existing; - uint8_t *dst = map_host; - size_t remaining = length; - off_t file_off = (off_t) offset; - while (remaining > 0) { - ssize_t nr = pread(host_backing_fd, dst, remaining, file_off); - if (nr < 0) { - if (errno == EINTR) - continue; - /* Real host I/O failure (not EINTR); previously the loop broke - * without setting ret and the syscall returned a "successful" - * partially-zero mapping. - */ - ret = linux_errno(); - goto fail; - } - if (nr == 0) - break; - dst += nr; - remaining -= (size_t) nr; - file_off += nr; - } + uint64_t gpa_for_addr = backing_gpa_start + (addr - va_start); + ret = read_file_range_to_guest(g, gpa_for_addr, host_backing_fd, offset, + length); + if (ret < 0) + goto fail; } /* Install L3 PTEs for the actual mapped range. Fresh blocks were fully @@ -968,11 +2360,39 @@ static int64_t sys_mmap_high_va(guest_t *g, return ret; } +/* Page-table high-water slot (mmap_rx_end / mmap_end) for a TTBR0 mmap offset, + * or NULL if @off is not in either mmap arena. + */ +static uint64_t *mmap_pt_end_for_off(guest_t *g, uint64_t off) +{ + if (off >= MMAP_RX_BASE && off < MMAP_BASE) + return &g->mmap_rx_end; + if (off >= MMAP_BASE) + return &g->mmap_end; + return NULL; +} + +/* Advance the allocation high-water mark (mmap_rx_next / mmap_next) that fork + * IPC state transfer replays, for whichever arena @off belongs to. + */ +static void mmap_bump_next(guest_t *g, uint64_t off, uint64_t end) +{ + if (off >= MMAP_RX_BASE && off < MMAP_BASE) { + if (end > g->mmap_rx_next) + g->mmap_rx_next = end; + } else if (off >= MMAP_BASE) { + if (end > g->mmap_next) + g->mmap_next = end; + } +} + static int mremap_extend_range(guest_t *g, uint64_t off, uint64_t size, int prot) { + uint64_t *pt_end = mmap_pt_end_for_off(g, off); + if (prot == LINUX_PROT_NONE) { guest_invalidate_ptes(g, off, off + size); return 0; @@ -983,10 +2403,67 @@ static int mremap_extend_range(guest_t *g, uint64_t ext_end = ALIGN_UP(off + size, BLOCK_2MIB); if (ext_end > g->guest_size) ext_end = g->guest_size; - if (guest_extend_page_tables(g, ext_start, ext_end, page_perms) < 0) + size_t nblocks = pt_end ? (size_t) ((ext_end - ext_start) / BLOCK_2MIB) : 0; + bool *block_preexisting = NULL; + if (nblocks) { + block_preexisting = calloc(nblocks, sizeof(*block_preexisting)); + if (!block_preexisting) + return -1; + for (size_t i = 0; i < nblocks; i++) + block_preexisting[i] = + guest_va_block_mapped(g, ext_start + (uint64_t) i * BLOCK_2MIB); + } + if (guest_extend_page_tables(g, ext_start, ext_end, page_perms) < 0) { + free(block_preexisting); return -1; - guest_update_perms(g, off, off + size, page_perms); + } + uint64_t saved_pt_end = pt_end ? *pt_end : 0; + if (pt_end && ext_end > *pt_end) + *pt_end = ext_end; + + for (size_t i = 0; i < nblocks; i++) { + if (block_preexisting[i]) + continue; + uint64_t b = ext_start + (uint64_t) i * BLOCK_2MIB; + uint64_t bend = b + BLOCK_2MIB; + if (bend > ext_end) + bend = ext_end; + uint64_t keep_start = off > b ? off : b; + uint64_t keep_end = off + size < bend ? off + size : bend; + if (keep_start <= b && keep_end >= bend) + continue; + if (guest_split_block(g, b) < 0) + goto fail; + if (b < keep_start && guest_invalidate_ptes(g, b, keep_start) < 0) + goto fail; + if (keep_end < bend && guest_invalidate_ptes(g, keep_end, bend) < 0) + goto fail; + } + + if (guest_update_perms(g, off, off + size, page_perms) < 0) + goto fail; + free(block_preexisting); return 0; + + /* Roll back: re-invalidate every fresh block whole (idempotent -- the + * forward pass already cleared the non-kept subranges) plus the kept [off, + * off+size) span, returning the range to its pre-extend state. + */ +fail: + for (size_t i = 0; i < nblocks; i++) { + if (block_preexisting[i]) + continue; + uint64_t b = ext_start + (uint64_t) i * BLOCK_2MIB; + uint64_t bend = b + BLOCK_2MIB; + if (bend > ext_end) + bend = ext_end; + (void) guest_invalidate_ptes(g, b, bend); + } + (void) guest_invalidate_ptes(g, off, off + size); + if (pt_end) + *pt_end = saved_pt_end; + free(block_preexisting); + return -1; } static int hvf_apply_file_overlay(guest_t *g, @@ -1017,6 +2494,11 @@ static int read_file_range_to_guest(guest_t *g, uint8_t *dst = host_ptr_for_gpa(g, gpa); if (!dst) return -LINUX_EFAULT; + /* A short read, EOF, or later error may still leave nonzero bytes in the + * destination. Mark before the first pread so every exit is conservative. + */ + if (len <= UINT64_MAX - gpa) + guest_dirty_mark_range(g, gpa, gpa + len); size_t remaining = len; while (remaining > 0) { @@ -1305,12 +2787,16 @@ static int rollback_fresh_mmap_allocation(guest_t *g, { if (overlay_installed) hvf_remove_file_overlay(g, overlay_ipa, overlay_len); - if (guest_invalidate_ptes(g, start, start + length) < 0) + uint64_t end = start + length; + uint64_t cur_mmap_end = g->mmap_end; + uint64_t cur_mmap_rx_end = g->mmap_rx_end; + if (guest_invalidate_ptes(g, start, end) < 0) return -LINUX_ENOMEM; g->mmap_next = saved_mmap_next; - g->mmap_end = saved_mmap_end; + g->mmap_end = cur_mmap_end > saved_mmap_end ? cur_mmap_end : saved_mmap_end; g->mmap_rx_next = saved_mmap_rx_next; - g->mmap_rx_end = saved_mmap_rx_end; + g->mmap_rx_end = cur_mmap_rx_end > saved_mmap_rx_end ? cur_mmap_rx_end + : saved_mmap_rx_end; g->mmap_rw_gap_hint = saved_rw_gap_hint; g->mmap_rx_gap_hint = saved_rx_gap_hint; return 0; @@ -1531,6 +3017,101 @@ static void hvf_remap_segments_best_effort(guest_t *g, } } +/* Replace a retired slab range with fresh zero-fill pages and refresh HVF's + * cached host VA->PA translation. Split only the two range boundaries, just + * as the file-overlay path does, so the expensive detach/remap covers the + * retired bytes rather than an initial slab-sized HVF segment. Reusing the + * same arena bounds is a no-op on later calls. If many distinct boundaries + * exhaust GUEST_MAX_HVF_SEGMENTS, splitting fails safely and the caller falls + * back to memset. + * + * The shared-slab case punches a hole in the backing file before re-establishing + * its MAP_SHARED host mapping. This keeps future clonefile fork snapshots in + * sync with the new zero state. A private slab can be replaced directly by a + * fresh MAP_ANON mapping. + * + * Caller holds mmap_lock, has invalidated every stage-1 PTE in the target, and + * has quiesced sibling vCPUs. + */ +static int hvf_replace_slab_zero_range_quiesced(guest_t *g, + uint64_t ipa, + uint64_t len) +{ + if (!g || !len || (ipa & (BLOCK_2MIB - 1)) || + (len & (BLOCK_2MIB - 1)) || ipa >= g->guest_size || + len > g->guest_size - ipa) + return -LINUX_EINVAL; + + uint64_t end = ipa + len; + hvf_segment_t segments[GUEST_MAX_HVF_SEGMENTS]; + int err = hvf_segment_split_range_boundaries(g, ipa, end); + if (err < 0) + return err; + int nsegments = hvf_segment_collect_range( + g, ipa, end, segments, GUEST_MAX_HVF_SEGMENTS); + if (nsegments < 0) + return nsegments; + + int unmapped = 0; + for (int i = 0; i < nsegments; i++) { + if (hv_vm_unmap(segments[i].ipa, segments[i].len) != HV_SUCCESS) { + hvf_remap_segments_best_effort(g, segments, unmapped); + return -LINUX_EIO; + } + unmapped++; + } + + err = 0; + if (g->shm_fd >= 0) { + struct fpunchhole hole = { + .fp_flags = 0, + .reserved = 0, + .fp_offset = (off_t) ipa, + .fp_length = (off_t) len, + }; + if (fcntl(g->shm_fd, F_PUNCHHOLE, &hole) < 0) + err = (int) linux_errno(); + } + + if (err == 0) { + void *target = (uint8_t *) g->host_base + ipa; + int flags = MAP_FIXED; + int fd = -1; + off_t offset = 0; + if (g->shm_fd >= 0) { + flags |= MAP_SHARED; + fd = g->shm_fd; + offset = (off_t) ipa; + } else { + flags |= MAP_ANON | MAP_PRIVATE; + } + if (mmap(target, len, PROT_READ | PROT_WRITE, flags, fd, offset) == + MAP_FAILED) + err = (int) linux_errno(); + } + + /* Whether replacement succeeded or failed, restore every detached segment. + * On a remap failure, retry the failed and remaining segments best-effort; + * already-restored prefixes must not be mapped twice. + */ + for (int i = 0; i < nsegments; i++) { + hv_return_t r = + hv_vm_map((uint8_t *) g->host_base + segments[i].ipa, + segments[i].ipa, segments[i].len, HVF_SEGMENT_FLAGS); + if (r == HV_SUCCESS) + continue; + log_error( + "munmap retire: hv_vm_map(0x%llx, 0x%llx) failed with 0x%x " + "after backing replacement", + (unsigned long long) segments[i].ipa, + (unsigned long long) segments[i].len, (int) r); + hvf_remap_segments_best_effort(g, &segments[i], nsegments - i); + return -LINUX_EIO; + } + + return err; +} + /* Apply a real MAP_SHARED file overlay at [ipa, ipa+len) backed by [fd, * file_off). The IPA range may be sub-2 MiB; the containing 2 MiB segment is * split out first if it is not already isolated. Caller holds mmap_lock and has @@ -1639,6 +3220,8 @@ static int hvf_apply_file_overlay(guest_t *g, thread_quiesce_siblings(); int err = hvf_apply_file_overlay_quiesced(g, ipa, len, fd, file_off); thread_resume_siblings(); + if (err == 0 && len <= UINT64_MAX - ipa) + guest_dirty_mark_range(g, ipa, ipa + len); return err; } @@ -1677,6 +3260,12 @@ static int hvf_remove_file_overlay_quiesced(guest_t *g, hvf_remap_segments_best_effort(g, segments, nsegments); return err; } + /* Restoring shm-backed slab pages may reveal an older nonzero snapshot. The + * following munmap/MAP_FIXED path will clear the bit only after it has + * actually zeroed a complete 2 MiB block. + */ + if (len <= UINT64_MAX - ipa) + guest_dirty_mark_range(g, ipa, ipa + len); for (int i = 0; i < nsegments; i++) { if (hv_vm_map((uint8_t *) g->host_base + segments[i].ipa, @@ -1860,6 +3449,19 @@ int64_t sys_mmap(guest_t *g, bool needs_exec = (prot & LINUX_PROT_EXEC) != 0; bool is_prot_none = (prot == LINUX_PROT_NONE); bool is_noreserve = is_anon && (flags & LINUX_MAP_NORESERVE) != 0; + /* Anonymous mappings defer page-table creation and zeroing to first touch + * (guest fault or host-side access), like MAP_NORESERVE always has. This + * keeps mmap()/munmap() cost independent of length: a multi-GiB reservation + * costs neither an eager PTE walk nor a full-length memset, and + * never-touched blocks consume no page-table pool. PROT_NONE stays a pure + * reservation (faults deliver SIGSEGV, not materialization), and MAP_FIXED + * keeps the eager path because it must atomically replace live mappings. + * Shared anonymous memory stays eager unless the caller opted into + * MAP_NORESERVE (the historical lazy set), since deferred zeroing has never + * been exercised against the fork snapshot paths for it. + */ + bool is_lazy = is_anon && !is_prot_none && + ((flags & LINUX_MAP_SHARED) == 0 || is_noreserve); host_fd_ref_t backing_ref = {.fd = -1, .owned = 0}; int host_backing_fd = -1, track_backing_fd = -1; /* Tracks whether hvf_apply_file_overlay has installed a host @@ -1887,13 +3489,24 @@ int64_t sys_mmap(guest_t *g, region_snapshot_t *replaced_snaps = NULL; int replaced_nsnaps = 0; bool replaced_regions_removed = false; + /* Linux kernel rejects MAP_FIXED with non-page-aligned address (checked + * below); the flag itself is needed early because it gates the lazy path. + */ + bool is_fixed = + (flags & LINUX_MAP_FIXED) || (flags & LINUX_MAP_FIXED_NOREPLACE); + if (is_fixed) + is_lazy = false; int track_flags = ((flags & LINUX_MAP_SHARED) ? LINUX_MAP_SHARED : LINUX_MAP_PRIVATE); if (is_anon) track_flags |= LINUX_MAP_ANONYMOUS; - /* Preserve MAP_NORESERVE in region metadata before merge checks run. */ - if (is_noreserve) + /* Preserve MAP_NORESERVE in region metadata before merge checks run. The + * same bit doubles as the internal lazy marker: guest_region_add_ex derives + * the region's deferred-PTE flag from it, and it is not guest visible + * (/proc/self/maps prints only prot and shared/private). + */ + if (is_noreserve || is_lazy) track_flags |= LINUX_MAP_NORESERVE; /* The memory syscall layer handles all mmap variants. Aligned file-backed @@ -1928,9 +3541,17 @@ int64_t sys_mmap(guest_t *g, if (length == 0) return -LINUX_ENOMEM; + /* A non-fixed nonzero address is a strong Linux hint. If it lands in the + * current (stopped) vCPU's invisible arena tail, release that tail before + * gap finding so implementation-only VA preparation does not perturb the + * address the application observes. Sibling arenas stay immutable without + * quiesce; their disjoint high-water placement makes self-overlap the + * normal and important case (allocator hinting near its previous result). + */ + if (!is_fixed && addr != 0) + mmap_fastpath_release_current_hint_locked(g, addr, length); + /* Linux kernel rejects MAP_FIXED with non-page-aligned address */ - bool is_fixed = - (flags & LINUX_MAP_FIXED) || (flags & LINUX_MAP_FIXED_NOREPLACE); if (is_fixed && (addr & 4095)) return -LINUX_EINVAL; @@ -1939,6 +3560,14 @@ int64_t sys_mmap(guest_t *g, */ bool is_noreplace = (flags & LINUX_MAP_FIXED_NOREPLACE) != 0; + /* A fixed mapping may replace an address previously handed out by an EL1 + * arena with a file/shared/stack-like mapping. Revoke all descriptors + * before making that semantic transition so a later fast munmap cannot + * classify it from the stale arena bounds. + */ + if (is_fixed) + mmap_fastpath_revoke_all_locked(g, false); + uint64_t result_off; /* Result as offset (0-based) */ if (is_fixed) { /* Addresses above TASK_SIZE (bit 63 set or beyond user VA range) are @@ -1979,6 +3608,7 @@ int64_t sys_mmap(guest_t *g, uint64_t fix_end = off + length; if (guest_range_hits_infra(g, off, fix_end)) return -LINUX_EINVAL; + guest_materialize_wait_range_locked(g, off, fix_end); result_off = off; @@ -2288,9 +3918,7 @@ int64_t sys_mmap(guest_t *g, return -LINUX_ENOMEM; } /* High-water mark for fork IPC state transfer */ - uint64_t rx_hwm = result_off + length; - if (rx_hwm > g->mmap_rx_next) - g->mmap_rx_next = rx_hwm; + mmap_bump_next(g, result_off, result_off + length); } else { /* RW (or PROT_NONE, or PROT_READ): allocate from main mmap region. * Honor the address hint if provided and within bounds. Some @@ -2349,9 +3977,7 @@ int64_t sys_mmap(guest_t *g, return -LINUX_ENOMEM; } /* High-water mark for fork IPC state transfer */ - uint64_t rw_hwm = result_off + length; - if (rw_hwm > g->mmap_next) - g->mmap_next = rw_hwm; + mmap_bump_next(g, result_off, result_off + length); } if (!region_has_capacity_after_removes(g, NULL, 0, 1)) { host_fd_ref_close(&backing_ref); @@ -2376,7 +4002,7 @@ int64_t sys_mmap(guest_t *g, guest_invalidate_ptes(g, result_off, result_off + length); } - if (!is_prot_none && !is_fixed && !is_noreserve) { + if (!is_prot_none && !is_fixed && !is_lazy) { /* Extend page tables for this specific allocation range only. * guest_extend_page_tables skips already-mapped blocks, so calling it * on pre-mapped regions is a no-op. This avoids creating entries for @@ -2435,16 +4061,24 @@ int64_t sys_mmap(guest_t *g, g->mmap_end = ext_end; } - /* Zero the mapped region */ + /* Zero the mapped region. RX mappings cannot be dirtied through their + * published PTEs, so a complete-block zero can make them clean again. + * Other mappings currently use writable stage-1 entries and must stay + * conservatively dirty even if their requested Linux prot is read-only. + */ memset((uint8_t *) g->host_base + result_off, 0, length); + if (needs_exec && !(prot & LINUX_PROT_WRITE)) + guest_dirty_clear_zeroed_range(g, result_off, result_off + length); } - /* MAP_NORESERVE: invalidate any stale PTEs (like PROT_NONE path) but track - * the region for lazy materialization on first fault. Page table entries - * will be created by guest_materialize_lazy() when the guest first touches - * a page in this range. + /* Lazy (private anonymous, incl. MAP_NORESERVE): invalidate any stale PTEs + * (like the PROT_NONE path) but track the region for lazy materialization + * on first fault. Page table entries will be created by + * guest_materialize_lazy() when the guest first touches a page in this + * range, or by the host-access fault-in path when a syscall targets the + * range before the guest ever touches it. */ - if (is_noreserve && !is_fixed) { + if (is_lazy) { guest_invalidate_ptes(g, result_off, result_off + length); } @@ -2512,6 +4146,7 @@ int64_t sys_mmap(guest_t *g, overlay_ipa = result_off; overlay_len = nf_overlay_len; } else { + guest_dirty_mark_range(g, result_off, result_off + length); uint8_t *dst = (uint8_t *) g->host_base + result_off; size_t remaining = length; off_t file_off = offset; @@ -2688,6 +4323,11 @@ int64_t sys_mremap(guest_t *g, */ if (guest_range_hits_infra(g, old_off, old_off + old_size)) return -LINUX_EINVAL; + if (old_off < g->guest_size) + guest_materialize_wait_range_locked(g, old_off, + old_size > g->guest_size - old_off + ? g->guest_size + : old_off + old_size); /* Verify the whole source range is covered by one tracked VMA. mremap() * must not copy holes or unrelated adjacent mappings. @@ -2721,10 +4361,13 @@ int64_t sys_mremap(guest_t *g, /* Zero the trimmed region on its real backing (high-VA tails live at * gpa_base, not host_base + tail_off). */ - memset(host_ptr_for_gpa(g, src_gpa_base + (tail_off - src_start)), 0, - tail_end - tail_off); + uint64_t tail_gpa = src_gpa_base + (tail_off - src_start); + if (guest_invalidate_ptes(g, tail_off, tail_end) < 0) + return -LINUX_ENOMEM; + memset(host_ptr_for_gpa(g, tail_gpa), 0, tail_end - tail_off); + guest_dirty_clear_zeroed_range(g, tail_gpa, + tail_gpa + (tail_end - tail_off)); guest_region_remove(g, tail_off, tail_end); - guest_invalidate_ptes(g, tail_off, tail_end); if (tail_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = tail_off; if (tail_off < g->mmap_rx_gap_hint) @@ -2750,6 +4393,7 @@ int64_t sys_mremap(guest_t *g, */ if (guest_range_hits_infra(g, new_off, new_off + new_size)) return -LINUX_EINVAL; + guest_materialize_wait_range_locked(g, new_off, new_off + new_size); /* Linux rejects MREMAP_FIXED when old and new ranges overlap */ uint64_t old_end = old_off + old_size, new_end = new_off + new_size; @@ -2870,10 +4514,16 @@ int64_t sys_mremap(guest_t *g, } (void) restore_snapshot_overlays_in_place(g, dest_snaps, dest_nsnaps); + int pt_err = restore_snapshot_page_tables( + g, new_off, new_off + new_size, dest_snaps, dest_nsnaps); + if (pt_err < 0) + restore_err = pt_err; dispose_region_snapshots(&dest_snaps, &dest_nsnaps); dispose_region_snapshots(&source_snaps, &source_nsnaps); if (track_backing_fd >= 0) close(track_backing_fd); + if (restore_err < 0) + return restore_err; return -LINUX_ENOMEM; } @@ -2936,12 +4586,20 @@ int64_t sys_mremap(guest_t *g, memset((uint8_t *) g->host_base + new_off + old_size, 0, new_size - old_size); + if (prot == LINUX_PROT_NONE) + guest_dirty_clear_zeroed_range(g, new_off, new_off + new_size); + else + guest_dirty_mark_range(g, new_off, new_off + new_size); + /* Remove old mapping */ if (old_size > 0) { - memset(host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), 0, - old_size); + uint64_t old_gpa = src_gpa_base + (old_off - src_start); + bool invalidated = + guest_invalidate_ptes(g, old_off, old_off + old_size) == 0; + memset(host_ptr_for_gpa(g, old_gpa), 0, old_size); + if (invalidated) + guest_dirty_clear_zeroed_range(g, old_gpa, old_gpa + old_size); guest_region_remove(g, old_off, old_off + old_size); - guest_invalidate_ptes(g, old_off, old_off + old_size); if (old_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = old_off; if (old_off < g->mmap_rx_gap_hint) @@ -3025,6 +4683,9 @@ int64_t sys_mremap(guest_t *g, } memset((uint8_t *) g->host_base + grow_off, 0, grow_len); + if (!(prot & LINUX_PROT_WRITE)) + guest_dirty_clear_zeroed_range(g, grow_off, + grow_off + grow_len); /* Update region tracking: remove old, add extended */ guest_region_remove(g, old_off, old_off + old_size); @@ -3041,14 +4702,7 @@ int64_t sys_mremap(guest_t *g, mark_region_backing_ro(g, old_off, old_off + new_size); /* Update high-water marks */ - uint64_t hwm = old_off + new_size; - if (old_off >= MMAP_RX_BASE && old_off < MMAP_BASE) { - if (hwm > g->mmap_rx_next) - g->mmap_rx_next = hwm; - } else if (old_off >= MMAP_BASE) { - if (hwm > g->mmap_next) - g->mmap_next = hwm; - } + mmap_bump_next(g, old_off, old_off + new_size); return (int64_t) old_addr; } @@ -3176,13 +4830,21 @@ int64_t sys_mremap(guest_t *g, memset((uint8_t *) g->host_base + new_off + old_size, 0, new_size - old_size); + if (prot == LINUX_PROT_NONE) + guest_dirty_clear_zeroed_range(g, new_off, new_off + new_size); + else + guest_dirty_mark_range(g, new_off, new_off + new_size); + /* Remove old mapping. Any live source overlay was already torn down * before the destination range was touched. */ - memset(host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), 0, - old_size); + uint64_t old_gpa = src_gpa_base + (old_off - src_start); + bool invalidated = + guest_invalidate_ptes(g, old_off, old_off + old_size) == 0; + memset(host_ptr_for_gpa(g, old_gpa), 0, old_size); + if (invalidated) + guest_dirty_clear_zeroed_range(g, old_gpa, old_gpa + old_size); guest_region_remove(g, old_off, old_off + old_size); - guest_invalidate_ptes(g, old_off, old_off + old_size); if (old_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = old_off; if (old_off < g->mmap_rx_gap_hint) @@ -3197,14 +4859,7 @@ int64_t sys_mremap(guest_t *g, mark_region_backing_ro(g, new_off, new_off + new_size); /* Update high-water marks */ - uint64_t hwm = new_off + new_size; - if (new_off >= MMAP_RX_BASE && new_off < MMAP_BASE) { - if (hwm > g->mmap_rx_next) - g->mmap_rx_next = hwm; - } else if (new_off >= MMAP_BASE) { - if (hwm > g->mmap_next) - g->mmap_next = hwm; - } + mmap_bump_next(g, new_off, new_off + new_size); return (int64_t) guest_ipa(g, new_off); } @@ -3293,6 +4948,8 @@ int64_t sys_madvise(guest_t *g, uint64_t addr, uint64_t length, int advice) */ if (!madvise_range_mapped(g, off, length)) return -LINUX_ENOMEM; + if (in_primary) + guest_materialize_wait_range_locked(g, off, off + length); uint64_t end = off + length; for (int i = 0; i < g->nregions; i++) { @@ -3406,11 +5063,38 @@ static int compare_range_pair(const void *a, const void *b) return 0; } +/* Coalesced sub-ranges of a munmap that must be zeroed. Sized so that even a + * pathologically fragmented lazy mapping (alternating materialized and + * untouched blocks) rarely overflows; on overflow the remainder of the region + * overlap is zeroed wholesale, which is always correct (zeroing already-zero + * slab bytes), just slower. + */ +#define MUNMAP_ZERO_RANGES_MAX 128 + +typedef struct { + uint64_t lo, hi; +} zero_range_t; + +static void zero_range_push(zero_range_t *ranges, + int *n, + uint64_t lo, + uint64_t hi) +{ + if (lo >= hi) + return; + if (*n > 0 && ranges[*n - 1].hi == lo) { + ranges[*n - 1].hi = hi; + return; + } + ranges[(*n)++] = (zero_range_t) {lo, hi}; +} + static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) { /* Reject munmap targeting VM infrastructure regions. */ if (guest_range_hits_infra(g, unmap_off, end)) return -LINUX_EINVAL; + guest_materialize_wait_range_locked(g, unmap_off, end); /* Restore slab backing under any active MAP_SHARED file overlay before * zeroing the host VA. Without this, the memset below would write zeros @@ -3420,14 +5104,19 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) if (cleanup_err < 0) return cleanup_err; - /* Invalidate PTEs first. This may need to split a 2MiB block which can fail - * if the page table pool is exhausted. Failing before region removal keeps - * metadata consistent. + /* Record which sub-ranges need zeroing BEFORE the PTE invalidation below + * destroys the evidence. Eager regions are zeroed across the whole overlap, + * as before. Lazy (deferred-PTE) regions only need their materialized 2MiB + * blocks zeroed: a block with no L2 mapping was never touched through PTEs, + * host-side fault-in materializes before writing, and the previous unmap of + * that slab range zeroed it -- so its bytes are still zero. This keeps + * munmap cost proportional to memory actually touched instead of to the + * mapping length. */ - if (guest_invalidate_ptes(g, unmap_off, end) < 0) - return -LINUX_ENOMEM; + zero_range_t zr[MUNMAP_ZERO_RANGES_MAX]; + int nzr = 0; for (int i = 0; i < g->nregions; i++) { - guest_region_t *r = &g->regions[i]; + const guest_region_t *r = &g->regions[i]; if (r->start >= end) break; if (r->end <= unmap_off) @@ -3436,7 +5125,52 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) continue; uint64_t zstart = (r->start > unmap_off) ? r->start : unmap_off; uint64_t zend = (r->end < end) ? r->end : end; - memset((uint8_t *) g->host_base + zstart, 0, zend - zstart); + if (!r->noreserve) { + if (nzr >= MUNMAP_ZERO_RANGES_MAX) { + /* Out of slots: widen the last range instead of dropping any + * span that must be zeroed. Everything between ranges lies + * inside [unmap_off, end) and is being unmapped, so zeroing the + * gap as well is harmless. + */ + zr[nzr - 1].hi = zend; + continue; + } + zero_range_push(zr, &nzr, zstart, zend); + continue; + } + for (uint64_t b = zstart & ~(BLOCK_2MIB - 1); b < zend;) { + if (!guest_va_block_mapped(g, b)) { + /* Jump through the PTE occupancy index to the next materialized + * block. A huge untouched reservation therefore does no work + * proportional to its virtual length. + */ + b = guest_va_next_present_block(g, b + BLOCK_2MIB, zend); + continue; + } + uint64_t lo = (b > zstart) ? b : zstart; + uint64_t hi = (b + BLOCK_2MIB < zend) ? b + BLOCK_2MIB : zend; + if (nzr >= MUNMAP_ZERO_RANGES_MAX) { + /* Out of slots: fold the remainder of this overlap into the + * last range and stop scanning blocks. + */ + zr[nzr - 1].hi = zend; + break; + } + zero_range_push(zr, &nzr, lo, hi); + b += BLOCK_2MIB; + } + } + + /* Invalidate PTEs first. This may need to split a 2MiB block which can fail + * if the page table pool is exhausted. Failing before region removal keeps + * metadata consistent. + */ + if (guest_invalidate_ptes(g, unmap_off, end) < 0) + return -LINUX_ENOMEM; + for (int i = 0; i < nzr; i++) { + memset((uint8_t *) g->host_base + zr[i].lo, 0, + zr[i].hi - zr[i].lo); + guest_dirty_clear_zeroed_range(g, zr[i].lo, zr[i].hi); } guest_region_remove(g, unmap_off, end); if (unmap_off < g->mmap_rw_gap_hint) @@ -3461,7 +5195,7 @@ void mem_cleanup_deferred_stack_unmaps(guest_t *g, thread_entry_t *t) if (nranges <= 0) return; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); for (int i = 0; i < nranges; i++) { int rc = munmap_guest_range(g, starts[i], ends[i]); if (rc < 0) { @@ -3474,7 +5208,7 @@ void mem_cleanup_deferred_stack_unmaps(guest_t *g, thread_entry_t *t) } thread_drop_deferred_stack_unmap(t, starts[i], ends[i]); } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); } /* sys_munmap. */ @@ -3542,6 +5276,7 @@ int64_t sys_munmap(guest_t *g, uint64_t addr, uint64_t length) thread_finish_deferred_stack_ranges(txns, nranges); } } + mmap_fastpath_rewind_current_if_clean_locked(g); return 0; } @@ -3569,6 +5304,13 @@ int64_t sys_mprotect(guest_t *g, uint64_t addr, uint64_t length, int prot) if (addr > UINT64_MAX - length) return -LINUX_EINVAL; + /* mmap_lock acquisition already drained EL1 publications. Permission-only + * edits do not change an arena mapping's private/anonymous/noreserve + * classification, so keep its control enabled: once this call has created + * an L3 table, later single-page R<->RW transitions can stay in EL1. + */ + mmap_fastpath_mark_mprotect_locked(g, addr, addr + length); + if (addr <= 0x0000FFFFFFFFFFFFULL) { if (addr >= g->guest_size) { uint64_t mprot_end = addr + length; @@ -3624,6 +5366,7 @@ int64_t sys_mprotect(guest_t *g, uint64_t addr, uint64_t length, int prot) */ if (guest_range_hits_infra(g, mprot_off, mprot_end)) return -LINUX_EINVAL; + guest_materialize_wait_range_locked(g, mprot_off, mprot_end); /* Same max_prot check as the high-VA branch above. */ if ((prot & LINUX_PROT_WRITE) && @@ -3641,6 +5384,18 @@ int64_t sys_mprotect(guest_t *g, uint64_t addr, uint64_t length, int prot) if (prot != LINUX_PROT_NONE) { int page_perms = prot_to_perms(prot); + /* Materialize lazy blocks in the range at their region's + * current prot before the block-granular extend below. The + * extend stamps whole 2MiB blocks with page_perms; on an + * unmaterialized lazy region that would hand every neighbor + * page OUTSIDE [mprot_off, mprot_end) the sub-range's + * permissions (region says RW, PTE says R-only, host-side + * writes EFAULT). guest_materialize_lazy covers block-within- + * region at region prot, so after this the extend is a no-op + * for lazy regions and update_perms below adjusts only the + * requested range. + */ + guest_lazy_faultin_locked(g, mprot_off, mprot_end - mprot_off); if (guest_extend_page_tables(g, mprot_off, mprot_end, page_perms) < 0) return -LINUX_ENOMEM; @@ -3934,7 +5689,13 @@ int mmap_fork_prepare_anon_shared(guest_t *g, if (!txn) return -LINUX_ENOMEM; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); + /* fork callers have quiesced siblings. Drain their last publications, + * revoke every descriptor, and trim never-consumed arena tails before the + * legacy [MMAP_BASE,mmap_next) snapshot range is computed. + */ + mmap_fastpath_revoke_all_locked(g, true); + guest_materialize_wait_all_locked(g); size_t hps = host_page_size_cached(); @@ -4081,7 +5842,7 @@ int mmap_fork_prepare_anon_shared(guest_t *g, for (int k = 0; k < n_regions; k++) close(dup_fds[k]); close(fd); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); *txn_out = txn; return -LINUX_ENOMEM; } @@ -4094,7 +5855,7 @@ int mmap_fork_prepare_anon_shared(guest_t *g, for (int k = 0; k < n_regions; k++) close(dup_fds[k]); close(fd); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); *txn_out = txn; return nsnaps; } @@ -4134,7 +5895,7 @@ int mmap_fork_prepare_anon_shared(guest_t *g, close(fd); } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); *txn_out = txn; return 0; } @@ -4153,7 +5914,7 @@ int mmap_fork_abort_anon_shared(guest_t *g, mmap_fork_anon_shared_txn_t *txn = *txn_ptr; int rc = 0; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); for (int i = txn->noverlays - 1; i >= 0; i--) { const fork_overlay_snapshot_t *ovl = &txn->overlays[i]; @@ -4222,7 +5983,7 @@ int mmap_fork_abort_anon_shared(guest_t *g, } } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); mmap_fork_dispose_anon_shared_txn(txn_ptr); return rc; } @@ -4236,7 +5997,7 @@ int mmap_fork_restore_overlays(guest_t *g, const uint64_t *parent_ovl_start, const uint64_t *parent_ovl_end) { - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); int rc = 0; for (int i = 0; i < g->nregions; i++) { @@ -4325,6 +6086,6 @@ int mmap_fork_restore_overlays(guest_t *g, } } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); return rc; } diff --git a/src/syscall/proc.c b/src/syscall/proc.c index fe2aa533..13eb275e 100644 --- a/src/syscall/proc.c +++ b/src/syscall/proc.c @@ -37,6 +37,7 @@ #include "utils.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "core/vdso.h" #include "runtime/futex.h" @@ -1830,6 +1831,21 @@ int vcpu_run_loop(hv_vcpu_t vcpu, drain_external_guest_signal(); + /* Every return from HVF is a natural retirement point. Drain before + * dispatching syscalls, page faults, MAP_FIXED, fork/exec, signals, or + * exit so no host path can consult pre-munmap region metadata and + * rematerialize an EL1-invalidated page. The helper also drains mmap + * publications before the acquire-snapshotted retire entries. + */ + bool munmap_producer_active = mmap_fastpath_current_producer_active(g); + if (!munmap_producer_active) + mmap_fastpath_drain_vmexit(g); + else if (vexit->reason == HV_EXIT_REASON_EXCEPTION) + log_error( + "%s: exception exit interrupted an active EL1 munmap " + "producer", + prefix); + /* Main: disarm timeout */ if (is_main) alarm(0); @@ -2168,7 +2184,7 @@ int vcpu_run_loop(hv_vcpu_t vcpu, * lookups to prevent races with concurrent * mmap/mprotect/munmap from other vCPU threads. */ - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); /* Check if this is a genuine permission violation (not a * W^X toggle). If the guest region lacks the required @@ -2182,7 +2198,7 @@ int vcpu_run_loop(hv_vcpu_t vcpu, int required = (type == 1) ? LINUX_PROT_WRITE : LINUX_PROT_EXEC; if (reg && !(reg->prot & required)) { - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); uint64_t esr; hv_vcpu_get_sys_reg(vcpu, HV_SYS_REG_ESR_EL1, &esr); signal_set_fault_info(LINUX_SEGV_ACCERR, far, esr); @@ -2210,7 +2226,7 @@ int vcpu_run_loop(hv_vcpu_t vcpu, int sr = guest_split_block(g, block_start); int ur = guest_update_perms(g, page_start, page_end, new_perms); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); if (verbose && (sr < 0 || ur < 0)) log_warn( "%s: W^X toggle FAILED " @@ -2377,9 +2393,9 @@ int vcpu_run_loop(hv_vcpu_t vcpu, uint32_t fsc_type = (fsc >> 2) & 0xF; if (fsc_type == 0x01) { uint64_t fault_off = far_addr - g->ipa_base; - pthread_mutex_lock(&mmap_lock); - int mat = guest_materialize_lazy(g, fault_off); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_acquire(g); + int mat = guest_materialize_lazy_fault(g, fault_off); + mmap_lock_release(); if (mat == 0) { /* Page materialized; the helpers inside * guest_materialize_lazy populated the per-vCPU @@ -2393,6 +2409,25 @@ int vcpu_run_loop(hv_vcpu_t vcpu, * re-fault on the retry, looping until the entry * self-evicts. */ + shim_globals_counter_inc( + g, SHIM_COUNTER_FAULT_MATERIALIZE); + switch ((tlbi_kind_t) cpu_tlbi_req.kind) { + case TLBI_RANGE: + shim_globals_counter_inc( + g, SHIM_COUNTER_FAULT_TLBI_VAE); + break; + case TLBI_RANGE_LARGE: + shim_globals_counter_inc( + g, SHIM_COUNTER_FAULT_TLBI_RVAE); + break; + case TLBI_BROADCAST: + shim_globals_counter_inc( + g, SHIM_COUNTER_FAULT_TLBI_BCAST); + break; + case TLBI_NONE: + default: + break; + } tlbi_request_emit_to_vcpu(vcpu); break; } @@ -2446,10 +2481,10 @@ int vcpu_run_loop(hv_vcpu_t vcpu, uint64_t live_avail = 0; void *live_pt = NULL; if (stale_plausible) { - pthread_mutex_lock(&mmap_lock); - live_pt = guest_ptr_avail(g, far_addr, &live_avail, - want_perm); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_acquire(g); + live_pt = guest_ptr_avail_nofault( + g, far_addr, &live_avail, want_perm); + mmap_lock_release(); } if (live_pt) { /* Bound per vCPU and per (page, faulting PC). A diff --git a/src/syscall/signal.c b/src/syscall/signal.c index d6f628a9..51f23a59 100644 --- a/src/syscall/signal.c +++ b/src/syscall/signal.c @@ -30,12 +30,14 @@ #include "hvutil.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "core/vdso.h" #include "runtime/thread.h" #include "syscall/abi.h" #include "syscall/fd.h" /* signalfd_notify */ +#include "syscall/internal.h" #include "syscall/poll.h" /* wakeup_pipe_signal */ #include "syscall/proc.h" /* proc_get_pid, proc_get_uid, SYSCALL_EXEC_HAPPENED */ #include "syscall/signal.h" @@ -1369,6 +1371,13 @@ int64_t signal_sigaltstack(guest_t *g, uint64_t ss_gva, uint64_t old_ss_gva) } else { if (ss.ss_size < LINUX_MINSIGSTKSZ) return -LINUX_ENOMEM; + /* Alternate stacks have the same lifetime sensitivity as clone + * stacks: once registered, their unmap must take the host path + * instead of being classified only as an anonymous arena range. + */ + mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, false); + mmap_lock_release(); t->altstack_sp = ss.ss_sp; t->altstack_flags = 0; t->altstack_size = ss.ss_size; @@ -1748,8 +1757,31 @@ static int deliver_signal_locked(hv_vcpu_t vcpu, return 1; } +/* Pre-fault the candidate signal-frame windows (current stack and altstack + * top) before sig_lock is taken. The frame write in deliver_signal_locked + * runs under sig_lock; letting it materialize lazy stack pages there would + * acquire mmap_lock in descending lock order. The pre-fault is advisory -- + * the write path still faults in as a backstop -- but it makes the + * under-lock engagement unreachable in practice. Reading the altstack + * fields without sig_lock is benign for the same reason. + */ +static void signal_prefault_frame(hv_vcpu_t vcpu, guest_t *g) +{ + uint64_t need = sizeof(linux_rt_sigframe_t) + 512; + uint64_t sp = 0; + hv_vcpu_get_sys_reg(vcpu, HV_SYS_REG_SP_EL0, &sp); + if (sp > need && sp <= g->guest_size) + guest_lazy_faultin(g, sp - need, need); + thread_entry_t *thr = current_thread; + if (thr && thr->altstack_sp != 0 && + !(thr->altstack_flags & LINUX_SS_DISABLE) && thr->altstack_size > need) + guest_lazy_faultin(g, thr->altstack_sp + thr->altstack_size - need, + need); +} + int signal_deliver(hv_vcpu_t vcpu, guest_t *g, int *exit_code) { + signal_prefault_frame(vcpu, g); pthread_mutex_lock(&sig_lock); uint64_t *blocked = thread_blocked_ptr(); /* Consider this thread's private (thread-directed) set plus the shared @@ -1811,6 +1843,7 @@ int signal_deliver_fault(hv_vcpu_t vcpu, guest_t *g, int signum, int *exit_code) * threads faulting on the same signal collapse into one bit so one fault is * lost. Deliver directly here, never touching sig_state.pending. */ + signal_prefault_frame(vcpu, g); pthread_mutex_lock(&sig_lock); /* Linux force_sig_info_to_task(): a forced synchronous fault cannot be diff --git a/src/syscall/syscall.c b/src/syscall/syscall.c index bac5168b..2a0ac4d1 100644 --- a/src/syscall/syscall.c +++ b/src/syscall/syscall.c @@ -66,6 +66,7 @@ #include "syscall/time.h" #include "core/shim-globals.h" +#include "core/mmap-fastpath.h" #include "debug/syscall-hist.h" @@ -174,9 +175,9 @@ typedef int64_t (*syscall_handler_t)(guest_t *g, { \ (void) g; (void) x0; (void) x1; (void) x2; \ (void) x3; (void) x4; (void) x5; (void) verbose; \ - pthread_mutex_lock(&mmap_lock); \ + mmap_lock_acquire(g); \ int64_t r = (body); \ - pthread_mutex_unlock(&mmap_lock); \ + mmap_lock_release(); \ return r; \ } @@ -484,16 +485,16 @@ static void sc_sync_regions_inline(guest_t *g) * position) cannot make us skip an entry permanently. */ for (int i = 0;; i++) { - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); if (i >= g->nregions) { - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); break; } const guest_region_t *r = &g->regions[i]; int duped = -1; if (r->shared && r->backing_fd >= 0) duped = dup(r->backing_fd); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); if (duped < 0) continue; (void) fsync(duped); @@ -524,7 +525,7 @@ static int64_t sc_sync_impl(guest_t *g) } pthread_mutex_unlock(&fd_lock); - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); for (int i = 0; i < g->nregions && n < (int) cap; i++) { const guest_region_t *r = &g->regions[i]; if (!r->shared || r->backing_fd < 0) @@ -534,7 +535,7 @@ static int64_t sc_sync_impl(guest_t *g) continue; hosts[n++] = duped; } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); /* fsync each dup outside both locks so a slow disk does not stall * concurrent FD or memory operations on other threads. @@ -708,7 +709,7 @@ static int64_t sc_mincore(guest_t *g, * never early-returns on a hole. */ uint8_t chunk[512]; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); int ri = guest_region_first_end_above(g, addr); for (uint64_t done = 0; done < npages;) { uint64_t batch = npages - done; @@ -723,13 +724,19 @@ static int64_t sc_mincore(guest_t *g, if (!mapped) has_hole = true; } - if (guest_write(g, vec + done, chunk, batch) < 0) { - pthread_mutex_unlock(&mmap_lock); + /* sc_mincore holds mmap_lock while regions[] is swept. Materialize a + * valid lazy output block through the locked entry point, then use a + * no-fault copy so an invalid vec returns EFAULT instead of trying to + * acquire mmap_lock recursively. + */ + (void) guest_lazy_faultin_locked(g, vec + done, batch); + if (guest_write_nofault(g, vec + done, chunk, batch) < 0) { + mmap_lock_release(); return -LINUX_EFAULT; } done += batch; } - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); return has_hole ? -LINUX_ENOMEM : 0; } @@ -942,6 +949,19 @@ static int64_t sc_set_tid_address(guest_t *g, return proc_get_pid(); } +static uint64_t mmap_fastpath_eligible_length(uint64_t addr, + uint64_t length, + uint64_t prot, + uint64_t flags) +{ + if (addr != 0 || prot != (LINUX_PROT_READ | LINUX_PROT_WRITE) || + (flags & ~(uint64_t) LINUX_MAP_NORESERVE) != + (LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS) || + length == 0 || length > UINT64_MAX - (GUEST_PAGE_SIZE - 1)) + return 0; + return (length + GUEST_PAGE_SIZE - 1) & ~(GUEST_PAGE_SIZE - 1); +} + static int64_t sc_mmap(guest_t *g, uint64_t x0, uint64_t x1, @@ -951,9 +971,23 @@ static int64_t sc_mmap(guest_t *g, uint64_t x5, bool verbose) { - pthread_mutex_lock(&mmap_lock); - int64_t r = sys_mmap(g, x0, x1, (int) x2, (int) x3, (int) x4, (int64_t) x5); - pthread_mutex_unlock(&mmap_lock); + uint64_t refill_len = mmap_fastpath_eligible_length(x0, x1, x2, x3); + uint64_t arena_addr = 0; + if (refill_len && mmap_fastpath_allocate_current_publication_only( + g, refill_len, &arena_addr)) + return (int64_t) arena_addr; + + mmap_lock_acquire(g); + int64_t r; + if (refill_len && + mmap_fastpath_allocate_current_locked(g, refill_len, &arena_addr)) { + r = (int64_t) arena_addr; + } else { + r = sys_mmap(g, x0, x1, (int) x2, (int) x3, (int) x4, (int64_t) x5); + if (r >= 0 && refill_len) + mmap_fastpath_refill_current_locked(g, refill_len); + } + mmap_lock_release(); log_debug(" mmap(0x%llx, 0x%llx) \xe2\x86\x92 0x%llx", (unsigned long long) x0, (unsigned long long) x1, (unsigned long long) (uint64_t) r); @@ -970,9 +1004,10 @@ static int64_t sc_mremap(guest_t *g, bool verbose) { (void) x5; - pthread_mutex_lock(&mmap_lock); + mmap_lock_acquire(g); + mmap_fastpath_revoke_all_locked(g, false); int64_t r = sys_mremap(g, x0, x1, x2, (int) x3, x4); - pthread_mutex_unlock(&mmap_lock); + mmap_lock_release(); log_debug(" mremap(0x%llx, 0x%llx, 0x%llx, 0x%x) \xe2\x86\x92 0x%llx", (unsigned long long) x0, (unsigned long long) x1, (unsigned long long) x2, (int) x3, @@ -2090,10 +2125,7 @@ static int64_t sc_execve(guest_t *g, (void) x3; (void) x4; (void) x5; - pthread_mutex_lock(&mmap_lock); - int64_t r = sys_execve(current_thread->vcpu, g, x0, x1, x2, verbose, NULL); - pthread_mutex_unlock(&mmap_lock); - return r; + return sys_execve(current_thread->vcpu, g, x0, x1, x2, verbose, NULL); } static int64_t sc_execveat(guest_t *g, @@ -2109,8 +2141,9 @@ static int64_t sc_execveat(guest_t *g, hv_vcpu_t vcpu = current_thread->vcpu; int dirfd = (int) x0, flags = (int) x4; - /* Resolve the target path before taking mmap_lock (path resolution may call - * fd_to_host / openat which do not need mmap_lock). + /* Resolve the target path before entering the exec transaction. Path + * resolution may call fd_to_host / openat and does not need mmap_lock; + * sys_execve takes it at the point of no return. */ uint64_t path_gva = x1; char resolved[LINUX_PATH_MAX]; @@ -2150,7 +2183,6 @@ static int64_t sc_execveat(guest_t *g, need_resolve = true; } - pthread_mutex_lock(&mmap_lock); int64_t r; if (need_resolve) { /* Use the host-resolved path directly so execveat does not copy a host @@ -2160,7 +2192,6 @@ static int64_t sc_execveat(guest_t *g, } else { r = sys_execve(vcpu, g, path_gva, x2, x3, verbose, NULL); } - pthread_mutex_unlock(&mmap_lock); return r; } diff --git a/src/syscall/sysvipc.c b/src/syscall/sysvipc.c index 25ad7280..fb659f9a 100644 --- a/src/syscall/sysvipc.c +++ b/src/syscall/sysvipc.c @@ -253,7 +253,12 @@ int64_t sys_shmat(guest_t *g, int shmid, uint64_t shmaddr_gva, int shmflg) return gva; /* propagate mmap error */ } - /* Copy host shm content into guest memory */ + /* Copy host shm content into guest memory. sys_shmat runs under + * mmap_lock (SC_LOCKED), so the resolve-time lazy fault-in inside + * guest_write would self-deadlock on it; materialize the fresh anonymous + * mapping through the locked variant first. + */ + guest_lazy_faultin_locked(g, (uint64_t) gva, seg_size); if (guest_write(g, (uint64_t) gva, host_addr, seg_size) < 0) { shmdt(host_addr); return -LINUX_EFAULT; @@ -312,7 +317,11 @@ int64_t sys_shmdt(guest_t *g, uint64_t shmaddr_gva) /* Write back guest modifications to host shm (unless read-only) */ if (!entry.rdonly) { - /* Read guest memory back to host shm buffer */ + /* Read guest memory back to host shm buffer. Same SC_LOCKED + * self-deadlock hazard as the shmat copy-in: pages the guest never + * touched may still be unmaterialized. + */ + guest_lazy_faultin_locked(g, entry.guest_gva, entry.size); guest_read(g, entry.guest_gva, entry.host_addr, entry.size); } diff --git a/tests/bench-mmap-fresh b/tests/bench-mmap-fresh new file mode 100755 index 00000000..554b0b34 --- /dev/null +++ b/tests/bench-mmap-fresh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Accumulate fresh bump-tail mmap timing across independent elfuse processes. +# One process cannot retain enough large fresh mappings to overcome CNTVCT's +# 41.7-ns tick without exhausting guest VA. Each invocation below gets a new +# guest, while its in-guest timer excludes process startup from the measurement. + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ELFUSE="${ELFUSE:-${ROOT_DIR}/build/elfuse}" +BENCH="${BENCH_MMAP_BIN:-${ROOT_DIR}/build/bench-mmap}" +# 1024 ticks = about 42.7 us on Apple Silicon. This bounds the aggregate +# counter quantization to roughly 0.1%, while keeping the largest cases timely. +TARGET_TICKS="${BENCH_MMAP_FRESH_TARGET_TICKS:-1024}" +MAX_RUNS="${BENCH_MMAP_FRESH_MAX_RUNS:-8192}" + +# size in bytes : mappings per fresh guest run +CASES=( + "4096:1000" + "65536:1000" + "1048576:1000" + "2097152:1000" + "8388608:256" + "134217728:16" + "1073741824:8" + "8589934592:4" + "34359738368:2" +) + +# Optional positional cases use the same size:count spelling as CASES, e.g. +# tests/bench-mmap-fresh 1073741824:8 8589934592:4 +if [ "$#" -gt 0 ]; then + CASES=("$@") +fi + +if [ ! -x "$ELFUSE" ] || [ ! -x "$BENCH" ]; then + echo "build build/elfuse and build/bench-mmap first" >&2 + exit 2 +fi + +printf 'fresh bump-tail mmap: target %s aggregate CNTVCT ticks\n' "$TARGET_TICKS" +printf '%-10s %8s %6s %14s\n' size count runs 'fresh mmap ns' + +for case in "${CASES[@]}"; do + size="${case%%:*}" + count="${case##*:}" + total_ticks=0 + total_ops=0 + runs=0 + ns_per_tick= + + while awk -v ticks="$total_ticks" -v target="$TARGET_TICKS" \ + 'BEGIN { exit !(ticks < target) }'; do + if [ "$runs" -ge "$MAX_RUNS" ]; then + echo "fresh benchmark exceeded ${MAX_RUNS} runs for ${size}" >&2 + exit 1 + fi + if ! output="$("$ELFUSE" "$BENCH" fresh "$size" "$count" 2>&1)"; then + echo "fresh guest run failed for size=${size}, count=${count}:" >&2 + echo "$output" >&2 + exit 1 + fi + raw_ticks="$(awk '{for (i = 1; i <= NF; i++) if ($i ~ /^ticks=/) {sub(/^ticks=/, "", $i); print $i}}' <<< "$output")" + read_ticks="$(awk '{for (i = 1; i <= NF; i++) if ($i ~ /^read_ticks=/) {sub(/^read_ticks=/, "", $i); print $i}}' <<< "$output")" + ns_per_tick="$(awk '{for (i = 1; i <= NF; i++) if ($i ~ /^ns_per_tick=/) {sub(/^ns_per_tick=/, "", $i); print $i}}' <<< "$output")" + if [ -z "$raw_ticks" ] || [ -z "$read_ticks" ] || [ -z "$ns_per_tick" ]; then + echo "unexpected benchmark output: $output" >&2 + exit 1 + fi + total_ticks="$(awk -v total="$total_ticks" -v raw="$raw_ticks" -v read="$read_ticks" 'BEGIN { print total + raw - read }')" + total_ops=$((total_ops + count)) + runs=$((runs + 1)) + done + + ns_per_op="$(awk -v ticks="$total_ticks" -v ops="$total_ops" -v ns="$ns_per_tick" 'BEGIN { printf "%.2f", ticks * ns / ops }')" + human="$(awk -v n="$size" 'BEGIN { if (n >= 1073741824) printf "%g GiB", n / 1073741824; else if (n >= 1048576) printf "%g MiB", n / 1048576; else printf "%g KiB", n / 1024 }')" + printf '%-10s %8s %6s %14s\n' "$human" "$count" "$runs" "$ns_per_op" +done diff --git a/tests/bench-mmap-lazy.c b/tests/bench-mmap-lazy.c new file mode 100644 index 00000000..377aae70 --- /dev/null +++ b/tests/bench-mmap-lazy.c @@ -0,0 +1,111 @@ +/* + * Guest microbenchmark for anonymous private mmap latency vs size. + * + * Measures mmap(), first-touch, and munmap() latency for MAP_PRIVATE | + * MAP_ANONYMOUS mappings from 4 KiB to 32 GiB. A lazy (deferred page-table) + * implementation should show size-independent mmap/munmap cost; an eager + * implementation scales linearly with length and exhausts resources on the + * multi-GiB sizes. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static uint64_t now_ns(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t) ts.tv_sec * 1000000000ull + (uint64_t) ts.tv_nsec; +} + +static void bench_size(uint64_t size, int iters) +{ + uint64_t t_map = 0, t_touch = 0, t_unmap = 0; + int ok = 0; + + for (int i = 0; i < iters; i++) { + uint64_t t0 = now_ns(); + void *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = now_ns(); + if (p == MAP_FAILED) { + printf("%10llu KiB: mmap failed: %s\n", + (unsigned long long) (size >> 10), strerror(errno)); + return; + } + /* First touch: one write at the start and one mid-mapping. */ + volatile char *c = p; + c[0] = 1; + c[size / 2] = 1; + uint64_t t2 = now_ns(); + int rc = munmap(p, size); + uint64_t t3 = now_ns(); + if (rc != 0) { + printf("%10llu KiB: munmap failed: %s\n", + (unsigned long long) (size >> 10), strerror(errno)); + return; + } + t_map += t1 - t0; + t_touch += t2 - t1; + t_unmap += t3 - t2; + ok++; + } + printf( + "%10llu KiB: mmap %10llu ns touch2 %10llu ns munmap %10llu ns " + "(%d iters)\n", + (unsigned long long) (size >> 10), + (unsigned long long) (t_map / (uint64_t) ok), + (unsigned long long) (t_touch / (uint64_t) ok), + (unsigned long long) (t_unmap / (uint64_t) ok), ok); +} + +int main(int argc, char **argv) +{ + static const struct { + uint64_t size; + int iters; + } cases[] = { + {4ull << 10, 200}, {64ull << 10, 200}, {2ull << 20, 100}, + {64ull << 20, 20}, {512ull << 20, 10}, {2ull << 30, 5}, + {8ull << 30, 3}, {16ull << 30, 3}, {32ull << 30, 3}, + {64ull << 30, 1}, {128ull << 30, 1}, {256ull << 30, 1}, + }; + /* Optional argv[1]: cap size in GiB (eager implementations commit host + * memory for every byte mapped; the full matrix would thrash small hosts). + */ + uint64_t cap = ~0ull; + if (argc > 1) + cap = (uint64_t) atoll(argv[1]) << 30; + + setvbuf(stdout, NULL, _IONBF, 0); + printf("anon private mmap latency vs size\n"); + for (unsigned i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) + if (cases[i].size <= cap) + bench_size(cases[i].size, cases[i].iters); + + /* Full-touch throughput sanity: 64 MiB written end to end. */ + uint64_t size = 64ull << 20; + uint64_t t0 = now_ns(); + void *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf("full-touch mmap failed: %s\n", strerror(errno)); + return 1; + } + memset(p, 0xa5, size); + uint64_t t1 = now_ns(); + munmap(p, size); + printf("mmap+memset 64MiB: %llu ns (%.2f GiB/s)\n", + (unsigned long long) (t1 - t0), + (double) size / 1.073741824 / (double) (t1 - t0)); + return 0; +} diff --git a/tests/bench-mmap.c b/tests/bench-mmap.c new file mode 100644 index 00000000..a7da49f5 --- /dev/null +++ b/tests/bench-mmap.c @@ -0,0 +1,809 @@ +/* + * Comprehensive anonymous-mmap microbenchmark for elfuse. + * + * Measures the guest-visible cost of the mmap subsystem in isolation: + * allocation, teardown, first-touch faults, permission splitting, and remap. It + * is self-contained -- no external harness -- and is meant to be run under + * elfuse (./build/elfuse ./build/bench-mmap) but also runs on any aarch64-linux + * host for a ground-truth comparison. + * + * Timing: reads CNTVCT_EL0 directly at EL0 (enabled by CNTKCTL_EL1.EL0VCTEN in + * bootstrap.c), so a measurement costs an isb + mrs, not a clock_gettime SVC. + * On Apple Silicon CNTFRQ is ~24 MHz (~41.7 ns/tick); amortizing over an + * adaptive batch drives the effective resolution well below one tick. This is + * the key fairness property: clock_gettime on a static guest falls through to + * the ~2 us SVC path and swamps any sub-us operation. + * + * Every case takes one untimed warmup pass (to pay the one-time arena carve and + * page-table extension). Most sections report aggregate samples; section H + * retains every operation so its normal latency and long tail remain visible. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* CNTVCT timing */ + +static double ns_per_tick; + +static inline uint64_t rd(void) +{ + uint64_t v; + __asm__ volatile("isb\n\tmrs %0, cntvct_el0" : "=r"(v)); + return v; +} + +static void clock_init(void) +{ + uint64_t f; + __asm__ volatile("mrs %0, cntfrq_el0" : "=r"(f)); + if (f == 0) + f = 24000000; /* defensive: assume 24 MHz if RES0 */ + ns_per_tick = 1e9 / (double) f; +} + +static double ns(uint64_t ticks) +{ + return (double) ticks * ns_per_tick; +} + +static int cmp_d(const void *a, const void *b) +{ + double x = *(const double *) a, y = *(const double *) b; + return (x > y) - (x < y); +} + +static double median(double *v, int n) +{ + qsort(v, n, sizeof(*v), cmp_d); + return (n & 1) ? v[n / 2] : 0.5 * (v[n / 2 - 1] + v[n / 2]); +} + +/* R-7/sample quantile, matching the common (n - 1) * p interpolation. The + * input must already be sorted. */ +static double sorted_quantile(const double *v, unsigned n, double p) +{ + double pos = (double) (n - 1) * p; + unsigned lo = (unsigned) pos; + unsigned hi = lo + (lo + 1 < n); + return v[lo] + (v[hi] - v[lo]) * (pos - lo); +} + +#define ITERS 15 +#define MIN_TIMED_TICKS 4096 +#define KIB (1ULL << 10) +#define MIB (1ULL << 20) +#define GIB (1ULL << 30) +#define DIRTY_VMEXIT_STRIDE (8 * MIB) + +/* The counter is too coarse to time a single mmap reliably. A section-A + * sample therefore repeats independent mmap -> munmap pairs until the + * aggregate interval is long enough. This avoids batching altogether: each + * allocation is immediately returned to the steady-state free list. + * + * Per-operation timestamps introduce one rd()/rd() interval per result. Its + * median cost is calibrated over long runs and subtracted, so a one-tick + * counter cannot turn a 1-GiB mmap into a spurious 83.3-ns (two-tick) result. + */ +static double rd_pair_ticks(void) +{ + enum { CAL_SAMPLES = 15, CAL_OPS = 4096 }; + double samples[CAL_SAMPLES]; + for (int sample = 0; sample < CAL_SAMPLES; sample++) { + uint64_t total = 0; + for (int op = 0; op < CAL_OPS; op++) { + uint64_t t0 = rd(); + uint64_t t1 = rd(); + total += t1 - t0; + } + samples[sample] = (double) total / CAL_OPS; + } + return median(samples, CAL_SAMPLES); +} + +static const char *human(uint64_t s, char *buf) +{ + if (s >= GIB) + sprintf(buf, "%llu GiB", (unsigned long long) (s / GIB)); + else if (s >= MIB) + sprintf(buf, "%llu MiB", (unsigned long long) (s / MIB)); + else + sprintf(buf, "%llu KiB", (unsigned long long) (s / KIB)); + return buf; +} + +/* A. mmap + munmap latency vs size (steady state). Each operation allocates + * with a NULL hint and immediately frees the result. This measures the + * repeated-allocation path a workload sees, not the one-shot fresh case + * (section B), while avoiding a size-dependent batch policy. + */ +static void bench_size_sweep(void) +{ + static const uint64_t sizes[] = { + 4 * KIB, 16 * KIB, 64 * KIB, 256 * KIB, MIB, 2 * MIB, 8 * MIB, + 64 * MIB, 256 * MIB, GIB, 4 * GIB, 16 * GIB, 32 * GIB, + }; + printf( + "== A. mmap / munmap latency vs size (steady state, NULL hint) ==\n"); + double timer_ticks = rd_pair_ticks(); + printf("%-10s %8s %12s %12s\n", "size", "ops", "mmap ns", "munmap ns"); + for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { + uint64_t size = sizes[s]; + double mm[ITERS], um[ITERS]; + unsigned reported_ops = 0; + int ok = 1; + for (int it = -1; it < ITERS && ok; it++) { + uint64_t mmap_ticks = 0, munmap_ticks = 0; + unsigned ops = 0; + do { + uint64_t t0 = rd(); + void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = rd(); + mmap_ticks += t1 - t0; + if (ptr == MAP_FAILED) { + ok = 0; + break; + } + t0 = rd(); + int rc = munmap(ptr, size); + t1 = rd(); + munmap_ticks += t1 - t0; + if (rc != 0) { + ok = 0; + break; + } + ops++; + } while (ok && + (mmap_ticks < MIN_TIMED_TICKS || munmap_ticks < MIN_TIMED_TICKS)); + if (!ok) + break; + if (it >= 0) { + mm[it] = ((double) mmap_ticks / ops - timer_ticks) * ns_per_tick; + um[it] = + ((double) munmap_ticks / ops - timer_ticks) * ns_per_tick; + reported_ops = ops; + } + } + char hb[16]; + if (!ok) { + printf("%-10s %8s %12s %12s\n", human(size, hb), "-", "FAILED", + "-"); + continue; + } + printf("%-10s %8u %12.1f %12.1f\n", human(size, hb), reported_ops, + median(mm, ITERS), median(um, ITERS)); + } + printf("\n"); +} + +/* B. fresh bump-tail mmap (isolates the lazy_fresh_range path). Allocate a + * sequential run WITHOUT freeing, so every mapping lands at or above the arena + * high-water -- exactly the case lazy_fresh_range skips the stale-PTE scan for. + * Small mappings use the original 2-GiB footprint cap; large mappings use a + * minimum count chosen to retain multiple samples without exceeding 64 GiB of + * live fresh VA. + * Run this binary against an opt-off build to read the skip's contribution as + * the difference on this identical code path -- a MAP_FIXED "recycled" compare + * would instead measure the region-snapshot replacement path, not the skip. + */ +static void bench_fresh(void) +{ + static const uint64_t sizes[] = {4 * KIB, 64 * KIB, MIB, + 2 * MIB, 8 * MIB, 128 * MIB, + GIB, 8 * GIB, 32 * GIB}; + printf( + "== B. fresh bump-tail mmap, per-mmap ns (lazy_fresh_range path) ==\n"); + printf("%-10s %8s %14s\n", "size", "count", "fresh mmap ns"); + void *run[1000]; + for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { + uint64_t size = sizes[s]; + int n = (int) (2 * GIB / size); + if (n < 1) + n = 1; + if (n > 1000) + n = 1000; + if (size == GIB) + n = 8; + else if (size == 8 * GIB) + n = 4; + else if (size == 32 * GIB) + n = 2; + /* warmup one fresh mapping so the arena high-water is already primed */ + void *w = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (w != MAP_FAILED) + munmap(w, size); + uint64_t t0 = rd(); + for (int i = 0; i < n; i++) + run[i] = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = rd(); + int failed = 0; + for (int i = 0; i < n; i++) { + if (run[i] == MAP_FAILED) + failed++; + else + munmap(run[i], size); + } + char hb[16]; + if (failed) { + printf("%-10s %8d %14s (%d failed)\n", human(size, hb), n, + "PARTIAL", failed); + continue; + } + printf("%-10s %8d %14.1f\n", human(size, hb), n, ns(t1 - t0) / n); + } + printf("\n"); +} + +/* One fresh bump-tail run for the host-side driver. A new elfuse process is + * used for each invocation, so the driver can accumulate many counter ticks + * without exhausting one guest's VA space. */ +static int bench_fresh_one(uint64_t size, int n) +{ + void *run[1000]; + if (n < 1 || n > (int) (sizeof(run) / sizeof(run[0]))) + return 2; + + void *warmup = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (warmup == MAP_FAILED) + return 1; + munmap(warmup, size); + + uint64_t t0 = rd(); + for (int i = 0; i < n; i++) + run[i] = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = rd(); + + int failed = 0; + for (int i = 0; i < n; i++) { + if (run[i] == MAP_FAILED) + failed++; + else + munmap(run[i], size); + } + if (failed) + return 1; + + printf("fresh size=%llu count=%d ticks=%llu read_ticks=%.6f " + "ns_per_tick=%.12f\n", + (unsigned long long) size, n, (unsigned long long) (t1 - t0), + rd_pair_ticks(), ns_per_tick); + return 0; +} + +/* C. first-touch cost. Touch one byte per macOS 16 KiB host page. Each access + * demands a distinct physical backing page, but it is not necessarily a + * distinct HVC: elfuse installs Stage-1 descriptors in 2 MiB windows and the + * fault-around policy may install several windows per exit. Report both the + * whole sweep and its per-host-page amortization; calling the latter a + * "per-fault" cost would substantially overcount guest translation faults. + */ +static void bench_fault(int pages, int drain_between) +{ + const uint64_t stride = 16 * KIB; + uint64_t size = stride * (uint64_t) (pages + 1); + printf("== C. first-touch fault cost (16 KiB stride, %d pages%s) ==\n", + pages, drain_between ? ", forced retire drain" : ""); + double sweep[ITERS]; + for (int it = -1; it < ITERS; it++) { + volatile uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED: %s\n\n", strerror(errno)); + return; + } + uint64_t t0 = rd(); + for (int i = 0; i < pages; i++) + p[(uint64_t) i * stride] = 1; + uint64_t t1 = rd(); + munmap((void *) p, size); + if (drain_between) + (void) fcntl(-1, F_GETFD); + if (it >= 0) + sweep[it] = ns(t1 - t0); + } + qsort(sweep, ITERS, sizeof(*sweep), cmp_d); + printf(" sweep: p50 %.1f us p95 %.1f us max %.1f us\n", + sorted_quantile(sweep, ITERS, 0.50) / 1000.0, + sorted_quantile(sweep, ITERS, 0.95) / 1000.0, + sweep[ITERS - 1] / 1000.0); + printf(" amortized/touch: p50 %.1f ns p95 %.1f ns max %.1f ns\n\n", + sorted_quantile(sweep, ITERS, 0.50) / pages, + sorted_quantile(sweep, ITERS, 0.95) / pages, + sweep[ITERS - 1] / pages); +} + +/* D. mprotect split cost. Touch the mapping before timing so lazy mmap has an + * actual L2 block descriptor; otherwise the EL1 lazy-metadata specialization + * can correctly defer PTE creation and this would measure a 1-2 tick metadata + * publication rather than a block-to-table split. Flip the middle 4 KiB to + * PROT_READ, forcing conversion into 512 L3 pages. Keep every mapping live + * until the samples finish so allocator reuse cannot hand a later iteration + * an already-existing (but empty) L3 table. + */ +static void bench_mprotect_split(void) +{ + printf( + "== D. mprotect split (2 MiB block -> L3, protect middle 4 KiB) ==\n"); + double sp[ITERS]; + uint8_t *held[ITERS + 1]; + int nheld = 0; + for (int it = -1; it < ITERS; it++) { + uint8_t *p = mmap(NULL, 2 * MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + for (int i = 0; i < nheld; i++) + munmap(held[i], 2 * MIB); + printf(" mmap FAILED\n\n"); + return; + } + held[nheld++] = p; + p[0] = 1; /* materialize a real 2 MiB L2 block before timing */ + uint8_t *mid = p + MIB; + uint64_t t0 = rd(); + int rc = mprotect(mid, 4 * KIB, PROT_READ); + uint64_t t1 = rd(); + if (rc != 0) { + for (int i = 0; i < nheld; i++) + munmap(held[i], 2 * MIB); + printf(" mprotect FAILED: %s\n\n", strerror(errno)); + return; + } + if (it >= 0) + sp[it] = ns(t1 - t0); + } + for (int i = 0; i < nheld; i++) + munmap(held[i], 2 * MIB); + printf(" split: median %.1f ns min %.1f ns\n", median(sp, ITERS), + sp[0]); +} + +/* Once the block has an L3 table, R<->RW on one anonymous arena page can stay + * in EL1 and defer only region metadata through the publication ring. + */ +static void bench_mprotect_fast_leaf(void) +{ + double leaf[ITERS]; + uint8_t *p = mmap(NULL, 2 * MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" fast leaf mmap FAILED\n\n"); + return; + } + p[0] = 1; /* ensure setup below creates an L3 table, not lazy metadata */ + if (mprotect(p + MIB, 4 * KIB, PROT_READ) != 0) { + munmap(p, 2 * MIB); + printf(" fast leaf setup FAILED: %s\n\n", strerror(errno)); + return; + } + for (int it = -1; it < ITERS; it++) { + int prot = (it & 1) ? PROT_READ : PROT_READ | PROT_WRITE; + uint64_t t0 = rd(); + int rc = mprotect(p + MIB, 4 * KIB, prot); + uint64_t t1 = rd(); + if (rc != 0) { + munmap(p, 2 * MIB); + printf(" fast leaf mprotect FAILED: %s\n\n", strerror(errno)); + return; + } + if (it >= 0) + leaf[it] = ns(t1 - t0); + } + munmap(p, 2 * MIB); + printf(" fast leaf: median %.1f ns min %.1f ns\n\n", median(leaf, ITERS), + leaf[0]); +} + +/* E. mremap grow: in-place vs forced move */ +static void bench_mremap(void) +{ + printf("== E. mremap grow 4 KiB -> 8 KiB ==\n"); + double inp[ITERS], mov[ITERS]; + + /* In-place: no blocker, the following page is free. */ + for (int it = -1; it < ITERS; it++) { + void *p = mmap(NULL, 4 * KIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED\n\n"); + return; + } + uint64_t t0 = rd(); + void *q = mremap(p, 4 * KIB, 8 * KIB, MREMAP_MAYMOVE); + uint64_t t1 = rd(); + if (q == MAP_FAILED) { + munmap(p, 4 * KIB); + printf(" mremap in-place FAILED\n\n"); + return; + } + munmap(q, 8 * KIB); + if (it >= 0) + inp[it] = ns(t1 - t0); + } + + /* Forced move: a PROT_READ blocker sits immediately after, so the grow must + * relocate. + */ + for (int it = -1; it < ITERS; it++) { + uint8_t *p = mmap(NULL, 8 * KIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + printf(" mmap FAILED\n\n"); + return; + } + /* free the tail page and pin it read-only so in-place growth is blocked + * but the head is still a 4 KiB mapping. + */ + munmap(p + 4 * KIB, 4 * KIB); + void *blk = mmap(p + 4 * KIB, 4 * KIB, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + uint64_t t0 = rd(); + void *q = mremap(p, 4 * KIB, 8 * KIB, MREMAP_MAYMOVE); + uint64_t t1 = rd(); + if (q == MAP_FAILED) { + printf(" mremap move FAILED\n\n"); + return; + } + munmap(q, 8 * KIB); + if (blk != MAP_FAILED) + munmap(blk, 4 * KIB); + if (it >= 0) + mov[it] = ns(t1 - t0); + } + printf(" in-place: median %.1f ns min %.1f ns\n", median(inp, ITERS), + inp[0]); + printf(" move: median %.1f ns min %.1f ns\n\n", median(mov, ITERS), + mov[0]); +} + +/* F. multi-threaded fresh mmap under mmap_lock Several threads hammer fresh + * bump-tail mmaps concurrently. mmap serializes on mmap_lock, so this exposes + * both lock contention and any per-mmap TLBI shootdown cost -- the one place a + * "skip the invalidate on fresh ranges" optimization could pay off that a + * single-threaded run cannot see. Threads do not free during the timed run + * (every mapping stays fresh); total live regions are capped under + * GUEST_MAX_REGIONS. Compare against an opt-off build to read the skip's + * multi-threaded contribution. + */ +typedef struct { + uint64_t size; + int n; + void **buf; + double per_op_ns; + int failed; +} mt_arg_t; + +static pthread_barrier_t mt_barrier; + +/* CNTVCT_EL0 reads a constant on worker vCPUs (EL0VCTEN is set for the main + * vCPU only), so the MT worker brackets its whole loop with clock_gettime and + * amortizes the one SVC pair over n mmaps. + */ +static uint64_t mono_ns(void) +{ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t) ts.tv_sec * 1000000000ull + (uint64_t) ts.tv_nsec; +} + +static void *mt_worker(void *p) +{ + mt_arg_t *a = p; + pthread_barrier_wait(&mt_barrier); + uint64_t t0 = mono_ns(); + for (int i = 0; i < a->n; i++) + a->buf[i] = mmap(NULL, a->size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + uint64_t t1 = mono_ns(); + a->per_op_ns = (double) (t1 - t0) / a->n; + for (int i = 0; i < a->n; i++) { + if (a->buf[i] == MAP_FAILED) + a->failed++; + else + munmap(a->buf[i], a->size); + } + return NULL; +} + +#define MT_MAX_THREADS 4 +#define MT_REGION_CAP 3000 /* keep T*n well under GUEST_MAX_REGIONS (4096) */ + +static void bench_mt(void) +{ + static const uint64_t sizes[] = {4 * KIB, 2 * MIB}; + static const int threads[] = {2, 4}; + printf( + "== F. multi-threaded fresh mmap, per-op ns (mmap_lock contention) " + "==\n"); + printf("%-10s %8s %12s %12s\n", "size", "threads", "mean ns", "max ns"); + for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { + uint64_t size = sizes[s]; + for (unsigned t = 0; t < sizeof(threads) / sizeof(threads[0]); t++) { + int T = threads[t]; + int n = MT_REGION_CAP / T; + if (n < 1) + n = 1; + mt_arg_t arg[MT_MAX_THREADS]; + pthread_t th[MT_MAX_THREADS]; + int ok = 1; + for (int i = 0; i < T; i++) { + arg[i].size = size; + arg[i].n = n; + arg[i].per_op_ns = 0; + arg[i].failed = 0; + arg[i].buf = calloc(n, sizeof(void *)); + if (!arg[i].buf) + ok = 0; + } + /* prime the arena high-water so the timed run is genuinely fresh */ + void *w = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (w != MAP_FAILED) + munmap(w, size); + pthread_barrier_init(&mt_barrier, NULL, (unsigned) T); + for (int i = 0; i < T && ok; i++) + if (pthread_create(&th[i], NULL, mt_worker, &arg[i]) != 0) + ok = 0; + double sum = 0, mx = 0; + int failed = 0; + for (int i = 0; i < T; i++) { + pthread_join(th[i], NULL); + sum += arg[i].per_op_ns; + if (arg[i].per_op_ns > mx) + mx = arg[i].per_op_ns; + failed += arg[i].failed; + free(arg[i].buf); + } + pthread_barrier_destroy(&mt_barrier); + char hb[16]; + if (!ok || failed) + printf("%-10s %8d %12s\n", human(size, hb), T, "FAILED"); + else + printf("%-10s %8d %12.1f %12.1f\n", human(size, hb), T, sum / T, + mx); + } + } + printf("\n"); +} + +/* G. Teardown after materialization. The store is deliberately outside the + * timed interval: it takes the lazy first-touch fault and installs the first + * page, then the counter brackets only munmap(). The size sweep matches A, + * so each row compares a wholly untouched mapping with one that has exactly + * one materialized 4-KiB page; touching every page would instead benchmark + * faulting and zeroing gigabytes of memory. */ +static void bench_munmap_materialized(void) +{ + double timer_ticks = rd_pair_ticks(); + static const uint64_t sizes[] = { + 4 * KIB, 16 * KIB, 64 * KIB, 256 * KIB, MIB, 2 * MIB, 8 * MIB, + 64 * MIB, 256 * MIB, GIB, 4 * GIB, 16 * GIB, 32 * GIB, + }; + + printf("== G. munmap after materializing one 4 KiB page ==\n"); + printf("%-10s %8s %12s\n", "size", "ops", "munmap ns"); + for (unsigned s = 0; s < sizeof(sizes) / sizeof(sizes[0]); s++) { + uint64_t size = sizes[s]; + double unmap_ns[ITERS]; + unsigned reported_ops = 0; + int ok = 1; + for (int it = -1; it < ITERS && ok; it++) { + uint64_t unmap_ticks = 0; + unsigned ops = 0; + do { + volatile uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + ok = 0; + break; + } + p[0] = 1; /* materialize before starting the timed interval */ + uint64_t t0 = rd(); + int rc = munmap((void *) p, size); + uint64_t t1 = rd(); + unmap_ticks += t1 - t0; + if (rc != 0) { + ok = 0; + break; + } + ops++; + } while (unmap_ticks < MIN_TIMED_TICKS); + + if (it >= 0 && ok) { + unmap_ns[it] = + ((double) unmap_ticks / ops - timer_ticks) * ns_per_tick; + reported_ops = ops; + } + } + + char hb[16]; + if (!ok) + printf("%-10s %8s %12s\n", human(size, hb), "-", "FAILED"); + else + printf("%-10s %8u %12.1f\n", human(size, hb), reported_ops, + median(unmap_ns, ITERS)); + } + printf("\n"); +} + +/* H. Teardown after every page was dirtied. Page stores happen before the + * timed interval, so this reports only munmap's handling of the materialized, + * dirty mapping. Each size has a fixed operation count and every munmap is + * retained separately. In particular, one slow first operation cannot end an + * adaptive batch and become the whole sample. Counts decrease with size to + * bound total dirtying work. The 1-GiB case uses eight operations, and the + * untimed store loop takes periodic VM exits so every vCPU-run interval stays + * below elfuse's watchdog; interpolated p95 remains distinct from max. + */ +static void bench_munmap_dirty(void) +{ + static const struct { + uint64_t size; + unsigned ops; + } cases[] = { + {4 * KIB, 2048}, {16 * KIB, 2048}, {64 * KIB, 2048}, + {256 * KIB, 1024}, {MIB, 512}, {2 * MIB, 256}, + {8 * MIB, 128}, {64 * MIB, 32}, {256 * MIB, 24}, + {GIB, 8}, + }; + double timer_ticks = rd_pair_ticks(); + + printf("== H. munmap after dirtying every 4 KiB page ==\n"); + printf("%-10s %8s %12s %12s %12s\n", "size", "ops", "p50 ns", + "p95 ns", "max ns"); + for (unsigned s = 0; s < sizeof(cases) / sizeof(cases[0]); s++) { + uint64_t size = cases[s].size; + unsigned ops = cases[s].ops; + double *unmap_ns = malloc((size_t) ops * sizeof(*unmap_ns)); + int ok = 1; + if (!unmap_ns) + ok = 0; + + /* One full untimed warmup pays setup without consuming an observation. */ + for (int64_t op = -1; op < (int64_t) ops && ok; op++) { + volatile uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + ok = 0; + break; + } + for (uint64_t off = 0; off < size; off += 4 * KIB) { + p[off] = (uint8_t) (off >> 12); + /* Large already-backed runs can execute in EL0 long enough + * for the benchmark's 10-second vCPU watchdog to fire under + * macOS memory pressure. This deliberately failing fcntl is + * a guaranteed, side-effect-free HVC outside the timed region. + * It also drains the preceding retirement in bounded chunks. + */ + if (off != 0 && (off & (DIRTY_VMEXIT_STRIDE - 1)) == 0) + (void) fcntl(-1, F_GETFD); + } + + /* Establish an identical clean host-retirement boundary before + * every measurement. The interval below then contains EL1's + * descriptor/TLBI path, never the previous operation's cleanup. + */ + (void) fcntl(-1, F_GETFD); + + uint64_t t0 = rd(); + int rc = munmap((void *) p, size); + uint64_t t1 = rd(); + if (rc != 0) { + ok = 0; + break; + } + if (op >= 0) { + double ticks = (double) (t1 - t0) - timer_ticks; + if (ticks < 0.0) + ticks = 0.0; + unmap_ns[op] = ticks * ns_per_tick; + } + } + + char hb[16]; + if (!ok) { + printf("%-10s %8u %12s %12s %12s\n", human(size, hb), ops, + "FAILED", "-", "-"); + } else { + qsort(unmap_ns, ops, sizeof(*unmap_ns), cmp_d); + printf("%-10s %8u %12.1f %12.1f %12.1f\n", human(size, hb), ops, + sorted_quantile(unmap_ns, ops, 0.50), + sorted_quantile(unmap_ns, ops, 0.95), unmap_ns[ops - 1]); + } + free(unmap_ns); + } + printf("\n"); +} + +int main(int argc, char **argv) +{ + clock_init(); + if (argc == 2 && strcmp(argv[1], "d") == 0) { + bench_mprotect_split(); + bench_mprotect_fast_leaf(); + return 0; + } + if (argc == 2 && strcmp(argv[1], "a") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", + ns_per_tick); + bench_size_sweep(); + return 0; + } + if (argc == 2 && strcmp(argv[1], "h") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", + ns_per_tick); + bench_munmap_dirty(); + return 0; + } + if (argc == 2 && strcmp(argv[1], "f") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", + ns_per_tick); + bench_mt(); + return 0; + } + if (argc == 2 && strcmp(argv[1], "c") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", + ns_per_tick); + bench_fault(512, 0); + return 0; + } + if (argc == 3 && strcmp(argv[1], "c") == 0) { + char *end = NULL; + errno = 0; + long pages = strtol(argv[2], &end, 0); + if (errno || !end || *end || pages < 1 || pages > 1048576) + return 2; + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", + ns_per_tick); + bench_fault((int) pages, 0); + return 0; + } + if (argc == 2 && strcmp(argv[1], "c-drain") == 0) { + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", + ns_per_tick); + bench_fault(512, 1); + return 0; + } + if (argc == 4 && strcmp(argv[1], "fresh") == 0) { + char *end = NULL; + errno = 0; + uint64_t size = strtoull(argv[2], &end, 0); + if (errno || !end || *end || size == 0) + return 2; + errno = 0; + long count = strtol(argv[3], &end, 0); + if (errno || !end || *end || count < 1 || count > 1000) + return 2; + return bench_fresh_one(size, (int) count); + } + if (argc != 1) + return 2; + printf("elfuse mmap benchmark (CNTVCT %.2f ns/tick)\n\n", ns_per_tick); + bench_size_sweep(); + bench_fresh(); + bench_fault(512, 0); + bench_mprotect_split(); + bench_mprotect_fast_leaf(); + bench_mremap(); + bench_mt(); + bench_munmap_materialized(); + bench_munmap_dirty(); + return 0; +} diff --git a/tests/manifest.txt b/tests/manifest.txt index 871557d3..4c0de5f5 100644 --- a/tests/manifest.txt +++ b/tests/manifest.txt @@ -105,6 +105,10 @@ test-fork-synthetic-fd test-guard-page test-mmap-hint +[section] Lazy anonymous mmap tests +test-mmap-lazy +test-mmap-fastpath + [section] mremap tests test-mremap test-mremap-infra diff --git a/tests/test-fork-ipc-protocol-host.c b/tests/test-fork-ipc-protocol-host.c index 115ce16a..b17f45c5 100644 --- a/tests/test-fork-ipc-protocol-host.c +++ b/tests/test-fork-ipc-protocol-host.c @@ -17,14 +17,17 @@ #include "runtime/fork-state.h" #define LEGACY_ELFK_MAGIC 0x454C464BU +#define LEGACY_ELFL_MAGIC 0x454C464CU -_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C464CU, - "fork IPC protocol magic must remain ELFL until the next " +_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C464DU, + "fork IPC protocol magic must remain ELFM until the next " "incompatible wire-format change"); _Static_assert(IPC_MAGIC_HEADER == FORK_IPC_PROTOCOL_MAGIC, "header magic must be the protocol identity"); _Static_assert(FORK_IPC_PROTOCOL_MAGIC != LEGACY_ELFK_MAGIC, "current protocol must reject old ELFK children/parents"); +_Static_assert(FORK_IPC_PROTOCOL_MAGIC != LEGACY_ELFL_MAGIC, + "dirty-bitmap wire must reject old ELFL children/parents"); _Static_assert(IPC_MAGIC_SENTINEL != FORK_IPC_PROTOCOL_MAGIC, "process-state sentinel must not alias the header protocol"); diff --git a/tests/test-mmap-dirty-stats.sh b/tests/test-mmap-dirty-stats.sh new file mode 100755 index 00000000..9d8e2715 --- /dev/null +++ b/tests/test-mmap-dirty-stats.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# Counter-backed dirty-map materialization integration checks. + +set -eu + +ELFUSE=${1:-build/elfuse} +TEST_BIN=${2:-build/test-mmap-lazy} +TMPDIR_CASE=$(mktemp -d "${TMPDIR:-/tmp}/elfuse-dirty-map.XXXXXX") +trap 'rm -rf "$TMPDIR_CASE"' EXIT INT TERM + +ELFUSE_SHIM_STATS=1 "$ELFUSE" "$TEST_BIN" \ + > "$TMPDIR_CASE/out" 2> "$TMPDIR_CASE/err" + +counter() +{ + key=$1 + awk -v key="$key" \ + '$1 == key { print $2; found = 1 } END { if (!found) exit 1 }' \ + "$TMPDIR_CASE/err" +} + +require_ge() +{ + key=$1 + floor=$2 + value=$(counter "$key") || { + printf 'missing dirty-map counter %s\n' "$key" >&2 + return 1 + } + if [ "$value" -lt "$floor" ]; then + printf '%s=%s, expected >= %s\n' "$key" "$value" "$floor" >&2 + return 1 + fi +} + +require_ge FAULT_CLEAN_SKIP 1 +require_ge FAULT_DIRTY_MEMSET 1 +require_ge FAULT_ALREADY_VALID 1 +require_ge FAULT_WINDOW_BYTES 2097152 + +printf ' clean-block zero skip OK\n' +printf ' dirty-block selective memset OK\n' +printf ' already-valid early return OK\n' +printf ' materialized-window bytes OK\n' +printf 'test-mmap-dirty-stats: PASS\n' diff --git a/tests/test-mmap-fastpath-stats.sh b/tests/test-mmap-fastpath-stats.sh new file mode 100755 index 00000000..e557d720 --- /dev/null +++ b/tests/test-mmap-fastpath-stats.sh @@ -0,0 +1,129 @@ +#!/bin/sh +# Counter-backed refill, adaptive sizing, giant-request guard, and VA recycle +# integration checks for the EL1 anonymous-mmap consumer fast path. + +set -eu + +ELFUSE=${1:-build/elfuse} +TEST_BIN=${2:-build/test-mmap-fastpath} +TMPDIR_CASE=$(mktemp -d "${TMPDIR:-/tmp}/elfuse-mmap-stats.XXXXXX") +trap 'rm -rf "$TMPDIR_CASE"' EXIT INT TERM + +run_case() +{ + case_name=$1 + out="$TMPDIR_CASE/$case_name.out" + err="$TMPDIR_CASE/$case_name.err" + ELFUSE_SHIM_STATS=1 "$ELFUSE" "$TEST_BIN" "--stats-$case_name" \ + > "$out" 2> "$err" +} + +counter() +{ + case_name=$1 + key=$2 + value=$(awk -v key="$key" '$1 == key { print $2; found = 1 } END { if (!found) exit 1 }' \ + "$TMPDIR_CASE/$case_name.err") || { + printf 'missing counter %s in case %s\n' "$key" "$case_name" >&2 + return 1 + } + printf '%s\n' "$value" +} + +require_ge() +{ + case_name=$1 + key=$2 + floor=$3 + value=$(counter "$case_name" "$key") + if [ "$value" -lt "$floor" ]; then + printf '%s: %s=%s, expected >= %s\n' \ + "$case_name" "$key" "$value" "$floor" >&2 + return 1 + fi +} + +require_eq() +{ + case_name=$1 + key=$2 + expected=$3 + value=$(counter "$case_name" "$key") + if [ "$value" -ne "$expected" ]; then + printf '%s: %s=%s, expected %s\n' \ + "$case_name" "$key" "$value" "$expected" >&2 + return 1 + fi +} + +require_le() +{ + case_name=$1 + key=$2 + ceiling=$3 + value=$(counter "$case_name" "$key") + if [ "$value" -gt "$ceiling" ]; then + printf '%s: %s=%s, expected <= %s\n' \ + "$case_name" "$key" "$value" "$ceiling" >&2 + return 1 + fi +} + +run_case ring-full +require_ge ring-full MMAP_HIT 32 +require_ge ring-full MMAP_RING_FULL 1 +printf ' 32-entry ring fallback OK\n' + +run_case np2-10m +require_ge np2-10m MMAP_HIT 80 +require_ge np2-10m MMAP_CAPACITY_MISS 1 +printf ' sustained 10 MiB stream OK\n' + +run_case np2-48m +require_ge np2-48m MMAP_HIT 40 +require_ge np2-48m MMAP_CAPACITY_MISS 1 +printf ' sustained 48 MiB stream OK\n' + +run_case np2-100m +require_ge np2-100m MMAP_HIT 24 +require_ge np2-100m MMAP_CAPACITY_MISS 1 +printf ' sustained 100 MiB stream OK\n' + +run_case escalation +require_ge escalation MMAP_HIT 45 +require_eq escalation MMAP_ARENA_CURRENT 17179869184 +printf ' 10 MiB -> 512 MiB escalation OK\n' + +run_case giant-guard +require_ge giant-guard MMAP_HIT 34 +require_eq giant-guard MMAP_ARENA_PEAK 34359738368 +printf ' 2 GiB request uses fast path OK\n' + +run_case adaptive-small +require_eq adaptive-small MMAP_ARENA_CURRENT 67108864 +require_eq adaptive-small MMAP_ARENA_PEAK 67108864 +printf ' small-stream arena floor OK\n' + +run_case adaptive-retention +require_eq adaptive-retention MMAP_ARENA_CURRENT 17179869184 +require_eq adaptive-retention MMAP_ARENA_PEAK 17179869184 +printf ' large arena retained OK\n' + +run_case adaptive-rewind-growth +require_ge adaptive-rewind-growth MMAP_CAPACITY_MISS 1 +require_eq adaptive-rewind-growth MMAP_ARENA_CURRENT 268435456 +printf ' rewound arena grows to target OK\n' + +run_case recycle +require_ge recycle MMAP_RECYCLE 1 +require_le recycle MMAP_HIGH_WATER 201326592 +printf ' arena VA recycling OK\n' + +run_case reuse-mixed +require_ge reuse-mixed MMAP_REUSE_PUBLISH 100 +require_ge reuse-mixed MMAP_REUSE_HIT 100 +require_eq reuse-mixed MMAP_REUSE_DROP 0 +require_le reuse-mixed MMAP_HIGH_WATER 134217728 +printf ' mixed-size committed reuse OK\n' + +printf 'test-mmap-fastpath-stats: PASS\n' diff --git a/tests/test-mmap-fastpath.c b/tests/test-mmap-fastpath.c new file mode 100644 index 00000000..12e4eb1a --- /dev/null +++ b/tests/test-mmap-fastpath.c @@ -0,0 +1,591 @@ +/* + * EL1 consumer-mmap fast-path integration tests. + * + * Run through the dedicated make target without an ELFUSE_MMAP_FASTPATH + * override so the default-enabled configuration is exercised. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +static sigjmp_buf segv_jmp; + +static inline void spin_hint(void); + +static void segv_handler(int sig) +{ + (void) sig; + siglongjmp(segv_jmp, 1); +} + +static int maps_extent_for(uintptr_t needle, + uintptr_t *lo_out, + uintptr_t *hi_out) +{ + int fd = open("/proc/self/maps", O_RDONLY); + if (fd < 0) + return -1; + char buf[16384]; + ssize_t n = read(fd, buf, sizeof(buf) - 1); + close(fd); + if (n <= 0) + return -1; + buf[n] = '\0'; + + char *line = buf; + while (*line) { + unsigned long long lo, hi; + if (sscanf(line, "%llx-%llx", &lo, &hi) == 2 && needle >= lo && + needle < hi) { + *lo_out = (uintptr_t) lo; + *hi_out = (uintptr_t) hi; + return 0; + } + char *nl = strchr(line, '\n'); + if (!nl) + break; + line = nl + 1; + } + return -1; +} + +static void test_fidelity(void) +{ + TEST("unconsumed arena is absent and faults"); + struct sigaction sa = {.sa_handler = segv_handler}; + struct sigaction old_sa; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) { + FAIL("sigaction"); + return; + } + + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + p[0] = 0x5a; /* drains the publication through the fault-side lock */ + + volatile uint8_t *unconsumed = p + 4096; + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) *unconsumed; + FAIL("wild read into unconsumed arena did not SIGSEGV"); + munmap(p, 4096); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + + uintptr_t lo = 0, hi = 0; + if (maps_extent_for((uintptr_t) p, &lo, &hi) < 0 || lo != (uintptr_t) p || + hi != (uintptr_t) p + 4096) { + FAIL("/proc/self/maps exposed more than the consumed page"); + munmap(p, 4096); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + if (munmap(p, 4096) != 0) { + FAIL("munmap"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + + /* No syscall may intervene between munmap and this load: EL1 must have + * invalidated the Stage-1 descriptor and completed broadcast TLBI before + * returning, even though host region cleanup is still deferred. + */ + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) *(volatile uint8_t *) p; + FAIL("access immediately after munmap did not SIGSEGV"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + sigaction(SIGSEGV, &old_sa, NULL); + PASS(); +} + +static void test_exhaustion_fallback(void) +{ + TEST("arena exhaustion falls back to host mmap"); + const size_t len = 80ULL << 20; /* larger than the first 64MiB arena */ + uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("80MiB mmap"); + return; + } + if (p[0] != 0 || p[len - 1] != 0) { + FAIL("fallback mapping was not zero-filled"); + munmap(p, len); + return; + } + p[0] = 1; + p[len - 1] = 2; + if (munmap(p, len) != 0) { + FAIL("munmap"); + return; + } + PASS(); +} + +typedef struct { + volatile uint8_t *p; + _Atomic int ready; + _Atomic int go; + _Atomic int result; +} large_tlbi_arg_t; + +static void *large_tlbi_worker(void *opaque) +{ + large_tlbi_arg_t *arg = opaque; + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) arg->p[0]; /* seed a translation on this sibling vCPU */ + atomic_store_explicit(&arg->ready, 1, memory_order_release); + while (!atomic_load_explicit(&arg->go, memory_order_acquire)) + spin_hint(); + (void) arg->p[0]; + atomic_store_explicit(&arg->result, -1, memory_order_release); + } else { + atomic_store_explicit(&arg->result, 1, memory_order_release); + } + return NULL; +} + +static void test_large_l2_range_tlbi(void) +{ + TEST("SCALE=3 RVAE1IS invalidates sibling L2 translation"); + const size_t len = 320ULL << 20; /* exceeds SCALE=2's 256MiB maximum */ + struct sigaction sa = {.sa_handler = segv_handler}; + struct sigaction old_sa; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) { + FAIL("sigaction"); + return; + } + + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + /* One touch per 2MiB materializes L2 block descriptors without making the + * test resident at every guest page. The resulting TLBI envelope requires + * SCALE=3 when FEAT_TLBIRANGE is enabled. + */ + for (size_t off = 0; off < len; off += 2ULL << 20) + p[off] = (uint8_t) (off >> 21); + + large_tlbi_arg_t arg = {.p = p}; + pthread_t worker; + if (pthread_create(&worker, NULL, large_tlbi_worker, &arg) != 0) { + FAIL("pthread_create"); + munmap((void *) p, len); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + while (!atomic_load_explicit(&arg.ready, memory_order_acquire)) + spin_hint(); + + if (munmap((void *) p, len) != 0) { + FAIL("munmap"); + atomic_store_explicit(&arg.go, 1, memory_order_release); + pthread_join(worker, NULL); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + + /* No syscall may intervene here: the sibling must observe EL1's broadcast + * invalidation before any host drain gets a chance to remove metadata. + */ + atomic_store_explicit(&arg.go, 1, memory_order_release); + int result; + while (!(result = atomic_load_explicit(&arg.result, + memory_order_acquire))) + spin_hint(); + pthread_join(worker, NULL); + sigaction(SIGSEGV, &old_sa, NULL); + if (result < 0) { + FAIL("stale sibling translation survived large-range TLBI"); + return; + } + PASS(); +} + +typedef struct { + _Atomic uintptr_t ptr; + _Atomic int ready; + _Atomic int done; + _Atomic int release; +} handoff_arg_t; + +static inline void spin_hint(void) +{ + __asm__ volatile("yield" ::: "memory"); +} + +static void *handoff_worker(void *opaque) +{ + handoff_arg_t *arg = opaque; + atomic_store_explicit(&arg->ready, 1, memory_order_release); + uintptr_t ptr; + while (!(ptr = atomic_load_explicit(&arg->ptr, memory_order_acquire))) + spin_hint(); + int rc = munmap((void *) ptr, 4096); + atomic_store_explicit(&arg->done, rc == 0 ? 1 : -1, + memory_order_release); + while (!atomic_load_explicit(&arg->release, memory_order_acquire)) + spin_hint(); + return NULL; +} + +static void test_cross_vcpu_handoff(void) +{ + TEST("cross-vCPU mmap publication then munmap retirement"); + struct sigaction sa = {.sa_handler = segv_handler}; + struct sigaction old_sa; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) { + FAIL("sigaction"); + return; + } + + handoff_arg_t arg = {0}; + pthread_t worker; + if (pthread_create(&worker, NULL, handoff_worker, &arg) != 0) { + FAIL("pthread_create"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + while (!atomic_load_explicit(&arg.ready, memory_order_acquire)) + spin_hint(); + + /* Keep the worker alive after munmap so its next thread-exit syscall + * cannot drain either ring. The fault below is the first natural VM exit + * after A's mmap publication and B's retirement. + */ + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + atomic_store_explicit(&arg.release, 1, memory_order_release); + pthread_join(worker, NULL); + FAIL("mmap"); + sigaction(SIGSEGV, &old_sa, NULL); + return; + } + atomic_store_explicit(&arg.ptr, (uintptr_t) p, memory_order_release); + int done; + while (!(done = atomic_load_explicit(&arg.done, memory_order_acquire))) + spin_hint(); + + int faulted = 0; + if (done > 0 && sigsetjmp(segv_jmp, 1) == 0) + (void) *(volatile uint8_t *) p; + else if (done > 0) + faulted = 1; + + atomic_store_explicit(&arg.release, 1, memory_order_release); + pthread_join(worker, NULL); + sigaction(SIGSEGV, &old_sa, NULL); + if (done < 0) { + FAIL("worker munmap"); + return; + } + if (!faulted) { + FAIL("retired cross-vCPU mapping remained accessible"); + return; + } + PASS(); +} + +static void test_mixed_size_committed_reuse(void) +{ + TEST("mixed-size committed extents split and read zero"); + static const size_t sizes[] = { + 64ULL << 10, 3ULL << 20, 20ULL << 10, 1ULL << 20, + 5ULL << 20, 96ULL << 10, 2ULL << 20, 512ULL << 10, + }; + + for (int i = 0; i < 128; i++) { + size_t len = sizes[i % (int) (sizeof(sizes) / sizeof(sizes[0]))]; + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + if (p[0] != 0 || p[len / 2] != 0 || p[len - 1] != 0) { + munmap((void *) p, len); + FAIL("reused extent exposed stale bytes"); + return; + } + p[0] = (uint8_t) (i + 1); + p[len / 2] = (uint8_t) (i ^ 0x5a); + p[len - 1] = (uint8_t) (i ^ 0xa5); + if (munmap((void *) p, len) != 0) { + FAIL("munmap"); + return; + } + } + PASS(); +} + +static void test_repeated_munmap_not_double_reused(void) +{ + TEST("repeated munmap does not double-publish an extent"); + const size_t len = 64ULL << 10; + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("initial mmap"); + return; + } + p[0] = 1; + if (munmap((void *) p, len) != 0 || munmap((void *) p, len) != 0) { + FAIL("repeated munmap"); + return; + } + + /* This fault drains both retire records. Only the first still has region + * coverage and may publish a reusable extent. */ + volatile uint8_t *bridge = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (bridge == MAP_FAILED) { + FAIL("bridge mmap"); + return; + } + bridge[0] = 2; + + volatile uint8_t *a = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + volatile uint8_t *b = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (a == MAP_FAILED || b == MAP_FAILED || a == b) { + if (a != MAP_FAILED) + munmap((void *) a, len); + if (b != MAP_FAILED && b != a) + munmap((void *) b, len); + munmap((void *) bridge, len); + FAIL("duplicate extent allocation"); + return; + } + a[0] = 0x31; + b[0] = 0x42; + if (a[0] != 0x31 || b[0] != 0x42) { + FAIL("distinct mappings aliased"); + return; + } + munmap((void *) a, len); + munmap((void *) b, len); + munmap((void *) bridge, len); + PASS(); +} + +typedef struct { + int iterations; + _Atomic int *failed; +} storm_arg_t; + +static void *storm_worker(void *opaque) +{ + storm_arg_t *arg = opaque; + for (int i = 0; i < arg->iterations; i++) { + size_t len = (size_t) ((i & 7) + 1) * 4096; + uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + __atomic_store_n(arg->failed, 1, __ATOMIC_RELAXED); + break; + } + p[0] = (uint8_t) i; + p[len - 1] = (uint8_t) (i ^ 0x5a); + if (munmap(p, len) != 0) { + __atomic_store_n(arg->failed, 1, __ATOMIC_RELAXED); + break; + } + } + return NULL; +} + +static void test_mt_storm_and_fork_exec(void) +{ + TEST("multi-vCPU mmap storm with fork+exec revocation"); + enum { NTHREADS = 8 }; + pthread_t threads[NTHREADS]; + _Atomic int failed = 0; + storm_arg_t arg = {.iterations = 400, .failed = &failed}; + + int made = 0; + for (; made < NTHREADS; made++) { + if (pthread_create(&threads[made], NULL, storm_worker, &arg) != 0) { + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + break; + } + } + + pid_t pid = fork(); + if (pid == 0) { + char *const argv[] = {(char *) "/proc/self/exe", NULL}; + char *const envp[] = {(char *) "ELFUSE_FASTPATH_EXEC_CHILD=1", NULL}; + execve(argv[0], argv, envp); + _exit(111); + } + if (pid < 0) + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + + for (int i = 0; i < made; i++) + pthread_join(threads[i], NULL); + + if (pid > 0) { + int status = 0; + if (waitpid(pid, &status, 0) != pid || !WIFEXITED(status) || + WEXITSTATUS(status) != 0) + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + } + + /* The parent's arenas were revoked for the fork snapshot. This pair makes + * the first call take the generation fallback and verifies service resumes. + */ + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + __atomic_store_n(&failed, 1, __ATOMIC_RELAXED); + else { + p[0] = 7; + munmap(p, 4096); + } + + if (__atomic_load_n(&failed, __ATOMIC_RELAXED)) { + FAIL("storm/fork/exec worker failure"); + return; + } + PASS(); +} + +static int stats_stream(size_t len, int iterations, bool release_each) +{ + for (int i = 0; i < iterations; i++) { + void *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + if (release_each && munmap(p, len) != 0) + return 1; + } + return 0; +} + +static int stats_mixed_reuse(void) +{ + static const size_t sizes[] = { + 64ULL << 10, 3ULL << 20, 20ULL << 10, 1ULL << 20, + 5ULL << 20, 96ULL << 10, 2ULL << 20, 512ULL << 10, + }; + for (int i = 0; i < 160; i++) { + size_t len = sizes[i % (int) (sizeof(sizes) / sizeof(sizes[0]))]; + volatile uint8_t *p = mmap(NULL, len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + if (p[0] != 0 || p[len - 1] != 0) + return 1; + p[0] = 1; + p[len - 1] = 2; + if (munmap((void *) p, len) != 0) + return 1; + } + return 0; +} + +static int run_stats_case(const char *name) +{ + if (strcmp(name, "ring-full") == 0) + return stats_stream(64ULL << 10, 40, false); + if (strcmp(name, "np2-10m") == 0) + return stats_stream(10ULL << 20, 96, false); + if (strcmp(name, "np2-48m") == 0) + return stats_stream(48ULL << 20, 48, false); + if (strcmp(name, "np2-100m") == 0) + return stats_stream(100ULL << 20, 30, false); + if (strcmp(name, "escalation") == 0) { + if (stats_stream(10ULL << 20, 48, false) != 0) + return 1; + return stats_stream(512ULL << 20, 3, false); + } + if (strcmp(name, "giant-guard") == 0) { + if (stats_stream(10ULL << 20, 32, false) != 0) + return 1; + for (int i = 0; i < 6; i++) { + if (stats_stream(2ULL << 30, 1, false) != 0 || + stats_stream(10ULL << 20, 1, false) != 0) + return 1; + } + return 0; + } + if (strcmp(name, "adaptive-small") == 0) + return stats_stream(64ULL << 10, 1100, false); + if (strcmp(name, "adaptive-retention") == 0) { + if (stats_stream(64ULL << 10, 1100, false) != 0) + return 1; + /* The first request selects a 16GiB arena, the next 32 consume it, and + * the last forces a capacity rollover that must retain the target. + */ + return stats_stream(500ULL << 20, 34, false); + } + if (strcmp(name, "adaptive-rewind-growth") == 0) + /* The first 64MiB arena holds eight 8MiB mappings. The ninth mmap + * takes the capacity fallback after the matching munmaps let host + * drain rewind the arena; refill must grow it to the 32-entry target + * instead of retaining an arena that will miss every eight calls. + */ + return stats_stream(8ULL << 20, 41, true); + if (strcmp(name, "recycle") == 0) + return stats_stream(64ULL << 10, 6000, true); + if (strcmp(name, "reuse-mixed") == 0) + return stats_mixed_reuse(); + return 2; +} + +int main(int argc, char **argv) +{ + if (argc == 2 && strncmp(argv[1], "--stats-", 8) == 0) + return run_stats_case(argv[1] + 8); + + if (getenv("ELFUSE_FASTPATH_EXEC_CHILD")) { + uint8_t *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) + return 1; + p[0] = 0xa5; + return p[0] == 0xa5 ? 0 : 1; + } + + test_fidelity(); + test_exhaustion_fallback(); + test_large_l2_range_tlbi(); + test_cross_vcpu_handoff(); + test_mixed_size_committed_reuse(); + test_repeated_munmap_not_double_reused(); + test_mt_storm_and_fork_exec(); + + printf("\ntest-mmap-fastpath: %d passed, %d failed - %s\n", passes, fails, + fails ? "FAIL" : "PASS"); + return fails ? 1 : 0; +} diff --git a/tests/test-mmap-lazy.c b/tests/test-mmap-lazy.c new file mode 100644 index 00000000..173b850a --- /dev/null +++ b/tests/test-mmap-lazy.c @@ -0,0 +1,846 @@ +/* + * Lazy anonymous mmap regression tests + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Private anonymous mappings defer page-table creation and zeroing to first + * touch. These tests pin down the guest-visible contract of that laziness: + * huge reservations succeed and read as zeros, address reuse never leaks + * stale bytes, host-side syscall access (read/write/futex) works on memory + * the guest never touched, PROT_NONE stays a faulting reservation, data + * survives PROT_NONE round trips and fork, and concurrent first touch from + * multiple threads never loses a write to the deferred zeroing. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +#define BLOCK_2MIB (2ULL << 20) + +#ifndef FUTEX_WAIT +#define FUTEX_WAIT 0 +#define FUTEX_WAKE 1 +#endif + +/* Largest plain anonymous RW mapping the kernel grants. On elfuse the lazy + * path must take this well past physical memory; on real Linux the result + * depends on the overcommit heuristic, so the tests only require >= 1 GiB + * and probe downward. + */ +static void *map_largest(size_t *out_size) +{ + static const size_t sizes[] = { + 64ULL << 30, + 16ULL << 30, + 4ULL << 30, + 1ULL << 30, + }; + for (unsigned i = 0; i < sizeof(sizes) / sizeof(sizes[0]); i++) { + void *p = mmap(NULL, sizes[i], PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p != MAP_FAILED) { + *out_size = sizes[i]; + return p; + } + } + return NULL; +} + +static void test_huge_sparse(void) +{ + TEST("huge mmap + sparse touch"); + size_t size = 0; + volatile uint8_t *p = map_largest(&size); + if (!p || size < (1ULL << 30)) { + FAIL("no >=1GiB anonymous mapping granted"); + return; + } + /* Sparse probes: start, one per size/8 stride, last page. All must read + * zero and accept writes. + */ + for (unsigned i = 0; i < 8; i++) { + size_t off = (size / 8) * i; + if (p[off] != 0) { + FAIL("fresh mapping reads nonzero"); + munmap((void *) p, size); + return; + } + p[off] = (uint8_t) (i + 1); + } + if (p[size - 1] != 0) { + FAIL("last page reads nonzero"); + munmap((void *) p, size); + return; + } + for (unsigned i = 0; i < 8; i++) { + size_t off = (size / 8) * i; + if (p[off] != (uint8_t) (i + 1)) { + FAIL("sparse write lost"); + munmap((void *) p, size); + return; + } + } + if (munmap((void *) p, size) != 0) { + FAIL("munmap"); + return; + } + PASS(); +} + +static void test_zero_reuse(void) +{ + TEST("address reuse reads zero"); + size_t size = 4ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap 1"); + return; + } + memset(p, 0xa5, size); + munmap(p, size); + uint8_t *q = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED) { + FAIL("mmap 2"); + return; + } + /* The allocator typically reuses the freed range; either way no byte may + * be nonzero. Check one page per 2MiB block plus both ends. + */ + for (size_t off = 0; off < size; off += 4096) { + if (q[off] != 0) { + FAIL("stale data after reuse"); + munmap(q, size); + return; + } + } + munmap(q, size); + PASS(); +} + +static void test_hinted_tail_zero(void) +{ + TEST("hinted tail mmap reads zero"); + size_t size = 2ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap base"); + return; + } + p[0] = 0x3c; + p[size - 1] = 0xc3; + + uint8_t *hint = p + size; + uint8_t *q = mmap(hint, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED) { + FAIL("mmap hint"); + munmap(p, size); + return; + } + if (q[0] != 0 || q[size - 1] != 0 || p[0] != 0x3c || p[size - 1] != 0xc3) { + FAIL("hinted tail leaked stale bytes or clobbered neighbor"); + munmap(q, size); + munmap(p, size); + return; + } + munmap(q, size); + munmap(p, size); + PASS(); +} + +static void test_partial_block_reuse(void) +{ + TEST("partial-block reuse preserves neighbor"); + const size_t half = BLOCK_2MIB / 2; + uint8_t *p = mmap(NULL, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + p[17] = 0xa5; + p[half + 17] = 0x5a; + if (mprotect(p + half, half, PROT_READ) != 0 || munmap(p, half) != 0) { + FAIL("split/unmap"); + munmap(p, BLOCK_2MIB); + return; + } + + uint8_t *q = mmap(p, half, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED || q != p) { + FAIL("freed half was not reused at hint"); + if (q != MAP_FAILED) + munmap(q, half); + munmap(p + half, half); + return; + } + if (q[17] != 0 || q[half - 1] != 0 || p[half + 17] != 0x5a) { + FAIL("partial zero clobbered neighbor or leaked stale data"); + munmap(q, half); + munmap(p + half, half); + return; + } + munmap(q, half); + munmap(p + half, half); + PASS(); +} + +static void test_fork_clean_reuse(void) +{ + TEST("fork child sees zero on clean-block reuse"); + uint8_t *p = mmap(NULL, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap 1"); + return; + } + memset(p, 0xcc, BLOCK_2MIB); + if (munmap(p, BLOCK_2MIB) != 0) { + FAIL("munmap"); + return; + } + uint8_t *q = mmap(p, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED) { + FAIL("mmap 2"); + return; + } + pid_t pid = fork(); + if (pid == 0) { + if (q[0] != 0 || q[BLOCK_2MIB - 1] != 0) + _exit(1); + q[123] = 0x77; + _exit(q[124] == 0 ? 0 : 2); + } + int st = 0; + if (pid < 0 || waitpid(pid, &st, 0) != pid || !WIFEXITED(st) || + WEXITSTATUS(st) != 0 || q[123] != 0) { + FAIL("fork clean-block state"); + munmap(q, BLOCK_2MIB); + return; + } + munmap(q, BLOCK_2MIB); + PASS(); +} + +static void test_file_overlay_reuse(void) +{ + TEST("file overlay teardown then lazy reuse"); + char path[] = "/tmp/elfuse-dirty-map.XXXXXX"; + int fd = mkstemp(path); + if (fd < 0) { + FAIL("mkstemp"); + return; + } + unlink(path); + if (ftruncate(fd, BLOCK_2MIB) != 0) { + FAIL("ftruncate"); + close(fd); + return; + } + uint8_t first = 0xa7, last = 0x5c; + if (pwrite(fd, &first, 1, 17) != 1 || + pwrite(fd, &last, 1, BLOCK_2MIB - 1) != 1) { + FAIL("pwrite"); + close(fd); + return; + } + + uint8_t *reserve = mmap(NULL, 2 * BLOCK_2MIB, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserve == MAP_FAILED) { + FAIL("reserve"); + close(fd); + return; + } + uintptr_t aligned = + ((uintptr_t) reserve + BLOCK_2MIB - 1) & ~(BLOCK_2MIB - 1); + munmap(reserve, 2 * BLOCK_2MIB); + uint8_t *target = (uint8_t *) aligned; + + uint8_t *file = mmap(target, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, fd, 0); + if (file != target || file[17] != first || file[BLOCK_2MIB - 1] != last) { + FAIL("file mmap"); + if (file != MAP_FAILED) + munmap(file, BLOCK_2MIB); + close(fd); + return; + } + file[BLOCK_2MIB / 2] = 0xe1; + if (munmap(file, BLOCK_2MIB) != 0) { + FAIL("file munmap"); + close(fd); + return; + } + close(fd); + + uint8_t *anon = mmap(target, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (anon != target || anon[17] != 0 || anon[BLOCK_2MIB / 2] != 0 || + anon[BLOCK_2MIB - 1] != 0) { + FAIL("stale file bytes after lazy reuse"); + if (anon != MAP_FAILED) + munmap(anon, BLOCK_2MIB); + return; + } + munmap(anon, BLOCK_2MIB); + PASS(); +} + +static void test_read_into_lazy(void) +{ + TEST("read() into untouched mapping"); + size_t size = 6ULL << 20; + uint8_t *buf = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int fds[2]; + if (buf == MAP_FAILED || pipe(fds) != 0) { + FAIL("setup"); + return; + } + static const char msg[] = "lazy-host-access-payload"; + /* Unaligned target crossing into the mapping's third 2MiB block. */ + size_t off = (4ULL << 20) + 123; + if (write(fds[1], msg, sizeof(msg)) != (ssize_t) sizeof(msg) || + read(fds[0], buf + off, sizeof(msg)) != (ssize_t) sizeof(msg)) { + FAIL("pipe copy through untouched buffer"); + goto out; + } + if (memcmp(buf + off, msg, sizeof(msg)) != 0) { + FAIL("payload corrupted"); + goto out; + } + /* A guest touch elsewhere in the same 2MiB block must not re-zero the + * host-written payload (deferred-zeroing idempotence). + */ + buf[(4ULL << 20) + 64 * 1024] = 7; + if (memcmp(buf + off, msg, sizeof(msg)) != 0) { + FAIL("payload clobbered by later fault in same block"); + goto out; + } + /* Untouched parts of the mapping still read zero. */ + for (size_t i = 0; i < 4096; i++) { + if (buf[i] != 0) { + FAIL("nonzero byte in untouched block"); + goto out; + } + } + PASS(); +out: + close(fds[0]); + close(fds[1]); + munmap(buf, size); +} + +static void test_write_from_lazy(void) +{ + TEST("write() from untouched mapping"); + size_t size = 2ULL << 20; + uint8_t *src = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + int fds[2]; + if (src == MAP_FAILED || pipe(fds) != 0) { + FAIL("setup"); + return; + } + uint8_t back[512]; + memset(back, 0xff, sizeof(back)); + if (write(fds[1], src + 4096, sizeof(back)) != (ssize_t) sizeof(back) || + read(fds[0], back, sizeof(back)) != (ssize_t) sizeof(back)) { + FAIL("pipe copy from untouched buffer"); + goto out; + } + for (size_t i = 0; i < sizeof(back); i++) { + if (back[i] != 0) { + FAIL("untouched buffer sent nonzero bytes"); + goto out; + } + } + PASS(); +out: + close(fds[0]); + close(fds[1]); + munmap(src, size); +} + +static void test_prot_none_roundtrip(void) +{ + TEST("mprotect NONE round trip keeps data"); + size_t size = 4ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + memset(p, 0x5c, 8192); + p[size - 1] = 0x77; + if (mprotect(p, size, PROT_NONE) != 0 || + mprotect(p, size, PROT_READ | PROT_WRITE) != 0) { + FAIL("mprotect"); + munmap(p, size); + return; + } + if (p[0] != 0x5c || p[8191] != 0x5c || p[size - 1] != 0x77 || + p[16384] != 0) { + FAIL("data lost or stale bytes after round trip"); + munmap(p, size); + return; + } + munmap(p, size); + PASS(); +} + +static void test_reserve_commit(void) +{ + TEST("PROT_NONE reserve + mprotect commit"); + size_t size = 1ULL << 30; + uint8_t *p = mmap(NULL, size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + if (p == MAP_FAILED) { + FAIL("reserve"); + return; + } + uint8_t *slab = p + (512ULL << 20); + if (mprotect(slab, 8ULL << 20, PROT_READ | PROT_WRITE) != 0) { + FAIL("commit"); + munmap(p, size); + return; + } + for (size_t off = 0; off < (8ULL << 20); off += 4096) { + if (slab[off] != 0) { + FAIL("committed slab reads nonzero"); + munmap(p, size); + return; + } + } + slab[0] = 1; + slab[(8ULL << 20) - 1] = 2; + if (slab[0] != 1 || slab[(8ULL << 20) - 1] != 2) { + FAIL("committed slab write lost"); + munmap(p, size); + return; + } + munmap(p, size); + PASS(); +} + +static sigjmp_buf segv_jmp; + +static void segv_handler(int sig) +{ + (void) sig; + siglongjmp(segv_jmp, 1); +} + +static void test_prot_none_faults(void) +{ + TEST("PROT_NONE|NORESERVE still faults"); + size_t size = 16ULL << 20; + volatile uint8_t *p = + mmap(NULL, size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, + -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + struct sigaction sa = {0}, old_sa; + sa.sa_handler = segv_handler; + sigaction(SIGSEGV, &sa, &old_sa); + int faulted = 0; + if (sigsetjmp(segv_jmp, 1) == 0) { + (void) p[BLOCK_2MIB + 5]; + } else { + faulted = 1; + } + sigaction(SIGSEGV, &old_sa, NULL); + munmap((void *) p, size); + /* A lazy materializer that ignores prot would silently hand the guest a + * readable zero page here instead of SIGSEGV. + */ + EXPECT_TRUE(faulted, "read from PROT_NONE reservation did not fault"); +} + +static void test_fork_lazy(void) +{ + TEST("fork with partially touched mapping"); + size_t size = 8ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + memset(p, 0x42, 4096); /* touch only block 0 */ + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork"); + munmap(p, size); + return; + } + if (pid == 0) { + /* Child: inherited data intact, untouched block reads zero and is + * privately writable. + */ + if (p[0] != 0x42 || p[4095] != 0x42) + _exit(1); + if (p[4ULL << 20] != 0) + _exit(2); + p[4ULL << 20] = 0x99; + if (p[(4ULL << 20) + 1] != 0) + _exit(3); + _exit(0); + } + int st = 0; + if (waitpid(pid, &st, 0) != pid || !WIFEXITED(st) || WEXITSTATUS(st) != 0) { + FAIL("child saw wrong memory"); + munmap(p, size); + return; + } + /* Parent: child's private write must not leak back. */ + if (p[4ULL << 20] != 0) { + FAIL("child write leaked into parent"); + munmap(p, size); + return; + } + munmap(p, size); + PASS(); +} + +static void test_futex_untouched(void) +{ + TEST("futex on untouched mapping"); + size_t size = 4ULL << 20; + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + uint32_t *word = (uint32_t *) (p + (2ULL << 20) + 256); + /* WAKE on never-touched memory: no waiters, must not fault. */ + long r = syscall(SYS_futex, word, FUTEX_WAKE, 1, NULL, NULL, 0); + if (r != 0) { + FAIL("FUTEX_WAKE on untouched word"); + munmap(p, size); + return; + } + /* WAIT with expected=1: the word reads as zero, so EAGAIN. */ + r = syscall(SYS_futex, word, FUTEX_WAIT, 1, NULL, NULL, 0); + if (!(r == -1 && errno == EAGAIN)) { + FAIL("FUTEX_WAIT did not read zero from untouched word"); + munmap(p, size); + return; + } + munmap(p, size); + PASS(); +} + +/* Concurrent first touch: every thread writes its own slot in the same fresh + * 2MiB block, racing the deferred zeroing. A materializer that re-zeros an + * already-populated block loses some slots. + */ +#define MT_THREADS 4 +#define MT_ITERS 64 + +typedef struct { + uint8_t *base; + int idx; + pthread_barrier_t *barrier; +} mt_arg_t; + +static void *mt_touch(void *argp) +{ + mt_arg_t *a = argp; + pthread_barrier_wait(a->barrier); + a->base[a->idx * 64] = (uint8_t) (a->idx + 1); + /* Also touch a private block so several materializations race. */ + a->base[BLOCK_2MIB * (unsigned) (a->idx + 1) + 17] = + (uint8_t) (0x10 + a->idx); + return NULL; +} + +static void test_mt_first_touch(void) +{ + TEST("concurrent first touch"); + for (int iter = 0; iter < MT_ITERS; iter++) { + size_t size = BLOCK_2MIB * (MT_THREADS + 2); + uint8_t *p = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + pthread_barrier_t barrier; + pthread_barrier_init(&barrier, NULL, MT_THREADS); + pthread_t th[MT_THREADS]; + mt_arg_t args[MT_THREADS]; + for (int i = 0; i < MT_THREADS; i++) { + args[i] = (mt_arg_t) {p, i, &barrier}; + if (pthread_create(&th[i], NULL, mt_touch, &args[i]) != 0) { + FAIL("pthread_create"); + return; + } + } + for (int i = 0; i < MT_THREADS; i++) + pthread_join(th[i], NULL); + pthread_barrier_destroy(&barrier); + for (int i = 0; i < MT_THREADS; i++) { + if (p[i * 64] != (uint8_t) (i + 1) || + p[BLOCK_2MIB * (unsigned) (i + 1) + 17] != + (uint8_t) (0x10 + i)) { + FAIL("write lost to concurrent materialization"); + munmap(p, size); + return; + } + } + munmap(p, size); + } + PASS(); +} + +typedef struct { + uint8_t *base; + size_t half; + int idx; + pthread_barrier_t *barrier; + int *error; +} claim_race_arg_t; + +static void *claim_race_touch(void *argp) +{ + claim_race_arg_t *a = argp; + pthread_barrier_wait(a->barrier); + a->base[64 * (unsigned) a->idx] = (uint8_t) (a->idx + 1); + return NULL; +} + +static void *claim_race_mutate(void *argp) +{ + claim_race_arg_t *a = argp; + uint8_t *neighbor = a->base + a->half; + uint8_t *hole = neighbor + 4096; + pthread_barrier_wait(a->barrier); + for (int i = 0; i < 16; i++) { + if (mprotect(neighbor, a->half, PROT_NONE) != 0 || + mprotect(neighbor, a->half, PROT_READ) != 0 || + munmap(hole, 4096) != 0) { + *a->error = 1; + return NULL; + } + void *r = mmap(hole, 4096, PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (r != hole) { + *a->error = 1; + return NULL; + } + } + return NULL; +} + +static void test_claim_mutation_race(void) +{ + TEST("first-touch claim vs adjacent mutations"); + const size_t half = BLOCK_2MIB / 2; + uint8_t *p = mmap(NULL, BLOCK_2MIB, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + p[half + 17] = 0x6d; + if (mprotect(p + half, half, PROT_READ) != 0 || munmap(p, half) != 0) { + FAIL("split/unmap"); + munmap(p, BLOCK_2MIB); + return; + } + uint8_t *q = mmap(p, half, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q != p) { + FAIL("reuse"); + if (q != MAP_FAILED) + munmap(q, half); + munmap(p + half, half); + return; + } + + pthread_barrier_t barrier; + pthread_barrier_init(&barrier, NULL, MT_THREADS + 1); + pthread_t workers[MT_THREADS], mutator; + claim_race_arg_t args[MT_THREADS + 1]; + int error = 0; + for (int i = 0; i < MT_THREADS; i++) { + args[i] = (claim_race_arg_t) {q, half, i, &barrier, &error}; + pthread_create(&workers[i], NULL, claim_race_touch, &args[i]); + } + args[MT_THREADS] = (claim_race_arg_t) {q, half, 0, &barrier, &error}; + pthread_create(&mutator, NULL, claim_race_mutate, &args[MT_THREADS]); + for (int i = 0; i < MT_THREADS; i++) + pthread_join(workers[i], NULL); + pthread_join(mutator, NULL); + pthread_barrier_destroy(&barrier); + + for (int i = 0; i < MT_THREADS; i++) { + if (q[64 * (unsigned) i] != (uint8_t) (i + 1)) + error = 1; + } + if (p[half + 17] != 0x6d) + error = 1; + munmap(q, half); + munmap(p + half, half); + if (error) { + FAIL("claim/mutation race corrupted data"); + return; + } + PASS(); +} + +static void test_adjacent_region_extension(void) +{ + TEST("adjacent fast-mmap region extension"); + enum { N_PAGES = 64 }; + uint8_t *pages[N_PAGES]; + int allocated = 0; + bool ok = true; + + for (int i = 0; i < N_PAGES; i++) { + pages[i] = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (pages[i] == MAP_FAILED) { + ok = false; + break; + } + allocated++; + pages[i][0] = (uint8_t) (i + 1); + } + for (int i = 0; ok && i < allocated; i++) { + if (pages[i][0] != (uint8_t) (i + 1)) + ok = false; + } + for (int i = 0; i < allocated; i++) + munmap(pages[i], 4096); + + if (!ok) { + FAIL("adjacent lazy mappings did not materialize independently"); + return; + } + PASS(); +} + +static void test_large_retire_backing_replacement(void) +{ + TEST("large retire replacement preserves neighbors and fork zeroes"); + const size_t body_len = 48ULL << 20; + const size_t total_len = body_len + (6ULL << 20); + uint8_t *p = mmap(NULL, total_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap"); + return; + } + + uintptr_t aligned = ((uintptr_t) p + BLOCK_2MIB * 2 - 1) & + ~(uintptr_t) (BLOCK_2MIB - 1); + uint8_t *body = (uint8_t *) aligned; + size_t left_len = (size_t) (body - p); + size_t right_len = total_len - left_len - body_len; + if (left_len < BLOCK_2MIB || right_len < BLOCK_2MIB) { + FAIL("guard alignment"); + munmap(p, total_len); + return; + } + + memset(p, 0xa5, total_len); + if (munmap(body, body_len) != 0) { + FAIL("retire body"); + munmap(p, total_len); + return; + } + + /* MAP_FIXED is a metadata-reading slow path, so it must first drain the + * EL1 retirement. The 48 MiB aligned dirty body exercises the 32 MiB HVF + * backing-replacement policy rather than the short-range memset fallback. + */ + uint8_t *q = mmap(body, body_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (q != body) { + FAIL("fixed reuse"); + munmap(p, left_len); + munmap(body + body_len, right_len); + return; + } + + bool ok = p[0] == 0xa5 && body[-1] == 0xa5 && + body[body_len] == 0xa5 && p[total_len - 1] == 0xa5; + for (size_t off = 0; ok && off < body_len; off += 4096) + ok = q[off] == 0; + + pid_t pid = -1; + int st = 0; + if (ok) + pid = fork(); + if (pid == 0) { + for (size_t off = 0; off < body_len; off += BLOCK_2MIB) { + if (q[off] != 0) + _exit(1); + } + q[123] = 0x77; + _exit(q[124] == 0 ? 0 : 2); + } + if (pid < 0 || waitpid(pid, &st, 0) != pid || !WIFEXITED(st) || + WEXITSTATUS(st) != 0 || q[123] != 0) + ok = false; + + munmap(p, left_len); + munmap(q, body_len); + munmap(body + body_len, right_len); + if (!ok) { + FAIL("replacement leaked data, clobbered a neighbor, or broke fork"); + return; + } + PASS(); +} + +int main(void) +{ + test_huge_sparse(); + test_zero_reuse(); + test_hinted_tail_zero(); + test_partial_block_reuse(); + test_fork_clean_reuse(); + test_file_overlay_reuse(); + test_read_into_lazy(); + test_write_from_lazy(); + test_prot_none_roundtrip(); + test_reserve_commit(); + test_prot_none_faults(); + test_fork_lazy(); + test_futex_untouched(); + test_mt_first_touch(); + test_claim_mutation_race(); + test_adjacent_region_extension(); + test_large_retire_backing_replacement(); + + SUMMARY("test-mmap-lazy"); + return fails ? 1 : 0; +} diff --git a/tests/test-mremap.c b/tests/test-mremap.c index dc97321b..bf8abe33 100644 --- a/tests/test-mremap.c +++ b/tests/test-mremap.c @@ -120,6 +120,106 @@ static void test_grow_maymove(void) munmap(q, 4096 * 4); } +static void test_grow_move_adjacent_fault(void) +{ + TEST("mremap fixed move keeps adjacent page unmapped"); + void *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap source"); + return; + } + char *dest = + mmap(NULL, 4096 * 3, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (dest == MAP_FAILED) { + FAIL("mmap dest"); + munmap(p, 4096); + return; + } + if (munmap(dest + 8192, 4096) < 0) { + FAIL("munmap guard"); + munmap(dest, 8192); + munmap(p, 4096); + return; + } + ((char *) p)[0] = 0x5a; + + void *q = mremap(p, 4096, 8192, MREMAP_MAYMOVE | MREMAP_FIXED, dest); + if (q == MAP_FAILED) { + FAIL("mremap move"); + munmap(dest, 8192); + munmap(p, 4096); + return; + } + if (q != dest || ((char *) q)[0] != 0x5a || ((char *) q)[4096] != 0) { + FAIL("mremap data"); + munmap(q, 8192); + return; + } + + pid_t pid = fork(); + if (pid == 0) { + volatile unsigned char value = *((volatile unsigned char *) q + 8192); + (void) value; + _exit(1); + } + int status = 0; + if (pid < 0 || waitpid(pid, &status, 0) != pid) { + FAIL("fork/wait"); + munmap(q, 8192); + return; + } + if ((WIFSIGNALED(status) && WTERMSIG(status) == SIGSEGV) || + (WIFEXITED(status) && WEXITSTATUS(status) == 139)) + PASS(); + else + FAIL("adjacent page became readable"); + munmap(q, 8192); +} + +static void test_fixed_preserves_neighbor_l3(void) +{ + TEST("mremap fixed preserves neighboring lazy PTEs"); + char *source = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (source == MAP_FAILED) { + FAIL("mmap source"); + return; + } + char *anchor = mmap(NULL, 8192, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (anchor == MAP_FAILED) { + FAIL("mmap anchor"); + munmap(source, 4096); + return; + } + char *dest = anchor + 4096; + if (munmap(dest, 4096) < 0) { + FAIL("munmap dest"); + munmap(anchor, 4096); + munmap(source, 4096); + return; + } + source[0] = 0x11; + anchor[0] = 0x7e; + + void *moved = + mremap(source, 4096, 4096, MREMAP_MAYMOVE | MREMAP_FIXED, dest); + if (moved == MAP_FAILED) { + FAIL("mremap fixed"); + munmap(anchor, 4096); + munmap(source, 4096); + return; + } + if (moved != dest || dest[0] != 0x11 || anchor[0] != 0x7e) + FAIL("neighboring lazy page changed"); + else + PASS(); + + munmap(dest, 4096); + munmap(anchor, 4096); +} + /* Test 3: grow without MAYMOVE fails if blocked */ static void test_grow_no_maymove(void) @@ -403,6 +503,8 @@ int main(void) test_shrink(); test_grow_maymove(); + test_grow_move_adjacent_fault(); + test_fixed_preserves_neighbor_l3(); test_grow_no_maymove(); test_fixed(); test_same_size(); diff --git a/tests/test-negative.c b/tests/test-negative.c index 3470147e..3d5ef814 100644 --- a/tests/test-negative.c +++ b/tests/test-negative.c @@ -208,6 +208,110 @@ static void test_mmap_prot(void) munmap(p, 4096); EXPECT_TRUE(saw_segv, "write to read-only mapping succeeded"); } + + TEST("fast single-page mprotect drains publication ring"); + { + volatile int *p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap failed"); + return; + } + *p = 1; /* Materialize a partial block, which installs an L3 table. */ + for (int i = 0; i < 96; i++) { + int prot = (i & 1) ? PROT_READ : PROT_READ | PROT_WRITE; + if (mprotect((void *) p, 4096, prot) != 0) { + munmap((void *) p, 4096); + FAIL("mprotect toggle failed"); + return; + } + } + + struct sigaction old_sa; + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = on_sigsegv; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) { + munmap((void *) p, 4096); + FAIL("sigaction failed"); + return; + } + saw_segv = 0; + if (sigsetjmp(segv_jmp, 1) == 0) + *p = 2; + sigaction(SIGSEGV, &old_sa, NULL); + + int restored = mprotect((void *) p, 4096, PROT_READ | PROT_WRITE) == 0; + if (restored) + *p = 3; + int final = *p; + munmap((void *) p, 4096); + EXPECT_TRUE(saw_segv && restored && final == 3, + "fast mprotect permissions or ring drain were stale"); + } + + TEST("first lazy 2MiB block mprotect materializes safely"); + { + const size_t block_size = 2 * 1024 * 1024; + volatile unsigned char *seed = + mmap(NULL, block_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (seed == MAP_FAILED) { + FAIL("seed mmap failed"); + return; + } + seed[1024 * 1024] = 0x7f; + munmap((void *) seed, block_size); + /* Force retirement so the next mmap can reuse backing whose dirty bit + * remains conservative. mprotect must update lazy metadata without + * exposing the stale byte; the later read performs the zeroing fault. + */ + (void) fcntl(-1, F_GETFD); + + volatile unsigned char *p = + mmap(NULL, block_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (p == MAP_FAILED) { + FAIL("mmap failed"); + return; + } + volatile unsigned char *mid = p + 1024 * 1024; + if (mprotect((void *) mid, 4096, PROT_READ) != 0) { + munmap((void *) p, block_size); + FAIL("first-block mprotect failed"); + return; + } + + int zeroed = p[0] == 0 && mid[0] == 0 && p[block_size - 1] == 0; + p[0] = 0x31; + p[block_size - 1] = 0x32; + + struct sigaction old_sa; + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = on_sigsegv; + sigemptyset(&sa.sa_mask); + if (sigaction(SIGSEGV, &sa, &old_sa) != 0) { + munmap((void *) p, block_size); + FAIL("sigaction failed"); + return; + } + saw_segv = 0; + if (sigsetjmp(segv_jmp, 1) == 0) + mid[0] = 0x33; + sigaction(SIGSEGV, &old_sa, NULL); + + int restored = + mprotect((void *) mid, 4096, PROT_READ | PROT_WRITE) == 0; + if (restored) + mid[0] = 0x34; + int neighbors = p[0] == 0x31 && p[block_size - 1] == 0x32; + int target = mid[0] == 0x34; + munmap((void *) p, block_size); + EXPECT_TRUE(zeroed && saw_segv && restored && neighbors && target, + "first-block split exposed stale bytes or wrong perms"); + } } /* Test 7: fcntl on invalid FD */ diff --git a/tests/test-tlbi-encoder-host.c b/tests/test-tlbi-encoder-host.c index 4757a2f0..accc26b7 100644 --- a/tests/test-tlbi-encoder-host.c +++ b/tests/test-tlbi-encoder-host.c @@ -47,12 +47,12 @@ static void check_field(const char *label, uint64_t got, uint64_t expect) /* Decompose the operand per ARM ARM D8.7.6 and compare each field against the * expected value. baseADDR is VA>>12 masked to 37 bits; TG must be 01 (4 KiB); - * SCALE must be 0; TTL must be 0; ASID must be 0. NUM derives from the page - * count via the ceil(pages/2) - 1 SCALE=0 encoding. + * TTL and ASID must be 0. NUM derives from the selected SCALE unit. */ static void verify_operand(uint64_t start_va, uint16_t pages, - uint64_t expect_num) + uint64_t expect_num, + uint64_t expect_scale) { uint64_t op = tlbi_rvae1is_operand(start_va, pages); @@ -76,7 +76,7 @@ static void verify_operand(uint64_t start_va, check_field(label, num, expect_num); snprintf(label, sizeof(label), "SCALE (pages=%u)", (unsigned) pages); - check_field(label, scale, 0); + check_field(label, scale, expect_scale); snprintf(label, sizeof(label), "TG (start=0x%llx)", (unsigned long long) start_va); @@ -100,30 +100,47 @@ int main(void) * pages 63 -> NUM 31 (covers 64) * pages 64 -> NUM 31 (covers 64) */ - verify_operand(0x10000000ULL, 2, 0); - verify_operand(0x10000000ULL, 3, 1); - verify_operand(0x10000000ULL, 16, 7); - verify_operand(0x10000000ULL, 17, 8); - verify_operand(0x10000000ULL, 32, 15); - verify_operand(0x10000000ULL, 63, 31); - verify_operand(0x10000000ULL, 64, 31); + verify_operand(0x10000000ULL, 2, 0, 0); + verify_operand(0x10000000ULL, 3, 1, 0); + verify_operand(0x10000000ULL, 16, 7, 0); + verify_operand(0x10000000ULL, 17, 8, 0); + verify_operand(0x10000000ULL, 32, 15, 0); + verify_operand(0x10000000ULL, 63, 31, 0); + verify_operand(0x10000000ULL, 64, 31, 0); + + /* SCALE=1 covers 64 pages per NUM step; SCALE=2 covers 2048. A lazy + * 2 MiB block is 512 pages and must therefore encode as SCALE=1, NUM=7. + */ + verify_operand(0x10000000ULL, 512, 7, 1); + verify_operand(0x10000000ULL, 2048, 31, 1); + verify_operand(0x10000000ULL, 8192, 3, 2); /* Boundary VAs. 4 KiB-aligned, low-VA, MMAP_BASE (8 GiB), high-VA just * below the 48-bit BaseADDR truncation point. */ - verify_operand(0x00000000ULL, 32, 15); /* zero base */ - verify_operand(0x200000000ULL, 32, 15); /* MMAP_BASE */ - verify_operand(0x800000000000ULL, 32, 15); /* Rosetta image */ - verify_operand(0x0000FFFFF0000000ULL, 32, 15); /* KBUF_USER_VA */ + verify_operand(0x00000000ULL, 32, 15, 0); /* zero base */ + verify_operand(0x200000000ULL, 32, 15, 0); /* MMAP_BASE */ + verify_operand(0x800000000000ULL, 32, 15, 0); /* Rosetta image */ + verify_operand(0x0000FFFFF0000000ULL, 32, 15, 0); /* KBUF_USER_VA */ /* Pathological inputs the clamp must catch: * pages = 0 -> clamped to 2 -> NUM 0 * pages = 1 -> clamped to 2 -> NUM 0 (callers never reach here) * pages = UINT16_MAX -> NUM clamped to 31 (saturating) */ - verify_operand(0x10000000ULL, 0, 0); - verify_operand(0x10000000ULL, 1, 0); - verify_operand(0x10000000ULL, UINT16_MAX, 31); + verify_operand(0x10000000ULL, 0, 0, 0); + verify_operand(0x10000000ULL, 1, 0, 0); + verify_operand(0x10000000ULL, UINT16_MAX, 31, 2); + + /* The accumulator widens a 2 MiB request to the SCALE=1 granule and keeps + * it on the single-shot RVAE path instead of degrading to broadcast. + */ + g_tlbi_range_supported = true; + tlbi_request_clear(); + tlbi_request_range(0x200000000ULL, 0x200200000ULL); + check_field("2MiB accumulator kind", cpu_tlbi_req.kind, TLBI_RANGE_LARGE); + check_field("2MiB accumulator pages", cpu_tlbi_req.pages, 512); + check_field("2MiB accumulator start", cpu_tlbi_req.start, 0x200000000ULL); /* TG bit is the architectural lynchpin -- if the encoder ever drops it the * integration tests on Apple Silicon would still pass. Pin a direct bit-46