INotFail/Elyz
2
2
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: major memory managment & PMactor

Browse Source
This commit is contained in:
Faynot
2026-06-26 14:33:29 +03:00
parent ceedd21aee
commit 3f87a93d52
14 changed files with 2347 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
kernel/src/mem/allocator.rs
View File

@@ -25,7 +25,7 @@ impl<A> Locked<A> {
pub fn lock(&self) -> LockedGuard<'_, A> {
while self.lock.compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed).is_err() {
core::hint::spin_loop();
core::hint::spin_loop(); // Уступка в spinlock для снижения нагрузки на шину
}
LockedGuard {
lock: &self.lock,
@@ -49,41 +49,106 @@ impl<A> DerefMut for LockedGuard<'_, A> {
fn deref_mut(&mut self) -> &mut Self::Target { self.data }
}
pub struct BumpAllocator {
start: usize,
end: usize,
next: usize,
/// Узел односвязного списка свободных блоков
struct ListNode {
next: Option<&'static mut ListNode>,
}
impl BumpAllocator {
const BLOCK_SIZES: &[usize] = &[8, 16, 32, 64, 128, 256, 512, 1024, 2048];
/// Slab аллокатор для гранулярного выделения памяти
pub struct SlabAllocator {
list_heads: [Option<&'static mut ListNode>; BLOCK_SIZES.len()],
heap_start: usize,
heap_end: usize,
next_bump: usize,
}
impl SlabAllocator {
pub const fn new() -> Self {
Self { start: 0, end: 0, next: 0 }
}
pub fn init(&mut self, start: usize, size: usize) {
self.start = start;
self.next = start;
self.end = start + size;
}
}
unsafe impl GlobalAlloc for Locked<BumpAllocator> {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
let mut bump = self.lock();
let alloc_start = (bump.next + layout.align() - 1) & !(layout.align() - 1);
let alloc_end = alloc_start + layout.size();
if alloc_end > bump.end {
null_mut()
} else {
bump.next = alloc_end;
alloc_start as *mut u8
Self {
list_heads: [None, None, None, None, None, None, None, None, None],
heap_start: 0,
heap_end: 0,
next_bump: 0,
}
}
unsafe fn dealloc(&self, _ptr: *mut u8, _layout: Layout) {}
pub fn init(&mut self, start: usize, size: usize) {
self.heap_start = start;
self.next_bump = start;
self.heap_end = start + size;
}
/// Поиск индекса блока под требуемый размер
fn list_index(layout: &Layout) -> Option<usize> {
let required_block_size = layout.size().max(layout.align());
BLOCK_SIZES.iter().position(|&s| s >= required_block_size)
}
/// Резервный Bump-аллокатор для нарезки новых Slab-блоков
fn fallback_alloc(&mut self, layout: Layout) -> *mut u8 {
let alloc_start = (self.next_bump + layout.align() - 1) & !(layout.align() - 1);
let alloc_end = alloc_start.checked_add(layout.size()).unwrap_or(self.heap_end + 1);
if alloc_end > self.heap_end {
null_mut() // Out of memory
} else {
self.next_bump = alloc_end;
alloc_start as *mut u8
}
}
}
unsafe impl GlobalAlloc for Locked<SlabAllocator> {
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
let mut allocator = self.lock();
match SlabAllocator::list_index(&layout) {
Some(index) => {
match allocator.list_heads[index].take() {
Some(node) => {
allocator.list_heads[index] = node.next.take();
node as *mut ListNode as *mut u8
}
None => {
let block_size = BLOCK_SIZES[index];
let block_align = block_size;
let layout = Layout::from_size_align(block_size, block_align).unwrap();
allocator.fallback_alloc(layout)
}
}
}
None => allocator.fallback_alloc(layout)
}
}
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
let mut allocator = self.lock();
match SlabAllocator::list_index(&layout) {
Some(index) => {
let new_node = ListNode {
next: allocator.list_heads[index].take(),
};
assert!(layout.size() >= core::mem::size_of::<ListNode>());
let new_node_ptr = ptr as *mut ListNode;
// Production-fix для Rust 2024: явная изоляция unsafe-операций
unsafe {
new_node_ptr.write(new_node);
allocator.list_heads[index] = Some(&mut *new_node_ptr);
}
}
None => {
// Крупные регионы освобождаются через вызовы дескрипторов VMM/PMM,
// глобальный аллокатор ядра их не трекает.
}
}
}
}
#[global_allocator]
pub static ALLOCATOR: Locked<BumpAllocator> = Locked::new(BumpAllocator::new());
pub static ALLOCATOR: Locked<SlabAllocator> = Locked::new(SlabAllocator::new());

298
kernel/src/mem/buddy.rs Normal file
View File

@@ -0,0 +1,298 @@
//! # Per-Actor Buddy Allocator (`src/mem/buddy.rs`)
//!
//! A power-of-two page-block allocator for a fixed, pre-committed physical
//! region. Designed for **single-consumer** use inside a [`crate::mem::pm_manages::PMActor`]:
//! the owning actor is the only entity that ever mutates its buddy allocator, so
//! no locking is needed — the entire subsystem is inherently lock-free from the
//! consumer's perspective.
//!
//! ## Block Layout
//!
//! The region is subdivided into blocks of `2^order` pages (order 0 … MAX_ORDER).
//! Every block at order *k* is aligned to `2^k` pages within the region.
//!
//! ```text
//! order 0 → 1 page = 4 KiB
//! order 1 → 2 pages = 8 KiB
//! …
//! order 11 → 2 048 pages = 8 MiB (MAX_ORDER)
//! ```
//!
//! ## Complexity
//!
//! | Operation | Amortised | Worst-case |
//! |-----------|-----------|------------|
//! | `alloc` | O(log N) | O(MAX_ORDER · Lₖ) |
//! | `free` | O(log N) | O(MAX_ORDER · Lₖ) |
//!
//! where Lₖ = `free_lists[k].len()` ≤ `total_pages / 2^k`.
//!
//! ## Production note
//!
//! For managed ranges > 1 GiB (> 256 K pages), replace the per-order `Vec<usize>`
//! free lists with a radix tree or interval tree to bound Lₖ. For current
//! PMActor capital sizes (typically ≤ 128 MiB = 32 K pages at order 0), the
//! `Vec`-based implementation is fully adequate.
extern crate alloc;
use alloc::vec::Vec;
// ══════════════════════════════════════════════════════════════════════════════
// Constants & helpers
// ══════════════════════════════════════════════════════════════════════════════
/// Maximum allocation order.
/// `2^11 × 4 096 bytes = 8 MiB` per single allocation.
pub const MAX_ORDER: usize = 11;
/// ⌈log₂(n)⌉ — the minimum order whose block size covers `page_count` pages.
///
/// ```text
/// order_for(1) = 0 (2^0 = 1)
/// order_for(2) = 1 (2^1 = 2)
/// order_for(3) = 2 (2^2 = 4 ≥ 3)
/// order_for(2048) = 11
/// ```
#[inline]
pub fn order_for(page_count: usize) -> usize {
if page_count <= 1 {
0
} else {
// Number of bits needed to represent (page_count - 1).
usize::BITS as usize - (page_count - 1).leading_zeros() as usize
}
}
// ══════════════════════════════════════════════════════════════════════════════
// BuddyAllocator
// ══════════════════════════════════════════════════════════════════════════════
/// Buddy allocator over a contiguous, pre-committed range of physical pages.
///
/// All page indices stored in free lists are **relative to the start of the
/// managed range**. The owner (`PMActor`) translates to absolute `PhysAddr`
/// by adding `managed_range.0`.
///
/// # Invariants
/// - `free_pages ≤ total_pages` at all times.
/// - Every block in `free_lists[k]` is aligned to `2^k` pages (i.e.
/// `idx % (1 << k) == 0`).
/// - No block appears in more than one order's free list simultaneously.
pub struct BuddyAllocator {
/// `free_lists[k]` = relative page indices of free 2^k-page blocks.
/// Ordering within each list is irrelevant; `pop()` / `swap_remove()` are used.
free_lists: [Vec<usize>; MAX_ORDER + 1],
/// Total pages in the managed range (need not be a power of two).
total_pages: usize,
/// Running count of free pages. Always equals `Σ (2^k × free_lists[k].len())`.
free_pages: usize,
}
impl BuddyAllocator {
// ─── Construction ─────────────────────────────────────────────────────────
/// Create a new allocator over `total_pages` pages, **all initially free**.
///
/// Uses a greedy largest-first decomposition to build the initial free lists
/// in O(total_pages / 2^MAX_ORDER) iterations — essentially O(1) for
/// power-of-two sizes.
///
/// Example: 7 pages → blocks [4, 2, 1] → free_lists[2]=[0], [1]=[4], [0]=[6].
pub fn new(total_pages: usize) -> Self {
// core::array::from_fn is the idiomatic way to init a non-Copy array.
let free_lists: [Vec<usize>; MAX_ORDER + 1] =
core::array::from_fn(|_| Vec::new());
let mut this = Self {
free_lists,
total_pages,
free_pages: 0,
};
if total_pages == 0 {
return this;
}
let mut idx = 0usize;
while idx < total_pages {
let remaining = total_pages - idx;
// Alignment constraint: block at `idx` must be aligned to 2^order.
// trailing_zeros(0) is u32::MAX, so we clamp to MAX_ORDER.
let align_order = if idx == 0 {
MAX_ORDER
} else {
(idx.trailing_zeros() as usize).min(MAX_ORDER)
};
// Size constraint: 2^order ≤ remaining → order ≤ ⌊log₂(remaining)⌋.
let size_order = (usize::BITS as usize - 1)
- remaining.leading_zeros() as usize; // ⌊log₂(remaining)⌋
let order = MAX_ORDER.min(align_order).min(size_order);
let block_size = 1usize << order;
this.free_lists[order].push(idx);
this.free_pages += block_size;
idx += block_size;
}
this
}
// ─── Allocation ───────────────────────────────────────────────────────────
/// Allocate a 2^`order`-page block.
///
/// Returns the **relative** page index of the block's first page, or `None`
/// if insufficient contiguous memory remains.
///
/// The returned index can be converted to a physical address:
/// ```text
/// phys = actor.managed_range.0.0 + (idx as u64) * PAGE_SIZE
/// ```
pub fn alloc(&mut self, order: usize) -> Option<usize> {
if order > MAX_ORDER {
return None;
}
// Find the smallest available order ≥ requested.
let found_order = (order..=MAX_ORDER)
.find(|&o| !self.free_lists[o].is_empty())?;
// Consume one block from the found order.
let block_idx = self.free_lists[found_order]
.pop()
.expect("buddy: free_list non-empty but pop() returned None");
// Debit: we removed a 2^found_order block from free.
self.free_pages -= 1 << found_order;
// Split down to the requested order, putting buddies back into free lists.
//
// Invariant at each iteration:
// `block_idx` is the lower half of a 2^cur_order block.
// The upper half (= `block_idx + 2^(cur_order-1)`) is returned to the
// free list as an order-(cur_order-1) block.
let mut cur_order = found_order;
while cur_order > order {
cur_order -= 1;
let buddy_idx = block_idx + (1 << cur_order);
self.free_lists[cur_order].push(buddy_idx);
self.free_pages += 1 << cur_order;
}
// Net accounting: removed 2^found_order, added (2^found_order − 2^order).
// Result: free_pages decreased by exactly 2^order. ✓
Some(block_idx)
}
/// Allocate `page_count` pages, rounding up to the nearest power of two.
///
/// Returns `(relative_page_idx, actual_order)`. The caller **must** pass the
/// same `order` to [`BuddyAllocator::free`] — mismatched orders corrupt the
/// allocator.
///
/// The wasted "rounding up" pages remain unusable until the block is freed.
/// For tightly-packed allocations, callers should use `alloc(order)` directly.
pub fn alloc_pages(&mut self, page_count: usize) -> Option<(usize, usize)> {
if page_count == 0 {
return None;
}
let order = order_for(page_count);
if order > MAX_ORDER {
return None;
}
self.alloc(order).map(|idx| (idx, order))
}
// ─── Deallocation ─────────────────────────────────────────────────────────
/// Return a 2^`order`-page block at **relative** index `block_idx` to the
/// free pool, coalescing with free buddies up the order chain.
///
/// # Panics (debug builds only)
/// - `order > MAX_ORDER`
/// - `block_idx + 2^order > total_pages`
///
/// In release builds the checks are elided for performance; passing incorrect
/// arguments causes undefined bookkeeping, not UB (no unsafe indexing).
pub fn free(&mut self, mut block_idx: usize, mut order: usize) {
debug_assert!(
order <= MAX_ORDER,
"buddy::free: order {} > MAX_ORDER {}",
order,
MAX_ORDER
);
debug_assert!(
block_idx.checked_add(1 << order).map_or(false, |e| e <= self.total_pages),
"buddy::free: block [{}, +{}] out of range (total_pages={})",
block_idx,
1usize << order,
self.total_pages
);
// Try to coalesce with our buddy at each order.
//
// The buddy of a block at relative index `i` of order `k` is at:
// buddy_idx = i XOR 2^k
// This works because aligned buddy pairs always differ in exactly bit k.
while order < MAX_ORDER {
let buddy_idx = block_idx ^ (1 << order);
// Buddy must lie entirely within the managed range.
let buddy_end = match buddy_idx.checked_add(1 << order) {
Some(e) => e,
None => break,
};
if buddy_end > self.total_pages {
break;
}
// Search for the buddy in the free list for this order.
// `swap_remove` is O(1) and order-preserving is not required.
if let Some(pos) = self.free_lists[order].iter().position(|&b| b == buddy_idx) {
self.free_lists[order].swap_remove(pos);
self.free_pages -= 1 << order; // buddy was in free count; remove it
// Merged block starts at the lower address of the two.
block_idx = block_idx.min(buddy_idx);
order += 1;
// Continue trying to merge at the next level.
} else {
break; // buddy is allocated or out-of-range; stop coalescing
}
}
// Push the (possibly merged) block onto the appropriate free list.
self.free_pages += 1 << order;
self.free_lists[order].push(block_idx);
}
// ─── Introspection ────────────────────────────────────────────────────────
/// Number of pages currently available for allocation.
#[inline]
pub fn free_pages(&self) -> usize {
self.free_pages
}
/// Total pages in the managed range.
#[inline]
pub fn total_pages(&self) -> usize {
self.total_pages
}
/// `true` if the allocator has no free pages.
#[inline]
pub fn is_exhausted(&self) -> bool {
self.free_pages == 0
}
/// Dump free-list statistics for each order (useful in panic handlers).
///
/// Returns an array `[(order, free_block_count); MAX_ORDER + 1]`.
pub fn stats(&self) -> [(usize, usize); MAX_ORDER + 1] {
core::array::from_fn(|o| (o, self.free_lists[o].len()))
}
}

38
kernel/src/mem/mod.rs
View File

@@ -1,16 +1,34 @@
pub mod pmm;
pub mod address;
pub mod paging;
pub mod allocator;
//! `src/mem/mod.rs` — memory subsystem root
#[allow(dead_code)]
pub fn init(memmap: &limine::response::MemoryMapResponse, hhdm_offset: u64) {
unsafe {
pmm::BitmapPMM::init(memmap, hhdm_offset);
}
pub mod address;
pub mod allocator;
pub mod buddy; // ← new: per-actor buddy allocator
pub mod paging;
pub mod pm_manages;
pub mod pmm;
pub mod vmm;
/// Initialise the physical memory manager.
///
/// Must be called before any heap allocation or VMM operation.
pub fn init_pmm(memmap: &limine::response::MemoryMapResponse, hhdm_offset: u64) {
unsafe { pmm::BitmapPMM::init(memmap, hhdm_offset); }
}
#[allow(dead_code)]
/// Initialise CPU features required by the VMM (INVPCID detection).
///
/// Must be called before `init_vmm` and any `AddressSpace::activate()`.
pub fn init_cpu_features() {
vmm::init_cpu_features();
}
/// Initialise the kernel address space record (after PMM, heap, and the
/// initial P4 page table have been set up in `kmain`).
pub fn init_vmm(pml4_phys: address::PhysAddr, hhdm_offset: u64) {
vmm::init_kernel_space(pml4_phys, hhdm_offset);
}
/// Physical memory statistics: `(used_pages, total_pages)`.
pub fn get_stats() -> (usize, usize) {
pmm::get_stats()
}

194
kernel/src/mem/paging.rs
View File

@@ -20,13 +20,27 @@ bitflags! {
}
}
/// Mask for the physical frame address stored in a PTE (bits 51:12).
const PTE_ADDR_MASK: u64 = 0x000F_FFFF_FFFF_F000;
#[repr(C, align(4096))]
pub struct PageTable {
entries: [u64; 512],
}
impl PageTable {
pub fn map_region(&mut self, virt: VirtAddr, phys: PhysAddr, size: u64, flags: PageTableFlags, hhdm: u64) {
// ── Bulk mapping ─────────────────────────────────────────────────────────
/// Map a contiguous physical range to a contiguous virtual range.
/// Allocates intermediate page-table pages from the PMM as needed.
pub fn map_region(
&mut self,
virt: VirtAddr,
phys: PhysAddr,
size: u64,
flags: PageTableFlags,
hhdm: u64,
) {
let pages = size.div_ceil(4096);
for i in 0..pages {
let offset = i * 4096;
@@ -34,51 +48,163 @@ impl PageTable {
}
}
/// Loads the page table into the CR3 register
pub unsafe fn activate(&self, phys_addr: PhysAddr) {
unsafe {
asm!(
"mov cr3, {0}",
"jmp 2f",
"2:",
in(reg) phys_addr.0,
options(nostack, preserves_flags)
);
}
}
// ── Single-page operations ────────────────────────────────────────────────
pub fn map_page(&mut self, virt: VirtAddr, phys: PhysAddr, flags: PageTableFlags, hhdm_offset: u64) {
let p4_idx = (virt.0 >> 39) & 0x1FF;
let p3_idx = (virt.0 >> 30) & 0x1FF;
let p2_idx = (virt.0 >> 21) & 0x1FF;
let p1_idx = (virt.0 >> 12) & 0x1FF;
/// Map a single 4 KiB page.
/// Allocates intermediate PT pages from the PMM if they do not exist.
pub fn map_page(
&mut self,
virt: VirtAddr,
phys: PhysAddr,
flags: PageTableFlags,
hhdm: u64,
) {
let p1 = self
.walk_to_p1_mut(virt, hhdm, true /* create */)
.expect("map_page: OOM allocating intermediate page-table pages");
let p3 = self.get_or_create_next_table(p4_idx as usize, hhdm_offset);
let p2 = p3.get_or_create_next_table(p3_idx as usize, hhdm_offset);
let p1 = p2.get_or_create_next_table(p2_idx as usize, hhdm_offset);
let p1_idx = ((virt.0 >> 12) & 0x1FF) as usize;
p1.entries[p1_idx] = phys.0 | flags.bits();
p1.entries[p1_idx as usize] = phys.0 | flags.bits();
unsafe {
asm!("invlpg [{}]", in(reg) virt.0, options(nostack, preserves_flags));
}
}
fn get_or_create_next_table(&mut self, index: usize, hhdm: u64) -> &mut Self {
if self.entries[index] & PageTableFlags::PRESENT.bits() == 0 {
let pt_phys = pmm_alloc().expect("VMM: Out of memory for page tables");
let pt_virt = pt_phys.to_virt(hhdm);
unsafe { core::ptr::write_bytes(pt_virt.as_mut_ptr::<u8>(), 0, 4096); }
self.entries[index] = pt_phys.0 | (PageTableFlags::PRESENT | PageTableFlags::WRITABLE | PageTableFlags::USER).bits();
/// Clear the PTE for `virt` and issue an `INVLPG`.
///
/// Does **not** free the underlying physical frame — that is the caller's
/// responsibility (e.g. `VmaBacking::do_unmap`).
/// Silently returns if any level is not present (idempotent).
pub fn unmap_page(&mut self, virt: VirtAddr, hhdm: u64) {
let Some(p1) = self.walk_to_p1_mut(virt, hhdm, false /* no create */) else {
return; // already absent — nothing to do
};
let p1_idx = ((virt.0 >> 12) & 0x1FF) as usize;
if p1.entries[p1_idx] & PageTableFlags::PRESENT.bits() == 0 {
return;
}
p1.entries[p1_idx] = 0;
unsafe {
asm!("invlpg [{}]", in(reg) virt.0, options(nostack, preserves_flags));
}
}
/// Walk four page-table levels to translate `virt` → physical address.
///
/// Handles 1 GiB and 2 MiB huge pages transparently.
/// Returns `None` if any level is absent or the page is not present.
pub fn translate(&self, virt: VirtAddr, hhdm: u64) -> Option<PhysAddr> {
let p4_idx = ((virt.0 >> 39) & 0x1FF) as usize;
let p3_idx = ((virt.0 >> 30) & 0x1FF) as usize;
let p2_idx = ((virt.0 >> 21) & 0x1FF) as usize;
let p1_idx = ((virt.0 >> 12) & 0x1FF) as usize;
// Helper: safely follow a PTE to the next level.
macro_rules! descend_ref {
($entry:expr) => {{
let e = $entry;
if e & PageTableFlags::PRESENT.bits() == 0 { return None; }
unsafe { &*PhysAddr(e & PTE_ADDR_MASK).to_virt(hhdm).as_mut_ptr::<PageTable>() }
}};
}
let p3 = descend_ref!(self.entries[p4_idx]);
// 1 GiB page (PD pointer with HUGE_PAGE set)
let p3e = p3.entries[p3_idx];
if p3e & PageTableFlags::HUGE_PAGE.bits() != 0 {
let base = p3e & 0x000F_FFFF_C000_0000; // bits 51:30
return Some(PhysAddr(base | (virt.0 & 0x3FFF_FFFF)));
}
let p2 = descend_ref!(p3e);
// 2 MiB page (PD entry with HUGE_PAGE set)
let p2e = p2.entries[p2_idx];
if p2e & PageTableFlags::HUGE_PAGE.bits() != 0 {
let base = p2e & 0x000F_FFFF_FFE0_0000; // bits 51:21
return Some(PhysAddr(base | (virt.0 & 0x001F_FFFF)));
}
let p1 = descend_ref!(p2e);
let p1e = p1.entries[p1_idx];
if p1e & PageTableFlags::PRESENT.bits() == 0 { return None; }
Some(PhysAddr((p1e & PTE_ADDR_MASK) | (virt.0 & 0xFFF)))
}
// ── CR3 ──────────────────────────────────────────────────────────────────
/// Load this page table into CR3 (full TLB flush, no PCID).
///
/// Prefer `AddressSpace::activate` (which uses PCID/INVPCID) for process
/// context switches. Use this only for early boot before PCID is enabled.
pub unsafe fn activate(&self, phys_addr: PhysAddr) {
unsafe {
asm!(
"mov cr3, {0}",
in(reg) phys_addr.0,
options(nostack, preserves_flags),
);
}
let next_pt_phys = PhysAddr(self.entries[index] & 0x000F_FFFF_FFFF_F000);
unsafe { &mut *next_pt_phys.to_virt(hhdm).as_mut_ptr() }
}
}
// ── Private walk helpers ──────────────────────────────────────────────────────
impl PageTable {
/// Walk (or create) the path P4 → P3 → P2 → P1, returning a mutable
/// reference to the P1 (page table, final level).
///
/// If `create` is `false` and any intermediate entry is absent, returns `None`.
/// If `create` is `true`, allocates missing intermediate pages from the PMM.
fn walk_to_p1_mut(&mut self, virt: VirtAddr, hhdm: u64, create: bool)
-> Option<&mut Self>
{
let p4_idx = ((virt.0 >> 39) & 0x1FF) as usize;
let p3_idx = ((virt.0 >> 30) & 0x1FF) as usize;
let p2_idx = ((virt.0 >> 21) & 0x1FF) as usize;
let p3 = self.get_or_create_next_table(p4_idx, hhdm, create)?;
let p2 = p3.get_or_create_next_table(p3_idx, hhdm, create)?;
p2.get_or_create_next_table(p2_idx, hhdm, create)
}
/// Return a mutable reference to the next-level table at `index`.
///
/// - If the entry is present, follows the pointer.
/// - If absent and `create`, allocates a zeroed page and installs it.
/// - If absent and `!create`, returns `None`.
fn get_or_create_next_table(
&mut self,
index: usize,
hhdm: u64,
create: bool,
) -> Option<&mut Self> {
let entry = self.entries[index];
if entry & PageTableFlags::PRESENT.bits() == 0 {
if !create { return None; }
let pt_phys = pmm_alloc().expect("VMM: OOM allocating page-table page");
let pt_virt = pt_phys.to_virt(hhdm);
unsafe { core::ptr::write_bytes(pt_virt.as_mut_ptr::<u8>(), 0, 4096); }
// Install with USER so both kernel and user pages can live under it;
// actual user/kernel distinction is enforced at the P1 level.
self.entries[index] = pt_phys.0
| (PageTableFlags::PRESENT | PageTableFlags::WRITABLE | PageTableFlags::USER).bits();
}
let next_phys = PhysAddr(self.entries[index] & PTE_ADDR_MASK);
Some(unsafe { &mut *next_phys.to_virt(hhdm).as_mut_ptr::<Self>() })
}
}
// ── PMM shim ─────────────────────────────────────────────────────────────────
/// Allocate a single physical frame for page-table use.
/// This thin wrapper avoids a direct dependency cycle between paging ↔ pmm.
pub fn pmm_alloc() -> Option<PhysAddr> {
PMM.lock().as_mut()?.alloc_frame()
}

502
kernel/src/mem/pm_manages.rs
View File

@@ -1,25 +1,491 @@
use crate::cap::{Capability, CapObject, Relation, CapRights};
use crate::mem::address::PhysAddr;
//! # PM Actor (`src/mem/pm_manages.rs`)
//!
//! Distributed physical-memory actor that owns a fixed physical capital range
//! and processes allocation requests asynchronously via a lock-free MPSC queue.
//!
//! ## Key changes vs. previous version
//!
//! 1. **Real allocator**: `PMActor` now contains a `BuddyAllocator` instead of a
//! bare page counter. Allocations produce real relative frame indices that
//! translate to absolute `PhysAddr` values.
//!
//! 2. **Closed response loop**: `process_messages()` returns `Vec<PMResponse>`.
//! The caller (scheduler / boot code) routes each `PMResponse` to the
//! requesting entity identified by `channel_id`. `channel_id = 0` means
//! "discard" and is safe to ignore.
//!
//! 3. **Revised message packing** — fits all fields into a single `u64` without
//! truncating critical identifiers:
//! ```text
//! bits 63:56 OPCODE (8 bits) 1=Alloc, 2=Free, 3=Carve
//! bits 55:40 CHANNEL_ID (16 bits) response routing; 0 = discard
//! bits 39:20 ARG1 (20 bits) size_pages / local_frame_idx / offset_pages
//! bits 19: 0 ARG2 (20 bits) token_sig (low 20 bits) / size_pages
//! ```
//! Maximum ARG size: 2^20 = 1 048 576 pages = 4 GiB per allocation.
//! Maximum CHANNEL_ID: 65 535 simultaneous pending requesters.
//!
//! ## Ownership model
//!
//! `PMActor` holds a `Strong` capability over its entire managed range.
//! Sub-capabilities carved out via `Allocate` or `Carve` are `Strong` children;
//! freeing them (via `Free`) coalesces the buddy tree. The actor **never**
//! touches the global `BitmapPMM` — that belongs to the kernel boot path.
//!
//! ## Ballooning hook
//!
//! When `BuddyAllocator::alloc_pages` returns `None`, the actor emits a
//! `PMResponse::OutOfMemory { channel_id }`. The future ballooning subsystem
//! will intercept this before it reaches the requester and negotiate memory
//! transfer from a neighbouring actor.
pub struct PMActor {
pub root_untyped: Capability,
pub managed_range: (PhysAddr, PhysAddr),
use alloc::vec::Vec;
use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use crate::cap::{CapObject, CapRights, Capability, Relation};
use crate::mem::address::PhysAddr;
use crate::mem::buddy::BuddyAllocator;
use crate::mem::pmm::PAGE_SIZE;
extern crate alloc;
// ══════════════════════════════════════════════════════════════════════════════
// Message packing constants
// ══════════════════════════════════════════════════════════════════════════════
const QUEUE_SIZE: usize = 1024; // power-of-two
const QUEUE_MASK: usize = QUEUE_SIZE - 1;
/// Bit layout of a packed PMRequest u64.
mod packing {
pub const OPCODE_SHIFT: u32 = 56;
pub const CHAN_SHIFT: u32 = 40;
pub const ARG1_SHIFT: u32 = 20;
// ARG2 sits in bits 19:0 (no shift).
pub const CHAN_MASK: u64 = 0x0000_FFFF; // 16 bits
pub const ARG_MASK: u64 = 0x000F_FFFF; // 20 bits
}
impl PMActor {
pub fn carve_region(&self, offset: usize, size: usize) -> Capability {
if let CapObject::Memory { phys, .. } = self.root_untyped.object {
Capability {
object: CapObject::Memory {
phys: PhysAddr(phys.0 + offset as u64),
size_pages: size / 4096,
},
rights: CapRights::READ | CapRights::WRITE | CapRights::GRANT,
relation: Relation::Strong,
token_sig: 0xDEAD_BEEF,
// ══════════════════════════════════════════════════════════════════════════════
// Request type
// ══════════════════════════════════════════════════════════════════════════════
/// Asynchronous request submitted to a `PMActor` inbox.
///
/// `channel_id` identifies where the response should be routed.
/// Use `channel_id = 0` if you do not need a reply.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PMRequest {
/// Allocate `size_pages` pages; tag the resulting capability with `token_sig`.
Allocate {
size_pages: usize,
token_sig: u32,
channel_id: u16,
},
/// Free a previously allocated block.
///
/// `local_frame_idx` is the **relative** page index returned inside the
/// `Allocated` capability (i.e. `cap.object.phys - managed_range.start`).
/// `order` is the buddy order used at allocation time (stored in the cap).
Free {
local_frame_idx: usize,
order: usize,
},
/// Carve a fixed sub-region without consulting the buddy allocator.
/// Used for static layout decisions made at boot (e.g. framebuffer alias).
Carve {
offset_pages: usize,
size_pages: usize,
channel_id: u16,
},
/// Sentinel — never pushed onto the queue; result of unpack(0).
None,
}
impl PMRequest {
/// Encode the request into one `u64` atom for the lock-free buffer.
///
/// Opcode 0 (None) is reserved as the "slot empty" sentinel; `None` returns
/// 0 intentionally and the sender rejects it before pushing.
#[inline]
pub fn pack(self) -> u64 {
use packing::*;
match self {
Self::Allocate { size_pages, token_sig, channel_id } => {
(1u64 << OPCODE_SHIFT)
| ((channel_id as u64 & CHAN_MASK) << CHAN_SHIFT)
| ((size_pages as u64 & ARG_MASK) << ARG1_SHIFT)
| (token_sig as u64 & ARG_MASK)
}
} else {
panic!("PM: Root is not memory!");
Self::Free { local_frame_idx, order } => {
(2u64 << OPCODE_SHIFT)
// channel_id not needed — Free has no reply
| ((local_frame_idx as u64 & ARG_MASK) << ARG1_SHIFT)
| (order as u64 & ARG_MASK)
}
Self::Carve { offset_pages, size_pages, channel_id } => {
(3u64 << OPCODE_SHIFT)
| ((channel_id as u64 & CHAN_MASK) << CHAN_SHIFT)
| ((offset_pages as u64 & ARG_MASK) << ARG1_SHIFT)
| (size_pages as u64 & ARG_MASK)
}
Self::None => 0,
}
}
/// Decode a packed `u64` back into a `PMRequest`.
#[inline]
pub fn unpack(val: u64) -> Self {
use packing::*;
let opcode = val >> OPCODE_SHIFT;
let channel_id = ((val >> CHAN_SHIFT) & CHAN_MASK) as u16;
let arg1 = ((val >> ARG1_SHIFT) & ARG_MASK) as usize;
let arg2 = (val & ARG_MASK) as usize;
match opcode {
1 => Self::Allocate {
size_pages: arg1,
token_sig: arg2 as u32,
channel_id,
},
2 => Self::Free {
local_frame_idx: arg1,
order: arg2,
},
3 => Self::Carve {
offset_pages: arg1,
size_pages: arg2,
channel_id,
},
_ => Self::None,
}
}
}
// ══════════════════════════════════════════════════════════════════════════════
// Response type
// ══════════════════════════════════════════════════════════════════════════════
/// Result returned by `PMActor::process_messages()` for each completed request.
#[derive(Debug, Clone, Copy)]
pub struct PMResponse {
/// Channel to route this response to. `0` = discard.
pub channel_id: u16,
/// The actual outcome.
pub result: PMResult,
}
/// Outcome of a single PM operation.
#[derive(Debug, Clone, Copy)]
pub enum PMResult {
/// Memory was allocated. `cap` is the strong capability to the region.
/// `order` is the buddy order — **must** be passed back to `Free`.
Allocated { cap: Capability, order: usize },
/// The actor had insufficient free pages.
/// Future: ballooning subsystem intercepts this and retries.
OutOfMemory { size_pages: usize },
/// A fixed sub-region was carved successfully.
Carved { cap: Capability },
/// Free completed (no capability issued — memory returned to buddy pool).
Freed { pages_returned: usize },
}
// ══════════════════════════════════════════════════════════════════════════════
// Lock-free MPSC inbox
// ══════════════════════════════════════════════════════════════════════════════
/// MPSC ring buffer for PMRequest values.
///
/// Producers (any core, any context) call `send`; the owning PMActor calls
/// `pop` — exactly one consumer, no locking on the read side.
pub struct PMActorQueue {
buffer: [AtomicU64; QUEUE_SIZE],
/// Pad to separate producer-written `tail` from consumer-read `head`.
_pad0: [u8; 64],
head: AtomicUsize,
_pad1: [u8; 64],
tail: AtomicUsize,
}
impl PMActorQueue {
pub const fn new() -> Self {
Self {
#[allow(clippy::declare_interior_mutable_const)]
buffer: [const { AtomicU64::new(0) }; QUEUE_SIZE],
_pad0: [0u8; 64],
head: AtomicUsize::new(0),
_pad1: [0u8; 64],
tail: AtomicUsize::new(0),
}
}
/// Push a request into the inbox (Multi-Producer path).
///
/// Returns `Err` only if the ring buffer is full — which should trigger
/// backpressure or a ballooning request in production.
pub fn send(&self, req: PMRequest) -> Result<(), &'static str> {
let packed = req.pack();
if packed == 0 {
return Err("PMActorQueue::send: PMRequest::None is not sendable");
}
let mut tail = self.tail.load(Ordering::Relaxed);
loop {
let head = self.head.load(Ordering::Acquire);
if tail.wrapping_sub(head) >= QUEUE_SIZE {
return Err("PMActorQueue overflow — apply backpressure or balloon");
}
match self.tail.compare_exchange_weak(
tail,
tail.wrapping_add(1),
Ordering::AcqRel,
Ordering::Relaxed,
) {
Ok(_) => {
self.buffer[tail & QUEUE_MASK].store(packed, Ordering::Release);
return Ok(());
}
Err(actual) => tail = actual,
}
}
}
/// Pop one request (Single-Consumer path — actor's own context only).
pub fn pop(&self) -> Option<PMRequest> {
let head = self.head.load(Ordering::Relaxed);
let tail = self.tail.load(Ordering::Acquire);
if head == tail {
return None;
}
let packed = self.buffer[head & QUEUE_MASK].load(Ordering::Acquire);
self.head.store(head.wrapping_add(1), Ordering::Release);
Some(PMRequest::unpack(packed))
}
}
// ══════════════════════════════════════════════════════════════════════════════
// PM Actor
// ══════════════════════════════════════════════════════════════════════════════
/// An autonomous physical-memory actor.
///
/// Owns a `BuddyAllocator` over its capital range and a lock-free MPSC inbox.
/// The actor must be driven externally: when the scheduler grants it CPU time,
/// it calls `process_messages()` to drain the inbox and produce `PMResponse`s.
///
/// # Single-consumer guarantee
/// Only `process_messages()` ever mutates `buddy` and `queue.head`.
/// This is enforced by taking `&mut self` on `process_messages`.
pub struct PMActor {
/// Unique identity within the actor federation.
pub actor_id: u64,
/// Root strong capability over the entire managed physical range.
pub root_untyped: Capability,
/// `(inclusive_start, exclusive_end)` physical addresses.
pub managed_range: (PhysAddr, PhysAddr),
/// Inbox — producers write here, actor reads here.
queue: PMActorQueue,
/// Local buddy allocator. Only ever touched in `process_messages`.
buddy: BuddyAllocator,
}
impl PMActor {
/// Construct a new actor managing `size_bytes` bytes starting at `start`.
///
/// `size_bytes` must be a multiple of `PAGE_SIZE`; non-multiples are
/// rounded down silently.
pub fn new(
actor_id: u64,
root_cap: Capability,
start: PhysAddr,
size_bytes: u64,
) -> Self {
let total_pages = (size_bytes / PAGE_SIZE) as usize;
Self {
actor_id,
root_untyped: root_cap,
managed_range: (start, PhysAddr(start.0 + total_pages as u64 * PAGE_SIZE)),
queue: PMActorQueue::new(),
buddy: BuddyAllocator::new(total_pages),
}
}
// ─── Producer API (callable from any context) ──────────────────────────
/// Submit a request to this actor's inbox.
///
/// Thread-safe (MPSC producer side). Returns `Err` if the inbox is full.
#[inline]
pub fn submit_request(&self, req: PMRequest) -> Result<(), &'static str> {
self.queue.send(req)
}
// ─── Consumer API (actor's own scheduled context) ─────────────────────
/// Drain the inbox and execute all pending requests.
///
/// Returns a `Vec` of responses that the caller (scheduler) must route to
/// the appropriate process or actor mailbox based on `channel_id`.
///
/// Responses with `channel_id == 0` can be discarded.
///
/// # No locks held
/// `buddy` and `queue.head` are mutated — no other thread touches them.
/// The only shared state is the queue's `tail`, which is written by producers
/// via `AtomicUsize::compare_exchange_weak`, never by this path.
pub fn process_messages(&mut self) -> Vec<PMResponse> {
let mut responses = Vec::new();
while let Some(req) = self.queue.pop() {
let resp = match req {
PMRequest::Allocate { size_pages, token_sig, channel_id } => {
self.handle_allocate(size_pages, token_sig, channel_id)
}
PMRequest::Free { local_frame_idx, order } => {
self.handle_free(local_frame_idx, order)
}
PMRequest::Carve { offset_pages, size_pages, channel_id } => {
self.handle_carve(offset_pages, size_pages, channel_id)
}
PMRequest::None => continue,
};
// Only push responses that need routing.
// Free responses (channel_id == 0) are still pushed so callers can
// audit completion if needed; they may simply drop them.
if let Some(r) = resp {
responses.push(r);
}
}
responses
}
// ─── Introspection ────────────────────────────────────────────────────
/// Free pages remaining in this actor's buddy pool.
#[inline]
pub fn free_pages(&self) -> usize {
self.buddy.free_pages()
}
/// Total pages this actor was initialised with.
#[inline]
pub fn total_pages(&self) -> usize {
self.buddy.total_pages()
}
}
// ── Private handler implementations ──────────────────────────────────────────
impl PMActor {
fn handle_allocate(
&mut self,
size_pages: usize,
token_sig: u32,
channel_id: u16,
) -> Option<PMResponse> {
match self.buddy.alloc_pages(size_pages) {
Some((rel_idx, order)) => {
// Translate relative index → absolute physical address.
let phys = PhysAddr(self.managed_range.0.0 + rel_idx as u64 * PAGE_SIZE);
// Build a strong child capability.
// token_sig is widened and XOR'd with actor_id for uniqueness.
let cap = Capability {
object: CapObject::Memory {
phys,
// Report actual (rounded-up) page count so the caller
// passes the correct `order` back to Free.
size_pages: 1 << order,
},
rights: CapRights::READ | CapRights::WRITE | CapRights::GRANT,
relation: Relation::Strong,
token_sig: (token_sig as u64) ^ self.actor_id ^ (rel_idx as u64),
};
Some(PMResponse {
channel_id,
result: PMResult::Allocated { cap, order },
})
}
None => {
// TODO: emit a ballooning request to a neighbour actor before
// propagating OutOfMemory upward.
Some(PMResponse {
channel_id,
result: PMResult::OutOfMemory { size_pages },
})
}
}
}
fn handle_free(
&mut self,
local_frame_idx: usize,
order: usize,
) -> Option<PMResponse> {
// Validate bounds before touching the buddy tree.
let block_end = local_frame_idx.saturating_add(1usize << order);
if block_end > self.buddy.total_pages() {
// Corrupted or spoofed index — ignore silently in release, panic in debug.
debug_assert!(
false,
"PMActor {}: Free out of range (idx={}, order={}, total={})",
self.actor_id,
local_frame_idx,
order,
self.buddy.total_pages()
);
return None;
}
self.buddy.free(local_frame_idx, order);
// Free never needs routing — return None to skip Vec push, or push with
// channel_id=0 for audit purposes. We skip to avoid unnecessary allocation.
// Callers that need a Free-complete signal should use a separate mechanism.
None
}
fn handle_carve(
&mut self,
offset_pages: usize,
size_pages: usize,
channel_id: u16,
) -> Option<PMResponse> {
// Validate offset + size within range.
let range_pages = self.buddy.total_pages();
if offset_pages >= range_pages
|| size_pages == 0
|| offset_pages.saturating_add(size_pages) > range_pages
{
return Some(PMResponse {
channel_id,
result: PMResult::OutOfMemory { size_pages },
});
}
let phys = PhysAddr(self.managed_range.0.0 + offset_pages as u64 * PAGE_SIZE);
let cap = Capability {
object: CapObject::Memory { phys, size_pages },
rights: CapRights::READ | CapRights::WRITE | CapRights::GRANT,
relation: Relation::Strong,
token_sig: self.actor_id ^ offset_pages as u64,
};
Some(PMResponse {
channel_id,
result: PMResult::Carved { cap },
})
}
}

161
kernel/src/mem/pmm.rs
View File

@@ -4,19 +4,22 @@ use crate::mem::allocator::Locked;
pub const PAGE_SIZE: u64 = 4096;
pub struct BitmapPMM {
bitmap: &'static mut [u8],
bitmap: &'static mut [u8],
total_pages: usize,
used_pages: usize,
last_idx: usize,
used_pages: usize,
/// Byte index hint: next search starts here to amortise O(N) scans.
last_byte: usize,
}
pub static PMM: Locked<Option<BitmapPMM>> = Locked::new(None);
impl BitmapPMM {
#[allow(dead_code)]
pub fn used_pages(&self) -> usize { self.used_pages }
pub fn used_pages(&self) -> usize { self.used_pages }
#[allow(dead_code)]
pub fn total_pages(&self) -> usize { self.total_pages }
#[allow(dead_code)]
pub fn free_pages(&self) -> usize { self.total_pages.saturating_sub(self.used_pages) }
pub unsafe fn init(mmap: &limine::response::MemoryMapResponse, hhdm_offset: u64) {
let max_addr = mmap.entries().iter()
@@ -27,22 +30,28 @@ impl BitmapPMM {
let total_pages = (max_addr / PAGE_SIZE) as usize;
let bitmap_size = total_pages.div_ceil(8);
let bitmap_phys_addr = mmap.entries().iter()
.find(|e| e.entry_type == limine::memory_map::EntryType::USABLE && e.length >= bitmap_size as u64)
// Find a usable region large enough to hold the bitmap.
let bitmap_phys = mmap.entries().iter()
.find(|e| {
e.entry_type == limine::memory_map::EntryType::USABLE
&& e.length >= bitmap_size as u64
})
.map(|e| e.base)
.expect("PMM: Insufficient memory for bitmap");
.expect("PMM: no usable region large enough for the bitmap");
let bitmap_virt_ptr = (bitmap_phys_addr + hhdm_offset) as *mut u8;
let bitmap_slice = unsafe { core::slice::from_raw_parts_mut(bitmap_virt_ptr, bitmap_size) };
bitmap_slice.fill(0xFF);
let bitmap_ptr = (bitmap_phys + hhdm_offset) as *mut u8;
// Mark everything as used (all bits = 1) and free usable entries below.
let bitmap = unsafe { core::slice::from_raw_parts_mut(bitmap_ptr, bitmap_size) };
bitmap.fill(0xFF);
let mut pmm = Self {
bitmap: bitmap_slice,
bitmap,
total_pages,
used_pages: total_pages,
last_idx: 0,
last_byte: 0,
};
// Free all usable pages …
for entry in mmap.entries() {
if entry.entry_type == limine::memory_map::EntryType::USABLE {
for addr in (entry.base..entry.base + entry.length).step_by(PAGE_SIZE as usize) {
@@ -51,71 +60,131 @@ impl BitmapPMM {
}
}
for addr in (bitmap_phys_addr..bitmap_phys_addr + bitmap_size as u64).step_by(PAGE_SIZE as usize) {
// … then re-lock the bitmap pages themselves …
for addr in (bitmap_phys..bitmap_phys + bitmap_size as u64).step_by(PAGE_SIZE as usize) {
pmm.lock_frame(PhysAddr(addr));
}
// … and the null page (physical 0x0 must never be returned as a valid frame).
pmm.lock_frame(PhysAddr(0));
*PMM.lock() = Some(pmm);
*PMM.lock() = Some(pmm);
}
// ── Core operations ───────────────────────────────────────────────────────
/// Mark a frame as free. Idempotent (double-free is a no-op, not UB).
pub fn free_frame(&mut self, phys_addr: PhysAddr) {
let idx = (phys_addr.0 / PAGE_SIZE) as usize;
if idx < self.total_pages {
let byte_idx = idx / 8;
let bit_idx = idx % 8;
if (self.bitmap[byte_idx] & (1 << bit_idx)) != 0 {
self.bitmap[byte_idx] &= !(1 << bit_idx);
self.used_pages -= 1;
}
if idx >= self.total_pages { return; }
let byte = idx / 8;
let bit = idx % 8;
if self.bitmap[byte] & (1 << bit) != 0 {
self.bitmap[byte] &= !(1 << bit);
self.used_pages -= 1;
// Pull the hint back so the freed page can be found quickly.
if byte < self.last_byte { self.last_byte = byte; }
}
}
/// Mark a frame as allocated (reserved). Idempotent.
pub fn lock_frame(&mut self, phys_addr: PhysAddr) {
let idx = (phys_addr.0 / PAGE_SIZE) as usize;
if idx < self.total_pages {
let byte_idx = idx / 8;
let bit_idx = idx % 8;
if (self.bitmap[byte_idx] & (1 << bit_idx)) == 0 {
self.bitmap[byte_idx] |= 1 << bit_idx;
self.used_pages += 1;
}
if idx >= self.total_pages { return; }
let byte = idx / 8;
let bit = idx % 8;
if self.bitmap[byte] & (1 << bit) == 0 {
self.bitmap[byte] |= 1 << bit;
self.used_pages += 1;
}
}
pub fn alloc_frame(&mut self) -> Option<PhysAddr> {
let start_byte = self.last_idx / 8;
for i in start_byte..(self.bitmap.len()) {
if self.bitmap[i] != 0xFF {
for bit in 0..8 {
let idx = i * 8 + bit;
if idx >= self.total_pages { return None; }
let addr = PhysAddr(idx as u64 * PAGE_SIZE);
if !self.is_locked(addr) {
self.lock_frame(addr);
self.last_idx = idx;
return Some(addr);
/// Allocate one physical frame.
///
/// Uses a two-pass search (linear scan from `last_byte` hint, then wraps
/// to 0 if not found in the first pass) to avoid returning `None` when
/// free frames exist before the hint.
pub fn alloc_frame(&mut self) -> Option<PhysAddr> {
let len = self.bitmap.len();
for pass in 0..2usize {
let (from, to) = if pass == 0 {
(self.last_byte, len)
} else {
(0, self.last_byte)
};
for byte_idx in from..to {
// Fast path: skip fully-used bytes.
if self.bitmap[byte_idx] == 0xFF { continue; }
for bit in 0..8u8 {
if self.bitmap[byte_idx] & (1 << bit) == 0 {
let page_idx = byte_idx * 8 + bit as usize;
if page_idx >= self.total_pages { return None; }
// Mark allocated.
self.bitmap[byte_idx] |= 1 << bit;
self.used_pages += 1;
self.last_byte = byte_idx;
return Some(PhysAddr(page_idx as u64 * PAGE_SIZE));
}
}
}
}
None
None // genuinely out of memory
}
#[inline]
fn is_locked(&self, addr: PhysAddr) -> bool {
let idx = (addr.0 / PAGE_SIZE) as usize;
(self.bitmap[idx / 8] & (1 << (idx % 8))) != 0
/// Try to allocate `count` **contiguous** physical frames.
///
/// Returns the base physical address of the run, or `None` if no run of
/// sufficient length exists. This is needed for (e.g.) allocating 2 MiB
/// huge-page aligned regions or DMA buffers that must be physically
/// contiguous.
///
/// O(N) worst-case; use sparingly and prefer small counts.
pub fn alloc_contiguous(&mut self, count: usize) -> Option<PhysAddr> {
if count == 0 { return None; }
let mut run_start = 0usize;
let mut run_len = 0usize;
for page_idx in 0..self.total_pages {
let byte = page_idx / 8;
let bit = page_idx % 8;
if self.bitmap[byte] & (1 << bit) == 0 {
if run_len == 0 { run_start = page_idx; }
run_len += 1;
if run_len == count {
// Lock every frame in the run.
for i in run_start..run_start + count {
self.bitmap[i / 8] |= 1 << (i % 8);
}
self.used_pages += count;
self.last_byte = run_start / 8;
return Some(PhysAddr(run_start as u64 * PAGE_SIZE));
}
} else {
run_len = 0;
}
}
None
}
}
// ── Module-level convenience functions ───────────────────────────────────────
pub fn alloc_frame() -> Option<PhysAddr> {
PMM.lock().as_mut()?.alloc_frame()
}
pub fn alloc_contiguous(count: usize) -> Option<PhysAddr> {
PMM.lock().as_mut()?.alloc_contiguous(count)
}
pub fn free_frame(addr: PhysAddr) {
if let Some(pmm) = PMM.lock().as_mut() {
pmm.free_frame(addr);

730
kernel/src/mem/vmm.rs Normal file
View File

@@ -0,0 +1,730 @@
//! # Virtual Memory Manager (`src/mem/vmm.rs`)
//!
//! Manages address spaces (PML4 + VMA list) and provides the bridge between
//! the capability system and the hardware MMU.
//!
//! ## Changes vs. previous version
//!
//! 1. **ASID recycling** — replaced the monotonic `AtomicU16` with a 4096-bit
//! bitmap allocator (`AsidAllocator`). `AddressSpace::drop` returns the ASID
//! to the pool, making it available for future processes. The bitmap is
//! `const`-initializable (no heap), so no boot-ordering issues.
//!
//! 2. **INVPCID CPUID guard** — `init_cpu_features()` must be called once during
//! boot (before any `AddressSpace::activate()` or TLB flush). It checks
//! CPUID.07H:EBX[10] and sets `INVPCID_SUPPORTED`. `tlb_flush_asid` falls
//! back to a full CR3 reload on CPUs that lack the instruction.
//!
//! 3. **Zero-copy shared mappings** — new `AddressSpace::map_shared()` maps a
//! physical range owned by another capability into this address space without
//! copying the frames. `VmaBacking::Shared` stores the `phys_base` so the
//! physical address is retrievable at any time via `translate()` on the PT.
//! Frames are *not* freed on unmap — ownership stays with the issuing actor.
//!
//! All other behaviour (lazy demand paging, revocation, TLB management) is
//! unchanged from the previous version.
#![allow(dead_code)]
use alloc::vec::Vec;
use core::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use crate::mem::address::{PhysAddr, VirtAddr};
use crate::mem::allocator::Locked;
use crate::mem::paging::{PageTable, PageTableFlags};
use crate::mem::pmm;
use crate::events::MMU_REVOCATION_QUEUE;
extern crate alloc;
// ══════════════════════════════════════════════════════════════════════════════
// Error type
// ══════════════════════════════════════════════════════════════════════════════
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VmError {
/// PMM returned `None` — no physical frames available.
OutOfMemory,
/// The requested virtual range overlaps an existing VMA.
RegionOverlap,
/// No VMA covers the given address.
RegionNotFound,
/// Address or size is not a multiple of 4096.
InvalidAlignment,
/// Size is zero, or `virt + size` would overflow.
InvalidRange,
/// Write fault on a read-only VMA, or exec fault on a NX VMA.
PermissionDenied,
/// Page-fault in a non-lazy (already-eager or fixed) region — hardware bug
/// or an exploit attempt; the faulting task must be killed.
UnexpectedFault,
/// Address outside the x86-64 canonical range.
NonCanonical,
/// ASID pool exhausted (> 4094 simultaneous address spaces).
AsidExhausted,
}
impl core::fmt::Display for VmError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.write_str(match self {
Self::OutOfMemory => "physical memory exhausted",
Self::RegionOverlap => "virtual address range overlaps existing VMA",
Self::RegionNotFound => "no VMA at address",
Self::InvalidAlignment => "address/size not page-aligned",
Self::InvalidRange => "zero-size or overflowing range",
Self::PermissionDenied => "VMA permission denied",
Self::UnexpectedFault => "page fault in non-lazy region",
Self::NonCanonical => "non-canonical virtual address",
Self::AsidExhausted => "ASID/PCID pool exhausted",
})
}
}
// ══════════════════════════════════════════════════════════════════════════════
// VMA flags
// ══════════════════════════════════════════════════════════════════════════════
bitflags::bitflags! {
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct VmaFlags: u32 {
const READ = 1 << 0;
const WRITE = 1 << 1;
const EXEC = 1 << 2;
const USER = 1 << 3;
const LAZY = 1 << 4;
const SHARED = 1 << 5;
const PINNED = 1 << 6;
const NOCACHE = 1 << 7;
const MMIO = 1 << 8;
}
}
impl VmaFlags {
#[inline]
pub fn to_page_flags(self) -> PageTableFlags {
let mut f = PageTableFlags::PRESENT;
if self.contains(Self::WRITE) { f |= PageTableFlags::WRITABLE; }
if self.contains(Self::USER) { f |= PageTableFlags::USER; }
if !self.contains(Self::EXEC) { f |= PageTableFlags::NO_EXECUTE; }
if self.contains(Self::NOCACHE) || self.contains(Self::MMIO) {
f |= PageTableFlags::NO_CACHE | PageTableFlags::WRITE_THROUGH;
}
f
}
}
// ══════════════════════════════════════════════════════════════════════════════
// VMA backing
// ══════════════════════════════════════════════════════════════════════════════
#[derive(Debug)]
pub enum VmaBacking {
/// Anonymous pages (stack, heap, BSS).
/// Index `i` → frame for `virt_start + i * 4096`. `None` = not yet faulted in.
Anonymous(Vec<Option<PhysAddr>>),
/// Fixed physical range. Frames are **not** freed on unmap.
/// Used for MMIO, identity-mapped RAM, framebuffer, DMA buffers.
Physical(PhysAddr),
/// Zero-copy borrow of another actor's frames.
///
/// `phys_base` is the physical address of the first page; the mapping covers
/// exactly `(virt_end - virt_start) / 4096` pages.
/// Frames are owned by `owner_cap` and **never** freed by this VMA.
Shared {
/// Token of the capability that owns the frames.
owner_cap: u64,
/// Physical base (first page of the shared region).
phys_base: PhysAddr,
},
}
impl VmaBacking {
#[inline]
fn phys_for_page(&self, page_idx: usize) -> Option<PhysAddr> {
match self {
Self::Anonymous(frames) => frames.get(page_idx).copied().flatten(),
Self::Physical(base) => Some(PhysAddr(base.0 + page_idx as u64 * 4096)),
Self::Shared { phys_base, .. } => {
Some(PhysAddr(phys_base.0 + page_idx as u64 * 4096))
}
}
}
/// Whether this backing owns its frames (should the PMM free them on unmap).
#[inline]
fn owns_frames(&self) -> bool {
matches!(self, Self::Anonymous(_))
}
}
// ══════════════════════════════════════════════════════════════════════════════
// VMA region
// ══════════════════════════════════════════════════════════════════════════════
#[derive(Debug)]
pub struct VmaRegion {
pub virt_start: VirtAddr,
pub virt_end: VirtAddr,
pub flags: VmaFlags,
pub cap_token: u64,
pub backing: VmaBacking,
}
impl VmaRegion {
#[inline] pub fn size(&self) -> u64 { self.virt_end.0 - self.virt_start.0 }
#[inline] pub fn pages(&self) -> usize { (self.size() / 4096) as usize }
#[inline] pub fn contains(&self, a: VirtAddr) -> bool {
a.0 >= self.virt_start.0 && a.0 < self.virt_end.0
}
}
// ══════════════════════════════════════════════════════════════════════════════
// ASID / PCID allocator (bitmap-based, const-initializable, O(1) amortised)
// ══════════════════════════════════════════════════════════════════════════════
/// x86-64 PCIDs: 0 (kernel, no PCID tagging) and 4095 (reserved by spec).
/// Valid user ASIDs: 1 – 4094 inclusive.
///
/// Layout: `bitmap[i]` bit `j` represents ASID `i * 32 + j`.
/// Bit set = in use.
struct AsidAllocator {
/// 4096 bits in 128 × u32 words.
bitmap: [u32; 128],
/// Start-of-next-search hint (in ASID units, not word units).
next_hint: u16,
}
impl AsidAllocator {
const fn new() -> Self {
let mut bm = [0u32; 128];
// Mark ASID 0 (kernel) as permanently in-use.
bm[0] |= 1u32;
// Mark ASID 4095 (reserved by Intel spec) as permanently in-use.
// 4095 / 32 = 127, bit 31.
bm[127] |= 1u32 << 31;
Self { bitmap: bm, next_hint: 1 }
}
/// Allocate the lowest free ASID in [1, 4094]. O(1) amortised with hint.
fn alloc(&mut self) -> Option<u16> {
// Two-pass: start from hint, wrap around if necessary.
for pass in 0..2usize {
let start = if pass == 0 { self.next_hint as usize } else { 1 };
let end = if pass == 0 { 4095usize } else { self.next_hint as usize };
let mut asid = start;
while asid < end {
let word = asid / 32;
let bit = asid % 32;
if self.bitmap[word] & (1u32 << bit) == 0 {
self.bitmap[word] |= 1u32 << bit;
self.next_hint = ((asid + 1) as u16).min(4094);
return Some(asid as u16);
}
// Skip fully-used 32-bit words for amortised O(1).
if self.bitmap[word] == u32::MAX {
asid = (word + 1) * 32;
} else {
asid += 1;
}
}
}
None // pool truly exhausted
}
/// Return an ASID to the pool.
fn free(&mut self, asid: u16) {
if asid == 0 || asid >= 4095 {
return; // sentinel values — never freed
}
let word = asid as usize / 32;
let bit = asid as usize % 32;
// Clear the bit. Idempotent: double-free is safe but wasteful.
self.bitmap[word] &= !(1u32 << bit);
// Pull hint back so reuse happens quickly.
if asid < self.next_hint {
self.next_hint = asid;
}
}
}
static ASID_ALLOC: Locked<AsidAllocator> = Locked::new(AsidAllocator::new());
#[inline]
fn alloc_asid() -> Result<u16, VmError> {
ASID_ALLOC.lock().alloc().ok_or(VmError::AsidExhausted)
}
#[inline]
fn free_asid(asid: u16) {
ASID_ALLOC.lock().free(asid);
}
// ══════════════════════════════════════════════════════════════════════════════
// CPU feature detection (call once during boot, before first activate())
// ══════════════════════════════════════════════════════════════════════════════
/// Set to `true` at boot if CPUID.07H:EBX[10] = 1 (INVPCID supported).
static INVPCID_SUPPORTED: AtomicBool = AtomicBool::new(false);
/// Detect and record the availability of INVPCID.
///
/// Must be called **once** during kernel init (before `init_kernel_space` or
/// any `AddressSpace::activate`). Idempotent if called more than once.
pub fn init_cpu_features() {
// CPUID leaf 7, sub-leaf 0 — INVPCID is EBX[10].
//
// LLVM permanently reserves RBX for its own use in inline asm, so we
// cannot name it as an operand directly. The standard workaround is to
// spill RBX around the CPUID instruction ourselves.
let ebx: u32;
unsafe {
core::arch::asm!(
// Spill rbx (LLVM's reserved reg) to a caller-saved scratch reg.
"mov {tmp:r}, rbx",
"mov eax, 7",
"xor ecx, ecx",
"cpuid",
// Move the result out before restoring rbx.
"mov {out:e}, ebx",
"mov rbx, {tmp:r}",
tmp = out(reg) _, // any scratch register chosen by LLVM
out = out(reg) ebx,
out("eax") _,
out("ecx") _,
out("edx") _,
options(nostack, preserves_flags),
);
}
let supported = (ebx >> 10) & 1 == 1;
INVPCID_SUPPORTED.store(supported, Ordering::Relaxed);
}
// ══════════════════════════════════════════════════════════════════════════════
// Address space
// ══════════════════════════════════════════════════════════════════════════════
pub struct AddressSpace {
/// Hardware PCID written to CR3 bits 11:0.
pub asid: u16,
/// Physical address of the PML4 page.
pub pml4_phys: PhysAddr,
/// Sorted (by virt_start), non-overlapping VMA list.
regions: Vec<VmaRegion>,
/// HHDM offset for dereferencing page-table pages.
hhdm: u64,
}
// ── Private helpers ───────────────────────────────────────────────────────────
impl AddressSpace {
#[inline]
unsafe fn pml4_raw(&self) -> *mut PageTable {
self.pml4_phys.to_virt(self.hhdm).as_mut_ptr::<PageTable>()
}
fn check_overlap(&self, start: VirtAddr, end: VirtAddr) -> Result<(), VmError> {
for r in &self.regions {
if r.virt_start.0 < end.0 && r.virt_end.0 > start.0 {
return Err(VmError::RegionOverlap);
}
}
Ok(())
}
fn insert_sorted(&mut self, region: VmaRegion) {
let pos = self.regions
.partition_point(|r| r.virt_start.0 < region.virt_start.0);
self.regions.insert(pos, region);
}
fn find_idx(&self, addr: VirtAddr) -> Option<usize> {
let pos = self.regions.partition_point(|r| r.virt_end.0 <= addr.0);
self.regions.get(pos).filter(|r| r.contains(addr)).map(|_| pos)
}
/// Unmap pages + free frames (if owned) for one region.
fn do_unmap(pml4: *mut PageTable, region: &VmaRegion, hhdm: u64) {
let pml4 = unsafe { &mut *pml4 };
let own = region.backing.owns_frames();
for i in 0..region.pages() {
let virt = VirtAddr(region.virt_start.0 + i as u64 * 4096);
pml4.unmap_page(virt, hhdm);
if own {
if let Some(frame) = region.backing.phys_for_page(i) {
pmm::free_frame(frame);
}
}
}
}
fn do_revoke_by_token(&mut self, cap_token: u64) {
let hhdm = self.hhdm;
let pml4 = unsafe { self.pml4_raw() };
let indices: Vec<usize> = self.regions
.iter()
.enumerate()
.filter(|(_, r)| r.cap_token == cap_token && !r.flags.contains(VmaFlags::PINNED))
.map(|(i, _)| i)
.collect();
for idx in indices.into_iter().rev() {
let region = self.regions.remove(idx);
Self::do_unmap(pml4, &region, hhdm);
}
}
/// Drain the MMU revocation queue and apply all pending token revocations.
/// Called by the page-fault handler before acquiring the VMM lock to
/// prevent deadlocks with the capability subsystem.
pub fn process_pending_revocations(&mut self) {
let mut needs_flush = false;
while let Some(token) = MMU_REVOCATION_QUEUE.pop() {
self.do_revoke_by_token(token);
needs_flush = true;
}
if needs_flush {
tlb_flush_asid(self.asid);
}
}
}
// ── Public API ────────────────────────────────────────────────────────────────
impl AddressSpace {
// ─── Construction ──────────────────────────────────────────────────────
/// Allocate a fresh, empty address space with a zeroed PML4.
pub fn new(hhdm: u64) -> Result<Self, VmError> {
let pml4_phys = pmm::alloc_frame().ok_or(VmError::OutOfMemory)?;
unsafe {
core::ptr::write_bytes(pml4_phys.to_virt(hhdm).as_mut_ptr::<u8>(), 0, 4096);
}
Ok(Self {
asid: alloc_asid()?,
pml4_phys,
regions: Vec::new(),
hhdm,
})
}
/// Wrap an already-active PML4 (bootloader-provided kernel table).
/// ASID 0 = kernel "no PCID tagging"; it is never returned to the pool.
pub fn from_active(pml4_phys: PhysAddr, asid: u16, hhdm: u64) -> Self {
Self { asid, pml4_phys, regions: Vec::new(), hhdm }
}
// ─── Anonymous / fixed mapping ─────────────────────────────────────────
/// Map `size` bytes of virtual space starting at `virt`.
///
/// | `phys` | `VmaFlags::LAZY` | Behaviour |
/// |-----------|------------------|-----------------------------------------------|
/// | `Some(p)` | any | Fixed physical (MMIO / identity / DMA) |
/// | `None` | not set | Eager anonymous — allocate + zero + map now |
/// | `None` | set | Lazy anonymous — map frames on first fault |
pub fn map_region(
&mut self,
virt: VirtAddr,
phys: Option<PhysAddr>,
size: u64,
flags: VmaFlags,
cap_token: u64,
) -> Result<VirtAddr, VmError> {
if size == 0 { return Err(VmError::InvalidRange); }
if virt.0 & 0xFFF != 0 || size & 0xFFF != 0 {
return Err(VmError::InvalidAlignment);
}
let virt_end = VirtAddr(virt.0.checked_add(size).ok_or(VmError::InvalidRange)?);
self.check_overlap(virt, virt_end)?;
let pml4 = unsafe { &mut *self.pml4_raw() };
let page_flags = flags.to_page_flags();
let page_count = (size / 4096) as usize;
let hhdm = self.hhdm;
let backing = match phys {
Some(base) => {
pml4.map_region(virt, base, size, page_flags, hhdm);
VmaBacking::Physical(base)
}
None if flags.contains(VmaFlags::LAZY) => {
let mut frames = Vec::with_capacity(page_count);
frames.resize_with(page_count, || None);
VmaBacking::Anonymous(frames)
}
None => {
let mut frames = Vec::with_capacity(page_count);
for i in 0..page_count {
let frame = pmm::alloc_frame().ok_or(VmError::OutOfMemory)?;
unsafe {
core::ptr::write_bytes(frame.to_virt(hhdm).as_mut_ptr::<u8>(), 0, 4096);
}
pml4.map_page(
VirtAddr(virt.0 + i as u64 * 4096),
frame,
page_flags,
hhdm,
);
frames.push(Some(frame));
}
VmaBacking::Anonymous(frames)
}
};
self.insert_sorted(VmaRegion { virt_start: virt, virt_end, flags, cap_token, backing });
Ok(virt)
}
// ─── Zero-copy shared mapping (new) ───────────────────────────────────
/// Map `page_count` pages of physical memory owned by `owner_cap` into
/// this address space at `virt`.
///
/// This is the primitive behind Elyz's zero-copy transfer: the physical
/// frames stay in place; only the PTE entries (rights) move between address
/// spaces. The frames are **not** freed when this mapping is unmapped —
/// the owning actor's capability retains that responsibility.
///
/// # Revocation
/// Pass `owner_cap` as `cap_token` in the VMA so that `revoke_by_token`
/// finds and unmaps this shared view when the source capability is revoked.
pub fn map_shared(
&mut self,
virt: VirtAddr,
phys_base: PhysAddr,
page_count: usize,
flags: VmaFlags,
owner_cap: u64,
) -> Result<VirtAddr, VmError> {
if page_count == 0 { return Err(VmError::InvalidRange); }
if virt.0 & 0xFFF != 0 { return Err(VmError::InvalidAlignment); }
let size = page_count as u64 * 4096;
let virt_end = VirtAddr(virt.0.checked_add(size).ok_or(VmError::InvalidRange)?);
self.check_overlap(virt, virt_end)?;
let pml4 = unsafe { &mut *self.pml4_raw() };
let page_flags = flags.to_page_flags();
let hhdm = self.hhdm;
// Eagerly map all pages — the physical addresses are already known.
for i in 0..page_count {
pml4.map_page(
VirtAddr(virt.0 + i as u64 * 4096),
PhysAddr(phys_base.0 + i as u64 * 4096),
page_flags,
hhdm,
);
}
self.insert_sorted(VmaRegion {
virt_start: virt,
virt_end,
flags: flags | VmaFlags::SHARED,
cap_token: owner_cap,
backing: VmaBacking::Shared { owner_cap, phys_base },
});
Ok(virt)
}
// ─── Demand paging ─────────────────────────────────────────────────────
/// Handle a hardware page fault at `fault_addr`.
///
/// Returns `Ok(())` if the fault was a valid lazy demand-page (caller should
/// `iretq` to retry). Returns `Err` for illegal accesses.
pub fn handle_fault(&mut self, fault_addr: VirtAddr, write: bool) -> Result<(), VmError> {
let hhdm = self.hhdm;
let idx = self.find_idx(fault_addr).ok_or(VmError::RegionNotFound)?;
{
let region = &self.regions[idx];
if write && !region.flags.contains(VmaFlags::WRITE) {
return Err(VmError::PermissionDenied);
}
if !region.flags.contains(VmaFlags::LAZY) {
return Err(VmError::UnexpectedFault);
}
}
let region = &mut self.regions[idx];
let page_idx = ((fault_addr.0 - region.virt_start.0) / 4096) as usize;
let page_virt = VirtAddr(region.virt_start.0 + page_idx as u64 * 4096);
let page_flags = region.flags.to_page_flags();
let VmaBacking::Anonymous(ref mut frames) = region.backing else {
return Err(VmError::RegionNotFound);
};
if frames[page_idx].is_some() {
// SMP race: another core already mapped this page.
return Ok(());
}
let frame = pmm::alloc_frame().ok_or(VmError::OutOfMemory)?;
unsafe {
core::ptr::write_bytes(frame.to_virt(hhdm).as_mut_ptr::<u8>(), 0, 4096);
}
frames[page_idx] = Some(frame);
let pml4 = unsafe { &mut *self.pml4_raw() };
pml4.map_page(page_virt, frame, page_flags, hhdm);
Ok(())
}
// ─── Unmapping ─────────────────────────────────────────────────────────
/// Unmap the VMA containing `virt`, free its frames (if owned), flush TLB.
pub fn unmap_region(&mut self, virt: VirtAddr) -> Result<(), VmError> {
let idx = self.find_idx(virt).ok_or(VmError::RegionNotFound)?;
let region = self.regions.remove(idx);
let pml4 = unsafe { self.pml4_raw() };
Self::do_unmap(pml4, &region, self.hhdm);
tlb_flush_asid(self.asid);
Ok(())
}
// ─── Capability revocation ─────────────────────────────────────────────
/// Atomically unmap all VMAs associated with `cap_token` (skip PINNED).
///
/// Hardware access is terminated before this function returns.
/// Shared-backed VMAs are unmapped from the PT without freeing frames.
pub fn revoke_by_token(&mut self, cap_token: u64) {
let hhdm = self.hhdm;
let pml4 = unsafe { self.pml4_raw() };
let indices: Vec<usize> = self.regions
.iter()
.enumerate()
.filter(|(_, r)| {
r.cap_token == cap_token && !r.flags.contains(VmaFlags::PINNED)
})
.map(|(i, _)| i)
.collect();
for idx in indices.into_iter().rev() {
let region = self.regions.remove(idx);
Self::do_unmap(pml4, &region, hhdm);
}
tlb_flush_asid(self.asid);
}
// ─── Address translation ───────────────────────────────────────────────
/// Walk the live page table to translate `virt` → physical address.
pub fn translate(&self, virt: VirtAddr) -> Option<PhysAddr> {
unsafe { (*self.pml4_raw()).translate(virt, self.hhdm) }
}
// ─── Activation ────────────────────────────────────────────────────────
/// Load this address space into the CPU (context switch).
///
/// Sets CR3 with bit 63 = NOFLUSH to preserve PCID-tagged TLB entries.
/// Requires CR4.PCIDE = 1 (enabled by the bootloader/early init).
///
/// # Safety
/// Must only be called from a code path that is identity-mapped in every
/// address space (e.g. the kernel `.text` section reachable via HHDM).
pub unsafe fn activate(&self) {
let cr3 = self.pml4_phys.0 | u64::from(self.asid) | (1u64 << 63);
unsafe {
core::arch::asm!(
"mov cr3, {0}",
in(reg) cr3,
options(nostack, preserves_flags),
);
}
}
// ─── Introspection ─────────────────────────────────────────────────────
#[inline] pub fn regions(&self) -> &[VmaRegion] { &self.regions }
#[inline] pub fn region_count(&self) -> usize { self.regions.len() }
pub fn find_region(&self, addr: VirtAddr) -> Option<&VmaRegion> {
self.find_idx(addr).map(|i| &self.regions[i])
}
}
impl Drop for AddressSpace {
fn drop(&mut self) {
let hhdm = self.hhdm;
let pml4 = unsafe { self.pml4_raw() };
while let Some(region) = self.regions.pop() {
Self::do_unmap(pml4, &region, hhdm);
}
pmm::free_frame(self.pml4_phys);
// Return the ASID to the pool so it can be reused by future processes.
// ASID 0 is the kernel sentinel — never freed (guarded inside free_asid).
free_asid(self.asid);
}
}
// ══════════════════════════════════════════════════════════════════════════════
// TLB management
// ══════════════════════════════════════════════════════════════════════════════
/// Flush all TLB entries tagged with `asid` (PCID) on the current core.
///
/// Uses `INVPCID` type-1 (single-context flush) when available (Broadwell+,
/// CPUID.07H:EBX[10] = 1). Falls back to a full CR3 reload on older CPUs.
///
/// **SMP note**: on multi-core systems a TLB-shootdown IPI to all remote cores
/// must be added once the LAPIC driver and scheduler are online.
pub fn tlb_flush_asid(asid: u16) {
if INVPCID_SUPPORTED.load(Ordering::Relaxed) {
#[repr(C, packed)]
struct InvpcidDesc { pcid: u64, addr: u64 }
let desc = InvpcidDesc { pcid: asid as u64, addr: 0 };
unsafe {
core::arch::asm!(
"invpcid {ty}, [{desc}]",
ty = in(reg) 1u64, // type 1 = single-context flush
desc = in(reg) &desc,
options(nostack, preserves_flags),
);
}
} else {
// Fallback: full TLB flush via CR3 reload (clears all PCID entries).
tlb_flush_all();
}
}
/// Full TLB flush (all PCIDs, all addresses) via CR3 reload.
pub fn tlb_flush_all() {
unsafe {
let mut cr3: u64;
core::arch::asm!("mov {0}, cr3", out(reg) cr3,
options(nostack, preserves_flags));
// Writing CR3 without bit 63 forces a full flush.
core::arch::asm!("mov cr3, {0}",
in(reg) cr3 & !(1u64 << 63),
options(nostack, preserves_flags));
}
}
// ══════════════════════════════════════════════════════════════════════════════
// Global kernel address space
// ══════════════════════════════════════════════════════════════════════════════
/// The one kernel address space. Initialised once during boot.
pub static KERNEL_SPACE: Locked<Option<AddressSpace>> = Locked::new(None);
/// Register the already-active PML4 as the kernel address space.
///
/// ASID 0 = PCID 0 = kernel (no per-process PCID tagging).
/// Must be called after `BitmapPMM::init` and `init_cpu_features`.
pub fn init_kernel_space(pml4_phys: PhysAddr, hhdm: u64) {
*KERNEL_SPACE.lock() = Some(AddressSpace::from_active(pml4_phys, 0, hhdm));
}