diff --git a/kernel/src/cap/mod.rs b/kernel/src/cap/mod.rs index 47b66db..0f2eae0 100644 --- a/kernel/src/cap/mod.rs +++ b/kernel/src/cap/mod.rs @@ -1,8 +1,10 @@ pub mod descriptor; pub mod object; -use alloc::vec::Vec; +use crate::events::MMU_REVOCATION_QUEUE; use crate::mem::allocator::Locked; +use alloc::vec::Vec; + pub use descriptor::*; pub struct CNodeSlot { @@ -27,44 +29,109 @@ impl CNode { } pub fn insert(&self, slot: usize, cap: Capability) -> Result<(), &'static str> { - if slot >= self.slots.len() { return Err("Index out of bounds"); } + if slot >= self.slots.len() { + return Err("Index out of bounds"); + } let mut s = self.slots[slot].lock(); s.cap = cap; Ok(()) } pub fn mint(&self, src: usize, dest: usize, relation: Relation, rights: CapRights) -> Result<(), &'static str> { - let mut slots = (self.slots[src].lock(), self.slots[dest].lock()); - let (src_slot, mut dest_slot) = (&slots.0, &mut slots.1); + if src >= self.slots.len() || dest >= self.slots.len() { + return Err("Index out of bounds"); + } - if !src_slot.cap.is_valid() { return Err("Source empty"); } - if !src_slot.cap.rights.contains(CapRights::GRANT) { return Err("Insufficient rights to mint"); } + // Защита от дедлока: если src == dest, операция не имеет смысла + if src == dest { + return Err("Source and destination slots must be different"); + } + // Захватываем блокировки в строгом порядке индексов для предотвращения инверсии блокировок (Lock Ranking) + let mut _guard_low; + let mut _guard_high; + + let (src_slot, dest_slot) = if src < dest { + _guard_low = self.slots[src].lock(); + _guard_high = self.slots[dest].lock(); + (&mut *_guard_low, &mut *_guard_high) + } else { + _guard_low = self.slots[dest].lock(); + _guard_high = self.slots[src].lock(); + (&mut *_guard_high, &mut *_guard_low) + }; + + if !src_slot.cap.is_valid() { + return Err("Source slot is empty"); + } + if !src_slot.cap.rights.contains(CapRights::GRANT) { + return Err("Insufficient rights to mint (Missing GRANT flag)"); + } + + // Права дочернего дескриптора не могут превышать права родительского let final_rights = src_slot.cap.rights & rights; - + dest_slot.cap = src_slot.cap; dest_slot.cap.rights = final_rights; dest_slot.cap.relation = relation; dest_slot.parent_idx = Some(src); - + Ok(()) } - pub fn revoke(&self, slot_idx: usize) { - for i in 0..self.slots.len() { - let mut should_clear = false; - { - let child = self.slots[i].lock(); - if child.parent_idx == Some(slot_idx) { - should_clear = true; - } - } + /// Публичный метод отзыва прав. + /// Каскадно аннулирует все дочерние дескрипторы и отправляет их токены в lock-free очередь VMM. + pub fn revoke(&self, slot_idx: usize) -> Result<(), &'static str> { + if slot_idx >= self.slots.len() { + return Err("Index out of bounds"); + } - if should_clear { - self.revoke(i); - let mut child = self.slots[i].lock(); - child.cap = Capability::empty(); - child.parent_idx = None; + // Запускаем рекурсивный отзыв + self.revoke_internal(slot_idx); + + Ok(()) + } + + /// Внутренний метод каскадного удаления. + /// Вынесен отдельно, чтобы избежать удержания блокировок при переходе на следующий уровень рекурсии. + fn revoke_internal(&self, slot_idx: usize) { + // 1. Сначала рекурсивно ищем и уничтожаем всех потомков данного слота + for i in 0..self.slots.len() { + let is_child = { + let child = self.slots[i].lock(); + child.parent_idx == Some(slot_idx) + }; + + if is_child { + // Рекурсивный спуск. Блокировка с child[i] к этому моменту уже снята, дедлока нет. + self.revoke_internal(i); + } + } + + // 2. Теперь уничтожаем сам дескриптор в текущем слоте и отправляем его токен на отзыв в MMU + let token_to_revoke = { + let mut slot = self.slots[slot_idx].lock(); + if slot.cap.is_valid() { + let token = slot.cap.token_sig; + + // Перезаписываем пустой заглушкой (уничтожаем сильную ссылку ядра) + slot.cap = Capability::empty(); + slot.parent_idx = None; + + Some(token) + } else { + None + } + }; + + // 3. Если слот содержал валидный токен, асинхронно уведомляем VMM актёра через lock-free кольцевой буфер + if let Some(token) = token_to_revoke { + // В Elyz 0xDEAD_BEEF используется для сырого Untyped, его в MMU слать бессмысленно + if token != 0 && token != 0xDEAD_BEEF { + if let Err(e) = MMU_REVOCATION_QUEUE.push(token) { + // Переполнение очереди отзыва — критический сбой планировщика ресурсов ядра + panic!("FATAL: Критическое переполнение очереди отзыва MMU: {}", e); + } } } } diff --git a/kernel/src/cpu/idt.rs b/kernel/src/cpu/idt.rs new file mode 100644 index 0000000..ef8d12c --- /dev/null +++ b/kernel/src/cpu/idt.rs @@ -0,0 +1,69 @@ +use core::arch::asm; + +#[derive(Debug, Clone, Copy)] +#[repr(C, packed)] +pub struct IdtEntry { + offset_low: u16, + selector: u16, + ist: u8, + type_attr: u8, + offset_mid: u16, + offset_high: u32, + ignore: u32, +} + +impl IdtEntry { + pub const fn new() -> Self { + Self { + offset_low: 0, + selector: 0, + ist: 0, + type_attr: 0, + offset_mid: 0, + offset_high: 0, + ignore: 0, + } + } + + pub fn set_handler(&mut self, handler: u64, selector: u16, flags: u8) { + self.offset_low = handler as u16; + self.selector = selector; + self.ist = 0; + self.type_attr = flags | 0x80; // Present bit + self.offset_mid = (handler >> 16) as u16; + self.offset_high = (handler >> 32) as u32; + self.ignore = 0; + } +} + +#[repr(C, packed)] +pub struct IdtPtr { + limit: u16, + base: u64, +} + +pub struct InterruptDescriptorTable { + entries: [IdtEntry; 256], +} + +impl InterruptDescriptorTable { + pub const fn new() -> Self { + Self { entries: [IdtEntry::new(); 256] } + } + + pub fn set_handler(&mut self, vector: u8, handler: u64) { + // 0x8E = Interrupt Gate, Ring 0, Present + self.entries[vector as usize].set_handler(handler, 0x28, 0x8E); // 0x28 - Kernel CS в Limine (по стандарту 5-й сегмент) + } + + pub unsafe fn load(&'static self) { + let ptr = IdtPtr { + limit: (core::mem::size_of::() - 1) as u16, + base: self as *const _ as u64, + }; + // Исправление для Rust 2024: явный unsafe блок внутри unsafe fn + unsafe { + asm!("lidt [{}]", in(reg) &ptr, options(readonly, nostack, preserves_flags)); + } + } +} diff --git a/kernel/src/cpu/interrupts.rs b/kernel/src/cpu/interrupts.rs new file mode 100644 index 0000000..6bec560 --- /dev/null +++ b/kernel/src/cpu/interrupts.rs @@ -0,0 +1,113 @@ +use core::arch::global_asm; +use core::arch::asm; +use crate::mem::vmm::KERNEL_SPACE; // Оставили только один импорт +use crate::mem::address::VirtAddr; + +// Глобальная статическая таблица дескрипторов прерываний (IDT) +pub static mut IDT: crate::cpu::idt::InterruptDescriptorTable = crate::cpu::idt::InterruptDescriptorTable::new(); + +// Низкоуровневый ассемблерный трамплин (Context Save & Restore) +global_asm!( + ".global page_fault_stub", + "page_fault_stub:", + "push rax", + "push rcx", + "push rdx", + "push rbx", + "push rbp", + "push rsi", + "push rdi", + "push r8", + "push r9", + "push r10", + "push r11", + "push r12", + "push r13", + "push r14", + "push r15", + + "mov rdi, [rsp + 15*8]", + "call rust_page_fault_handler", + + "pop r15", + "pop r14", + "pop r13", + "pop r12", + "pop r11", + "pop r10", + "pop r9", + "pop r8", + "pop rdi", + "pop rsi", + "pop rbp", + "pop rbx", + "pop rdx", + "pop rcx", + "pop rax", + + "add rsp, 8", + "iretq" +); + +unsafe extern "C" { + fn page_fault_stub(); +} + +pub fn init_idt() { + unsafe { + let idt_mut_ptr = core::ptr::addr_of_mut!(IDT); + (*idt_mut_ptr).set_handler(14, page_fault_stub as u64); + let idt_static_ref: &'static crate::cpu::idt::InterruptDescriptorTable = &*core::ptr::addr_of!(IDT); + idt_static_ref.load(); + } +} + +pub fn process_deferred_mmu_events() { + let mut vmm_guard = KERNEL_SPACE.lock(); + if let Some(space) = vmm_guard.as_mut() { + space.process_pending_revocations(); + } +} + +#[unsafe(no_mangle)] +pub extern "C" fn rust_page_fault_handler(error_code: u64) { + let fault_addr: u64; + unsafe { + asm!("mov {}, cr2", out(reg) fault_addr, options(nomem, nostack, preserves_flags)); + } + + let write = (error_code & 0x2) != 0; + let present = (error_code & 0x1) != 0; + let virt_addr = VirtAddr(fault_addr); + + // 1. Сначала применяем все отложенные отзывы прав из других Actor'ов (чтобы избежать Deadlock) + process_deferred_mmu_events(); + + // 2. Теперь захватываем VMM для обработки текущего сбоя + let mut vmm_guard = KERNEL_SPACE.lock(); + + if let Some(space) = vmm_guard.as_mut() { + match space.handle_fault(virt_addr, write) { + Ok(_) => return, + Err(e) => { + panic!( + "KERNEL PANIC: Необработанный сбой виртуальной памяти (Page Fault)!\n\ + Адрес: {:#X}\n\ + Режим: {}\n\ + Присутствие: {}\n\ + Причина VMM: {:?}", + fault_addr, + if write { "WRITE" } else { "READ" }, + if present { "YES (Protection Violation)" } else { "NO (Not Present)" }, + e + ); + } + } + } else { + panic!( + "KERNEL PANIC: Критический Page Fault до аллокации глобального KERNEL_SPACE!\n\ + Адрес сбоя: {:#X}", + fault_addr + ); + } +} diff --git a/kernel/src/cpu/mod.rs b/kernel/src/cpu/mod.rs new file mode 100644 index 0000000..7a63647 --- /dev/null +++ b/kernel/src/cpu/mod.rs @@ -0,0 +1,2 @@ +pub mod idt; +pub mod interrupts; diff --git a/kernel/src/events.rs b/kernel/src/events.rs new file mode 100644 index 0000000..67e9bdc --- /dev/null +++ b/kernel/src/events.rs @@ -0,0 +1,75 @@ +use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + +const QUEUE_SIZE: usize = 1024; // Должно быть степенью двойки для быстрой битовой маски +const QUEUE_MASK: usize = QUEUE_SIZE - 1; + +/// Lock-free MPSC (Multi-Producer, Single-Consumer) очередь. +/// Оптимизирована для передачи токенов инвалидации без блокировок. +pub struct RevocationQueue { + buffer: [AtomicU64; QUEUE_SIZE], + // Используем паддинг (cache line size = 64 bytes) для предотвращения False Sharing + // между head (изменяется VMM) и tail (изменяется Cap-системой). + #[doc(hidden)] + _pad0: [u8; 64], + head: AtomicUsize, + #[doc(hidden)] + _pad1: [u8; 64], + tail: AtomicUsize, +} + +impl RevocationQueue { + pub const fn new() -> Self { + Self { + #[allow(clippy::declare_interior_mutable_const)] + buffer: [const { AtomicU64::new(0) }; QUEUE_SIZE], + _pad0: [0; 64], + head: AtomicUsize::new(0), + _pad1: [0; 64], + tail: AtomicUsize::new(0), + } + } + + /// Вызывается со стороны Actor'ов и подсистемы Capabilities (Multi-Producer) + pub fn push(&self, token_sig: u64) -> Result<(), &'static str> { + let mut tail = self.tail.load(Ordering::Relaxed); + loop { + let head = self.head.load(Ordering::Acquire); + + if tail.wrapping_sub(head) >= QUEUE_SIZE { + // Очередь переполнена. В проде здесь должен быть триггер ballooning'а + // или принудительный yield для VMM, но пока возвращаем ошибку. + return Err("Revocation queue overflow"); + } + + match self.tail.compare_exchange_weak( + tail, + tail.wrapping_add(1), + Ordering::AcqRel, + Ordering::Relaxed + ) { + Ok(_) => { + self.buffer[tail & QUEUE_MASK].store(token_sig, Ordering::Release); + return Ok(()); + } + Err(actual_tail) => tail = actual_tail, + } + } + } + + /// Вызывается только из активного VMM контекста (Single-Consumer) + pub fn pop(&self) -> Option { + let head = self.head.load(Ordering::Relaxed); + let tail = self.tail.load(Ordering::Acquire); + + if head == tail { + return None; // Очередь пуста + } + + let token = self.buffer[head & QUEUE_MASK].load(Ordering::Acquire); + self.head.store(head.wrapping_add(1), Ordering::Release); + + Some(token) + } +} + +pub static MMU_REVOCATION_QUEUE: RevocationQueue = RevocationQueue::new(); diff --git a/kernel/src/main.rs b/kernel/src/main.rs index ebc458a..93fc6e7 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -13,9 +13,13 @@ use limine::request::{FramebufferRequest, RequestsEndMarker, RequestsStartMarker use crate::mem::paging::{PageTable, PageTableFlags}; use crate::mem::address::{PhysAddr, VirtAddr}; use crate::cap::{Relation, CapRights, Capability, CapObject}; +use crate::mem::pm_manages::{PMActor, PMRequest, PMResult}; pub mod cap; +pub mod cpu; +pub mod events; mod mem; + #[macro_use] pub mod debug; @@ -208,9 +212,19 @@ unsafe extern "C" fn kmain() -> ! { let mut allocator = allocator::ALLOCATOR.lock(); allocator.init(heap_start as usize, heap_size); } - info!(console, "HEAP", "Global Allocator is online."); + info!(console, "HEAP", "Slab Allocator is online."); + mem::init_cpu_features(); + info!(console, "CPU", "INVPCID / CPU features detected."); + mem::vmm::init_kernel_space(p4_phys, hhdm_offset); + info!(console, "VMM", "Kernel Address Space registered."); + cpu::interrupts::init_idt(); + info!(console, "CPU", "Interrupt Descriptor Table (IDT) loaded."); + + unsafe { core::arch::asm!("sti", options(nomem, nostack, preserves_flags)); } + + // --- Тестирование подсистемы Capability --- let root_cnode = cap::CNode::new(256); if let Some(frame) = mem::pmm::alloc_frame() { @@ -237,6 +251,79 @@ unsafe extern "C" fn kmain() -> ! { info!(console, "CAP", "Slot 10 successfully revoked."); } } + + // --- Тестирование PMActor (Lock-Free Очереди) --- + info!(console, "PM", "--- PMActor Buddy Test ---"); + + let actor_base_phys = PhysAddr(0x4000_0000); + let actor_root_cap = Capability { + object: CapObject::Memory { phys: actor_base_phys, size_pages: 1024 }, + rights: CapRights::all(), + relation: Relation::Strong, + token_sig: 0xAAAA_BBBB, + }; + + let mut pm_actor = PMActor::new(1, actor_root_cap, actor_base_phys, 1024 * 4096); + info!(console, "PM", "PMActor ID:1 spawned ({} free pages).", pm_actor.free_pages()); + + // channel_id=1 → route response to "process 1" + pm_actor.submit_request(PMRequest::Allocate { + size_pages: 10, + token_sig: 0x123, + channel_id: 1, + }).unwrap(); + + // Carve a fixed sub-region (e.g. for a framebuffer alias) + pm_actor.submit_request(PMRequest::Carve { + offset_pages: 100, + size_pages: 4, + channel_id: 2, + }).unwrap(); + + // Process and collect responses + let responses = pm_actor.process_messages(); + for resp in &responses { + match resp.result { + PMResult::Allocated { cap, order } => { + info!(console, "PM", + "ch={} Allocated: phys={:#x} order={} pages={}", + resp.channel_id, + if let CapObject::Memory { phys, .. } = cap.object { phys.0 } else { 0 }, + order, + 1usize << order, + ); + + // Free it back (using the order returned in the response) + if let CapObject::Memory { phys, .. } = cap.object { + let rel_idx = ((phys.0 - actor_base_phys.0) / 4096) as usize; + pm_actor.submit_request(PMRequest::Free { + local_frame_idx: rel_idx, + order, + }).unwrap(); + } + } + PMResult::Carved { cap } => { + info!(console, "PM", + "ch={} Carved sub-cap: phys={:#x}", + resp.channel_id, + if let CapObject::Memory { phys, .. } = cap.object { phys.0 } else { 0 }, + ); + } + PMResult::OutOfMemory { size_pages } => { + info!(console, "PM", "ch={} OOM for {} pages!", resp.channel_id, size_pages); + } + PMResult::Freed { .. } => {} + } + } + + // Drain the Free request + let _ = pm_actor.process_messages(); + info!(console, "PM", "After free: {} free pages (should be 1024).", pm_actor.free_pages()); + + + + + // --- Логотип --- let logo = r#" ########### ################## diff --git a/kernel/src/mem/allocator.rs b/kernel/src/mem/allocator.rs index 0823f29..1fb1d51 100644 --- a/kernel/src/mem/allocator.rs +++ b/kernel/src/mem/allocator.rs @@ -25,7 +25,7 @@ impl Locked { pub fn lock(&self) -> LockedGuard<'_, A> { while self.lock.compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed).is_err() { - core::hint::spin_loop(); + core::hint::spin_loop(); // Уступка в spinlock для снижения нагрузки на шину } LockedGuard { lock: &self.lock, @@ -49,41 +49,106 @@ impl DerefMut for LockedGuard<'_, A> { fn deref_mut(&mut self) -> &mut Self::Target { self.data } } -pub struct BumpAllocator { - start: usize, - end: usize, - next: usize, +/// Узел односвязного списка свободных блоков +struct ListNode { + next: Option<&'static mut ListNode>, } -impl BumpAllocator { +const BLOCK_SIZES: &[usize] = &[8, 16, 32, 64, 128, 256, 512, 1024, 2048]; + +/// Slab аллокатор для гранулярного выделения памяти +pub struct SlabAllocator { + list_heads: [Option<&'static mut ListNode>; BLOCK_SIZES.len()], + heap_start: usize, + heap_end: usize, + next_bump: usize, +} + +impl SlabAllocator { pub const fn new() -> Self { - Self { start: 0, end: 0, next: 0 } - } - - pub fn init(&mut self, start: usize, size: usize) { - self.start = start; - self.next = start; - self.end = start + size; - } -} - -unsafe impl GlobalAlloc for Locked { - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - let mut bump = self.lock(); - - let alloc_start = (bump.next + layout.align() - 1) & !(layout.align() - 1); - let alloc_end = alloc_start + layout.size(); - - if alloc_end > bump.end { - null_mut() - } else { - bump.next = alloc_end; - alloc_start as *mut u8 + Self { + list_heads: [None, None, None, None, None, None, None, None, None], + heap_start: 0, + heap_end: 0, + next_bump: 0, } } - unsafe fn dealloc(&self, _ptr: *mut u8, _layout: Layout) {} + pub fn init(&mut self, start: usize, size: usize) { + self.heap_start = start; + self.next_bump = start; + self.heap_end = start + size; + } + + /// Поиск индекса блока под требуемый размер + fn list_index(layout: &Layout) -> Option { + let required_block_size = layout.size().max(layout.align()); + BLOCK_SIZES.iter().position(|&s| s >= required_block_size) + } + + /// Резервный Bump-аллокатор для нарезки новых Slab-блоков + fn fallback_alloc(&mut self, layout: Layout) -> *mut u8 { + let alloc_start = (self.next_bump + layout.align() - 1) & !(layout.align() - 1); + let alloc_end = alloc_start.checked_add(layout.size()).unwrap_or(self.heap_end + 1); + + if alloc_end > self.heap_end { + null_mut() // Out of memory + } else { + self.next_bump = alloc_end; + alloc_start as *mut u8 + } + } +} + +unsafe impl GlobalAlloc for Locked { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let mut allocator = self.lock(); + + match SlabAllocator::list_index(&layout) { + Some(index) => { + match allocator.list_heads[index].take() { + Some(node) => { + allocator.list_heads[index] = node.next.take(); + node as *mut ListNode as *mut u8 + } + None => { + let block_size = BLOCK_SIZES[index]; + let block_align = block_size; + let layout = Layout::from_size_align(block_size, block_align).unwrap(); + allocator.fallback_alloc(layout) + } + } + } + None => allocator.fallback_alloc(layout) + } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + let mut allocator = self.lock(); + + match SlabAllocator::list_index(&layout) { + Some(index) => { + let new_node = ListNode { + next: allocator.list_heads[index].take(), + }; + + assert!(layout.size() >= core::mem::size_of::()); + + let new_node_ptr = ptr as *mut ListNode; + + // Production-fix для Rust 2024: явная изоляция unsafe-операций + unsafe { + new_node_ptr.write(new_node); + allocator.list_heads[index] = Some(&mut *new_node_ptr); + } + } + None => { + // Крупные регионы освобождаются через вызовы дескрипторов VMM/PMM, + // глобальный аллокатор ядра их не трекает. + } + } + } } #[global_allocator] -pub static ALLOCATOR: Locked = Locked::new(BumpAllocator::new()); +pub static ALLOCATOR: Locked = Locked::new(SlabAllocator::new()); diff --git a/kernel/src/mem/buddy.rs b/kernel/src/mem/buddy.rs new file mode 100644 index 0000000..5e98617 --- /dev/null +++ b/kernel/src/mem/buddy.rs @@ -0,0 +1,298 @@ +//! # Per-Actor Buddy Allocator (`src/mem/buddy.rs`) +//! +//! A power-of-two page-block allocator for a fixed, pre-committed physical +//! region. Designed for **single-consumer** use inside a [`crate::mem::pm_manages::PMActor`]: +//! the owning actor is the only entity that ever mutates its buddy allocator, so +//! no locking is needed — the entire subsystem is inherently lock-free from the +//! consumer's perspective. +//! +//! ## Block Layout +//! +//! The region is subdivided into blocks of `2^order` pages (order 0 … MAX_ORDER). +//! Every block at order *k* is aligned to `2^k` pages within the region. +//! +//! ```text +//! order 0 → 1 page = 4 KiB +//! order 1 → 2 pages = 8 KiB +//! … +//! order 11 → 2 048 pages = 8 MiB (MAX_ORDER) +//! ``` +//! +//! ## Complexity +//! +//! | Operation | Amortised | Worst-case | +//! |-----------|-----------|------------| +//! | `alloc` | O(log N) | O(MAX_ORDER · Lₖ) | +//! | `free` | O(log N) | O(MAX_ORDER · Lₖ) | +//! +//! where Lₖ = `free_lists[k].len()` ≤ `total_pages / 2^k`. +//! +//! ## Production note +//! +//! For managed ranges > 1 GiB (> 256 K pages), replace the per-order `Vec` +//! free lists with a radix tree or interval tree to bound Lₖ. For current +//! PMActor capital sizes (typically ≤ 128 MiB = 32 K pages at order 0), the +//! `Vec`-based implementation is fully adequate. + +extern crate alloc; +use alloc::vec::Vec; + +// ══════════════════════════════════════════════════════════════════════════════ +// Constants & helpers +// ══════════════════════════════════════════════════════════════════════════════ + +/// Maximum allocation order. +/// `2^11 × 4 096 bytes = 8 MiB` per single allocation. +pub const MAX_ORDER: usize = 11; + +/// ⌈log₂(n)⌉ — the minimum order whose block size covers `page_count` pages. +/// +/// ```text +/// order_for(1) = 0 (2^0 = 1) +/// order_for(2) = 1 (2^1 = 2) +/// order_for(3) = 2 (2^2 = 4 ≥ 3) +/// order_for(2048) = 11 +/// ``` +#[inline] +pub fn order_for(page_count: usize) -> usize { + if page_count <= 1 { + 0 + } else { + // Number of bits needed to represent (page_count - 1). + usize::BITS as usize - (page_count - 1).leading_zeros() as usize + } +} + +// ══════════════════════════════════════════════════════════════════════════════ +// BuddyAllocator +// ══════════════════════════════════════════════════════════════════════════════ + +/// Buddy allocator over a contiguous, pre-committed range of physical pages. +/// +/// All page indices stored in free lists are **relative to the start of the +/// managed range**. The owner (`PMActor`) translates to absolute `PhysAddr` +/// by adding `managed_range.0`. +/// +/// # Invariants +/// - `free_pages ≤ total_pages` at all times. +/// - Every block in `free_lists[k]` is aligned to `2^k` pages (i.e. +/// `idx % (1 << k) == 0`). +/// - No block appears in more than one order's free list simultaneously. +pub struct BuddyAllocator { + /// `free_lists[k]` = relative page indices of free 2^k-page blocks. + /// Ordering within each list is irrelevant; `pop()` / `swap_remove()` are used. + free_lists: [Vec; MAX_ORDER + 1], + /// Total pages in the managed range (need not be a power of two). + total_pages: usize, + /// Running count of free pages. Always equals `Σ (2^k × free_lists[k].len())`. + free_pages: usize, +} + +impl BuddyAllocator { + // ─── Construction ───────────────────────────────────────────────────────── + + /// Create a new allocator over `total_pages` pages, **all initially free**. + /// + /// Uses a greedy largest-first decomposition to build the initial free lists + /// in O(total_pages / 2^MAX_ORDER) iterations — essentially O(1) for + /// power-of-two sizes. + /// + /// Example: 7 pages → blocks [4, 2, 1] → free_lists[2]=[0], [1]=[4], [0]=[6]. + pub fn new(total_pages: usize) -> Self { + // core::array::from_fn is the idiomatic way to init a non-Copy array. + let free_lists: [Vec; MAX_ORDER + 1] = + core::array::from_fn(|_| Vec::new()); + + let mut this = Self { + free_lists, + total_pages, + free_pages: 0, + }; + + if total_pages == 0 { + return this; + } + + let mut idx = 0usize; + while idx < total_pages { + let remaining = total_pages - idx; + + // Alignment constraint: block at `idx` must be aligned to 2^order. + // trailing_zeros(0) is u32::MAX, so we clamp to MAX_ORDER. + let align_order = if idx == 0 { + MAX_ORDER + } else { + (idx.trailing_zeros() as usize).min(MAX_ORDER) + }; + + // Size constraint: 2^order ≤ remaining → order ≤ ⌊log₂(remaining)⌋. + let size_order = (usize::BITS as usize - 1) + - remaining.leading_zeros() as usize; // ⌊log₂(remaining)⌋ + + let order = MAX_ORDER.min(align_order).min(size_order); + let block_size = 1usize << order; + + this.free_lists[order].push(idx); + this.free_pages += block_size; + idx += block_size; + } + + this + } + + // ─── Allocation ─────────────────────────────────────────────────────────── + + /// Allocate a 2^`order`-page block. + /// + /// Returns the **relative** page index of the block's first page, or `None` + /// if insufficient contiguous memory remains. + /// + /// The returned index can be converted to a physical address: + /// ```text + /// phys = actor.managed_range.0.0 + (idx as u64) * PAGE_SIZE + /// ``` + pub fn alloc(&mut self, order: usize) -> Option { + if order > MAX_ORDER { + return None; + } + + // Find the smallest available order ≥ requested. + let found_order = (order..=MAX_ORDER) + .find(|&o| !self.free_lists[o].is_empty())?; + + // Consume one block from the found order. + let block_idx = self.free_lists[found_order] + .pop() + .expect("buddy: free_list non-empty but pop() returned None"); + + // Debit: we removed a 2^found_order block from free. + self.free_pages -= 1 << found_order; + + // Split down to the requested order, putting buddies back into free lists. + // + // Invariant at each iteration: + // `block_idx` is the lower half of a 2^cur_order block. + // The upper half (= `block_idx + 2^(cur_order-1)`) is returned to the + // free list as an order-(cur_order-1) block. + let mut cur_order = found_order; + while cur_order > order { + cur_order -= 1; + let buddy_idx = block_idx + (1 << cur_order); + self.free_lists[cur_order].push(buddy_idx); + self.free_pages += 1 << cur_order; + } + + // Net accounting: removed 2^found_order, added (2^found_order − 2^order). + // Result: free_pages decreased by exactly 2^order. ✓ + Some(block_idx) + } + + /// Allocate `page_count` pages, rounding up to the nearest power of two. + /// + /// Returns `(relative_page_idx, actual_order)`. The caller **must** pass the + /// same `order` to [`BuddyAllocator::free`] — mismatched orders corrupt the + /// allocator. + /// + /// The wasted "rounding up" pages remain unusable until the block is freed. + /// For tightly-packed allocations, callers should use `alloc(order)` directly. + pub fn alloc_pages(&mut self, page_count: usize) -> Option<(usize, usize)> { + if page_count == 0 { + return None; + } + let order = order_for(page_count); + if order > MAX_ORDER { + return None; + } + self.alloc(order).map(|idx| (idx, order)) + } + + // ─── Deallocation ───────────────────────────────────────────────────────── + + /// Return a 2^`order`-page block at **relative** index `block_idx` to the + /// free pool, coalescing with free buddies up the order chain. + /// + /// # Panics (debug builds only) + /// - `order > MAX_ORDER` + /// - `block_idx + 2^order > total_pages` + /// + /// In release builds the checks are elided for performance; passing incorrect + /// arguments causes undefined bookkeeping, not UB (no unsafe indexing). + pub fn free(&mut self, mut block_idx: usize, mut order: usize) { + debug_assert!( + order <= MAX_ORDER, + "buddy::free: order {} > MAX_ORDER {}", + order, + MAX_ORDER + ); + debug_assert!( + block_idx.checked_add(1 << order).map_or(false, |e| e <= self.total_pages), + "buddy::free: block [{}, +{}] out of range (total_pages={})", + block_idx, + 1usize << order, + self.total_pages + ); + + // Try to coalesce with our buddy at each order. + // + // The buddy of a block at relative index `i` of order `k` is at: + // buddy_idx = i XOR 2^k + // This works because aligned buddy pairs always differ in exactly bit k. + while order < MAX_ORDER { + let buddy_idx = block_idx ^ (1 << order); + + // Buddy must lie entirely within the managed range. + let buddy_end = match buddy_idx.checked_add(1 << order) { + Some(e) => e, + None => break, + }; + if buddy_end > self.total_pages { + break; + } + + // Search for the buddy in the free list for this order. + // `swap_remove` is O(1) and order-preserving is not required. + if let Some(pos) = self.free_lists[order].iter().position(|&b| b == buddy_idx) { + self.free_lists[order].swap_remove(pos); + self.free_pages -= 1 << order; // buddy was in free count; remove it + + // Merged block starts at the lower address of the two. + block_idx = block_idx.min(buddy_idx); + order += 1; + // Continue trying to merge at the next level. + } else { + break; // buddy is allocated or out-of-range; stop coalescing + } + } + + // Push the (possibly merged) block onto the appropriate free list. + self.free_pages += 1 << order; + self.free_lists[order].push(block_idx); + } + + // ─── Introspection ──────────────────────────────────────────────────────── + + /// Number of pages currently available for allocation. + #[inline] + pub fn free_pages(&self) -> usize { + self.free_pages + } + + /// Total pages in the managed range. + #[inline] + pub fn total_pages(&self) -> usize { + self.total_pages + } + + /// `true` if the allocator has no free pages. + #[inline] + pub fn is_exhausted(&self) -> bool { + self.free_pages == 0 + } + + /// Dump free-list statistics for each order (useful in panic handlers). + /// + /// Returns an array `[(order, free_block_count); MAX_ORDER + 1]`. + pub fn stats(&self) -> [(usize, usize); MAX_ORDER + 1] { + core::array::from_fn(|o| (o, self.free_lists[o].len())) + } +} diff --git a/kernel/src/mem/mod.rs b/kernel/src/mem/mod.rs index bbba724..12f1574 100644 --- a/kernel/src/mem/mod.rs +++ b/kernel/src/mem/mod.rs @@ -1,16 +1,34 @@ -pub mod pmm; -pub mod address; -pub mod paging; -pub mod allocator; +//! `src/mem/mod.rs` — memory subsystem root -#[allow(dead_code)] -pub fn init(memmap: &limine::response::MemoryMapResponse, hhdm_offset: u64) { - unsafe { - pmm::BitmapPMM::init(memmap, hhdm_offset); - } +pub mod address; +pub mod allocator; +pub mod buddy; // ← new: per-actor buddy allocator +pub mod paging; +pub mod pm_manages; +pub mod pmm; +pub mod vmm; + +/// Initialise the physical memory manager. +/// +/// Must be called before any heap allocation or VMM operation. +pub fn init_pmm(memmap: &limine::response::MemoryMapResponse, hhdm_offset: u64) { + unsafe { pmm::BitmapPMM::init(memmap, hhdm_offset); } } -#[allow(dead_code)] +/// Initialise CPU features required by the VMM (INVPCID detection). +/// +/// Must be called before `init_vmm` and any `AddressSpace::activate()`. +pub fn init_cpu_features() { + vmm::init_cpu_features(); +} + +/// Initialise the kernel address space record (after PMM, heap, and the +/// initial P4 page table have been set up in `kmain`). +pub fn init_vmm(pml4_phys: address::PhysAddr, hhdm_offset: u64) { + vmm::init_kernel_space(pml4_phys, hhdm_offset); +} + +/// Physical memory statistics: `(used_pages, total_pages)`. pub fn get_stats() -> (usize, usize) { pmm::get_stats() } diff --git a/kernel/src/mem/paging.rs b/kernel/src/mem/paging.rs index 76cf468..fb5a30a 100644 --- a/kernel/src/mem/paging.rs +++ b/kernel/src/mem/paging.rs @@ -20,13 +20,27 @@ bitflags! { } } +/// Mask for the physical frame address stored in a PTE (bits 51:12). +const PTE_ADDR_MASK: u64 = 0x000F_FFFF_FFFF_F000; + #[repr(C, align(4096))] pub struct PageTable { entries: [u64; 512], } impl PageTable { - pub fn map_region(&mut self, virt: VirtAddr, phys: PhysAddr, size: u64, flags: PageTableFlags, hhdm: u64) { + // ── Bulk mapping ───────────────────────────────────────────────────────── + + /// Map a contiguous physical range to a contiguous virtual range. + /// Allocates intermediate page-table pages from the PMM as needed. + pub fn map_region( + &mut self, + virt: VirtAddr, + phys: PhysAddr, + size: u64, + flags: PageTableFlags, + hhdm: u64, + ) { let pages = size.div_ceil(4096); for i in 0..pages { let offset = i * 4096; @@ -34,51 +48,163 @@ impl PageTable { } } - /// Loads the page table into the CR3 register - pub unsafe fn activate(&self, phys_addr: PhysAddr) { - unsafe { - asm!( - "mov cr3, {0}", - "jmp 2f", - "2:", - in(reg) phys_addr.0, - options(nostack, preserves_flags) - ); - } - } + // ── Single-page operations ──────────────────────────────────────────────── - pub fn map_page(&mut self, virt: VirtAddr, phys: PhysAddr, flags: PageTableFlags, hhdm_offset: u64) { - let p4_idx = (virt.0 >> 39) & 0x1FF; - let p3_idx = (virt.0 >> 30) & 0x1FF; - let p2_idx = (virt.0 >> 21) & 0x1FF; - let p1_idx = (virt.0 >> 12) & 0x1FF; + /// Map a single 4 KiB page. + /// Allocates intermediate PT pages from the PMM if they do not exist. + pub fn map_page( + &mut self, + virt: VirtAddr, + phys: PhysAddr, + flags: PageTableFlags, + hhdm: u64, + ) { + let p1 = self + .walk_to_p1_mut(virt, hhdm, true /* create */) + .expect("map_page: OOM allocating intermediate page-table pages"); - let p3 = self.get_or_create_next_table(p4_idx as usize, hhdm_offset); - let p2 = p3.get_or_create_next_table(p3_idx as usize, hhdm_offset); - let p1 = p2.get_or_create_next_table(p2_idx as usize, hhdm_offset); + let p1_idx = ((virt.0 >> 12) & 0x1FF) as usize; + p1.entries[p1_idx] = phys.0 | flags.bits(); - p1.entries[p1_idx as usize] = phys.0 | flags.bits(); - unsafe { asm!("invlpg [{}]", in(reg) virt.0, options(nostack, preserves_flags)); } } -fn get_or_create_next_table(&mut self, index: usize, hhdm: u64) -> &mut Self { - if self.entries[index] & PageTableFlags::PRESENT.bits() == 0 { - let pt_phys = pmm_alloc().expect("VMM: Out of memory for page tables"); - let pt_virt = pt_phys.to_virt(hhdm); - - unsafe { core::ptr::write_bytes(pt_virt.as_mut_ptr::(), 0, 4096); } - - self.entries[index] = pt_phys.0 | (PageTableFlags::PRESENT | PageTableFlags::WRITABLE | PageTableFlags::USER).bits(); + /// Clear the PTE for `virt` and issue an `INVLPG`. + /// + /// Does **not** free the underlying physical frame — that is the caller's + /// responsibility (e.g. `VmaBacking::do_unmap`). + /// Silently returns if any level is not present (idempotent). + pub fn unmap_page(&mut self, virt: VirtAddr, hhdm: u64) { + let Some(p1) = self.walk_to_p1_mut(virt, hhdm, false /* no create */) else { + return; // already absent — nothing to do + }; + let p1_idx = ((virt.0 >> 12) & 0x1FF) as usize; + if p1.entries[p1_idx] & PageTableFlags::PRESENT.bits() == 0 { + return; + } + p1.entries[p1_idx] = 0; + unsafe { + asm!("invlpg [{}]", in(reg) virt.0, options(nostack, preserves_flags)); + } + } + + /// Walk four page-table levels to translate `virt` → physical address. + /// + /// Handles 1 GiB and 2 MiB huge pages transparently. + /// Returns `None` if any level is absent or the page is not present. + pub fn translate(&self, virt: VirtAddr, hhdm: u64) -> Option { + let p4_idx = ((virt.0 >> 39) & 0x1FF) as usize; + let p3_idx = ((virt.0 >> 30) & 0x1FF) as usize; + let p2_idx = ((virt.0 >> 21) & 0x1FF) as usize; + let p1_idx = ((virt.0 >> 12) & 0x1FF) as usize; + + // Helper: safely follow a PTE to the next level. + macro_rules! descend_ref { + ($entry:expr) => {{ + let e = $entry; + if e & PageTableFlags::PRESENT.bits() == 0 { return None; } + unsafe { &*PhysAddr(e & PTE_ADDR_MASK).to_virt(hhdm).as_mut_ptr::() } + }}; + } + + let p3 = descend_ref!(self.entries[p4_idx]); + + // 1 GiB page (PD pointer with HUGE_PAGE set) + let p3e = p3.entries[p3_idx]; + if p3e & PageTableFlags::HUGE_PAGE.bits() != 0 { + let base = p3e & 0x000F_FFFF_C000_0000; // bits 51:30 + return Some(PhysAddr(base | (virt.0 & 0x3FFF_FFFF))); + } + + let p2 = descend_ref!(p3e); + + // 2 MiB page (PD entry with HUGE_PAGE set) + let p2e = p2.entries[p2_idx]; + if p2e & PageTableFlags::HUGE_PAGE.bits() != 0 { + let base = p2e & 0x000F_FFFF_FFE0_0000; // bits 51:21 + return Some(PhysAddr(base | (virt.0 & 0x001F_FFFF))); + } + + let p1 = descend_ref!(p2e); + let p1e = p1.entries[p1_idx]; + if p1e & PageTableFlags::PRESENT.bits() == 0 { return None; } + + Some(PhysAddr((p1e & PTE_ADDR_MASK) | (virt.0 & 0xFFF))) + } + + // ── CR3 ────────────────────────────────────────────────────────────────── + + /// Load this page table into CR3 (full TLB flush, no PCID). + /// + /// Prefer `AddressSpace::activate` (which uses PCID/INVPCID) for process + /// context switches. Use this only for early boot before PCID is enabled. + pub unsafe fn activate(&self, phys_addr: PhysAddr) { + unsafe { + asm!( + "mov cr3, {0}", + in(reg) phys_addr.0, + options(nostack, preserves_flags), + ); } - let next_pt_phys = PhysAddr(self.entries[index] & 0x000F_FFFF_FFFF_F000); - unsafe { &mut *next_pt_phys.to_virt(hhdm).as_mut_ptr() } } } +// ── Private walk helpers ────────────────────────────────────────────────────── + +impl PageTable { + /// Walk (or create) the path P4 → P3 → P2 → P1, returning a mutable + /// reference to the P1 (page table, final level). + /// + /// If `create` is `false` and any intermediate entry is absent, returns `None`. + /// If `create` is `true`, allocates missing intermediate pages from the PMM. + fn walk_to_p1_mut(&mut self, virt: VirtAddr, hhdm: u64, create: bool) + -> Option<&mut Self> + { + let p4_idx = ((virt.0 >> 39) & 0x1FF) as usize; + let p3_idx = ((virt.0 >> 30) & 0x1FF) as usize; + let p2_idx = ((virt.0 >> 21) & 0x1FF) as usize; + + let p3 = self.get_or_create_next_table(p4_idx, hhdm, create)?; + let p2 = p3.get_or_create_next_table(p3_idx, hhdm, create)?; + p2.get_or_create_next_table(p2_idx, hhdm, create) + } + + /// Return a mutable reference to the next-level table at `index`. + /// + /// - If the entry is present, follows the pointer. + /// - If absent and `create`, allocates a zeroed page and installs it. + /// - If absent and `!create`, returns `None`. + fn get_or_create_next_table( + &mut self, + index: usize, + hhdm: u64, + create: bool, + ) -> Option<&mut Self> { + let entry = self.entries[index]; + if entry & PageTableFlags::PRESENT.bits() == 0 { + if !create { return None; } + + let pt_phys = pmm_alloc().expect("VMM: OOM allocating page-table page"); + let pt_virt = pt_phys.to_virt(hhdm); + unsafe { core::ptr::write_bytes(pt_virt.as_mut_ptr::(), 0, 4096); } + + // Install with USER so both kernel and user pages can live under it; + // actual user/kernel distinction is enforced at the P1 level. + self.entries[index] = pt_phys.0 + | (PageTableFlags::PRESENT | PageTableFlags::WRITABLE | PageTableFlags::USER).bits(); + } + + let next_phys = PhysAddr(self.entries[index] & PTE_ADDR_MASK); + Some(unsafe { &mut *next_phys.to_virt(hhdm).as_mut_ptr::() }) + } +} + +// ── PMM shim ───────────────────────────────────────────────────────────────── + +/// Allocate a single physical frame for page-table use. +/// This thin wrapper avoids a direct dependency cycle between paging ↔ pmm. pub fn pmm_alloc() -> Option { PMM.lock().as_mut()?.alloc_frame() } - diff --git a/kernel/src/mem/pm_manages.rs b/kernel/src/mem/pm_manages.rs index 30b86b1..b64a1d7 100644 --- a/kernel/src/mem/pm_manages.rs +++ b/kernel/src/mem/pm_manages.rs @@ -1,25 +1,491 @@ -use crate::cap::{Capability, CapObject, Relation, CapRights}; -use crate::mem::address::PhysAddr; +//! # PM Actor (`src/mem/pm_manages.rs`) +//! +//! Distributed physical-memory actor that owns a fixed physical capital range +//! and processes allocation requests asynchronously via a lock-free MPSC queue. +//! +//! ## Key changes vs. previous version +//! +//! 1. **Real allocator**: `PMActor` now contains a `BuddyAllocator` instead of a +//! bare page counter. Allocations produce real relative frame indices that +//! translate to absolute `PhysAddr` values. +//! +//! 2. **Closed response loop**: `process_messages()` returns `Vec`. +//! The caller (scheduler / boot code) routes each `PMResponse` to the +//! requesting entity identified by `channel_id`. `channel_id = 0` means +//! "discard" and is safe to ignore. +//! +//! 3. **Revised message packing** — fits all fields into a single `u64` without +//! truncating critical identifiers: +//! ```text +//! bits 63:56 OPCODE (8 bits) 1=Alloc, 2=Free, 3=Carve +//! bits 55:40 CHANNEL_ID (16 bits) response routing; 0 = discard +//! bits 39:20 ARG1 (20 bits) size_pages / local_frame_idx / offset_pages +//! bits 19: 0 ARG2 (20 bits) token_sig (low 20 bits) / size_pages +//! ``` +//! Maximum ARG size: 2^20 = 1 048 576 pages = 4 GiB per allocation. +//! Maximum CHANNEL_ID: 65 535 simultaneous pending requesters. +//! +//! ## Ownership model +//! +//! `PMActor` holds a `Strong` capability over its entire managed range. +//! Sub-capabilities carved out via `Allocate` or `Carve` are `Strong` children; +//! freeing them (via `Free`) coalesces the buddy tree. The actor **never** +//! touches the global `BitmapPMM` — that belongs to the kernel boot path. +//! +//! ## Ballooning hook +//! +//! When `BuddyAllocator::alloc_pages` returns `None`, the actor emits a +//! `PMResponse::OutOfMemory { channel_id }`. The future ballooning subsystem +//! will intercept this before it reaches the requester and negotiate memory +//! transfer from a neighbouring actor. -pub struct PMActor { - pub root_untyped: Capability, - pub managed_range: (PhysAddr, PhysAddr), +use alloc::vec::Vec; +use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + +use crate::cap::{CapObject, CapRights, Capability, Relation}; +use crate::mem::address::PhysAddr; +use crate::mem::buddy::BuddyAllocator; +use crate::mem::pmm::PAGE_SIZE; + +extern crate alloc; + +// ══════════════════════════════════════════════════════════════════════════════ +// Message packing constants +// ══════════════════════════════════════════════════════════════════════════════ + +const QUEUE_SIZE: usize = 1024; // power-of-two +const QUEUE_MASK: usize = QUEUE_SIZE - 1; + +/// Bit layout of a packed PMRequest u64. +mod packing { + pub const OPCODE_SHIFT: u32 = 56; + pub const CHAN_SHIFT: u32 = 40; + pub const ARG1_SHIFT: u32 = 20; + // ARG2 sits in bits 19:0 (no shift). + + pub const CHAN_MASK: u64 = 0x0000_FFFF; // 16 bits + pub const ARG_MASK: u64 = 0x000F_FFFF; // 20 bits } -impl PMActor { - pub fn carve_region(&self, offset: usize, size: usize) -> Capability { - if let CapObject::Memory { phys, .. } = self.root_untyped.object { - Capability { - object: CapObject::Memory { - phys: PhysAddr(phys.0 + offset as u64), - size_pages: size / 4096, - }, - rights: CapRights::READ | CapRights::WRITE | CapRights::GRANT, - relation: Relation::Strong, - token_sig: 0xDEAD_BEEF, +// ══════════════════════════════════════════════════════════════════════════════ +// Request type +// ══════════════════════════════════════════════════════════════════════════════ + +/// Asynchronous request submitted to a `PMActor` inbox. +/// +/// `channel_id` identifies where the response should be routed. +/// Use `channel_id = 0` if you do not need a reply. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PMRequest { + /// Allocate `size_pages` pages; tag the resulting capability with `token_sig`. + Allocate { + size_pages: usize, + token_sig: u32, + channel_id: u16, + }, + /// Free a previously allocated block. + /// + /// `local_frame_idx` is the **relative** page index returned inside the + /// `Allocated` capability (i.e. `cap.object.phys - managed_range.start`). + /// `order` is the buddy order used at allocation time (stored in the cap). + Free { + local_frame_idx: usize, + order: usize, + }, + /// Carve a fixed sub-region without consulting the buddy allocator. + /// Used for static layout decisions made at boot (e.g. framebuffer alias). + Carve { + offset_pages: usize, + size_pages: usize, + channel_id: u16, + }, + /// Sentinel — never pushed onto the queue; result of unpack(0). + None, +} + +impl PMRequest { + /// Encode the request into one `u64` atom for the lock-free buffer. + /// + /// Opcode 0 (None) is reserved as the "slot empty" sentinel; `None` returns + /// 0 intentionally and the sender rejects it before pushing. + #[inline] + pub fn pack(self) -> u64 { + use packing::*; + match self { + Self::Allocate { size_pages, token_sig, channel_id } => { + (1u64 << OPCODE_SHIFT) + | ((channel_id as u64 & CHAN_MASK) << CHAN_SHIFT) + | ((size_pages as u64 & ARG_MASK) << ARG1_SHIFT) + | (token_sig as u64 & ARG_MASK) } - } else { - panic!("PM: Root is not memory!"); + Self::Free { local_frame_idx, order } => { + (2u64 << OPCODE_SHIFT) + // channel_id not needed — Free has no reply + | ((local_frame_idx as u64 & ARG_MASK) << ARG1_SHIFT) + | (order as u64 & ARG_MASK) + } + Self::Carve { offset_pages, size_pages, channel_id } => { + (3u64 << OPCODE_SHIFT) + | ((channel_id as u64 & CHAN_MASK) << CHAN_SHIFT) + | ((offset_pages as u64 & ARG_MASK) << ARG1_SHIFT) + | (size_pages as u64 & ARG_MASK) + } + Self::None => 0, + } + } + + /// Decode a packed `u64` back into a `PMRequest`. + #[inline] + pub fn unpack(val: u64) -> Self { + use packing::*; + let opcode = val >> OPCODE_SHIFT; + let channel_id = ((val >> CHAN_SHIFT) & CHAN_MASK) as u16; + let arg1 = ((val >> ARG1_SHIFT) & ARG_MASK) as usize; + let arg2 = (val & ARG_MASK) as usize; + + match opcode { + 1 => Self::Allocate { + size_pages: arg1, + token_sig: arg2 as u32, + channel_id, + }, + 2 => Self::Free { + local_frame_idx: arg1, + order: arg2, + }, + 3 => Self::Carve { + offset_pages: arg1, + size_pages: arg2, + channel_id, + }, + _ => Self::None, } } } + +// ══════════════════════════════════════════════════════════════════════════════ +// Response type +// ══════════════════════════════════════════════════════════════════════════════ + +/// Result returned by `PMActor::process_messages()` for each completed request. +#[derive(Debug, Clone, Copy)] +pub struct PMResponse { + /// Channel to route this response to. `0` = discard. + pub channel_id: u16, + /// The actual outcome. + pub result: PMResult, +} + +/// Outcome of a single PM operation. +#[derive(Debug, Clone, Copy)] +pub enum PMResult { + /// Memory was allocated. `cap` is the strong capability to the region. + /// `order` is the buddy order — **must** be passed back to `Free`. + Allocated { cap: Capability, order: usize }, + + /// The actor had insufficient free pages. + /// Future: ballooning subsystem intercepts this and retries. + OutOfMemory { size_pages: usize }, + + /// A fixed sub-region was carved successfully. + Carved { cap: Capability }, + + /// Free completed (no capability issued — memory returned to buddy pool). + Freed { pages_returned: usize }, +} + +// ══════════════════════════════════════════════════════════════════════════════ +// Lock-free MPSC inbox +// ══════════════════════════════════════════════════════════════════════════════ + +/// MPSC ring buffer for PMRequest values. +/// +/// Producers (any core, any context) call `send`; the owning PMActor calls +/// `pop` — exactly one consumer, no locking on the read side. +pub struct PMActorQueue { + buffer: [AtomicU64; QUEUE_SIZE], + /// Pad to separate producer-written `tail` from consumer-read `head`. + _pad0: [u8; 64], + head: AtomicUsize, + _pad1: [u8; 64], + tail: AtomicUsize, +} + +impl PMActorQueue { + pub const fn new() -> Self { + Self { + #[allow(clippy::declare_interior_mutable_const)] + buffer: [const { AtomicU64::new(0) }; QUEUE_SIZE], + _pad0: [0u8; 64], + head: AtomicUsize::new(0), + _pad1: [0u8; 64], + tail: AtomicUsize::new(0), + } + } + + /// Push a request into the inbox (Multi-Producer path). + /// + /// Returns `Err` only if the ring buffer is full — which should trigger + /// backpressure or a ballooning request in production. + pub fn send(&self, req: PMRequest) -> Result<(), &'static str> { + let packed = req.pack(); + if packed == 0 { + return Err("PMActorQueue::send: PMRequest::None is not sendable"); + } + + let mut tail = self.tail.load(Ordering::Relaxed); + loop { + let head = self.head.load(Ordering::Acquire); + if tail.wrapping_sub(head) >= QUEUE_SIZE { + return Err("PMActorQueue overflow — apply backpressure or balloon"); + } + + match self.tail.compare_exchange_weak( + tail, + tail.wrapping_add(1), + Ordering::AcqRel, + Ordering::Relaxed, + ) { + Ok(_) => { + self.buffer[tail & QUEUE_MASK].store(packed, Ordering::Release); + return Ok(()); + } + Err(actual) => tail = actual, + } + } + } + + /// Pop one request (Single-Consumer path — actor's own context only). + pub fn pop(&self) -> Option { + let head = self.head.load(Ordering::Relaxed); + let tail = self.tail.load(Ordering::Acquire); + if head == tail { + return None; + } + let packed = self.buffer[head & QUEUE_MASK].load(Ordering::Acquire); + self.head.store(head.wrapping_add(1), Ordering::Release); + Some(PMRequest::unpack(packed)) + } +} + +// ══════════════════════════════════════════════════════════════════════════════ +// PM Actor +// ══════════════════════════════════════════════════════════════════════════════ + +/// An autonomous physical-memory actor. +/// +/// Owns a `BuddyAllocator` over its capital range and a lock-free MPSC inbox. +/// The actor must be driven externally: when the scheduler grants it CPU time, +/// it calls `process_messages()` to drain the inbox and produce `PMResponse`s. +/// +/// # Single-consumer guarantee +/// Only `process_messages()` ever mutates `buddy` and `queue.head`. +/// This is enforced by taking `&mut self` on `process_messages`. +pub struct PMActor { + /// Unique identity within the actor federation. + pub actor_id: u64, + + /// Root strong capability over the entire managed physical range. + pub root_untyped: Capability, + + /// `(inclusive_start, exclusive_end)` physical addresses. + pub managed_range: (PhysAddr, PhysAddr), + + /// Inbox — producers write here, actor reads here. + queue: PMActorQueue, + + /// Local buddy allocator. Only ever touched in `process_messages`. + buddy: BuddyAllocator, +} + +impl PMActor { + /// Construct a new actor managing `size_bytes` bytes starting at `start`. + /// + /// `size_bytes` must be a multiple of `PAGE_SIZE`; non-multiples are + /// rounded down silently. + pub fn new( + actor_id: u64, + root_cap: Capability, + start: PhysAddr, + size_bytes: u64, + ) -> Self { + let total_pages = (size_bytes / PAGE_SIZE) as usize; + Self { + actor_id, + root_untyped: root_cap, + managed_range: (start, PhysAddr(start.0 + total_pages as u64 * PAGE_SIZE)), + queue: PMActorQueue::new(), + buddy: BuddyAllocator::new(total_pages), + } + } + + // ─── Producer API (callable from any context) ────────────────────────── + + /// Submit a request to this actor's inbox. + /// + /// Thread-safe (MPSC producer side). Returns `Err` if the inbox is full. + #[inline] + pub fn submit_request(&self, req: PMRequest) -> Result<(), &'static str> { + self.queue.send(req) + } + + // ─── Consumer API (actor's own scheduled context) ───────────────────── + + /// Drain the inbox and execute all pending requests. + /// + /// Returns a `Vec` of responses that the caller (scheduler) must route to + /// the appropriate process or actor mailbox based on `channel_id`. + /// + /// Responses with `channel_id == 0` can be discarded. + /// + /// # No locks held + /// `buddy` and `queue.head` are mutated — no other thread touches them. + /// The only shared state is the queue's `tail`, which is written by producers + /// via `AtomicUsize::compare_exchange_weak`, never by this path. + pub fn process_messages(&mut self) -> Vec { + let mut responses = Vec::new(); + + while let Some(req) = self.queue.pop() { + let resp = match req { + PMRequest::Allocate { size_pages, token_sig, channel_id } => { + self.handle_allocate(size_pages, token_sig, channel_id) + } + PMRequest::Free { local_frame_idx, order } => { + self.handle_free(local_frame_idx, order) + } + PMRequest::Carve { offset_pages, size_pages, channel_id } => { + self.handle_carve(offset_pages, size_pages, channel_id) + } + PMRequest::None => continue, + }; + + // Only push responses that need routing. + // Free responses (channel_id == 0) are still pushed so callers can + // audit completion if needed; they may simply drop them. + if let Some(r) = resp { + responses.push(r); + } + } + + responses + } + + // ─── Introspection ──────────────────────────────────────────────────── + + /// Free pages remaining in this actor's buddy pool. + #[inline] + pub fn free_pages(&self) -> usize { + self.buddy.free_pages() + } + + /// Total pages this actor was initialised with. + #[inline] + pub fn total_pages(&self) -> usize { + self.buddy.total_pages() + } +} + +// ── Private handler implementations ────────────────────────────────────────── + +impl PMActor { + fn handle_allocate( + &mut self, + size_pages: usize, + token_sig: u32, + channel_id: u16, + ) -> Option { + match self.buddy.alloc_pages(size_pages) { + Some((rel_idx, order)) => { + // Translate relative index → absolute physical address. + let phys = PhysAddr(self.managed_range.0.0 + rel_idx as u64 * PAGE_SIZE); + + // Build a strong child capability. + // token_sig is widened and XOR'd with actor_id for uniqueness. + let cap = Capability { + object: CapObject::Memory { + phys, + // Report actual (rounded-up) page count so the caller + // passes the correct `order` back to Free. + size_pages: 1 << order, + }, + rights: CapRights::READ | CapRights::WRITE | CapRights::GRANT, + relation: Relation::Strong, + token_sig: (token_sig as u64) ^ self.actor_id ^ (rel_idx as u64), + }; + + Some(PMResponse { + channel_id, + result: PMResult::Allocated { cap, order }, + }) + } + None => { + // TODO: emit a ballooning request to a neighbour actor before + // propagating OutOfMemory upward. + Some(PMResponse { + channel_id, + result: PMResult::OutOfMemory { size_pages }, + }) + } + } + } + + fn handle_free( + &mut self, + local_frame_idx: usize, + order: usize, + ) -> Option { + // Validate bounds before touching the buddy tree. + let block_end = local_frame_idx.saturating_add(1usize << order); + if block_end > self.buddy.total_pages() { + // Corrupted or spoofed index — ignore silently in release, panic in debug. + debug_assert!( + false, + "PMActor {}: Free out of range (idx={}, order={}, total={})", + self.actor_id, + local_frame_idx, + order, + self.buddy.total_pages() + ); + return None; + } + + self.buddy.free(local_frame_idx, order); + + // Free never needs routing — return None to skip Vec push, or push with + // channel_id=0 for audit purposes. We skip to avoid unnecessary allocation. + // Callers that need a Free-complete signal should use a separate mechanism. + None + } + + fn handle_carve( + &mut self, + offset_pages: usize, + size_pages: usize, + channel_id: u16, + ) -> Option { + // Validate offset + size within range. + let range_pages = self.buddy.total_pages(); + if offset_pages >= range_pages + || size_pages == 0 + || offset_pages.saturating_add(size_pages) > range_pages + { + return Some(PMResponse { + channel_id, + result: PMResult::OutOfMemory { size_pages }, + }); + } + + let phys = PhysAddr(self.managed_range.0.0 + offset_pages as u64 * PAGE_SIZE); + let cap = Capability { + object: CapObject::Memory { phys, size_pages }, + rights: CapRights::READ | CapRights::WRITE | CapRights::GRANT, + relation: Relation::Strong, + token_sig: self.actor_id ^ offset_pages as u64, + }; + + Some(PMResponse { + channel_id, + result: PMResult::Carved { cap }, + }) + } +} diff --git a/kernel/src/mem/pmm.rs b/kernel/src/mem/pmm.rs index fa98a69..aacf4a7 100644 --- a/kernel/src/mem/pmm.rs +++ b/kernel/src/mem/pmm.rs @@ -4,19 +4,22 @@ use crate::mem::allocator::Locked; pub const PAGE_SIZE: u64 = 4096; pub struct BitmapPMM { - bitmap: &'static mut [u8], + bitmap: &'static mut [u8], total_pages: usize, - used_pages: usize, - last_idx: usize, + used_pages: usize, + /// Byte index hint: next search starts here to amortise O(N) scans. + last_byte: usize, } pub static PMM: Locked> = Locked::new(None); impl BitmapPMM { #[allow(dead_code)] - pub fn used_pages(&self) -> usize { self.used_pages } + pub fn used_pages(&self) -> usize { self.used_pages } #[allow(dead_code)] pub fn total_pages(&self) -> usize { self.total_pages } + #[allow(dead_code)] + pub fn free_pages(&self) -> usize { self.total_pages.saturating_sub(self.used_pages) } pub unsafe fn init(mmap: &limine::response::MemoryMapResponse, hhdm_offset: u64) { let max_addr = mmap.entries().iter() @@ -27,22 +30,28 @@ impl BitmapPMM { let total_pages = (max_addr / PAGE_SIZE) as usize; let bitmap_size = total_pages.div_ceil(8); - let bitmap_phys_addr = mmap.entries().iter() - .find(|e| e.entry_type == limine::memory_map::EntryType::USABLE && e.length >= bitmap_size as u64) + // Find a usable region large enough to hold the bitmap. + let bitmap_phys = mmap.entries().iter() + .find(|e| { + e.entry_type == limine::memory_map::EntryType::USABLE + && e.length >= bitmap_size as u64 + }) .map(|e| e.base) - .expect("PMM: Insufficient memory for bitmap"); + .expect("PMM: no usable region large enough for the bitmap"); - let bitmap_virt_ptr = (bitmap_phys_addr + hhdm_offset) as *mut u8; - let bitmap_slice = unsafe { core::slice::from_raw_parts_mut(bitmap_virt_ptr, bitmap_size) }; - bitmap_slice.fill(0xFF); + let bitmap_ptr = (bitmap_phys + hhdm_offset) as *mut u8; + // Mark everything as used (all bits = 1) and free usable entries below. + let bitmap = unsafe { core::slice::from_raw_parts_mut(bitmap_ptr, bitmap_size) }; + bitmap.fill(0xFF); let mut pmm = Self { - bitmap: bitmap_slice, + bitmap, total_pages, used_pages: total_pages, - last_idx: 0, + last_byte: 0, }; + // Free all usable pages … for entry in mmap.entries() { if entry.entry_type == limine::memory_map::EntryType::USABLE { for addr in (entry.base..entry.base + entry.length).step_by(PAGE_SIZE as usize) { @@ -51,71 +60,131 @@ impl BitmapPMM { } } - for addr in (bitmap_phys_addr..bitmap_phys_addr + bitmap_size as u64).step_by(PAGE_SIZE as usize) { + // … then re-lock the bitmap pages themselves … + for addr in (bitmap_phys..bitmap_phys + bitmap_size as u64).step_by(PAGE_SIZE as usize) { pmm.lock_frame(PhysAddr(addr)); } + // … and the null page (physical 0x0 must never be returned as a valid frame). pmm.lock_frame(PhysAddr(0)); - *PMM.lock() = Some(pmm); + *PMM.lock() = Some(pmm); } + // ── Core operations ─────────────────────────────────────────────────────── + + /// Mark a frame as free. Idempotent (double-free is a no-op, not UB). pub fn free_frame(&mut self, phys_addr: PhysAddr) { let idx = (phys_addr.0 / PAGE_SIZE) as usize; - if idx < self.total_pages { - let byte_idx = idx / 8; - let bit_idx = idx % 8; - if (self.bitmap[byte_idx] & (1 << bit_idx)) != 0 { - self.bitmap[byte_idx] &= !(1 << bit_idx); - self.used_pages -= 1; - } + if idx >= self.total_pages { return; } + let byte = idx / 8; + let bit = idx % 8; + if self.bitmap[byte] & (1 << bit) != 0 { + self.bitmap[byte] &= !(1 << bit); + self.used_pages -= 1; + // Pull the hint back so the freed page can be found quickly. + if byte < self.last_byte { self.last_byte = byte; } } } + /// Mark a frame as allocated (reserved). Idempotent. pub fn lock_frame(&mut self, phys_addr: PhysAddr) { let idx = (phys_addr.0 / PAGE_SIZE) as usize; - if idx < self.total_pages { - let byte_idx = idx / 8; - let bit_idx = idx % 8; - if (self.bitmap[byte_idx] & (1 << bit_idx)) == 0 { - self.bitmap[byte_idx] |= 1 << bit_idx; - self.used_pages += 1; - } + if idx >= self.total_pages { return; } + let byte = idx / 8; + let bit = idx % 8; + if self.bitmap[byte] & (1 << bit) == 0 { + self.bitmap[byte] |= 1 << bit; + self.used_pages += 1; } } -pub fn alloc_frame(&mut self) -> Option { - let start_byte = self.last_idx / 8; - for i in start_byte..(self.bitmap.len()) { - if self.bitmap[i] != 0xFF { - for bit in 0..8 { - let idx = i * 8 + bit; - if idx >= self.total_pages { return None; } - - let addr = PhysAddr(idx as u64 * PAGE_SIZE); - if !self.is_locked(addr) { - self.lock_frame(addr); - self.last_idx = idx; - return Some(addr); + /// Allocate one physical frame. + /// + /// Uses a two-pass search (linear scan from `last_byte` hint, then wraps + /// to 0 if not found in the first pass) to avoid returning `None` when + /// free frames exist before the hint. + pub fn alloc_frame(&mut self) -> Option { + let len = self.bitmap.len(); + + for pass in 0..2usize { + let (from, to) = if pass == 0 { + (self.last_byte, len) + } else { + (0, self.last_byte) + }; + + for byte_idx in from..to { + // Fast path: skip fully-used bytes. + if self.bitmap[byte_idx] == 0xFF { continue; } + + for bit in 0..8u8 { + if self.bitmap[byte_idx] & (1 << bit) == 0 { + let page_idx = byte_idx * 8 + bit as usize; + if page_idx >= self.total_pages { return None; } + + // Mark allocated. + self.bitmap[byte_idx] |= 1 << bit; + self.used_pages += 1; + self.last_byte = byte_idx; + + return Some(PhysAddr(page_idx as u64 * PAGE_SIZE)); } } } } - None + + None // genuinely out of memory } -#[inline] - fn is_locked(&self, addr: PhysAddr) -> bool { - let idx = (addr.0 / PAGE_SIZE) as usize; - (self.bitmap[idx / 8] & (1 << (idx % 8))) != 0 + /// Try to allocate `count` **contiguous** physical frames. + /// + /// Returns the base physical address of the run, or `None` if no run of + /// sufficient length exists. This is needed for (e.g.) allocating 2 MiB + /// huge-page aligned regions or DMA buffers that must be physically + /// contiguous. + /// + /// O(N) worst-case; use sparingly and prefer small counts. + pub fn alloc_contiguous(&mut self, count: usize) -> Option { + if count == 0 { return None; } + + let mut run_start = 0usize; + let mut run_len = 0usize; + + for page_idx in 0..self.total_pages { + let byte = page_idx / 8; + let bit = page_idx % 8; + if self.bitmap[byte] & (1 << bit) == 0 { + if run_len == 0 { run_start = page_idx; } + run_len += 1; + if run_len == count { + // Lock every frame in the run. + for i in run_start..run_start + count { + self.bitmap[i / 8] |= 1 << (i % 8); + } + self.used_pages += count; + self.last_byte = run_start / 8; + return Some(PhysAddr(run_start as u64 * PAGE_SIZE)); + } + } else { + run_len = 0; + } + } + + None } } +// ── Module-level convenience functions ─────────────────────────────────────── pub fn alloc_frame() -> Option { PMM.lock().as_mut()?.alloc_frame() } +pub fn alloc_contiguous(count: usize) -> Option { + PMM.lock().as_mut()?.alloc_contiguous(count) +} + pub fn free_frame(addr: PhysAddr) { if let Some(pmm) = PMM.lock().as_mut() { pmm.free_frame(addr); diff --git a/kernel/src/mem/vmm.rs b/kernel/src/mem/vmm.rs new file mode 100644 index 0000000..4f07d29 --- /dev/null +++ b/kernel/src/mem/vmm.rs @@ -0,0 +1,730 @@ +//! # Virtual Memory Manager (`src/mem/vmm.rs`) +//! +//! Manages address spaces (PML4 + VMA list) and provides the bridge between +//! the capability system and the hardware MMU. +//! +//! ## Changes vs. previous version +//! +//! 1. **ASID recycling** — replaced the monotonic `AtomicU16` with a 4096-bit +//! bitmap allocator (`AsidAllocator`). `AddressSpace::drop` returns the ASID +//! to the pool, making it available for future processes. The bitmap is +//! `const`-initializable (no heap), so no boot-ordering issues. +//! +//! 2. **INVPCID CPUID guard** — `init_cpu_features()` must be called once during +//! boot (before any `AddressSpace::activate()` or TLB flush). It checks +//! CPUID.07H:EBX[10] and sets `INVPCID_SUPPORTED`. `tlb_flush_asid` falls +//! back to a full CR3 reload on CPUs that lack the instruction. +//! +//! 3. **Zero-copy shared mappings** — new `AddressSpace::map_shared()` maps a +//! physical range owned by another capability into this address space without +//! copying the frames. `VmaBacking::Shared` stores the `phys_base` so the +//! physical address is retrievable at any time via `translate()` on the PT. +//! Frames are *not* freed on unmap — ownership stays with the issuing actor. +//! +//! All other behaviour (lazy demand paging, revocation, TLB management) is +//! unchanged from the previous version. + +#![allow(dead_code)] + +use alloc::vec::Vec; +use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; + +use crate::mem::address::{PhysAddr, VirtAddr}; +use crate::mem::allocator::Locked; +use crate::mem::paging::{PageTable, PageTableFlags}; +use crate::mem::pmm; +use crate::events::MMU_REVOCATION_QUEUE; + +extern crate alloc; + +// ══════════════════════════════════════════════════════════════════════════════ +// Error type +// ══════════════════════════════════════════════════════════════════════════════ + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VmError { + /// PMM returned `None` — no physical frames available. + OutOfMemory, + /// The requested virtual range overlaps an existing VMA. + RegionOverlap, + /// No VMA covers the given address. + RegionNotFound, + /// Address or size is not a multiple of 4096. + InvalidAlignment, + /// Size is zero, or `virt + size` would overflow. + InvalidRange, + /// Write fault on a read-only VMA, or exec fault on a NX VMA. + PermissionDenied, + /// Page-fault in a non-lazy (already-eager or fixed) region — hardware bug + /// or an exploit attempt; the faulting task must be killed. + UnexpectedFault, + /// Address outside the x86-64 canonical range. + NonCanonical, + /// ASID pool exhausted (> 4094 simultaneous address spaces). + AsidExhausted, +} + +impl core::fmt::Display for VmError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(match self { + Self::OutOfMemory => "physical memory exhausted", + Self::RegionOverlap => "virtual address range overlaps existing VMA", + Self::RegionNotFound => "no VMA at address", + Self::InvalidAlignment => "address/size not page-aligned", + Self::InvalidRange => "zero-size or overflowing range", + Self::PermissionDenied => "VMA permission denied", + Self::UnexpectedFault => "page fault in non-lazy region", + Self::NonCanonical => "non-canonical virtual address", + Self::AsidExhausted => "ASID/PCID pool exhausted", + }) + } +} + +// ══════════════════════════════════════════════════════════════════════════════ +// VMA flags +// ══════════════════════════════════════════════════════════════════════════════ + +bitflags::bitflags! { + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub struct VmaFlags: u32 { + const READ = 1 << 0; + const WRITE = 1 << 1; + const EXEC = 1 << 2; + const USER = 1 << 3; + const LAZY = 1 << 4; + const SHARED = 1 << 5; + const PINNED = 1 << 6; + const NOCACHE = 1 << 7; + const MMIO = 1 << 8; + } +} + +impl VmaFlags { + #[inline] + pub fn to_page_flags(self) -> PageTableFlags { + let mut f = PageTableFlags::PRESENT; + if self.contains(Self::WRITE) { f |= PageTableFlags::WRITABLE; } + if self.contains(Self::USER) { f |= PageTableFlags::USER; } + if !self.contains(Self::EXEC) { f |= PageTableFlags::NO_EXECUTE; } + if self.contains(Self::NOCACHE) || self.contains(Self::MMIO) { + f |= PageTableFlags::NO_CACHE | PageTableFlags::WRITE_THROUGH; + } + f + } +} + +// ══════════════════════════════════════════════════════════════════════════════ +// VMA backing +// ══════════════════════════════════════════════════════════════════════════════ + +#[derive(Debug)] +pub enum VmaBacking { + /// Anonymous pages (stack, heap, BSS). + /// Index `i` → frame for `virt_start + i * 4096`. `None` = not yet faulted in. + Anonymous(Vec>), + + /// Fixed physical range. Frames are **not** freed on unmap. + /// Used for MMIO, identity-mapped RAM, framebuffer, DMA buffers. + Physical(PhysAddr), + + /// Zero-copy borrow of another actor's frames. + /// + /// `phys_base` is the physical address of the first page; the mapping covers + /// exactly `(virt_end - virt_start) / 4096` pages. + /// Frames are owned by `owner_cap` and **never** freed by this VMA. + Shared { + /// Token of the capability that owns the frames. + owner_cap: u64, + /// Physical base (first page of the shared region). + phys_base: PhysAddr, + }, +} + +impl VmaBacking { + #[inline] + fn phys_for_page(&self, page_idx: usize) -> Option { + match self { + Self::Anonymous(frames) => frames.get(page_idx).copied().flatten(), + Self::Physical(base) => Some(PhysAddr(base.0 + page_idx as u64 * 4096)), + Self::Shared { phys_base, .. } => { + Some(PhysAddr(phys_base.0 + page_idx as u64 * 4096)) + } + } + } + + /// Whether this backing owns its frames (should the PMM free them on unmap). + #[inline] + fn owns_frames(&self) -> bool { + matches!(self, Self::Anonymous(_)) + } +} + +// ══════════════════════════════════════════════════════════════════════════════ +// VMA region +// ══════════════════════════════════════════════════════════════════════════════ + +#[derive(Debug)] +pub struct VmaRegion { + pub virt_start: VirtAddr, + pub virt_end: VirtAddr, + pub flags: VmaFlags, + pub cap_token: u64, + pub backing: VmaBacking, +} + +impl VmaRegion { + #[inline] pub fn size(&self) -> u64 { self.virt_end.0 - self.virt_start.0 } + #[inline] pub fn pages(&self) -> usize { (self.size() / 4096) as usize } + #[inline] pub fn contains(&self, a: VirtAddr) -> bool { + a.0 >= self.virt_start.0 && a.0 < self.virt_end.0 + } +} + +// ══════════════════════════════════════════════════════════════════════════════ +// ASID / PCID allocator (bitmap-based, const-initializable, O(1) amortised) +// ══════════════════════════════════════════════════════════════════════════════ + +/// x86-64 PCIDs: 0 (kernel, no PCID tagging) and 4095 (reserved by spec). +/// Valid user ASIDs: 1 – 4094 inclusive. +/// +/// Layout: `bitmap[i]` bit `j` represents ASID `i * 32 + j`. +/// Bit set = in use. +struct AsidAllocator { + /// 4096 bits in 128 × u32 words. + bitmap: [u32; 128], + /// Start-of-next-search hint (in ASID units, not word units). + next_hint: u16, +} + +impl AsidAllocator { + const fn new() -> Self { + let mut bm = [0u32; 128]; + // Mark ASID 0 (kernel) as permanently in-use. + bm[0] |= 1u32; + // Mark ASID 4095 (reserved by Intel spec) as permanently in-use. + // 4095 / 32 = 127, bit 31. + bm[127] |= 1u32 << 31; + Self { bitmap: bm, next_hint: 1 } + } + + /// Allocate the lowest free ASID in [1, 4094]. O(1) amortised with hint. + fn alloc(&mut self) -> Option { + // Two-pass: start from hint, wrap around if necessary. + for pass in 0..2usize { + let start = if pass == 0 { self.next_hint as usize } else { 1 }; + let end = if pass == 0 { 4095usize } else { self.next_hint as usize }; + + let mut asid = start; + while asid < end { + let word = asid / 32; + let bit = asid % 32; + if self.bitmap[word] & (1u32 << bit) == 0 { + self.bitmap[word] |= 1u32 << bit; + self.next_hint = ((asid + 1) as u16).min(4094); + return Some(asid as u16); + } + // Skip fully-used 32-bit words for amortised O(1). + if self.bitmap[word] == u32::MAX { + asid = (word + 1) * 32; + } else { + asid += 1; + } + } + } + None // pool truly exhausted + } + + /// Return an ASID to the pool. + fn free(&mut self, asid: u16) { + if asid == 0 || asid >= 4095 { + return; // sentinel values — never freed + } + let word = asid as usize / 32; + let bit = asid as usize % 32; + // Clear the bit. Idempotent: double-free is safe but wasteful. + self.bitmap[word] &= !(1u32 << bit); + // Pull hint back so reuse happens quickly. + if asid < self.next_hint { + self.next_hint = asid; + } + } +} + +static ASID_ALLOC: Locked = Locked::new(AsidAllocator::new()); + +#[inline] +fn alloc_asid() -> Result { + ASID_ALLOC.lock().alloc().ok_or(VmError::AsidExhausted) +} + +#[inline] +fn free_asid(asid: u16) { + ASID_ALLOC.lock().free(asid); +} + +// ══════════════════════════════════════════════════════════════════════════════ +// CPU feature detection (call once during boot, before first activate()) +// ══════════════════════════════════════════════════════════════════════════════ + +/// Set to `true` at boot if CPUID.07H:EBX[10] = 1 (INVPCID supported). +static INVPCID_SUPPORTED: AtomicBool = AtomicBool::new(false); + +/// Detect and record the availability of INVPCID. +/// +/// Must be called **once** during kernel init (before `init_kernel_space` or +/// any `AddressSpace::activate`). Idempotent if called more than once. +pub fn init_cpu_features() { + // CPUID leaf 7, sub-leaf 0 — INVPCID is EBX[10]. + // + // LLVM permanently reserves RBX for its own use in inline asm, so we + // cannot name it as an operand directly. The standard workaround is to + // spill RBX around the CPUID instruction ourselves. + let ebx: u32; + unsafe { + core::arch::asm!( + // Spill rbx (LLVM's reserved reg) to a caller-saved scratch reg. + "mov {tmp:r}, rbx", + "mov eax, 7", + "xor ecx, ecx", + "cpuid", + // Move the result out before restoring rbx. + "mov {out:e}, ebx", + "mov rbx, {tmp:r}", + tmp = out(reg) _, // any scratch register chosen by LLVM + out = out(reg) ebx, + out("eax") _, + out("ecx") _, + out("edx") _, + options(nostack, preserves_flags), + ); + } + let supported = (ebx >> 10) & 1 == 1; + INVPCID_SUPPORTED.store(supported, Ordering::Relaxed); +} + +// ══════════════════════════════════════════════════════════════════════════════ +// Address space +// ══════════════════════════════════════════════════════════════════════════════ + +pub struct AddressSpace { + /// Hardware PCID written to CR3 bits 11:0. + pub asid: u16, + /// Physical address of the PML4 page. + pub pml4_phys: PhysAddr, + /// Sorted (by virt_start), non-overlapping VMA list. + regions: Vec, + /// HHDM offset for dereferencing page-table pages. + hhdm: u64, +} + +// ── Private helpers ─────────────────────────────────────────────────────────── + +impl AddressSpace { + #[inline] + unsafe fn pml4_raw(&self) -> *mut PageTable { + self.pml4_phys.to_virt(self.hhdm).as_mut_ptr::() + } + + fn check_overlap(&self, start: VirtAddr, end: VirtAddr) -> Result<(), VmError> { + for r in &self.regions { + if r.virt_start.0 < end.0 && r.virt_end.0 > start.0 { + return Err(VmError::RegionOverlap); + } + } + Ok(()) + } + + fn insert_sorted(&mut self, region: VmaRegion) { + let pos = self.regions + .partition_point(|r| r.virt_start.0 < region.virt_start.0); + self.regions.insert(pos, region); + } + + fn find_idx(&self, addr: VirtAddr) -> Option { + let pos = self.regions.partition_point(|r| r.virt_end.0 <= addr.0); + self.regions.get(pos).filter(|r| r.contains(addr)).map(|_| pos) + } + + /// Unmap pages + free frames (if owned) for one region. + fn do_unmap(pml4: *mut PageTable, region: &VmaRegion, hhdm: u64) { + let pml4 = unsafe { &mut *pml4 }; + let own = region.backing.owns_frames(); + for i in 0..region.pages() { + let virt = VirtAddr(region.virt_start.0 + i as u64 * 4096); + pml4.unmap_page(virt, hhdm); + if own { + if let Some(frame) = region.backing.phys_for_page(i) { + pmm::free_frame(frame); + } + } + } + } + + fn do_revoke_by_token(&mut self, cap_token: u64) { + let hhdm = self.hhdm; + let pml4 = unsafe { self.pml4_raw() }; + + let indices: Vec = self.regions + .iter() + .enumerate() + .filter(|(_, r)| r.cap_token == cap_token && !r.flags.contains(VmaFlags::PINNED)) + .map(|(i, _)| i) + .collect(); + + for idx in indices.into_iter().rev() { + let region = self.regions.remove(idx); + Self::do_unmap(pml4, ®ion, hhdm); + } + } + + /// Drain the MMU revocation queue and apply all pending token revocations. + /// Called by the page-fault handler before acquiring the VMM lock to + /// prevent deadlocks with the capability subsystem. + pub fn process_pending_revocations(&mut self) { + let mut needs_flush = false; + while let Some(token) = MMU_REVOCATION_QUEUE.pop() { + self.do_revoke_by_token(token); + needs_flush = true; + } + if needs_flush { + tlb_flush_asid(self.asid); + } + } +} + +// ── Public API ──────────────────────────────────────────────────────────────── + +impl AddressSpace { + // ─── Construction ────────────────────────────────────────────────────── + + /// Allocate a fresh, empty address space with a zeroed PML4. + pub fn new(hhdm: u64) -> Result { + let pml4_phys = pmm::alloc_frame().ok_or(VmError::OutOfMemory)?; + unsafe { + core::ptr::write_bytes(pml4_phys.to_virt(hhdm).as_mut_ptr::(), 0, 4096); + } + Ok(Self { + asid: alloc_asid()?, + pml4_phys, + regions: Vec::new(), + hhdm, + }) + } + + /// Wrap an already-active PML4 (bootloader-provided kernel table). + /// ASID 0 = kernel "no PCID tagging"; it is never returned to the pool. + pub fn from_active(pml4_phys: PhysAddr, asid: u16, hhdm: u64) -> Self { + Self { asid, pml4_phys, regions: Vec::new(), hhdm } + } + + // ─── Anonymous / fixed mapping ───────────────────────────────────────── + + /// Map `size` bytes of virtual space starting at `virt`. + /// + /// | `phys` | `VmaFlags::LAZY` | Behaviour | + /// |-----------|------------------|-----------------------------------------------| + /// | `Some(p)` | any | Fixed physical (MMIO / identity / DMA) | + /// | `None` | not set | Eager anonymous — allocate + zero + map now | + /// | `None` | set | Lazy anonymous — map frames on first fault | + pub fn map_region( + &mut self, + virt: VirtAddr, + phys: Option, + size: u64, + flags: VmaFlags, + cap_token: u64, + ) -> Result { + if size == 0 { return Err(VmError::InvalidRange); } + if virt.0 & 0xFFF != 0 || size & 0xFFF != 0 { + return Err(VmError::InvalidAlignment); + } + + let virt_end = VirtAddr(virt.0.checked_add(size).ok_or(VmError::InvalidRange)?); + self.check_overlap(virt, virt_end)?; + + let pml4 = unsafe { &mut *self.pml4_raw() }; + let page_flags = flags.to_page_flags(); + let page_count = (size / 4096) as usize; + let hhdm = self.hhdm; + + let backing = match phys { + Some(base) => { + pml4.map_region(virt, base, size, page_flags, hhdm); + VmaBacking::Physical(base) + } + None if flags.contains(VmaFlags::LAZY) => { + let mut frames = Vec::with_capacity(page_count); + frames.resize_with(page_count, || None); + VmaBacking::Anonymous(frames) + } + None => { + let mut frames = Vec::with_capacity(page_count); + for i in 0..page_count { + let frame = pmm::alloc_frame().ok_or(VmError::OutOfMemory)?; + unsafe { + core::ptr::write_bytes(frame.to_virt(hhdm).as_mut_ptr::(), 0, 4096); + } + pml4.map_page( + VirtAddr(virt.0 + i as u64 * 4096), + frame, + page_flags, + hhdm, + ); + frames.push(Some(frame)); + } + VmaBacking::Anonymous(frames) + } + }; + + self.insert_sorted(VmaRegion { virt_start: virt, virt_end, flags, cap_token, backing }); + Ok(virt) + } + + // ─── Zero-copy shared mapping (new) ─────────────────────────────────── + + /// Map `page_count` pages of physical memory owned by `owner_cap` into + /// this address space at `virt`. + /// + /// This is the primitive behind Elyz's zero-copy transfer: the physical + /// frames stay in place; only the PTE entries (rights) move between address + /// spaces. The frames are **not** freed when this mapping is unmapped — + /// the owning actor's capability retains that responsibility. + /// + /// # Revocation + /// Pass `owner_cap` as `cap_token` in the VMA so that `revoke_by_token` + /// finds and unmaps this shared view when the source capability is revoked. + pub fn map_shared( + &mut self, + virt: VirtAddr, + phys_base: PhysAddr, + page_count: usize, + flags: VmaFlags, + owner_cap: u64, + ) -> Result { + if page_count == 0 { return Err(VmError::InvalidRange); } + if virt.0 & 0xFFF != 0 { return Err(VmError::InvalidAlignment); } + + let size = page_count as u64 * 4096; + let virt_end = VirtAddr(virt.0.checked_add(size).ok_or(VmError::InvalidRange)?); + self.check_overlap(virt, virt_end)?; + + let pml4 = unsafe { &mut *self.pml4_raw() }; + let page_flags = flags.to_page_flags(); + let hhdm = self.hhdm; + + // Eagerly map all pages — the physical addresses are already known. + for i in 0..page_count { + pml4.map_page( + VirtAddr(virt.0 + i as u64 * 4096), + PhysAddr(phys_base.0 + i as u64 * 4096), + page_flags, + hhdm, + ); + } + + self.insert_sorted(VmaRegion { + virt_start: virt, + virt_end, + flags: flags | VmaFlags::SHARED, + cap_token: owner_cap, + backing: VmaBacking::Shared { owner_cap, phys_base }, + }); + + Ok(virt) + } + + // ─── Demand paging ───────────────────────────────────────────────────── + + /// Handle a hardware page fault at `fault_addr`. + /// + /// Returns `Ok(())` if the fault was a valid lazy demand-page (caller should + /// `iretq` to retry). Returns `Err` for illegal accesses. + pub fn handle_fault(&mut self, fault_addr: VirtAddr, write: bool) -> Result<(), VmError> { + let hhdm = self.hhdm; + let idx = self.find_idx(fault_addr).ok_or(VmError::RegionNotFound)?; + + { + let region = &self.regions[idx]; + if write && !region.flags.contains(VmaFlags::WRITE) { + return Err(VmError::PermissionDenied); + } + if !region.flags.contains(VmaFlags::LAZY) { + return Err(VmError::UnexpectedFault); + } + } + + let region = &mut self.regions[idx]; + let page_idx = ((fault_addr.0 - region.virt_start.0) / 4096) as usize; + let page_virt = VirtAddr(region.virt_start.0 + page_idx as u64 * 4096); + let page_flags = region.flags.to_page_flags(); + + let VmaBacking::Anonymous(ref mut frames) = region.backing else { + return Err(VmError::RegionNotFound); + }; + + if frames[page_idx].is_some() { + // SMP race: another core already mapped this page. + return Ok(()); + } + + let frame = pmm::alloc_frame().ok_or(VmError::OutOfMemory)?; + unsafe { + core::ptr::write_bytes(frame.to_virt(hhdm).as_mut_ptr::(), 0, 4096); + } + frames[page_idx] = Some(frame); + + let pml4 = unsafe { &mut *self.pml4_raw() }; + pml4.map_page(page_virt, frame, page_flags, hhdm); + + Ok(()) + } + + // ─── Unmapping ───────────────────────────────────────────────────────── + + /// Unmap the VMA containing `virt`, free its frames (if owned), flush TLB. + pub fn unmap_region(&mut self, virt: VirtAddr) -> Result<(), VmError> { + let idx = self.find_idx(virt).ok_or(VmError::RegionNotFound)?; + let region = self.regions.remove(idx); + let pml4 = unsafe { self.pml4_raw() }; + Self::do_unmap(pml4, ®ion, self.hhdm); + tlb_flush_asid(self.asid); + Ok(()) + } + + // ─── Capability revocation ───────────────────────────────────────────── + + /// Atomically unmap all VMAs associated with `cap_token` (skip PINNED). + /// + /// Hardware access is terminated before this function returns. + /// Shared-backed VMAs are unmapped from the PT without freeing frames. + pub fn revoke_by_token(&mut self, cap_token: u64) { + let hhdm = self.hhdm; + let pml4 = unsafe { self.pml4_raw() }; + + let indices: Vec = self.regions + .iter() + .enumerate() + .filter(|(_, r)| { + r.cap_token == cap_token && !r.flags.contains(VmaFlags::PINNED) + }) + .map(|(i, _)| i) + .collect(); + + for idx in indices.into_iter().rev() { + let region = self.regions.remove(idx); + Self::do_unmap(pml4, ®ion, hhdm); + } + + tlb_flush_asid(self.asid); + } + + // ─── Address translation ─────────────────────────────────────────────── + + /// Walk the live page table to translate `virt` → physical address. + pub fn translate(&self, virt: VirtAddr) -> Option { + unsafe { (*self.pml4_raw()).translate(virt, self.hhdm) } + } + + // ─── Activation ──────────────────────────────────────────────────────── + + /// Load this address space into the CPU (context switch). + /// + /// Sets CR3 with bit 63 = NOFLUSH to preserve PCID-tagged TLB entries. + /// Requires CR4.PCIDE = 1 (enabled by the bootloader/early init). + /// + /// # Safety + /// Must only be called from a code path that is identity-mapped in every + /// address space (e.g. the kernel `.text` section reachable via HHDM). + pub unsafe fn activate(&self) { + let cr3 = self.pml4_phys.0 | u64::from(self.asid) | (1u64 << 63); + unsafe { + core::arch::asm!( + "mov cr3, {0}", + in(reg) cr3, + options(nostack, preserves_flags), + ); + } + } + + // ─── Introspection ───────────────────────────────────────────────────── + + #[inline] pub fn regions(&self) -> &[VmaRegion] { &self.regions } + #[inline] pub fn region_count(&self) -> usize { self.regions.len() } + + pub fn find_region(&self, addr: VirtAddr) -> Option<&VmaRegion> { + self.find_idx(addr).map(|i| &self.regions[i]) + } +} + +impl Drop for AddressSpace { + fn drop(&mut self) { + let hhdm = self.hhdm; + let pml4 = unsafe { self.pml4_raw() }; + while let Some(region) = self.regions.pop() { + Self::do_unmap(pml4, ®ion, hhdm); + } + pmm::free_frame(self.pml4_phys); + // Return the ASID to the pool so it can be reused by future processes. + // ASID 0 is the kernel sentinel — never freed (guarded inside free_asid). + free_asid(self.asid); + } +} + +// ══════════════════════════════════════════════════════════════════════════════ +// TLB management +// ══════════════════════════════════════════════════════════════════════════════ + +/// Flush all TLB entries tagged with `asid` (PCID) on the current core. +/// +/// Uses `INVPCID` type-1 (single-context flush) when available (Broadwell+, +/// CPUID.07H:EBX[10] = 1). Falls back to a full CR3 reload on older CPUs. +/// +/// **SMP note**: on multi-core systems a TLB-shootdown IPI to all remote cores +/// must be added once the LAPIC driver and scheduler are online. +pub fn tlb_flush_asid(asid: u16) { + if INVPCID_SUPPORTED.load(Ordering::Relaxed) { + #[repr(C, packed)] + struct InvpcidDesc { pcid: u64, addr: u64 } + + let desc = InvpcidDesc { pcid: asid as u64, addr: 0 }; + unsafe { + core::arch::asm!( + "invpcid {ty}, [{desc}]", + ty = in(reg) 1u64, // type 1 = single-context flush + desc = in(reg) &desc, + options(nostack, preserves_flags), + ); + } + } else { + // Fallback: full TLB flush via CR3 reload (clears all PCID entries). + tlb_flush_all(); + } +} + +/// Full TLB flush (all PCIDs, all addresses) via CR3 reload. +pub fn tlb_flush_all() { + unsafe { + let mut cr3: u64; + core::arch::asm!("mov {0}, cr3", out(reg) cr3, + options(nostack, preserves_flags)); + // Writing CR3 without bit 63 forces a full flush. + core::arch::asm!("mov cr3, {0}", + in(reg) cr3 & !(1u64 << 63), + options(nostack, preserves_flags)); + } +} + +// ══════════════════════════════════════════════════════════════════════════════ +// Global kernel address space +// ══════════════════════════════════════════════════════════════════════════════ + +/// The one kernel address space. Initialised once during boot. +pub static KERNEL_SPACE: Locked> = Locked::new(None); + +/// Register the already-active PML4 as the kernel address space. +/// +/// ASID 0 = PCID 0 = kernel (no per-process PCID tagging). +/// Must be called after `BitmapPMM::init` and `init_cpu_features`. +pub fn init_kernel_space(pml4_phys: PhysAddr, hhdm: u64) { + *KERNEL_SPACE.lock() = Some(AddressSpace::from_active(pml4_phys, 0, hhdm)); +} diff --git a/limine.conf b/limine.conf index 0978db7..9fcaf3c 100644 --- a/limine.conf +++ b/limine.conf @@ -2,7 +2,7 @@ timeout: 3 # The entry name that will be displayed in the boot menu. -/Elyz (RINA KERNEL) +/Elyz (LISA KERNEL) # We use the Limine boot protocol. protocol: limine