feat: major memory managment & PMactor
This commit is contained in:
Faynot
@@ -1,8 +1,10 @@
|
||||
pub mod descriptor;
|
||||
pub mod object;
|
||||
|
||||
use alloc::vec::Vec;
|
||||
use crate::events::MMU_REVOCATION_QUEUE;
|
||||
use crate::mem::allocator::Locked;
|
||||
use alloc::vec::Vec;
|
||||
|
||||
pub use descriptor::*;
|
||||
|
||||
pub struct CNodeSlot {
|
||||
@@ -27,44 +29,109 @@ impl CNode {
|
||||
}
|
||||
|
||||
pub fn insert(&self, slot: usize, cap: Capability) -> Result<(), &'static str> {
|
||||
if slot >= self.slots.len() { return Err("Index out of bounds"); }
|
||||
if slot >= self.slots.len() {
|
||||
return Err("Index out of bounds");
|
||||
}
|
||||
let mut s = self.slots[slot].lock();
|
||||
s.cap = cap;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn mint(&self, src: usize, dest: usize, relation: Relation, rights: CapRights) -> Result<(), &'static str> {
|
||||
let mut slots = (self.slots[src].lock(), self.slots[dest].lock());
|
||||
let (src_slot, mut dest_slot) = (&slots.0, &mut slots.1);
|
||||
if src >= self.slots.len() || dest >= self.slots.len() {
|
||||
return Err("Index out of bounds");
|
||||
}
|
||||
|
||||
if !src_slot.cap.is_valid() { return Err("Source empty"); }
|
||||
if !src_slot.cap.rights.contains(CapRights::GRANT) { return Err("Insufficient rights to mint"); }
|
||||
// Защита от дедлока: если src == dest, операция не имеет смысла
|
||||
if src == dest {
|
||||
return Err("Source and destination slots must be different");
|
||||
}
|
||||
|
||||
// Захватываем блокировки в строгом порядке индексов для предотвращения инверсии блокировок (Lock Ranking)
|
||||
let mut _guard_low;
|
||||
let mut _guard_high;
|
||||
|
||||
let (src_slot, dest_slot) = if src < dest {
|
||||
_guard_low = self.slots[src].lock();
|
||||
_guard_high = self.slots[dest].lock();
|
||||
(&mut *_guard_low, &mut *_guard_high)
|
||||
} else {
|
||||
_guard_low = self.slots[dest].lock();
|
||||
_guard_high = self.slots[src].lock();
|
||||
(&mut *_guard_high, &mut *_guard_low)
|
||||
};
|
||||
|
||||
if !src_slot.cap.is_valid() {
|
||||
return Err("Source slot is empty");
|
||||
}
|
||||
if !src_slot.cap.rights.contains(CapRights::GRANT) {
|
||||
return Err("Insufficient rights to mint (Missing GRANT flag)");
|
||||
}
|
||||
|
||||
// Права дочернего дескриптора не могут превышать права родительского
|
||||
let final_rights = src_slot.cap.rights & rights;
|
||||
|
||||
|
||||
dest_slot.cap = src_slot.cap;
|
||||
dest_slot.cap.rights = final_rights;
|
||||
dest_slot.cap.relation = relation;
|
||||
dest_slot.parent_idx = Some(src);
|
||||
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn revoke(&self, slot_idx: usize) {
|
||||
for i in 0..self.slots.len() {
|
||||
let mut should_clear = false;
|
||||
{
|
||||
let child = self.slots[i].lock();
|
||||
if child.parent_idx == Some(slot_idx) {
|
||||
should_clear = true;
|
||||
}
|
||||
}
|
||||
/// Публичный метод отзыва прав.
|
||||
/// Каскадно аннулирует все дочерние дескрипторы и отправляет их токены в lock-free очередь VMM.
|
||||
pub fn revoke(&self, slot_idx: usize) -> Result<(), &'static str> {
|
||||
if slot_idx >= self.slots.len() {
|
||||
return Err("Index out of bounds");
|
||||
}
|
||||
|
||||
if should_clear {
|
||||
self.revoke(i);
|
||||
let mut child = self.slots[i].lock();
|
||||
child.cap = Capability::empty();
|
||||
child.parent_idx = None;
|
||||
// Запускаем рекурсивный отзыв
|
||||
self.revoke_internal(slot_idx);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Внутренний метод каскадного удаления.
|
||||
/// Вынесен отдельно, чтобы избежать удержания блокировок при переходе на следующий уровень рекурсии.
|
||||
fn revoke_internal(&self, slot_idx: usize) {
|
||||
// 1. Сначала рекурсивно ищем и уничтожаем всех потомков данного слота
|
||||
for i in 0..self.slots.len() {
|
||||
let is_child = {
|
||||
let child = self.slots[i].lock();
|
||||
child.parent_idx == Some(slot_idx)
|
||||
};
|
||||
|
||||
if is_child {
|
||||
// Рекурсивный спуск. Блокировка с child[i] к этому моменту уже снята, дедлока нет.
|
||||
self.revoke_internal(i);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Теперь уничтожаем сам дескриптор в текущем слоте и отправляем его токен на отзыв в MMU
|
||||
let token_to_revoke = {
|
||||
let mut slot = self.slots[slot_idx].lock();
|
||||
if slot.cap.is_valid() {
|
||||
let token = slot.cap.token_sig;
|
||||
|
||||
// Перезаписываем пустой заглушкой (уничтожаем сильную ссылку ядра)
|
||||
slot.cap = Capability::empty();
|
||||
slot.parent_idx = None;
|
||||
|
||||
Some(token)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
// 3. Если слот содержал валидный токен, асинхронно уведомляем VMM актёра через lock-free кольцевой буфер
|
||||
if let Some(token) = token_to_revoke {
|
||||
// В Elyz 0xDEAD_BEEF используется для сырого Untyped, его в MMU слать бессмысленно
|
||||
if token != 0 && token != 0xDEAD_BEEF {
|
||||
if let Err(e) = MMU_REVOCATION_QUEUE.push(token) {
|
||||
// Переполнение очереди отзыва — критический сбой планировщика ресурсов ядра
|
||||
panic!("FATAL: Критическое переполнение очереди отзыва MMU: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
69
kernel/src/cpu/idt.rs
Normal file
69
kernel/src/cpu/idt.rs
Normal file
@@ -0,0 +1,69 @@
|
||||
use core::arch::asm;
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
#[repr(C, packed)]
|
||||
pub struct IdtEntry {
|
||||
offset_low: u16,
|
||||
selector: u16,
|
||||
ist: u8,
|
||||
type_attr: u8,
|
||||
offset_mid: u16,
|
||||
offset_high: u32,
|
||||
ignore: u32,
|
||||
}
|
||||
|
||||
impl IdtEntry {
|
||||
pub const fn new() -> Self {
|
||||
Self {
|
||||
offset_low: 0,
|
||||
selector: 0,
|
||||
ist: 0,
|
||||
type_attr: 0,
|
||||
offset_mid: 0,
|
||||
offset_high: 0,
|
||||
ignore: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_handler(&mut self, handler: u64, selector: u16, flags: u8) {
|
||||
self.offset_low = handler as u16;
|
||||
self.selector = selector;
|
||||
self.ist = 0;
|
||||
self.type_attr = flags | 0x80; // Present bit
|
||||
self.offset_mid = (handler >> 16) as u16;
|
||||
self.offset_high = (handler >> 32) as u32;
|
||||
self.ignore = 0;
|
||||
}
|
||||
}
|
||||
|
||||
#[repr(C, packed)]
|
||||
pub struct IdtPtr {
|
||||
limit: u16,
|
||||
base: u64,
|
||||
}
|
||||
|
||||
pub struct InterruptDescriptorTable {
|
||||
entries: [IdtEntry; 256],
|
||||
}
|
||||
|
||||
impl InterruptDescriptorTable {
|
||||
pub const fn new() -> Self {
|
||||
Self { entries: [IdtEntry::new(); 256] }
|
||||
}
|
||||
|
||||
pub fn set_handler(&mut self, vector: u8, handler: u64) {
|
||||
// 0x8E = Interrupt Gate, Ring 0, Present
|
||||
self.entries[vector as usize].set_handler(handler, 0x28, 0x8E); // 0x28 - Kernel CS в Limine (по стандарту 5-й сегмент)
|
||||
}
|
||||
|
||||
pub unsafe fn load(&'static self) {
|
||||
let ptr = IdtPtr {
|
||||
limit: (core::mem::size_of::<Self>() - 1) as u16,
|
||||
base: self as *const _ as u64,
|
||||
};
|
||||
// Исправление для Rust 2024: явный unsafe блок внутри unsafe fn
|
||||
unsafe {
|
||||
asm!("lidt [{}]", in(reg) &ptr, options(readonly, nostack, preserves_flags));
|
||||
}
|
||||
}
|
||||
}
|
||||
113
kernel/src/cpu/interrupts.rs
Normal file
113
kernel/src/cpu/interrupts.rs
Normal file
@@ -0,0 +1,113 @@
|
||||
use core::arch::global_asm;
|
||||
use core::arch::asm;
|
||||
use crate::mem::vmm::KERNEL_SPACE; // Оставили только один импорт
|
||||
use crate::mem::address::VirtAddr;
|
||||
|
||||
// Глобальная статическая таблица дескрипторов прерываний (IDT)
|
||||
pub static mut IDT: crate::cpu::idt::InterruptDescriptorTable = crate::cpu::idt::InterruptDescriptorTable::new();
|
||||
|
||||
// Низкоуровневый ассемблерный трамплин (Context Save & Restore)
|
||||
global_asm!(
|
||||
".global page_fault_stub",
|
||||
"page_fault_stub:",
|
||||
"push rax",
|
||||
"push rcx",
|
||||
"push rdx",
|
||||
"push rbx",
|
||||
"push rbp",
|
||||
"push rsi",
|
||||
"push rdi",
|
||||
"push r8",
|
||||
"push r9",
|
||||
"push r10",
|
||||
"push r11",
|
||||
"push r12",
|
||||
"push r13",
|
||||
"push r14",
|
||||
"push r15",
|
||||
|
||||
"mov rdi, [rsp + 15*8]",
|
||||
"call rust_page_fault_handler",
|
||||
|
||||
"pop r15",
|
||||
"pop r14",
|
||||
"pop r13",
|
||||
"pop r12",
|
||||
"pop r11",
|
||||
"pop r10",
|
||||
"pop r9",
|
||||
"pop r8",
|
||||
"pop rdi",
|
||||
"pop rsi",
|
||||
"pop rbp",
|
||||
"pop rbx",
|
||||
"pop rdx",
|
||||
"pop rcx",
|
||||
"pop rax",
|
||||
|
||||
"add rsp, 8",
|
||||
"iretq"
|
||||
);
|
||||
|
||||
unsafe extern "C" {
|
||||
fn page_fault_stub();
|
||||
}
|
||||
|
||||
pub fn init_idt() {
|
||||
unsafe {
|
||||
let idt_mut_ptr = core::ptr::addr_of_mut!(IDT);
|
||||
(*idt_mut_ptr).set_handler(14, page_fault_stub as u64);
|
||||
let idt_static_ref: &'static crate::cpu::idt::InterruptDescriptorTable = &*core::ptr::addr_of!(IDT);
|
||||
idt_static_ref.load();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn process_deferred_mmu_events() {
|
||||
let mut vmm_guard = KERNEL_SPACE.lock();
|
||||
if let Some(space) = vmm_guard.as_mut() {
|
||||
space.process_pending_revocations();
|
||||
}
|
||||
}
|
||||
|
||||
#[unsafe(no_mangle)]
|
||||
pub extern "C" fn rust_page_fault_handler(error_code: u64) {
|
||||
let fault_addr: u64;
|
||||
unsafe {
|
||||
asm!("mov {}, cr2", out(reg) fault_addr, options(nomem, nostack, preserves_flags));
|
||||
}
|
||||
|
||||
let write = (error_code & 0x2) != 0;
|
||||
let present = (error_code & 0x1) != 0;
|
||||
let virt_addr = VirtAddr(fault_addr);
|
||||
|
||||
// 1. Сначала применяем все отложенные отзывы прав из других Actor'ов (чтобы избежать Deadlock)
|
||||
process_deferred_mmu_events();
|
||||
|
||||
// 2. Теперь захватываем VMM для обработки текущего сбоя
|
||||
let mut vmm_guard = KERNEL_SPACE.lock();
|
||||
|
||||
if let Some(space) = vmm_guard.as_mut() {
|
||||
match space.handle_fault(virt_addr, write) {
|
||||
Ok(_) => return,
|
||||
Err(e) => {
|
||||
panic!(
|
||||
"KERNEL PANIC: Необработанный сбой виртуальной памяти (Page Fault)!\n\
|
||||
Адрес: {:#X}\n\
|
||||
Режим: {}\n\
|
||||
Присутствие: {}\n\
|
||||
Причина VMM: {:?}",
|
||||
fault_addr,
|
||||
if write { "WRITE" } else { "READ" },
|
||||
if present { "YES (Protection Violation)" } else { "NO (Not Present)" },
|
||||
e
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
panic!(
|
||||
"KERNEL PANIC: Критический Page Fault до аллокации глобального KERNEL_SPACE!\n\
|
||||
Адрес сбоя: {:#X}",
|
||||
fault_addr
|
||||
);
|
||||
}
|
||||
}
|
||||
2
kernel/src/cpu/mod.rs
Normal file
2
kernel/src/cpu/mod.rs
Normal file
@@ -0,0 +1,2 @@
|
||||
pub mod idt;
|
||||
pub mod interrupts;
|
||||
75
kernel/src/events.rs
Normal file
75
kernel/src/events.rs
Normal file
@@ -0,0 +1,75 @@
|
||||
use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
|
||||
const QUEUE_SIZE: usize = 1024; // Должно быть степенью двойки для быстрой битовой маски
|
||||
const QUEUE_MASK: usize = QUEUE_SIZE - 1;
|
||||
|
||||
/// Lock-free MPSC (Multi-Producer, Single-Consumer) очередь.
|
||||
/// Оптимизирована для передачи токенов инвалидации без блокировок.
|
||||
pub struct RevocationQueue {
|
||||
buffer: [AtomicU64; QUEUE_SIZE],
|
||||
// Используем паддинг (cache line size = 64 bytes) для предотвращения False Sharing
|
||||
// между head (изменяется VMM) и tail (изменяется Cap-системой).
|
||||
#[doc(hidden)]
|
||||
_pad0: [u8; 64],
|
||||
head: AtomicUsize,
|
||||
#[doc(hidden)]
|
||||
_pad1: [u8; 64],
|
||||
tail: AtomicUsize,
|
||||
}
|
||||
|
||||
impl RevocationQueue {
|
||||
pub const fn new() -> Self {
|
||||
Self {
|
||||
#[allow(clippy::declare_interior_mutable_const)]
|
||||
buffer: [const { AtomicU64::new(0) }; QUEUE_SIZE],
|
||||
_pad0: [0; 64],
|
||||
head: AtomicUsize::new(0),
|
||||
_pad1: [0; 64],
|
||||
tail: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Вызывается со стороны Actor'ов и подсистемы Capabilities (Multi-Producer)
|
||||
pub fn push(&self, token_sig: u64) -> Result<(), &'static str> {
|
||||
let mut tail = self.tail.load(Ordering::Relaxed);
|
||||
loop {
|
||||
let head = self.head.load(Ordering::Acquire);
|
||||
|
||||
if tail.wrapping_sub(head) >= QUEUE_SIZE {
|
||||
// Очередь переполнена. В проде здесь должен быть триггер ballooning'а
|
||||
// или принудительный yield для VMM, но пока возвращаем ошибку.
|
||||
return Err("Revocation queue overflow");
|
||||
}
|
||||
|
||||
match self.tail.compare_exchange_weak(
|
||||
tail,
|
||||
tail.wrapping_add(1),
|
||||
Ordering::AcqRel,
|
||||
Ordering::Relaxed
|
||||
) {
|
||||
Ok(_) => {
|
||||
self.buffer[tail & QUEUE_MASK].store(token_sig, Ordering::Release);
|
||||
return Ok(());
|
||||
}
|
||||
Err(actual_tail) => tail = actual_tail,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Вызывается только из активного VMM контекста (Single-Consumer)
|
||||
pub fn pop(&self) -> Option<u64> {
|
||||
let head = self.head.load(Ordering::Relaxed);
|
||||
let tail = self.tail.load(Ordering::Acquire);
|
||||
|
||||
if head == tail {
|
||||
return None; // Очередь пуста
|
||||
}
|
||||
|
||||
let token = self.buffer[head & QUEUE_MASK].load(Ordering::Acquire);
|
||||
self.head.store(head.wrapping_add(1), Ordering::Release);
|
||||
|
||||
Some(token)
|
||||
}
|
||||
}
|
||||
|
||||
pub static MMU_REVOCATION_QUEUE: RevocationQueue = RevocationQueue::new();
|
||||
@@ -13,9 +13,13 @@ use limine::request::{FramebufferRequest, RequestsEndMarker, RequestsStartMarker
|
||||
use crate::mem::paging::{PageTable, PageTableFlags};
|
||||
use crate::mem::address::{PhysAddr, VirtAddr};
|
||||
use crate::cap::{Relation, CapRights, Capability, CapObject};
|
||||
use crate::mem::pm_manages::{PMActor, PMRequest, PMResult};
|
||||
|
||||
pub mod cap;
|
||||
pub mod cpu;
|
||||
pub mod events;
|
||||
mod mem;
|
||||
|
||||
#[macro_use]
|
||||
pub mod debug;
|
||||
|
||||
@@ -208,9 +212,19 @@ unsafe extern "C" fn kmain() -> ! {
|
||||
let mut allocator = allocator::ALLOCATOR.lock();
|
||||
allocator.init(heap_start as usize, heap_size);
|
||||
}
|
||||
info!(console, "HEAP", "Global Allocator is online.");
|
||||
info!(console, "HEAP", "Slab Allocator is online.");
|
||||
|
||||
mem::init_cpu_features();
|
||||
info!(console, "CPU", "INVPCID / CPU features detected.");
|
||||
mem::vmm::init_kernel_space(p4_phys, hhdm_offset);
|
||||
info!(console, "VMM", "Kernel Address Space registered.");
|
||||
|
||||
cpu::interrupts::init_idt();
|
||||
info!(console, "CPU", "Interrupt Descriptor Table (IDT) loaded.");
|
||||
|
||||
unsafe { core::arch::asm!("sti", options(nomem, nostack, preserves_flags)); }
|
||||
|
||||
// --- Тестирование подсистемы Capability ---
|
||||
let root_cnode = cap::CNode::new(256);
|
||||
|
||||
if let Some(frame) = mem::pmm::alloc_frame() {
|
||||
@@ -237,6 +251,79 @@ unsafe extern "C" fn kmain() -> ! {
|
||||
info!(console, "CAP", "Slot 10 successfully revoked.");
|
||||
}
|
||||
}
|
||||
|
||||
// --- Тестирование PMActor (Lock-Free Очереди) ---
|
||||
info!(console, "PM", "--- PMActor Buddy Test ---");
|
||||
|
||||
let actor_base_phys = PhysAddr(0x4000_0000);
|
||||
let actor_root_cap = Capability {
|
||||
object: CapObject::Memory { phys: actor_base_phys, size_pages: 1024 },
|
||||
rights: CapRights::all(),
|
||||
relation: Relation::Strong,
|
||||
token_sig: 0xAAAA_BBBB,
|
||||
};
|
||||
|
||||
let mut pm_actor = PMActor::new(1, actor_root_cap, actor_base_phys, 1024 * 4096);
|
||||
info!(console, "PM", "PMActor ID:1 spawned ({} free pages).", pm_actor.free_pages());
|
||||
|
||||
// channel_id=1 → route response to "process 1"
|
||||
pm_actor.submit_request(PMRequest::Allocate {
|
||||
size_pages: 10,
|
||||
token_sig: 0x123,
|
||||
channel_id: 1,
|
||||
}).unwrap();
|
||||
|
||||
// Carve a fixed sub-region (e.g. for a framebuffer alias)
|
||||
pm_actor.submit_request(PMRequest::Carve {
|
||||
offset_pages: 100,
|
||||
size_pages: 4,
|
||||
channel_id: 2,
|
||||
}).unwrap();
|
||||
|
||||
// Process and collect responses
|
||||
let responses = pm_actor.process_messages();
|
||||
for resp in &responses {
|
||||
match resp.result {
|
||||
PMResult::Allocated { cap, order } => {
|
||||
info!(console, "PM",
|
||||
"ch={} Allocated: phys={:#x} order={} pages={}",
|
||||
resp.channel_id,
|
||||
if let CapObject::Memory { phys, .. } = cap.object { phys.0 } else { 0 },
|
||||
order,
|
||||
1usize << order,
|
||||
);
|
||||
|
||||
// Free it back (using the order returned in the response)
|
||||
if let CapObject::Memory { phys, .. } = cap.object {
|
||||
let rel_idx = ((phys.0 - actor_base_phys.0) / 4096) as usize;
|
||||
pm_actor.submit_request(PMRequest::Free {
|
||||
local_frame_idx: rel_idx,
|
||||
order,
|
||||
}).unwrap();
|
||||
}
|
||||
}
|
||||
PMResult::Carved { cap } => {
|
||||
info!(console, "PM",
|
||||
"ch={} Carved sub-cap: phys={:#x}",
|
||||
resp.channel_id,
|
||||
if let CapObject::Memory { phys, .. } = cap.object { phys.0 } else { 0 },
|
||||
);
|
||||
}
|
||||
PMResult::OutOfMemory { size_pages } => {
|
||||
info!(console, "PM", "ch={} OOM for {} pages!", resp.channel_id, size_pages);
|
||||
}
|
||||
PMResult::Freed { .. } => {}
|
||||
}
|
||||
}
|
||||
|
||||
// Drain the Free request
|
||||
let _ = pm_actor.process_messages();
|
||||
info!(console, "PM", "After free: {} free pages (should be 1024).", pm_actor.free_pages());
|
||||
|
||||
|
||||
|
||||
|
||||
// --- Логотип ---
|
||||
let logo = r#"
|
||||
###########
|
||||
##################
|
||||
|
||||
@@ -25,7 +25,7 @@ impl<A> Locked<A> {
|
||||
|
||||
pub fn lock(&self) -> LockedGuard<'_, A> {
|
||||
while self.lock.compare_exchange_weak(false, true, Ordering::Acquire, Ordering::Relaxed).is_err() {
|
||||
core::hint::spin_loop();
|
||||
core::hint::spin_loop(); // Уступка в spinlock для снижения нагрузки на шину
|
||||
}
|
||||
LockedGuard {
|
||||
lock: &self.lock,
|
||||
@@ -49,41 +49,106 @@ impl<A> DerefMut for LockedGuard<'_, A> {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target { self.data }
|
||||
}
|
||||
|
||||
pub struct BumpAllocator {
|
||||
start: usize,
|
||||
end: usize,
|
||||
next: usize,
|
||||
/// Узел односвязного списка свободных блоков
|
||||
struct ListNode {
|
||||
next: Option<&'static mut ListNode>,
|
||||
}
|
||||
|
||||
impl BumpAllocator {
|
||||
const BLOCK_SIZES: &[usize] = &[8, 16, 32, 64, 128, 256, 512, 1024, 2048];
|
||||
|
||||
/// Slab аллокатор для гранулярного выделения памяти
|
||||
pub struct SlabAllocator {
|
||||
list_heads: [Option<&'static mut ListNode>; BLOCK_SIZES.len()],
|
||||
heap_start: usize,
|
||||
heap_end: usize,
|
||||
next_bump: usize,
|
||||
}
|
||||
|
||||
impl SlabAllocator {
|
||||
pub const fn new() -> Self {
|
||||
Self { start: 0, end: 0, next: 0 }
|
||||
}
|
||||
|
||||
pub fn init(&mut self, start: usize, size: usize) {
|
||||
self.start = start;
|
||||
self.next = start;
|
||||
self.end = start + size;
|
||||
}
|
||||
}
|
||||
|
||||
unsafe impl GlobalAlloc for Locked<BumpAllocator> {
|
||||
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
|
||||
let mut bump = self.lock();
|
||||
|
||||
let alloc_start = (bump.next + layout.align() - 1) & !(layout.align() - 1);
|
||||
let alloc_end = alloc_start + layout.size();
|
||||
|
||||
if alloc_end > bump.end {
|
||||
null_mut()
|
||||
} else {
|
||||
bump.next = alloc_end;
|
||||
alloc_start as *mut u8
|
||||
Self {
|
||||
list_heads: [None, None, None, None, None, None, None, None, None],
|
||||
heap_start: 0,
|
||||
heap_end: 0,
|
||||
next_bump: 0,
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn dealloc(&self, _ptr: *mut u8, _layout: Layout) {}
|
||||
pub fn init(&mut self, start: usize, size: usize) {
|
||||
self.heap_start = start;
|
||||
self.next_bump = start;
|
||||
self.heap_end = start + size;
|
||||
}
|
||||
|
||||
/// Поиск индекса блока под требуемый размер
|
||||
fn list_index(layout: &Layout) -> Option<usize> {
|
||||
let required_block_size = layout.size().max(layout.align());
|
||||
BLOCK_SIZES.iter().position(|&s| s >= required_block_size)
|
||||
}
|
||||
|
||||
/// Резервный Bump-аллокатор для нарезки новых Slab-блоков
|
||||
fn fallback_alloc(&mut self, layout: Layout) -> *mut u8 {
|
||||
let alloc_start = (self.next_bump + layout.align() - 1) & !(layout.align() - 1);
|
||||
let alloc_end = alloc_start.checked_add(layout.size()).unwrap_or(self.heap_end + 1);
|
||||
|
||||
if alloc_end > self.heap_end {
|
||||
null_mut() // Out of memory
|
||||
} else {
|
||||
self.next_bump = alloc_end;
|
||||
alloc_start as *mut u8
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unsafe impl GlobalAlloc for Locked<SlabAllocator> {
|
||||
unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
|
||||
let mut allocator = self.lock();
|
||||
|
||||
match SlabAllocator::list_index(&layout) {
|
||||
Some(index) => {
|
||||
match allocator.list_heads[index].take() {
|
||||
Some(node) => {
|
||||
allocator.list_heads[index] = node.next.take();
|
||||
node as *mut ListNode as *mut u8
|
||||
}
|
||||
None => {
|
||||
let block_size = BLOCK_SIZES[index];
|
||||
let block_align = block_size;
|
||||
let layout = Layout::from_size_align(block_size, block_align).unwrap();
|
||||
allocator.fallback_alloc(layout)
|
||||
}
|
||||
}
|
||||
}
|
||||
None => allocator.fallback_alloc(layout)
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) {
|
||||
let mut allocator = self.lock();
|
||||
|
||||
match SlabAllocator::list_index(&layout) {
|
||||
Some(index) => {
|
||||
let new_node = ListNode {
|
||||
next: allocator.list_heads[index].take(),
|
||||
};
|
||||
|
||||
assert!(layout.size() >= core::mem::size_of::<ListNode>());
|
||||
|
||||
let new_node_ptr = ptr as *mut ListNode;
|
||||
|
||||
// Production-fix для Rust 2024: явная изоляция unsafe-операций
|
||||
unsafe {
|
||||
new_node_ptr.write(new_node);
|
||||
allocator.list_heads[index] = Some(&mut *new_node_ptr);
|
||||
}
|
||||
}
|
||||
None => {
|
||||
// Крупные регионы освобождаются через вызовы дескрипторов VMM/PMM,
|
||||
// глобальный аллокатор ядра их не трекает.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[global_allocator]
|
||||
pub static ALLOCATOR: Locked<BumpAllocator> = Locked::new(BumpAllocator::new());
|
||||
pub static ALLOCATOR: Locked<SlabAllocator> = Locked::new(SlabAllocator::new());
|
||||
|
||||
298
kernel/src/mem/buddy.rs
Normal file
298
kernel/src/mem/buddy.rs
Normal file
@@ -0,0 +1,298 @@
|
||||
//! # Per-Actor Buddy Allocator (`src/mem/buddy.rs`)
|
||||
//!
|
||||
//! A power-of-two page-block allocator for a fixed, pre-committed physical
|
||||
//! region. Designed for **single-consumer** use inside a [`crate::mem::pm_manages::PMActor`]:
|
||||
//! the owning actor is the only entity that ever mutates its buddy allocator, so
|
||||
//! no locking is needed — the entire subsystem is inherently lock-free from the
|
||||
//! consumer's perspective.
|
||||
//!
|
||||
//! ## Block Layout
|
||||
//!
|
||||
//! The region is subdivided into blocks of `2^order` pages (order 0 … MAX_ORDER).
|
||||
//! Every block at order *k* is aligned to `2^k` pages within the region.
|
||||
//!
|
||||
//! ```text
|
||||
//! order 0 → 1 page = 4 KiB
|
||||
//! order 1 → 2 pages = 8 KiB
|
||||
//! …
|
||||
//! order 11 → 2 048 pages = 8 MiB (MAX_ORDER)
|
||||
//! ```
|
||||
//!
|
||||
//! ## Complexity
|
||||
//!
|
||||
//! | Operation | Amortised | Worst-case |
|
||||
//! |-----------|-----------|------------|
|
||||
//! | `alloc` | O(log N) | O(MAX_ORDER · Lₖ) |
|
||||
//! | `free` | O(log N) | O(MAX_ORDER · Lₖ) |
|
||||
//!
|
||||
//! where Lₖ = `free_lists[k].len()` ≤ `total_pages / 2^k`.
|
||||
//!
|
||||
//! ## Production note
|
||||
//!
|
||||
//! For managed ranges > 1 GiB (> 256 K pages), replace the per-order `Vec<usize>`
|
||||
//! free lists with a radix tree or interval tree to bound Lₖ. For current
|
||||
//! PMActor capital sizes (typically ≤ 128 MiB = 32 K pages at order 0), the
|
||||
//! `Vec`-based implementation is fully adequate.
|
||||
|
||||
extern crate alloc;
|
||||
use alloc::vec::Vec;
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
// Constants & helpers
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/// Maximum allocation order.
|
||||
/// `2^11 × 4 096 bytes = 8 MiB` per single allocation.
|
||||
pub const MAX_ORDER: usize = 11;
|
||||
|
||||
/// ⌈log₂(n)⌉ — the minimum order whose block size covers `page_count` pages.
|
||||
///
|
||||
/// ```text
|
||||
/// order_for(1) = 0 (2^0 = 1)
|
||||
/// order_for(2) = 1 (2^1 = 2)
|
||||
/// order_for(3) = 2 (2^2 = 4 ≥ 3)
|
||||
/// order_for(2048) = 11
|
||||
/// ```
|
||||
#[inline]
|
||||
pub fn order_for(page_count: usize) -> usize {
|
||||
if page_count <= 1 {
|
||||
0
|
||||
} else {
|
||||
// Number of bits needed to represent (page_count - 1).
|
||||
usize::BITS as usize - (page_count - 1).leading_zeros() as usize
|
||||
}
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
// BuddyAllocator
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/// Buddy allocator over a contiguous, pre-committed range of physical pages.
|
||||
///
|
||||
/// All page indices stored in free lists are **relative to the start of the
|
||||
/// managed range**. The owner (`PMActor`) translates to absolute `PhysAddr`
|
||||
/// by adding `managed_range.0`.
|
||||
///
|
||||
/// # Invariants
|
||||
/// - `free_pages ≤ total_pages` at all times.
|
||||
/// - Every block in `free_lists[k]` is aligned to `2^k` pages (i.e.
|
||||
/// `idx % (1 << k) == 0`).
|
||||
/// - No block appears in more than one order's free list simultaneously.
|
||||
pub struct BuddyAllocator {
|
||||
/// `free_lists[k]` = relative page indices of free 2^k-page blocks.
|
||||
/// Ordering within each list is irrelevant; `pop()` / `swap_remove()` are used.
|
||||
free_lists: [Vec<usize>; MAX_ORDER + 1],
|
||||
/// Total pages in the managed range (need not be a power of two).
|
||||
total_pages: usize,
|
||||
/// Running count of free pages. Always equals `Σ (2^k × free_lists[k].len())`.
|
||||
free_pages: usize,
|
||||
}
|
||||
|
||||
impl BuddyAllocator {
|
||||
// ─── Construction ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Create a new allocator over `total_pages` pages, **all initially free**.
|
||||
///
|
||||
/// Uses a greedy largest-first decomposition to build the initial free lists
|
||||
/// in O(total_pages / 2^MAX_ORDER) iterations — essentially O(1) for
|
||||
/// power-of-two sizes.
|
||||
///
|
||||
/// Example: 7 pages → blocks [4, 2, 1] → free_lists[2]=[0], [1]=[4], [0]=[6].
|
||||
pub fn new(total_pages: usize) -> Self {
|
||||
// core::array::from_fn is the idiomatic way to init a non-Copy array.
|
||||
let free_lists: [Vec<usize>; MAX_ORDER + 1] =
|
||||
core::array::from_fn(|_| Vec::new());
|
||||
|
||||
let mut this = Self {
|
||||
free_lists,
|
||||
total_pages,
|
||||
free_pages: 0,
|
||||
};
|
||||
|
||||
if total_pages == 0 {
|
||||
return this;
|
||||
}
|
||||
|
||||
let mut idx = 0usize;
|
||||
while idx < total_pages {
|
||||
let remaining = total_pages - idx;
|
||||
|
||||
// Alignment constraint: block at `idx` must be aligned to 2^order.
|
||||
// trailing_zeros(0) is u32::MAX, so we clamp to MAX_ORDER.
|
||||
let align_order = if idx == 0 {
|
||||
MAX_ORDER
|
||||
} else {
|
||||
(idx.trailing_zeros() as usize).min(MAX_ORDER)
|
||||
};
|
||||
|
||||
// Size constraint: 2^order ≤ remaining → order ≤ ⌊log₂(remaining)⌋.
|
||||
let size_order = (usize::BITS as usize - 1)
|
||||
- remaining.leading_zeros() as usize; // ⌊log₂(remaining)⌋
|
||||
|
||||
let order = MAX_ORDER.min(align_order).min(size_order);
|
||||
let block_size = 1usize << order;
|
||||
|
||||
this.free_lists[order].push(idx);
|
||||
this.free_pages += block_size;
|
||||
idx += block_size;
|
||||
}
|
||||
|
||||
this
|
||||
}
|
||||
|
||||
// ─── Allocation ───────────────────────────────────────────────────────────
|
||||
|
||||
/// Allocate a 2^`order`-page block.
|
||||
///
|
||||
/// Returns the **relative** page index of the block's first page, or `None`
|
||||
/// if insufficient contiguous memory remains.
|
||||
///
|
||||
/// The returned index can be converted to a physical address:
|
||||
/// ```text
|
||||
/// phys = actor.managed_range.0.0 + (idx as u64) * PAGE_SIZE
|
||||
/// ```
|
||||
pub fn alloc(&mut self, order: usize) -> Option<usize> {
|
||||
if order > MAX_ORDER {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Find the smallest available order ≥ requested.
|
||||
let found_order = (order..=MAX_ORDER)
|
||||
.find(|&o| !self.free_lists[o].is_empty())?;
|
||||
|
||||
// Consume one block from the found order.
|
||||
let block_idx = self.free_lists[found_order]
|
||||
.pop()
|
||||
.expect("buddy: free_list non-empty but pop() returned None");
|
||||
|
||||
// Debit: we removed a 2^found_order block from free.
|
||||
self.free_pages -= 1 << found_order;
|
||||
|
||||
// Split down to the requested order, putting buddies back into free lists.
|
||||
//
|
||||
// Invariant at each iteration:
|
||||
// `block_idx` is the lower half of a 2^cur_order block.
|
||||
// The upper half (= `block_idx + 2^(cur_order-1)`) is returned to the
|
||||
// free list as an order-(cur_order-1) block.
|
||||
let mut cur_order = found_order;
|
||||
while cur_order > order {
|
||||
cur_order -= 1;
|
||||
let buddy_idx = block_idx + (1 << cur_order);
|
||||
self.free_lists[cur_order].push(buddy_idx);
|
||||
self.free_pages += 1 << cur_order;
|
||||
}
|
||||
|
||||
// Net accounting: removed 2^found_order, added (2^found_order − 2^order).
|
||||
// Result: free_pages decreased by exactly 2^order. ✓
|
||||
Some(block_idx)
|
||||
}
|
||||
|
||||
/// Allocate `page_count` pages, rounding up to the nearest power of two.
|
||||
///
|
||||
/// Returns `(relative_page_idx, actual_order)`. The caller **must** pass the
|
||||
/// same `order` to [`BuddyAllocator::free`] — mismatched orders corrupt the
|
||||
/// allocator.
|
||||
///
|
||||
/// The wasted "rounding up" pages remain unusable until the block is freed.
|
||||
/// For tightly-packed allocations, callers should use `alloc(order)` directly.
|
||||
pub fn alloc_pages(&mut self, page_count: usize) -> Option<(usize, usize)> {
|
||||
if page_count == 0 {
|
||||
return None;
|
||||
}
|
||||
let order = order_for(page_count);
|
||||
if order > MAX_ORDER {
|
||||
return None;
|
||||
}
|
||||
self.alloc(order).map(|idx| (idx, order))
|
||||
}
|
||||
|
||||
// ─── Deallocation ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Return a 2^`order`-page block at **relative** index `block_idx` to the
|
||||
/// free pool, coalescing with free buddies up the order chain.
|
||||
///
|
||||
/// # Panics (debug builds only)
|
||||
/// - `order > MAX_ORDER`
|
||||
/// - `block_idx + 2^order > total_pages`
|
||||
///
|
||||
/// In release builds the checks are elided for performance; passing incorrect
|
||||
/// arguments causes undefined bookkeeping, not UB (no unsafe indexing).
|
||||
pub fn free(&mut self, mut block_idx: usize, mut order: usize) {
|
||||
debug_assert!(
|
||||
order <= MAX_ORDER,
|
||||
"buddy::free: order {} > MAX_ORDER {}",
|
||||
order,
|
||||
MAX_ORDER
|
||||
);
|
||||
debug_assert!(
|
||||
block_idx.checked_add(1 << order).map_or(false, |e| e <= self.total_pages),
|
||||
"buddy::free: block [{}, +{}] out of range (total_pages={})",
|
||||
block_idx,
|
||||
1usize << order,
|
||||
self.total_pages
|
||||
);
|
||||
|
||||
// Try to coalesce with our buddy at each order.
|
||||
//
|
||||
// The buddy of a block at relative index `i` of order `k` is at:
|
||||
// buddy_idx = i XOR 2^k
|
||||
// This works because aligned buddy pairs always differ in exactly bit k.
|
||||
while order < MAX_ORDER {
|
||||
let buddy_idx = block_idx ^ (1 << order);
|
||||
|
||||
// Buddy must lie entirely within the managed range.
|
||||
let buddy_end = match buddy_idx.checked_add(1 << order) {
|
||||
Some(e) => e,
|
||||
None => break,
|
||||
};
|
||||
if buddy_end > self.total_pages {
|
||||
break;
|
||||
}
|
||||
|
||||
// Search for the buddy in the free list for this order.
|
||||
// `swap_remove` is O(1) and order-preserving is not required.
|
||||
if let Some(pos) = self.free_lists[order].iter().position(|&b| b == buddy_idx) {
|
||||
self.free_lists[order].swap_remove(pos);
|
||||
self.free_pages -= 1 << order; // buddy was in free count; remove it
|
||||
|
||||
// Merged block starts at the lower address of the two.
|
||||
block_idx = block_idx.min(buddy_idx);
|
||||
order += 1;
|
||||
// Continue trying to merge at the next level.
|
||||
} else {
|
||||
break; // buddy is allocated or out-of-range; stop coalescing
|
||||
}
|
||||
}
|
||||
|
||||
// Push the (possibly merged) block onto the appropriate free list.
|
||||
self.free_pages += 1 << order;
|
||||
self.free_lists[order].push(block_idx);
|
||||
}
|
||||
|
||||
// ─── Introspection ────────────────────────────────────────────────────────
|
||||
|
||||
/// Number of pages currently available for allocation.
|
||||
#[inline]
|
||||
pub fn free_pages(&self) -> usize {
|
||||
self.free_pages
|
||||
}
|
||||
|
||||
/// Total pages in the managed range.
|
||||
#[inline]
|
||||
pub fn total_pages(&self) -> usize {
|
||||
self.total_pages
|
||||
}
|
||||
|
||||
/// `true` if the allocator has no free pages.
|
||||
#[inline]
|
||||
pub fn is_exhausted(&self) -> bool {
|
||||
self.free_pages == 0
|
||||
}
|
||||
|
||||
/// Dump free-list statistics for each order (useful in panic handlers).
|
||||
///
|
||||
/// Returns an array `[(order, free_block_count); MAX_ORDER + 1]`.
|
||||
pub fn stats(&self) -> [(usize, usize); MAX_ORDER + 1] {
|
||||
core::array::from_fn(|o| (o, self.free_lists[o].len()))
|
||||
}
|
||||
}
|
||||
@@ -1,16 +1,34 @@
|
||||
pub mod pmm;
|
||||
pub mod address;
|
||||
pub mod paging;
|
||||
pub mod allocator;
|
||||
//! `src/mem/mod.rs` — memory subsystem root
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn init(memmap: &limine::response::MemoryMapResponse, hhdm_offset: u64) {
|
||||
unsafe {
|
||||
pmm::BitmapPMM::init(memmap, hhdm_offset);
|
||||
}
|
||||
pub mod address;
|
||||
pub mod allocator;
|
||||
pub mod buddy; // ← new: per-actor buddy allocator
|
||||
pub mod paging;
|
||||
pub mod pm_manages;
|
||||
pub mod pmm;
|
||||
pub mod vmm;
|
||||
|
||||
/// Initialise the physical memory manager.
|
||||
///
|
||||
/// Must be called before any heap allocation or VMM operation.
|
||||
pub fn init_pmm(memmap: &limine::response::MemoryMapResponse, hhdm_offset: u64) {
|
||||
unsafe { pmm::BitmapPMM::init(memmap, hhdm_offset); }
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
/// Initialise CPU features required by the VMM (INVPCID detection).
|
||||
///
|
||||
/// Must be called before `init_vmm` and any `AddressSpace::activate()`.
|
||||
pub fn init_cpu_features() {
|
||||
vmm::init_cpu_features();
|
||||
}
|
||||
|
||||
/// Initialise the kernel address space record (after PMM, heap, and the
|
||||
/// initial P4 page table have been set up in `kmain`).
|
||||
pub fn init_vmm(pml4_phys: address::PhysAddr, hhdm_offset: u64) {
|
||||
vmm::init_kernel_space(pml4_phys, hhdm_offset);
|
||||
}
|
||||
|
||||
/// Physical memory statistics: `(used_pages, total_pages)`.
|
||||
pub fn get_stats() -> (usize, usize) {
|
||||
pmm::get_stats()
|
||||
}
|
||||
|
||||
@@ -20,13 +20,27 @@ bitflags! {
|
||||
}
|
||||
}
|
||||
|
||||
/// Mask for the physical frame address stored in a PTE (bits 51:12).
|
||||
const PTE_ADDR_MASK: u64 = 0x000F_FFFF_FFFF_F000;
|
||||
|
||||
#[repr(C, align(4096))]
|
||||
pub struct PageTable {
|
||||
entries: [u64; 512],
|
||||
}
|
||||
|
||||
impl PageTable {
|
||||
pub fn map_region(&mut self, virt: VirtAddr, phys: PhysAddr, size: u64, flags: PageTableFlags, hhdm: u64) {
|
||||
// ── Bulk mapping ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Map a contiguous physical range to a contiguous virtual range.
|
||||
/// Allocates intermediate page-table pages from the PMM as needed.
|
||||
pub fn map_region(
|
||||
&mut self,
|
||||
virt: VirtAddr,
|
||||
phys: PhysAddr,
|
||||
size: u64,
|
||||
flags: PageTableFlags,
|
||||
hhdm: u64,
|
||||
) {
|
||||
let pages = size.div_ceil(4096);
|
||||
for i in 0..pages {
|
||||
let offset = i * 4096;
|
||||
@@ -34,51 +48,163 @@ impl PageTable {
|
||||
}
|
||||
}
|
||||
|
||||
/// Loads the page table into the CR3 register
|
||||
pub unsafe fn activate(&self, phys_addr: PhysAddr) {
|
||||
unsafe {
|
||||
asm!(
|
||||
"mov cr3, {0}",
|
||||
"jmp 2f",
|
||||
"2:",
|
||||
in(reg) phys_addr.0,
|
||||
options(nostack, preserves_flags)
|
||||
);
|
||||
}
|
||||
}
|
||||
// ── Single-page operations ────────────────────────────────────────────────
|
||||
|
||||
pub fn map_page(&mut self, virt: VirtAddr, phys: PhysAddr, flags: PageTableFlags, hhdm_offset: u64) {
|
||||
let p4_idx = (virt.0 >> 39) & 0x1FF;
|
||||
let p3_idx = (virt.0 >> 30) & 0x1FF;
|
||||
let p2_idx = (virt.0 >> 21) & 0x1FF;
|
||||
let p1_idx = (virt.0 >> 12) & 0x1FF;
|
||||
/// Map a single 4 KiB page.
|
||||
/// Allocates intermediate PT pages from the PMM if they do not exist.
|
||||
pub fn map_page(
|
||||
&mut self,
|
||||
virt: VirtAddr,
|
||||
phys: PhysAddr,
|
||||
flags: PageTableFlags,
|
||||
hhdm: u64,
|
||||
) {
|
||||
let p1 = self
|
||||
.walk_to_p1_mut(virt, hhdm, true /* create */)
|
||||
.expect("map_page: OOM allocating intermediate page-table pages");
|
||||
|
||||
let p3 = self.get_or_create_next_table(p4_idx as usize, hhdm_offset);
|
||||
let p2 = p3.get_or_create_next_table(p3_idx as usize, hhdm_offset);
|
||||
let p1 = p2.get_or_create_next_table(p2_idx as usize, hhdm_offset);
|
||||
let p1_idx = ((virt.0 >> 12) & 0x1FF) as usize;
|
||||
p1.entries[p1_idx] = phys.0 | flags.bits();
|
||||
|
||||
p1.entries[p1_idx as usize] = phys.0 | flags.bits();
|
||||
|
||||
unsafe {
|
||||
asm!("invlpg [{}]", in(reg) virt.0, options(nostack, preserves_flags));
|
||||
}
|
||||
}
|
||||
|
||||
fn get_or_create_next_table(&mut self, index: usize, hhdm: u64) -> &mut Self {
|
||||
if self.entries[index] & PageTableFlags::PRESENT.bits() == 0 {
|
||||
let pt_phys = pmm_alloc().expect("VMM: Out of memory for page tables");
|
||||
let pt_virt = pt_phys.to_virt(hhdm);
|
||||
|
||||
unsafe { core::ptr::write_bytes(pt_virt.as_mut_ptr::<u8>(), 0, 4096); }
|
||||
|
||||
self.entries[index] = pt_phys.0 | (PageTableFlags::PRESENT | PageTableFlags::WRITABLE | PageTableFlags::USER).bits();
|
||||
/// Clear the PTE for `virt` and issue an `INVLPG`.
|
||||
///
|
||||
/// Does **not** free the underlying physical frame — that is the caller's
|
||||
/// responsibility (e.g. `VmaBacking::do_unmap`).
|
||||
/// Silently returns if any level is not present (idempotent).
|
||||
pub fn unmap_page(&mut self, virt: VirtAddr, hhdm: u64) {
|
||||
let Some(p1) = self.walk_to_p1_mut(virt, hhdm, false /* no create */) else {
|
||||
return; // already absent — nothing to do
|
||||
};
|
||||
let p1_idx = ((virt.0 >> 12) & 0x1FF) as usize;
|
||||
if p1.entries[p1_idx] & PageTableFlags::PRESENT.bits() == 0 {
|
||||
return;
|
||||
}
|
||||
p1.entries[p1_idx] = 0;
|
||||
unsafe {
|
||||
asm!("invlpg [{}]", in(reg) virt.0, options(nostack, preserves_flags));
|
||||
}
|
||||
}
|
||||
|
||||
/// Walk four page-table levels to translate `virt` → physical address.
|
||||
///
|
||||
/// Handles 1 GiB and 2 MiB huge pages transparently.
|
||||
/// Returns `None` if any level is absent or the page is not present.
|
||||
pub fn translate(&self, virt: VirtAddr, hhdm: u64) -> Option<PhysAddr> {
|
||||
let p4_idx = ((virt.0 >> 39) & 0x1FF) as usize;
|
||||
let p3_idx = ((virt.0 >> 30) & 0x1FF) as usize;
|
||||
let p2_idx = ((virt.0 >> 21) & 0x1FF) as usize;
|
||||
let p1_idx = ((virt.0 >> 12) & 0x1FF) as usize;
|
||||
|
||||
// Helper: safely follow a PTE to the next level.
|
||||
macro_rules! descend_ref {
|
||||
($entry:expr) => {{
|
||||
let e = $entry;
|
||||
if e & PageTableFlags::PRESENT.bits() == 0 { return None; }
|
||||
unsafe { &*PhysAddr(e & PTE_ADDR_MASK).to_virt(hhdm).as_mut_ptr::<PageTable>() }
|
||||
}};
|
||||
}
|
||||
|
||||
let p3 = descend_ref!(self.entries[p4_idx]);
|
||||
|
||||
// 1 GiB page (PD pointer with HUGE_PAGE set)
|
||||
let p3e = p3.entries[p3_idx];
|
||||
if p3e & PageTableFlags::HUGE_PAGE.bits() != 0 {
|
||||
let base = p3e & 0x000F_FFFF_C000_0000; // bits 51:30
|
||||
return Some(PhysAddr(base | (virt.0 & 0x3FFF_FFFF)));
|
||||
}
|
||||
|
||||
let p2 = descend_ref!(p3e);
|
||||
|
||||
// 2 MiB page (PD entry with HUGE_PAGE set)
|
||||
let p2e = p2.entries[p2_idx];
|
||||
if p2e & PageTableFlags::HUGE_PAGE.bits() != 0 {
|
||||
let base = p2e & 0x000F_FFFF_FFE0_0000; // bits 51:21
|
||||
return Some(PhysAddr(base | (virt.0 & 0x001F_FFFF)));
|
||||
}
|
||||
|
||||
let p1 = descend_ref!(p2e);
|
||||
let p1e = p1.entries[p1_idx];
|
||||
if p1e & PageTableFlags::PRESENT.bits() == 0 { return None; }
|
||||
|
||||
Some(PhysAddr((p1e & PTE_ADDR_MASK) | (virt.0 & 0xFFF)))
|
||||
}
|
||||
|
||||
// ── CR3 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Load this page table into CR3 (full TLB flush, no PCID).
|
||||
///
|
||||
/// Prefer `AddressSpace::activate` (which uses PCID/INVPCID) for process
|
||||
/// context switches. Use this only for early boot before PCID is enabled.
|
||||
pub unsafe fn activate(&self, phys_addr: PhysAddr) {
|
||||
unsafe {
|
||||
asm!(
|
||||
"mov cr3, {0}",
|
||||
in(reg) phys_addr.0,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
let next_pt_phys = PhysAddr(self.entries[index] & 0x000F_FFFF_FFFF_F000);
|
||||
unsafe { &mut *next_pt_phys.to_virt(hhdm).as_mut_ptr() }
|
||||
}
|
||||
}
|
||||
|
||||
// ── Private walk helpers ──────────────────────────────────────────────────────
|
||||
|
||||
impl PageTable {
|
||||
/// Walk (or create) the path P4 → P3 → P2 → P1, returning a mutable
|
||||
/// reference to the P1 (page table, final level).
|
||||
///
|
||||
/// If `create` is `false` and any intermediate entry is absent, returns `None`.
|
||||
/// If `create` is `true`, allocates missing intermediate pages from the PMM.
|
||||
fn walk_to_p1_mut(&mut self, virt: VirtAddr, hhdm: u64, create: bool)
|
||||
-> Option<&mut Self>
|
||||
{
|
||||
let p4_idx = ((virt.0 >> 39) & 0x1FF) as usize;
|
||||
let p3_idx = ((virt.0 >> 30) & 0x1FF) as usize;
|
||||
let p2_idx = ((virt.0 >> 21) & 0x1FF) as usize;
|
||||
|
||||
let p3 = self.get_or_create_next_table(p4_idx, hhdm, create)?;
|
||||
let p2 = p3.get_or_create_next_table(p3_idx, hhdm, create)?;
|
||||
p2.get_or_create_next_table(p2_idx, hhdm, create)
|
||||
}
|
||||
|
||||
/// Return a mutable reference to the next-level table at `index`.
|
||||
///
|
||||
/// - If the entry is present, follows the pointer.
|
||||
/// - If absent and `create`, allocates a zeroed page and installs it.
|
||||
/// - If absent and `!create`, returns `None`.
|
||||
fn get_or_create_next_table(
|
||||
&mut self,
|
||||
index: usize,
|
||||
hhdm: u64,
|
||||
create: bool,
|
||||
) -> Option<&mut Self> {
|
||||
let entry = self.entries[index];
|
||||
if entry & PageTableFlags::PRESENT.bits() == 0 {
|
||||
if !create { return None; }
|
||||
|
||||
let pt_phys = pmm_alloc().expect("VMM: OOM allocating page-table page");
|
||||
let pt_virt = pt_phys.to_virt(hhdm);
|
||||
unsafe { core::ptr::write_bytes(pt_virt.as_mut_ptr::<u8>(), 0, 4096); }
|
||||
|
||||
// Install with USER so both kernel and user pages can live under it;
|
||||
// actual user/kernel distinction is enforced at the P1 level.
|
||||
self.entries[index] = pt_phys.0
|
||||
| (PageTableFlags::PRESENT | PageTableFlags::WRITABLE | PageTableFlags::USER).bits();
|
||||
}
|
||||
|
||||
let next_phys = PhysAddr(self.entries[index] & PTE_ADDR_MASK);
|
||||
Some(unsafe { &mut *next_phys.to_virt(hhdm).as_mut_ptr::<Self>() })
|
||||
}
|
||||
}
|
||||
|
||||
// ── PMM shim ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Allocate a single physical frame for page-table use.
|
||||
/// This thin wrapper avoids a direct dependency cycle between paging ↔ pmm.
|
||||
pub fn pmm_alloc() -> Option<PhysAddr> {
|
||||
PMM.lock().as_mut()?.alloc_frame()
|
||||
}
|
||||
|
||||
|
||||
@@ -1,25 +1,491 @@
|
||||
use crate::cap::{Capability, CapObject, Relation, CapRights};
|
||||
use crate::mem::address::PhysAddr;
|
||||
//! # PM Actor (`src/mem/pm_manages.rs`)
|
||||
//!
|
||||
//! Distributed physical-memory actor that owns a fixed physical capital range
|
||||
//! and processes allocation requests asynchronously via a lock-free MPSC queue.
|
||||
//!
|
||||
//! ## Key changes vs. previous version
|
||||
//!
|
||||
//! 1. **Real allocator**: `PMActor` now contains a `BuddyAllocator` instead of a
|
||||
//! bare page counter. Allocations produce real relative frame indices that
|
||||
//! translate to absolute `PhysAddr` values.
|
||||
//!
|
||||
//! 2. **Closed response loop**: `process_messages()` returns `Vec<PMResponse>`.
|
||||
//! The caller (scheduler / boot code) routes each `PMResponse` to the
|
||||
//! requesting entity identified by `channel_id`. `channel_id = 0` means
|
||||
//! "discard" and is safe to ignore.
|
||||
//!
|
||||
//! 3. **Revised message packing** — fits all fields into a single `u64` without
|
||||
//! truncating critical identifiers:
|
||||
//! ```text
|
||||
//! bits 63:56 OPCODE (8 bits) 1=Alloc, 2=Free, 3=Carve
|
||||
//! bits 55:40 CHANNEL_ID (16 bits) response routing; 0 = discard
|
||||
//! bits 39:20 ARG1 (20 bits) size_pages / local_frame_idx / offset_pages
|
||||
//! bits 19: 0 ARG2 (20 bits) token_sig (low 20 bits) / size_pages
|
||||
//! ```
|
||||
//! Maximum ARG size: 2^20 = 1 048 576 pages = 4 GiB per allocation.
|
||||
//! Maximum CHANNEL_ID: 65 535 simultaneous pending requesters.
|
||||
//!
|
||||
//! ## Ownership model
|
||||
//!
|
||||
//! `PMActor` holds a `Strong` capability over its entire managed range.
|
||||
//! Sub-capabilities carved out via `Allocate` or `Carve` are `Strong` children;
|
||||
//! freeing them (via `Free`) coalesces the buddy tree. The actor **never**
|
||||
//! touches the global `BitmapPMM` — that belongs to the kernel boot path.
|
||||
//!
|
||||
//! ## Ballooning hook
|
||||
//!
|
||||
//! When `BuddyAllocator::alloc_pages` returns `None`, the actor emits a
|
||||
//! `PMResponse::OutOfMemory { channel_id }`. The future ballooning subsystem
|
||||
//! will intercept this before it reaches the requester and negotiate memory
|
||||
//! transfer from a neighbouring actor.
|
||||
|
||||
pub struct PMActor {
|
||||
pub root_untyped: Capability,
|
||||
pub managed_range: (PhysAddr, PhysAddr),
|
||||
use alloc::vec::Vec;
|
||||
use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
|
||||
use crate::cap::{CapObject, CapRights, Capability, Relation};
|
||||
use crate::mem::address::PhysAddr;
|
||||
use crate::mem::buddy::BuddyAllocator;
|
||||
use crate::mem::pmm::PAGE_SIZE;
|
||||
|
||||
extern crate alloc;
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
// Message packing constants
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
const QUEUE_SIZE: usize = 1024; // power-of-two
|
||||
const QUEUE_MASK: usize = QUEUE_SIZE - 1;
|
||||
|
||||
/// Bit layout of a packed PMRequest u64.
|
||||
mod packing {
|
||||
pub const OPCODE_SHIFT: u32 = 56;
|
||||
pub const CHAN_SHIFT: u32 = 40;
|
||||
pub const ARG1_SHIFT: u32 = 20;
|
||||
// ARG2 sits in bits 19:0 (no shift).
|
||||
|
||||
pub const CHAN_MASK: u64 = 0x0000_FFFF; // 16 bits
|
||||
pub const ARG_MASK: u64 = 0x000F_FFFF; // 20 bits
|
||||
}
|
||||
|
||||
impl PMActor {
|
||||
pub fn carve_region(&self, offset: usize, size: usize) -> Capability {
|
||||
if let CapObject::Memory { phys, .. } = self.root_untyped.object {
|
||||
Capability {
|
||||
object: CapObject::Memory {
|
||||
phys: PhysAddr(phys.0 + offset as u64),
|
||||
size_pages: size / 4096,
|
||||
},
|
||||
rights: CapRights::READ | CapRights::WRITE | CapRights::GRANT,
|
||||
relation: Relation::Strong,
|
||||
token_sig: 0xDEAD_BEEF,
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
// Request type
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/// Asynchronous request submitted to a `PMActor` inbox.
|
||||
///
|
||||
/// `channel_id` identifies where the response should be routed.
|
||||
/// Use `channel_id = 0` if you do not need a reply.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PMRequest {
|
||||
/// Allocate `size_pages` pages; tag the resulting capability with `token_sig`.
|
||||
Allocate {
|
||||
size_pages: usize,
|
||||
token_sig: u32,
|
||||
channel_id: u16,
|
||||
},
|
||||
/// Free a previously allocated block.
|
||||
///
|
||||
/// `local_frame_idx` is the **relative** page index returned inside the
|
||||
/// `Allocated` capability (i.e. `cap.object.phys - managed_range.start`).
|
||||
/// `order` is the buddy order used at allocation time (stored in the cap).
|
||||
Free {
|
||||
local_frame_idx: usize,
|
||||
order: usize,
|
||||
},
|
||||
/// Carve a fixed sub-region without consulting the buddy allocator.
|
||||
/// Used for static layout decisions made at boot (e.g. framebuffer alias).
|
||||
Carve {
|
||||
offset_pages: usize,
|
||||
size_pages: usize,
|
||||
channel_id: u16,
|
||||
},
|
||||
/// Sentinel — never pushed onto the queue; result of unpack(0).
|
||||
None,
|
||||
}
|
||||
|
||||
impl PMRequest {
|
||||
/// Encode the request into one `u64` atom for the lock-free buffer.
|
||||
///
|
||||
/// Opcode 0 (None) is reserved as the "slot empty" sentinel; `None` returns
|
||||
/// 0 intentionally and the sender rejects it before pushing.
|
||||
#[inline]
|
||||
pub fn pack(self) -> u64 {
|
||||
use packing::*;
|
||||
match self {
|
||||
Self::Allocate { size_pages, token_sig, channel_id } => {
|
||||
(1u64 << OPCODE_SHIFT)
|
||||
| ((channel_id as u64 & CHAN_MASK) << CHAN_SHIFT)
|
||||
| ((size_pages as u64 & ARG_MASK) << ARG1_SHIFT)
|
||||
| (token_sig as u64 & ARG_MASK)
|
||||
}
|
||||
} else {
|
||||
panic!("PM: Root is not memory!");
|
||||
Self::Free { local_frame_idx, order } => {
|
||||
(2u64 << OPCODE_SHIFT)
|
||||
// channel_id not needed — Free has no reply
|
||||
| ((local_frame_idx as u64 & ARG_MASK) << ARG1_SHIFT)
|
||||
| (order as u64 & ARG_MASK)
|
||||
}
|
||||
Self::Carve { offset_pages, size_pages, channel_id } => {
|
||||
(3u64 << OPCODE_SHIFT)
|
||||
| ((channel_id as u64 & CHAN_MASK) << CHAN_SHIFT)
|
||||
| ((offset_pages as u64 & ARG_MASK) << ARG1_SHIFT)
|
||||
| (size_pages as u64 & ARG_MASK)
|
||||
}
|
||||
Self::None => 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Decode a packed `u64` back into a `PMRequest`.
|
||||
#[inline]
|
||||
pub fn unpack(val: u64) -> Self {
|
||||
use packing::*;
|
||||
let opcode = val >> OPCODE_SHIFT;
|
||||
let channel_id = ((val >> CHAN_SHIFT) & CHAN_MASK) as u16;
|
||||
let arg1 = ((val >> ARG1_SHIFT) & ARG_MASK) as usize;
|
||||
let arg2 = (val & ARG_MASK) as usize;
|
||||
|
||||
match opcode {
|
||||
1 => Self::Allocate {
|
||||
size_pages: arg1,
|
||||
token_sig: arg2 as u32,
|
||||
channel_id,
|
||||
},
|
||||
2 => Self::Free {
|
||||
local_frame_idx: arg1,
|
||||
order: arg2,
|
||||
},
|
||||
3 => Self::Carve {
|
||||
offset_pages: arg1,
|
||||
size_pages: arg2,
|
||||
channel_id,
|
||||
},
|
||||
_ => Self::None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
// Response type
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/// Result returned by `PMActor::process_messages()` for each completed request.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct PMResponse {
|
||||
/// Channel to route this response to. `0` = discard.
|
||||
pub channel_id: u16,
|
||||
/// The actual outcome.
|
||||
pub result: PMResult,
|
||||
}
|
||||
|
||||
/// Outcome of a single PM operation.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum PMResult {
|
||||
/// Memory was allocated. `cap` is the strong capability to the region.
|
||||
/// `order` is the buddy order — **must** be passed back to `Free`.
|
||||
Allocated { cap: Capability, order: usize },
|
||||
|
||||
/// The actor had insufficient free pages.
|
||||
/// Future: ballooning subsystem intercepts this and retries.
|
||||
OutOfMemory { size_pages: usize },
|
||||
|
||||
/// A fixed sub-region was carved successfully.
|
||||
Carved { cap: Capability },
|
||||
|
||||
/// Free completed (no capability issued — memory returned to buddy pool).
|
||||
Freed { pages_returned: usize },
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
// Lock-free MPSC inbox
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/// MPSC ring buffer for PMRequest values.
|
||||
///
|
||||
/// Producers (any core, any context) call `send`; the owning PMActor calls
|
||||
/// `pop` — exactly one consumer, no locking on the read side.
|
||||
pub struct PMActorQueue {
|
||||
buffer: [AtomicU64; QUEUE_SIZE],
|
||||
/// Pad to separate producer-written `tail` from consumer-read `head`.
|
||||
_pad0: [u8; 64],
|
||||
head: AtomicUsize,
|
||||
_pad1: [u8; 64],
|
||||
tail: AtomicUsize,
|
||||
}
|
||||
|
||||
impl PMActorQueue {
|
||||
pub const fn new() -> Self {
|
||||
Self {
|
||||
#[allow(clippy::declare_interior_mutable_const)]
|
||||
buffer: [const { AtomicU64::new(0) }; QUEUE_SIZE],
|
||||
_pad0: [0u8; 64],
|
||||
head: AtomicUsize::new(0),
|
||||
_pad1: [0u8; 64],
|
||||
tail: AtomicUsize::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Push a request into the inbox (Multi-Producer path).
|
||||
///
|
||||
/// Returns `Err` only if the ring buffer is full — which should trigger
|
||||
/// backpressure or a ballooning request in production.
|
||||
pub fn send(&self, req: PMRequest) -> Result<(), &'static str> {
|
||||
let packed = req.pack();
|
||||
if packed == 0 {
|
||||
return Err("PMActorQueue::send: PMRequest::None is not sendable");
|
||||
}
|
||||
|
||||
let mut tail = self.tail.load(Ordering::Relaxed);
|
||||
loop {
|
||||
let head = self.head.load(Ordering::Acquire);
|
||||
if tail.wrapping_sub(head) >= QUEUE_SIZE {
|
||||
return Err("PMActorQueue overflow — apply backpressure or balloon");
|
||||
}
|
||||
|
||||
match self.tail.compare_exchange_weak(
|
||||
tail,
|
||||
tail.wrapping_add(1),
|
||||
Ordering::AcqRel,
|
||||
Ordering::Relaxed,
|
||||
) {
|
||||
Ok(_) => {
|
||||
self.buffer[tail & QUEUE_MASK].store(packed, Ordering::Release);
|
||||
return Ok(());
|
||||
}
|
||||
Err(actual) => tail = actual,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Pop one request (Single-Consumer path — actor's own context only).
|
||||
pub fn pop(&self) -> Option<PMRequest> {
|
||||
let head = self.head.load(Ordering::Relaxed);
|
||||
let tail = self.tail.load(Ordering::Acquire);
|
||||
if head == tail {
|
||||
return None;
|
||||
}
|
||||
let packed = self.buffer[head & QUEUE_MASK].load(Ordering::Acquire);
|
||||
self.head.store(head.wrapping_add(1), Ordering::Release);
|
||||
Some(PMRequest::unpack(packed))
|
||||
}
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
// PM Actor
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/// An autonomous physical-memory actor.
|
||||
///
|
||||
/// Owns a `BuddyAllocator` over its capital range and a lock-free MPSC inbox.
|
||||
/// The actor must be driven externally: when the scheduler grants it CPU time,
|
||||
/// it calls `process_messages()` to drain the inbox and produce `PMResponse`s.
|
||||
///
|
||||
/// # Single-consumer guarantee
|
||||
/// Only `process_messages()` ever mutates `buddy` and `queue.head`.
|
||||
/// This is enforced by taking `&mut self` on `process_messages`.
|
||||
pub struct PMActor {
|
||||
/// Unique identity within the actor federation.
|
||||
pub actor_id: u64,
|
||||
|
||||
/// Root strong capability over the entire managed physical range.
|
||||
pub root_untyped: Capability,
|
||||
|
||||
/// `(inclusive_start, exclusive_end)` physical addresses.
|
||||
pub managed_range: (PhysAddr, PhysAddr),
|
||||
|
||||
/// Inbox — producers write here, actor reads here.
|
||||
queue: PMActorQueue,
|
||||
|
||||
/// Local buddy allocator. Only ever touched in `process_messages`.
|
||||
buddy: BuddyAllocator,
|
||||
}
|
||||
|
||||
impl PMActor {
|
||||
/// Construct a new actor managing `size_bytes` bytes starting at `start`.
|
||||
///
|
||||
/// `size_bytes` must be a multiple of `PAGE_SIZE`; non-multiples are
|
||||
/// rounded down silently.
|
||||
pub fn new(
|
||||
actor_id: u64,
|
||||
root_cap: Capability,
|
||||
start: PhysAddr,
|
||||
size_bytes: u64,
|
||||
) -> Self {
|
||||
let total_pages = (size_bytes / PAGE_SIZE) as usize;
|
||||
Self {
|
||||
actor_id,
|
||||
root_untyped: root_cap,
|
||||
managed_range: (start, PhysAddr(start.0 + total_pages as u64 * PAGE_SIZE)),
|
||||
queue: PMActorQueue::new(),
|
||||
buddy: BuddyAllocator::new(total_pages),
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Producer API (callable from any context) ──────────────────────────
|
||||
|
||||
/// Submit a request to this actor's inbox.
|
||||
///
|
||||
/// Thread-safe (MPSC producer side). Returns `Err` if the inbox is full.
|
||||
#[inline]
|
||||
pub fn submit_request(&self, req: PMRequest) -> Result<(), &'static str> {
|
||||
self.queue.send(req)
|
||||
}
|
||||
|
||||
// ─── Consumer API (actor's own scheduled context) ─────────────────────
|
||||
|
||||
/// Drain the inbox and execute all pending requests.
|
||||
///
|
||||
/// Returns a `Vec` of responses that the caller (scheduler) must route to
|
||||
/// the appropriate process or actor mailbox based on `channel_id`.
|
||||
///
|
||||
/// Responses with `channel_id == 0` can be discarded.
|
||||
///
|
||||
/// # No locks held
|
||||
/// `buddy` and `queue.head` are mutated — no other thread touches them.
|
||||
/// The only shared state is the queue's `tail`, which is written by producers
|
||||
/// via `AtomicUsize::compare_exchange_weak`, never by this path.
|
||||
pub fn process_messages(&mut self) -> Vec<PMResponse> {
|
||||
let mut responses = Vec::new();
|
||||
|
||||
while let Some(req) = self.queue.pop() {
|
||||
let resp = match req {
|
||||
PMRequest::Allocate { size_pages, token_sig, channel_id } => {
|
||||
self.handle_allocate(size_pages, token_sig, channel_id)
|
||||
}
|
||||
PMRequest::Free { local_frame_idx, order } => {
|
||||
self.handle_free(local_frame_idx, order)
|
||||
}
|
||||
PMRequest::Carve { offset_pages, size_pages, channel_id } => {
|
||||
self.handle_carve(offset_pages, size_pages, channel_id)
|
||||
}
|
||||
PMRequest::None => continue,
|
||||
};
|
||||
|
||||
// Only push responses that need routing.
|
||||
// Free responses (channel_id == 0) are still pushed so callers can
|
||||
// audit completion if needed; they may simply drop them.
|
||||
if let Some(r) = resp {
|
||||
responses.push(r);
|
||||
}
|
||||
}
|
||||
|
||||
responses
|
||||
}
|
||||
|
||||
// ─── Introspection ────────────────────────────────────────────────────
|
||||
|
||||
/// Free pages remaining in this actor's buddy pool.
|
||||
#[inline]
|
||||
pub fn free_pages(&self) -> usize {
|
||||
self.buddy.free_pages()
|
||||
}
|
||||
|
||||
/// Total pages this actor was initialised with.
|
||||
#[inline]
|
||||
pub fn total_pages(&self) -> usize {
|
||||
self.buddy.total_pages()
|
||||
}
|
||||
}
|
||||
|
||||
// ── Private handler implementations ──────────────────────────────────────────
|
||||
|
||||
impl PMActor {
|
||||
fn handle_allocate(
|
||||
&mut self,
|
||||
size_pages: usize,
|
||||
token_sig: u32,
|
||||
channel_id: u16,
|
||||
) -> Option<PMResponse> {
|
||||
match self.buddy.alloc_pages(size_pages) {
|
||||
Some((rel_idx, order)) => {
|
||||
// Translate relative index → absolute physical address.
|
||||
let phys = PhysAddr(self.managed_range.0.0 + rel_idx as u64 * PAGE_SIZE);
|
||||
|
||||
// Build a strong child capability.
|
||||
// token_sig is widened and XOR'd with actor_id for uniqueness.
|
||||
let cap = Capability {
|
||||
object: CapObject::Memory {
|
||||
phys,
|
||||
// Report actual (rounded-up) page count so the caller
|
||||
// passes the correct `order` back to Free.
|
||||
size_pages: 1 << order,
|
||||
},
|
||||
rights: CapRights::READ | CapRights::WRITE | CapRights::GRANT,
|
||||
relation: Relation::Strong,
|
||||
token_sig: (token_sig as u64) ^ self.actor_id ^ (rel_idx as u64),
|
||||
};
|
||||
|
||||
Some(PMResponse {
|
||||
channel_id,
|
||||
result: PMResult::Allocated { cap, order },
|
||||
})
|
||||
}
|
||||
None => {
|
||||
// TODO: emit a ballooning request to a neighbour actor before
|
||||
// propagating OutOfMemory upward.
|
||||
Some(PMResponse {
|
||||
channel_id,
|
||||
result: PMResult::OutOfMemory { size_pages },
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_free(
|
||||
&mut self,
|
||||
local_frame_idx: usize,
|
||||
order: usize,
|
||||
) -> Option<PMResponse> {
|
||||
// Validate bounds before touching the buddy tree.
|
||||
let block_end = local_frame_idx.saturating_add(1usize << order);
|
||||
if block_end > self.buddy.total_pages() {
|
||||
// Corrupted or spoofed index — ignore silently in release, panic in debug.
|
||||
debug_assert!(
|
||||
false,
|
||||
"PMActor {}: Free out of range (idx={}, order={}, total={})",
|
||||
self.actor_id,
|
||||
local_frame_idx,
|
||||
order,
|
||||
self.buddy.total_pages()
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
self.buddy.free(local_frame_idx, order);
|
||||
|
||||
// Free never needs routing — return None to skip Vec push, or push with
|
||||
// channel_id=0 for audit purposes. We skip to avoid unnecessary allocation.
|
||||
// Callers that need a Free-complete signal should use a separate mechanism.
|
||||
None
|
||||
}
|
||||
|
||||
fn handle_carve(
|
||||
&mut self,
|
||||
offset_pages: usize,
|
||||
size_pages: usize,
|
||||
channel_id: u16,
|
||||
) -> Option<PMResponse> {
|
||||
// Validate offset + size within range.
|
||||
let range_pages = self.buddy.total_pages();
|
||||
if offset_pages >= range_pages
|
||||
|| size_pages == 0
|
||||
|| offset_pages.saturating_add(size_pages) > range_pages
|
||||
{
|
||||
return Some(PMResponse {
|
||||
channel_id,
|
||||
result: PMResult::OutOfMemory { size_pages },
|
||||
});
|
||||
}
|
||||
|
||||
let phys = PhysAddr(self.managed_range.0.0 + offset_pages as u64 * PAGE_SIZE);
|
||||
let cap = Capability {
|
||||
object: CapObject::Memory { phys, size_pages },
|
||||
rights: CapRights::READ | CapRights::WRITE | CapRights::GRANT,
|
||||
relation: Relation::Strong,
|
||||
token_sig: self.actor_id ^ offset_pages as u64,
|
||||
};
|
||||
|
||||
Some(PMResponse {
|
||||
channel_id,
|
||||
result: PMResult::Carved { cap },
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,19 +4,22 @@ use crate::mem::allocator::Locked;
|
||||
pub const PAGE_SIZE: u64 = 4096;
|
||||
|
||||
pub struct BitmapPMM {
|
||||
bitmap: &'static mut [u8],
|
||||
bitmap: &'static mut [u8],
|
||||
total_pages: usize,
|
||||
used_pages: usize,
|
||||
last_idx: usize,
|
||||
used_pages: usize,
|
||||
/// Byte index hint: next search starts here to amortise O(N) scans.
|
||||
last_byte: usize,
|
||||
}
|
||||
|
||||
pub static PMM: Locked<Option<BitmapPMM>> = Locked::new(None);
|
||||
|
||||
impl BitmapPMM {
|
||||
#[allow(dead_code)]
|
||||
pub fn used_pages(&self) -> usize { self.used_pages }
|
||||
pub fn used_pages(&self) -> usize { self.used_pages }
|
||||
#[allow(dead_code)]
|
||||
pub fn total_pages(&self) -> usize { self.total_pages }
|
||||
#[allow(dead_code)]
|
||||
pub fn free_pages(&self) -> usize { self.total_pages.saturating_sub(self.used_pages) }
|
||||
|
||||
pub unsafe fn init(mmap: &limine::response::MemoryMapResponse, hhdm_offset: u64) {
|
||||
let max_addr = mmap.entries().iter()
|
||||
@@ -27,22 +30,28 @@ impl BitmapPMM {
|
||||
let total_pages = (max_addr / PAGE_SIZE) as usize;
|
||||
let bitmap_size = total_pages.div_ceil(8);
|
||||
|
||||
let bitmap_phys_addr = mmap.entries().iter()
|
||||
.find(|e| e.entry_type == limine::memory_map::EntryType::USABLE && e.length >= bitmap_size as u64)
|
||||
// Find a usable region large enough to hold the bitmap.
|
||||
let bitmap_phys = mmap.entries().iter()
|
||||
.find(|e| {
|
||||
e.entry_type == limine::memory_map::EntryType::USABLE
|
||||
&& e.length >= bitmap_size as u64
|
||||
})
|
||||
.map(|e| e.base)
|
||||
.expect("PMM: Insufficient memory for bitmap");
|
||||
.expect("PMM: no usable region large enough for the bitmap");
|
||||
|
||||
let bitmap_virt_ptr = (bitmap_phys_addr + hhdm_offset) as *mut u8;
|
||||
let bitmap_slice = unsafe { core::slice::from_raw_parts_mut(bitmap_virt_ptr, bitmap_size) };
|
||||
bitmap_slice.fill(0xFF);
|
||||
let bitmap_ptr = (bitmap_phys + hhdm_offset) as *mut u8;
|
||||
// Mark everything as used (all bits = 1) and free usable entries below.
|
||||
let bitmap = unsafe { core::slice::from_raw_parts_mut(bitmap_ptr, bitmap_size) };
|
||||
bitmap.fill(0xFF);
|
||||
|
||||
let mut pmm = Self {
|
||||
bitmap: bitmap_slice,
|
||||
bitmap,
|
||||
total_pages,
|
||||
used_pages: total_pages,
|
||||
last_idx: 0,
|
||||
last_byte: 0,
|
||||
};
|
||||
|
||||
// Free all usable pages …
|
||||
for entry in mmap.entries() {
|
||||
if entry.entry_type == limine::memory_map::EntryType::USABLE {
|
||||
for addr in (entry.base..entry.base + entry.length).step_by(PAGE_SIZE as usize) {
|
||||
@@ -51,71 +60,131 @@ impl BitmapPMM {
|
||||
}
|
||||
}
|
||||
|
||||
for addr in (bitmap_phys_addr..bitmap_phys_addr + bitmap_size as u64).step_by(PAGE_SIZE as usize) {
|
||||
// … then re-lock the bitmap pages themselves …
|
||||
for addr in (bitmap_phys..bitmap_phys + bitmap_size as u64).step_by(PAGE_SIZE as usize) {
|
||||
pmm.lock_frame(PhysAddr(addr));
|
||||
}
|
||||
|
||||
// … and the null page (physical 0x0 must never be returned as a valid frame).
|
||||
pmm.lock_frame(PhysAddr(0));
|
||||
|
||||
*PMM.lock() = Some(pmm);
|
||||
*PMM.lock() = Some(pmm);
|
||||
}
|
||||
|
||||
// ── Core operations ───────────────────────────────────────────────────────
|
||||
|
||||
/// Mark a frame as free. Idempotent (double-free is a no-op, not UB).
|
||||
pub fn free_frame(&mut self, phys_addr: PhysAddr) {
|
||||
let idx = (phys_addr.0 / PAGE_SIZE) as usize;
|
||||
if idx < self.total_pages {
|
||||
let byte_idx = idx / 8;
|
||||
let bit_idx = idx % 8;
|
||||
if (self.bitmap[byte_idx] & (1 << bit_idx)) != 0 {
|
||||
self.bitmap[byte_idx] &= !(1 << bit_idx);
|
||||
self.used_pages -= 1;
|
||||
}
|
||||
if idx >= self.total_pages { return; }
|
||||
let byte = idx / 8;
|
||||
let bit = idx % 8;
|
||||
if self.bitmap[byte] & (1 << bit) != 0 {
|
||||
self.bitmap[byte] &= !(1 << bit);
|
||||
self.used_pages -= 1;
|
||||
// Pull the hint back so the freed page can be found quickly.
|
||||
if byte < self.last_byte { self.last_byte = byte; }
|
||||
}
|
||||
}
|
||||
|
||||
/// Mark a frame as allocated (reserved). Idempotent.
|
||||
pub fn lock_frame(&mut self, phys_addr: PhysAddr) {
|
||||
let idx = (phys_addr.0 / PAGE_SIZE) as usize;
|
||||
if idx < self.total_pages {
|
||||
let byte_idx = idx / 8;
|
||||
let bit_idx = idx % 8;
|
||||
if (self.bitmap[byte_idx] & (1 << bit_idx)) == 0 {
|
||||
self.bitmap[byte_idx] |= 1 << bit_idx;
|
||||
self.used_pages += 1;
|
||||
}
|
||||
if idx >= self.total_pages { return; }
|
||||
let byte = idx / 8;
|
||||
let bit = idx % 8;
|
||||
if self.bitmap[byte] & (1 << bit) == 0 {
|
||||
self.bitmap[byte] |= 1 << bit;
|
||||
self.used_pages += 1;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn alloc_frame(&mut self) -> Option<PhysAddr> {
|
||||
let start_byte = self.last_idx / 8;
|
||||
for i in start_byte..(self.bitmap.len()) {
|
||||
if self.bitmap[i] != 0xFF {
|
||||
for bit in 0..8 {
|
||||
let idx = i * 8 + bit;
|
||||
if idx >= self.total_pages { return None; }
|
||||
|
||||
let addr = PhysAddr(idx as u64 * PAGE_SIZE);
|
||||
if !self.is_locked(addr) {
|
||||
self.lock_frame(addr);
|
||||
self.last_idx = idx;
|
||||
return Some(addr);
|
||||
/// Allocate one physical frame.
|
||||
///
|
||||
/// Uses a two-pass search (linear scan from `last_byte` hint, then wraps
|
||||
/// to 0 if not found in the first pass) to avoid returning `None` when
|
||||
/// free frames exist before the hint.
|
||||
pub fn alloc_frame(&mut self) -> Option<PhysAddr> {
|
||||
let len = self.bitmap.len();
|
||||
|
||||
for pass in 0..2usize {
|
||||
let (from, to) = if pass == 0 {
|
||||
(self.last_byte, len)
|
||||
} else {
|
||||
(0, self.last_byte)
|
||||
};
|
||||
|
||||
for byte_idx in from..to {
|
||||
// Fast path: skip fully-used bytes.
|
||||
if self.bitmap[byte_idx] == 0xFF { continue; }
|
||||
|
||||
for bit in 0..8u8 {
|
||||
if self.bitmap[byte_idx] & (1 << bit) == 0 {
|
||||
let page_idx = byte_idx * 8 + bit as usize;
|
||||
if page_idx >= self.total_pages { return None; }
|
||||
|
||||
// Mark allocated.
|
||||
self.bitmap[byte_idx] |= 1 << bit;
|
||||
self.used_pages += 1;
|
||||
self.last_byte = byte_idx;
|
||||
|
||||
return Some(PhysAddr(page_idx as u64 * PAGE_SIZE));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
|
||||
None // genuinely out of memory
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn is_locked(&self, addr: PhysAddr) -> bool {
|
||||
let idx = (addr.0 / PAGE_SIZE) as usize;
|
||||
(self.bitmap[idx / 8] & (1 << (idx % 8))) != 0
|
||||
/// Try to allocate `count` **contiguous** physical frames.
|
||||
///
|
||||
/// Returns the base physical address of the run, or `None` if no run of
|
||||
/// sufficient length exists. This is needed for (e.g.) allocating 2 MiB
|
||||
/// huge-page aligned regions or DMA buffers that must be physically
|
||||
/// contiguous.
|
||||
///
|
||||
/// O(N) worst-case; use sparingly and prefer small counts.
|
||||
pub fn alloc_contiguous(&mut self, count: usize) -> Option<PhysAddr> {
|
||||
if count == 0 { return None; }
|
||||
|
||||
let mut run_start = 0usize;
|
||||
let mut run_len = 0usize;
|
||||
|
||||
for page_idx in 0..self.total_pages {
|
||||
let byte = page_idx / 8;
|
||||
let bit = page_idx % 8;
|
||||
if self.bitmap[byte] & (1 << bit) == 0 {
|
||||
if run_len == 0 { run_start = page_idx; }
|
||||
run_len += 1;
|
||||
if run_len == count {
|
||||
// Lock every frame in the run.
|
||||
for i in run_start..run_start + count {
|
||||
self.bitmap[i / 8] |= 1 << (i % 8);
|
||||
}
|
||||
self.used_pages += count;
|
||||
self.last_byte = run_start / 8;
|
||||
return Some(PhysAddr(run_start as u64 * PAGE_SIZE));
|
||||
}
|
||||
} else {
|
||||
run_len = 0;
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
// ── Module-level convenience functions ───────────────────────────────────────
|
||||
|
||||
pub fn alloc_frame() -> Option<PhysAddr> {
|
||||
PMM.lock().as_mut()?.alloc_frame()
|
||||
}
|
||||
|
||||
pub fn alloc_contiguous(count: usize) -> Option<PhysAddr> {
|
||||
PMM.lock().as_mut()?.alloc_contiguous(count)
|
||||
}
|
||||
|
||||
pub fn free_frame(addr: PhysAddr) {
|
||||
if let Some(pmm) = PMM.lock().as_mut() {
|
||||
pmm.free_frame(addr);
|
||||
|
||||
730
kernel/src/mem/vmm.rs
Normal file
730
kernel/src/mem/vmm.rs
Normal file
@@ -0,0 +1,730 @@
|
||||
//! # Virtual Memory Manager (`src/mem/vmm.rs`)
|
||||
//!
|
||||
//! Manages address spaces (PML4 + VMA list) and provides the bridge between
|
||||
//! the capability system and the hardware MMU.
|
||||
//!
|
||||
//! ## Changes vs. previous version
|
||||
//!
|
||||
//! 1. **ASID recycling** — replaced the monotonic `AtomicU16` with a 4096-bit
|
||||
//! bitmap allocator (`AsidAllocator`). `AddressSpace::drop` returns the ASID
|
||||
//! to the pool, making it available for future processes. The bitmap is
|
||||
//! `const`-initializable (no heap), so no boot-ordering issues.
|
||||
//!
|
||||
//! 2. **INVPCID CPUID guard** — `init_cpu_features()` must be called once during
|
||||
//! boot (before any `AddressSpace::activate()` or TLB flush). It checks
|
||||
//! CPUID.07H:EBX[10] and sets `INVPCID_SUPPORTED`. `tlb_flush_asid` falls
|
||||
//! back to a full CR3 reload on CPUs that lack the instruction.
|
||||
//!
|
||||
//! 3. **Zero-copy shared mappings** — new `AddressSpace::map_shared()` maps a
|
||||
//! physical range owned by another capability into this address space without
|
||||
//! copying the frames. `VmaBacking::Shared` stores the `phys_base` so the
|
||||
//! physical address is retrievable at any time via `translate()` on the PT.
|
||||
//! Frames are *not* freed on unmap — ownership stays with the issuing actor.
|
||||
//!
|
||||
//! All other behaviour (lazy demand paging, revocation, TLB management) is
|
||||
//! unchanged from the previous version.
|
||||
|
||||
#![allow(dead_code)]
|
||||
|
||||
use alloc::vec::Vec;
|
||||
use core::sync::atomic::{AtomicBool, AtomicU32, Ordering};
|
||||
|
||||
use crate::mem::address::{PhysAddr, VirtAddr};
|
||||
use crate::mem::allocator::Locked;
|
||||
use crate::mem::paging::{PageTable, PageTableFlags};
|
||||
use crate::mem::pmm;
|
||||
use crate::events::MMU_REVOCATION_QUEUE;
|
||||
|
||||
extern crate alloc;
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
// Error type
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum VmError {
|
||||
/// PMM returned `None` — no physical frames available.
|
||||
OutOfMemory,
|
||||
/// The requested virtual range overlaps an existing VMA.
|
||||
RegionOverlap,
|
||||
/// No VMA covers the given address.
|
||||
RegionNotFound,
|
||||
/// Address or size is not a multiple of 4096.
|
||||
InvalidAlignment,
|
||||
/// Size is zero, or `virt + size` would overflow.
|
||||
InvalidRange,
|
||||
/// Write fault on a read-only VMA, or exec fault on a NX VMA.
|
||||
PermissionDenied,
|
||||
/// Page-fault in a non-lazy (already-eager or fixed) region — hardware bug
|
||||
/// or an exploit attempt; the faulting task must be killed.
|
||||
UnexpectedFault,
|
||||
/// Address outside the x86-64 canonical range.
|
||||
NonCanonical,
|
||||
/// ASID pool exhausted (> 4094 simultaneous address spaces).
|
||||
AsidExhausted,
|
||||
}
|
||||
|
||||
impl core::fmt::Display for VmError {
|
||||
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
|
||||
f.write_str(match self {
|
||||
Self::OutOfMemory => "physical memory exhausted",
|
||||
Self::RegionOverlap => "virtual address range overlaps existing VMA",
|
||||
Self::RegionNotFound => "no VMA at address",
|
||||
Self::InvalidAlignment => "address/size not page-aligned",
|
||||
Self::InvalidRange => "zero-size or overflowing range",
|
||||
Self::PermissionDenied => "VMA permission denied",
|
||||
Self::UnexpectedFault => "page fault in non-lazy region",
|
||||
Self::NonCanonical => "non-canonical virtual address",
|
||||
Self::AsidExhausted => "ASID/PCID pool exhausted",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
// VMA flags
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
bitflags::bitflags! {
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct VmaFlags: u32 {
|
||||
const READ = 1 << 0;
|
||||
const WRITE = 1 << 1;
|
||||
const EXEC = 1 << 2;
|
||||
const USER = 1 << 3;
|
||||
const LAZY = 1 << 4;
|
||||
const SHARED = 1 << 5;
|
||||
const PINNED = 1 << 6;
|
||||
const NOCACHE = 1 << 7;
|
||||
const MMIO = 1 << 8;
|
||||
}
|
||||
}
|
||||
|
||||
impl VmaFlags {
|
||||
#[inline]
|
||||
pub fn to_page_flags(self) -> PageTableFlags {
|
||||
let mut f = PageTableFlags::PRESENT;
|
||||
if self.contains(Self::WRITE) { f |= PageTableFlags::WRITABLE; }
|
||||
if self.contains(Self::USER) { f |= PageTableFlags::USER; }
|
||||
if !self.contains(Self::EXEC) { f |= PageTableFlags::NO_EXECUTE; }
|
||||
if self.contains(Self::NOCACHE) || self.contains(Self::MMIO) {
|
||||
f |= PageTableFlags::NO_CACHE | PageTableFlags::WRITE_THROUGH;
|
||||
}
|
||||
f
|
||||
}
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
// VMA backing
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum VmaBacking {
|
||||
/// Anonymous pages (stack, heap, BSS).
|
||||
/// Index `i` → frame for `virt_start + i * 4096`. `None` = not yet faulted in.
|
||||
Anonymous(Vec<Option<PhysAddr>>),
|
||||
|
||||
/// Fixed physical range. Frames are **not** freed on unmap.
|
||||
/// Used for MMIO, identity-mapped RAM, framebuffer, DMA buffers.
|
||||
Physical(PhysAddr),
|
||||
|
||||
/// Zero-copy borrow of another actor's frames.
|
||||
///
|
||||
/// `phys_base` is the physical address of the first page; the mapping covers
|
||||
/// exactly `(virt_end - virt_start) / 4096` pages.
|
||||
/// Frames are owned by `owner_cap` and **never** freed by this VMA.
|
||||
Shared {
|
||||
/// Token of the capability that owns the frames.
|
||||
owner_cap: u64,
|
||||
/// Physical base (first page of the shared region).
|
||||
phys_base: PhysAddr,
|
||||
},
|
||||
}
|
||||
|
||||
impl VmaBacking {
|
||||
#[inline]
|
||||
fn phys_for_page(&self, page_idx: usize) -> Option<PhysAddr> {
|
||||
match self {
|
||||
Self::Anonymous(frames) => frames.get(page_idx).copied().flatten(),
|
||||
Self::Physical(base) => Some(PhysAddr(base.0 + page_idx as u64 * 4096)),
|
||||
Self::Shared { phys_base, .. } => {
|
||||
Some(PhysAddr(phys_base.0 + page_idx as u64 * 4096))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this backing owns its frames (should the PMM free them on unmap).
|
||||
#[inline]
|
||||
fn owns_frames(&self) -> bool {
|
||||
matches!(self, Self::Anonymous(_))
|
||||
}
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
// VMA region
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct VmaRegion {
|
||||
pub virt_start: VirtAddr,
|
||||
pub virt_end: VirtAddr,
|
||||
pub flags: VmaFlags,
|
||||
pub cap_token: u64,
|
||||
pub backing: VmaBacking,
|
||||
}
|
||||
|
||||
impl VmaRegion {
|
||||
#[inline] pub fn size(&self) -> u64 { self.virt_end.0 - self.virt_start.0 }
|
||||
#[inline] pub fn pages(&self) -> usize { (self.size() / 4096) as usize }
|
||||
#[inline] pub fn contains(&self, a: VirtAddr) -> bool {
|
||||
a.0 >= self.virt_start.0 && a.0 < self.virt_end.0
|
||||
}
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
// ASID / PCID allocator (bitmap-based, const-initializable, O(1) amortised)
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/// x86-64 PCIDs: 0 (kernel, no PCID tagging) and 4095 (reserved by spec).
|
||||
/// Valid user ASIDs: 1 – 4094 inclusive.
|
||||
///
|
||||
/// Layout: `bitmap[i]` bit `j` represents ASID `i * 32 + j`.
|
||||
/// Bit set = in use.
|
||||
struct AsidAllocator {
|
||||
/// 4096 bits in 128 × u32 words.
|
||||
bitmap: [u32; 128],
|
||||
/// Start-of-next-search hint (in ASID units, not word units).
|
||||
next_hint: u16,
|
||||
}
|
||||
|
||||
impl AsidAllocator {
|
||||
const fn new() -> Self {
|
||||
let mut bm = [0u32; 128];
|
||||
// Mark ASID 0 (kernel) as permanently in-use.
|
||||
bm[0] |= 1u32;
|
||||
// Mark ASID 4095 (reserved by Intel spec) as permanently in-use.
|
||||
// 4095 / 32 = 127, bit 31.
|
||||
bm[127] |= 1u32 << 31;
|
||||
Self { bitmap: bm, next_hint: 1 }
|
||||
}
|
||||
|
||||
/// Allocate the lowest free ASID in [1, 4094]. O(1) amortised with hint.
|
||||
fn alloc(&mut self) -> Option<u16> {
|
||||
// Two-pass: start from hint, wrap around if necessary.
|
||||
for pass in 0..2usize {
|
||||
let start = if pass == 0 { self.next_hint as usize } else { 1 };
|
||||
let end = if pass == 0 { 4095usize } else { self.next_hint as usize };
|
||||
|
||||
let mut asid = start;
|
||||
while asid < end {
|
||||
let word = asid / 32;
|
||||
let bit = asid % 32;
|
||||
if self.bitmap[word] & (1u32 << bit) == 0 {
|
||||
self.bitmap[word] |= 1u32 << bit;
|
||||
self.next_hint = ((asid + 1) as u16).min(4094);
|
||||
return Some(asid as u16);
|
||||
}
|
||||
// Skip fully-used 32-bit words for amortised O(1).
|
||||
if self.bitmap[word] == u32::MAX {
|
||||
asid = (word + 1) * 32;
|
||||
} else {
|
||||
asid += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
None // pool truly exhausted
|
||||
}
|
||||
|
||||
/// Return an ASID to the pool.
|
||||
fn free(&mut self, asid: u16) {
|
||||
if asid == 0 || asid >= 4095 {
|
||||
return; // sentinel values — never freed
|
||||
}
|
||||
let word = asid as usize / 32;
|
||||
let bit = asid as usize % 32;
|
||||
// Clear the bit. Idempotent: double-free is safe but wasteful.
|
||||
self.bitmap[word] &= !(1u32 << bit);
|
||||
// Pull hint back so reuse happens quickly.
|
||||
if asid < self.next_hint {
|
||||
self.next_hint = asid;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static ASID_ALLOC: Locked<AsidAllocator> = Locked::new(AsidAllocator::new());
|
||||
|
||||
#[inline]
|
||||
fn alloc_asid() -> Result<u16, VmError> {
|
||||
ASID_ALLOC.lock().alloc().ok_or(VmError::AsidExhausted)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn free_asid(asid: u16) {
|
||||
ASID_ALLOC.lock().free(asid);
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
// CPU feature detection (call once during boot, before first activate())
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/// Set to `true` at boot if CPUID.07H:EBX[10] = 1 (INVPCID supported).
|
||||
static INVPCID_SUPPORTED: AtomicBool = AtomicBool::new(false);
|
||||
|
||||
/// Detect and record the availability of INVPCID.
|
||||
///
|
||||
/// Must be called **once** during kernel init (before `init_kernel_space` or
|
||||
/// any `AddressSpace::activate`). Idempotent if called more than once.
|
||||
pub fn init_cpu_features() {
|
||||
// CPUID leaf 7, sub-leaf 0 — INVPCID is EBX[10].
|
||||
//
|
||||
// LLVM permanently reserves RBX for its own use in inline asm, so we
|
||||
// cannot name it as an operand directly. The standard workaround is to
|
||||
// spill RBX around the CPUID instruction ourselves.
|
||||
let ebx: u32;
|
||||
unsafe {
|
||||
core::arch::asm!(
|
||||
// Spill rbx (LLVM's reserved reg) to a caller-saved scratch reg.
|
||||
"mov {tmp:r}, rbx",
|
||||
"mov eax, 7",
|
||||
"xor ecx, ecx",
|
||||
"cpuid",
|
||||
// Move the result out before restoring rbx.
|
||||
"mov {out:e}, ebx",
|
||||
"mov rbx, {tmp:r}",
|
||||
tmp = out(reg) _, // any scratch register chosen by LLVM
|
||||
out = out(reg) ebx,
|
||||
out("eax") _,
|
||||
out("ecx") _,
|
||||
out("edx") _,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
let supported = (ebx >> 10) & 1 == 1;
|
||||
INVPCID_SUPPORTED.store(supported, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
// Address space
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
pub struct AddressSpace {
|
||||
/// Hardware PCID written to CR3 bits 11:0.
|
||||
pub asid: u16,
|
||||
/// Physical address of the PML4 page.
|
||||
pub pml4_phys: PhysAddr,
|
||||
/// Sorted (by virt_start), non-overlapping VMA list.
|
||||
regions: Vec<VmaRegion>,
|
||||
/// HHDM offset for dereferencing page-table pages.
|
||||
hhdm: u64,
|
||||
}
|
||||
|
||||
// ── Private helpers ───────────────────────────────────────────────────────────
|
||||
|
||||
impl AddressSpace {
|
||||
#[inline]
|
||||
unsafe fn pml4_raw(&self) -> *mut PageTable {
|
||||
self.pml4_phys.to_virt(self.hhdm).as_mut_ptr::<PageTable>()
|
||||
}
|
||||
|
||||
fn check_overlap(&self, start: VirtAddr, end: VirtAddr) -> Result<(), VmError> {
|
||||
for r in &self.regions {
|
||||
if r.virt_start.0 < end.0 && r.virt_end.0 > start.0 {
|
||||
return Err(VmError::RegionOverlap);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn insert_sorted(&mut self, region: VmaRegion) {
|
||||
let pos = self.regions
|
||||
.partition_point(|r| r.virt_start.0 < region.virt_start.0);
|
||||
self.regions.insert(pos, region);
|
||||
}
|
||||
|
||||
fn find_idx(&self, addr: VirtAddr) -> Option<usize> {
|
||||
let pos = self.regions.partition_point(|r| r.virt_end.0 <= addr.0);
|
||||
self.regions.get(pos).filter(|r| r.contains(addr)).map(|_| pos)
|
||||
}
|
||||
|
||||
/// Unmap pages + free frames (if owned) for one region.
|
||||
fn do_unmap(pml4: *mut PageTable, region: &VmaRegion, hhdm: u64) {
|
||||
let pml4 = unsafe { &mut *pml4 };
|
||||
let own = region.backing.owns_frames();
|
||||
for i in 0..region.pages() {
|
||||
let virt = VirtAddr(region.virt_start.0 + i as u64 * 4096);
|
||||
pml4.unmap_page(virt, hhdm);
|
||||
if own {
|
||||
if let Some(frame) = region.backing.phys_for_page(i) {
|
||||
pmm::free_frame(frame);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn do_revoke_by_token(&mut self, cap_token: u64) {
|
||||
let hhdm = self.hhdm;
|
||||
let pml4 = unsafe { self.pml4_raw() };
|
||||
|
||||
let indices: Vec<usize> = self.regions
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, r)| r.cap_token == cap_token && !r.flags.contains(VmaFlags::PINNED))
|
||||
.map(|(i, _)| i)
|
||||
.collect();
|
||||
|
||||
for idx in indices.into_iter().rev() {
|
||||
let region = self.regions.remove(idx);
|
||||
Self::do_unmap(pml4, ®ion, hhdm);
|
||||
}
|
||||
}
|
||||
|
||||
/// Drain the MMU revocation queue and apply all pending token revocations.
|
||||
/// Called by the page-fault handler before acquiring the VMM lock to
|
||||
/// prevent deadlocks with the capability subsystem.
|
||||
pub fn process_pending_revocations(&mut self) {
|
||||
let mut needs_flush = false;
|
||||
while let Some(token) = MMU_REVOCATION_QUEUE.pop() {
|
||||
self.do_revoke_by_token(token);
|
||||
needs_flush = true;
|
||||
}
|
||||
if needs_flush {
|
||||
tlb_flush_asid(self.asid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Public API ────────────────────────────────────────────────────────────────
|
||||
|
||||
impl AddressSpace {
|
||||
// ─── Construction ──────────────────────────────────────────────────────
|
||||
|
||||
/// Allocate a fresh, empty address space with a zeroed PML4.
|
||||
pub fn new(hhdm: u64) -> Result<Self, VmError> {
|
||||
let pml4_phys = pmm::alloc_frame().ok_or(VmError::OutOfMemory)?;
|
||||
unsafe {
|
||||
core::ptr::write_bytes(pml4_phys.to_virt(hhdm).as_mut_ptr::<u8>(), 0, 4096);
|
||||
}
|
||||
Ok(Self {
|
||||
asid: alloc_asid()?,
|
||||
pml4_phys,
|
||||
regions: Vec::new(),
|
||||
hhdm,
|
||||
})
|
||||
}
|
||||
|
||||
/// Wrap an already-active PML4 (bootloader-provided kernel table).
|
||||
/// ASID 0 = kernel "no PCID tagging"; it is never returned to the pool.
|
||||
pub fn from_active(pml4_phys: PhysAddr, asid: u16, hhdm: u64) -> Self {
|
||||
Self { asid, pml4_phys, regions: Vec::new(), hhdm }
|
||||
}
|
||||
|
||||
// ─── Anonymous / fixed mapping ─────────────────────────────────────────
|
||||
|
||||
/// Map `size` bytes of virtual space starting at `virt`.
|
||||
///
|
||||
/// | `phys` | `VmaFlags::LAZY` | Behaviour |
|
||||
/// |-----------|------------------|-----------------------------------------------|
|
||||
/// | `Some(p)` | any | Fixed physical (MMIO / identity / DMA) |
|
||||
/// | `None` | not set | Eager anonymous — allocate + zero + map now |
|
||||
/// | `None` | set | Lazy anonymous — map frames on first fault |
|
||||
pub fn map_region(
|
||||
&mut self,
|
||||
virt: VirtAddr,
|
||||
phys: Option<PhysAddr>,
|
||||
size: u64,
|
||||
flags: VmaFlags,
|
||||
cap_token: u64,
|
||||
) -> Result<VirtAddr, VmError> {
|
||||
if size == 0 { return Err(VmError::InvalidRange); }
|
||||
if virt.0 & 0xFFF != 0 || size & 0xFFF != 0 {
|
||||
return Err(VmError::InvalidAlignment);
|
||||
}
|
||||
|
||||
let virt_end = VirtAddr(virt.0.checked_add(size).ok_or(VmError::InvalidRange)?);
|
||||
self.check_overlap(virt, virt_end)?;
|
||||
|
||||
let pml4 = unsafe { &mut *self.pml4_raw() };
|
||||
let page_flags = flags.to_page_flags();
|
||||
let page_count = (size / 4096) as usize;
|
||||
let hhdm = self.hhdm;
|
||||
|
||||
let backing = match phys {
|
||||
Some(base) => {
|
||||
pml4.map_region(virt, base, size, page_flags, hhdm);
|
||||
VmaBacking::Physical(base)
|
||||
}
|
||||
None if flags.contains(VmaFlags::LAZY) => {
|
||||
let mut frames = Vec::with_capacity(page_count);
|
||||
frames.resize_with(page_count, || None);
|
||||
VmaBacking::Anonymous(frames)
|
||||
}
|
||||
None => {
|
||||
let mut frames = Vec::with_capacity(page_count);
|
||||
for i in 0..page_count {
|
||||
let frame = pmm::alloc_frame().ok_or(VmError::OutOfMemory)?;
|
||||
unsafe {
|
||||
core::ptr::write_bytes(frame.to_virt(hhdm).as_mut_ptr::<u8>(), 0, 4096);
|
||||
}
|
||||
pml4.map_page(
|
||||
VirtAddr(virt.0 + i as u64 * 4096),
|
||||
frame,
|
||||
page_flags,
|
||||
hhdm,
|
||||
);
|
||||
frames.push(Some(frame));
|
||||
}
|
||||
VmaBacking::Anonymous(frames)
|
||||
}
|
||||
};
|
||||
|
||||
self.insert_sorted(VmaRegion { virt_start: virt, virt_end, flags, cap_token, backing });
|
||||
Ok(virt)
|
||||
}
|
||||
|
||||
// ─── Zero-copy shared mapping (new) ───────────────────────────────────
|
||||
|
||||
/// Map `page_count` pages of physical memory owned by `owner_cap` into
|
||||
/// this address space at `virt`.
|
||||
///
|
||||
/// This is the primitive behind Elyz's zero-copy transfer: the physical
|
||||
/// frames stay in place; only the PTE entries (rights) move between address
|
||||
/// spaces. The frames are **not** freed when this mapping is unmapped —
|
||||
/// the owning actor's capability retains that responsibility.
|
||||
///
|
||||
/// # Revocation
|
||||
/// Pass `owner_cap` as `cap_token` in the VMA so that `revoke_by_token`
|
||||
/// finds and unmaps this shared view when the source capability is revoked.
|
||||
pub fn map_shared(
|
||||
&mut self,
|
||||
virt: VirtAddr,
|
||||
phys_base: PhysAddr,
|
||||
page_count: usize,
|
||||
flags: VmaFlags,
|
||||
owner_cap: u64,
|
||||
) -> Result<VirtAddr, VmError> {
|
||||
if page_count == 0 { return Err(VmError::InvalidRange); }
|
||||
if virt.0 & 0xFFF != 0 { return Err(VmError::InvalidAlignment); }
|
||||
|
||||
let size = page_count as u64 * 4096;
|
||||
let virt_end = VirtAddr(virt.0.checked_add(size).ok_or(VmError::InvalidRange)?);
|
||||
self.check_overlap(virt, virt_end)?;
|
||||
|
||||
let pml4 = unsafe { &mut *self.pml4_raw() };
|
||||
let page_flags = flags.to_page_flags();
|
||||
let hhdm = self.hhdm;
|
||||
|
||||
// Eagerly map all pages — the physical addresses are already known.
|
||||
for i in 0..page_count {
|
||||
pml4.map_page(
|
||||
VirtAddr(virt.0 + i as u64 * 4096),
|
||||
PhysAddr(phys_base.0 + i as u64 * 4096),
|
||||
page_flags,
|
||||
hhdm,
|
||||
);
|
||||
}
|
||||
|
||||
self.insert_sorted(VmaRegion {
|
||||
virt_start: virt,
|
||||
virt_end,
|
||||
flags: flags | VmaFlags::SHARED,
|
||||
cap_token: owner_cap,
|
||||
backing: VmaBacking::Shared { owner_cap, phys_base },
|
||||
});
|
||||
|
||||
Ok(virt)
|
||||
}
|
||||
|
||||
// ─── Demand paging ─────────────────────────────────────────────────────
|
||||
|
||||
/// Handle a hardware page fault at `fault_addr`.
|
||||
///
|
||||
/// Returns `Ok(())` if the fault was a valid lazy demand-page (caller should
|
||||
/// `iretq` to retry). Returns `Err` for illegal accesses.
|
||||
pub fn handle_fault(&mut self, fault_addr: VirtAddr, write: bool) -> Result<(), VmError> {
|
||||
let hhdm = self.hhdm;
|
||||
let idx = self.find_idx(fault_addr).ok_or(VmError::RegionNotFound)?;
|
||||
|
||||
{
|
||||
let region = &self.regions[idx];
|
||||
if write && !region.flags.contains(VmaFlags::WRITE) {
|
||||
return Err(VmError::PermissionDenied);
|
||||
}
|
||||
if !region.flags.contains(VmaFlags::LAZY) {
|
||||
return Err(VmError::UnexpectedFault);
|
||||
}
|
||||
}
|
||||
|
||||
let region = &mut self.regions[idx];
|
||||
let page_idx = ((fault_addr.0 - region.virt_start.0) / 4096) as usize;
|
||||
let page_virt = VirtAddr(region.virt_start.0 + page_idx as u64 * 4096);
|
||||
let page_flags = region.flags.to_page_flags();
|
||||
|
||||
let VmaBacking::Anonymous(ref mut frames) = region.backing else {
|
||||
return Err(VmError::RegionNotFound);
|
||||
};
|
||||
|
||||
if frames[page_idx].is_some() {
|
||||
// SMP race: another core already mapped this page.
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let frame = pmm::alloc_frame().ok_or(VmError::OutOfMemory)?;
|
||||
unsafe {
|
||||
core::ptr::write_bytes(frame.to_virt(hhdm).as_mut_ptr::<u8>(), 0, 4096);
|
||||
}
|
||||
frames[page_idx] = Some(frame);
|
||||
|
||||
let pml4 = unsafe { &mut *self.pml4_raw() };
|
||||
pml4.map_page(page_virt, frame, page_flags, hhdm);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ─── Unmapping ─────────────────────────────────────────────────────────
|
||||
|
||||
/// Unmap the VMA containing `virt`, free its frames (if owned), flush TLB.
|
||||
pub fn unmap_region(&mut self, virt: VirtAddr) -> Result<(), VmError> {
|
||||
let idx = self.find_idx(virt).ok_or(VmError::RegionNotFound)?;
|
||||
let region = self.regions.remove(idx);
|
||||
let pml4 = unsafe { self.pml4_raw() };
|
||||
Self::do_unmap(pml4, ®ion, self.hhdm);
|
||||
tlb_flush_asid(self.asid);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ─── Capability revocation ─────────────────────────────────────────────
|
||||
|
||||
/// Atomically unmap all VMAs associated with `cap_token` (skip PINNED).
|
||||
///
|
||||
/// Hardware access is terminated before this function returns.
|
||||
/// Shared-backed VMAs are unmapped from the PT without freeing frames.
|
||||
pub fn revoke_by_token(&mut self, cap_token: u64) {
|
||||
let hhdm = self.hhdm;
|
||||
let pml4 = unsafe { self.pml4_raw() };
|
||||
|
||||
let indices: Vec<usize> = self.regions
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, r)| {
|
||||
r.cap_token == cap_token && !r.flags.contains(VmaFlags::PINNED)
|
||||
})
|
||||
.map(|(i, _)| i)
|
||||
.collect();
|
||||
|
||||
for idx in indices.into_iter().rev() {
|
||||
let region = self.regions.remove(idx);
|
||||
Self::do_unmap(pml4, ®ion, hhdm);
|
||||
}
|
||||
|
||||
tlb_flush_asid(self.asid);
|
||||
}
|
||||
|
||||
// ─── Address translation ───────────────────────────────────────────────
|
||||
|
||||
/// Walk the live page table to translate `virt` → physical address.
|
||||
pub fn translate(&self, virt: VirtAddr) -> Option<PhysAddr> {
|
||||
unsafe { (*self.pml4_raw()).translate(virt, self.hhdm) }
|
||||
}
|
||||
|
||||
// ─── Activation ────────────────────────────────────────────────────────
|
||||
|
||||
/// Load this address space into the CPU (context switch).
|
||||
///
|
||||
/// Sets CR3 with bit 63 = NOFLUSH to preserve PCID-tagged TLB entries.
|
||||
/// Requires CR4.PCIDE = 1 (enabled by the bootloader/early init).
|
||||
///
|
||||
/// # Safety
|
||||
/// Must only be called from a code path that is identity-mapped in every
|
||||
/// address space (e.g. the kernel `.text` section reachable via HHDM).
|
||||
pub unsafe fn activate(&self) {
|
||||
let cr3 = self.pml4_phys.0 | u64::from(self.asid) | (1u64 << 63);
|
||||
unsafe {
|
||||
core::arch::asm!(
|
||||
"mov cr3, {0}",
|
||||
in(reg) cr3,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── Introspection ─────────────────────────────────────────────────────
|
||||
|
||||
#[inline] pub fn regions(&self) -> &[VmaRegion] { &self.regions }
|
||||
#[inline] pub fn region_count(&self) -> usize { self.regions.len() }
|
||||
|
||||
pub fn find_region(&self, addr: VirtAddr) -> Option<&VmaRegion> {
|
||||
self.find_idx(addr).map(|i| &self.regions[i])
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for AddressSpace {
|
||||
fn drop(&mut self) {
|
||||
let hhdm = self.hhdm;
|
||||
let pml4 = unsafe { self.pml4_raw() };
|
||||
while let Some(region) = self.regions.pop() {
|
||||
Self::do_unmap(pml4, ®ion, hhdm);
|
||||
}
|
||||
pmm::free_frame(self.pml4_phys);
|
||||
// Return the ASID to the pool so it can be reused by future processes.
|
||||
// ASID 0 is the kernel sentinel — never freed (guarded inside free_asid).
|
||||
free_asid(self.asid);
|
||||
}
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
// TLB management
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/// Flush all TLB entries tagged with `asid` (PCID) on the current core.
|
||||
///
|
||||
/// Uses `INVPCID` type-1 (single-context flush) when available (Broadwell+,
|
||||
/// CPUID.07H:EBX[10] = 1). Falls back to a full CR3 reload on older CPUs.
|
||||
///
|
||||
/// **SMP note**: on multi-core systems a TLB-shootdown IPI to all remote cores
|
||||
/// must be added once the LAPIC driver and scheduler are online.
|
||||
pub fn tlb_flush_asid(asid: u16) {
|
||||
if INVPCID_SUPPORTED.load(Ordering::Relaxed) {
|
||||
#[repr(C, packed)]
|
||||
struct InvpcidDesc { pcid: u64, addr: u64 }
|
||||
|
||||
let desc = InvpcidDesc { pcid: asid as u64, addr: 0 };
|
||||
unsafe {
|
||||
core::arch::asm!(
|
||||
"invpcid {ty}, [{desc}]",
|
||||
ty = in(reg) 1u64, // type 1 = single-context flush
|
||||
desc = in(reg) &desc,
|
||||
options(nostack, preserves_flags),
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// Fallback: full TLB flush via CR3 reload (clears all PCID entries).
|
||||
tlb_flush_all();
|
||||
}
|
||||
}
|
||||
|
||||
/// Full TLB flush (all PCIDs, all addresses) via CR3 reload.
|
||||
pub fn tlb_flush_all() {
|
||||
unsafe {
|
||||
let mut cr3: u64;
|
||||
core::arch::asm!("mov {0}, cr3", out(reg) cr3,
|
||||
options(nostack, preserves_flags));
|
||||
// Writing CR3 without bit 63 forces a full flush.
|
||||
core::arch::asm!("mov cr3, {0}",
|
||||
in(reg) cr3 & !(1u64 << 63),
|
||||
options(nostack, preserves_flags));
|
||||
}
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
// Global kernel address space
|
||||
// ══════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
/// The one kernel address space. Initialised once during boot.
|
||||
pub static KERNEL_SPACE: Locked<Option<AddressSpace>> = Locked::new(None);
|
||||
|
||||
/// Register the already-active PML4 as the kernel address space.
|
||||
///
|
||||
/// ASID 0 = PCID 0 = kernel (no per-process PCID tagging).
|
||||
/// Must be called after `BitmapPMM::init` and `init_cpu_features`.
|
||||
pub fn init_kernel_space(pml4_phys: PhysAddr, hhdm: u64) {
|
||||
*KERNEL_SPACE.lock() = Some(AddressSpace::from_active(pml4_phys, 0, hhdm));
|
||||
}
|
||||
Reference in New Issue
Block a user