feat: docs & ARCH 2.2, 2.3, 2.4
This commit is contained in:
Faynot
@@ -1,6 +1,7 @@
|
||||
// src/cpu/lapic.rs
|
||||
|
||||
use core::sync::atomic::{AtomicU64, Ordering};
|
||||
use crate::mem::address::get_hhdm;
|
||||
|
||||
pub const LAPIC_DEFAULT_BASE: u64 = 0xFEE00_000;
|
||||
const LAPIC_EOI: u64 = 0x0B0;
|
||||
@@ -8,8 +9,8 @@ const LAPIC_ICR_LOW: u64 = 0x300;
|
||||
|
||||
static LAPIC_VIRT_BASE: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
pub fn init(hhdm_offset: u64) {
|
||||
LAPIC_VIRT_BASE.store(LAPIC_DEFAULT_BASE + hhdm_offset, Ordering::SeqCst);
|
||||
pub fn init() {
|
||||
LAPIC_VIRT_BASE.store(LAPIC_DEFAULT_BASE + get_hhdm(), Ordering::SeqCst);
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
|
||||
@@ -14,6 +14,7 @@ use crate::mem::paging::{PageTable, PageTableFlags};
|
||||
use crate::mem::address::{PhysAddr, VirtAddr};
|
||||
use crate::cap::{Relation, CapRights, Capability, CapObject};
|
||||
use crate::mem::pm_manages::{PMActor, PMRequest, PMResult};
|
||||
use crate::mem::buddy::BuddyAllocator;
|
||||
|
||||
pub mod cap;
|
||||
pub mod cpu;
|
||||
@@ -175,6 +176,7 @@ unsafe extern "C" fn kmain() -> ! {
|
||||
let hhdm_res = HHDM_REQUEST.get_response().expect("Limine: No HHDM");
|
||||
let kaddr_res = KERNEL_ADDR_REQUEST.get_response().expect("Limine: No Kernel Address");
|
||||
let hhdm_offset = hhdm_res.offset();
|
||||
crate::mem::address::init_hhdm(hhdm_offset);
|
||||
|
||||
let fb = fb_res.framebuffers().next().expect("Limine: No active framebuffer found");
|
||||
let mut console = tty::Console::new(&fb, KERNEL_FONT);
|
||||
@@ -185,17 +187,17 @@ unsafe extern "C" fn kmain() -> ! {
|
||||
cpu::interrupts::init_early_exceptions();
|
||||
info!(console, "BOOT", "LIS4 Kernel Starting...");
|
||||
|
||||
unsafe { mem::pmm::BitmapPMM::init(&mmap_res, hhdm_offset); }
|
||||
unsafe { mem::pmm::BitmapPMM::init(&mmap_res); }
|
||||
info!(console, "MEM", "Primary Physical Memory Manager (BitmapPMM) initialized.");
|
||||
|
||||
cpu::lapic::init(hhdm_offset);
|
||||
cpu::lapic::init();
|
||||
info!(console, "LAPIC", "Local APIC initialized.");
|
||||
|
||||
// debug: locate the free page
|
||||
info!(console, "BOOT", "alloc_frame...");
|
||||
let p4_phys = mem::pmm::alloc_frame().expect("OOM: Failed to allocate P4 table");
|
||||
info!(console, "BOOT", "alloc_frame ok: phys=0x{:x}", p4_phys.0);
|
||||
let virt = p4_phys.to_virt(hhdm_offset);
|
||||
let virt = p4_phys.to_virt();
|
||||
info!(console, "BOOT", "virt=0x{:x}", virt.0);
|
||||
let p4 = unsafe { &mut *virt.as_mut_ptr::<PageTable>() };
|
||||
info!(console, "BOOT", "zeroing page...");
|
||||
@@ -206,10 +208,10 @@ unsafe extern "C" fn kmain() -> ! {
|
||||
|
||||
for (i, entry) in mmap_res.entries().iter().enumerate() {
|
||||
let phys = PhysAddr(entry.base);
|
||||
let virt_hhdm = phys.to_virt(hhdm_offset);
|
||||
p4.map_region(virt_hhdm, phys, entry.length, flags, hhdm_offset);
|
||||
let virt_hhdm = phys.to_virt();
|
||||
p4.map_region(virt_hhdm, phys, entry.length, flags);
|
||||
if entry.entry_type != limine::memory_map::EntryType::RESERVED {
|
||||
p4.map_region(VirtAddr(entry.base), phys, entry.length, flags, hhdm_offset);
|
||||
p4.map_region(VirtAddr(entry.base), phys, entry.length, flags);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -218,7 +220,6 @@ unsafe extern "C" fn kmain() -> ! {
|
||||
PhysAddr(kaddr_res.physical_base()),
|
||||
0x1000 * 1024,
|
||||
flags,
|
||||
hhdm_offset
|
||||
);
|
||||
|
||||
info!(console, "MMU", "Activating Kernel Page Tables...");
|
||||
@@ -228,7 +229,7 @@ unsafe extern "C" fn kmain() -> ! {
|
||||
let heap_size = 8 * 1024 * 1024;
|
||||
for i in (0..heap_size).step_by(4096) {
|
||||
let frame = mem::pmm::alloc_frame().expect("OOM: Heap allocation failed");
|
||||
p4.map_page(VirtAddr(heap_start + i as u64), frame, flags, hhdm_offset);
|
||||
p4.map_page(VirtAddr(heap_start + i as u64), frame, flags);
|
||||
}
|
||||
|
||||
{
|
||||
@@ -238,7 +239,7 @@ unsafe extern "C" fn kmain() -> ! {
|
||||
info!(console, "HEAP", "Kernel Slab Allocator is online.");
|
||||
|
||||
mem::init_cpu_features();
|
||||
mem::vmm::init_kernel_space(p4_phys, hhdm_offset);
|
||||
mem::vmm::init_kernel_space(p4_phys);
|
||||
info!(console, "VMM", "Kernel Address Space registered successfully.");
|
||||
|
||||
cpu::interrupts::init_idt();
|
||||
@@ -367,6 +368,74 @@ unsafe extern "C" fn kmain() -> ! {
|
||||
panic!("CRITICAL STATE LOSS: Memory leak detected inside PMActor context!");
|
||||
}
|
||||
|
||||
// === ARCH 2.2: Buddy Allocator Intrusive List — Direct Test ===
|
||||
info!(console, "PM", "=-= Buddy Allocator Direct Test (Intrusive List) =-=");
|
||||
|
||||
let buddy_test_pages = 128;
|
||||
let buddy_test_base = PhysAddr(0x5000_0000);
|
||||
let mut buddy = BuddyAllocator::new(buddy_test_pages, buddy_test_base.0);
|
||||
|
||||
let b1 = buddy.alloc(0).expect("buddy: alloc order 0");
|
||||
let b2 = buddy.alloc(0).expect("buddy: alloc order 0 #2");
|
||||
buddy.free(b1, 0);
|
||||
buddy.free(b2, 0);
|
||||
assert_eq!(buddy.free_pages(), buddy_test_pages, "buddy: pages not fully recovered after simple alloc/free");
|
||||
info!(console, "PM", " [OK] Simple alloc/free");
|
||||
|
||||
let b3 = buddy.alloc(1).expect("buddy: alloc order 1");
|
||||
assert_eq!(b3, 0, "buddy: first order 1 block at 0");
|
||||
let b4 = buddy.alloc(1).expect("buddy: alloc order 1 #2");
|
||||
assert_eq!(b4, 2, "buddy: second order 1 block at 2");
|
||||
buddy.free(b3, 1);
|
||||
buddy.free(b4, 1);
|
||||
let b5 = buddy.alloc(2).expect("buddy: order 2 after coalescing");
|
||||
assert_eq!(b5, 0, "buddy: coalesced block at 0");
|
||||
buddy.free(b5, 2);
|
||||
info!(console, "PM", " [OK] Coalescing across orders (O(1) intrusive list)");
|
||||
|
||||
let mut allocs = Vec::new();
|
||||
loop {
|
||||
match buddy.alloc(0) {
|
||||
Some(idx) => allocs.push(idx),
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
assert!(buddy.is_exhausted(), "buddy: should be exhausted");
|
||||
info!(console, "PM", " [OK] Exhaustion after {} allocs", allocs.len());
|
||||
|
||||
for (i, idx) in allocs.iter().enumerate() {
|
||||
buddy.free(*idx, 0);
|
||||
}
|
||||
assert_eq!(buddy.free_pages(), buddy_test_pages, "buddy: not fully recovered");
|
||||
info!(console, "PM", " [OK] Full recovery after freeing {} pages", allocs.len());
|
||||
|
||||
info!(console, "PM", "~) BUDDY DIRECT TEST PASSED (~");
|
||||
|
||||
// === ARCH 2.3: PMM Tree Bitmap — alloc/free stress test ===
|
||||
info!(console, "PM", "=-= PMM Tree Bitmap Alloc/Free Stress Test =-=");
|
||||
|
||||
const PMM_TEST_COUNT: usize = 64;
|
||||
let mut frames = [PhysAddr(0); PMM_TEST_COUNT];
|
||||
for i in 0..PMM_TEST_COUNT {
|
||||
frames[i] = mem::pmm::alloc_frame().expect("PMM tree: OOM during alloc test");
|
||||
}
|
||||
info!(console, "PM", " [OK] Allocated {} frames", PMM_TEST_COUNT);
|
||||
|
||||
for f in &frames {
|
||||
mem::pmm::free_frame(*f);
|
||||
}
|
||||
info!(console, "PM", " [OK] Freed {} frames back", PMM_TEST_COUNT);
|
||||
|
||||
for i in 0..PMM_TEST_COUNT {
|
||||
frames[i] = mem::pmm::alloc_frame().expect("PMM tree: OOM after free cycle");
|
||||
}
|
||||
info!(console, "PM", " [OK] Re-allocated {} frames after free cycle", PMM_TEST_COUNT);
|
||||
|
||||
for f in &frames {
|
||||
mem::pmm::free_frame(*f);
|
||||
}
|
||||
info!(console, "PM", "~) PMM TREE BITMAP TEST PASSED (~");
|
||||
|
||||
let logo = r#"
|
||||
###########
|
||||
##################
|
||||
|
||||
@@ -1,3 +1,19 @@
|
||||
use core::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
static HHDM_OFFSET: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
/// Store the Higher-Half Direct Map offset obtained from the bootloader.
|
||||
/// Must be called once during early boot, before any address translation.
|
||||
pub fn init_hhdm(offset: u64) {
|
||||
HHDM_OFFSET.store(offset, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Return the HHDM offset (kernel virtual base for physical memory).
|
||||
#[inline]
|
||||
pub fn get_hhdm() -> u64 {
|
||||
HHDM_OFFSET.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
|
||||
#[repr(transparent)]
|
||||
pub struct PhysAddr(pub u64);
|
||||
@@ -8,8 +24,8 @@ pub struct VirtAddr(pub u64);
|
||||
|
||||
impl PhysAddr {
|
||||
/// Convert physical address to virtual via HHDM offset
|
||||
pub fn to_virt(self, hhdm_offset: u64) -> VirtAddr {
|
||||
VirtAddr(self.0 + hhdm_offset)
|
||||
pub fn to_virt(self) -> VirtAddr {
|
||||
VirtAddr(self.0 + get_hhdm())
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
@@ -24,9 +40,10 @@ impl PhysAddr {
|
||||
|
||||
impl VirtAddr {
|
||||
#[allow(dead_code)]
|
||||
pub fn to_phys(self, hhdm_offset: u64) -> Option<PhysAddr> {
|
||||
if self.0 < hhdm_offset { return None; }
|
||||
Some(PhysAddr(self.0 - hhdm_offset))
|
||||
pub fn to_phys(self) -> Option<PhysAddr> {
|
||||
let hhdm = get_hhdm();
|
||||
if self.0 < hhdm { return None; }
|
||||
Some(PhysAddr(self.0 - hhdm))
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
|
||||
@@ -1,105 +1,99 @@
|
||||
//! # Per-Actor Buddy Allocator (`src/mem/buddy.rs`)
|
||||
//!
|
||||
//! A power-of-two page-block allocator for a fixed, pre-committed physical
|
||||
//! region. Designed for **single-consumer** use inside a [`crate::mem::pm_manages::PMActor`]:
|
||||
//! the owning actor is the only entity that ever mutates its buddy allocator, so
|
||||
//! no locking is needed — the entire subsystem is inherently lock-free from the
|
||||
//! consumer's perspective.
|
||||
//!
|
||||
//! ## Block Layout
|
||||
//!
|
||||
//! The region is subdivided into blocks of `2^order` pages (order 0 … MAX_ORDER).
|
||||
//! Every block at order *k* is aligned to `2^k` pages within the region.
|
||||
//!
|
||||
//! ```text
|
||||
//! order 0 → 1 page = 4 KiB
|
||||
//! order 1 → 2 pages = 8 KiB
|
||||
//! …
|
||||
//! order 11 → 2 048 pages = 8 MiB (MAX_ORDER)
|
||||
//! ```
|
||||
//!
|
||||
//! ## Complexity
|
||||
//!
|
||||
//! | Operation | Amortised | Worst-case |
|
||||
//! |-----------|-----------|------------|
|
||||
//! | `alloc` | O(log N) | O(MAX_ORDER · Lₖ) |
|
||||
//! | `free` | O(log N) | O(MAX_ORDER · Lₖ) |
|
||||
//!
|
||||
//! where Lₖ = `free_lists[k].len()` ≤ `total_pages / 2^k`.
|
||||
//!
|
||||
//! ## Production note
|
||||
//!
|
||||
//! For managed ranges > 1 GiB (> 256 K pages), replace the per-order `Vec<usize>`
|
||||
//! free lists with a radix tree or interval tree to bound Lₖ. For current
|
||||
//! PMActor capital sizes (typically ≤ 128 MiB = 32 K pages at order 0), the
|
||||
//! `Vec`-based implementation is fully adequate.
|
||||
use crate::mem::address::{PhysAddr, VirtAddr};
|
||||
|
||||
extern crate alloc;
|
||||
use alloc::vec::Vec;
|
||||
|
||||
// Constants & helpers
|
||||
/// Maximum allocation order.
|
||||
/// `2^11 × 4 096 bytes = 8 MiB` per single allocation.
|
||||
pub const MAX_ORDER: usize = 11;
|
||||
|
||||
/// ⌈log2(n)⌉ — the minimum order whose block size covers `page_count` pages.
|
||||
///
|
||||
/// ```text
|
||||
/// order_for(1) = 0 (2^0 = 1)
|
||||
/// order_for(2) = 1 (2^1 = 2)
|
||||
/// order_for(3) = 2 (2^2 = 4 ≥ 3)
|
||||
/// order_for(2048) = 11
|
||||
/// ```
|
||||
const NEXT_SENTINEL: usize = usize::MAX;
|
||||
|
||||
#[inline]
|
||||
pub fn order_for(page_count: usize) -> usize {
|
||||
if page_count <= 1 {
|
||||
0
|
||||
} else {
|
||||
// Number of bits needed to represent (page_count - 1).
|
||||
usize::BITS as usize - (page_count - 1).leading_zeros() as usize
|
||||
}
|
||||
}
|
||||
|
||||
// BuddyAllocator
|
||||
/// Buddy allocator over a contiguous, pre-committed range of physical pages.
|
||||
///
|
||||
/// All page indices stored in free lists are **relative to the start of the
|
||||
/// managed range**. The owner (`PMActor`) translates to absolute `PhysAddr`
|
||||
/// by adding `managed_range.0`.
|
||||
///
|
||||
/// # Invariants
|
||||
/// - `free_pages ≤ total_pages` at all times.
|
||||
/// - Every block in `free_lists[k]` is aligned to `2^k` pages (i.e.
|
||||
/// `idx % (1 << k) == 0`).
|
||||
/// - No block appears in more than one order's free list simultaneously.
|
||||
pub struct BuddyAllocator {
|
||||
/// `free_lists[k]` = relative page indices of free 2^k-page blocks.
|
||||
/// Ordering within each list is irrelevant; `pop()` / `swap_remove()` are used.
|
||||
free_lists: [Vec<usize>; MAX_ORDER + 1],
|
||||
/// Total pages in the managed range (need not be a power of two).
|
||||
free_heads: [usize; MAX_ORDER + 1],
|
||||
total_pages: usize,
|
||||
/// Running count of free pages. Always equals `Σ (2^k × free_lists[k].len())`.
|
||||
free_pages: usize,
|
||||
base_phys: u64,
|
||||
}
|
||||
|
||||
impl BuddyAllocator {
|
||||
//Construction
|
||||
/// Create a new allocator over `total_pages` pages, **all initially free**.
|
||||
///
|
||||
/// Uses a greedy largest-first decomposition to build the initial free lists
|
||||
/// in O(total_pages / 2^MAX_ORDER) iterations — essentially O(1) for
|
||||
/// power-of-two sizes.
|
||||
///
|
||||
/// Example: 7 pages → blocks [4, 2, 1] → free_lists[2]=[0], [1]=[4], [0]=[6].
|
||||
pub fn new(total_pages: usize) -> Self {
|
||||
// core::array::from_fn is the idiomatic way to init a non-Copy array.
|
||||
let free_lists: [Vec<usize>; MAX_ORDER + 1] =
|
||||
core::array::from_fn(|_| Vec::new());
|
||||
#[inline]
|
||||
fn page_virt(&self, idx: usize) -> VirtAddr {
|
||||
PhysAddr(self.base_phys + idx as u64 * 4096).to_virt()
|
||||
}
|
||||
|
||||
unsafe fn write_node(&self, idx: usize, next: usize, prev: usize) {
|
||||
unsafe {
|
||||
let ptr = self.page_virt(idx).as_mut_ptr::<usize>();
|
||||
ptr.write(next);
|
||||
ptr.add(1).write(prev);
|
||||
}
|
||||
}
|
||||
|
||||
unsafe fn read_next(&self, idx: usize) -> usize {
|
||||
unsafe { self.page_virt(idx).as_ptr::<usize>().read() }
|
||||
}
|
||||
|
||||
unsafe fn read_prev(&self, idx: usize) -> usize {
|
||||
unsafe { self.page_virt(idx).as_ptr::<usize>().add(1).read() }
|
||||
}
|
||||
|
||||
fn flist_push(&mut self, order: usize, idx: usize) {
|
||||
let head = self.free_heads[order];
|
||||
unsafe {
|
||||
self.write_node(idx, head, NEXT_SENTINEL);
|
||||
if head != NEXT_SENTINEL {
|
||||
self.write_node(head, self.read_next(head), idx);
|
||||
}
|
||||
}
|
||||
self.free_heads[order] = idx;
|
||||
}
|
||||
|
||||
fn flist_remove(&mut self, order: usize, idx: usize) {
|
||||
unsafe {
|
||||
let next = self.read_next(idx);
|
||||
let prev = self.read_prev(idx);
|
||||
|
||||
self.write_node(idx, NEXT_SENTINEL, NEXT_SENTINEL);
|
||||
|
||||
if prev != NEXT_SENTINEL {
|
||||
self.write_node(prev, next, self.read_prev(prev));
|
||||
} else {
|
||||
self.free_heads[order] = next;
|
||||
}
|
||||
|
||||
if next != NEXT_SENTINEL {
|
||||
self.write_node(next, self.read_next(next), prev);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn flist_pop(&mut self, order: usize) -> Option<usize> {
|
||||
let head = self.free_heads[order];
|
||||
if head == NEXT_SENTINEL {
|
||||
return None;
|
||||
}
|
||||
self.flist_remove(order, head);
|
||||
Some(head)
|
||||
}
|
||||
|
||||
fn flist_contains(&self, order: usize, idx: usize) -> bool {
|
||||
if self.free_heads[order] == idx {
|
||||
return true;
|
||||
}
|
||||
unsafe { self.read_prev(idx) != NEXT_SENTINEL }
|
||||
}
|
||||
|
||||
pub fn new(total_pages: usize, base_phys: u64) -> Self {
|
||||
let mut this = Self {
|
||||
free_lists,
|
||||
free_heads: [NEXT_SENTINEL; MAX_ORDER + 1],
|
||||
total_pages,
|
||||
free_pages: 0,
|
||||
base_phys,
|
||||
};
|
||||
|
||||
if total_pages == 0 {
|
||||
@@ -110,22 +104,19 @@ impl BuddyAllocator {
|
||||
while idx < total_pages {
|
||||
let remaining = total_pages - idx;
|
||||
|
||||
// Alignment constraint: block at `idx` must be aligned to 2^order.
|
||||
// trailing_zeros(0) is u32::MAX, so we clamp to MAX_ORDER.
|
||||
let align_order = if idx == 0 {
|
||||
MAX_ORDER
|
||||
} else {
|
||||
(idx.trailing_zeros() as usize).min(MAX_ORDER)
|
||||
};
|
||||
|
||||
// Size constraint: 2^order ≤ remaining → order ≤ ⌊log₂(remaining)⌋.
|
||||
let size_order = (usize::BITS as usize - 1)
|
||||
- remaining.leading_zeros() as usize; // ⌊log2(remaining)⌋
|
||||
- remaining.leading_zeros() as usize;
|
||||
|
||||
let order = MAX_ORDER.min(align_order).min(size_order);
|
||||
let order = MAX_ORDER.min(align_order).min(size_order);
|
||||
let block_size = 1usize << order;
|
||||
|
||||
this.free_lists[order].push(idx);
|
||||
this.flist_push(order, idx);
|
||||
this.free_pages += block_size;
|
||||
idx += block_size;
|
||||
}
|
||||
@@ -133,60 +124,30 @@ impl BuddyAllocator {
|
||||
this
|
||||
}
|
||||
|
||||
//Allocation
|
||||
/// Allocate a 2^`order`-page block.
|
||||
///
|
||||
/// Returns the **relative** page index of the block's first page, or `None`
|
||||
/// if insufficient contiguous memory remains.
|
||||
///
|
||||
/// The returned index can be converted to a physical address:
|
||||
/// ```text
|
||||
/// phys = actor.managed_range.0.0 + (idx as u64) * PAGE_SIZE
|
||||
/// ```
|
||||
pub fn alloc(&mut self, order: usize) -> Option<usize> {
|
||||
if order > MAX_ORDER {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Find the smallest available order ≥ requested.
|
||||
let found_order = (order..=MAX_ORDER)
|
||||
.find(|&o| !self.free_lists[o].is_empty())?;
|
||||
.find(|&o| self.free_heads[o] != NEXT_SENTINEL)?;
|
||||
|
||||
// Consume one block from the found order.
|
||||
let block_idx = self.free_lists[found_order]
|
||||
.pop()
|
||||
.expect("buddy: free_list non-empty but pop() returned None");
|
||||
let block_idx = self.flist_pop(found_order)
|
||||
.expect("buddy: free_list non-empty but flist_pop returned None");
|
||||
|
||||
// Debit: we removed a 2^found_order block from free.
|
||||
self.free_pages -= 1 << found_order;
|
||||
|
||||
// Split down to the requested order, putting buddies back into free lists.
|
||||
//
|
||||
// Invariant at each iteration:
|
||||
// `block_idx` is the lower half of a 2^cur_order block.
|
||||
// The upper half (= `block_idx + 2^(cur_order-1)`) is returned to the
|
||||
// free list as an order-(cur_order-1) block.
|
||||
let mut cur_order = found_order;
|
||||
while cur_order > order {
|
||||
cur_order -= 1;
|
||||
let buddy_idx = block_idx + (1 << cur_order);
|
||||
self.free_lists[cur_order].push(buddy_idx);
|
||||
self.flist_push(cur_order, buddy_idx);
|
||||
self.free_pages += 1 << cur_order;
|
||||
}
|
||||
|
||||
// Net accounting: removed 2^found_order, added (2^found_order − 2^order).
|
||||
// Result: free_pages decreased by exactly 2^order. ✓
|
||||
Some(block_idx)
|
||||
}
|
||||
|
||||
/// Allocate `page_count` pages, rounding up to the nearest power of two.
|
||||
///
|
||||
/// Returns `(relative_page_idx, actual_order)`. The caller **must** pass the
|
||||
/// same `order` to [`BuddyAllocator::free`] — mismatched orders corrupt the
|
||||
/// allocator.
|
||||
///
|
||||
/// The wasted "rounding up" pages remain unusable until the block is freed.
|
||||
/// For tightly-packed allocations, callers should use `alloc(order)` directly.
|
||||
pub fn alloc_pages(&mut self, page_count: usize) -> Option<(usize, usize)> {
|
||||
if page_count == 0 {
|
||||
return None;
|
||||
@@ -198,17 +159,6 @@ impl BuddyAllocator {
|
||||
self.alloc(order).map(|idx| (idx, order))
|
||||
}
|
||||
|
||||
//Deallocation
|
||||
|
||||
/// Return a 2^`order`-page block at **relative** index `block_idx` to the
|
||||
/// free pool, coalescing with free buddies up the order chain.
|
||||
///
|
||||
/// # Panics (debug builds only)
|
||||
/// - `order > MAX_ORDER`
|
||||
/// - `block_idx + 2^order > total_pages`
|
||||
///
|
||||
/// In release builds the checks are elided for performance; passing incorrect
|
||||
/// arguments causes undefined bookkeeping, not UB (no unsafe indexing).
|
||||
pub fn free(&mut self, mut block_idx: usize, mut order: usize) {
|
||||
debug_assert!(
|
||||
order <= MAX_ORDER,
|
||||
@@ -224,67 +174,60 @@ impl BuddyAllocator {
|
||||
self.total_pages
|
||||
);
|
||||
|
||||
// Try to coalesce with our buddy at each order.
|
||||
//
|
||||
// The buddy of a block at relative index `i` of order `k` is at:
|
||||
// buddy_idx = i XOR 2^k
|
||||
// This works because aligned buddy pairs always differ in exactly bit k.
|
||||
while order < MAX_ORDER {
|
||||
let buddy_idx = block_idx ^ (1 << order);
|
||||
|
||||
// Buddy must lie entirely within the managed range.
|
||||
let buddy_end = match buddy_idx.checked_add(1 << order) {
|
||||
Some(e) => e,
|
||||
None => break,
|
||||
None => break,
|
||||
};
|
||||
if buddy_end > self.total_pages {
|
||||
break;
|
||||
}
|
||||
|
||||
// Search for the buddy in the free list for this order.
|
||||
// `swap_remove` is O(1) and order-preserving is not required.
|
||||
if let Some(pos) = self.free_lists[order].iter().position(|&b| b == buddy_idx) {
|
||||
self.free_lists[order].swap_remove(pos);
|
||||
self.free_pages -= 1 << order; // buddy was in free count; remove it
|
||||
|
||||
// Merged block starts at the lower address of the two.
|
||||
if self.flist_contains(order, buddy_idx) {
|
||||
self.flist_remove(order, buddy_idx);
|
||||
self.free_pages -= 1 << order;
|
||||
block_idx = block_idx.min(buddy_idx);
|
||||
order += 1;
|
||||
// Continue trying to merge at the next level.
|
||||
} else {
|
||||
break; // buddy is allocated or out-of-range; stop coalescing
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Push the (possibly merged) block onto the appropriate free list.
|
||||
self.free_pages += 1 << order;
|
||||
self.free_lists[order].push(block_idx);
|
||||
self.flist_push(order, block_idx);
|
||||
}
|
||||
|
||||
//Introspection
|
||||
|
||||
/// Number of pages currently available for allocation.
|
||||
#[inline]
|
||||
pub fn free_pages(&self) -> usize {
|
||||
self.free_pages
|
||||
}
|
||||
|
||||
/// Total pages in the managed range.
|
||||
#[inline]
|
||||
pub fn total_pages(&self) -> usize {
|
||||
self.total_pages
|
||||
}
|
||||
|
||||
/// `true` if the allocator has no free pages.
|
||||
#[inline]
|
||||
pub fn is_exhausted(&self) -> bool {
|
||||
self.free_pages == 0
|
||||
}
|
||||
|
||||
/// Dump free-list statistics for each order (useful in panic handlers).
|
||||
///
|
||||
/// Returns an array `[(order, free_block_count); MAX_ORDER + 1]`.
|
||||
pub fn stats(&self) -> [(usize, usize); MAX_ORDER + 1] {
|
||||
core::array::from_fn(|o| (o, self.free_lists[o].len()))
|
||||
core::array::from_fn(|o| {
|
||||
let count = if self.free_heads[o] == NEXT_SENTINEL {
|
||||
0
|
||||
} else {
|
||||
let mut cnt = 0;
|
||||
let mut cur = self.free_heads[o];
|
||||
while cur != NEXT_SENTINEL {
|
||||
cnt += 1;
|
||||
cur = unsafe { self.read_next(cur) };
|
||||
}
|
||||
cnt
|
||||
};
|
||||
(o, count)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,8 +12,8 @@ pub mod pm_router;
|
||||
/// Initialise the physical memory manager.
|
||||
///
|
||||
/// Must be called before any heap allocation or VMM operation.
|
||||
pub fn init_pmm(memmap: &limine::response::MemoryMapResponse, hhdm_offset: u64) {
|
||||
unsafe { pmm::BitmapPMM::init(memmap, hhdm_offset); }
|
||||
pub fn init_pmm(memmap: &limine::response::MemoryMapResponse) {
|
||||
unsafe { pmm::BitmapPMM::init(memmap); }
|
||||
}
|
||||
|
||||
/// Initialise CPU features required by the VMM (INVPCID detection).
|
||||
@@ -25,8 +25,8 @@ pub fn init_cpu_features() {
|
||||
|
||||
/// Initialise the kernel address space record (after PMM, heap, and the
|
||||
/// initial P4 page table have been set up in `kmain`).
|
||||
pub fn init_vmm(pml4_phys: address::PhysAddr, hhdm_offset: u64) {
|
||||
vmm::init_kernel_space(pml4_phys, hhdm_offset);
|
||||
pub fn init_vmm(pml4_phys: address::PhysAddr) {
|
||||
vmm::init_kernel_space(pml4_phys);
|
||||
}
|
||||
|
||||
/// Physical memory statistics: `(used_pages, total_pages)`.
|
||||
|
||||
@@ -40,16 +40,15 @@ impl PageTable {
|
||||
phys: PhysAddr,
|
||||
size: u64,
|
||||
flags: PageTableFlags,
|
||||
hhdm: u64,
|
||||
) {
|
||||
let pages = size.div_ceil(4096);
|
||||
for i in 0..pages {
|
||||
let offset = i * 4096;
|
||||
self.map_page(VirtAddr(virt.0 + offset), PhysAddr(phys.0 + offset), flags, hhdm);
|
||||
self.map_page(VirtAddr(virt.0 + offset), PhysAddr(phys.0 + offset), flags);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_flags(&self, virt: VirtAddr, hhdm: u64) -> Option<PageTableFlags> {
|
||||
pub fn get_flags(&self, virt: VirtAddr) -> Option<PageTableFlags> {
|
||||
let p4_idx = ((virt.0 >> 39) & 0x1FF) as usize;
|
||||
let p3_idx = ((virt.0 >> 30) & 0x1FF) as usize;
|
||||
let p2_idx = ((virt.0 >> 21) & 0x1FF) as usize;
|
||||
@@ -59,7 +58,7 @@ impl PageTable {
|
||||
($entry:expr) => {{
|
||||
let e = $entry;
|
||||
if e & PageTableFlags::PRESENT.bits() == 0 { return None; }
|
||||
unsafe { &*PhysAddr(e & PTE_ADDR_MASK).to_virt(hhdm).as_mut_ptr::<PageTable>() }
|
||||
unsafe { &*PhysAddr(e & PTE_ADDR_MASK).to_virt().as_mut_ptr::<PageTable>() }
|
||||
}};
|
||||
}
|
||||
|
||||
@@ -82,8 +81,8 @@ impl PageTable {
|
||||
Some(PageTableFlags::from_bits_truncate(p1e))
|
||||
}
|
||||
|
||||
pub fn update_flags(&mut self, virt: VirtAddr, flags: PageTableFlags, hhdm: u64) -> Result<(), ()> {
|
||||
let Some(p1) = self.walk_to_p1_mut(virt, hhdm, false) else { return Err(()); };
|
||||
pub fn update_flags(&mut self, virt: VirtAddr, flags: PageTableFlags) -> Result<(), ()> {
|
||||
let Some(p1) = self.walk_to_p1_mut(virt, false) else { return Err(()); };
|
||||
let p1_idx = ((virt.0 >> 12) & 0x1FF) as usize;
|
||||
let entry = p1.entries[p1_idx];
|
||||
|
||||
@@ -107,10 +106,9 @@ impl PageTable {
|
||||
virt: VirtAddr,
|
||||
phys: PhysAddr,
|
||||
flags: PageTableFlags,
|
||||
hhdm: u64,
|
||||
) {
|
||||
let p1 = self
|
||||
.walk_to_p1_mut(virt, hhdm, true /* create */)
|
||||
.walk_to_p1_mut(virt, true /* create */)
|
||||
.expect("map_page: OOM allocating intermediate page-table pages");
|
||||
|
||||
let p1_idx = ((virt.0 >> 12) & 0x1FF) as usize;
|
||||
@@ -126,8 +124,8 @@ impl PageTable {
|
||||
/// Does **not** free the underlying physical frame — that is the caller's
|
||||
/// responsibility (e.g. `VmaBacking::do_unmap`).
|
||||
/// Silently returns if any level is not present (idempotent).
|
||||
pub fn unmap_page(&mut self, virt: VirtAddr, hhdm: u64) {
|
||||
let Some(p1) = self.walk_to_p1_mut(virt, hhdm, false /* no create */) else {
|
||||
pub fn unmap_page(&mut self, virt: VirtAddr) {
|
||||
let Some(p1) = self.walk_to_p1_mut(virt, false /* no create */) else {
|
||||
return; // already absent — nothing to do
|
||||
};
|
||||
let p1_idx = ((virt.0 >> 12) & 0x1FF) as usize;
|
||||
@@ -144,7 +142,7 @@ impl PageTable {
|
||||
///
|
||||
/// Handles 1 GiB and 2 MiB huge pages transparently.
|
||||
/// Returns `None` if any level is absent or the page is not present.
|
||||
pub fn translate(&self, virt: VirtAddr, hhdm: u64) -> Option<PhysAddr> {
|
||||
pub fn translate(&self, virt: VirtAddr) -> Option<PhysAddr> {
|
||||
let p4_idx = ((virt.0 >> 39) & 0x1FF) as usize;
|
||||
let p3_idx = ((virt.0 >> 30) & 0x1FF) as usize;
|
||||
let p2_idx = ((virt.0 >> 21) & 0x1FF) as usize;
|
||||
@@ -155,7 +153,7 @@ impl PageTable {
|
||||
($entry:expr) => {{
|
||||
let e = $entry;
|
||||
if e & PageTableFlags::PRESENT.bits() == 0 { return None; }
|
||||
unsafe { &*PhysAddr(e & PTE_ADDR_MASK).to_virt(hhdm).as_mut_ptr::<PageTable>() }
|
||||
unsafe { &*PhysAddr(e & PTE_ADDR_MASK).to_virt().as_mut_ptr::<PageTable>() }
|
||||
}};
|
||||
}
|
||||
|
||||
@@ -209,16 +207,16 @@ impl PageTable {
|
||||
///
|
||||
/// If `create` is `false` and any intermediate entry is absent, returns `None`.
|
||||
/// If `create` is `true`, allocates missing intermediate pages from the PMM.
|
||||
fn walk_to_p1_mut(&mut self, virt: VirtAddr, hhdm: u64, create: bool)
|
||||
fn walk_to_p1_mut(&mut self, virt: VirtAddr, create: bool)
|
||||
-> Option<&mut Self>
|
||||
{
|
||||
let p4_idx = ((virt.0 >> 39) & 0x1FF) as usize;
|
||||
let p3_idx = ((virt.0 >> 30) & 0x1FF) as usize;
|
||||
let p2_idx = ((virt.0 >> 21) & 0x1FF) as usize;
|
||||
|
||||
let p3 = self.get_or_create_next_table(p4_idx, hhdm, create)?;
|
||||
let p2 = p3.get_or_create_next_table(p3_idx, hhdm, create)?;
|
||||
p2.get_or_create_next_table(p2_idx, hhdm, create)
|
||||
let p3 = self.get_or_create_next_table(p4_idx, create)?;
|
||||
let p2 = p3.get_or_create_next_table(p3_idx, create)?;
|
||||
p2.get_or_create_next_table(p2_idx, create)
|
||||
}
|
||||
|
||||
/// Return a mutable reference to the next-level table at `index`.
|
||||
@@ -229,7 +227,6 @@ impl PageTable {
|
||||
fn get_or_create_next_table(
|
||||
&mut self,
|
||||
index: usize,
|
||||
hhdm: u64,
|
||||
create: bool,
|
||||
) -> Option<&mut Self> {
|
||||
let entry = self.entries[index];
|
||||
@@ -237,7 +234,7 @@ impl PageTable {
|
||||
if !create { return None; }
|
||||
|
||||
let pt_phys = pmm_alloc().expect("VMM: OOM allocating page-table page");
|
||||
let pt_virt = pt_phys.to_virt(hhdm);
|
||||
let pt_virt = pt_phys.to_virt();
|
||||
unsafe { core::ptr::write_bytes(pt_virt.as_mut_ptr::<u8>(), 0, 4096); }
|
||||
|
||||
// Install with USER so both kernel and user pages can live under it;
|
||||
@@ -247,7 +244,7 @@ impl PageTable {
|
||||
}
|
||||
|
||||
let next_phys = PhysAddr(self.entries[index] & PTE_ADDR_MASK);
|
||||
Some(unsafe { &mut *next_phys.to_virt(hhdm).as_mut_ptr::<Self>() })
|
||||
Some(unsafe { &mut *next_phys.to_virt().as_mut_ptr::<Self>() })
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -279,7 +279,7 @@ impl PMActor {
|
||||
root_untyped: root_cap,
|
||||
managed_range: (start, PhysAddr(start.0 + total_pages as u64 * PAGE_SIZE)),
|
||||
queue: PMActorQueue::new(),
|
||||
buddy: BuddyAllocator::new(total_pages),
|
||||
buddy: BuddyAllocator::new(total_pages, start.0),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -140,9 +140,10 @@ impl PMRouter {
|
||||
let channel = &self.channels[id as usize];
|
||||
|
||||
while channel.state.load(Ordering::Acquire) != STATE_READY {
|
||||
core::hint::spin_loop();
|
||||
// TODO: Для "Focus Mode" и полноценного планировщика:
|
||||
// scheduler::yield_to_actor();
|
||||
// HLT-ожидание вместо busy-wait: CPU останавливается до ближайшего
|
||||
// прерывания (timer tick, IPI от route_responses и т.д.).
|
||||
// Когда появится планировщик — заменить на yield_to_actor().
|
||||
unsafe { core::arch::asm!("hlt", options(nomem, nostack, preserves_flags)); }
|
||||
}
|
||||
|
||||
let result = unsafe {
|
||||
|
||||
@@ -1,19 +1,36 @@
|
||||
use crate::mem::address::PhysAddr;
|
||||
use crate::mem::address::{PhysAddr, get_hhdm};
|
||||
use crate::mem::allocator::Locked;
|
||||
|
||||
pub const PAGE_SIZE: u64 = 4096;
|
||||
|
||||
pub struct BitmapPMM {
|
||||
bitmap: &'static mut [u8],
|
||||
l1_bitmap: &'static mut [u64],
|
||||
ref_counts: &'static mut [u16],
|
||||
total_pages: usize,
|
||||
used_pages: usize,
|
||||
last_byte: usize,
|
||||
last_word: usize,
|
||||
}
|
||||
|
||||
pub static PMM: Locked<Option<BitmapPMM>> = Locked::new(None);
|
||||
|
||||
impl BitmapPMM {
|
||||
unsafe fn read_word(&self, word_idx: usize) -> u64 {
|
||||
let byte_off = word_idx * 8;
|
||||
unsafe { (self.bitmap.as_ptr().add(byte_off) as *const u64).read() }
|
||||
}
|
||||
|
||||
fn l1_update_word(&mut self, word_idx: usize) {
|
||||
let word = unsafe { self.read_word(word_idx) };
|
||||
let l1_idx = word_idx / 64;
|
||||
let l1_bit = word_idx % 64;
|
||||
if word != !0u64 {
|
||||
self.l1_bitmap[l1_idx] |= 1 << l1_bit;
|
||||
} else {
|
||||
self.l1_bitmap[l1_idx] &= !(1 << l1_bit);
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub fn used_pages(&self) -> usize { self.used_pages }
|
||||
#[allow(dead_code)]
|
||||
@@ -21,7 +38,7 @@ impl BitmapPMM {
|
||||
#[allow(dead_code)]
|
||||
pub fn free_pages(&self) -> usize { self.total_pages.saturating_sub(self.used_pages) }
|
||||
|
||||
pub unsafe fn init(mmap: &limine::response::MemoryMapResponse, hhdm_offset: u64) {
|
||||
pub unsafe fn init(mmap: &limine::response::MemoryMapResponse) {
|
||||
let max_addr = mmap.entries().iter()
|
||||
.map(|e| e.base + e.length)
|
||||
.max()
|
||||
@@ -31,7 +48,10 @@ impl BitmapPMM {
|
||||
|
||||
let bitmap_size = total_pages.div_ceil(8);
|
||||
let ref_counts_size = total_pages * core::mem::size_of::<u16>();
|
||||
let total_meta_size = bitmap_size + ref_counts_size;
|
||||
let num_words = total_pages.div_ceil(64);
|
||||
let l1_u64_count = num_words.div_ceil(64);
|
||||
let l1_byte_size = l1_u64_count * 8;
|
||||
let total_meta_size = bitmap_size + ref_counts_size + l1_byte_size;
|
||||
|
||||
let meta_phys = mmap.entries().iter()
|
||||
.find(|e| {
|
||||
@@ -41,8 +61,10 @@ impl BitmapPMM {
|
||||
.map(|e| e.base)
|
||||
.expect("PMM: no usable region large enough for metadata");
|
||||
|
||||
let bitmap_ptr = (meta_phys + hhdm_offset) as *mut u8;
|
||||
let ref_counts_ptr = (meta_phys + hhdm_offset + bitmap_size as u64) as *mut u16;
|
||||
let hhdm = get_hhdm();
|
||||
let bitmap_ptr = (meta_phys + hhdm) as *mut u8;
|
||||
let ref_counts_ptr = (meta_phys + hhdm + bitmap_size as u64) as *mut u16;
|
||||
let l1_ptr = (meta_phys + hhdm + bitmap_size as u64 + ref_counts_size as u64) as *mut u64;
|
||||
|
||||
let bitmap = unsafe { core::slice::from_raw_parts_mut(bitmap_ptr, bitmap_size) };
|
||||
bitmap.fill(0xFF);
|
||||
@@ -50,15 +72,18 @@ impl BitmapPMM {
|
||||
let ref_counts = unsafe { core::slice::from_raw_parts_mut(ref_counts_ptr, total_pages) };
|
||||
ref_counts.fill(1);
|
||||
|
||||
let l1_bitmap = unsafe { core::slice::from_raw_parts_mut(l1_ptr, l1_u64_count) };
|
||||
l1_bitmap.fill(0);
|
||||
|
||||
let mut pmm = Self {
|
||||
bitmap,
|
||||
l1_bitmap,
|
||||
ref_counts,
|
||||
total_pages,
|
||||
used_pages: total_pages,
|
||||
last_byte: 0,
|
||||
last_word: 0,
|
||||
};
|
||||
|
||||
// Free all USABLE pages (gap pages not covered by any entry stay locked).
|
||||
for entry in mmap.entries() {
|
||||
if entry.entry_type == limine::memory_map::EntryType::USABLE {
|
||||
for addr in (entry.base..entry.base + entry.length).step_by(PAGE_SIZE as usize) {
|
||||
@@ -67,7 +92,6 @@ impl BitmapPMM {
|
||||
}
|
||||
}
|
||||
|
||||
// Metadata pages sit inside a USABLE region — lock them back.
|
||||
let meta_end = (meta_phys + total_meta_size as u64 + PAGE_SIZE - 1) & !(PAGE_SIZE - 1);
|
||||
for addr in (meta_phys..meta_end).step_by(PAGE_SIZE as usize) {
|
||||
pmm.lock_frame(PhysAddr(addr));
|
||||
@@ -78,8 +102,6 @@ impl BitmapPMM {
|
||||
*PMM.lock() = Some(pmm);
|
||||
}
|
||||
|
||||
//Core operations
|
||||
|
||||
pub fn free_frame(&mut self, phys_addr: PhysAddr) {
|
||||
let idx = (phys_addr.0 / PAGE_SIZE) as usize;
|
||||
if idx >= self.total_pages { return; }
|
||||
@@ -92,8 +114,11 @@ impl BitmapPMM {
|
||||
|
||||
if self.ref_counts[idx] == 0 {
|
||||
self.bitmap[byte] &= !(1 << bit);
|
||||
self.used_pages -= 1;
|
||||
if byte < self.last_byte { self.last_byte = byte; }
|
||||
self.used_pages -= 1;
|
||||
|
||||
let word_idx = idx / 64;
|
||||
self.l1_update_word(word_idx);
|
||||
if word_idx < self.last_word { self.last_word = word_idx; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -108,10 +133,13 @@ impl BitmapPMM {
|
||||
if self.bitmap[byte] & (1 << bit) == 0 {
|
||||
self.bitmap[byte] |= 1 << bit;
|
||||
self.ref_counts[idx] = 1;
|
||||
self.used_pages += 1;
|
||||
self.used_pages += 1;
|
||||
} else if self.ref_counts[idx] == 0 {
|
||||
self.ref_counts[idx] = 1;
|
||||
}
|
||||
|
||||
let word_idx = idx / 64;
|
||||
self.l1_update_word(word_idx);
|
||||
}
|
||||
|
||||
pub fn inc_ref_frame(&mut self, phys_addr: PhysAddr) {
|
||||
@@ -127,31 +155,45 @@ impl BitmapPMM {
|
||||
}
|
||||
|
||||
pub fn alloc_frame(&mut self) -> Option<PhysAddr> {
|
||||
let len = self.bitmap.len();
|
||||
let l1_len = self.l1_bitmap.len();
|
||||
|
||||
for pass in 0..2usize {
|
||||
let (from, to) = if pass == 0 {
|
||||
(self.last_byte, len)
|
||||
(self.last_word / 64, l1_len)
|
||||
} else {
|
||||
(0, self.last_byte)
|
||||
(0, self.last_word / 64)
|
||||
};
|
||||
|
||||
for byte_idx in from..to {
|
||||
if self.bitmap[byte_idx] == 0xFF { continue; }
|
||||
for l1_idx in from..to {
|
||||
let l1_word = self.l1_bitmap[l1_idx];
|
||||
if l1_word == 0 { continue; }
|
||||
|
||||
for bit in 0..8u8 {
|
||||
if self.bitmap[byte_idx] & (1 << bit) == 0 {
|
||||
let page_idx = byte_idx * 8 + bit as usize;
|
||||
if page_idx >= self.total_pages { return None; }
|
||||
let word_offset = l1_word.trailing_zeros() as usize;
|
||||
let word_idx = l1_idx * 64 + word_offset;
|
||||
|
||||
self.bitmap[byte_idx] |= 1 << bit;
|
||||
self.ref_counts[page_idx] = 1;
|
||||
self.used_pages += 1;
|
||||
self.last_byte = byte_idx;
|
||||
|
||||
return Some(PhysAddr(page_idx as u64 * PAGE_SIZE));
|
||||
}
|
||||
let word = unsafe { self.read_word(word_idx) };
|
||||
if word == !0u64 {
|
||||
self.l1_bitmap[l1_idx] &= !(1 << word_offset);
|
||||
continue;
|
||||
}
|
||||
|
||||
let free_bit = (!word).trailing_zeros() as usize;
|
||||
let page_idx = word_idx * 64 + free_bit;
|
||||
if page_idx >= self.total_pages { continue; }
|
||||
|
||||
let byte = page_idx / 8;
|
||||
let bit = page_idx % 8;
|
||||
|
||||
self.bitmap[byte] |= 1 << bit;
|
||||
self.ref_counts[page_idx] = 1;
|
||||
self.used_pages += 1;
|
||||
self.last_word = word_idx;
|
||||
|
||||
if unsafe { self.read_word(word_idx) } == !0u64 {
|
||||
self.l1_bitmap[l1_idx] &= !(1 << word_offset);
|
||||
}
|
||||
|
||||
return Some(PhysAddr(page_idx as u64 * PAGE_SIZE));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -176,9 +218,11 @@ impl BitmapPMM {
|
||||
for i in run_start..run_start + count {
|
||||
self.bitmap[i / 8] |= 1 << (i % 8);
|
||||
self.ref_counts[i] = 1;
|
||||
let word_idx = i / 64;
|
||||
self.l1_update_word(word_idx);
|
||||
}
|
||||
self.used_pages += count;
|
||||
self.last_byte = run_start / 8;
|
||||
self.last_word = run_start / 64;
|
||||
return Some(PhysAddr(run_start as u64 * PAGE_SIZE));
|
||||
}
|
||||
} else {
|
||||
@@ -190,8 +234,6 @@ impl BitmapPMM {
|
||||
}
|
||||
}
|
||||
|
||||
//Module-level convenience functions
|
||||
|
||||
pub fn alloc_frame() -> Option<PhysAddr> {
|
||||
PMM.lock().as_mut()?.alloc_frame()
|
||||
}
|
||||
@@ -219,7 +261,3 @@ pub fn get_stats() -> (usize, usize) {
|
||||
(0, 0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -306,14 +306,12 @@ pub struct AddressSpace {
|
||||
pub pml4_phys: PhysAddr,
|
||||
/// Sorted (by virt_start), non-overlapping VMA list.
|
||||
regions: Vec<VmaRegion>,
|
||||
/// HHDM offset for dereferencing page-table pages.
|
||||
hhdm: u64,
|
||||
}
|
||||
|
||||
impl AddressSpace {
|
||||
#[inline]
|
||||
unsafe fn pml4_raw(&self) -> *mut PageTable {
|
||||
self.pml4_phys.to_virt(self.hhdm).as_mut_ptr::<PageTable>()
|
||||
self.pml4_phys.to_virt().as_mut_ptr::<PageTable>()
|
||||
}
|
||||
|
||||
fn check_overlap(&self, start: VirtAddr, end: VirtAddr) -> Result<(), VmError> {
|
||||
@@ -326,10 +324,9 @@ impl AddressSpace {
|
||||
}
|
||||
|
||||
pub fn clone_for_fork(&mut self, child_cap_token: u64) -> Result<Self, VmError> {
|
||||
let mut child = AddressSpace::new(self.hhdm)?;
|
||||
let mut child = AddressSpace::new()?;
|
||||
let child_pml4 = unsafe { &mut *child.pml4_raw() };
|
||||
let parent_pml4 = unsafe { &mut *self.pml4_raw() };
|
||||
let hhdm = self.hhdm;
|
||||
|
||||
for region in &mut self.regions {
|
||||
let mut child_region = VmaRegion {
|
||||
@@ -350,10 +347,10 @@ impl AddressSpace {
|
||||
|
||||
match &mut region.backing {
|
||||
VmaBacking::Physical(base) => {
|
||||
child_pml4.map_region(region.virt_start, *base, region.size(), region.flags.to_page_flags(), hhdm);
|
||||
child_pml4.map_region(region.virt_start, *base, region.size(), region.flags.to_page_flags());
|
||||
}
|
||||
VmaBacking::Shared { phys_base, .. } => {
|
||||
child_pml4.map_region(region.virt_start, *phys_base, region.size(), region.flags.to_page_flags(), hhdm);
|
||||
child_pml4.map_region(region.virt_start, *phys_base, region.size(), region.flags.to_page_flags());
|
||||
}
|
||||
VmaBacking::Anonymous(frames) => {
|
||||
let cow_needed = region.flags.contains(VmaFlags::WRITE);
|
||||
@@ -377,10 +374,10 @@ impl AddressSpace {
|
||||
page_flags.remove(PageTableFlags::WRITABLE);
|
||||
page_flags.insert(PageTableFlags::COW);
|
||||
|
||||
let _ = parent_pml4.update_flags(virt, page_flags, hhdm);
|
||||
let _ = parent_pml4.update_flags(virt, page_flags);
|
||||
}
|
||||
|
||||
child_pml4.map_page(virt, *frame, page_flags, hhdm);
|
||||
child_pml4.map_page(virt, *frame, page_flags);
|
||||
child_frames[i] = Some(*frame);
|
||||
}
|
||||
}
|
||||
@@ -408,12 +405,12 @@ impl AddressSpace {
|
||||
}
|
||||
|
||||
/// Unmap pages + free frames (if owned) for one region.
|
||||
fn do_unmap(pml4: *mut PageTable, region: &VmaRegion, hhdm: u64) {
|
||||
fn do_unmap(pml4: *mut PageTable, region: &VmaRegion) {
|
||||
let pml4 = unsafe { &mut *pml4 };
|
||||
let own = region.backing.owns_frames();
|
||||
for i in 0..region.pages() {
|
||||
let virt = VirtAddr(region.virt_start.0 + i as u64 * 4096);
|
||||
pml4.unmap_page(virt, hhdm);
|
||||
pml4.unmap_page(virt);
|
||||
if own {
|
||||
if let Some(frame) = region.backing.phys_for_page(i) {
|
||||
pmm::free_frame(frame);
|
||||
@@ -423,7 +420,6 @@ impl AddressSpace {
|
||||
}
|
||||
|
||||
fn do_revoke_by_token(&mut self, cap_token: u64) {
|
||||
let hhdm = self.hhdm;
|
||||
let pml4 = unsafe { self.pml4_raw() };
|
||||
|
||||
let indices: Vec<usize> = self.regions
|
||||
@@ -435,7 +431,7 @@ impl AddressSpace {
|
||||
|
||||
for idx in indices.into_iter().rev() {
|
||||
let region = self.regions.remove(idx);
|
||||
Self::do_unmap(pml4, ®ion, hhdm);
|
||||
Self::do_unmap(pml4, ®ion);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -456,23 +452,22 @@ impl AddressSpace {
|
||||
|
||||
impl AddressSpace {
|
||||
/// Allocate a fresh, empty address space with a zeroed PML4.
|
||||
pub fn new(hhdm: u64) -> Result<Self, VmError> {
|
||||
pub fn new() -> Result<Self, VmError> {
|
||||
let pml4_phys = pmm::alloc_frame().ok_or(VmError::OutOfMemory)?;
|
||||
unsafe {
|
||||
core::ptr::write_bytes(pml4_phys.to_virt(hhdm).as_mut_ptr::<u8>(), 0, 4096);
|
||||
core::ptr::write_bytes(pml4_phys.to_virt().as_mut_ptr::<u8>(), 0, 4096);
|
||||
}
|
||||
Ok(Self {
|
||||
asid: alloc_asid()?,
|
||||
pml4_phys,
|
||||
regions: Vec::new(),
|
||||
hhdm,
|
||||
})
|
||||
}
|
||||
|
||||
/// Wrap an already-active PML4 (bootloader-provided kernel table).
|
||||
/// ASID 0 = kernel "no PCID tagging"; it is never returned to the pool.
|
||||
pub fn from_active(pml4_phys: PhysAddr, asid: u16, hhdm: u64) -> Self {
|
||||
Self { asid, pml4_phys, regions: Vec::new(), hhdm }
|
||||
pub fn from_active(pml4_phys: PhysAddr, asid: u16) -> Self {
|
||||
Self { asid, pml4_phys, regions: Vec::new() }
|
||||
}
|
||||
|
||||
pub fn map_region(
|
||||
@@ -494,11 +489,10 @@ impl AddressSpace {
|
||||
let pml4 = unsafe { &mut *self.pml4_raw() };
|
||||
let page_flags = flags.to_page_flags();
|
||||
let page_count = (size / 4096) as usize;
|
||||
let hhdm = self.hhdm;
|
||||
|
||||
let backing = match phys {
|
||||
Some(base) => {
|
||||
pml4.map_region(virt, base, size, page_flags, hhdm);
|
||||
pml4.map_region(virt, base, size, page_flags);
|
||||
VmaBacking::Physical(base)
|
||||
}
|
||||
None if flags.contains(VmaFlags::LAZY) => {
|
||||
@@ -511,13 +505,12 @@ impl AddressSpace {
|
||||
for i in 0..page_count {
|
||||
let frame = pmm::alloc_frame().ok_or(VmError::OutOfMemory)?;
|
||||
unsafe {
|
||||
core::ptr::write_bytes(frame.to_virt(hhdm).as_mut_ptr::<u8>(), 0, 4096);
|
||||
core::ptr::write_bytes(frame.to_virt().as_mut_ptr::<u8>(), 0, 4096);
|
||||
}
|
||||
pml4.map_page(
|
||||
VirtAddr(virt.0 + i as u64 * 4096),
|
||||
frame,
|
||||
page_flags,
|
||||
hhdm,
|
||||
);
|
||||
frames.push(Some(frame));
|
||||
}
|
||||
@@ -557,7 +550,6 @@ impl AddressSpace {
|
||||
|
||||
let pml4 = unsafe { &mut *self.pml4_raw() };
|
||||
let page_flags = flags.to_page_flags();
|
||||
let hhdm = self.hhdm;
|
||||
|
||||
// Eagerly map all pages — the physical addresses are already known.
|
||||
for i in 0..page_count {
|
||||
@@ -565,7 +557,6 @@ impl AddressSpace {
|
||||
VirtAddr(virt.0 + i as u64 * 4096),
|
||||
PhysAddr(phys_base.0 + i as u64 * 4096),
|
||||
page_flags,
|
||||
hhdm,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -585,7 +576,6 @@ impl AddressSpace {
|
||||
/// Returns `Ok(())` if the fault was a valid lazy demand-page (caller should
|
||||
/// `iretq` to retry). Returns `Err` for illegal accesses.
|
||||
pub fn handle_fault(&mut self, fault_addr: VirtAddr, write: bool) -> Result<(), VmError> {
|
||||
let hhdm = self.hhdm;
|
||||
let idx = self.find_idx(fault_addr).ok_or(VmError::RegionNotFound)?;
|
||||
|
||||
let (virt_start, region_flags) = {
|
||||
@@ -601,7 +591,7 @@ impl AddressSpace {
|
||||
let page_virt = VirtAddr(virt_start.0 + page_idx as u64 * 4096);
|
||||
let pml4 = unsafe { &mut *self.pml4_raw() };
|
||||
|
||||
let current_pte_flags = pml4.get_flags(page_virt, hhdm);
|
||||
let current_pte_flags = pml4.get_flags(page_virt);
|
||||
let is_cow = current_pte_flags.map_or(false, |f| f.contains(PageTableFlags::COW));
|
||||
|
||||
if write && is_cow {
|
||||
@@ -617,8 +607,8 @@ impl AddressSpace {
|
||||
|
||||
unsafe {
|
||||
core::ptr::copy_nonoverlapping(
|
||||
old_frame.to_virt(hhdm).as_ptr::<u8>(),
|
||||
new_frame.to_virt(hhdm).as_mut_ptr::<u8>(),
|
||||
old_frame.to_virt().as_ptr::<u8>(),
|
||||
new_frame.to_virt().as_mut_ptr::<u8>(),
|
||||
4096,
|
||||
);
|
||||
}
|
||||
@@ -629,7 +619,7 @@ impl AddressSpace {
|
||||
target_flags.remove(PageTableFlags::COW);
|
||||
target_flags.insert(PageTableFlags::WRITABLE);
|
||||
|
||||
pml4.map_page(page_virt, new_frame, target_flags, hhdm);
|
||||
pml4.map_page(page_virt, new_frame, target_flags);
|
||||
|
||||
pmm::free_frame(old_frame);
|
||||
|
||||
@@ -651,7 +641,7 @@ impl AddressSpace {
|
||||
|
||||
let frame = pmm::alloc_frame().ok_or(VmError::OutOfMemory)?;
|
||||
unsafe {
|
||||
core::ptr::write_bytes(frame.to_virt(hhdm).as_mut_ptr::<u8>(), 0, 4096);
|
||||
core::ptr::write_bytes(frame.to_virt().as_mut_ptr::<u8>(), 0, 4096);
|
||||
}
|
||||
frames[page_idx] = Some(frame);
|
||||
|
||||
@@ -660,7 +650,7 @@ impl AddressSpace {
|
||||
target_flags.remove(PageTableFlags::WRITABLE);
|
||||
}
|
||||
|
||||
pml4.map_page(page_virt, frame, target_flags, hhdm);
|
||||
pml4.map_page(page_virt, frame, target_flags);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -669,7 +659,7 @@ impl AddressSpace {
|
||||
let idx = self.find_idx(virt).ok_or(VmError::RegionNotFound)?;
|
||||
let region = self.regions.remove(idx);
|
||||
let pml4 = unsafe { self.pml4_raw() };
|
||||
Self::do_unmap(pml4, ®ion, self.hhdm);
|
||||
Self::do_unmap(pml4, ®ion);
|
||||
tlb_flush_asid(self.asid);
|
||||
Ok(())
|
||||
}
|
||||
@@ -679,7 +669,6 @@ impl AddressSpace {
|
||||
/// Hardware access is terminated before this function returns.
|
||||
/// Shared-backed VMAs are unmapped from the PT without freeing frames.
|
||||
pub fn revoke_by_token(&mut self, cap_token: u64) {
|
||||
let hhdm = self.hhdm;
|
||||
let pml4 = unsafe { self.pml4_raw() };
|
||||
|
||||
let indices: Vec<usize> = self.regions
|
||||
@@ -693,7 +682,7 @@ impl AddressSpace {
|
||||
|
||||
for idx in indices.into_iter().rev() {
|
||||
let region = self.regions.remove(idx);
|
||||
Self::do_unmap(pml4, ®ion, hhdm);
|
||||
Self::do_unmap(pml4, ®ion);
|
||||
}
|
||||
|
||||
tlb_flush_asid(self.asid);
|
||||
@@ -703,7 +692,7 @@ impl AddressSpace {
|
||||
|
||||
/// Walk the live page table to translate `virt` → physical address.
|
||||
pub fn translate(&self, virt: VirtAddr) -> Option<PhysAddr> {
|
||||
unsafe { (*self.pml4_raw()).translate(virt, self.hhdm) }
|
||||
unsafe { (*self.pml4_raw()).translate(virt) }
|
||||
}
|
||||
|
||||
//Activation
|
||||
@@ -737,10 +726,9 @@ impl AddressSpace {
|
||||
|
||||
impl Drop for AddressSpace {
|
||||
fn drop(&mut self) {
|
||||
let hhdm = self.hhdm;
|
||||
let pml4 = unsafe { self.pml4_raw() };
|
||||
while let Some(region) = self.regions.pop() {
|
||||
Self::do_unmap(pml4, ®ion, hhdm);
|
||||
Self::do_unmap(pml4, ®ion);
|
||||
}
|
||||
pmm::free_frame(self.pml4_phys);
|
||||
// Return the ASID to the pool so it can be reused by future processes.
|
||||
@@ -823,6 +811,6 @@ pub static KERNEL_SPACE: Locked<Option<AddressSpace>> = Locked::new(None);
|
||||
///
|
||||
/// ASID 0 = PCID 0 = kernel (no per-process PCID tagging).
|
||||
/// Must be called after `BitmapPMM::init` and `init_cpu_features`.
|
||||
pub fn init_kernel_space(pml4_phys: PhysAddr, hhdm: u64) {
|
||||
*KERNEL_SPACE.lock() = Some(AddressSpace::from_active(pml4_phys, 0, hhdm));
|
||||
pub fn init_kernel_space(pml4_phys: PhysAddr) {
|
||||
*KERNEL_SPACE.lock() = Some(AddressSpace::from_active(pml4_phys, 0));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user