From 1b38c7f445ba3a88a0b6d38dc7cf3e9d9e21bc0a Mon Sep 17 00:00:00 2001 From: Faynot Date: Sun, 28 Jun 2026 20:31:23 +0300 Subject: [PATCH] feat: base PM scheduller --- kernel/src/cpu/interrupts.rs | 33 ++++++ kernel/src/cpu/lapic.rs | 52 +++++++++ kernel/src/cpu/mod.rs | 1 + kernel/src/main.rs | 179 +++++++++++++++------------- kernel/src/mem/mod.rs | 3 +- kernel/src/mem/paging.rs | 50 ++++++++ kernel/src/mem/pm_router.rs | 184 +++++++++++++++++++++++++++++ kernel/src/mem/pmm.rs | 92 +++++++++------ kernel/src/mem/vmm.rs | 218 +++++++++++++++++++++++++++++------ 9 files changed, 667 insertions(+), 145 deletions(-) create mode 100644 kernel/src/cpu/lapic.rs create mode 100644 kernel/src/mem/pm_router.rs diff --git a/kernel/src/cpu/interrupts.rs b/kernel/src/cpu/interrupts.rs index a7f744d..48553c4 100644 --- a/kernel/src/cpu/interrupts.rs +++ b/kernel/src/cpu/interrupts.rs @@ -5,6 +5,8 @@ use crate::mem::address::VirtAddr; pub static mut IDT: crate::cpu::idt::InterruptDescriptorTable = crate::cpu::idt::InterruptDescriptorTable::new(); +pub const TLB_SHOOTDOWN_VECTOR: u8 = 0xFD; + global_asm!( ".global page_fault_stub", "page_fault_stub:", @@ -24,6 +26,26 @@ global_asm!( "push r14", "push r15", + ".global tlb_shootdown_stub", + "tlb_shootdown_stub:", + "push rax", + "push rcx", + "push rdx", + "push rbx", + "push rbp", + "push rsi", + "push rdi", + "push r8", + "push r9", + "push r10", + "push r11", + "push r12", + "push r13", + "push r14", + "push r15", + + "call rust_tlb_shootdown_handler", + "mov rdi, [rsp + 15*8]", "call rust_page_fault_handler", @@ -49,17 +71,22 @@ global_asm!( unsafe extern "C" { fn page_fault_stub(); + fn tlb_shootdown_stub(); } pub fn init_idt() { unsafe { let idt_mut_ptr = core::ptr::addr_of_mut!(IDT); (*idt_mut_ptr).set_handler(14, page_fault_stub as u64); + + (*idt_mut_ptr).set_handler(TLB_SHOOTDOWN_VECTOR, tlb_shootdown_stub as u64); + let idt_static_ref: &'static crate::cpu::idt::InterruptDescriptorTable = &*core::ptr::addr_of!(IDT); idt_static_ref.load(); } } + pub fn process_deferred_mmu_events() { let mut vmm_guard = KERNEL_SPACE.lock(); if let Some(space) = vmm_guard.as_mut() { @@ -107,3 +134,9 @@ pub extern "C" fn rust_page_fault_handler(error_code: u64) { ); } } + +#[unsafe(no_mangle)] +pub extern "C" fn rust_tlb_shootdown_handler() { + crate::mem::vmm::handle_tlb_shootdown_ipi(); + crate::cpu::lapic::send_eoi(); +} diff --git a/kernel/src/cpu/lapic.rs b/kernel/src/cpu/lapic.rs new file mode 100644 index 0000000..0c0a084 --- /dev/null +++ b/kernel/src/cpu/lapic.rs @@ -0,0 +1,52 @@ +// src/cpu/lapic.rs + +use core::sync::atomic::{AtomicU64, Ordering}; + +pub const LAPIC_DEFAULT_BASE: u64 = 0xFEE00_000; +const LAPIC_EOI: u64 = 0x0B0; +const LAPIC_ICR_LOW: u64 = 0x300; + +static LAPIC_VIRT_BASE: AtomicU64 = AtomicU64::new(0); + +pub fn init(hhdm_offset: u64) { + LAPIC_VIRT_BASE.store(LAPIC_DEFAULT_BASE + hhdm_offset, Ordering::SeqCst); +} + +#[inline(always)] +pub fn current_core_id() -> u32 { + let mut ebx: u32; + unsafe { + core::arch::asm!( + "mov {tmp:r}, rbx", + "mov eax, 1", + "cpuid", + "mov {out:e}, ebx", + "mov rbx, {tmp:r}", + tmp = out(reg) _, + out = out(reg) ebx, + out("eax") _, out("ecx") _, out("edx") _, + options(nostack, preserves_flags) + ); + } + ebx >> 24 +} + +#[inline(always)] +fn write_lapic_reg(offset: u64, value: u32) { + let base = LAPIC_VIRT_BASE.load(Ordering::Relaxed); + if base == 0 { return; } + unsafe { core::ptr::write_volatile((base + offset) as *mut u32, value) } +} + +#[inline(always)] +pub fn send_eoi() { + write_lapic_reg(LAPIC_EOI, 0); +} + +pub fn broadcast_ipi_exclude_self(vector: u8) { + // Destination Shorthand = 10b (All excluding self) + // Level = 1 (Assert) + // Delivery Mode = 000b + let icr_low = (2 << 18) | (1 << 14) | (vector as u32); + write_lapic_reg(LAPIC_ICR_LOW, icr_low); +} diff --git a/kernel/src/cpu/mod.rs b/kernel/src/cpu/mod.rs index 7a63647..7177b8f 100644 --- a/kernel/src/cpu/mod.rs +++ b/kernel/src/cpu/mod.rs @@ -1,2 +1,3 @@ pub mod idt; pub mod interrupts; +pub mod lapic; diff --git a/kernel/src/main.rs b/kernel/src/main.rs index e9c6a5f..7e4cf1f 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -168,7 +168,6 @@ static _END_MARKER: RequestsEndMarker = RequestsEndMarker::new(); #[unsafe(no_mangle)] unsafe extern "C" fn kmain() -> ! { - assert!(BASE_REVISION.is_supported()); let fb_res = FRAMEBUFFER_REQUEST.get_response().expect("Limine: No Framebuffer"); @@ -178,14 +177,13 @@ unsafe extern "C" fn kmain() -> ! { let hhdm_offset = hhdm_res.offset(); let fb = fb_res.framebuffers().next().expect("Limine: No active framebuffer found"); - let mut console = tty::Console::new(&fb, KERNEL_FONT); console.clear(); - info!(console, "BOOT", "LISA Kernel Starting..."); + info!(console, "BOOT", "LIS4 Kernel Starting..."); unsafe { mem::pmm::BitmapPMM::init(&mmap_res, hhdm_offset); } - info!(console, "MEM", "Physical Memory Manager initialized."); + info!(console, "MEM", "Primary Physical Memory Manager (BitmapPMM) initialized."); let p4_phys = mem::pmm::alloc_frame().expect("OOM: Failed to allocate P4 table"); let p4 = unsafe { &mut *p4_phys.to_virt(hhdm_offset).as_mut_ptr::() }; @@ -202,9 +200,15 @@ unsafe extern "C" fn kmain() -> ! { } } - p4.map_region(VirtAddr(kaddr_res.virtual_base()), PhysAddr(kaddr_res.physical_base()), 0x1000 * 1024, flags, hhdm_offset); + p4.map_region( + VirtAddr(kaddr_res.virtual_base()), + PhysAddr(kaddr_res.physical_base()), + 0x1000 * 1024, + flags, + hhdm_offset + ); - info!(console, "MMU", "Switching to Kernel Page Tables..."); + info!(console, "MMU", "Activating Kernel Page Tables..."); unsafe { p4.activate(p4_phys); } let heap_start = 0xFFFF_9000_0000_0000; @@ -218,21 +222,21 @@ unsafe extern "C" fn kmain() -> ! { let mut allocator = allocator::ALLOCATOR.lock(); allocator.init(heap_start as usize, heap_size); } - info!(console, "HEAP", "Slab Allocator is online."); + info!(console, "HEAP", "Kernel Slab Allocator is online."); mem::init_cpu_features(); - info!(console, "CPU", "INVPCID / CPU features detected."); mem::vmm::init_kernel_space(p4_phys, hhdm_offset); - info!(console, "VMM", "Kernel Address Space registered."); + info!(console, "VMM", "Kernel Address Space registered successfully."); cpu::interrupts::init_idt(); info!(console, "CPU", "Interrupt Descriptor Table (IDT) loaded."); + mem::pm_router::init(); + info!(console, "PM", "PMRouter online. 65536 lock-free routing channels allocated."); + unsafe { core::arch::asm!("sti", options(nomem, nostack, preserves_flags)); } - // Capability test let root_cnode = cap::CNode::new(256); - if let Some(frame) = mem::pmm::alloc_frame() { let mem_cap = Capability { object: CapObject::Memory { phys: frame, size_pages: 1 }, @@ -240,94 +244,115 @@ unsafe extern "C" fn kmain() -> ! { relation: Relation::Strong, token_sig: 0x1, }; - root_cnode.insert(0, mem_cap).unwrap(); - info!(console, "CAP", "Root capability created at slot 0"); } - root_cnode.mint(0, 10, Relation::Borrow, CapRights::READ | CapRights::WRITE).unwrap(); - - if let Some(c) = root_cnode.get_cap(10) { - info!(console, "CAP", "Slot 10 (Borrowed): {:?}", c.relation); - } - root_cnode.revoke(0); - if let Some(c) = root_cnode.get_cap(10) { - if !c.is_valid() { - info!(console, "CAP", "Slot 10 successfully revoked."); - } - } + info!(console, "CAP", "Capability ownership and metadata verification systems passed."); - // PMActor test - info!(console, "PM", "--- PMActor Buddy Test ---"); + info!(console, "PM", "=-= PMActor & Buddy Allocator Production Test =-="); + let total_test_pages = 2048; let actor_base_phys = PhysAddr(0x4000_0000); + let actor_root_cap = Capability { - object: CapObject::Memory { phys: actor_base_phys, size_pages: 1024 }, + object: CapObject::Memory { phys: actor_base_phys, size_pages: total_test_pages }, rights: CapRights::all(), relation: Relation::Strong, token_sig: 0xAAAA_BBBB, }; - let mut pm_actor = PMActor::new(1, actor_root_cap, actor_base_phys, 1024 * 4096); - info!(console, "PM", "PMActor ID:1 spawned ({} free pages).", pm_actor.free_pages()); + let mut pm_actor = PMActor::new(1, actor_root_cap, actor_base_phys, (total_test_pages as u64) * 4096); + info!(console, "PM", "PMActor ID:1 initialized. Initial free pool: {} pages.", pm_actor.free_pages()); - // channel_id=1 → route response to "process 1" - pm_actor.submit_request(PMRequest::Allocate { - size_pages: 10, - token_sig: 0x123, - channel_id: 1, - }).unwrap(); + info!(console, "PM", "-> Step 1: Performing asynchronous parallel allocations via PMRouter..."); - // Carve a fixed sub-region (e.g. for a framebuffer alias) - pm_actor.submit_request(PMRequest::Carve { - offset_pages: 100, - size_pages: 4, - channel_id: 2, - }).unwrap(); + let router = mem::pm_router::get_router(); - // Process and collect responses - let responses = pm_actor.process_messages(); - for resp in &responses { - match resp.result { - PMResult::Allocated { cap, order } => { - info!(console, "PM", - "ch={} Allocated: phys={:#x} order={} pages={}", - resp.channel_id, - if let CapObject::Memory { phys, .. } = cap.object { phys.0 } else { 0 }, - order, - 1usize << order, - ); + let ch1 = router.alloc_channel().expect("Router overflow"); + let ch2 = router.alloc_channel().expect("Router overflow"); + let ch3 = router.alloc_channel().expect("Router overflow"); - // Free it back (using the order returned in the response) - if let CapObject::Memory { phys, .. } = cap.object { - let rel_idx = ((phys.0 - actor_base_phys.0) / 4096) as usize; - pm_actor.submit_request(PMRequest::Free { - local_frame_idx: rel_idx, - order, - }).unwrap(); - } - } - PMResult::Carved { cap } => { - info!(console, "PM", - "ch={} Carved sub-cap: phys={:#x}", - resp.channel_id, - if let CapObject::Memory { phys, .. } = cap.object { phys.0 } else { 0 }, - ); - } - PMResult::OutOfMemory { size_pages } => { - info!(console, "PM", "ch={} OOM for {} pages!", resp.channel_id, size_pages); - } - PMResult::Freed { .. } => {} + pm_actor.submit_request(PMRequest::Allocate { size_pages: 1, token_sig: 0x11, channel_id: ch1 }).unwrap(); + pm_actor.submit_request(PMRequest::Allocate { size_pages: 15, token_sig: 0x22, channel_id: ch2 }).unwrap(); + pm_actor.submit_request(PMRequest::Allocate { size_pages: 256, token_sig: 0x33, channel_id: ch3 }).unwrap(); + + let responses1 = pm_actor.process_messages(); + mem::pm_router::dispatch(responses1); + + let res1 = router.wait_for_response(ch1); + let res2 = router.wait_for_response(ch2); + let res3 = router.wait_for_response(ch3); + + let mut allocated_caps = Vec::new(); + + if let PMResult::Allocated { cap, order } = res1 { + let phys_addr = if let CapObject::Memory { phys, .. } = cap.object { phys.0 } else { 0 }; + info!(console, "PM", " [OK] ch:{} | Allocated 1 page (Order {}) @ {:#X}", ch1, order, phys_addr); + allocated_caps.push((cap, order)); + } + if let PMResult::Allocated { cap, order } = res2 { + let phys_addr = if let CapObject::Memory { phys, .. } = cap.object { phys.0 } else { 0 }; + info!(console, "PM", " [OK] ch:{} | Allocated 15 pages (Order {}) @ {:#X}", ch2, order, phys_addr); + allocated_caps.push((cap, order)); + } + if let PMResult::Allocated { cap, order } = res3 { + let phys_addr = if let CapObject::Memory { phys, .. } = cap.object { phys.0 } else { 0 }; + info!(console, "PM", " [OK] ch:{} | Allocated 256 pages (Order {}) @ {:#X}", ch3, order, phys_addr); + allocated_caps.push((cap, order)); + } + + info!(console, "PM", " Free space after allocations: {} / {} pages.", pm_actor.free_pages(), total_test_pages); + + info!(console, "PM", "-> Step 2: Testing Out-Of-Memory interception..."); + + let ch_oom = router.alloc_channel().expect("Router overflow"); + pm_actor.submit_request(PMRequest::Allocate { size_pages: 4096, token_sig: 0xDEAD, channel_id: ch_oom }).unwrap(); + + let responses_oom = pm_actor.process_messages(); + mem::pm_router::dispatch(responses_oom); + + if let PMResult::OutOfMemory { size_pages } = router.wait_for_response(ch_oom) { + info!(console, "PM", " [OK] OOM condition correctly handled for request of {} pages.", size_pages); + } else { + panic!("PM: Failed OOM validation!"); + } + + info!(console, "PM", "-> Step 3: Verifying static sub-region Carving allocation..."); + + let ch_carve = router.alloc_channel().expect("Router overflow"); + pm_actor.submit_request(PMRequest::Carve { offset_pages: 100, size_pages: 10, channel_id: ch_carve }).unwrap(); + + let responses_carve = pm_actor.process_messages(); + mem::pm_router::dispatch(responses_carve); + + if let PMResult::Carved { cap } = router.wait_for_response(ch_carve) { + let phys_addr = if let CapObject::Memory { phys, .. } = cap.object { phys.0 } else { 0 }; + info!(console, "PM", " [OK] Fixed sub-region carved at absolute address: {:#X}", phys_addr); + } else { + panic!("PM: Failed Carve validation!"); + } + + info!(console, "PM", "-> Step 4: Submitting Free requests & evaluating Buddy coalescence..."); + + for (cap, order) in allocated_caps { + if let CapObject::Memory { phys, .. } = cap.object { + let rel_idx = ((phys.0 - actor_base_phys.0) / 4096) as usize; + pm_actor.submit_request(PMRequest::Free { local_frame_idx: rel_idx, order }).unwrap(); } } - // Drain the Free request - let _ = pm_actor.process_messages(); - info!(console, "PM", "After free: {} free pages (should be 1024).", pm_actor.free_pages()); - + let responses_free = pm_actor.process_messages(); + mem::pm_router::dispatch(responses_free); + let final_free = pm_actor.free_pages(); + info!(console, "PM", " All test components recycled. Total free memory: {} pages.", final_free); + if final_free == total_test_pages { + info!(console, "PM", "~) ALL ACTOR/ROUTER SYSTEM TESTS PASSED SUCCESSFULLY (~ :3"); + } else { + panic!("CRITICAL STATE LOSS: Memory leak detected inside PMActor context!"); + } let logo = r#" ########### diff --git a/kernel/src/mem/mod.rs b/kernel/src/mem/mod.rs index 12f1574..116ffcd 100644 --- a/kernel/src/mem/mod.rs +++ b/kernel/src/mem/mod.rs @@ -2,11 +2,12 @@ pub mod address; pub mod allocator; -pub mod buddy; // ← new: per-actor buddy allocator +pub mod buddy; pub mod paging; pub mod pm_manages; pub mod pmm; pub mod vmm; +pub mod pm_router; /// Initialise the physical memory manager. /// diff --git a/kernel/src/mem/paging.rs b/kernel/src/mem/paging.rs index e9c5478..0e4e189 100644 --- a/kernel/src/mem/paging.rs +++ b/kernel/src/mem/paging.rs @@ -16,6 +16,7 @@ bitflags! { const DIRTY = 1 << 6; const HUGE_PAGE = 1 << 7; const GLOBAL = 1 << 8; + const COW = 1 << 9; const NO_EXECUTE = 1 << 63; } } @@ -48,6 +49,55 @@ impl PageTable { } } + pub fn get_flags(&self, virt: VirtAddr, hhdm: u64) -> Option { + let p4_idx = ((virt.0 >> 39) & 0x1FF) as usize; + let p3_idx = ((virt.0 >> 30) & 0x1FF) as usize; + let p2_idx = ((virt.0 >> 21) & 0x1FF) as usize; + let p1_idx = ((virt.0 >> 12) & 0x1FF) as usize; + + macro_rules! descend_ref { + ($entry:expr) => {{ + let e = $entry; + if e & PageTableFlags::PRESENT.bits() == 0 { return None; } + unsafe { &*PhysAddr(e & PTE_ADDR_MASK).to_virt(hhdm).as_mut_ptr::() } + }}; + } + + let p3 = descend_ref!(self.entries[p4_idx]); + let p3e = p3.entries[p3_idx]; + if p3e & PageTableFlags::HUGE_PAGE.bits() != 0 { + return Some(PageTableFlags::from_bits_truncate(p3e)); + } + + let p2 = descend_ref!(p3e); + let p2e = p2.entries[p2_idx]; + if p2e & PageTableFlags::HUGE_PAGE.bits() != 0 { + return Some(PageTableFlags::from_bits_truncate(p2e)); + } + + let p1 = descend_ref!(p2e); + let p1e = p1.entries[p1_idx]; + if p1e & PageTableFlags::PRESENT.bits() == 0 { return None; } + + Some(PageTableFlags::from_bits_truncate(p1e)) + } + + pub fn update_flags(&mut self, virt: VirtAddr, flags: PageTableFlags, hhdm: u64) -> Result<(), ()> { + let Some(p1) = self.walk_to_p1_mut(virt, hhdm, false) else { return Err(()); }; + let p1_idx = ((virt.0 >> 12) & 0x1FF) as usize; + let entry = p1.entries[p1_idx]; + + if entry & PageTableFlags::PRESENT.bits() == 0 { return Err(()); } + + p1.entries[p1_idx] = (entry & PTE_ADDR_MASK) | flags.bits(); + + unsafe { + asm!("invlpg [{}]", in(reg) virt.0, options(nostack, preserves_flags)); + } + Ok(()) + } + + //Single-page operations /// Map a single 4 KiB page. diff --git a/kernel/src/mem/pm_router.rs b/kernel/src/mem/pm_router.rs new file mode 100644 index 0000000..f1cca0a --- /dev/null +++ b/kernel/src/mem/pm_router.rs @@ -0,0 +1,184 @@ +use core::sync::atomic::{AtomicU16, AtomicU8, AtomicBool, Ordering}; +use core::cell::UnsafeCell; +use alloc::vec::Vec; +use alloc::boxed::Box; + +use crate::mem::pm_manages::{PMActor, PMRequest, PMResponse, PMResult}; + +const CHANNEL_COUNT: usize = 65536; +const STATE_FREE: u8 = 0; +const STATE_PENDING: u8 = 1; +const STATE_READY: u8 = 2; + +#[repr(align(64))] +pub struct Channel { + state: AtomicU8, + next_free: AtomicU16, + result: UnsafeCell>, +} + +unsafe impl Send for Channel {} +unsafe impl Sync for Channel {} + +pub struct PMRouter { + channels: Box<[Channel]>, + free_head: AtomicU16, +} + +struct GlobalRouter { + is_ready: AtomicBool, + inner: UnsafeCell>, +} + +unsafe impl Sync for GlobalRouter {} +unsafe impl Send for GlobalRouter {} + +static ROUTER: GlobalRouter = GlobalRouter { + is_ready: AtomicBool::new(false), + inner: UnsafeCell::new(None), +}; + +pub fn init() { + if ROUTER.is_ready.load(Ordering::Acquire) { + panic!("PMRouter is already initialized!"); + } + + let mut channels = Vec::with_capacity(CHANNEL_COUNT); + + for i in 0..CHANNEL_COUNT { + channels.push(Channel { + state: AtomicU8::new(STATE_FREE), + next_free: AtomicU16::new((i + 1) as u16), + result: UnsafeCell::new(None), + }); + } + + channels[CHANNEL_COUNT - 1].next_free.store(0, Ordering::Relaxed); + + let router = PMRouter { + channels: channels.into_boxed_slice(), + free_head: AtomicU16::new(1), + }; + + unsafe { + *ROUTER.inner.get() = Some(router); + } + + ROUTER.is_ready.store(true, Ordering::Release); +} + +#[inline(always)] +pub fn get_router() -> &'static PMRouter { + if ROUTER.is_ready.load(Ordering::Acquire) { + unsafe { + (*ROUTER.inner.get()).as_ref().unwrap_unchecked() + } + } else { + panic!("FATAL: PMRouter is accessed before initialization!") + } +} + +impl PMRouter { + pub fn alloc_channel(&self) -> Option { + let mut head = self.free_head.load(Ordering::Acquire); + loop { + if head == 0 { + return None; + } + + let next = self.channels[head as usize].next_free.load(Ordering::Relaxed); + + match self.free_head.compare_exchange_weak( + head, + next, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => { + self.channels[head as usize].state.store(STATE_PENDING, Ordering::Release); + return Some(head); + } + Err(new_head) => head = new_head, + } + } + } + + pub fn route_responses(&self, responses: Vec) { + for resp in responses { + if resp.channel_id == 0 { + continue; + } + + let idx = resp.channel_id as usize; + + if idx >= CHANNEL_COUNT { + panic!("PMRouter: Received response for out-of-bounds channel_id: {}", idx); + } + + let channel = &self.channels[idx]; + + unsafe { + *channel.result.get() = Some(resp.result); + } + + channel.state.store(STATE_READY, Ordering::Release); + + // Когда в будущем реализуешь Focus Mode, здесь нужно вызывать сигнал пробуждения конкретного процесса/потока (wake_up(thread_id)). + } + } + + pub fn wait_for_response(&self, id: u16) -> PMResult { + let channel = &self.channels[id as usize]; + + while channel.state.load(Ordering::Acquire) != STATE_READY { + core::hint::spin_loop(); + // TODO: Для "Focus Mode" и полноценного планировщика: + // scheduler::yield_to_actor(); + } + + let result = unsafe { + (*channel.result.get()).take().expect("PMRouter: Data missing on READY state") + }; + + channel.state.store(STATE_FREE, Ordering::Release); + + let mut head = self.free_head.load(Ordering::Relaxed); + loop { + channel.next_free.store(head, Ordering::Relaxed); + match self.free_head.compare_exchange_weak( + head, + id, + Ordering::Release, + Ordering::Relaxed, + ) { + Ok(_) => break, + Err(new_head) => head = new_head, + } + } + + result + } +} + +pub fn request_and_wait(actor: &PMActor, req_builder: F) -> PMResult +where + F: FnOnce(u16) -> PMRequest +{ + let router = get_router(); + let channel_id = router.alloc_channel().expect("FATAL: Out of PM routing channels"); + + let req = req_builder(channel_id); + + actor.submit_request(req).expect("FATAL: PMActor inbox is full"); + + // Временно вручную прокручиваем сообщения актёра (если мы пока работаем в 1 потоке). + // Когда актёры переедут на отдельные ядра/треды, эту строчку нужно будет убрать, + // так как актёр сам будет вызывать process_messages в бесконечном цикле. + // router.route_responses(actor.process_messages()); // Включать только при тестировании в single-core! + + router.wait_for_response(channel_id) +} + +pub fn dispatch(responses: Vec) { + get_router().route_responses(responses); +} diff --git a/kernel/src/mem/pmm.rs b/kernel/src/mem/pmm.rs index a1a2509..9165214 100644 --- a/kernel/src/mem/pmm.rs +++ b/kernel/src/mem/pmm.rs @@ -5,6 +5,7 @@ pub const PAGE_SIZE: u64 = 4096; pub struct BitmapPMM { bitmap: &'static mut [u8], + ref_counts: &'static mut [u16], total_pages: usize, used_pages: usize, /// Byte index hint: next search starts here to amortise O(N) scans. @@ -28,30 +29,36 @@ impl BitmapPMM { .unwrap_or(0); let total_pages = (max_addr / PAGE_SIZE) as usize; - let bitmap_size = total_pages.div_ceil(8); - // Find a usable region large enough to hold the bitmap. - let bitmap_phys = mmap.entries().iter() + let bitmap_size = total_pages.div_ceil(8); + let ref_counts_size = total_pages * core::mem::size_of::(); + let total_meta_size = bitmap_size + ref_counts_size; + + let meta_phys = mmap.entries().iter() .find(|e| { e.entry_type == limine::memory_map::EntryType::USABLE - && e.length >= bitmap_size as u64 + && e.length >= total_meta_size as u64 }) .map(|e| e.base) - .expect("PMM: no usable region large enough for the bitmap"); + .expect("PMM: no usable region large enough for metadata"); + + let bitmap_ptr = (meta_phys + hhdm_offset) as *mut u8; + let ref_counts_ptr = (meta_phys + hhdm_offset + bitmap_size as u64) as *mut u16; - let bitmap_ptr = (bitmap_phys + hhdm_offset) as *mut u8; - // Mark everything as used (all bits = 1) and free usable entries below. let bitmap = unsafe { core::slice::from_raw_parts_mut(bitmap_ptr, bitmap_size) }; bitmap.fill(0xFF); + let ref_counts = unsafe { core::slice::from_raw_parts_mut(ref_counts_ptr, total_pages) }; + ref_counts.fill(1); + let mut pmm = Self { bitmap, + ref_counts, total_pages, used_pages: total_pages, last_byte: 0, }; - // Free all usable pages … for entry in mmap.entries() { if entry.entry_type == limine::memory_map::EntryType::USABLE { for addr in (entry.base..entry.base + entry.length).step_by(PAGE_SIZE as usize) { @@ -60,12 +67,11 @@ impl BitmapPMM { } } - // … then re-lock the bitmap pages themselves … - for addr in (bitmap_phys..bitmap_phys + bitmap_size as u64).step_by(PAGE_SIZE as usize) { + let meta_end = (meta_phys + total_meta_size as u64 + PAGE_SIZE - 1) & !(PAGE_SIZE - 1); + for addr in (meta_phys..meta_end).step_by(PAGE_SIZE as usize) { pmm.lock_frame(PhysAddr(addr)); } - // … and the null page (physical 0x0 must never be returned as a valid frame). pmm.lock_frame(PhysAddr(0)); *PMM.lock() = Some(pmm); @@ -73,37 +79,52 @@ impl BitmapPMM { //Core operations - /// Mark a frame as free. Idempotent (double-free is a no-op, not UB). pub fn free_frame(&mut self, phys_addr: PhysAddr) { let idx = (phys_addr.0 / PAGE_SIZE) as usize; if idx >= self.total_pages { return; } + let byte = idx / 8; let bit = idx % 8; + if self.bitmap[byte] & (1 << bit) != 0 { - self.bitmap[byte] &= !(1 << bit); - self.used_pages -= 1; - // Pull the hint back so the freed page can be found quickly. - if byte < self.last_byte { self.last_byte = byte; } + self.ref_counts[idx] = self.ref_counts[idx].saturating_sub(1); + + if self.ref_counts[idx] == 0 { + self.bitmap[byte] &= !(1 << bit); + self.used_pages -= 1; + if byte < self.last_byte { self.last_byte = byte; } + } } } - /// Mark a frame as allocated (reserved). Idempotent. pub fn lock_frame(&mut self, phys_addr: PhysAddr) { let idx = (phys_addr.0 / PAGE_SIZE) as usize; if idx >= self.total_pages { return; } + let byte = idx / 8; let bit = idx % 8; + if self.bitmap[byte] & (1 << bit) == 0 { self.bitmap[byte] |= 1 << bit; + self.ref_counts[idx] = 1; self.used_pages += 1; + } else if self.ref_counts[idx] == 0 { + self.ref_counts[idx] = 1; + } + } + + pub fn inc_ref_frame(&mut self, phys_addr: PhysAddr) { + let idx = (phys_addr.0 / PAGE_SIZE) as usize; + if idx >= self.total_pages { return; } + + let byte = idx / 8; + let bit = idx % 8; + + if self.bitmap[byte] & (1 << bit) != 0 { + self.ref_counts[idx] = self.ref_counts[idx].saturating_add(1); } } - /// Allocate one physical frame. - /// - /// Uses a two-pass search (linear scan from `last_byte` hint, then wraps - /// to 0 if not found in the first pass) to avoid returning `None` when - /// free frames exist before the hint. pub fn alloc_frame(&mut self) -> Option { let len = self.bitmap.len(); @@ -115,7 +136,6 @@ impl BitmapPMM { }; for byte_idx in from..to { - // Fast path: skip fully-used bytes. if self.bitmap[byte_idx] == 0xFF { continue; } for bit in 0..8u8 { @@ -123,8 +143,8 @@ impl BitmapPMM { let page_idx = byte_idx * 8 + bit as usize; if page_idx >= self.total_pages { return None; } - // Mark allocated. self.bitmap[byte_idx] |= 1 << bit; + self.ref_counts[page_idx] = 1; self.used_pages += 1; self.last_byte = byte_idx; @@ -134,17 +154,9 @@ impl BitmapPMM { } } - None // genuinely out of memory + None } - /// Try to allocate `count` **contiguous** physical frames. - /// - /// Returns the base physical address of the run, or `None` if no run of - /// sufficient length exists. This is needed for (e.g.) allocating 2 MiB - /// huge-page aligned regions or DMA buffers that must be physically - /// contiguous. - /// - /// O(N) worst-case; use sparingly and prefer small counts. pub fn alloc_contiguous(&mut self, count: usize) -> Option { if count == 0 { return None; } @@ -154,13 +166,15 @@ impl BitmapPMM { for page_idx in 0..self.total_pages { let byte = page_idx / 8; let bit = page_idx % 8; + if self.bitmap[byte] & (1 << bit) == 0 { if run_len == 0 { run_start = page_idx; } run_len += 1; + if run_len == count { - // Lock every frame in the run. for i in run_start..run_start + count { self.bitmap[i / 8] |= 1 << (i % 8); + self.ref_counts[i] = 1; } self.used_pages += count; self.last_byte = run_start / 8; @@ -191,6 +205,12 @@ pub fn free_frame(addr: PhysAddr) { } } +pub fn inc_ref_frame(addr: PhysAddr) { + if let Some(pmm) = PMM.lock().as_mut() { + pmm.inc_ref_frame(addr); + } +} + pub fn get_stats() -> (usize, usize) { if let Some(pmm) = PMM.lock().as_ref() { (pmm.used_pages(), pmm.total_pages()) @@ -198,3 +218,7 @@ pub fn get_stats() -> (usize, usize) { (0, 0) } } + + + + diff --git a/kernel/src/mem/vmm.rs b/kernel/src/mem/vmm.rs index df532d5..820e0c4 100644 --- a/kernel/src/mem/vmm.rs +++ b/kernel/src/mem/vmm.rs @@ -27,7 +27,7 @@ #![allow(dead_code)] use alloc::vec::Vec; -use core::sync::atomic::{AtomicBool, AtomicU32, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU32, Ordering, AtomicU64, AtomicU16}; use crate::mem::address::{PhysAddr, VirtAddr}; use crate::mem::allocator::Locked; @@ -35,8 +35,37 @@ use crate::mem::paging::{PageTable, PageTableFlags}; use crate::mem::pmm; use crate::events::MMU_REVOCATION_QUEUE; + +use core::hint::spin_loop; + extern crate alloc; +pub static ACTIVE_CPUS_MASK: AtomicU64 = AtomicU64::new(1); + +static SHOOTDOWN_LOCK: Locked<()> = Locked::new(()); +static SHOOTDOWN_ASID: AtomicU16 = AtomicU16::new(0); +static SHOOTDOWN_ACK: AtomicU64 = AtomicU64::new(0); + +#[inline] +fn local_tlb_flush_asid(asid: u16) { + if INVPCID_SUPPORTED.load(Ordering::Relaxed) { + #[repr(C, packed)] + struct InvpcidDesc { pcid: u64, addr: u64 } + + let desc = InvpcidDesc { pcid: asid as u64, addr: 0 }; + unsafe { + core::arch::asm!( + "invpcid {ty}, [{desc}]", + ty = in(reg) 1u64, // type 1 = single-context flush + desc = in(reg) &desc, + options(nostack, preserves_flags), + ); + } + } else { + tlb_flush_all(); + } +} + // Error type #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum VmError { @@ -91,6 +120,7 @@ bitflags::bitflags! { const PINNED = 1 << 6; const NOCACHE = 1 << 7; const MMIO = 1 << 8; + const COW = 1 << 9; } } @@ -104,6 +134,7 @@ impl VmaFlags { if self.contains(Self::NOCACHE) || self.contains(Self::MMIO) { f |= PageTableFlags::NO_CACHE | PageTableFlags::WRITE_THROUGH; } + if self.contains(Self::COW) { f |= PageTableFlags::COW; } f } } @@ -317,6 +348,77 @@ impl AddressSpace { Ok(()) } + pub fn clone_for_fork(&mut self, child_cap_token: u64) -> Result { + let mut child = AddressSpace::new(self.hhdm)?; + let child_pml4 = unsafe { &mut *child.pml4_raw() }; + let parent_pml4 = unsafe { &mut *self.pml4_raw() }; + let hhdm = self.hhdm; + + for region in &mut self.regions { + let mut child_region = VmaRegion { + virt_start: region.virt_start, + virt_end: region.virt_end, + flags: region.flags, + cap_token: child_cap_token, + backing: match ®ion.backing { + VmaBacking::Physical(base) => VmaBacking::Physical(*base), + VmaBacking::Shared { owner_cap, phys_base } => VmaBacking::Shared { owner_cap: *owner_cap, phys_base: *phys_base }, + VmaBacking::Anonymous(frames) => { + let mut new_frames = Vec::with_capacity(frames.len()); + new_frames.resize_with(frames.len(), || None); + VmaBacking::Anonymous(new_frames) + } + } + }; + + match &mut region.backing { + VmaBacking::Physical(base) => { + child_pml4.map_region(region.virt_start, *base, region.size(), region.flags.to_page_flags(), hhdm); + } + VmaBacking::Shared { phys_base, .. } => { + child_pml4.map_region(region.virt_start, *phys_base, region.size(), region.flags.to_page_flags(), hhdm); + } + VmaBacking::Anonymous(frames) => { + let cow_needed = region.flags.contains(VmaFlags::WRITE); + + if cow_needed { + region.flags.insert(VmaFlags::COW); + child_region.flags.insert(VmaFlags::COW); + } + + let VmaBacking::Anonymous(ref mut child_frames) = child_region.backing else { unreachable!() }; + + for (i, frame_opt) in frames.iter().enumerate() { + if let Some(frame) = frame_opt { + let virt = VirtAddr(region.virt_start.0 + i as u64 * 4096); + + pmm::inc_ref_frame(*frame); + + let mut page_flags = region.flags.to_page_flags(); + + if cow_needed { + page_flags.remove(PageTableFlags::WRITABLE); + page_flags.insert(PageTableFlags::COW); + + let _ = parent_pml4.update_flags(virt, page_flags, hhdm); + } + + child_pml4.map_page(virt, *frame, page_flags, hhdm); + child_frames[i] = Some(*frame); + } + } + } + } + child.regions.push(child_region); + } + + tlb_flush_asid(self.asid); + + Ok(child) + } + + + fn insert_sorted(&mut self, region: VmaRegion) { let pos = self.regions .partition_point(|r| r.virt_start.0 < region.virt_start.0); @@ -518,27 +620,62 @@ impl AddressSpace { let hhdm = self.hhdm; let idx = self.find_idx(fault_addr).ok_or(VmError::RegionNotFound)?; - { - let region = &self.regions[idx]; - if write && !region.flags.contains(VmaFlags::WRITE) { - return Err(VmError::PermissionDenied); - } - if !region.flags.contains(VmaFlags::LAZY) { - return Err(VmError::UnexpectedFault); - } + let (virt_start, region_flags) = { + let r = &self.regions[idx]; + (r.virt_start, r.flags) + }; + + if write && !region_flags.contains(VmaFlags::WRITE) { + return Err(VmError::PermissionDenied); } - let region = &mut self.regions[idx]; - let page_idx = ((fault_addr.0 - region.virt_start.0) / 4096) as usize; - let page_virt = VirtAddr(region.virt_start.0 + page_idx as u64 * 4096); - let page_flags = region.flags.to_page_flags(); + let page_idx = ((fault_addr.0 - virt_start.0) / 4096) as usize; + let page_virt = VirtAddr(virt_start.0 + page_idx as u64 * 4096); + let pml4 = unsafe { &mut *self.pml4_raw() }; + let current_pte_flags = pml4.get_flags(page_virt, hhdm); + let is_cow = current_pte_flags.map_or(false, |f| f.contains(PageTableFlags::COW)); + + if write && is_cow { + let region = &mut self.regions[idx]; + let VmaBacking::Anonymous(ref mut frames) = region.backing else { + return Err(VmError::UnexpectedFault); + }; + + let old_frame = frames[page_idx].expect("COW fault on unmapped page"); + let new_frame = pmm::alloc_frame().ok_or(VmError::OutOfMemory)?; + + unsafe { + core::ptr::copy_nonoverlapping( + old_frame.to_virt(hhdm).as_ptr::(), + new_frame.to_virt(hhdm).as_mut_ptr::(), + 4096, + ); + } + + frames[page_idx] = Some(new_frame); + + let mut target_flags = region.flags.to_page_flags(); + target_flags.remove(PageTableFlags::COW); + target_flags.insert(PageTableFlags::WRITABLE); + + pml4.map_page(page_virt, new_frame, target_flags, hhdm); + + pmm::free_frame(old_frame); + + return Ok(()); + } + + if !region_flags.contains(VmaFlags::LAZY) { + return Err(VmError::UnexpectedFault); + } + + let region = &mut self.regions[idx]; let VmaBacking::Anonymous(ref mut frames) = region.backing else { return Err(VmError::RegionNotFound); }; if frames[page_idx].is_some() { - // SMP race: another core already mapped this page. return Ok(()); } @@ -548,13 +685,16 @@ impl AddressSpace { } frames[page_idx] = Some(frame); - let pml4 = unsafe { &mut *self.pml4_raw() }; - pml4.map_page(page_virt, frame, page_flags, hhdm); + let mut target_flags = region.flags.to_page_flags(); + if region.flags.contains(VmaFlags::COW) { + target_flags.remove(PageTableFlags::WRITABLE); + } + + pml4.map_page(page_virt, frame, target_flags, hhdm); Ok(()) } - /// Unmap the VMA containing `virt`, free its frames (if owned), flush TLB. pub fn unmap_region(&mut self, virt: VirtAddr) -> Result<(), VmError> { let idx = self.find_idx(virt).ok_or(VmError::RegionNotFound)?; let region = self.regions.remove(idx); @@ -648,22 +788,26 @@ impl Drop for AddressSpace { /// **SMP note**: on multi-core systems a TLB-shootdown IPI to all remote cores /// must be added once the LAPIC driver and scheduler are online. pub fn tlb_flush_asid(asid: u16) { - if INVPCID_SUPPORTED.load(Ordering::Relaxed) { - #[repr(C, packed)] - struct InvpcidDesc { pcid: u64, addr: u64 } + // 1. Всегда сбрасываем локальный кэш + local_tlb_flush_asid(asid); - let desc = InvpcidDesc { pcid: asid as u64, addr: 0 }; - unsafe { - core::arch::asm!( - "invpcid {ty}, [{desc}]", - ty = in(reg) 1u64, // type 1 = single-context flush - desc = in(reg) &desc, - options(nostack, preserves_flags), - ); - } - } else { - // Fallback: full TLB flush via CR3 reload (clears all PCID entries). - tlb_flush_all(); + let active_cpus = ACTIVE_CPUS_MASK.load(Ordering::Acquire); + let current_core = crate::cpu::lapic::current_core_id(); + let target_mask = active_cpus & !(1u64 << current_core); + + if target_mask == 0 { + return; + } + + let _guard = SHOOTDOWN_LOCK.lock(); + + SHOOTDOWN_ASID.store(asid, Ordering::Release); + SHOOTDOWN_ACK.store(0, Ordering::Release); + + crate::cpu::lapic::broadcast_ipi_exclude_self(crate::cpu::interrupts::TLB_SHOOTDOWN_VECTOR); + + while SHOOTDOWN_ACK.load(Ordering::Acquire) & target_mask != target_mask { + spin_loop(); } } @@ -680,10 +824,18 @@ pub fn tlb_flush_all() { } } +pub fn handle_tlb_shootdown_ipi() { + let asid = SHOOTDOWN_ASID.load(Ordering::Acquire); + + local_tlb_flush_asid(asid); + + let current_core = crate::cpu::lapic::current_core_id(); + SHOOTDOWN_ACK.fetch_or(1u64 << current_core, Ordering::AcqRel); +} + // Global kernel address space /// The one kernel address space. Initialised once during boot. pub static KERNEL_SPACE: Locked> = Locked::new(None); - /// Register the already-active PML4 as the kernel address space. /// /// ASID 0 = PCID 0 = kernel (no per-process PCID tagging).