implement pagefault and tlb_flush for x86_64 to support mmap.

api/ruxos_posix_api/src/imp/mmap/mod.rs

@@ -8,7 +8,8 @@
  */
 
 cfg_if::cfg_if! {
-    if #[cfg(all(feature = "paging", target_arch = "aarch64"))] {
+    // for X86_64 with SMP, it must flush TLB via IPI
+    if #[cfg( all(feature = "paging", any(target_arch = "aarch64", any( all(target_arch = "x86_64", feature = "irq", feature = "smp"), all(target_arch = "x86_64", not(feature = "smp")) ) ) ))] {
         #[macro_use]
         mod utils;
         mod api;

api/ruxos_posix_api/src/imp/mmap/trap.rs

@@ -89,7 +89,7 @@ impl ruxhal::trap::TrapHandler for TrapHandlerImpl {
                 preload_page_with_swap(&mut memory_map, &mut swaped_map, &mut off_pool);
 
             // Fill target data to assigned physical addresses, from file or zero according to mapping type
-            let dst = fake_vaddr.as_mut_ptr();
+            let dst: *mut u8 = fake_vaddr.as_mut_ptr();
             #[cfg(feature = "fs")]
             {
                 if let Some(off) = swaped_map.remove(&vaddr) {

crates/allocator/Cargo.toml

@@ -22,6 +22,7 @@ buddy = ["dep:buddy_system_allocator"]
 allocator_api = []
 
 [dependencies]
+log = "0.4"
 buddy_system_allocator = { version = "0.9", default-features = false, optional = true }
 slab_allocator = { path = "../slab_allocator", optional = true }
 rlsf = { version = "0.2", optional = true }

crates/scheduler/Cargo.toml

@@ -11,3 +11,4 @@ documentation = "https://rcore-os.github.io/arceos/scheduler/index.html"
 
 [dependencies]
 linked_list = { path = "../linked_list" }
+log = "0.4"

modules/ruxhal/src/arch/aarch64/trap.rs

@@ -96,18 +96,16 @@ fn handle_sync_exception(tf: &mut TrapFrame) {
 
                 // this cause is coded like linux.
                 let cause: PageFaultCause = match esr.read_as_enum(ESR_EL1::EC) {
-                    Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => {
-                        if iss & 0x40 != 0 {
-                            PageFaultCause::WRITE // = store
-                        } else {
-                            PageFaultCause::READ //  = load
-                        }
+                    Some(ESR_EL1::EC::Value::DataAbortCurrentEL) if iss & 0x40 != 0 => {
+                        PageFaultCause::WRITE // = store
+                    }
+                    Some(ESR_EL1::EC::Value::DataAbortCurrentEL) if iss & 0x40 == 0 => {
+                        PageFaultCause::READ //  = load
                     }
                     _ => {
                         PageFaultCause::INSTRUCTION // = instruction fetch
                     }
                 };
-                debug!("mapped vaddr in Page Fault: {:X} {:?}", vaddr, cause);
                 if crate::trap::handle_page_fault(vaddr, cause) {
                     return;
                 }

modules/ruxhal/src/arch/x86_64/mod.rs

@@ -11,7 +11,6 @@ mod context;
 mod gdt;
 mod idt;
 
-#[cfg(target_os = "none")]
 mod trap;
 
 use core::arch::asm;
@@ -23,6 +22,20 @@ use x86_64::instructions::interrupts;
 #[cfg(feature = "musl")]
 use x86_64::registers::model_specific::EferFlags;
 
+#[cfg(feature = "irq")]
+extern crate alloc;
+
+#[cfg(all(feature = "irq", feature = "paging", feature = "smp"))]
+use {
+    crate::{
+        cpu::this_cpu_id,
+        platform::irq::{end_of_interrupt, send_ipi_excluding_self},
+    },
+    alloc::vec::Vec,
+    ruxconfig::SMP,
+    spinlock::SpinRaw,
+};
+
 pub use self::context::{ExtendedState, FxsaveArea, TaskContext, TrapFrame};
 pub use self::gdt::GdtStruct;
 pub use self::idt::IdtStruct;
@@ -88,17 +101,94 @@ pub unsafe fn write_page_table_root(root_paddr: PhysAddr) {
     }
 }
 
+/// data structure for communicating between IPI for the TLB flushing.
+///
+/// `Vaddr` means flushing the TLB entry that maps the given virtual address.
+/// `All` means flushing the entire TLB.
+#[cfg(all(feature = "irq", feature = "paging", feature = "smp"))]
+enum FlushTlbIpiData {
+    Vaddr(usize),
+    All,
+}
+
+// const variable for every CPU's flushing addresses.
+#[cfg(all(feature = "irq", feature = "paging", feature = "smp"))]
+#[allow(clippy::declare_interior_mutable_const)]
+const FLUSH_QUEUES: SpinRaw<Vec<FlushTlbIpiData>> = SpinRaw::new(Vec::new());
+/// static variable for communicating between IPI for the TLB flushing.
+#[cfg(all(feature = "irq", feature = "paging", feature = "smp"))]
+static FLUSHING_ADDRESSES: [SpinRaw<Vec<FlushTlbIpiData>>; SMP] = [FLUSH_QUEUES; SMP];
+/// const irq vector for TLB flushing IPI.
+#[cfg(all(feature = "irq", feature = "paging", feature = "smp"))]
+pub const INVALID_TLB_VECTOR: u8 = 0xff; // SPURIOUS APIC INTERRUPT
+
 /// Flushes the TLB.
 ///
 /// If `vaddr` is [`None`], flushes the entire TLB. Otherwise, flushes the TLB
 /// entry that maps the given virtual address.
 #[inline]
 pub fn flush_tlb(vaddr: Option<VirtAddr>) {
     if let Some(vaddr) = vaddr {
-        unsafe { tlb::flush(vaddr.into()) }
+        trace!("flush TLB entry: {:#x}", vaddr);
+        unsafe {
+            tlb::flush(vaddr.into());
+        }
+        #[cfg(all(feature = "irq", feature = "paging", feature = "smp"))]
+        {
+            for (i, flushing_vec) in FLUSHING_ADDRESSES.iter().enumerate().take(SMP) {
+                if i != this_cpu_id() {
+                    flushing_vec
+                        .lock()
+                        .push(FlushTlbIpiData::Vaddr(vaddr.into()));
+                }
+            }
+            unsafe {
+                send_ipi_excluding_self(INVALID_TLB_VECTOR);
+            }
+        }
     } else {
-        unsafe { tlb::flush_all() }
+        trace!("flush all TLB entry");
+        unsafe {
+            tlb::flush_all();
+        }
+        #[cfg(all(feature = "irq", feature = "paging", feature = "smp"))]
+        {
+            for (i, flushing_vec) in FLUSHING_ADDRESSES.iter().enumerate().take(SMP) {
+                if i != this_cpu_id() {
+                    let mut flushing_addresses = flushing_vec.lock();
+                    // clear all flushing addresses to avoid flushing again in IPI handler.
+                    flushing_addresses.clear();
+                    flushing_addresses.push(FlushTlbIpiData::All);
+                }
+            }
+            unsafe {
+                send_ipi_excluding_self(INVALID_TLB_VECTOR);
+            }
+        }
+    }
+}
+
+/// Flushes the TLB in IPI handler.
+///
+/// This function is called in IPI handler, and it flushes the TLB entry that maps the given virtual address.
+#[inline]
+#[cfg(all(feature = "irq", feature = "paging", feature = "smp"))]
+pub(crate) fn flush_tlb_ipi_handler() {
+    // error!("flush TLB entry in IPI handler");
+    let guard = kernel_guard::NoPreempt::new();
+    unsafe {
+        let mut flushing_addresses = FLUSHING_ADDRESSES[this_cpu_id()].lock();
+        while let Some(flush_data) = flushing_addresses.pop() {
+            if let FlushTlbIpiData::Vaddr(vaddr) = flush_data {
+                tlb::flush(vaddr);
+            } else {
+                tlb::flush_all();
+                flushing_addresses.clear();
+            }
+        }
+        end_of_interrupt();
     }
+    drop(guard);
 }
 
 /// Reads the thread pointer of the current CPU.

modules/ruxhal/src/arch/x86_64/trap.rs

@@ -10,6 +10,13 @@
 use x86::{controlregs::cr2, irq::*};
 
 use super::context::TrapFrame;
+#[cfg(all(feature = "paging", feature = "irq", feature = "smp"))]
+use crate::arch::{flush_tlb_ipi_handler, INVALID_TLB_VECTOR};
+#[cfg(any(
+    all(feature = "paging", feature = "irq", feature = "smp"),
+    all(feature = "paging", not(feature = "smp"))
+))]
+use crate::trap::PageFaultCause;
 
 core::arch::global_asm!(include_str!("trap.S"));
 
@@ -28,12 +35,25 @@ fn x86_trap_handler(tf: &TrapFrame) {
                     tf.error_code,
                 );
             } else {
+                let vaddr = unsafe { cr2() };
+                #[cfg(any(
+                    all(feature = "paging", feature = "irq", feature = "smp"),
+                    all(feature = "paging", not(feature = "smp"))
+                ))]
+                {
+                    // this cause is coded like linux.
+                    let cause: PageFaultCause = match tf.error_code {
+                        x if x & 0x10 != 0 => PageFaultCause::INSTRUCTION,
+                        x if x & 0x02 != 0 => PageFaultCause::WRITE,
+                        _ => PageFaultCause::READ,
+                    };
+                    if crate::trap::handle_page_fault(vaddr, cause) {
+                        return;
+                    }
+                }
                 panic!(
                     "Kernel #PF @ {:#x}, fault_vaddr={:#x}, error_code={:#x}:\n{:#x?}",
-                    tf.rip,
-                    unsafe { cr2() },
-                    tf.error_code,
-                    tf,
+                    tf.rip, vaddr, tf.error_code, tf,
                 );
             }
         }
@@ -44,6 +64,8 @@ fn x86_trap_handler(tf: &TrapFrame) {
                 tf.rip, tf.error_code, tf
             );
         }
+        #[cfg(all(feature = "paging", feature = "irq", feature = "smp"))]
+        INVALID_TLB_VECTOR => flush_tlb_ipi_handler(),
         IRQ_VECTOR_START..=IRQ_VECTOR_END => crate::trap::handle_irq_extern(tf.vector as _),
         _ => {
             panic!(

modules/ruxhal/src/platform/x86_pc/apic.rs

@@ -13,6 +13,8 @@ use lazy_init::LazyInit;
 use memory_addr::PhysAddr;
 use spinlock::SpinNoIrq;
 use x2apic::ioapic::IoApic;
+#[cfg(feature = "irq")]
+use x2apic::lapic::IpiAllShorthand;
 use x2apic::lapic::{xapic_base, LocalApic, LocalApicBuilder};
 use x86_64::instructions::port::Port;
 
@@ -71,8 +73,22 @@ pub fn dispatch_irq(vector: usize) {
     crate::irq::dispatch_irq_common(vector);
     unsafe { local_apic().end_of_interrupt() };
 }
+#[cfg(feature = "irq")]
+pub(crate) unsafe fn end_of_interrupt() {
+    local_apic().end_of_interrupt()
+}
+
+#[cfg(feature = "irq")]
+pub(crate) unsafe fn send_ipi_excluding_self(vector: u8) {
+    local_apic().send_ipi_all(vector, IpiAllShorthand::AllExcludingSelf);
+}
+
+#[cfg(feature = "irq")]
+pub(crate) unsafe fn send_ipi_including_self(vector: u8) {
+    local_apic().send_ipi_all(vector, IpiAllShorthand::AllIncludingSelf);
+}
 
-pub(super) fn local_apic<'a>() -> &'a mut LocalApic {
+pub fn local_apic<'a>() -> &'a mut LocalApic {
     // It's safe as LAPIC is per-cpu.
     unsafe { LOCAL_APIC.as_mut().unwrap() }
 }

scripts/make/build_c.mk

@@ -29,6 +29,7 @@ endif
 
 ifeq ($(ARCH), x86_64)
   LDFLAGS += --no-relax
+  CFLAGS += -mno-red-zone
 else ifeq ($(ARCH), riscv64)
   CFLAGS += -march=rv64gc -mabi=lp64d -mcmodel=medany
 endif

scripts/make/build_musl.mk

@@ -29,6 +29,7 @@ endif
 
 ifeq ($(ARCH), x86_64)
   LDFLAGS += --no-relax
+  CFLAGS += -mno-red-zone
 else ifeq ($(ARCH), riscv64)
   CFLAGS += -march=rv64gc -mabi=lp64d -mcmodel=medany
 endif

scripts/make/features.mk

@@ -18,7 +18,7 @@ ifeq ($(APP_TYPE),c)
   else
     lib_feat_prefix := ruxlibc/
   endif
-  lib_features := fp_simd alloc paging multitask fs net fd pipe select poll epoll random-hw signal
+  lib_features := fp_simd alloc irq sched_rr paging multitask fs net fd pipe select poll epoll random-hw signal
 else
   # TODO: it's better to use `ruxfeat/` as `ax_feat_prefix`, but all apps need to have `ruxfeat` as a dependency
   ax_feat_prefix := axstd/
@@ -27,7 +27,7 @@ else
 endif
 ifeq ($(APP_TYPE),c)
   ifeq ($(MUSL), y)
-    lib_features += irq musl sched_rr
+    lib_features += musl
   endif
 endif
 

ulib/ruxlibc/Cargo.toml

@@ -52,6 +52,11 @@ poll = ["ruxos_posix_api/poll"]
 epoll = ["ruxos_posix_api/epoll"]
 random-hw = ["ruxos_posix_api/random-hw"]
 
+# Interrupts
+irq = ["ruxos_posix_api/irq", "ruxfeat/irq"]
+
+sched_rr = ["irq", "ruxfeat/sched_rr"]
+
 [dependencies]
 ruxfeat = { path = "../../api/ruxfeat" }
 ruxos_posix_api = { path = "../../api/ruxos_posix_api" }