x86_64: kernel task context switches worked

.github/workflows/build.yml

@@ -22,7 +22,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        arch: [aarch64]
+        arch: [x86_64, aarch64]
     steps:
     - uses: actions/checkout@v2
     - uses: actions-rs/toolchain@v1
@@ -35,11 +35,13 @@ jobs:
 
     - name: Install musl toolchain
       run: |
-        if [ "${{ matrix.arch }}" = "aarch64" ]; then
+        if [ "${{ matrix.arch }}" = "x86_64" ]; then
+          export MUSL_PATH="x86_64-linux-musl-cross"
+        elif [ "${{ matrix.arch }}" = "aarch64" ]; then
           export MUSL_PATH="aarch64-linux-musl-cross"
         fi
         wget https://musl.cc/$MUSL_PATH.tgz
-        tar -xf aarch64-linux-musl-cross.tgz
+        tar -xf $MUSL_PATH.tgz
         mv $MUSL_PATH musl
 
     - name: Build kernel

kernel/Cargo.toml

@@ -9,7 +9,7 @@ edition = "2021"
 [features]
 platform-pc = []
 platform-qemu-virt-arm = []
-default = []
+default = ["platform-pc"]
 
 [dependencies]
 log = "0.4"

kernel/Makefile

@@ -22,7 +22,7 @@ kernel_bin := $(kernel_elf).bin
 # Cargo features and build args
 features := platform-$(PLATFORM)
 
-build_args := --features "$(features)" --target $(target) -Zbuild-std=core,alloc -Zbuild-std-features=compiler-builtins-mem
+build_args := --no-default-features --features "$(features)" --target $(target) -Zbuild-std=core,alloc -Zbuild-std-features=compiler-builtins-mem
 ifeq ($(MODE), release)
   build_args += --release
 endif
@@ -34,7 +34,7 @@ GDB := gdb-multiarch
 
 # QEMU
 qemu := qemu-system-$(ARCH)
-qemu_args := -nographic
+qemu_args := -nographic -m 128M
 ifeq ($(ARCH), x86_64)
   qemu_args += \
     -machine q35 \
@@ -55,7 +55,6 @@ GDB := gdb-multiarch
 build: $(kernel_bin)
 
 env:
-	(rustup target list | grep "$(target) (installed)") || rustup target add $(target)
 	cargo install cargo-binutils --vers =0.3.3
 	rustup component add rust-src
 	rustup component add llvm-tools-preview

kernel/src/arch/aarch64/context.rs

@@ -2,7 +2,7 @@ use core::arch::asm;
 
 use cortex_a::registers::SPSR_EL1;
 
-use crate::mm::PhysAddr;
+use crate::mm::{PhysAddr, VirtAddr};
 
 #[repr(C)]
 #[derive(Debug, Default, Clone, Copy)]
@@ -18,12 +18,12 @@ pub struct TrapFrame {
 }
 
 impl TrapFrame {
-    pub fn new_user(entry: usize, ustack_top: usize, arg0: usize) -> Self {
+    pub fn new_user(entry: VirtAddr, ustack_top: VirtAddr, arg0: usize) -> Self {
         let mut regs = [0; 31];
         regs[0] = arg0 as _;
         Self {
-            usp: ustack_top as _,
-            elr: entry as _,
+            usp: ustack_top.as_usize() as _,
+            elr: entry.as_usize() as _,
             spsr: (SPSR_EL1::M::EL0t
                 + SPSR_EL1::D::Masked
                 + SPSR_EL1::A::Masked
@@ -34,9 +34,9 @@ impl TrapFrame {
         }
     }
 
-    pub const fn new_clone(&self, ustack_top: usize) -> Self {
+    pub const fn new_clone(&self, ustack_top: VirtAddr) -> Self {
         let mut tf = *self;
-        tf.usp = ustack_top as _;
+        tf.usp = ustack_top.as_usize() as _;
         tf.r[0] = 0; // for child thread, clone returns 0
         tf
     }
@@ -47,7 +47,7 @@ impl TrapFrame {
         tf
     }
 
-    pub unsafe fn exec(&self, kstack_top: usize) -> ! {
+    pub unsafe fn exec(&self, kstack_top: VirtAddr) -> ! {
         asm!("
             mov     sp, x1
             ldp     x30, x9, [x0, 30 * 8]
@@ -74,7 +74,7 @@ impl TrapFrame {
 
             eret",
             in("x0") self,
-            in("x1") kstack_top,
+            in("x1") kstack_top.as_usize(),
             options(noreturn),
         )
     }
@@ -105,15 +105,25 @@ impl TaskContext {
         unsafe { core::mem::MaybeUninit::zeroed().assume_init() }
     }
 
-    pub fn init(&mut self, entry: usize, kstack_top: usize, page_table_root: PhysAddr) {
-        self.sp = kstack_top as u64;
+    pub fn init(
+        &mut self,
+        entry: usize,
+        kstack_top: VirtAddr,
+        page_table_root: PhysAddr,
+        is_kernel: bool,
+    ) {
+        self.sp = kstack_top.as_usize() as u64;
         self.lr = entry as u64;
-        self.ttbr0_el1 = page_table_root.as_usize() as u64;
+        self.ttbr0_el1 = if is_kernel {
+            0
+        } else {
+            page_table_root.as_usize() as u64
+        };
     }
 
     pub fn switch_to(&mut self, next_ctx: &Self) {
         unsafe {
-            crate::arch::instructions::activate_paging(next_ctx.ttbr0_el1 as usize, false);
+            crate::arch::instructions::set_user_page_table_root(next_ctx.ttbr0_el1 as usize);
             context_switch(self, next_ctx)
         }
     }

kernel/src/arch/aarch64/instructions.rs

@@ -23,17 +23,24 @@ pub unsafe fn set_thread_pointer(tp: usize) {
     TPIDR_EL1.set(tp as _)
 }
 
-pub unsafe fn activate_paging(page_table_root: usize, is_kernel: bool) {
-    if is_kernel {
-        // kernel space use TTBR1 (0xffff_0000_0000_0000..0xffff_ffff_ffff_ffff)
-        TTBR1_EL1.set(page_table_root as _);
-    } else {
-        // user space use TTBR0 (0x0..0xffff_ffff_ffff)
-        TTBR0_EL1.set(page_table_root as _);
-    }
+pub unsafe fn set_kernel_page_table_root(root_paddr: usize) {
+    // kernel space page table use TTBR1 (0xffff_0000_0000_0000..0xffff_ffff_ffff_ffff)
+    TTBR1_EL1.set(root_paddr as _);
+    // clear user page table root when initialize kernel page table.
+    TTBR0_EL1.set(0);
     flush_tlb_all();
 }
 
+pub unsafe fn set_user_page_table_root(root_paddr: usize) {
+    // user space page table use TTBR0 (0x0..0xffff_ffff_ffff)
+    let old_root = TTBR0_EL1.get();
+    debug!("Set page table root: {:#x} => {:#x}", old_root, root_paddr);
+    if old_root != root_paddr as u64 {
+        TTBR0_EL1.set(root_paddr as u64);
+        flush_tlb_all();
+    }
+}
+
 pub fn flush_tlb_all() {
     unsafe { asm!("tlbi vmalle1; dsb sy; isb") };
 }

kernel/src/arch/aarch64/mod.rs

@@ -2,7 +2,6 @@ mod context;
 mod page_table;
 mod trap;
 
-pub mod consts;
 pub mod instructions;
 
 pub use self::context::{TaskContext, TrapFrame};

kernel/src/arch/x86_64/context.rs

@@ -1,6 +1,6 @@
 use core::arch::asm;
 
-use crate::mm::PhysAddr;
+use crate::mm::{PhysAddr, VirtAddr};
 
 #[repr(C)]
 #[derive(Debug, Default, Clone, Copy)]
@@ -36,21 +36,21 @@ pub struct TrapFrame {
 }
 
 impl TrapFrame {
-    pub fn new_user(entry: usize, ustack_top: usize, arg0: usize) -> Self {
+    pub fn new_user(entry: VirtAddr, ustack_top: VirtAddr, arg0: usize) -> Self {
         Self {
             rdi: arg0 as _,
-            rip: entry as _,
+            rip: entry.as_usize() as _,
             cs: 0x20 | 3,
-            rflags: 0x3000 | 0x200 | 0x2, // IOPL = 3, IF = 1 (FIXME: set IOPL = 0 when IO port bitmap is supported)
-            user_rsp: ustack_top as _,
+            rflags: 0x3202, // IOPL = 3, IF = 1 (FIXME: set IOPL = 0 when IO port bitmap is supported)
+            user_rsp: ustack_top.as_usize() as _,
             user_ss: 0x28 | 3,
             ..Default::default()
         }
     }
 
-    pub const fn new_clone(&self, ustack_top: usize) -> Self {
+    pub const fn new_clone(&self, ustack_top: VirtAddr) -> Self {
         let mut tf = *self;
-        tf.user_rsp = ustack_top as _;
+        tf.user_rsp = ustack_top.as_usize() as _;
         tf.rax = 0; // for child thread, clone returns 0
         tf
     }
@@ -61,54 +61,28 @@ impl TrapFrame {
         tf
     }
 
-    pub unsafe fn exec(&self, kstack_top: usize) -> ! {
-        asm!(
-            "
-            // mov     sp, x1
-            // ldp     x30, x9, [x0, 30 * 8]
-            // ldp     x10, x11, [x0, 32 * 8]
-            // msr     sp_el0, x9
-            // msr     elr_el1, x10
-            // msr     spsr_el1, x11
-
-            // ldp     x28, x29, [x0, 28 * 8]
-            // ldp     x26, x27, [x0, 26 * 8]
-            // ldp     x24, x25, [x0, 24 * 8]
-            // ldp     x22, x23, [x0, 22 * 8]
-            // ldp     x20, x21, [x0, 20 * 8]
-            // ldp     x18, x19, [x0, 18 * 8]
-            // ldp     x16, x17, [x0, 16 * 8]
-            // ldp     x14, x15, [x0, 14 * 8]
-            // ldp     x12, x13, [x0, 12 * 8]
-            // ldp     x10, x11, [x0, 10 * 8]
-            // ldp     x8, x9, [x0, 8 * 8]
-            // ldp     x6, x7, [x0, 6 * 8]
-            // ldp     x4, x5, [x0, 4 * 8]
-            // ldp     x2, x3, [x0, 2 * 8]
-            // ldp     x0, x1, [x0]
-
-            iret",
-            // in("x0") self,
-            // in("x1") kstack_top,
-            options(noreturn),
-        )
+    pub unsafe fn exec(&self, _kstack_top: VirtAddr) -> ! {
+        unimplemented!()
     }
 }
 
+#[repr(C)]
+#[derive(Debug, Default)]
+struct ContextSwitchFrame {
+    r15: u64,
+    r14: u64,
+    r13: u64,
+    r12: u64,
+    rbx: u64,
+    rbp: u64,
+    rflags: u64,
+    rip: u64,
+}
+
 #[repr(C)]
 #[derive(Debug)]
 pub struct TaskContext {
-    pub r12: u64,
-    pub r13: u64,
-    pub r14: u64,
-    pub r15: u64,
-    pub rbx: u64,
-    pub rbp: u64,
-
-    pub rflags: u64,
     pub rsp: u64,
-    pub rip: u64,
-
     pub fs_base: u64,
     pub cr3: u64,
 }
@@ -118,46 +92,58 @@ impl TaskContext {
         unsafe { core::mem::MaybeUninit::zeroed().assume_init() }
     }
 
-    pub fn init(&mut self, entry: usize, kstack_top: usize, page_table_root: PhysAddr) {
-        self.rsp = kstack_top as u64;
-        self.rip = entry as u64;
+    pub fn init(
+        &mut self,
+        entry: usize,
+        kstack_top: VirtAddr,
+        page_table_root: PhysAddr,
+        _is_kernel: bool,
+    ) {
+        unsafe {
+            let frame_ptr = (kstack_top.as_mut_ptr() as *mut ContextSwitchFrame).sub(1);
+            core::ptr::write(
+                frame_ptr,
+                ContextSwitchFrame {
+                    rip: entry as _,
+                    rflags: 0x3002, // IOPL = 3, IF = 0
+                    ..Default::default()
+                },
+            );
+            self.rsp = frame_ptr as u64;
+        }
         self.cr3 = page_table_root.as_usize() as u64;
     }
 
     pub fn switch_to(&mut self, next_ctx: &Self) {
         unsafe {
-            crate::arch::instructions::activate_paging(next_ctx.cr3 as usize, false);
-            context_switch(self, next_ctx)
+            crate::arch::instructions::set_user_page_table_root(next_ctx.cr3 as usize);
+            // TODO: swtich fs_base
+            context_switch(&mut self.rsp, &next_ctx.rsp)
         }
     }
 }
 
 #[naked]
-unsafe extern "C" fn context_switch(_current_task: &mut TaskContext, _next_task: &TaskContext) {
+unsafe extern "C" fn context_switch(_current_stack: &mut u64, _next_stack: &u64) {
     asm!(
         "
-        // save old context (callee-saved registers)
-        // stp     x29, x30, [x0, 12 * 8]
-        // stp     x27, x28, [x0, 10 * 8]
-        // stp     x25, x26, [x0, 8 * 8]
-        // stp     x23, x24, [x0, 6 * 8]
-        // stp     x21, x22, [x0, 4 * 8]
-        // stp     x19, x20, [x0, 2 * 8]
-        // mov     x19, sp
-        // mrs     x20, tpidr_el0
-        // stp     x19, x20, [x0]
-
-        // restore new context
-        // ldp     x19, x20, [x1]
-        // mov     sp, x19
-        // msr     tpidr_el0, x20
-        // ldp     x19, x20, [x1, 2 * 8]
-        // ldp     x21, x22, [x1, 4 * 8]
-        // ldp     x23, x24, [x1, 6 * 8]
-        // ldp     x25, x26, [x1, 8 * 8]
-        // ldp     x27, x28, [x1, 10 * 8]
-        // ldp     x29, x30, [x1, 12 * 8]
-
+        pushf
+        push    rbp
+        push    rbx
+        push    r12
+        push    r13
+        push    r14
+        push    r15
+        mov     [rdi], rsp
+
+        mov     rsp, [rsi]
+        pop     r15
+        pop     r14
+        pop     r13
+        pop     r12
+        pop     rbx
+        pop     rbp
+        popf
         ret",
         options(noreturn),
     )