x86_64: support HPET to calibrate TSC and APIC frequency

kernel/Cargo.toml

@@ -18,6 +18,8 @@ log = "0.4"
 cfg-if = "1.0"
 bitflags = "1.3"
 xmas-elf = "0.8"
+bit_field = "0.10"
+tock-registers = { version = "0.7", default-features = false, features = ["register_types"] }
 memoffset = { version = "0.6", features = ["unstable_const"] }
 buddy_system_allocator = { version = "0.8", default-features = false }
 
@@ -28,7 +30,6 @@ x2apic = "0.4"
 raw-cpuid = "10.3"
 
 [target.'cfg(target_arch = "aarch64")'.dependencies]
-tock-registers = "0.7"
 cortex-a = "7.2"
 
 [build-dependencies]

kernel/platforms/pc-rvm.toml

@@ -3,6 +3,7 @@ phys-memory-size = "0x800_0000"  # 128M
 kernel-base-paddr = "0x4200_0000"
 kernel-base-vaddr = "0xffff_ff80_4200_0000"
 mmio-regions = [
-    ["0xfec00000", "0x1000"],   # IO APIC
-    ["0xfee00000", "0x1000"],   # Local APIC
+    ["0xfec0_0000", "0x1000"],   # IO APIC
+    ["0xfed0_0000", "0x1000"],   # HPET
+    ["0xfee0_0000", "0x1000"],   # Local APIC
 ]

kernel/platforms/pc.toml

@@ -4,5 +4,6 @@ kernel-base-paddr = "0x20_0000"
 kernel-base-vaddr = "0xffff_ff80_0020_0000"
 mmio-regions = [
     ["0xfec0_0000", "0x1000"],   # IO APIC
+    ["0xfed0_0000", "0x1000"],   # HPET
     ["0xfee0_0000", "0x1000"],   # Local APIC
 ]

kernel/src/drivers/interrupt/apic.rs

@@ -2,27 +2,57 @@
 
 #![allow(dead_code)]
 
+use x2apic::ioapic::{IoApic, IrqFlags, IrqMode};
 use x2apic::lapic::{xapic_base, LocalApic, LocalApicBuilder};
 
 use crate::mm::PhysAddr;
-use crate::sync::{LazyInit, PerCpuData};
+use crate::sync::{LazyInit, PerCpuData, SpinNoIrqLock};
 use crate::utils::irq_handler::{IrqHandler, IrqHandlerTable};
 
-const APIC_TIMER_VECTOR: usize = 0xf0;
-const APIC_SPURIOUS_VECTOR: usize = 0xf1;
-const APIC_ERROR_VECTOR: usize = 0xf2;
+pub mod vectors {
+    pub const PIT_GSI: usize = 2; // TODO: lookup ACPI tables
+    pub const PIT_VECTOR: usize = 0x20;
+
+    pub const APIC_TIMER_VECTOR: usize = 0xf0;
+    pub const APIC_SPURIOUS_VECTOR: usize = 0xf1;
+    pub const APIC_ERROR_VECTOR: usize = 0xf2;
+}
+
+use self::vectors::*;
 
 const IRQ_COUNT: usize = 256;
 
+const IO_APIC_BASE: PhysAddr = PhysAddr::new(0xFEC0_0000);
+
 static LOCAL_APIC: LazyInit<PerCpuData<LocalApic>> = LazyInit::new();
+static IO_APIC: LazyInit<SpinNoIrqLock<IoApic>> = LazyInit::new();
 static HANDLERS: IrqHandlerTable<IRQ_COUNT> = IrqHandlerTable::new();
 
 fn lapic_eoi() {
     unsafe { local_apic().end_of_interrupt() };
 }
 
-pub fn set_enable(_vector: usize, _enable: bool) {
-    // TODO: implement IOAPIC
+pub fn set_enable(gsi: usize, enable: bool) {
+    unsafe {
+        if enable {
+            IO_APIC.lock().enable_irq(gsi as u8);
+        } else {
+            IO_APIC.lock().disable_irq(gsi as u8);
+        }
+    }
+}
+
+#[allow(dead_code)]
+fn configure_irq(gsi: usize, vector: usize) {
+    let mut io_apic = IO_APIC.lock();
+    unsafe {
+        let mut entry = io_apic.table_entry(gsi as u8);
+        entry.set_dest((local_apic().id() >> 24) as u8); // TODO: distinguish x2apic/x2apic
+        entry.set_vector(vector as u8);
+        entry.set_mode(IrqMode::Fixed);
+        entry.set_flags(IrqFlags::MASKED);
+        io_apic.set_table_entry(gsi as u8, entry);
+    }
 }
 
 pub fn handle_irq(vector: usize) {
@@ -35,6 +65,7 @@ pub fn register_handler(vector: usize, handler: IrqHandler) {
 }
 
 pub fn init() {
+    println!("Initializing Local APIC...");
     super::i8259_pic::init();
 
     let base_vaddr = PhysAddr::new(unsafe { xapic_base() } as usize).into_kvaddr();
@@ -48,6 +79,9 @@ pub fn init() {
     unsafe { lapic.enable() };
     LOCAL_APIC.init_by(PerCpuData::new(lapic));
 
+    let io_apic = unsafe { IoApic::new(IO_APIC_BASE.into_kvaddr().as_usize() as u64) };
+    IO_APIC.init_by(SpinNoIrqLock::new(io_apic));
+
     super::register_handler(APIC_TIMER_VECTOR, crate::timer::handle_timer_irq);
 }
 

kernel/src/drivers/interrupt/mod.rs

@@ -4,6 +4,7 @@ cfg_if! {
         mod i8259_pic;
         use apic as imp;
         pub use apic::local_apic;
+        pub use apic::vectors::*;
     } else if #[cfg(target_arch = "aarch64")] {
         mod gicv2;
         use gicv2 as imp;

kernel/src/drivers/timer/arm_generic_timer.rs

@@ -5,33 +5,22 @@ use tock_registers::interfaces::{Readable, Writeable};
 
 use crate::drivers::interrupt;
 use crate::sync::LazyInit;
-use crate::timer::TimeValue;
 
 const PHYS_TIMER_IRQ_NUM: usize = 30;
 
-const NANOS_PER_SEC: u64 = 1_000_000_000;
+static CLOCK_FREQ_HZ: LazyInit<u64> = LazyInit::new();
 
-static CLOCK_FREQ: LazyInit<u64> = LazyInit::new();
-
-fn ticks_to_nanos(ticks: u64) -> u64 {
-    ticks * NANOS_PER_SEC / *CLOCK_FREQ
-}
-
-fn nanos_to_ticks(nanos: u64) -> u64 {
-    nanos * *CLOCK_FREQ / NANOS_PER_SEC
-}
-
-pub fn current_time_nanos() -> u64 {
-    ticks_to_nanos(CNTPCT_EL0.get())
+pub fn current_ticks() -> u64 {
+    CNTPCT_EL0.get()
 }
 
-pub fn current_time() -> TimeValue {
-    TimeValue::from_nanos(current_time_nanos())
+pub fn frequency_hz() -> u64 {
+    *CLOCK_FREQ_HZ
 }
 
 pub fn set_oneshot_timer(deadline_ns: u64) {
     let cnptct = CNTPCT_EL0.get();
-    let cnptct_deadline = nanos_to_ticks(deadline_ns);
+    let cnptct_deadline = crate::timer::nanos_to_ticks(deadline_ns, frequency_hz());
     if cnptct < cnptct_deadline {
         let interval = cnptct_deadline - cnptct;
         debug_assert!(interval <= u32::MAX as u64);
@@ -42,7 +31,7 @@ pub fn set_oneshot_timer(deadline_ns: u64) {
 }
 
 pub fn init() {
-    CLOCK_FREQ.init_by(CNTFRQ_EL0.get());
+    CLOCK_FREQ_HZ.init_by(CNTFRQ_EL0.get());
     CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET);
     interrupt::register_handler(PHYS_TIMER_IRQ_NUM, crate::timer::handle_timer_irq);
     interrupt::set_enable(PHYS_TIMER_IRQ_NUM, true);

kernel/src/drivers/timer/mod.rs

@@ -1,12 +1,14 @@
 cfg_if! {
     if #[cfg(target_arch = "x86_64")] {
-        mod x86_lapic;
-        use x86_lapic as imp;
+        mod x86_hpet;
+        mod x86_tsc;
+        mod x86_common;
+        use x86_common as imp;
     } else if #[cfg(target_arch = "aarch64")] {
         mod arm_generic_timer;
         use arm_generic_timer as imp;
     }
 }
 
 pub(super) use self::imp::init;
-pub use self::imp::{current_time, current_time_nanos, set_oneshot_timer};
+pub use self::imp::{current_ticks, frequency_hz, set_oneshot_timer};

kernel/src/drivers/timer/x86_common.rs

@@ -0,0 +1,55 @@
+use x2apic::lapic::{TimerDivide, TimerMode};
+
+use super::{x86_hpet, x86_tsc};
+use crate::drivers::interrupt::local_apic;
+use crate::sync::LazyInit;
+use crate::timer::{current_time_nanos, nanos_to_ticks};
+
+pub use x86_tsc::{current_ticks, frequency_hz}; // use TSC as the clock source.
+
+static LAPIC_FREQ_HZ: LazyInit<u64> = LazyInit::new();
+
+pub fn set_oneshot_timer(deadline_ns: u64) {
+    let now_ns = current_time_nanos();
+    unsafe {
+        if now_ns < deadline_ns {
+            let apic_ticks = nanos_to_ticks(deadline_ns - now_ns, *LAPIC_FREQ_HZ);
+            debug_assert!(apic_ticks <= u32::MAX as u64);
+            local_apic().set_timer_initial(apic_ticks.max(1) as u32);
+        } else {
+            local_apic().set_timer_initial(1);
+        }
+    }
+}
+
+fn calibrate_lapic_timer() {
+    let lapic = local_apic();
+    unsafe {
+        lapic.set_timer_mode(TimerMode::OneShot);
+        lapic.set_timer_divide(TimerDivide::Div256); // divide 1
+    }
+
+    let mut best_freq_hz = u64::MAX;
+    for _ in 0..5 {
+        unsafe { lapic.set_timer_initial(u32::MAX) };
+        x86_hpet::wait_millis(10);
+        let ticks_per_sec = unsafe { (u32::MAX - lapic.timer_current()) as u64 * 100 };
+        if ticks_per_sec < best_freq_hz {
+            best_freq_hz = ticks_per_sec;
+        }
+    }
+    println!(
+        "Calibrated LAPIC frequency: {}.{:03} MHz",
+        best_freq_hz / 1_000_000,
+        best_freq_hz % 1_000_000 / 1_000,
+    );
+
+    LAPIC_FREQ_HZ.init_by(best_freq_hz);
+    unsafe { lapic.enable_timer() } // enable APIC timer IRQ
+}
+
+pub fn init() {
+    super::x86_hpet::init();
+    x86_tsc::calibrate_tsc();
+    calibrate_lapic_timer();
+}