Make local APIC timer really local; improve timers initialization; interrupts restore

arch/x86/apic.c

@@ -25,7 +25,6 @@
 #include <apic.h>
 #include <console.h>
 #include <cpu.h>
-#include <drivers/pit.h>
 #include <ktf.h>
 #include <lib.h>
 #include <percpu.h>
@@ -179,6 +178,9 @@ void init_apic_timer(void) {
     apic_lvt_timer_t timer;
     printk("Initializing local APIC timer\n");
 
+    /* Enable interrupts for calibration */
+    unsigned long flags = interrupts_enable_save();
+
     /* Spend 200ms calibrating the timer, 10 iterations of 20ms each */
     for (i = 0; i < CAL_ITERATIONS; ++i) {
         /* Set the counter to the max value (0xFFFFFFFF) */
@@ -197,6 +199,8 @@ void init_apic_timer(void) {
         min_ticks = min(min_ticks, elapsed_ticks);
     }
 
+    interrupts_restore(flags);
+
     /* Interrupt every min_ticks ticks */
     apic_write(APIC_TMR_DCR, APIC_TIMER_DIVIDE_BY_16);
     apic_write(APIC_TMR_ICR, min_ticks);
@@ -206,5 +210,5 @@ void init_apic_timer(void) {
     timer.timer_mode = APIC_LVT_TIMER_PERIODIC;
     apic_write(APIC_LVT_TIMER, timer.reg);
 
-    pit_disable();
+    PERCPU_SET(apic_timer_enabled, true);
 }

arch/x86/entry.S

@@ -162,6 +162,7 @@ GLOBAL(end_exception_handlers)
 
 GLOBAL(interrupt_handlers)
 interrupt_handler timer timer_interrupt_handler timer_irq
+interrupt_handler apic_timer apic_timer_interrupt_handler apic_timer_irq
 interrupt_handler uart1 uart_interrupt_handler serial_com1_irq
 interrupt_handler uart2 uart_interrupt_handler serial_com2_irq
 interrupt_handler keyboard keyboard_interrupt_handler kb_port1_irq

arch/x86/traps.c

@@ -44,6 +44,7 @@ extern void asm_interrupt_handler_uart2(void);
 extern void asm_interrupt_handler_keyboard(void);
 extern void asm_interrupt_handler_timer(void);
 extern void asm_interrupt_handler_dummy(void);
+extern void asm_interrupt_handler_apic_timer(void);
 
 extern void terminate_user_task(void);
 
@@ -165,7 +166,7 @@ void init_traps(const cpu_t *cpu) {
     set_intr_gate(&percpu->idt[KB_PORT1_IRQ], __KERN_CS,
                   _ul(asm_interrupt_handler_keyboard), GATE_DPL0, GATE_PRESENT, 0);
     set_intr_gate(&percpu->idt[APIC_TIMER_IRQ], __KERN_CS,
-                  _ul(asm_interrupt_handler_timer), GATE_DPL0, GATE_PRESENT, 0);
+                  _ul(asm_interrupt_handler_apic_timer), GATE_DPL0, GATE_PRESENT, 0);
     set_intr_gate(&percpu->idt[APIC_SPI_VECTOR], __KERN_CS,
                   _ul(asm_interrupt_handler_dummy), GATE_DPL0, GATE_PRESENT, 0);
 

common/acpi.c

@@ -750,7 +750,7 @@ ACPI_STATUS init_acpi(void) {
 
 void acpi_power_off(void) {
     AcpiEnterSleepStatePrep(ACPI_STATE_S5);
-    cli();
+    interrupts_disable();
     AcpiEnterSleepState(ACPI_STATE_S5);
     panic("Power Off");
 }

common/setup.c

@@ -165,18 +165,18 @@ static void __text_init init_vga_console(void) {
 }
 
 void __text_init init_timers(cpu_t *cpu) {
-    static bool __data_init hpet_initialized = false;
-
     if (is_cpu_bsp(cpu)) {
         if (opt_hpet)
-            hpet_initialized = init_hpet(cpu);
+            boot_flags.timer_global = init_hpet(cpu);
 
-        if (!hpet_initialized && opt_pit)
+        if (!boot_flags.timer_global && opt_pit) {
             init_pit(cpu);
+            boot_flags.timer_global = true;
+        }
     }
 
     if (opt_apic_timer) {
-        if (hpet_initialized || opt_pit) /* Needed for APIC timer calibration */
+        if (boot_flags.timer_global) /* Needed for APIC timer calibration */
             init_apic_timer();
         else {
             warning("CPU%u: Unable to initialize APIC timer - no calibration timers!",
@@ -214,9 +214,6 @@ void __noreturn __text_init kernel_start(uint32_t multiboot_magic, unsigned long
     /* Initialize Programmable Interrupt Controller */
     init_pic();
 
-    /* PIC is initialized - enable local interrupts */
-    sti();
-
     /* Initialize Physical Memory Manager */
     init_regions();
     init_pmm();
@@ -264,8 +261,9 @@ void __noreturn __text_init kernel_start(uint32_t multiboot_magic, unsigned long
 
     init_ioapic();
 
-    /* Initialize timers */
+    /* Initialize timers and enable interrupts */
     init_timers(bsp);
+    interrupts_enable();
 
     if (!boot_flags.nosmp)
         init_smp();

drivers/keyboard.c

@@ -67,7 +67,7 @@ void keyboard_reboot(void) {
     if (!i8042_present)
         return;
 
-    cli();
+    interrupts_disable();
 
     while ((inb(KEYBOARD_PORT_CMD) & KEYBOARD_STATUS_IN_FULL) != 0)
         cpu_relax();

include/arch/x86/asm-macros.h

@@ -323,8 +323,10 @@ name ## _end:
     "pop %%" STR(_ASM_BX) "\n"
 
 #if defined(__x86_64__)
+#define PUSHF() "pushfq\n"
 #define POPF() "popfq\n"
 #else
+#define PUSHF() "pushf\n"
 #define POPF() "popf\n"
 #endif
 

include/lib.h

@@ -171,22 +171,49 @@ static inline void int3(void) {
     asm volatile("int3");
 }
 
-static inline unsigned long read_flags(void) {
+static inline unsigned long read_eflags(void) {
     unsigned long flags;
 
-    asm volatile(
-#if defined(__i386__)
-        "pushfd;"
-        "popl %0"
-#elif defined(__x86_64__)
-        "pushfq;"
-        "popq %0"
-#endif
-        : "=r"(flags));
+    asm volatile(PUSHF() "pop %0" : "=r"(flags));
+
+    return flags;
+}
+
+static inline void write_eflags(unsigned long flags) {
+    asm volatile("push %0\n" POPF()::"r"(flags));
+}
+
+static inline void interrupts_enable(void) {
+    sti();
+}
 
+static inline unsigned long interrupts_enable_save(void) {
+    unsigned long flags = read_eflags();
+    interrupts_enable();
     return flags;
 }
 
+static inline void interrupts_disable(void) {
+    cli();
+}
+
+static inline unsigned long interrupts_disable_save(void) {
+    unsigned long flags = read_eflags();
+    interrupts_disable();
+    return flags;
+}
+
+static inline void interrupts_restore(unsigned long flags) {
+    if (flags & X86_EFLAGS_IF)
+        sti();
+    else
+        cli();
+}
+
+static inline bool interrupts_enabled(void) {
+    return read_eflags() & X86_EFLAGS_IF;
+}
+
 static inline unsigned long read_cs(void) {
     unsigned long cs;
 

include/percpu.h

@@ -58,6 +58,8 @@ struct percpu {
 #endif
 
     unsigned long usermode_private;
+    volatile unsigned long apic_ticks;
+    bool apic_timer_enabled;
 } __aligned(PAGE_SIZE);
 typedef struct percpu percpu_t;
 

include/setup.h

@@ -34,7 +34,7 @@
 
 struct boot_flags {
     uint64_t virt : 1, legacy_devs : 1, i8042 : 1, vga : 1, msi : 1, aspm : 1, rtc : 1,
-        nosmp : 1, rsvd : 56;
+        nosmp : 1, timer_global : 1, rsvd : 55;
 };
 typedef struct boot_flags boot_flags_t;
 

include/time.h

@@ -27,13 +27,19 @@
 
 typedef uint64_t time_t;
 
-extern void msleep(time_t ms);
+extern int msleep(time_t ms);
+extern int msleep_local(time_t ms);
 extern time_t get_timer_ticks(void);
+extern time_t get_local_ticks(void);
 
 /* Static declarations */
 
-static inline void sleep(time_t s) {
-    msleep(s * 1000);
+static inline int sleep(time_t s) {
+    return msleep(s * 1000);
+}
+
+static inline int sleep_local(time_t s) {
+    return msleep_local(s * 1000);
 }
 
 #endif

lib/lib.c

@@ -28,7 +28,7 @@
 #include <extables.h>
 
 void __noreturn halt(void) {
-    cli();
+    interrupts_disable();
 
     while (1) {
         hlt();
@@ -40,7 +40,7 @@ void __noreturn halt(void) {
 void __noreturn hard_reboot(void) {
     idt_ptr_t idt_ptr = {0};
 
-    cli();
+    interrupts_disable();
     lidt(&idt_ptr);
     int3();
 

lib/time.c

@@ -23,22 +23,56 @@
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include <apic.h>
+#include <errno.h>
+#include <percpu.h>
+#include <setup.h>
 #include <time.h>
 
-static volatile time_t ticks = 0;
+extern boot_flags_t boot_flags;
+
+static __aligned(16) volatile time_t ticks = 0;
 
 void timer_interrupt_handler(void) {
-    ++ticks;
+    asm volatile("lock incq %[ticks]" : [ ticks ] "=m"(ACCESS_ONCE(ticks)));
+    apic_EOI();
+}
+
+void apic_timer_interrupt_handler(void) {
+    asm volatile("lock incq %%gs:%[ticks]"
+                 : [ ticks ] "=m"(ACCESS_ONCE(PERCPU_VAR(apic_ticks))));
     apic_EOI();
 }
 
-void msleep(time_t ms) {
-    time_t end = ticks + ms;
-    while (ticks < end) {
+int msleep(time_t ms) {
+    time_t end;
+
+    if (!boot_flags.timer_global)
+        return -ENODEV;
+
+    end = ACCESS_ONCE(ticks) + ms;
+    while (ACCESS_ONCE(ticks) < end)
         cpu_relax();
-    }
+
+    return 0;
+}
+
+int msleep_local(time_t ms) {
+    time_t end;
+
+    if (!PERCPU_GET(apic_timer_enabled))
+        return -ENODEV;
+
+    end = PERCPU_GET(apic_ticks) + ms;
+    while (PERCPU_GET(apic_ticks) < end)
+        cpu_relax();
+
+    return 0;
 }
 
 time_t get_timer_ticks(void) {
-    return ticks;
+    return ACCESS_ONCE(ticks);
+}
+
+time_t get_local_ticks(void) {
+    return PERCPU_GET(apic_ticks);
 }

smp/smp.c

@@ -56,8 +56,9 @@ void __noreturn ap_startup(void) {
     init_traps(cpu);
     init_apic(ap_cpuid, apic_get_mode());
 
+    /* Initialize timers and enable interrupts */
     init_timers(cpu);
-    sti();
+    interrupts_enable();
 
     if (opt_fpu)
         enable_fpu();

tests/test_cond_branch_mispredictions.c

@@ -215,13 +215,13 @@ int __aligned(PAGE_SIZE) test_cond_branch_mispredictions(void *unused) {
     printk("Testing conditional branch %s BTB flushing\n",
            WITH_BTB_FLUSH ? "with" : "without");
 
-    cli();
+    unsigned long flags = interrupts_disable_save();
     test_cond_forward_branch_cl0(LOOP_ITERATIONS);
     test_cond_forward_branch_cl1(LOOP_ITERATIONS);
 
     test_cond_backward_branch_cl0(LOOP_ITERATIONS);
     test_cond_backward_branch_cl1(LOOP_ITERATIONS);
-    sti();
+    interrupts_restore(flags);
 
     return 0;
 }

tests/test_uncond_branch_mispredictions.c

@@ -231,13 +231,13 @@ int __aligned(PAGE_SIZE) test_uncond_branch_mispredictions(void *unused) {
     printk("Testing direct unconditional " STR(BRANCH) " %s BTB flushing\n",
            WITH_BTB_FLUSH ? "with" : "without");
 
-    cli();
+    unsigned long flags = interrupts_disable_save();
     test_uncond_forward_branch_cl0(LOOP_ITERATIONS);
     test_uncond_forward_branch_cl1(LOOP_ITERATIONS);
 
     test_uncond_backward_branch_cl0(LOOP_ITERATIONS);
     test_uncond_backward_branch_cl1(LOOP_ITERATIONS);
-    sti();
+    interrupts_restore(flags);
 
     return 0;
 }