treewide: Improve SMP support

cheshire.mk

@@ -63,7 +63,7 @@ chs-clean-deps:
 ######################
 
 CHS_NONFREE_REMOTE ?= [email protected]:pulp-restricted/cheshire-nonfree.git
-CHS_NONFREE_COMMIT ?= 92f6f02
+CHS_NONFREE_COMMIT ?= a111e47
 
 CHS_PHONY += chs-nonfree-init
 chs-nonfree-init:
@@ -82,17 +82,26 @@ include $(CHS_ROOT)/sw/sw.mk
 # Generate HW #
 ###############
 
+# `CHS_NUM_IRQ_HARTS` and `CHS_NUM_PLIC_SRCS` are used to generate register files.
+# They must match the corresponding SystemVerilog parameters.
+CHS_NUM_IRQ_HARTS  ?= 1
+CHS_NUM_PLIC_SRCS  ?= 58
+CHS_NUM_PLIC_PRIOW ?= 7
+
 # SoC registers
 $(CHS_ROOT)/hw/regs/cheshire_reg_pkg.sv $(CHS_ROOT)/hw/regs/cheshire_reg_top.sv: $(CHS_ROOT)/hw/regs/cheshire_regs.hjson
 	$(REGTOOL) -r $< --outdir $(dir $@)
 
 # CLINT
-CLINTCORES ?= 1
+CLINTCORES ?= $(CHS_NUM_IRQ_HARTS)
 include $(CLINTROOT)/clint.mk
 $(CLINTROOT)/.generated:
 	flock -x $@ $(MAKE) clint && touch $@
 
 # OpenTitan peripherals
+$(CHS_ROOT)/hw/rv_plic.cfg.hjson: $(CHS_ROOT)/util/gen_pliccfg.py
+	$< $(CHS_NUM_IRQ_HARTS) $(CHS_NUM_PLIC_SRCS) $(CHS_NUM_PLIC_PRIOW) > $@
+
 include $(OTPROOT)/otp.mk
 $(OTPROOT)/.generated: $(CHS_ROOT)/hw/rv_plic.cfg.hjson
 	flock -x $@ sh -c "cp $< $(dir $@)/src/rv_plic/; $(MAKE) -j1 otp" && touch $@

docs/um/arch.md

@@ -185,7 +185,7 @@ Cheshire provides a flexible RISC-V interrupt architecture that can route and mu
 
 First, all internal (`intr.intn`) and external (`intr_ext_i`) interrupt sources are collected (`intr`). From here, they either pass through an *interrupt router* if enabled (`IrqRouter`) or are simply fanned out to interrupt *targets*, which may support as many or fewer interrupt sources as provided by `intr`. If a target supports fewer sources, its interrupt sources are *truncated*.
 
-Cheshire provides both a core-local interruptor (CLINT), grouping all per-core interrupts in one module, and a shared platform-level interrupt controller (PLIC). The former is used only for inter-processor and timer interrupts, while the latter is a proper interrupt target. If enabled (`Clic`), each CVA6 core also has a core-local interrupt controller (CLIC), another interrupt target. In addition to the PLIC and CLICs, any number external interrupt targets may be defined (`NumExtOutIntrTgts`) with their own number of incoming sources (`NumExtIrqHarts`).
+Cheshire provides both a core-local interruptor (CLINT), grouping all per-core interrupts in one module, and a shared platform-level interrupt controller (PLIC). The former is used only for inter-processor and timer interrupts, while the latter is a proper interrupt target. If enabled (`Clic`), each CVA6 core also has a core-local interrupt controller (CLIC), another interrupt target. In addition to the PLIC and CLICs, any number of external interrupt targets may be defined (`NumExtOutIntrTgts`) with their own number of incoming sources (`NumExtOutIntrs`).
 
 Finally, the PLIC and grouped CLINT also support allocating external harts for which to manage interrupts (`NumExtIrqHarts`), i.e. harts without interrupt controllers of themselves.
 

hw/bootrom/cheshire_bootrom.S

@@ -5,17 +5,20 @@
 // Nicole Narr <[email protected]>
 // Christopher Reinwardt <[email protected]>
 // Paul Scheffler <[email protected]>
+// Enrico Zelioli <[email protected]>
 
-// TODO: Avoid hardcoding in addresses and offsets
-
-#include "smp.h"
-
+#include <regs/cheshire.h>
+#include <regs/axi_llc.h>
 .section .text._start
 
 // Minimal bootrom loader
 .global _start
 .align 4
 _start:
+    // Globally disable Machine and Supervisor interrupts (MIE, SIE = 0).
+    // Note that this will *not* stop WFI from resuming on enabled interrupts.
+    csrrc zero, mstatus, 10
+
     // Reset all integer GPRs; we do *not* assume FP in the boot ROM.
     li x1, 0
     li x4, 0
@@ -47,8 +50,12 @@ _start:
     li x30, 0
     li x31, 0
 
-    // Pause SMP harts
-    smp_pause(t0, t1)
+    // Pause SMP harts: enable only MSIE, disable all other interrupt sources
+    li   t1, 0x8
+    csrw mie, t1
+    li   t0, 0
+    csrr t1, mhartid
+    bne  t0, t1, _smp_wait
 
     // Init stack and global pointer with safe, linked values
     la sp, __stack_pointer$
@@ -58,33 +65,35 @@ _start:
     .option pop
 
     // If LLC present: Wait for end of BIST, then extend stack and set to all SPM
-    la t0, __base_regs
-    lw t0, 80(t0)   // regs.HW_FEATURES
-    andi t0, t0, 2  // regs.HW_FEATURES.llc
+    la   t0, __base_regs
+    lwu  t0, CHESHIRE_HW_FEATURES_REG_OFFSET(t0)
+    andi t0, t0, (1 << CHESHIRE_HW_FEATURES_LLC_BIT)
     beqz t0, _prom_check_run
-    la t0, __base_llc
+    la   t0, __base_llc
 _wait_llc_bist:
-    lw t1, 72(t0)   // llc.BIST_STATUS_DONE_BIT
+    lwu  t1, AXI_LLC_BIST_STATUS_REG_OFFSET(t0) // Check BIST status done bit
     beqz t1, _wait_llc_bist
-    li t1, -1
-    sw t1, 0(t0)    // llc.CFG_SPM_LOW
-    sw t1, 4(t0)    // llc.CFG_SPM_HIGH
-    li t1, 1
-    sw t1, 16(t0)   // llc.CFG_COMMIT
+    li   t1, -1
+    sw   t1, AXI_LLC_CFG_SPM_LOW_REG_OFFSET(t0)
+    sw   t1, AXI_LLC_CFG_SPM_HIGH_REG_OFFSET(t0)
+    li   t1, 1
+    sw   t1, AXI_LLC_COMMIT_CFG_REG_OFFSET(t0)
     // Correct stack to start at end of SPM
-    la t0, __base_regs
-    la sp, __base_spm
-    lw t0, 84(t0)   // regs.LLC_SIZE
-    add sp, sp, t0
+    la   t0, __base_regs
+    la   sp, __base_spm
+    lwu  t0, CHESHIRE_LLC_SIZE_REG_OFFSET(t0)
+    add  sp, sp, t0
     addi sp, sp, -8
 
-// Enter Platform ROM if present.
+// Enter Platform ROM if present. Falls through to `_boot`.
 _prom_check_run:
     // Note that we have internal access to SPM here *if and only if* there is an LLC.
-    la t0, __base_regs
-    lw t0, 72(t0)   // regs.PLATFORM_ROM
-    beqz t0, _boot
-    jalr t0
+    li   t0, 0
+    li   t1, 0
+    la   ra, __base_regs
+    lwu  ra, CHESHIRE_PLATFORM_ROM_REG_OFFSET(ra)
+    beqz ra, _boot
+    jalr ra, 0(ra)
 
 // Reset regs, full fence, then jump to main
 _boot:
@@ -99,34 +108,44 @@ _boot:
 .align 4
 _exit:
     // Save the return value to scratch register 2, try `ebreak`, then wait forever
+    // Set bit 0 to signal that the execution is done.
     slli a0, a0, 1
     ori  a0, a0, 1
     la t0, __base_regs
-    sw a0, 8(t0)     // regs.SCRATCH[2]
+    sw a0, CHESHIRE_SCRATCH_2_REG_OFFSET(t0)
     ebreak
 1:  wfi
     j 1b
 
-.global boot_next_stage
-.align 4
-boot_next_stage:
-    // Non-SMP hart: Write boot address into global scratch registers
-    la t0, __base_regs
-    sw a0, 16(t0)   // regs.SCRATCH[4]
-    srli a0, a0, 32
-    sw a0, 20(t0)   // regs.SCRATCH[5]
-    fence
-    // Resume SMP harts
-    smp_resume(t0, t1, t2)
-    // Load boot address from global scratch registers
-    la t0, __base_regs
-    lwu t1, 20(t0)  // regs.SCRATCH[5]
+// Nonzero harts wait for and handle wakeup here
+_smp_wait:
+    // Set mtvec, sleep, and check MSIP; repeat unless MSIP is set (M IPI is pending).
+    la   t0, 1f
+    csrw t0, mtvec
+    wfi
+1:  csrr t0, mip
+    andi t0, t0, 0x8
+    beqz t0, _smp_wait
+    // Clear CLINT M IPI register for this hart
+    la   t0, __base_clint
+    csrr ra, mhartid
+    slli t1, ra, 2
+    add  t1, t1, t0
+    sw   zero, 0(t1) // *(CLINT_BASE + hart_id * 4) = 0
+    // Check SMP enable (scratch[6][0]).
+    // If this is not set, go back to sleep, as we are not meant to resume.
+    la   t0, __base_regs
+    lwu  t0, CHESHIRE_SCRATCH_6_REG_OFFSET(t0)
+    andi t0, t0, 1
+    beqz t0, _smp_wait
+    // Jump to scratch[5:4]. Synchronization, if any, should be done there.
+    la   t1, __base_regs
+    lwu  t0, CHESHIRE_SCRATCH_4_REG_OFFSET(t1)
+    lwu  t1, CHESHIRE_SCRATCH_5_REG_OFFSET(t1)
     slli t1, t1, 32
-    lwu t0, 16(t0)  // regs.SCRATCH[4]
-    or t0, t0, t1
-    // Store hartid to a0
-    csrr a0, mhartid
-    // Jump to boot address
-    jalr ra, 0(t0)
-    // We should never get here
-    ret
+    ori  ra, t1, t0
+    li   t0, 0
+    li   t1, 1
+    jalr ra, 0(ra)
+    // Should we return, go back to `_start` to be parked again.
+    j _start

hw/bootrom/cheshire_bootrom.c

@@ -18,8 +18,6 @@
 #include "hal/uart_debug.h"
 #include "gpt.h"
 
-extern int boot_next_stage(void *);
-
 int boot_passive(uint64_t core_freq) {
     // Initialize UART with debug settings
     uart_debug_init(&__base_uart, core_freq);
@@ -30,7 +28,7 @@ int boot_passive(uint64_t core_freq) {
         if (uart_debug_check(&__base_uart)) return uart_debug_serve(&__base_uart);
     // No UART (or JTAG) requests came in, but scratch[2][2] was set --> run code at scratch[1:0]
     scratch[2] = 0;
-    return boot_next_stage((void *)(uintptr_t)(((uint64_t)scratch[1] << 32) | scratch[0]));
+    return invoke((void *)(uintptr_t)(((uint64_t)scratch[1] << 32) | scratch[0]));
 }
 
 int boot_spi_sdcard(uint64_t core_freq, uint64_t rtc_freq) {

hw/bootrom/cheshire_bootrom.sv

@@ -29,7 +29,7 @@ module cheshire_bootrom #(
     always_comb begin
         data_o = '0;
         unique case (word)
-        000: data_o = 32'h42014081 /* 0x0000 */;
+            000: data_o = 32'h42014081 /* 0x0000 */;
             001: data_o = 32'h43014281 /* 0x0004 */;
             002: data_o = 32'h44014381 /* 0x0008 */;
             003: data_o = 32'h45014481 /* 0x000c */;

hw/cheshire_pkg.sv

@@ -30,7 +30,6 @@ package cheshire_pkg;
   localparam int unsigned SlinkNumChan    = serial_link_single_channel_reg_pkg::NumChannels;
   localparam int unsigned SlinkNumLanes   = serial_link_single_channel_reg_pkg::NumBits/2;
   localparam int unsigned SlinkMaxClkDiv  = 1 << serial_link_single_channel_reg_pkg::Log2MaxClkDiv;
-  localparam int unsigned ClintNumCores   = clint_reg_pkg::NumCores;
   localparam int unsigned UsbNumPorts     = spinal_usb_ohci_pkg::NumPhyPorts;
 
   // Default JTAG ID code type

hw/cheshire_soc.sv

@@ -1714,19 +1714,25 @@ module cheshire_soc import cheshire_pkg::*; #(
 
   end
 
-  //////////////////
-  //  Assertions  //
-  //////////////////
+  ///////////////////////////////
+  //  Elaboration-Time Checks  //
+  ///////////////////////////////
+
+  // Check that CLINT core count is equal to `NumIrqHarts`
+  if (clint_reg_pkg::NumCores != NumIrqHarts)
+    $fatal(1, "CLIC core count (%d) does not match `NumIrqHarts` (%d)",
+      clint_reg_pkg::NumCores, NumIrqHarts);
+
+  // Check that PLIC target count is equal to `2*NumIrqHarts` (two privilege levels)
+  if (rv_plic_reg_pkg::NumTarget != 2 * NumIrqHarts)
+    $fatal(1, "PLIC target count (%d) does not match `2*NumIrqHarts` (%d)",
+      rv_plic_reg_pkg::NumTarget, 2 * NumIrqHarts);
 
-  // TODO: check that CVA6 and Cheshire config agree
   // TODO: check that all interconnect params agree
   // TODO: check that params with min/max values are within legal range
-  // TODO: check that CLINT and PLIC target counts are both `NumIntHarts + Cfg.NumExtHarts`
-  // TODO: check that (for now) `NumIntHarts == 1`
   // TODO: check that available user bits suffice to identify all masters
   // TODO: check that atomics user domain is nonzero
   // TODO: check that `ext` (IO) and internal types agree
-  // TODO: many other things I most likely forgot
   // TODO: check that LLC only exists if its output is connected (the reverse is allowed)
 
 endmodule

hw/rv_plic.cfg.hjson

@@ -3,13 +3,15 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 // Paul Scheffler <[email protected]>
+// Enrico Zelioli <[email protected]>
+// AUTOMATICALLY GENERATED by gen_pliccfg.py; edit the script instead.
 
 {
     instance_name: "rv_plic",
     param_values: {
         src: 58,
-        target: 2,  // We need *two targets* per hart: M and S modes
+        target: 2,
         prio: 7,
-        nonstd_regs: 0  // Do *not* include these: MSIPs are not used and we use a 64 MiB address space
+        nonstd_regs: 0
     },
 }

sw/boot/zsl.c

@@ -14,6 +14,7 @@
 #include "gpt.h"
 #include "dif/uart.h"
 #include "printf.h"
+#include "smp.h"
 
 // Type for firmware payload
 typedef int (*payload_t)(uint64_t, uint64_t, uint64_t);
@@ -51,40 +52,48 @@ static inline void load_part_or_spin(void *priv, const uint64_t *pguid, void *co
 }
 
 int main(void) {
-    // Get system parameters
-    uint32_t bootmode = *reg32(&__base_regs, CHESHIRE_BOOT_MODE_REG_OFFSET);
-    uint32_t rtc_freq = *reg32(&__base_regs, CHESHIRE_RTC_FREQ_REG_OFFSET);
-    uint64_t core_freq = clint_get_core_freq(rtc_freq, 2500);
-    rgp = (void *)(uintptr_t)*reg32(&__base_regs, CHESHIRE_SCRATCH_3_REG_OFFSET);
-    uint32_t read = *reg32(&__base_regs, CHESHIRE_SCRATCH_0_REG_OFFSET);
-    void *priv = (void *)(uintptr_t)*reg32(&__base_regs, CHESHIRE_SCRATCH_1_REG_OFFSET);
 
-    // Initialize UART
-    uart_init(&__base_uart, core_freq, __BOOT_BAUDRATE);
+    uint64_t hart_id = get_mhartid();
 
-    // Print boot-critical cat, and also parameters
-    printf(" /\\___/\\       Boot mode:       %d\r\n"
-           "( o   o )      Real-time clock: %d Hz\r\n"
-           "(  =^=  )      System clock:    %d Hz\r\n"
-           "(        )     Read global ptr: 0x%08x\r\n"
-           "(    P    )    Read pointer:    0x%08x\r\n"
-           "(  U # L   )   Read argument:   0x%08x\r\n"
-           "(    P      )\r\n"
-           "(           ))))))))))\r\n\r\n",
-           bootmode, rtc_freq, core_freq, rgp, read, priv);
+    if (hart_id == 0) {
+        // Get system parameters
+        uint32_t bootmode = *reg32(&__base_regs, CHESHIRE_BOOT_MODE_REG_OFFSET);
+        uint32_t rtc_freq = *reg32(&__base_regs, CHESHIRE_RTC_FREQ_REG_OFFSET);
+        uint64_t core_freq = clint_get_core_freq(rtc_freq, 2500);
+        rgp = (void *)(uintptr_t)*reg32(&__base_regs, CHESHIRE_SCRATCH_3_REG_OFFSET);
+        uint32_t read = *reg32(&__base_regs, CHESHIRE_SCRATCH_0_REG_OFFSET);
+        void *priv = (void *)(uintptr_t)*reg32(&__base_regs, CHESHIRE_SCRATCH_1_REG_OFFSET);
 
-    // If this is a GPT disk boot, load payload and device tree
-    if (read & 1) {
-        rread = (gpt_read_t)(void *)(uintptr_t)(read & ~1);
-        load_part_or_spin(priv, __BOOT_DTB_TYPE_GUID, __BOOT_ZSL_DTB, "device tree", 64);
-        load_part_or_spin(priv, __BOOT_FW_TYPE_GUID, __BOOT_ZSL_FW, "firmware", 8192);
+        // Initialize UART
+        uart_init(&__base_uart, core_freq, __BOOT_BAUDRATE);
+
+        // Print boot-critical cat, and also parameters
+        printf(" /\\___/\\       Boot mode:       %d\r\n"
+               "( o   o )      Real-time clock: %d Hz\r\n"
+               "(  =^=  )      System clock:    %d Hz\r\n"
+               "(        )     Read global ptr: 0x%08x\r\n"
+               "(    P    )    Read pointer:    0x%08x\r\n"
+               "(  U # L   )   Read argument:   0x%08x\r\n"
+               "(    P      )\r\n"
+               "(           ))))))))))\r\n\r\n",
+               bootmode, rtc_freq, core_freq, rgp, read, priv);
+
+        // If this is a GPT disk boot, load payload and device tree
+        if (read & 1) {
+            rread = (gpt_read_t)(void *)(uintptr_t)(read & ~1);
+            load_part_or_spin(priv, __BOOT_DTB_TYPE_GUID, __BOOT_ZSL_DTB, "device tree", 64);
+            load_part_or_spin(priv, __BOOT_FW_TYPE_GUID, __BOOT_ZSL_FW, "firmware", 8192);
+        }
+
+        // Launch payload
+        printf("[ZSL] Launch firmware at %lx with device tree at %lx\r\n", __BOOT_ZSL_FW,
+               __BOOT_ZSL_DTB);
+        smp_resume();
     }
 
-    // Launch payload
     payload_t fw = __BOOT_ZSL_FW;
-    printf("[ZSL] Launch firmware at %lx with device tree at %lx\r\n", fw, __BOOT_ZSL_DTB);
     fencei();
-    return fw(0, (uintptr_t)__BOOT_ZSL_DTB, 0);
+    return fw(hart_id, (uintptr_t)__BOOT_ZSL_DTB, 0);
 }
 
 // On trap, report relevant CSRs and spin

sw/include/gpt.h

@@ -10,8 +10,6 @@
 
 #include <stdint.h>
 
-extern int boot_next_stage(void *);
-
 typedef int (*gpt_read_t)(void *priv, void *buf, uint64_t addr, uint64_t len);
 
 int gpt_check_signature(gpt_read_t read, void *priv);