aarch64: Add support for "near" in LoadExtName

cranelift/codegen/src/binemit/mod.rs

@@ -82,6 +82,11 @@ pub enum Reloc {
     /// This is equivalent to `R_AARCH64_ADR_GOT_PAGE` (311) in the  [aaelf64](https://github.com/ARM-software/abi-aa/blob/2bcab1e3b22d55170c563c3c7940134089176746/aaelf64/aaelf64.rst#static-aarch64-relocations)
     Aarch64AdrGotPage21,
 
+    /// Equivalent of `R_AARCH64_ADR_PREL_PG_HI21`.
+    Aarch64AdrPrelPgHi21,
+    /// Equivalent of `R_AARCH64_ADD_ABS_LO12_NC`.
+    Aarch64AddAbsLo12Nc,
+
     /// AArch64 GOT Low bits
 
     /// Set the LD/ST immediate field to bits 11:3 of X. No overflow check; check that X&7 = 0
@@ -155,6 +160,8 @@ impl fmt::Display for Reloc {
             Self::Aarch64TlsDescCall => write!(f, "Aarch64TlsDescCall"),
             Self::Aarch64AdrGotPage21 => write!(f, "Aarch64AdrGotPage21"),
             Self::Aarch64Ld64GotLo12Nc => write!(f, "Aarch64AdrGotLo12Nc"),
+            Self::Aarch64AdrPrelPgHi21 => write!(f, "Aarch64AdrPrelPgHi21"),
+            Self::Aarch64AddAbsLo12Nc => write!(f, "Aarch64AddAbsLo12Nc"),
             Self::S390xTlsGd64 => write!(f, "TlsGd64"),
             Self::S390xTlsGdCall => write!(f, "TlsGdCall"),
             Self::PulleyCallIndirectHost => write!(f, "PulleyCallIndirectHost"),

cranelift/codegen/src/isa/aarch64/inst.isle

@@ -918,7 +918,14 @@
         (rtmp2 WritableReg))
 
        ;; Load an inline symbol reference.
-       (LoadExtName
+       (LoadExtNameGot
+        (rd WritableReg)
+        (name BoxExternalName))
+       (LoadExtNameNear
+        (rd WritableReg)
+        (name BoxExternalName)
+        (offset i64))
+       (LoadExtNameFar
         (rd WritableReg)
         (name BoxExternalName)
         (offset i64))
@@ -3808,11 +3815,51 @@
             (_ Unit (emit (MInst.VecLoadReplicate dst src size flags))))
         dst))
 
-;; Helper for emitting `MInst.LoadExtName` instructions.
-(decl load_ext_name (BoxExternalName i64) Reg)
-(rule (load_ext_name extname offset)
+(decl pure is_pic () bool)
+(extern constructor is_pic is_pic)
+
+;; Helper loading an external name into a register via `MInst.LoadExt*`
+(decl load_ext_name (BoxExternalName i64 RelocDistance) Reg)
+
+;; When `is_pic` is true all names are referenced through the GOT. Note that
+;; the relocation is applied to the address of the GOT itself so the offset
+;; requested must be manually added in (or skipped if it's 0).
+(rule (load_ext_name name offset _)
+  (if-let true (is_pic))
+  (add $I64 (load_ext_name_got name) (imm $I64 (ImmExtend.Zero) (i64_cast_unsigned offset))))
+(rule 1 (load_ext_name name 0 _)
+  (if-let true (is_pic))
+  (load_ext_name_got name))
+
+;; Relocations that are "near" do an `adr;add` combo.
+(rule 1 (load_ext_name name offset (RelocDistance.Near))
+  (if-let false (is_pic))
+  (load_ext_name_near name offset))
+
+;; Relocations that are "far" do an `Abs8` relocation.
+(rule 1 (load_ext_name name offset (RelocDistance.Far))
+  (if-let false (is_pic))
+  (load_ext_name_far name offset))
+
+;; Helper for emitting `MInst.LoadExtNameGot` instructions.
+(decl load_ext_name_got (BoxExternalName) Reg)
+(rule (load_ext_name_got extname)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.LoadExtNameGot dst extname))))
+        dst))
+
+;; Helper for emitting `MInst.LoadExtNameNear` instructions.
+(decl load_ext_name_near (BoxExternalName i64) Reg)
+(rule (load_ext_name_near extname offset)
+      (let ((dst WritableReg (temp_writable_reg $I64))
+            (_ Unit (emit (MInst.LoadExtNameNear dst extname offset))))
+        dst))
+
+;; Helper for emitting `MInst.LoadExtNameFar` instructions.
+(decl load_ext_name_far (BoxExternalName i64) Reg)
+(rule (load_ext_name_far extname offset)
       (let ((dst WritableReg (temp_writable_reg $I64))
-            (_ Unit (emit (MInst.LoadExtName dst extname offset))))
+            (_ Unit (emit (MInst.LoadExtNameFar dst extname offset))))
         dst))
 
 ;; Lower the address of a load or a store.

cranelift/codegen/src/isa/aarch64/inst/emit.rs

@@ -3230,56 +3230,82 @@ impl MachInstEmit for Inst {
                 // disable the worst-case-size check in this case.
                 start_off = sink.cur_offset();
             }
-            &Inst::LoadExtName {
+            &Inst::LoadExtNameGot { rd, ref name } => {
+                // See this CE Example for the variations of this with and without BTI & PAUTH
+                // https://godbolt.org/z/ncqjbbvvn
+                //
+                // Emit the following code:
+                //   adrp    rd, :got:X
+                //   ldr     rd, [rd, :got_lo12:X]
+
+                // adrp rd, symbol
+                sink.add_reloc(Reloc::Aarch64AdrGotPage21, &**name, 0);
+                let inst = Inst::Adrp { rd, off: 0 };
+                inst.emit(sink, emit_info, state);
+
+                // ldr rd, [rd, :got_lo12:X]
+                sink.add_reloc(Reloc::Aarch64Ld64GotLo12Nc, &**name, 0);
+                let inst = Inst::ULoad64 {
+                    rd,
+                    mem: AMode::reg(rd.to_reg()),
+                    flags: MemFlags::trusted(),
+                };
+                inst.emit(sink, emit_info, state);
+            }
+            &Inst::LoadExtNameNear {
                 rd,
                 ref name,
                 offset,
             } => {
-                if emit_info.0.is_pic() {
-                    // See this CE Example for the variations of this with and without BTI & PAUTH
-                    // https://godbolt.org/z/ncqjbbvvn
-                    //
-                    // Emit the following code:
-                    //   adrp    rd, :got:X
-                    //   ldr     rd, [rd, :got_lo12:X]
-
-                    // adrp rd, symbol
-                    sink.add_reloc(Reloc::Aarch64AdrGotPage21, &**name, 0);
-                    let inst = Inst::Adrp { rd, off: 0 };
-                    inst.emit(sink, emit_info, state);
+                // Emit the following code:
+                //   adrp    rd, X
+                //   add     rd, rd, :lo12:X
+                //
+                // See https://godbolt.org/z/855KEvM5r for an example.
 
-                    // ldr rd, [rd, :got_lo12:X]
-                    sink.add_reloc(Reloc::Aarch64Ld64GotLo12Nc, &**name, 0);
-                    let inst = Inst::ULoad64 {
-                        rd,
-                        mem: AMode::reg(rd.to_reg()),
-                        flags: MemFlags::trusted(),
-                    };
-                    inst.emit(sink, emit_info, state);
-                } else {
-                    // With absolute offsets we set up a load from a preallocated space, and then jump
-                    // over it.
-                    //
-                    // Emit the following code:
-                    //   ldr     rd, #8
-                    //   b       #0x10
-                    //   <8 byte space>
-
-                    let inst = Inst::ULoad64 {
-                        rd,
-                        mem: AMode::Label {
-                            label: MemLabel::PCRel(8),
-                        },
-                        flags: MemFlags::trusted(),
-                    };
-                    inst.emit(sink, emit_info, state);
-                    let inst = Inst::Jump {
-                        dest: BranchTarget::ResolvedOffset(12),
-                    };
-                    inst.emit(sink, emit_info, state);
-                    sink.add_reloc(Reloc::Abs8, &**name, offset);
-                    sink.put8(0);
-                }
+                // adrp rd, symbol
+                sink.add_reloc(Reloc::Aarch64AdrPrelPgHi21, &**name, offset);
+                let inst = Inst::Adrp { rd, off: 0 };
+                inst.emit(sink, emit_info, state);
+
+                // add rd, rd, :lo12:X
+                sink.add_reloc(Reloc::Aarch64AddAbsLo12Nc, &**name, offset);
+                let inst = Inst::AluRRImm12 {
+                    alu_op: ALUOp::Add,
+                    size: OperandSize::Size64,
+                    rd: rd,
+                    rn: rd.to_reg(),
+                    imm12: Imm12::ZERO,
+                };
+                inst.emit(sink, emit_info, state);
+            }
+            &Inst::LoadExtNameFar {
+                rd,
+                ref name,
+                offset,
+            } => {
+                // With absolute offsets we set up a load from a preallocated space, and then jump
+                // over it.
+                //
+                // Emit the following code:
+                //   ldr     rd, #8
+                //   b       #0x10
+                //   <8 byte space>
+
+                let inst = Inst::ULoad64 {
+                    rd,
+                    mem: AMode::Label {
+                        label: MemLabel::PCRel(8),
+                    },
+                    flags: MemFlags::trusted(),
+                };
+                inst.emit(sink, emit_info, state);
+                let inst = Inst::Jump {
+                    dest: BranchTarget::ResolvedOffset(12),
+                };
+                inst.emit(sink, emit_info, state);
+                sink.add_reloc(Reloc::Abs8, &**name, offset);
+                sink.put8(0);
             }
             &Inst::LoadAddr { rd, ref mem } => {
                 let mem = mem.clone();

cranelift/codegen/src/isa/aarch64/inst/imms.rs

@@ -270,6 +270,12 @@ pub struct Imm12 {
 }
 
 impl Imm12 {
+    /// Handy 0-value constant.
+    pub const ZERO: Imm12 = Imm12 {
+        bits: 0,
+        shift12: false,
+    };
+
     /// Compute a Imm12 from raw bits, if possible.
     pub fn maybe_from_u64(val: u64) -> Option<Imm12> {
         if val & !0xfff == 0 {

cranelift/codegen/src/isa/aarch64/inst/mod.rs

@@ -879,7 +879,9 @@ fn aarch64_get_operands(inst: &mut Inst, collector: &mut impl OperandVisitor) {
             collector.reg_early_def(rtmp1);
             collector.reg_early_def(rtmp2);
         }
-        Inst::LoadExtName { rd, .. } => {
+        Inst::LoadExtNameGot { rd, .. }
+        | Inst::LoadExtNameNear { rd, .. }
+        | Inst::LoadExtNameFar { rd, .. } => {
             collector.reg_def(rd);
         }
         Inst::LoadAddr { rd, mem } => {
@@ -2746,13 +2748,25 @@ impl Inst {
                     targets
                 )
             }
-            &Inst::LoadExtName {
+            &Inst::LoadExtNameGot { rd, ref name } => {
+                let rd = pretty_print_reg(rd.to_reg());
+                format!("load_ext_name_got {rd}, {name:?}")
+            }
+            &Inst::LoadExtNameNear {
+                rd,
+                ref name,
+                offset,
+            } => {
+                let rd = pretty_print_reg(rd.to_reg());
+                format!("load_ext_name_near {rd}, {name:?}+{offset}")
+            }
+            &Inst::LoadExtNameFar {
                 rd,
                 ref name,
                 offset,
             } => {
                 let rd = pretty_print_reg(rd.to_reg());
-                format!("load_ext_name {rd}, {name:?}+{offset}")
+                format!("load_ext_name_far {rd}, {name:?}+{offset}")
             }
             &Inst::LoadAddr { rd, ref mem } => {
                 // TODO: we really should find a better way to avoid duplication of

cranelift/codegen/src/isa/aarch64/lower.isle

@@ -2484,13 +2484,13 @@
 
 ;;;; Rules for `func_addr` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(rule (lower (func_addr (func_ref_data _ extname _)))
-      (load_ext_name (box_external_name extname) 0))
+(rule (lower (func_addr (func_ref_data _ extname dist)))
+      (load_ext_name (box_external_name extname) 0 dist))
 
 ;;;; Rules for `symbol_value` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(rule (lower (symbol_value (symbol_value_data extname _ offset)))
-      (load_ext_name (box_external_name extname) offset))
+(rule (lower (symbol_value (symbol_value_data extname dist offset)))
+      (load_ext_name (box_external_name extname) offset dist))
 
 ;;; Rules for `get_{frame,stack}_pointer` and `get_return_address` ;;;;;;;;;;;;;
 
@@ -2506,7 +2506,7 @@
 ;; Rules for `call` and `call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Direct call to an in-range function.
-(rule 1 (lower (call (func_ref_data sig_ref name (reloc_distance_near)) args))
+(rule 1 (lower (call (func_ref_data sig_ref name (RelocDistance.Near)) args))
       (let ((output ValueRegsVec (gen_call_output sig_ref))
             (abi Sig (abi_sig sig_ref))
             (uses CallArgList (gen_call_args abi args))
@@ -2516,12 +2516,12 @@
         output))
 
 ;; Direct call to an out-of-range function (implicitly via pointer).
-(rule (lower (call (func_ref_data sig_ref name _) args))
+(rule (lower (call (func_ref_data sig_ref name dist) args))
       (let ((output ValueRegsVec (gen_call_output sig_ref))
             (abi Sig (abi_sig sig_ref))
             (uses CallArgList (gen_call_args abi args))
             (defs CallRetList (gen_call_rets abi output))
-            (target Reg (load_ext_name name 0))
+            (target Reg (load_ext_name name 0 dist))
             (info BoxCallIndInfo (gen_call_ind_info abi target uses defs (try_call_none)))
             (_ Unit (emit_side_effect (call_ind_impl info))))
         output))
@@ -2540,7 +2540,7 @@
 ;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Direct call to an in-range function.
-(rule 1 (lower_branch (try_call (func_ref_data sig_ref name (reloc_distance_near)) args et) targets)
+(rule 1 (lower_branch (try_call (func_ref_data sig_ref name (RelocDistance.Near)) args et) targets)
       (let ((abi Sig (abi_sig sig_ref))
             (trycall OptionTryCallInfo (try_call_info et targets))
             (uses CallArgList (gen_call_args abi args))
@@ -2549,12 +2549,12 @@
         (emit_side_effect (call_impl info))))
 
 ;; Direct call to an out-of-range function (implicitly via pointer).
-(rule (lower_branch (try_call (func_ref_data sig_ref name _) args et) targets)
+(rule (lower_branch (try_call (func_ref_data sig_ref name dist) args et) targets)
       (let ((abi Sig (abi_sig sig_ref))
             (trycall OptionTryCallInfo (try_call_info et targets))
             (uses CallArgList (gen_call_args abi args))
             (defs CallRetList (gen_try_call_rets abi))
-            (target Reg (load_ext_name name 0))
+            (target Reg (load_ext_name name 0 dist))
             (info BoxCallIndInfo (gen_call_ind_info abi target uses defs trycall)))
         (emit_side_effect (call_ind_impl info))))
 
@@ -2578,17 +2578,17 @@
 ;;;; Rules for `return_call` and `return_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Direct call to an in-range function.
-(rule 1 (lower (return_call (func_ref_data sig_ref name (reloc_distance_near)) args))
+(rule 1 (lower (return_call (func_ref_data sig_ref name (RelocDistance.Near)) args))
       (let ((abi Sig (abi_sig sig_ref))
             (uses CallArgList (gen_return_call_args abi args))
             (info BoxReturnCallInfo (gen_return_call_info abi name uses)))
         (side_effect (return_call_impl info))))
 
 ;; Direct call to an out-of-range function (implicitly via pointer).
-(rule (lower (return_call (func_ref_data sig_ref name _) args))
+(rule (lower (return_call (func_ref_data sig_ref name dist) args))
       (let ((abi Sig (abi_sig sig_ref))
             (uses CallArgList (gen_return_call_args abi args))
-            (target Reg (load_ext_name name 0))
+            (target Reg (load_ext_name name 0 dist))
             (info BoxReturnCallIndInfo (gen_return_call_ind_info abi target uses)))
         (side_effect (return_call_ind_impl info))))
 

cranelift/codegen/src/isa/aarch64/lower/isle.rs

@@ -852,4 +852,8 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {
             ShiftOpShiftImm::maybe_from_shift(shift.value().into()).unwrap(),
         )
     }
+
+    fn is_pic(&mut self) -> bool {
+        self.backend.flags.is_pic()
+    }
 }

cranelift/codegen/src/isa/pulley_shared/lower.isle

@@ -141,7 +141,7 @@
 ;; Rules for `call` and `call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Direct call to a pulley function.
-(rule 1 (lower (call (func_ref_data sig_ref name (reloc_distance_near)) args))
+(rule 1 (lower (call (func_ref_data sig_ref name (RelocDistance.Near)) args))
       (let ((output ValueRegsVec (gen_call_output sig_ref))
             (abi Sig (abi_sig sig_ref))
             (uses CallArgList (gen_call_args abi args))
@@ -174,7 +174,7 @@
 ;;;; Rules for `try_call` and `try_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Direct call to a pulley function.
-(rule 1 (lower_branch (try_call (func_ref_data sig_ref name (reloc_distance_near)) args et) targets)
+(rule 1 (lower_branch (try_call (func_ref_data sig_ref name (RelocDistance.Near)) args et) targets)
       (let ((abi Sig (abi_sig sig_ref))
             (trycall OptionTryCallInfo (try_call_info et targets))
             (uses CallArgList (gen_call_args abi args))
@@ -205,7 +205,7 @@
 ;;;; Rules for `return_call` and `return_call_indirect` ;;;;;;;;;;;;;;;;;;;;;;;;
 
 ;; Direct call to a pulley function.
-(rule 1 (lower (return_call (func_ref_data sig_ref name (reloc_distance_near)) args))
+(rule 1 (lower (return_call (func_ref_data sig_ref name (RelocDistance.Near)) args))
       (let ((abi Sig (abi_sig sig_ref))
             (uses CallArgList (gen_return_call_args abi args))
             (info BoxReturnCallInfo (gen_return_call_info abi name uses)))