@@ -2086,25 +2086,79 @@ static int emit_kfunc_args(const struct bpf_insn *insn, int *ret_size, struct ji
20862086 return stack_size ;
20872087}
20882088
2089+ /* Look up helper prototype since kernel exposes no such function. */
2090+ static const struct bpf_func_proto *
2091+ bpf_jit_get_helper_proto (const s32 imm , const struct jit_ctx * ctx )
2092+ {
2093+ static const struct bpf_func_proto * helpers [__BPF_FUNC_MAX_ID ];
2094+ const struct bpf_func_proto * proto ;
2095+ static bool scanned = false;
2096+ int i ;
2097+
2098+ if (!scanned ) {
2099+ for (i = 1 ; i < ARRAY_SIZE (helpers ); i ++ ) {
2100+ helpers [i ]= bpf_base_func_proto (i , ctx -> prog );
2101+ //FIXME pr_info("helpers: id=%d proto=%px imm=%d\n",
2102+ //FIXME i, helpers[i], helpers[i] ? helpers[i]->func - __bpf_call_base : 0);
2103+ }
2104+ scanned = true;
2105+ }
2106+
2107+ for (i = 1 ; i < ARRAY_SIZE (helpers ); i ++ ) {
2108+ proto = helpers [i ];
2109+ if (proto && imm == (proto -> func - __bpf_call_base ))
2110+ return proto ;
2111+ }
2112+
2113+ return NULL ;
2114+ }
2115+
20892116/* Set up callee args for helper function, return used stack. */
2090- static inline int emit_helper_args (struct jit_ctx * ctx )
2117+ static int emit_helper_args (const s32 imm , struct jit_ctx * ctx )
20912118{
2119+ const struct bpf_func_proto * proto ;
20922120 const s8 * r0 = bpf2a32 [BPF_REG_0 ];
20932121 const s8 * r1 = bpf2a32 [BPF_REG_1 ];
20942122 const s8 * r2 = bpf2a32 [BPF_REG_2 ];
20952123 const s8 * r3 = bpf2a32 [BPF_REG_3 ];
20962124 const s8 * r4 = bpf2a32 [BPF_REG_4 ];
20972125 const s8 * r5 = bpf2a32 [BPF_REG_5 ];
2126+ int nargs = 5 , stack = 0 ;
2127+
2128+ proto = bpf_jit_get_helper_proto (imm , ctx );
2129+ if (!proto ) {
2130+ //FIXME pr_warn("emit_helper_args: no proto for imm=%d\n", imm);
2131+ goto no_proto ;
2132+ }
20982133
2099- /* Move BPF_R1, BPF_R2 -> ARM_R0:ARM_R1, ARM_R2:ARM_R3 */
2100- emit_a32_mov_r64 (true, r0 , r1 , ctx );
2101- emit_a32_mov_r64 (true, r1 , r2 , ctx );
2102- /* Remaining 3 args on stack */
2103- emit_push_r64 (r5 , ctx );
2104- emit_push_r64 (r4 , ctx );
2105- emit_push_r64 (r3 , ctx );
2134+ for (nargs = 0 ; nargs < 5 ; nargs ++ )
2135+ if (proto -> arg_type [nargs ] == ARG_DONTCARE )
2136+ break ;
2137+ //FIXME pr_info("emit_helper_args: nargs=%d for imm=%d\n", nargs, imm);
2138+ no_proto :
2139+ switch (nargs ) {
2140+ case 5 :
2141+ emit_push_r64 (r5 , ctx );
2142+ stack += sizeof (u64 );
2143+ fallthrough ;
2144+ case 4 :
2145+ emit_push_r64 (r4 , ctx );
2146+ stack += sizeof (u64 );
2147+ fallthrough ;
2148+ case 3 :
2149+ emit_push_r64 (r3 , ctx );
2150+ stack += sizeof (u64 );
2151+ fallthrough ;
2152+ case 2 :
2153+ case 1 :
2154+ emit_a32_mov_r64 (true, r0 , r1 , ctx );
2155+ if (nargs > 1 )
2156+ emit_a32_mov_r64 (true, r1 , r2 , ctx );
2157+ default :
2158+ break ;
2159+ }
21062160
2107- return 3 * sizeof ( u64 ) ;
2161+ return stack ;
21082162}
21092163
21102164/*
@@ -2634,20 +2688,21 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
26342688 case BPF_PSEUDO_KFUNC_CALL :
26352689 /* BPF kfunc call */
26362690 stack_size = emit_kfunc_args (insn , & ret_size , ctx );
2637- if (stack_size < 0 )
2638- return stack_size ;
26392691 break ;
26402692 case BPF_PSEUDO_CALL : /* BPF to BPF call, same ABI */
26412693 break ;
26422694 case 0 : /* BPF helper call, u64 args & regs */
2643- stack_size = emit_helper_args (ctx );
2695+ stack_size = emit_helper_args (insn -> imm , ctx );
26442696 break ;
26452697 default :
26462698 pr_err_once ("unknown BPF_CALL type 0x%02x\n" ,
26472699 insn -> src_reg );
26482700 return - EINVAL ;
26492701 }
26502702
2703+ if (stack_size < 0 )
2704+ return stack_size ;
2705+
26512706 /* Use fixed number of insns in case func_addr varies
26522707 * with pass (e.g. bpf2bpf call).
26532708 */
0 commit comments