Skip to content

Commit

Permalink
powerpc64/bpf: Store temp registers' bpf to ppc mapping
Browse files Browse the repository at this point in the history 
In bpf_jit_build_body(), the mapping of TMP_REG_1 and TMP_REG_2's bpf
register to ppc register is evalulated at every use despite not
changing. Instead, determine the ppc register once and store the result.
This will be more useful when a later patch introduces a more complex
mapping from bpf registers to ppc registers.

Signed-off-by: Jordan Niethe <jniethe5@gmail.com>
  • Loading branch information
@iamjpn @intel-lab-lkp
iamjpn authored and intel-lab-lkp committed Jul 27, 2021
1 parent e73f0f0 commit f15769c
Showing 1 changed file with 69 additions and 94 deletions.
163 changes: 69 additions & 94 deletions arch/powerpc/net/bpf_jit_comp64.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,8 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
u32 code = insn[i].code;
u32 dst_reg = b2p[insn[i].dst_reg];
u32 src_reg = b2p[insn[i].src_reg];
u32 tmp1_reg = b2p[TMP_REG_1];
u32 tmp2_reg = b2p[TMP_REG_2];
s16 off = insn[i].off;
s32 imm = insn[i].imm;
bool func_addr_fixed;
Expand Down Expand Up @@ -337,8 +339,8 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
if (imm >= -32768 && imm < 32768)
EMIT(PPC_RAW_ADDI(dst_reg, dst_reg, IMM_L(imm)));
else {
PPC_LI32(b2p[TMP_REG_1], imm);
EMIT(PPC_RAW_ADD(dst_reg, dst_reg, b2p[TMP_REG_1]));
PPC_LI32(tmp1_reg, imm);
EMIT(PPC_RAW_ADD(dst_reg, dst_reg, tmp1_reg));
}
}
goto bpf_alu32_trunc;
Expand All @@ -354,32 +356,28 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
if (imm >= -32768 && imm < 32768)
EMIT(PPC_RAW_MULI(dst_reg, dst_reg, IMM_L(imm)));
else {
PPC_LI32(b2p[TMP_REG_1], imm);
PPC_LI32(tmp1_reg, imm);
if (BPF_CLASS(code) == BPF_ALU)
EMIT(PPC_RAW_MULW(dst_reg, dst_reg,
b2p[TMP_REG_1]));
EMIT(PPC_RAW_MULW(dst_reg, dst_reg, tmp1_reg));
else
EMIT(PPC_RAW_MULD(dst_reg, dst_reg,
b2p[TMP_REG_1]));
EMIT(PPC_RAW_MULD(dst_reg, dst_reg, tmp1_reg));
}
goto bpf_alu32_trunc;
case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */
case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */
if (BPF_OP(code) == BPF_MOD) {
EMIT(PPC_RAW_DIVWU(b2p[TMP_REG_1], dst_reg, src_reg));
EMIT(PPC_RAW_MULW(b2p[TMP_REG_1], src_reg,
b2p[TMP_REG_1]));
EMIT(PPC_RAW_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]));
EMIT(PPC_RAW_DIVWU(tmp1_reg, dst_reg, src_reg));
EMIT(PPC_RAW_MULW(tmp1_reg, src_reg, tmp1_reg));
EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
} else
EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, src_reg));
goto bpf_alu32_trunc;
case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */
case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */
if (BPF_OP(code) == BPF_MOD) {
EMIT(PPC_RAW_DIVDU(b2p[TMP_REG_1], dst_reg, src_reg));
EMIT(PPC_RAW_MULD(b2p[TMP_REG_1], src_reg,
b2p[TMP_REG_1]));
EMIT(PPC_RAW_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]));
EMIT(PPC_RAW_DIVDU(tmp1_reg, dst_reg, src_reg));
EMIT(PPC_RAW_MULD(tmp1_reg, src_reg, tmp1_reg));
EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
} else
EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg, src_reg));
break;
Expand All @@ -392,35 +390,23 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
else if (imm == 1)
goto bpf_alu32_trunc;

PPC_LI32(b2p[TMP_REG_1], imm);
PPC_LI32(tmp1_reg, imm);
switch (BPF_CLASS(code)) {
case BPF_ALU:
if (BPF_OP(code) == BPF_MOD) {
EMIT(PPC_RAW_DIVWU(b2p[TMP_REG_2],
dst_reg,
b2p[TMP_REG_1]));
EMIT(PPC_RAW_MULW(b2p[TMP_REG_1],
b2p[TMP_REG_1],
b2p[TMP_REG_2]));
EMIT(PPC_RAW_SUB(dst_reg, dst_reg,
b2p[TMP_REG_1]));
EMIT(PPC_RAW_DIVWU(tmp2_reg, dst_reg, tmp1_reg));
EMIT(PPC_RAW_MULW(tmp1_reg, tmp1_reg, tmp2_reg));
EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
} else
EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg,
b2p[TMP_REG_1]));
EMIT(PPC_RAW_DIVWU(dst_reg, dst_reg, tmp1_reg));
break;
case BPF_ALU64:
if (BPF_OP(code) == BPF_MOD) {
EMIT(PPC_RAW_DIVDU(b2p[TMP_REG_2],
dst_reg,
b2p[TMP_REG_1]));
EMIT(PPC_RAW_MULD(b2p[TMP_REG_1],
b2p[TMP_REG_1],
b2p[TMP_REG_2]));
EMIT(PPC_RAW_SUB(dst_reg, dst_reg,
b2p[TMP_REG_1]));
EMIT(PPC_RAW_DIVDU(tmp2_reg, dst_reg, tmp1_reg));
EMIT(PPC_RAW_MULD(tmp1_reg, tmp1_reg, tmp2_reg));
EMIT(PPC_RAW_SUB(dst_reg, dst_reg, tmp1_reg));
} else
EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg,
b2p[TMP_REG_1]));
EMIT(PPC_RAW_DIVDU(dst_reg, dst_reg, tmp1_reg));
break;
}
goto bpf_alu32_trunc;
Expand All @@ -442,8 +428,8 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
EMIT(PPC_RAW_ANDI(dst_reg, dst_reg, IMM_L(imm)));
else {
/* Sign-extended */
PPC_LI32(b2p[TMP_REG_1], imm);
EMIT(PPC_RAW_AND(dst_reg, dst_reg, b2p[TMP_REG_1]));
PPC_LI32(tmp1_reg, imm);
EMIT(PPC_RAW_AND(dst_reg, dst_reg, tmp1_reg));
}
goto bpf_alu32_trunc;
case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
Expand All @@ -454,8 +440,8 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */
if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
/* Sign-extended */
PPC_LI32(b2p[TMP_REG_1], imm);
EMIT(PPC_RAW_OR(dst_reg, dst_reg, b2p[TMP_REG_1]));
PPC_LI32(tmp1_reg, imm);
EMIT(PPC_RAW_OR(dst_reg, dst_reg, tmp1_reg));
} else {
if (IMM_L(imm))
EMIT(PPC_RAW_ORI(dst_reg, dst_reg, IMM_L(imm)));
Expand All @@ -471,8 +457,8 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */
if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
/* Sign-extended */
PPC_LI32(b2p[TMP_REG_1], imm);
EMIT(PPC_RAW_XOR(dst_reg, dst_reg, b2p[TMP_REG_1]));
PPC_LI32(tmp1_reg, imm);
EMIT(PPC_RAW_XOR(dst_reg, dst_reg, tmp1_reg));
} else {
if (IMM_L(imm))
EMIT(PPC_RAW_XORI(dst_reg, dst_reg, IMM_L(imm)));
Expand Down Expand Up @@ -573,24 +559,24 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
switch (imm) {
case 16:
/* Rotate 8 bits left & mask with 0x0000ff00 */
EMIT(PPC_RAW_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 16, 23));
EMIT(PPC_RAW_RLWINM(tmp1_reg, dst_reg, 8, 16, 23));
/* Rotate 8 bits right & insert LSB to reg */
EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 24, 31));
EMIT(PPC_RAW_RLWIMI(tmp1_reg, dst_reg, 24, 24, 31));
/* Move result back to dst_reg */
EMIT(PPC_RAW_MR(dst_reg, b2p[TMP_REG_1]));
EMIT(PPC_RAW_MR(dst_reg, tmp1_reg));
break;
case 32:
/*
* Rotate word left by 8 bits:
* 2 bytes are already in their final position
* -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
*/
EMIT(PPC_RAW_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 0, 31));
EMIT(PPC_RAW_RLWINM(tmp1_reg, dst_reg, 8, 0, 31));
/* Rotate 24 bits and insert byte 1 */
EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 0, 7));
EMIT(PPC_RAW_RLWIMI(tmp1_reg, dst_reg, 24, 0, 7));
/* Rotate 24 bits and insert byte 3 */
EMIT(PPC_RAW_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 16, 23));
EMIT(PPC_RAW_MR(dst_reg, b2p[TMP_REG_1]));
EMIT(PPC_RAW_RLWIMI(tmp1_reg, dst_reg, 24, 16, 23));
EMIT(PPC_RAW_MR(dst_reg, tmp1_reg));
break;
case 64:
/*
Expand All @@ -602,8 +588,8 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
* same across all passes
*/
PPC_BPF_STL(dst_reg, 1, bpf_jit_stack_local(ctx));
EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx)));
EMIT(PPC_RAW_LDBRX(dst_reg, 0, b2p[TMP_REG_1]));
EMIT(PPC_RAW_ADDI(tmp1_reg, 1, bpf_jit_stack_local(ctx)));
EMIT(PPC_RAW_LDBRX(dst_reg, 0, tmp1_reg));
break;
}
break;
Expand Down Expand Up @@ -633,32 +619,32 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */
case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
if (BPF_CLASS(code) == BPF_ST) {
EMIT(PPC_RAW_LI(b2p[TMP_REG_1], imm));
src_reg = b2p[TMP_REG_1];
EMIT(PPC_RAW_LI(tmp1_reg, imm));
src_reg = tmp1_reg;
}
EMIT(PPC_RAW_STB(src_reg, dst_reg, off));
break;
case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
if (BPF_CLASS(code) == BPF_ST) {
EMIT(PPC_RAW_LI(b2p[TMP_REG_1], imm));
src_reg = b2p[TMP_REG_1];
EMIT(PPC_RAW_LI(tmp1_reg, imm));
src_reg = tmp1_reg;
}
EMIT(PPC_RAW_STH(src_reg, dst_reg, off));
break;
case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
if (BPF_CLASS(code) == BPF_ST) {
PPC_LI32(b2p[TMP_REG_1], imm);
src_reg = b2p[TMP_REG_1];
PPC_LI32(tmp1_reg, imm);
src_reg = tmp1_reg;
}
EMIT(PPC_RAW_STW(src_reg, dst_reg, off));
break;
case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
if (BPF_CLASS(code) == BPF_ST) {
PPC_LI32(b2p[TMP_REG_1], imm);
src_reg = b2p[TMP_REG_1];
PPC_LI32(tmp1_reg, imm);
src_reg = tmp1_reg;
}
PPC_BPF_STL(src_reg, dst_reg, off);
break;
Expand All @@ -677,14 +663,14 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
/* *(u32 *)(dst + off) += src */

/* Get EA into TMP_REG_1 */
EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], dst_reg, off));
EMIT(PPC_RAW_ADDI(tmp1_reg, dst_reg, off));
tmp_idx = ctx->idx * 4;
/* load value from memory into TMP_REG_2 */
EMIT(PPC_RAW_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0));
EMIT(PPC_RAW_LWARX(tmp2_reg, 0, tmp1_reg, 0));
/* add value from src_reg into this */
EMIT(PPC_RAW_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg));
EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg));
/* store result back */
EMIT(PPC_RAW_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]));
EMIT(PPC_RAW_STWCX(tmp2_reg, 0, tmp1_reg));
/* we're done if this succeeded */
PPC_BCC_SHORT(COND_NE, tmp_idx);
break;
Expand All @@ -697,11 +683,11 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
}
/* *(u64 *)(dst + off) += src */

EMIT(PPC_RAW_ADDI(b2p[TMP_REG_1], dst_reg, off));
EMIT(PPC_RAW_ADDI(tmp1_reg, dst_reg, off));
tmp_idx = ctx->idx * 4;
EMIT(PPC_RAW_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0));
EMIT(PPC_RAW_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg));
EMIT(PPC_RAW_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]));
EMIT(PPC_RAW_LDARX(tmp2_reg, 0, tmp1_reg, 0));
EMIT(PPC_RAW_ADD(tmp2_reg, tmp2_reg, src_reg));
EMIT(PPC_RAW_STDCX(tmp2_reg, 0, tmp1_reg));
PPC_BCC_SHORT(COND_NE, tmp_idx);
break;

Expand Down Expand Up @@ -879,14 +865,10 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
case BPF_JMP | BPF_JSET | BPF_X:
case BPF_JMP32 | BPF_JSET | BPF_X:
if (BPF_CLASS(code) == BPF_JMP) {
EMIT(PPC_RAW_AND_DOT(b2p[TMP_REG_1], dst_reg,
src_reg));
EMIT(PPC_RAW_AND_DOT(tmp1_reg, dst_reg, src_reg));
} else {
int tmp_reg = b2p[TMP_REG_1];

EMIT(PPC_RAW_AND(tmp_reg, dst_reg, src_reg));
EMIT(PPC_RAW_RLWINM_DOT(tmp_reg, tmp_reg, 0, 0,
31));
EMIT(PPC_RAW_AND(tmp1_reg, dst_reg, src_reg));
EMIT(PPC_RAW_RLWINM_DOT(tmp1_reg, tmp1_reg, 0, 0, 31));
}
break;
case BPF_JMP | BPF_JNE | BPF_K:
Expand Down Expand Up @@ -915,14 +897,12 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
EMIT(PPC_RAW_CMPLDI(dst_reg, imm));
} else {
/* sign-extending load */
PPC_LI32(b2p[TMP_REG_1], imm);
PPC_LI32(tmp1_reg, imm);
/* ... but unsigned comparison */
if (is_jmp32)
EMIT(PPC_RAW_CMPLW(dst_reg,
b2p[TMP_REG_1]));
EMIT(PPC_RAW_CMPLW(dst_reg, tmp1_reg));
else
EMIT(PPC_RAW_CMPLD(dst_reg,
b2p[TMP_REG_1]));
EMIT(PPC_RAW_CMPLD(dst_reg, tmp1_reg));
}
break;
}
Expand All @@ -947,13 +927,11 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
else
EMIT(PPC_RAW_CMPDI(dst_reg, imm));
} else {
PPC_LI32(b2p[TMP_REG_1], imm);
PPC_LI32(tmp1_reg, imm);
if (is_jmp32)
EMIT(PPC_RAW_CMPW(dst_reg,
b2p[TMP_REG_1]));
EMIT(PPC_RAW_CMPW(dst_reg, tmp1_reg));
else
EMIT(PPC_RAW_CMPD(dst_reg,
b2p[TMP_REG_1]));
EMIT(PPC_RAW_CMPD(dst_reg, tmp1_reg));
}
break;
}
Expand All @@ -962,19 +940,16 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, struct codegen_context *
/* andi does not sign-extend the immediate */
if (imm >= 0 && imm < 32768)
/* PPC_ANDI is _only/always_ dot-form */
EMIT(PPC_RAW_ANDI(b2p[TMP_REG_1], dst_reg, imm));
EMIT(PPC_RAW_ANDI(tmp1_reg, dst_reg, imm));
else {
int tmp_reg = b2p[TMP_REG_1];

PPC_LI32(tmp_reg, imm);
PPC_LI32(tmp1_reg, imm);
if (BPF_CLASS(code) == BPF_JMP) {
EMIT(PPC_RAW_AND_DOT(tmp_reg, dst_reg,
tmp_reg));
EMIT(PPC_RAW_AND_DOT(tmp1_reg, dst_reg,
tmp1_reg));
} else {
EMIT(PPC_RAW_AND(tmp_reg, dst_reg,
tmp_reg));
EMIT(PPC_RAW_RLWINM_DOT(tmp_reg, tmp_reg,
0, 0, 31));
EMIT(PPC_RAW_AND(tmp1_reg, dst_reg, tmp1_reg));
EMIT(PPC_RAW_RLWINM_DOT(tmp1_reg, tmp1_reg,
0, 0, 31));
}
}
break;
Expand Down

0 comments on commit f15769c

Please sign in to comment.