#ifdef __aarch64__ #include "ibm.h" #include "codegen.h" #include "codegen_backend.h" #include "codegen_backend_arm64_defs.h" #include "codegen_ir_defs.h" static inline void codegen_addlong(codeblock_t *block, uint32_t val) { *(uint32_t *)&block->data[block_pos] = val; block_pos += 4; if (block_pos >= BLOCK_MAX) { fatal("codegen_addlong over! %i\n", block_pos); CPU_BLOCK_END(); } } #define Rt(x) (x) #define Rd(x) (x) #define Rn(x) ((x) << 5) #define Rt2(x) ((x) << 10) #define Rm(x) ((x) << 16) #define shift_imm6(x) ((x) << 10) #define DATA_OFFSET_UP (1 << 23) #define DATA_OFFSET_DOWN (0 << 23) #define COND_SHIFT 28 #define COND_NE (0x1 << COND_SHIFT) #define COND_AL (0xe << COND_SHIFT) #define OPCODE_SHIFT 24 #define OPCODE_ADD_IMM (0x11 << OPCODE_SHIFT) #define OPCODE_AND_IMM (0x11 << OPCODE_SHIFT) #define OPCODE_CBNZ (0xb5 << OPCODE_SHIFT) #define OPCODE_LDR_LITERAL_W (0x18 << OPCODE_SHIFT) #define OPCODE_LDR_LITERAL_X (0x58 << OPCODE_SHIFT) #define OPCODE_MOVK_W (0x0e5 << 23) #define OPCODE_MOVZ_W (0x0a5 << 23) #define OPCODE_LDR_IMM_W (0x2e5 << 22) #define OPCODE_LDP_POSTIDX_X (0x2a3 << 22) #define OPCODE_STP_PREIDX_X (0x2a6 << 22) #define OPCODE_STR_IMM_W (0x2e4 << 22) #define OPCODE_STR_IMM_Q (0x3e4 << 22) #define OPCODE_STRB_IMM (0x0e4 << 22) #define OPCODE_ADD_LSL (0x058 << 21) #define OPCODE_AND_LSL (0x050 << 21) #define OPCODE_ORR_LSL (0x150 << 21) #define OPCODE_BLR (0xd63f0000) #define OPCODE_NOP (0xd503201f) #define OPCODE_RET (0xd65f0000) #define DATPROC_SHIFT(sh) (sh << 10) #define DATPROC_IMM_SHIFT(sh) (sh << 22) #define MOV_WIDE_HW(hw) (hw << 21) #define IMM7_X(imm_data) (((imm_data >> 3) & 0x7f) << 15) #define IMM12(imm_data) ((imm_data) << 10) #define IMM16(imm_data) ((imm_data) << 5) #define OFFSET19(offset) (((offset >> 2) << 5) & 0x00ffffe0) #define OFFSET12_B(offset) (offset << 10) #define OFFSET12_W(offset) ((offset >> 2) << 10) #define OFFSET12_Q(offset) ((offset >> 3) << 10) static int literal_offset = 0; void codegen_reset_literal_pool(codeblock_t *block) { literal_offset = 0; } /*Returns true if offset fits into 19 bits*/ static int offset_is_19bit(int offset) { if (offset >= (1 << (18+2))) return 0; if (offset < -(1 << (18+2))) return 0; return 1; } #define in_range7_x(offset) (((offset) >= -0x200) && ((offset) < (0x200)) && !((offset) & 7)) #define in_range12_b(offset) (((offset) >= 0) && ((offset) < 0x1000)) #define in_range12_w(offset) (((offset) >= 0) && ((offset) < 0x4000) && !((offset) & 3)) #define in_range12_q(offset) (((offset) >= 0) && ((offset) < 0x8000) && !((offset) & 7)) int add_literal(codeblock_t *block, uint32_t data) { if (literal_offset >= 4096) fatal("add_literal - literal pool full\n"); *(uint32_t *)&block->data[ARM_LITERAL_POOL_OFFSET + literal_offset] = data; literal_offset += 4; return literal_offset - 4; } int add_literal_q(codeblock_t *block, uint64_t data) { if (literal_offset & 4) literal_offset += 4; if (literal_offset >= 4096-4) fatal("add_literal - literal pool full\n"); *(uint64_t *)&block->data[ARM_LITERAL_POOL_OFFSET + literal_offset] = data; literal_offset += 8; return literal_offset - 8; } static inline int imm_is_imm16(uint32_t imm_data) { if (!(imm_data & 0xffff0000) || !(imm_data & 0x0000ffff)) return 1; return 0; } static inline int imm_is_imm12(uint32_t imm_data) { if (!(imm_data & 0xfffff000) || !(imm_data & 0xff000fff)) return 1; return 0; } void host_arm64_ADD_IMM(codeblock_t *block, int dst_reg, int src_n_reg, uint32_t imm_data) { if (!(imm_data & 0xff000000)) { if (imm_data & 0xfff) codegen_addlong(block, OPCODE_ADD_IMM | Rd(dst_reg) | Rn(src_n_reg) | IMM12(imm_data & 0xfff) | DATPROC_IMM_SHIFT(0)); if (imm_data & 0xfff000) codegen_addlong(block, OPCODE_ADD_IMM | Rd(dst_reg) | Rn(src_n_reg) | IMM12((imm_data >> 12) & 0xfff) | DATPROC_IMM_SHIFT(1)); } else { host_arm64_MOVZ_IMM(block, REG_W16, imm_data & 0xffff); host_arm64_MOVK_IMM(block, REG_W16, imm_data & 0xffff0000); codegen_addlong(block, OPCODE_ADD_LSL | Rd(dst_reg) | Rn(src_n_reg) | Rm(REG_W16) | DATPROC_SHIFT(0)); } } void host_arm64_ADD_REG(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg, int shift) { codegen_addlong(block, OPCODE_ADD_LSL | Rd(dst_reg) | Rn(src_n_reg) | Rm(src_m_reg) | DATPROC_SHIFT(shift)); } void host_arm64_AND_IMM(codeblock_t *block, int dst_reg, int src_n_reg, uint32_t imm_data) { host_arm64_mov_imm(block, REG_W16, imm_data); codegen_addlong(block, OPCODE_AND_LSL | Rd(dst_reg) | Rn(src_n_reg) | Rm(REG_W16) | DATPROC_SHIFT(0)); } void host_arm64_BLR(codeblock_t *block, int addr_reg) { codegen_addlong(block, OPCODE_BLR | Rn(addr_reg)); } void host_arm64_CBNZ(codeblock_t *block, int reg, uintptr_t dest) { int offset = dest - (uintptr_t)&block->data[block_pos]; if (!offset_is_19bit(offset)) fatal("host_arm64_CBNZ - offset out of range %x\n", offset); codegen_addlong(block, OPCODE_CBNZ | OFFSET19(offset) | Rt(reg)); } void host_arm64_LDP_POSTIDX_X(codeblock_t *block, int src_reg1, int src_reg2, int base_reg, int offset) { if (!in_range7_x(offset)) fatal("host_arm64_LDP_POSTIDX out of range7 %i\n", offset); codegen_addlong(block, OPCODE_LDP_POSTIDX_X | IMM7_X(offset) | Rn(base_reg) | Rt(src_reg1) | Rt2(src_reg2)); } void host_arm64_LDR_IMM_W(codeblock_t *block, int dest_reg, int base_reg, int offset) { if (!in_range12_w(offset)) fatal("host_arm64_LDR_IMM_W out of range12 %i\n", offset); codegen_addlong(block, OPCODE_LDR_IMM_W | OFFSET12_W(offset) | Rn(base_reg) | Rt(dest_reg)); } void host_arm64_LDR_LITERAL_W(codeblock_t *block, int dest_reg, int literal_offset) { int offset = (ARM_LITERAL_POOL_OFFSET + literal_offset) - block_pos; if (!offset_is_19bit(offset)) fatal("host_arm64_CBNZ - offset out of range %x\n", offset); codegen_addlong(block, OPCODE_LDR_LITERAL_W | OFFSET19(offset) | Rt(dest_reg)); } void host_arm64_LDR_LITERAL_X(codeblock_t *block, int dest_reg, int literal_offset) { int offset = (ARM_LITERAL_POOL_OFFSET + literal_offset) - block_pos; if (!offset_is_19bit(offset)) fatal("host_arm64_CBNZ - offset out of range %x\n", offset); codegen_addlong(block, OPCODE_LDR_LITERAL_X | OFFSET19(offset) | Rt(dest_reg)); } void host_arm64_MOV_REG(codeblock_t *block, int dst_reg, int src_m_reg, int shift) { codegen_addlong(block, OPCODE_ORR_LSL | Rd(dst_reg) | Rn(REG_WZR) | Rm(src_m_reg) | DATPROC_SHIFT(shift)); } void host_arm64_MOVZ_IMM(codeblock_t *block, int reg, uint32_t imm_data) { int hw; if (!imm_is_imm16(imm_data)) fatal("MOVZ_IMM - imm not representable %08x\n", imm_data); hw = (imm_data & 0xffff0000) ? 1 : 0; if (hw) imm_data >>= 16; codegen_addlong(block, OPCODE_MOVZ_W | MOV_WIDE_HW(hw) | IMM16(imm_data) | Rd(reg)); } void host_arm64_MOVK_IMM(codeblock_t *block, int reg, uint32_t imm_data) { int hw; if (!imm_is_imm16(imm_data)) fatal("MOVK_IMM - imm not representable %08x\n", imm_data); hw = (imm_data & 0xffff0000) ? 1 : 0; if (hw) imm_data >>= 16; codegen_addlong(block, OPCODE_MOVK_W | MOV_WIDE_HW(hw) | IMM16(imm_data) | Rd(reg)); } void host_arm64_NOP(codeblock_t *block) { codegen_addlong(block, OPCODE_NOP); } void host_arm64_RET(codeblock_t *block, int reg) { codegen_addlong(block, OPCODE_RET | Rn(reg)); } void host_arm64_STP_PREIDX_X(codeblock_t *block, int src_reg1, int src_reg2, int base_reg, int offset) { if (!in_range7_x(offset)) fatal("host_arm64_STP_PREIDX out of range7 %i\n", offset); codegen_addlong(block, OPCODE_STP_PREIDX_X | IMM7_X(offset) | Rn(base_reg) | Rt(src_reg1) | Rt2(src_reg2)); } void host_arm64_STR_IMM_W(codeblock_t *block, int dest_reg, int base_reg, int offset) { if (!in_range12_w(offset)) fatal("host_arm64_STR_IMM_W out of range12 %i\n", offset); codegen_addlong(block, OPCODE_STR_IMM_W | OFFSET12_W(offset) | Rn(base_reg) | Rt(dest_reg)); } void host_arm64_STR_IMM_Q(codeblock_t *block, int dest_reg, int base_reg, int offset) { if (!in_range12_q(offset)) fatal("host_arm64_STR_IMM_W out of range12 %i\n", offset); codegen_addlong(block, OPCODE_STR_IMM_Q | OFFSET12_Q(offset) | Rn(base_reg) | Rt(dest_reg)); } void host_arm64_STRB_IMM(codeblock_t *block, int dest_reg, int base_reg, int offset) { if (!in_range12_b(offset)) fatal("host_arm64_STRB_IMM out of range12 %i\n", offset); codegen_addlong(block, OPCODE_STRB_IMM | OFFSET12_B(offset) | Rn(base_reg) | Rt(dest_reg)); } void host_arm64_call(codeblock_t *block, void *dst_addr) { int offset = add_literal_q(block, (uintptr_t)dst_addr); host_arm64_LDR_LITERAL_X(block, REG_X16, offset); host_arm64_BLR(block, REG_X16); } void host_arm64_mov_imm(codeblock_t *block, int reg, uint32_t imm_data) { if (imm_is_imm16(imm_data)) host_arm64_MOVZ_IMM(block, reg, imm_data); else { host_arm64_MOVZ_IMM(block, reg, imm_data & 0xffff); host_arm64_MOVK_IMM(block, reg, imm_data & 0xffff0000); } } static int codegen_ADD(codeblock_t *block, uop_t *uop) { host_arm64_ADD_REG(block, uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real, 0); return 0; } static int codegen_ADD_IMM(codeblock_t *block, uop_t *uop) { host_arm64_ADD_IMM(block, uop->dest_reg_a_real, uop->src_reg_a_real, uop->imm_data); return 0; } static int codegen_ADD_LSHIFT(codeblock_t *block, uop_t *uop) { host_arm64_ADD_REG(block, uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real, uop->imm_data); return 0; } static int codegen_AND_IMM(codeblock_t *block, uop_t *uop) { host_arm64_AND_IMM(block, uop->dest_reg_a_real, uop->src_reg_a_real, uop->imm_data); return 0; } static int codegen_CALL_INSTRUCTION_FUNC(codeblock_t *block, uop_t *uop) { host_arm64_call(block, uop->p); host_arm64_CBNZ(block, REG_X0, (uintptr_t)&block->data[BLOCK_EXIT_OFFSET]); return 0; } static int codegen_LOAD_FUNC_ARG0_IMM(codeblock_t *block, uop_t *uop) { host_arm64_mov_imm(block, REG_ARG0, uop->imm_data); return 0; } static int codegen_LOAD_FUNC_ARG1_IMM(codeblock_t *block, uop_t *uop) { host_arm64_mov_imm(block, REG_ARG1, uop->imm_data); return 0; } static int codegen_LOAD_FUNC_ARG2_IMM(codeblock_t *block, uop_t *uop) { host_arm64_mov_imm(block, REG_ARG2, uop->imm_data); return 0; } static int codegen_LOAD_FUNC_ARG3_IMM(codeblock_t *block, uop_t *uop) { host_arm64_mov_imm(block, REG_ARG3, uop->imm_data); return 0; } static int codegen_MOV(codeblock_t *block, uop_t *uop) { host_arm64_MOV_REG(block, uop->dest_reg_a_real, uop->src_reg_a_real, 0); return 0; } static int codegen_MOV_IMM(codeblock_t *block, uop_t *uop) { host_arm64_mov_imm(block, uop->dest_reg_a_real, uop->imm_data); return 0; } static int codegen_MOV_PTR(codeblock_t *block, uop_t *uop) { int offset = add_literal_q(block, (uintptr_t)uop->p); host_arm64_LDR_LITERAL_X(block, uop->dest_reg_a_real, offset); return 0; } static int codegen_STORE_PTR_IMM(codeblock_t *block, uop_t *uop) { host_arm64_mov_imm(block, REG_W16, uop->imm_data); if (in_range12_w((uintptr_t)uop->p - (uintptr_t)&cpu_state)) host_arm64_STR_IMM_W(block, REG_W16, REG_CPUSTATE, (uintptr_t)uop->p - (uintptr_t)&cpu_state); else fatal("codegen_STORE_PTR_IMM - not in range\n"); return 0; } static int codegen_STORE_PTR_IMM_8(codeblock_t *block, uop_t *uop) { host_arm64_mov_imm(block, REG_W16, uop->imm_data); if (in_range12_b((uintptr_t)uop->p - (uintptr_t)&cpu_state)) host_arm64_STRB_IMM(block, REG_W16, REG_CPUSTATE, (uintptr_t)uop->p - (uintptr_t)&cpu_state); else fatal("codegen_STORE_PTR_IMM - not in range\n"); return 0; } const uOpFn uop_handlers[UOP_MAX] = { [UOP_CALL_INSTRUCTION_FUNC & UOP_MASK] = codegen_CALL_INSTRUCTION_FUNC, [UOP_LOAD_FUNC_ARG_0_IMM & UOP_MASK] = codegen_LOAD_FUNC_ARG0_IMM, [UOP_LOAD_FUNC_ARG_1_IMM & UOP_MASK] = codegen_LOAD_FUNC_ARG1_IMM, [UOP_LOAD_FUNC_ARG_2_IMM & UOP_MASK] = codegen_LOAD_FUNC_ARG2_IMM, [UOP_LOAD_FUNC_ARG_3_IMM & UOP_MASK] = codegen_LOAD_FUNC_ARG3_IMM, [UOP_STORE_P_IMM & UOP_MASK] = codegen_STORE_PTR_IMM, [UOP_STORE_P_IMM_8 & UOP_MASK] = codegen_STORE_PTR_IMM_8, [UOP_MOV & UOP_MASK] = codegen_MOV, [UOP_MOV_PTR & UOP_MASK] = codegen_MOV_PTR, [UOP_MOV_IMM & UOP_MASK] = codegen_MOV_IMM, [UOP_ADD & UOP_MASK] = codegen_ADD, [UOP_ADD_IMM & UOP_MASK] = codegen_ADD_IMM, [UOP_ADD_LSHIFT & UOP_MASK] = codegen_ADD_LSHIFT, [UOP_AND_IMM & UOP_MASK] = codegen_AND_IMM }; void codegen_direct_read_32(codeblock_t *block, int host_reg, void *p) { if (in_range12_w((uintptr_t)p - (uintptr_t)&cpu_state)) host_arm64_LDR_IMM_W(block, host_reg, REG_CPUSTATE, (uintptr_t)p - (uintptr_t)&cpu_state); else fatal("codegen_direct_read_32 - not in range\n"); } void codegen_direct_write_8(codeblock_t *block, void *p, int host_reg) { if (in_range12_b((uintptr_t)p - (uintptr_t)&cpu_state)) host_arm64_STRB_IMM(block, host_reg, REG_CPUSTATE, (uintptr_t)p - (uintptr_t)&cpu_state); else fatal("codegen_direct_write_8 - not in range\n"); } void codegen_direct_write_32(codeblock_t *block, void *p, int host_reg) { if (in_range12_w((uintptr_t)p - (uintptr_t)&cpu_state)) host_arm64_STR_IMM_W(block, host_reg, REG_CPUSTATE, (uintptr_t)p - (uintptr_t)&cpu_state); else fatal("codegen_direct_write_32 - not in range\n"); } void codegen_direct_write_ptr(codeblock_t *block, void *p, int host_reg) { if (in_range12_q((uintptr_t)p - (uintptr_t)&cpu_state)) host_arm64_STR_IMM_Q(block, host_reg, REG_CPUSTATE, (uintptr_t)p - (uintptr_t)&cpu_state); else fatal("codegen_direct_write_ptr - not in range\n"); } #endif