From 856f0b447196a3dc1c419341724ec84b862a92b4 Mon Sep 17 00:00:00 2001 From: SarahW Date: Sun, 25 Nov 2018 17:59:38 +0000 Subject: [PATCH] ARM64 MMX fixes --- src/codegen_backend_arm64_ops.c | 81 +++++++++++++++----- src/codegen_backend_arm64_ops.h | 16 ++-- src/codegen_backend_arm64_uops.c | 125 ++++++++++++++++++------------- 3 files changed, 146 insertions(+), 76 deletions(-) diff --git a/src/codegen_backend_arm64_ops.c b/src/codegen_backend_arm64_ops.c index 1ae57d9..9dabfe4 100644 --- a/src/codegen_backend_arm64_ops.c +++ b/src/codegen_backend_arm64_ops.c @@ -107,6 +107,7 @@ static inline void codegen_addlong(codeblock_t *block, uint32_t val) #define OPCODE_ADD_V8B (0x0e208400) #define OPCODE_ADD_V4H (0x0e608400) #define OPCODE_ADD_V2S (0x0ea08400) +#define OPCODE_ADDP_V4S (0x4ea0bc00) #define OPCODE_AND_V (0x0e201c00) #define OPCODE_ASR (0x1ac02800) #define OPCODE_BIC_V (0x0e601c00) @@ -164,12 +165,12 @@ static inline void codegen_addlong(codeblock_t *block, uint32_t val) #define OPCODE_SQSUB_V4H (0x0e602c00) #define OPCODE_SQXTN_V8B_8H (0x0e214800) #define OPCODE_SQXTN_V4H_4S (0x0e614800) -#define OPCODE_SHL_VD (0x0e005400) -#define OPCODE_SHL_VQ (0x4e005400) +#define OPCODE_SHL_VD (0x0f005400) +#define OPCODE_SHL_VQ (0x4f005400) #define OPCODE_SHRN (0x0f008400) #define OPCODE_SMULL_V4S_4H (0x0e60c000) -#define OPCODE_SSHR_VD (0x0e004400) -#define OPCODE_SSHR_VQ (0x4e004400) +#define OPCODE_SSHR_VD (0x0f000400) +#define OPCODE_SSHR_VQ (0x4f000400) #define OPCODE_STR_REG (0xb8206800) #define OPCODE_STRB_REG (0x38206800) #define OPCODE_STRH_REG (0x78206800) @@ -184,8 +185,9 @@ static inline void codegen_addlong(codeblock_t *block, uint32_t val) #define OPCODE_UQSUB_V8B (0x2e202c00) #define OPCODE_UQSUB_V4H (0x2e602c00) #define OPCODE_UQXTN_V8B_8H (0x2e214800) -#define OPCODE_USHR_VD (0x2e004400) -#define OPCODE_USHR_VQ (0x6e004400) +#define OPCODE_UQXTN_V4H_4S (0x2e614800) +#define OPCODE_USHR_VD (0x2f000400) +#define OPCODE_USHR_VQ (0x6f000400) #define OPCODE_ZIP1_V8B (0x0e003800) #define OPCODE_ZIP1_V4H (0x0e403800) #define OPCODE_ZIP1_V2S (0x0e803800) @@ -220,6 +222,8 @@ static inline void codegen_addlong(codeblock_t *block, uint32_t val) #define SHIFT_IMM_V2S(shift) (((shift) | 0x20) << 16) #define SHIFT_IMM_V2D(shift) (((shift) | 0x40) << 16) +#define SHRN_SHIFT_IMM_V4S(shift) (((shift) | 0x10) << 16) + static int literal_offset = 0; void codegen_reset_literal_pool(codeblock_t *block) { @@ -312,6 +316,22 @@ void host_arm64_ADD_IMM(codeblock_t *block, int dst_reg, int src_n_reg, uint32_t codegen_addlong(block, OPCODE_ADD_LSL | Rd(dst_reg) | Rn(src_n_reg) | Rm(REG_W16) | DATPROC_SHIFT(0)); } } +void host_arm64_ADDX_IMM(codeblock_t *block, int dst_reg, int src_n_reg, uint64_t imm_data) +{ + if (!(imm_data & ~0xffffffull)) + { + if (imm_data & 0xfff) + { + codegen_addlong(block, OPCODE_ADDX_IMM | Rd(dst_reg) | Rn(src_n_reg) | IMM12(imm_data & 0xfff) | DATPROC_IMM_SHIFT(0)); + if (imm_data & 0xfff000) + codegen_addlong(block, OPCODE_ADDX_IMM | Rd(dst_reg) | Rn(dst_reg) | IMM12((imm_data >> 12) & 0xfff) | DATPROC_IMM_SHIFT(1)); + } + else if (imm_data & 0xfff000) + codegen_addlong(block, OPCODE_ADDX_IMM | Rd(dst_reg) | Rn(src_n_reg) | IMM12((imm_data >> 12) & 0xfff) | DATPROC_IMM_SHIFT(1)); + } + else + fatal("ADD_IMM_X %016llx\n", imm_data); +} void host_arm64_ADD_REG(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg, int shift) { codegen_addlong(block, OPCODE_ADD_LSL | Rd(dst_reg) | Rn(src_n_reg) | Rm(src_m_reg) | DATPROC_SHIFT(shift)); @@ -333,6 +353,11 @@ void host_arm64_ADD_V2S(codeblock_t *block, int dst_reg, int src_n_reg, int src_ codegen_addlong(block, OPCODE_ADD_V2S | Rd(dst_reg) | Rn(src_n_reg) | Rm(src_m_reg)); } +void host_arm64_ADDP_V4S(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg) +{ + codegen_addlong(block, OPCODE_ADDP_V4S | Rd(dst_reg) | Rn(src_n_reg) | Rm(src_m_reg)); +} + void host_arm64_ADR(codeblock_t *block, int dst_reg, int offset) { codegen_addlong(block, OPCODE_ADR | Rd(dst_reg) | OFFSET20(offset)); @@ -971,9 +996,11 @@ void host_arm64_SCVTF_D_W(codeblock_t *block, int dst_reg, int src_reg) codegen_addlong(block, OPCODE_SCVTF_D_W | Rd(dst_reg) | Rn(src_reg)); } -void host_arm64_SHRN_V4S_4H(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg, int shift) +void host_arm64_SHRN_V4H_4S(codeblock_t *block, int dst_reg, int src_n_reg, int shift) { - codegen_addlong(block, OPCODE_SHRN | Rd(dst_reg) | Rn(src_n_reg) | Rm(src_m_reg) | SHIFT_IMM_V2S(shift)); + if (shift > 16) + fatal("host_arm64_SHRN_V4H_4S : shift > 16\n"); + codegen_addlong(block, OPCODE_SHRN | Rd(dst_reg) | Rn(src_n_reg) | SHRN_SHIFT_IMM_V4S(16-shift)); } void host_arm64_SMULL_V4S_4H(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg) @@ -1002,9 +1029,9 @@ void host_arm64_SQXTN_V8B_8H(codeblock_t *block, int dst_reg, int src_reg) { codegen_addlong(block, OPCODE_SQXTN_V8B_8H | Rd(dst_reg) | Rn(src_reg)); } -void host_arm64_SQXTN_V4B_4H(codeblock_t *block, int dst_reg, int src_reg) +void host_arm64_SQXTN_V4H_4S(codeblock_t *block, int dst_reg, int src_reg) { - codegen_addlong(block, OPCODE_SQXTN_V4B_4H | Rd(dst_reg) | Rn(src_reg)); + codegen_addlong(block, OPCODE_SQXTN_V4H_4S | Rd(dst_reg) | Rn(src_reg)); } void host_arm64_SHL_V4H(codeblock_t *block, int dst_reg, int src_n_reg, int shift) @@ -1022,15 +1049,21 @@ void host_arm64_SHL_V2D(codeblock_t *block, int dst_reg, int src_n_reg, int shif void host_arm64_SSHR_V4H(codeblock_t *block, int dst_reg, int src_n_reg, int shift) { - codegen_addlong(block, OPCODE_SSHR_VD | Rd(dst_reg) | Rn(src_n_reg) | SHIFT_IMM_V4H(shift)); + if (shift > 16) + fatal("host_arm_USHR_V4H : shift > 16\n"); + codegen_addlong(block, OPCODE_SSHR_VD | Rd(dst_reg) | Rn(src_n_reg) | SHIFT_IMM_V4H(16-shift)); } void host_arm64_SSHR_V2S(codeblock_t *block, int dst_reg, int src_n_reg, int shift) { - codegen_addlong(block, OPCODE_SSHR_VD | Rd(dst_reg) | Rn(src_n_reg) | SHIFT_IMM_V2S(shift)); + if (shift > 32) + fatal("host_arm_SSHR_V2S : shift > 32\n"); + codegen_addlong(block, OPCODE_SSHR_VD | Rd(dst_reg) | Rn(src_n_reg) | SHIFT_IMM_V2S(32-shift)); } void host_arm64_SSHR_V2D(codeblock_t *block, int dst_reg, int src_n_reg, int shift) { - codegen_addlong(block, OPCODE_SSHR_VQ | Rd(dst_reg) | Rn(src_n_reg) | SHIFT_IMM_V2D(shift)); + if (shift > 64) + fatal("host_arm_SSHR_V2D : shift > 64\n"); + codegen_addlong(block, OPCODE_SSHR_VQ | Rd(dst_reg) | Rn(src_n_reg) | SHIFT_IMM_V2D(64-shift)); } void host_arm64_STP_PREIDX_X(codeblock_t *block, int src_reg1, int src_reg2, int base_reg, int offset) @@ -1175,18 +1208,28 @@ void host_arm64_UQXTN_V8B_8H(codeblock_t *block, int dst_reg, int src_reg) { codegen_addlong(block, OPCODE_UQXTN_V8B_8H | Rd(dst_reg) | Rn(src_reg)); } +void host_arm64_UQXTN_V4H_4S(codeblock_t *block, int dst_reg, int src_reg) +{ + codegen_addlong(block, OPCODE_UQXTN_V4H_4S | Rd(dst_reg) | Rn(src_reg)); +} -void host_arm64_USHR_V4H(codeblock_t *block, int dst_reg, int src_n_reg, shift) +void host_arm64_USHR_V4H(codeblock_t *block, int dst_reg, int src_n_reg, int shift) { - codegen_addlong(block, OPCODE_USHR_VD | Rd(dst_reg) | Rn(src_n_reg) | SHIFT_IMM_V4H(shift)); + if (shift > 16) + fatal("host_arm_USHR_V4H : shift > 16\n"); + codegen_addlong(block, OPCODE_USHR_VD | Rd(dst_reg) | Rn(src_n_reg) | SHIFT_IMM_V4H(16-shift)); } -void host_arm64_USHR_V2S(codeblock_t *block, int dst_reg, int src_n_reg, shift) +void host_arm64_USHR_V2S(codeblock_t *block, int dst_reg, int src_n_reg, int shift) { - codegen_addlong(block, OPCODE_USHR_VD | Rd(dst_reg) | Rn(src_n_reg) | SHIFT_IMM_V2S(shift)); + if (shift > 32) + fatal("host_arm_USHR_V4S : shift > 32\n"); + codegen_addlong(block, OPCODE_USHR_VD | Rd(dst_reg) | Rn(src_n_reg) | SHIFT_IMM_V2S(32-shift)); } -void host_arm64_USHR_V2D(codeblock_t *block, int dst_reg, int src_n_reg, shift) +void host_arm64_USHR_V2D(codeblock_t *block, int dst_reg, int src_n_reg, int shift) { - codegen_addlong(block, OPCODE_USHR_VQ | Rd(dst_reg) | Rn(src_n_reg) | SHIFT_IMM_V2D(shift)); + if (shift > 64) + fatal("host_arm_USHR_V2D : shift > 64\n"); + codegen_addlong(block, OPCODE_USHR_VQ | Rd(dst_reg) | Rn(src_n_reg) | SHIFT_IMM_V2D(64-shift)); } void host_arm64_ZIP1_V8B(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg) diff --git a/src/codegen_backend_arm64_ops.h b/src/codegen_backend_arm64_ops.h index 01dd50c..179c5af 100644 --- a/src/codegen_backend_arm64_ops.h +++ b/src/codegen_backend_arm64_ops.h @@ -4,6 +4,9 @@ void host_arm64_ADD_REG_LSR(codeblock_t *block, int dst_reg, int src_n_reg, int void host_arm64_ADD_V8B(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg); void host_arm64_ADD_V4H(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg); void host_arm64_ADD_V2S(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg); +void host_arm64_ADDX_IMM(codeblock_t *block, int dst_reg, int src_n_reg, uint64_t imm_data); + +void host_arm64_ADDP_V4S(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg); void host_arm64_ADR(codeblock_t *block, int dst_reg, int offset); @@ -157,13 +160,13 @@ void host_arm64_SQSUB_V8B(codeblock_t *block, int dst_reg, int src_n_reg, int sr void host_arm64_SQSUB_V4H(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg); void host_arm64_SQXTN_V8B_8H(codeblock_t *block, int dst_reg, int src_reg); -void host_arm64_SQXTN_V4H_4D(codeblock_t *block, int dst_reg, int src_reg); +void host_arm64_SQXTN_V4H_4S(codeblock_t *block, int dst_reg, int src_reg); void host_arm64_SHL_V4H(codeblock_t *block, int dst_reg, int src_reg, int shift); void host_arm64_SHL_V2S(codeblock_t *block, int dst_reg, int src_reg, int shift); void host_arm64_SHL_V2D(codeblock_t *block, int dst_reg, int src_reg, int shift); -void host_arm64_SHRN_V4H_4S(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg, int shift); +void host_arm64_SHRN_V4H_4S(codeblock_t *block, int dst_reg, int src_n_reg, int shift); void host_arm64_SMULL_V4S_4H(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg); @@ -201,12 +204,13 @@ uint32_t *host_arm64_TBNZ(codeblock_t *block, int reg, int bit); void host_arm64_UBFX(codeblock_t *block, int dst_reg, int src_reg, int lsb, int width); -void host_arm64_UQADD_V8B(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg, int shift); -void host_arm64_UQADD_V4H(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg, int shift); -void host_arm64_UQSUB_V8B(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg, int shift); -void host_arm64_UQSUB_V4H(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg, int shift); +void host_arm64_UQADD_V8B(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg); +void host_arm64_UQADD_V4H(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg); +void host_arm64_UQSUB_V8B(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg); +void host_arm64_UQSUB_V4H(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg); void host_arm64_UQXTN_V8B_8H(codeblock_t *block, int dst_reg, int src_reg); +void host_arm64_UQXTN_V4H_4S(codeblock_t *block, int dst_reg, int src_reg); void host_arm64_USHR_V4H(codeblock_t *block, int dst_reg, int src_reg, int shift); void host_arm64_USHR_V2S(codeblock_t *block, int dst_reg, int src_reg, int shift); diff --git a/src/codegen_backend_arm64_uops.c b/src/codegen_backend_arm64_uops.c index 18b1d48..4f34ea3 100644 --- a/src/codegen_backend_arm64_uops.c +++ b/src/codegen_backend_arm64_uops.c @@ -1386,14 +1386,14 @@ static int codegen_OR_IMM(codeblock_t *block, uop_t *uop) static int codegen_PACKSSWB(codeblock_t *block, uop_t *uop) { - int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_a = HOST_REG_GET(uop->src_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real); - int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_a = IREG_GET_SIZE(uop->src_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real); + int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real); + int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real); - if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_a) && REG_IS_Q(src_size_b)) + if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_b) && uop->dest_reg_a_real == uop->src_reg_a_real) { - host_arm64_SQXTN_V8B_8H(block, dest_reg, dest_reg); host_arm64_SQXTN_V8B_8H(block, REG_V_TEMP, src_reg_b); - host_arm64_VZIP1_V2S(block, dest_reg, dest_reg, REG_V_TEMP); + host_arm64_SQXTN_V8B_8H(block, dest_reg, dest_reg); + host_arm64_ZIP1_V2S(block, dest_reg, dest_reg, REG_V_TEMP); } else fatal("PACKSSWB %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real); @@ -1407,9 +1407,9 @@ static int codegen_PACKSSDW(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_b) && uop->dest_reg_a_real == uop->src_reg_a_real) { - host_arm64_UQXTN_V4H_4D(block, dest_reg, dest_reg); - host_arm64_UQXTN_V4H_4D(block, REG_V_TEMP, src_reg_b); - host_arm64_VZIP1_V2S(block, dest_reg, dest_reg, REG_V_TEMP); + host_arm64_SQXTN_V4H_4S(block, REG_V_TEMP, src_reg_b); + host_arm64_SQXTN_V4H_4S(block, dest_reg, dest_reg); + host_arm64_ZIP1_V2S(block, dest_reg, dest_reg, REG_V_TEMP); } else fatal("PACKSSDW %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real); @@ -1423,9 +1423,9 @@ static int codegen_PACKUSWB(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_b) && uop->dest_reg_a_real == uop->src_reg_a_real) { - host_arm64_UQXTN_V8B_8H(block, dest_reg, dest_reg); host_arm64_UQXTN_V8B_8H(block, REG_V_TEMP, src_reg_b); - host_arm64_VZIP1_V2S(block, dest_reg, dest_reg, REG_V_TEMP); + host_arm64_UQXTN_V8B_8H(block, dest_reg, dest_reg); + host_arm64_ZIP1_V2S(block, dest_reg, dest_reg, REG_V_TEMP); } else fatal("PACKUSWB %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real); @@ -1510,7 +1510,7 @@ static int codegen_PADDUSB(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_a) && REG_IS_Q(src_size_b)) { - host_arm64_UQADD_V8H(block, dest_reg, src_reg_a, src_reg_b); + host_arm64_UQADD_V8B(block, dest_reg, src_reg_a, src_reg_b); } else fatal("PADDUSB %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real); @@ -1553,7 +1553,7 @@ static int codegen_PCMPEQW(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_a) && REG_IS_Q(src_size_b)) { - host_arm_CMEQ_V4H(block, dest_reg, src_reg_a, src_reg_b); + host_arm64_CMEQ_V4H(block, dest_reg, src_reg_a, src_reg_b); } else fatal("PCMPEQW %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real); @@ -1567,7 +1567,7 @@ static int codegen_PCMPEQD(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_a) && REG_IS_Q(src_size_b)) { - host_arm_CMEQ_V2S(block, dest_reg, src_reg_a, src_reg_b); + host_arm64_CMEQ_V2S(block, dest_reg, src_reg_a, src_reg_b); } else fatal("PCMPEQD %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real); @@ -1581,7 +1581,7 @@ static int codegen_PCMPGTB(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_a) && REG_IS_Q(src_size_b)) { - host_arm_CMGT_V8B(block, dest_reg, src_reg_a, src_reg_b); + host_arm64_CMGT_V8B(block, dest_reg, src_reg_a, src_reg_b); } else fatal("PCMPGTB %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real); @@ -1595,7 +1595,7 @@ static int codegen_PCMPGTW(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_a) && REG_IS_Q(src_size_b)) { - host_arm_CMGT_V4H(block, dest_reg, src_reg_a, src_reg_b); + host_arm64_CMGT_V4H(block, dest_reg, src_reg_a, src_reg_b); } else fatal("PCMPGTW %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real); @@ -1609,7 +1609,7 @@ static int codegen_PCMPGTD(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_a) && REG_IS_Q(src_size_b)) { - host_arm_CMGT_V2S(block, dest_reg, src_reg_a, src_reg_b); + host_arm64_CMGT_V2S(block, dest_reg, src_reg_a, src_reg_b); } else fatal("PCMPGTD %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real); @@ -1619,13 +1619,13 @@ static int codegen_PCMPGTD(codeblock_t *block, uop_t *uop) static int codegen_PMADDWD(codeblock_t *block, uop_t *uop) { - int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real); - int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real); + int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_a = HOST_REG_GET(uop->src_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real); + int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_a = IREG_GET_SIZE(uop->src_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real); - if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_b) && uop->dest_reg_a_real == uop->src_reg_a_real) + if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_a) && REG_IS_Q(src_size_b)) { - host_arm64_MUL_V4H(block, dest_reg, src_reg_a, src_reg_b); - host_arm64_SADDLP_V2S_4H(block, dest_reg, dest_reg); + host_arm64_SMULL_V4S_4H(block, REG_V_TEMP, src_reg_a, src_reg_b); + host_arm64_ADDP_V4S(block, dest_reg, REG_V_TEMP, REG_V_TEMP); } else fatal("PMULHW %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real); @@ -1634,10 +1634,10 @@ static int codegen_PMADDWD(codeblock_t *block, uop_t *uop) } static int codegen_PMULHW(codeblock_t *block, uop_t *uop) { - int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real); - int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real); + int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_a = HOST_REG_GET(uop->src_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real); + int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_a = IREG_GET_SIZE(uop->src_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real); - if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_b) && uop->dest_reg_a_real == uop->src_reg_a_real) + if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_a) && REG_IS_Q(src_size_b)) { host_arm64_SMULL_V4S_4H(block, dest_reg, src_reg_a, src_reg_b); host_arm64_SHRN_V4H_4S(block, dest_reg, dest_reg, 16); @@ -1649,10 +1649,10 @@ static int codegen_PMULHW(codeblock_t *block, uop_t *uop) } static int codegen_PMULLW(codeblock_t *block, uop_t *uop) { - int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real); - int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real); + int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_a = HOST_REG_GET(uop->src_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real); + int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_a = IREG_GET_SIZE(uop->src_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real); - if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_b) && uop->dest_reg_a_real == uop->src_reg_a_real) + if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_a) && REG_IS_Q(src_size_b)) { host_arm64_MUL_V4H(block, dest_reg, src_reg_a, src_reg_b); } @@ -1669,7 +1669,9 @@ static int codegen_PSLLW_IMM(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size)) { - if (uop->imm_data > 15) + if (uop->imm_data == 0) + host_arm64_FMOV_D_D(block, dest_reg, src_reg); + else if (uop->imm_data > 15) host_arm64_EOR_REG_V(block, dest_reg, dest_reg, dest_reg); else host_arm64_SHL_V4H(block, dest_reg, src_reg, uop->imm_data); @@ -1686,7 +1688,9 @@ static int codegen_PSLLD_IMM(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size)) { - if (uop->imm_data > 31) + if (uop->imm_data == 0) + host_arm64_FMOV_D_D(block, dest_reg, src_reg); + else if (uop->imm_data > 31) host_arm64_EOR_REG_V(block, dest_reg, dest_reg, dest_reg); else host_arm64_SHL_V2S(block, dest_reg, src_reg, uop->imm_data); @@ -1703,7 +1707,9 @@ static int codegen_PSLLQ_IMM(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size)) { - if (uop->imm_data > 63) + if (uop->imm_data == 0) + host_arm64_FMOV_D_D(block, dest_reg, src_reg); + else if (uop->imm_data > 63) host_arm64_EOR_REG_V(block, dest_reg, dest_reg, dest_reg); else host_arm64_SHL_V2D(block, dest_reg, src_reg, uop->imm_data); @@ -1720,7 +1726,9 @@ static int codegen_PSRAW_IMM(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size)) { - if (uop->imm_data > 15) + if (uop->imm_data == 0) + host_arm64_FMOV_D_D(block, dest_reg, src_reg); + else if (uop->imm_data > 15) host_arm64_SSHR_V4H(block, dest_reg, src_reg, 15); else host_arm64_SSHR_V4H(block, dest_reg, src_reg, uop->imm_data); @@ -1737,7 +1745,9 @@ static int codegen_PSRAD_IMM(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size)) { - if (uop->imm_data > 31) + if (uop->imm_data == 0) + host_arm64_FMOV_D_D(block, dest_reg, src_reg); + else if (uop->imm_data > 31) host_arm64_SSHR_V2S(block, dest_reg, src_reg, 31); else host_arm64_SSHR_V2S(block, dest_reg, src_reg, uop->imm_data); @@ -1754,7 +1764,9 @@ static int codegen_PSRAQ_IMM(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size)) { - if (uop->imm_data > 63) + if (uop->imm_data == 0) + host_arm64_FMOV_D_D(block, dest_reg, src_reg); + else if (uop->imm_data > 63) host_arm64_SSHR_V2D(block, dest_reg, src_reg, 63); else host_arm64_SSHR_V2D(block, dest_reg, src_reg, uop->imm_data); @@ -1771,7 +1783,9 @@ static int codegen_PSRLW_IMM(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size)) { - if (uop->imm_data > 15) + if (uop->imm_data == 0) + host_arm64_FMOV_D_D(block, dest_reg, src_reg); + else if (uop->imm_data > 15) host_arm64_EOR_REG_V(block, dest_reg, dest_reg, dest_reg); else host_arm64_USHR_V4H(block, dest_reg, src_reg, uop->imm_data); @@ -1788,7 +1802,9 @@ static int codegen_PSRLD_IMM(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size)) { - if (uop->imm_data > 31) + if (uop->imm_data == 0) + host_arm64_FMOV_D_D(block, dest_reg, src_reg); + else if (uop->imm_data > 31) host_arm64_EOR_REG_V(block, dest_reg, dest_reg, dest_reg); else host_arm64_USHR_V2S(block, dest_reg, src_reg, uop->imm_data); @@ -1805,7 +1821,9 @@ static int codegen_PSRLQ_IMM(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size)) { - if (uop->imm_data > 63) + if (uop->imm_data == 0) + host_arm64_FMOV_D_D(block, dest_reg, src_reg); + else if (uop->imm_data > 63) host_arm64_EOR_REG_V(block, dest_reg, dest_reg, dest_reg); else host_arm64_USHR_V2D(block, dest_reg, src_reg, uop->imm_data); @@ -1846,9 +1864,6 @@ static int codegen_PSUBW(codeblock_t *block, uop_t *uop) } static int codegen_PSUBD(codeblock_t *block, uop_t *uop) { - int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real); - int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real); - int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_a = HOST_REG_GET(uop->src_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real); int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_a = IREG_GET_SIZE(uop->src_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real); @@ -1925,7 +1940,7 @@ static int codegen_PUNPCKHBW(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_a) && REG_IS_Q(src_size_b)) { - host_arm64_VZIP2_V8B(block, dest_reg, src_reg_a, src_reg_b); + host_arm64_ZIP2_V8B(block, dest_reg, src_reg_a, src_reg_b); } else fatal("PUNPCKHBW %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real); @@ -1939,7 +1954,7 @@ static int codegen_PUNPCKHWD(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_a) && REG_IS_Q(src_size_b)) { - host_arm64_VZIP2_V4H(block, dest_reg, src_reg_a, src_reg_b); + host_arm64_ZIP2_V4H(block, dest_reg, src_reg_a, src_reg_b); } else fatal("PUNPCKHWD %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real); @@ -1953,7 +1968,7 @@ static int codegen_PUNPCKHDQ(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_a) && REG_IS_Q(src_size_b)) { - host_arm64_VZIP2_V2S(block, dest_reg, src_reg_a, src_reg_b); + host_arm64_ZIP2_V2S(block, dest_reg, src_reg_a, src_reg_b); } else fatal("PUNPCKHDQ %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real); @@ -1967,7 +1982,7 @@ static int codegen_PUNPCKLBW(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_a) && REG_IS_Q(src_size_b)) { - host_arm64_VZIP1_V8B(block, dest_reg, src_reg_a, src_reg_b); + host_arm64_ZIP1_V8B(block, dest_reg, src_reg_a, src_reg_b); } else fatal("PUNPCKLBW %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real); @@ -1981,7 +1996,7 @@ static int codegen_PUNPCKLWD(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_a) && REG_IS_Q(src_size_b)) { - host_arm64_VZIP1_V4H(block, dest_reg, src_reg_a, src_reg_b); + host_arm64_ZIP1_V4H(block, dest_reg, src_reg_a, src_reg_b); } else fatal("PUNPCKLWD %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real); @@ -1995,7 +2010,7 @@ static int codegen_PUNPCKLDQ(codeblock_t *block, uop_t *uop) if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_a) && REG_IS_Q(src_size_b)) { - host_arm64_VZIP1_V2S(block, dest_reg, src_reg_a, src_reg_b); + host_arm64_ZIP1_V2S(block, dest_reg, src_reg_a, src_reg_b); } else fatal("PUNPCKLDQ %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real); @@ -2613,7 +2628,7 @@ void codegen_direct_read_st_8(codeblock_t *block, int host_reg, void *base, int { host_arm64_LDR_IMM_W(block, REG_TEMP, REG_SP, IREG_TOP_diff_stack_offset); host_arm64_ADD_IMM(block, REG_TEMP, REG_TEMP, reg_idx); - host_arm64_ADD_IMM(block, REG_TEMP2, REG_CPUSTATE, (uintptr_t)base - (uintptr_t)&cpu_state); + host_arm64_ADDX_IMM(block, REG_TEMP2, REG_CPUSTATE, (uintptr_t)base - (uintptr_t)&cpu_state); host_arm64_AND_IMM(block, REG_TEMP, REG_TEMP, 7); host_arm64_LDRB_REG(block, host_reg, REG_TEMP2, REG_TEMP); } @@ -2621,7 +2636,7 @@ void codegen_direct_read_st_64(codeblock_t *block, int host_reg, void *base, int { host_arm64_LDR_IMM_W(block, REG_TEMP, REG_SP, IREG_TOP_diff_stack_offset); host_arm64_ADD_IMM(block, REG_TEMP, REG_TEMP, reg_idx); - host_arm64_ADD_IMM(block, REG_TEMP2, REG_CPUSTATE, (uintptr_t)base - (uintptr_t)&cpu_state); + host_arm64_ADDX_IMM(block, REG_TEMP2, REG_CPUSTATE, (uintptr_t)base - (uintptr_t)&cpu_state); host_arm64_AND_IMM(block, REG_TEMP, REG_TEMP, 7); host_arm64_LDR_REG_F64_S(block, host_reg, REG_TEMP2, REG_TEMP); } @@ -2629,7 +2644,7 @@ void codegen_direct_read_st_double(codeblock_t *block, int host_reg, void *base, { host_arm64_LDR_IMM_W(block, REG_TEMP, REG_SP, IREG_TOP_diff_stack_offset); host_arm64_ADD_IMM(block, REG_TEMP, REG_TEMP, reg_idx); - host_arm64_ADD_IMM(block, REG_TEMP2, REG_CPUSTATE, (uintptr_t)base - (uintptr_t)&cpu_state); + host_arm64_ADDX_IMM(block, REG_TEMP2, REG_CPUSTATE, (uintptr_t)base - (uintptr_t)&cpu_state); host_arm64_AND_IMM(block, REG_TEMP, REG_TEMP, 7); host_arm64_LDR_REG_F64_S(block, host_reg, REG_TEMP2, REG_TEMP); } @@ -2673,7 +2688,7 @@ void codegen_direct_write_st_8(codeblock_t *block, void *base, int reg_idx, int { host_arm64_LDR_IMM_W(block, REG_TEMP, REG_SP, IREG_TOP_diff_stack_offset); host_arm64_ADD_IMM(block, REG_TEMP, REG_TEMP, reg_idx); - host_arm64_ADD_IMM(block, REG_TEMP2, REG_CPUSTATE, (uintptr_t)base - (uintptr_t)&cpu_state); + host_arm64_ADDX_IMM(block, REG_TEMP2, REG_CPUSTATE, (uintptr_t)base - (uintptr_t)&cpu_state); host_arm64_AND_IMM(block, REG_TEMP, REG_TEMP, 7); host_arm64_STRB_REG(block, host_reg, REG_TEMP2, REG_TEMP); } @@ -2681,7 +2696,7 @@ void codegen_direct_write_st_64(codeblock_t *block, void *base, int reg_idx, int { host_arm64_LDR_IMM_W(block, REG_TEMP, REG_SP, IREG_TOP_diff_stack_offset); host_arm64_ADD_IMM(block, REG_TEMP, REG_TEMP, reg_idx); - host_arm64_ADD_IMM(block, REG_TEMP2, REG_CPUSTATE, (uintptr_t)base - (uintptr_t)&cpu_state); + host_arm64_ADDX_IMM(block, REG_TEMP2, REG_CPUSTATE, (uintptr_t)base - (uintptr_t)&cpu_state); host_arm64_AND_IMM(block, REG_TEMP, REG_TEMP, 7); host_arm64_STR_REG_F64_S(block, host_reg, REG_TEMP2, REG_TEMP); } @@ -2689,7 +2704,7 @@ void codegen_direct_write_st_double(codeblock_t *block, void *base, int reg_idx, { host_arm64_LDR_IMM_W(block, REG_TEMP, REG_SP, IREG_TOP_diff_stack_offset); host_arm64_ADD_IMM(block, REG_TEMP, REG_TEMP, reg_idx); - host_arm64_ADD_IMM(block, REG_TEMP2, REG_CPUSTATE, (uintptr_t)base - (uintptr_t)&cpu_state); + host_arm64_ADDX_IMM(block, REG_TEMP2, REG_CPUSTATE, (uintptr_t)base - (uintptr_t)&cpu_state); host_arm64_AND_IMM(block, REG_TEMP, REG_TEMP, 7); host_arm64_STR_REG_F64_S(block, host_reg, REG_TEMP2, REG_TEMP); } @@ -2716,6 +2731,10 @@ void codegen_direct_read_32_stack(codeblock_t *block, int host_reg, int stack_of else fatal("codegen_direct_read_32_stack - not in range\n"); } +void codegen_direct_read_64_stack(codeblock_t *block, int host_reg, int stack_offset) +{ + host_arm64_LDR_IMM_F64(block, host_reg, REG_SP, stack_offset); +} void codegen_direct_read_double_stack(codeblock_t *block, int host_reg, int stack_offset) { host_arm64_LDR_IMM_F64(block, host_reg, REG_SP, stack_offset); @@ -2728,6 +2747,10 @@ void codegen_direct_write_32_stack(codeblock_t *block, int stack_offset, int hos else fatal("codegen_direct_write_32_stack - not in range\n"); } +void codegen_direct_write_64_stack(codeblock_t *block, int stack_offset, int host_reg) +{ + host_arm64_STR_IMM_F64(block, host_reg, REG_SP, stack_offset); +} void codegen_direct_write_double_stack(codeblock_t *block, int stack_offset, int host_reg) { host_arm64_STR_IMM_F64(block, host_reg, REG_SP, stack_offset);