From 086422b79fec0a1bcad6b39cb1e88f63187e6438 Mon Sep 17 00:00:00 2001
From: SarahW <pcem@pcem-emulator.co.uk>
Date: Sat, 24 Nov 2018 12:52:08 +0000
Subject: [PATCH] Added recompiled versions of PMULLW, PMULHW and PMADDWD.

---
 src/codegen_backend_arm64_ops.c   | 36 +++++++++++++++++++----
 src/codegen_backend_arm64_ops.h   |  8 +++++
 src/codegen_backend_arm64_uops.c  | 49 +++++++++++++++++++++++++++++++
 src/codegen_backend_arm_ops.c     | 18 ++++++++++++
 src/codegen_backend_arm_ops.h     |  7 +++++
 src/codegen_backend_arm_uops.c    | 49 +++++++++++++++++++++++++++++++
 src/codegen_backend_x86-64_ops.c  | 13 ++++++++
 src/codegen_backend_x86-64_ops.h  |  4 +++
 src/codegen_backend_x86-64_uops.c | 47 +++++++++++++++++++++++++++++
 src/codegen_backend_x86_ops.c     | 13 ++++++++
 src/codegen_backend_x86_ops.h     |  4 +++
 src/codegen_backend_x86_uops.c    | 47 +++++++++++++++++++++++++++++
 src/codegen_ir_defs.h             | 12 +++++++-
 src/codegen_ops.c                 | 12 ++++----
 src/codegen_ops_mmx_arith.c       |  4 +++
 src/codegen_ops_mmx_arith.h       |  4 +++
 16 files changed, 314 insertions(+), 13 deletions(-)

diff --git a/src/codegen_backend_arm64_ops.c b/src/codegen_backend_arm64_ops.c
index c5e7fb0..1ae57d9 100644
--- a/src/codegen_backend_arm64_ops.c
+++ b/src/codegen_backend_arm64_ops.c
@@ -151,9 +151,11 @@ static inline void codegen_addlong(codeblock_t *block, uint32_t val)
 #define OPCODE_LSL           (0x1ac02000)
 #define OPCODE_LSR           (0x1ac02400)
 #define OPCODE_MSR_FPCR      (0xd51b4400)
+#define OPCODE_MUL_V4H       (0x0e609c00)
 #define OPCODE_NOP           (0xd503201f)
 #define OPCODE_ORR_V         (0x0ea01c00)
 #define OPCODE_RET           (0xd65f0000)
+#define OPCODE_SADDLP_V2S_4H (0x0e602800)
 #define OPCODE_SCVTF_D_Q     (0x9e620000)
 #define OPCODE_SCVTF_D_W     (0x1e620000)
 #define OPCODE_SQADD_V8B     (0x0e200c00)
@@ -164,6 +166,8 @@ static inline void codegen_addlong(codeblock_t *block, uint32_t val)
 #define OPCODE_SQXTN_V4H_4S  (0x0e614800)
 #define OPCODE_SHL_VD        (0x0e005400)
 #define OPCODE_SHL_VQ        (0x4e005400)
+#define OPCODE_SHRN          (0x0f008400)
+#define OPCODE_SMULL_V4S_4H  (0x0e60c000)
 #define OPCODE_SSHR_VD       (0x0e004400)
 #define OPCODE_SSHR_VQ       (0x4e004400)
 #define OPCODE_STR_REG       (0xb8206800)
@@ -900,6 +904,11 @@ void host_arm64_MSR_FPCR(codeblock_t *block, int src_reg)
 	codegen_addlong(block, OPCODE_MSR_FPCR | Rd(src_reg));
 }
 
+void host_arm64_MUL_V4H(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg)
+{
+	codegen_addlong(block, OPCODE_MUL_V4H | Rd(dst_reg) | Rn(src_n_reg) | Rm(src_m_reg));
+}
+
 void host_arm64_NOP(codeblock_t *block)
 {
 	codegen_addlong(block, OPCODE_NOP);
@@ -943,6 +952,11 @@ void host_arm64_RET(codeblock_t *block, int reg)
 	codegen_addlong(block, OPCODE_RET | Rn(reg));
 }
 
+void host_arm64_SADDLP_V2S_4H(codeblock_t *block, int dst_reg, int src_n_reg)
+{
+	codegen_addlong(block, OPCODE_SADDLP_V2S_4H | Rd(dst_reg) | Rn(src_n_reg));
+}
+
 void host_arm64_SBFX(codeblock_t *block, int dst_reg, int src_reg, int lsb, int width)
 {
 	codegen_addlong(block, OPCODE_SBFX | Rd(dst_reg) | Rn(src_reg) | IMMN(0) | IMMR(lsb) | IMMS((lsb+width-1) & 31));
@@ -957,6 +971,16 @@ void host_arm64_SCVTF_D_W(codeblock_t *block, int dst_reg, int src_reg)
 	codegen_addlong(block, OPCODE_SCVTF_D_W | Rd(dst_reg) | Rn(src_reg));
 }
 
+void host_arm64_SHRN_V4S_4H(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg, int shift)
+{
+	codegen_addlong(block, OPCODE_SHRN | Rd(dst_reg) | Rn(src_n_reg) | Rm(src_m_reg) | SHIFT_IMM_V2S(shift));
+}
+
+void host_arm64_SMULL_V4S_4H(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg)
+{
+	codegen_addlong(block, OPCODE_SMULL_V4S_4H | Rd(dst_reg) | Rn(src_n_reg) | Rm(src_m_reg));
+}
+
 void host_arm64_SQADD_V8B(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg)
 {
 	codegen_addlong(block, OPCODE_SQADD_V8B | Rd(dst_reg) | Rn(src_n_reg) | Rm(src_m_reg));
@@ -983,28 +1007,28 @@ void host_arm64_SQXTN_V4B_4H(codeblock_t *block, int dst_reg, int src_reg)
 	codegen_addlong(block, OPCODE_SQXTN_V4B_4H | Rd(dst_reg) | Rn(src_reg));
 }
 
-void host_arm64_SHL_V4H(codeblock_t *block, int dst_reg, int src_n_reg, shift)
+void host_arm64_SHL_V4H(codeblock_t *block, int dst_reg, int src_n_reg, int shift)
 {
 	codegen_addlong(block, OPCODE_SHL_VD | Rd(dst_reg) | Rn(src_n_reg) | SHIFT_IMM_V4H(shift));
 }
-void host_arm64_SHL_V2S(codeblock_t *block, int dst_reg, int src_n_reg, shift)
+void host_arm64_SHL_V2S(codeblock_t *block, int dst_reg, int src_n_reg, int shift)
 {
 	codegen_addlong(block, OPCODE_SHL_VD | Rd(dst_reg) | Rn(src_n_reg) | SHIFT_IMM_V2S(shift));
 }
-void host_arm64_SHL_V2D(codeblock_t *block, int dst_reg, int src_n_reg, shift)
+void host_arm64_SHL_V2D(codeblock_t *block, int dst_reg, int src_n_reg, int shift)
 {
 	codegen_addlong(block, OPCODE_SHL_VQ | Rd(dst_reg) | Rn(src_n_reg) | SHIFT_IMM_V2D(shift));
 }
 
-void host_arm64_SSHR_V4H(codeblock_t *block, int dst_reg, int src_n_reg, shift)
+void host_arm64_SSHR_V4H(codeblock_t *block, int dst_reg, int src_n_reg, int shift)
 {
 	codegen_addlong(block, OPCODE_SSHR_VD | Rd(dst_reg) | Rn(src_n_reg) | SHIFT_IMM_V4H(shift));
 }
-void host_arm64_SSHR_V2S(codeblock_t *block, int dst_reg, int src_n_reg, shift)
+void host_arm64_SSHR_V2S(codeblock_t *block, int dst_reg, int src_n_reg, int shift)
 {
 	codegen_addlong(block, OPCODE_SSHR_VD | Rd(dst_reg) | Rn(src_n_reg) | SHIFT_IMM_V2S(shift));
 }
-void host_arm64_SSHR_V2D(codeblock_t *block, int dst_reg, int src_n_reg, shift)
+void host_arm64_SSHR_V2D(codeblock_t *block, int dst_reg, int src_n_reg, int shift)
 {
 	codegen_addlong(block, OPCODE_SSHR_VQ | Rd(dst_reg) | Rn(src_n_reg) | SHIFT_IMM_V2D(shift));
 }
diff --git a/src/codegen_backend_arm64_ops.h b/src/codegen_backend_arm64_ops.h
index 77c02c7..01dd50c 100644
--- a/src/codegen_backend_arm64_ops.h
+++ b/src/codegen_backend_arm64_ops.h
@@ -134,6 +134,8 @@ void host_arm64_MOVK_IMM(codeblock_t *block, int reg, uint32_t imm_data);
 
 void host_arm64_MSR_FPCR(codeblock_t *block, int src_reg);
 
+void host_arm64_MUL_V4H(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg);
+
 void host_arm64_NOP(codeblock_t *block);
 
 void host_arm64_ORR_IMM(codeblock_t *block, int dst_reg, int src_n_reg, uint32_t imm_data);
@@ -142,6 +144,8 @@ void host_arm64_ORR_REG_V(codeblock_t *block, int dst_reg, int src_n_reg, int sr
 
 void host_arm64_RET(codeblock_t *block, int reg);
 
+void host_arm64_SADDLP_V2S_4H(codeblock_t *block, int dst_reg, int src_n_reg);
+
 void host_arm64_SBFX(codeblock_t *block, int dst_reg, int src_reg, int lsb, int width);
 
 void host_arm64_SCVTF_D_Q(codeblock_t *block, int dst_reg, int src_reg);
@@ -159,6 +163,10 @@ void host_arm64_SHL_V4H(codeblock_t *block, int dst_reg, int src_reg, int shift)
 void host_arm64_SHL_V2S(codeblock_t *block, int dst_reg, int src_reg, int shift);
 void host_arm64_SHL_V2D(codeblock_t *block, int dst_reg, int src_reg, int shift);
 
+void host_arm64_SHRN_V4H_4S(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg, int shift);
+
+void host_arm64_SMULL_V4S_4H(codeblock_t *block, int dst_reg, int src_n_reg, int src_m_reg);
+
 void host_arm64_SSHR_V4H(codeblock_t *block, int dst_reg, int src_reg, int shift);
 void host_arm64_SSHR_V2S(codeblock_t *block, int dst_reg, int src_reg, int shift);
 void host_arm64_SSHR_V2D(codeblock_t *block, int dst_reg, int src_reg, int shift);
diff --git a/src/codegen_backend_arm64_uops.c b/src/codegen_backend_arm64_uops.c
index 68f1122..18b1d48 100644
--- a/src/codegen_backend_arm64_uops.c
+++ b/src/codegen_backend_arm64_uops.c
@@ -1617,6 +1617,51 @@ static int codegen_PCMPGTD(codeblock_t *block, uop_t *uop)
         return 0;
 }
 
+static int codegen_PMADDWD(codeblock_t *block, uop_t *uop)
+{
+        int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real);
+        int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real);
+
+        if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_b) && uop->dest_reg_a_real == uop->src_reg_a_real)
+        {
+                host_arm64_MUL_V4H(block, dest_reg, src_reg_a, src_reg_b);
+                host_arm64_SADDLP_V2S_4H(block, dest_reg, dest_reg);
+        }
+        else
+                fatal("PMULHW %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real);
+
+        return 0;
+}
+static int codegen_PMULHW(codeblock_t *block, uop_t *uop)
+{
+        int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real);
+        int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real);
+
+        if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_b) && uop->dest_reg_a_real == uop->src_reg_a_real)
+        {
+                host_arm64_SMULL_V4S_4H(block, dest_reg, src_reg_a, src_reg_b);
+                host_arm64_SHRN_V4H_4S(block, dest_reg, dest_reg, 16);
+        }
+        else
+                fatal("PMULHW %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real);
+
+        return 0;
+}
+static int codegen_PMULLW(codeblock_t *block, uop_t *uop)
+{
+        int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real);
+        int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real);
+
+        if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_b) && uop->dest_reg_a_real == uop->src_reg_a_real)
+        {
+                host_arm64_MUL_V4H(block, dest_reg, src_reg_a, src_reg_b);
+        }
+        else
+                fatal("PMULLW %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real);
+
+        return 0;
+}
+
 static int codegen_PSLLW_IMM(codeblock_t *block, uop_t *uop)
 {
         int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg = HOST_REG_GET(uop->src_reg_a_real);
@@ -2499,6 +2544,10 @@ const uOpFn uop_handlers[UOP_MAX] =
         [UOP_PCMPGTW & UOP_MASK] = codegen_PCMPGTW,
         [UOP_PCMPGTD & UOP_MASK] = codegen_PCMPGTD,
 
+        [UOP_PMADDWD & UOP_MASK] = codegen_PMADDWD,
+        [UOP_PMULHW & UOP_MASK]  = codegen_PMULHW,
+        [UOP_PMULLW & UOP_MASK]  = codegen_PMULLW,
+
         [UOP_PSLLW_IMM & UOP_MASK] = codegen_PSLLW_IMM,
         [UOP_PSLLD_IMM & UOP_MASK] = codegen_PSLLD_IMM,
         [UOP_PSLLQ_IMM & UOP_MASK] = codegen_PSLLQ_IMM,
diff --git a/src/codegen_backend_arm_ops.c b/src/codegen_backend_arm_ops.c
index 815c894..c0a5a6a 100644
--- a/src/codegen_backend_arm_ops.c
+++ b/src/codegen_backend_arm_ops.c
@@ -116,7 +116,10 @@ static inline void codegen_addlong(codeblock_t *block, uint32_t val)
 #define OPCODE_VMRS_APSR 0xeef1fa10
 #define OPCODE_VMSR_FPSCR 0xeee10a10
 #define OPCODE_VMUL   0xee200b00
+#define OPCODE_VMUL_S16  0xf2100510
+#define OPCODE_VMULL_S16 0xf2900c00
 #define OPCODE_VORR_D 0xf2200110
+#define OPCODE_VPADDL_S16 0xf3b40200
 #define OPCODE_VQADD_S8  0xf2000010
 #define OPCODE_VQADD_S16 0xf2100010
 #define OPCODE_VQADD_U8  0xf3000010
@@ -907,15 +910,30 @@ void host_arm_VMRS_APSR(codeblock_t *block)
 {
 	codegen_addlong(block, COND_AL | OPCODE_VMRS_APSR);
 }
+
 void host_arm_VMUL_D(codeblock_t *block, int dst_reg, int src_reg_n, int src_reg_m)
 {
 	codegen_addlong(block, COND_AL | OPCODE_VMUL | Rd(dst_reg) | Rn(src_reg_n) | Rm(src_reg_m));
 }
+void host_arm_VMUL_S16(codeblock_t *block, int dst_reg, int src_reg_n, int src_reg_m)
+{
+	codegen_addlong(block, OPCODE_VMUL_S16 | Rd(dst_reg) | Rn(src_reg_n) | Rm(src_reg_m));
+}
+void host_arm_VMULL_S16(codeblock_t *block, int dst_reg, int src_reg_n, int src_reg_m)
+{
+	codegen_addlong(block, OPCODE_VMULL_S16 | Rd(dst_reg) | Rn(src_reg_n) | Rm(src_reg_m));
+}
+
 void host_arm_VORR_D(codeblock_t *block, int dst_reg, int src_reg_n, int src_reg_m)
 {
 	codegen_addlong(block, OPCODE_VORR_D | Rd(dst_reg) | Rn(src_reg_n) | Rm(src_reg_m));
 }
 
+void host_arm_VPADDL_S16(codeblock_t *block, int dst_reg, int src_reg)
+{
+	codegen_addlong(block, OPCODE_VPADDL_S16 | Vd(dst_reg) | Vm(src_reg));
+}
+
 void host_arm_VQADD_S8(codeblock_t *block, int dst_reg, int src_reg_n, int src_reg_m)
 {
 	codegen_addlong(block, OPCODE_VQADD_S8 | Rd(dst_reg) | Rn(src_reg_n) | Rm(src_reg_m));
diff --git a/src/codegen_backend_arm_ops.h b/src/codegen_backend_arm_ops.h
index 28c3299..946d7d5 100644
--- a/src/codegen_backend_arm_ops.h
+++ b/src/codegen_backend_arm_ops.h
@@ -170,8 +170,15 @@ void host_arm_VMOV_S_32(codeblock_t *block, int dest_reg, int src_reg);
 void host_arm_VMOV_D_D(codeblock_t *block, int dest_reg, int src_reg);
 void host_arm_VMRS_APSR(codeblock_t *block);
 void host_arm_VMSR_FPSCR(codeblock_t *block, int src_reg);
+
 void host_arm_VMUL_D(codeblock_t *block, int dst_reg, int src_reg_n, int src_reg_m);
+void host_arm_VMUL_S16(codeblock_t *block, int dest_reg, int src_reg_n, int src_reg_m);
+void host_arm_VMULL_S16(codeblock_t *block, int dest_reg, int src_reg_n, int src_reg_m);
+
 void host_arm_VORR_D(codeblock_t *block, int dst_reg, int src_reg_n, int src_reg_m);
+
+void host_arm_VPADDL_S16(codeblock_t *block, int dst_reg, int src_reg);
+
 void host_arm_VQADD_S8(codeblock_t *block, int dst_reg, int src_reg_n, int src_reg_m);
 void host_arm_VQADD_U8(codeblock_t *block, int dst_reg, int src_reg_n, int src_reg_m);
 void host_arm_VQADD_S16(codeblock_t *block, int dst_reg, int src_reg_n, int src_reg_m);
diff --git a/src/codegen_backend_arm_uops.c b/src/codegen_backend_arm_uops.c
index 7e9b6f8..11e9f8f 100644
--- a/src/codegen_backend_arm_uops.c
+++ b/src/codegen_backend_arm_uops.c
@@ -1737,6 +1737,51 @@ static int codegen_PCMPGTD(codeblock_t *block, uop_t *uop)
         return 0;
 }
 
+static int codegen_PMADDWD(codeblock_t *block, uop_t *uop)
+{
+        int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real);
+        int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real);
+
+        if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_b) && uop->dest_reg_a_real == uop->src_reg_a_real)
+        {
+                host_arm_VMUL_S16(block, dest_reg, src_reg_a, src_reg_b);
+                host_arm_VPADDL_S16(block, dest_reg, dest_reg);
+        }
+        else
+                fatal("PMULHW %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real);
+
+        return 0;
+}
+static int codegen_PMULHW(codeblock_t *block, uop_t *uop)
+{
+        int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real);
+        int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real);
+
+        if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_b) && uop->dest_reg_a_real == uop->src_reg_a_real)
+        {
+                host_arm_VMULL_S16(block, dest_reg, src_reg_a, src_reg_b);
+                host_arm_VSHRN_16(block, dest_reg, dest_reg, 16);
+        }
+        else
+                fatal("PMULHW %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real);
+
+        return 0;
+}
+static int codegen_PMULLW(codeblock_t *block, uop_t *uop)
+{
+        int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real);
+        int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real);
+
+        if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_b) && uop->dest_reg_a_real == uop->src_reg_a_real)
+        {
+                host_arm_VMUL_S16(block, dest_reg, src_reg_a, src_reg_b);
+        }
+        else
+                fatal("PMULLW %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real);
+
+        return 0;
+}
+
 static int codegen_PSLLW_IMM(codeblock_t *block, uop_t *uop)
 {
         int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg = HOST_REG_GET(uop->src_reg_a_real);
@@ -2640,6 +2685,10 @@ const uOpFn uop_handlers[UOP_MAX] =
         [UOP_PCMPGTW & UOP_MASK] = codegen_PCMPGTW,
         [UOP_PCMPGTD & UOP_MASK] = codegen_PCMPGTD,
 
+        [UOP_PMADDWD & UOP_MASK] = codegen_PMADDWD,
+        [UOP_PMULHW & UOP_MASK]  = codegen_PMULHW,
+        [UOP_PMULLW & UOP_MASK]  = codegen_PMULLW,
+
         [UOP_PSLLW_IMM & UOP_MASK] = codegen_PSLLW_IMM,
         [UOP_PSLLD_IMM & UOP_MASK] = codegen_PSLLD_IMM,
         [UOP_PSLLQ_IMM & UOP_MASK] = codegen_PSLLQ_IMM,
diff --git a/src/codegen_backend_x86-64_ops.c b/src/codegen_backend_x86-64_ops.c
index 9aabc0e..ed57f2a 100644
--- a/src/codegen_backend_x86-64_ops.c
+++ b/src/codegen_backend_x86-64_ops.c
@@ -1319,6 +1319,19 @@ void host_x86_PCMPGTD_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg)
         codegen_addbyte4(block, 0x66, 0x0f, 0x66, 0xc0 | src_reg | (dst_reg << 3)); /*PCMPGTD dst_reg, src_reg*/
 }
 
+void host_x86_PMADDWD_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg)
+{
+        codegen_addbyte4(block, 0x66, 0x0f, 0xf5, 0xc0 | src_reg | (dst_reg << 3)); /*PMULLW dst_reg, src_reg*/
+}
+void host_x86_PMULHW_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg)
+{
+        codegen_addbyte4(block, 0x66, 0x0f, 0xe5, 0xc0 | src_reg | (dst_reg << 3)); /*PMULLW dst_reg, src_reg*/
+}
+void host_x86_PMULLW_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg)
+{
+        codegen_addbyte4(block, 0x66, 0x0f, 0xd5, 0xc0 | src_reg | (dst_reg << 3)); /*PMULLW dst_reg, src_reg*/
+}
+
 void host_x86_PSLLW_XREG_IMM(codeblock_t *block, int dst_reg, int shift)
 {
         codegen_addbyte4(block, 0x66, 0x0f, 0x71, 0xc0 | 0x30 | dst_reg); /*PSLLW dst_reg, imm*/
diff --git a/src/codegen_backend_x86-64_ops.h b/src/codegen_backend_x86-64_ops.h
index 67bccb7..35510d0 100644
--- a/src/codegen_backend_x86-64_ops.h
+++ b/src/codegen_backend_x86-64_ops.h
@@ -177,6 +177,10 @@ void host_x86_PCMPGTB_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg);
 void host_x86_PCMPGTW_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg);
 void host_x86_PCMPGTD_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg);
 
+void host_x86_PMADDWD_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg);
+void host_x86_PMULHW_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg);
+void host_x86_PMULLW_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg);
+
 void host_x86_PSLLW_XREG_IMM(codeblock_t *block, int dst_reg, int shift);
 void host_x86_PSLLD_XREG_IMM(codeblock_t *block, int dst_reg, int shift);
 void host_x86_PSLLQ_XREG_IMM(codeblock_t *block, int dst_reg, int shift);
diff --git a/src/codegen_backend_x86-64_uops.c b/src/codegen_backend_x86-64_uops.c
index d960dca..faf5c33 100644
--- a/src/codegen_backend_x86-64_uops.c
+++ b/src/codegen_backend_x86-64_uops.c
@@ -1497,6 +1497,49 @@ static int codegen_PCMPGTD(codeblock_t *block, uop_t *uop)
         return 0;
 }
 
+static int codegen_PMADDWD(codeblock_t *block, uop_t *uop)
+{
+        int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real);
+        int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real);
+
+        if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_b) && uop->dest_reg_a_real == uop->src_reg_a_real)
+        {
+                host_x86_PMADDWD_XREG_XREG(block, dest_reg, src_reg_b);
+        }
+        else
+                fatal("PMULHW %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real);
+
+        return 0;
+}
+static int codegen_PMULHW(codeblock_t *block, uop_t *uop)
+{
+        int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real);
+        int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real);
+
+        if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_b) && uop->dest_reg_a_real == uop->src_reg_a_real)
+        {
+                host_x86_PMULHW_XREG_XREG(block, dest_reg, src_reg_b);
+        }
+        else
+                fatal("PMULHW %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real);
+
+        return 0;
+}
+static int codegen_PMULLW(codeblock_t *block, uop_t *uop)
+{
+        int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real);
+        int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real);
+
+        if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_b) && uop->dest_reg_a_real == uop->src_reg_a_real)
+        {
+                host_x86_PMULLW_XREG_XREG(block, dest_reg, src_reg_b);
+        }
+        else
+                fatal("PMULLW %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real);
+
+        return 0;
+}
+
 static int codegen_PSLLW_IMM(codeblock_t *block, uop_t *uop)
 {
         int dest_reg = HOST_REG_GET(uop->dest_reg_a_real);
@@ -2264,6 +2307,10 @@ const uOpFn uop_handlers[UOP_MAX] =
         [UOP_PCMPGTW & UOP_MASK] = codegen_PCMPGTW,
         [UOP_PCMPGTD & UOP_MASK] = codegen_PCMPGTD,
 
+        [UOP_PMADDWD & UOP_MASK] = codegen_PMADDWD,
+        [UOP_PMULHW & UOP_MASK]  = codegen_PMULHW,
+        [UOP_PMULLW & UOP_MASK]  = codegen_PMULLW,
+
         [UOP_PSLLW_IMM & UOP_MASK] = codegen_PSLLW_IMM,
         [UOP_PSLLD_IMM & UOP_MASK] = codegen_PSLLD_IMM,
         [UOP_PSLLQ_IMM & UOP_MASK] = codegen_PSLLQ_IMM,
diff --git a/src/codegen_backend_x86_ops.c b/src/codegen_backend_x86_ops.c
index 8376908..e477122 100644
--- a/src/codegen_backend_x86_ops.c
+++ b/src/codegen_backend_x86_ops.c
@@ -1177,6 +1177,19 @@ void host_x86_PCMPGTD_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg)
         codegen_addbyte4(block, 0x66, 0x0f, 0x66, 0xc0 | src_reg | (dst_reg << 3)); /*PCMPGTD dst_reg, src_reg*/
 }
 
+void host_x86_PMADDWD_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg)
+{
+        codegen_addbyte4(block, 0x66, 0x0f, 0xf5, 0xc0 | src_reg | (dst_reg << 3)); /*PMULLW dst_reg, src_reg*/
+}
+void host_x86_PMULHW_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg)
+{
+        codegen_addbyte4(block, 0x66, 0x0f, 0xe5, 0xc0 | src_reg | (dst_reg << 3)); /*PMULLW dst_reg, src_reg*/
+}
+void host_x86_PMULLW_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg)
+{
+        codegen_addbyte4(block, 0x66, 0x0f, 0xd5, 0xc0 | src_reg | (dst_reg << 3)); /*PMULLW dst_reg, src_reg*/
+}
+
 void host_x86_PSLLW_XREG_IMM(codeblock_t *block, int dst_reg, int shift)
 {
         codegen_addbyte4(block, 0x66, 0x0f, 0x71, 0xc0 | 0x30 | dst_reg); /*PSLLW dst_reg, imm*/
diff --git a/src/codegen_backend_x86_ops.h b/src/codegen_backend_x86_ops.h
index 491c719..cd9eb4e 100644
--- a/src/codegen_backend_x86_ops.h
+++ b/src/codegen_backend_x86_ops.h
@@ -178,6 +178,10 @@ void host_x86_PCMPGTB_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg);
 void host_x86_PCMPGTW_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg);
 void host_x86_PCMPGTD_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg);
 
+void host_x86_PMADDWD_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg);
+void host_x86_PMULHW_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg);
+void host_x86_PMULLW_XREG_XREG(codeblock_t *block, int dst_reg, int src_reg);
+
 void host_x86_PSLLW_XREG_IMM(codeblock_t *block, int dst_reg, int shift);
 void host_x86_PSLLD_XREG_IMM(codeblock_t *block, int dst_reg, int shift);
 void host_x86_PSLLQ_XREG_IMM(codeblock_t *block, int dst_reg, int shift);
diff --git a/src/codegen_backend_x86_uops.c b/src/codegen_backend_x86_uops.c
index 654843b..c977bae 100644
--- a/src/codegen_backend_x86_uops.c
+++ b/src/codegen_backend_x86_uops.c
@@ -1500,6 +1500,49 @@ static int codegen_PCMPGTD(codeblock_t *block, uop_t *uop)
         return 0;
 }
 
+static int codegen_PMADDWD(codeblock_t *block, uop_t *uop)
+{
+        int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real);
+        int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real);
+
+        if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_b) && uop->dest_reg_a_real == uop->src_reg_a_real)
+        {
+                host_x86_PMADDWD_XREG_XREG(block, dest_reg, src_reg_b);
+        }
+        else
+                fatal("PMULHW %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real);
+
+        return 0;
+}
+static int codegen_PMULHW(codeblock_t *block, uop_t *uop)
+{
+        int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real);
+        int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real);
+
+        if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_b) && uop->dest_reg_a_real == uop->src_reg_a_real)
+        {
+                host_x86_PMULHW_XREG_XREG(block, dest_reg, src_reg_b);
+        }
+        else
+                fatal("PMULHW %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real);
+
+        return 0;
+}
+static int codegen_PMULLW(codeblock_t *block, uop_t *uop)
+{
+        int dest_reg = HOST_REG_GET(uop->dest_reg_a_real), src_reg_b = HOST_REG_GET(uop->src_reg_b_real);
+        int dest_size = IREG_GET_SIZE(uop->dest_reg_a_real), src_size_b = IREG_GET_SIZE(uop->src_reg_b_real);
+
+        if (REG_IS_Q(dest_size) && REG_IS_Q(src_size_b) && uop->dest_reg_a_real == uop->src_reg_a_real)
+        {
+                host_x86_PMULLW_XREG_XREG(block, dest_reg, src_reg_b);
+        }
+        else
+                fatal("PMULLW %02x %02x %02x\n", uop->dest_reg_a_real, uop->src_reg_a_real, uop->src_reg_b_real);
+
+        return 0;
+}
+
 static int codegen_PSLLW_IMM(codeblock_t *block, uop_t *uop)
 {
         int dest_reg = HOST_REG_GET(uop->dest_reg_a_real);
@@ -2264,6 +2307,10 @@ const uOpFn uop_handlers[UOP_MAX] =
         [UOP_PCMPGTW & UOP_MASK] = codegen_PCMPGTW,
         [UOP_PCMPGTD & UOP_MASK] = codegen_PCMPGTD,
 
+        [UOP_PMADDWD & UOP_MASK] = codegen_PMADDWD,
+        [UOP_PMULHW & UOP_MASK]  = codegen_PMULHW,
+        [UOP_PMULLW & UOP_MASK]  = codegen_PMULLW,
+        
         [UOP_PSLLW_IMM & UOP_MASK] = codegen_PSLLW_IMM,
         [UOP_PSLLD_IMM & UOP_MASK] = codegen_PSLLD_IMM,
         [UOP_PSLLQ_IMM & UOP_MASK] = codegen_PSLLQ_IMM,
diff --git a/src/codegen_ir_defs.h b/src/codegen_ir_defs.h
index 4029c2c..eea588f 100644
--- a/src/codegen_ir_defs.h
+++ b/src/codegen_ir_defs.h
@@ -260,8 +260,14 @@
 #define UOP_PACKSSDW              (UOP_TYPE_PARAMS_REGS | 0xb5)
 /*UOP_PACKUSWB - dest_reg = interleave src_reg_a/src_reg_b, converting words to bytes with unsigned saturation*/
 #define UOP_PACKUSWB              (UOP_TYPE_PARAMS_REGS | 0xb6)
+/*UOP_PMULLW - (packed word) dest_reg = (src_reg_a * src_reg_b) & 0xffff*/
+#define UOP_PMULLW                (UOP_TYPE_PARAMS_REGS | 0xb7)
+/*UOP_PMULHW - (packed word) dest_reg = (src_reg_a * src_reg_b) >> 16*/
+#define UOP_PMULHW                (UOP_TYPE_PARAMS_REGS | 0xb8)
+/*UOP_PMADDWD - (packed word) dest_reg = (src_reg_a * src_reg_b) >> 16*/
+#define UOP_PMADDWD               (UOP_TYPE_PARAMS_REGS | 0xb9)
 
-#define UOP_MAX 0xb7
+#define UOP_MAX 0xba
 
 #define UOP_MASK 0xffff
 
@@ -620,6 +626,10 @@ static inline void uop_gen_reg_src_pointer_imm(uint32_t uop_type, ir_data_t *ir,
 #define uop_PCMPGTW(ir, dst_reg, src_reg_a, src_reg_b) uop_gen_reg_dst_src2(UOP_PCMPGTW, ir, dst_reg, src_reg_a, src_reg_b)
 #define uop_PCMPGTD(ir, dst_reg, src_reg_a, src_reg_b) uop_gen_reg_dst_src2(UOP_PCMPGTD, ir, dst_reg, src_reg_a, src_reg_b)
 
+#define uop_PMADDWD(ir, dst_reg, src_reg_a, src_reg_b)  uop_gen_reg_dst_src2(UOP_PMADDWD, ir, dst_reg, src_reg_a, src_reg_b)
+#define uop_PMULHW(ir, dst_reg, src_reg_a, src_reg_b)  uop_gen_reg_dst_src2(UOP_PMULHW, ir, dst_reg, src_reg_a, src_reg_b)
+#define uop_PMULLW(ir, dst_reg, src_reg_a, src_reg_b)  uop_gen_reg_dst_src2(UOP_PMULLW, ir, dst_reg, src_reg_a, src_reg_b)
+
 #define uop_PSLLW_IMM(ir, dst_reg, src_reg, imm)       uop_gen_reg_dst_src_imm(UOP_PSLLW_IMM, ir, dst_reg, src_reg, imm)
 #define uop_PSLLD_IMM(ir, dst_reg, src_reg, imm)       uop_gen_reg_dst_src_imm(UOP_PSLLD_IMM, ir, dst_reg, src_reg, imm)
 #define uop_PSLLQ_IMM(ir, dst_reg, src_reg, imm)       uop_gen_reg_dst_src_imm(UOP_PSLLQ_IMM, ir, dst_reg, src_reg, imm)
diff --git a/src/codegen_ops.c b/src/codegen_ops.c
index 71a2e86..00e4018 100644
--- a/src/codegen_ops.c
+++ b/src/codegen_ops.c
@@ -89,9 +89,9 @@ RecompOpFn recomp_opcodes_0f[512] =
 /*b0*/  NULL,           NULL,           ropLSS_16,      NULL,           ropLFS_16,      ropLGS_16,      ropMOVZX_16_8,  NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           ropMOVSX_16_8,  NULL,
 
 /*c0*/  NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,
-/*d0*/  NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           ropPSUBUSB,     ropPSUBUSW,     NULL,           ropPAND,        ropPADDUSB,     ropPADDUSW,     NULL,           ropPANDN,
-/*e0*/  NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           ropPSUBSB,      ropPSUBSW,      NULL,           ropPOR,         ropPADDSB,      ropPADDSW,      NULL,           ropPXOR,
-/*f0*/  NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           ropPSUBB,       ropPSUBW,       ropPSUBD,       NULL,           ropPADDB,       ropPADDW,       ropPADDD,       NULL,
+/*d0*/  NULL,           NULL,           NULL,           NULL,           NULL,           ropPMULLW,      NULL,           NULL,           ropPSUBUSB,     ropPSUBUSW,     NULL,           ropPAND,        ropPADDUSB,     ropPADDUSW,     NULL,           ropPANDN,
+/*e0*/  NULL,           NULL,           NULL,           NULL,           NULL,           ropPMULHW,      NULL,           NULL,           ropPSUBSB,      ropPSUBSW,      NULL,           ropPOR,         ropPADDSB,      ropPADDSW,      NULL,           ropPXOR,
+/*f0*/  NULL,           NULL,           NULL,           NULL,           NULL,           ropPMADDWD,     NULL,           NULL,           ropPSUBB,       ropPSUBW,       ropPSUBD,       NULL,           ropPADDB,       ropPADDW,       ropPADDD,       NULL,
 
         /*32-bit data*/
 /*      00              01              02              03              04              05              06              07              08              09              0a              0b              0c              0d              0e              0f*/
@@ -111,9 +111,9 @@ RecompOpFn recomp_opcodes_0f[512] =
 /*b0*/  NULL,           NULL,           ropLSS_32,      NULL,           ropLFS_32,      ropLGS_32,      ropMOVZX_32_8,  ropMOVZX_32_16, NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           ropMOVSX_32_8,  ropMOVSX_32_16,
 
 /*c0*/  NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,
-/*d0*/  NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           ropPSUBUSB,     ropPSUBUSW,     NULL,           ropPAND,        ropPADDUSB,     ropPADDUSW,     NULL,           ropPANDN,
-/*e0*/  NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           ropPSUBSB,      ropPSUBSW,      NULL,           ropPOR,         ropPADDSB,      ropPADDSW,      NULL,           ropPXOR,
-/*f0*/  NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           NULL,           ropPSUBB,       ropPSUBW,       ropPSUBD,       NULL,           ropPADDB,       ropPADDW,       ropPADDD,       NULL,
+/*d0*/  NULL,           NULL,           NULL,           NULL,           NULL,           ropPMULLW,      NULL,           NULL,           ropPSUBUSB,     ropPSUBUSW,     NULL,           ropPAND,        ropPADDUSB,     ropPADDUSW,     NULL,           ropPANDN,
+/*e0*/  NULL,           NULL,           NULL,           NULL,           NULL,           ropPMULHW,      NULL,           NULL,           ropPSUBSB,      ropPSUBSW,      NULL,           ropPOR,         ropPADDSB,      ropPADDSW,      NULL,           ropPXOR,
+/*f0*/  NULL,           NULL,           NULL,           NULL,           NULL,           ropPMADDWD,     NULL,           NULL,           ropPSUBB,       ropPSUBW,       ropPSUBD,       NULL,           ropPADDB,       ropPADDW,       ropPADDD,       NULL,
 };
 
 RecompOpFn recomp_opcodes_d8[512] =
diff --git a/src/codegen_ops_mmx_arith.c b/src/codegen_ops_mmx_arith.c
index 848c046..cd23e3c 100644
--- a/src/codegen_ops_mmx_arith.c
+++ b/src/codegen_ops_mmx_arith.c
@@ -50,3 +50,7 @@ ropParith(PSUBSB)
 ropParith(PSUBSW)
 ropParith(PSUBUSB)
 ropParith(PSUBUSW)
+
+ropParith(PMADDWD)
+ropParith(PMULHW)
+ropParith(PMULLW)
diff --git a/src/codegen_ops_mmx_arith.h b/src/codegen_ops_mmx_arith.h
index e848c8d..5f67f0a 100644
--- a/src/codegen_ops_mmx_arith.h
+++ b/src/codegen_ops_mmx_arith.h
@@ -13,3 +13,7 @@ uint32_t ropPSUBSB(codeblock_t *block, ir_data_t *ir, uint8_t opcode, uint32_t f
 uint32_t ropPSUBSW(codeblock_t *block, ir_data_t *ir, uint8_t opcode, uint32_t fetchdat, uint32_t op_32, uint32_t op_pc);
 uint32_t ropPSUBUSB(codeblock_t *block, ir_data_t *ir, uint8_t opcode, uint32_t fetchdat, uint32_t op_32, uint32_t op_pc);
 uint32_t ropPSUBUSW(codeblock_t *block, ir_data_t *ir, uint8_t opcode, uint32_t fetchdat, uint32_t op_32, uint32_t op_pc);
+
+uint32_t ropPMADDWD(codeblock_t *block, ir_data_t *ir, uint8_t opcode, uint32_t fetchdat, uint32_t op_32, uint32_t op_pc);
+uint32_t ropPMULHW(codeblock_t *block, ir_data_t *ir, uint8_t opcode, uint32_t fetchdat, uint32_t op_32, uint32_t op_pc);
+uint32_t ropPMULLW(codeblock_t *block, ir_data_t *ir, uint8_t opcode, uint32_t fetchdat, uint32_t op_32, uint32_t op_pc);