(V8SI "TARGET_AVX2") V4SI
(V4DI "TARGET_AVX2") V2DI])
-(define_mode_iterator VI48_AVX2
- [V8SI V4SI V4DI V2DI])
-
(define_mode_iterator VI4SD_AVX2
[V4SI V4DI])
;; Random 256bit vector integer mode combinations
(define_mode_iterator VI124_256 [V32QI V16HI V8SI])
-(define_mode_iterator VI248_256 [V16HI V8SI V4DI])
;; Int-float size matches
(define_mode_iterator VI4F_128 [V4SI V4SF])
(V8SI "V8SI") (V4DI "V4DI")
(V4SI "V4SI") (V2DI "V2DI")
(V16HI "V16HI") (V8HI "V8HI")
- (V32QI "V32QI") (V16QI "V16QI")
- ])
+ (V32QI "V32QI") (V16QI "V16QI")])
;; Mapping of vector modes to a vector mode of double size
(define_mode_attr ssedoublevecmode
(V8SF "8") (V4DF "4")
(V4SF "4") (V2DF "2")])
+;; SSE prefix for integer vector modes
+(define_mode_attr sseintprefix
+ [(V2DI "p") (V2DF "")
+ (V4DI "p") (V4DF "")
+ (V4SI "p") (V4SF "")
+ (V8SI "p") (V8SF "")])
+
;; SSE scalar suffix for vector modes
(define_mode_attr ssescalarmodesuffix
[(SF "ss") (DF "sd")
(V4DI "V4DI") (V4DF "V4DI")
(V4SI "V2DI") (V4SF "V2DI")
(V8SI "V4DI") (V8SF "V4DI")])
-(define_mode_attr gthrfirstp
- [(V2DI "p") (V2DF "")
- (V4DI "p") (V4DF "")
- (V4SI "p") (V4SF "")
- (V8SI "p") (V8SF "")])
-(define_mode_attr gthrlastp
- [(V2DI "q") (V2DF "pd")
- (V4DI "q") (V4DF "pd")
- (V4SI "d") (V4SF "ps")
- (V8SI "d") (V8SF "ps")])
(define_mode_iterator FMAMODE [SF DF V4SF V2DF V8SF V4DF])
(match_operand:V4SI 1 "nonimmediate_operand" "xm")
(parallel [(const_int 0) (const_int 1)]))))]
"TARGET_SSE2"
- "%vcvtdq2pd\t{%1, %0|%0, %1}"
+ "%vcvtdq2pd\t{%1, %0|%0, %q1}"
[(set_attr "type" "ssecvt")
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "V2DF")])
(match_operand:V4SF 1 "nonimmediate_operand" "xm")
(parallel [(const_int 0) (const_int 1)]))))]
"TARGET_SSE2"
- "%vcvtps2pd\t{%1, %0|%0, %1}"
+ "%vcvtps2pd\t{%1, %0|%0, %q1}"
[(set_attr "type" "ssecvt")
(set_attr "amdfam10_decode" "direct")
(set_attr "athlon_decode" "double")
DONE;
})
+(define_insn_and_split "*sse4_1_extractps"
+ [(set (match_operand:SF 0 "nonimmediate_operand" "=rm,x,x")
+ (vec_select:SF
+ (match_operand:V4SF 1 "register_operand" "x,0,x")
+ (parallel [(match_operand:SI 2 "const_0_to_3_operand" "n,n,n")])))]
+ "TARGET_SSE4_1"
+ "@
+ %vextractps\t{%2, %1, %0|%0, %1, %2}
+ #
+ #"
+ "&& reload_completed && SSE_REG_P (operands[0])"
+ [(const_int 0)]
+{
+ rtx dest = gen_rtx_REG (V4SFmode, REGNO (operands[0]));
+ switch (INTVAL (operands[2]))
+ {
+ case 1:
+ case 3:
+ emit_insn (gen_sse_shufps_v4sf (dest, operands[1], operands[1],
+ operands[2], operands[2],
+ GEN_INT (INTVAL (operands[2]) + 4),
+ GEN_INT (INTVAL (operands[2]) + 4)));
+ break;
+ case 2:
+ emit_insn (gen_vec_interleave_highv4sf (dest, operands[1], operands[1]));
+ break;
+ default:
+ /* 0 should be handled by the *vec_extractv4sf_0 pattern above. */
+ gcc_unreachable ();
+ }
+ DONE;
+}
+ [(set_attr "isa" "*,noavx,avx")
+ (set_attr "type" "sselog,*,*")
+ (set_attr "prefix_data16" "1,*,*")
+ (set_attr "prefix_extra" "1,*,*")
+ (set_attr "length_immediate" "1,*,*")
+ (set_attr "prefix" "maybe_vex,*,*")
+ (set_attr "mode" "V4SF,*,*")])
+
+(define_insn_and_split "*vec_extract_v4sf_mem"
+ [(set (match_operand:SF 0 "register_operand" "=x,*r,f")
+ (vec_select:SF
+ (match_operand:V4SF 1 "memory_operand" "o,o,o")
+ (parallel [(match_operand 2 "const_0_to_3_operand" "n,n,n")])))]
+ "TARGET_SSE"
+ "#"
+ "&& reload_completed"
+ [(const_int 0)]
+{
+ int i = INTVAL (operands[2]);
+
+ emit_move_insn (operands[0], adjust_address (operands[1], SFmode, i*4));
+ DONE;
+})
+
(define_expand "avx_vextractf128<mode>"
[(match_operand:<ssehalfvecmode> 0 "nonimmediate_operand" "")
(match_operand:V_256 1 "register_operand" "")
(set_attr "prefix" "vex")
(set_attr "mode" "OI")])
-(define_insn_and_split "*sse4_1_extractps"
- [(set (match_operand:SF 0 "nonimmediate_operand" "=rm,x,x")
- (vec_select:SF
- (match_operand:V4SF 1 "register_operand" "x,0,x")
- (parallel [(match_operand:SI 2 "const_0_to_3_operand" "n,n,n")])))]
- "TARGET_SSE4_1"
- "@
- %vextractps\t{%2, %1, %0|%0, %1, %2}
- #
- #"
- "&& reload_completed && SSE_REG_P (operands[0])"
- [(const_int 0)]
-{
- rtx dest = gen_rtx_REG (V4SFmode, REGNO (operands[0]));
- switch (INTVAL (operands[2]))
- {
- case 1:
- case 3:
- emit_insn (gen_sse_shufps_v4sf (dest, operands[1], operands[1],
- operands[2], operands[2],
- GEN_INT (INTVAL (operands[2]) + 4),
- GEN_INT (INTVAL (operands[2]) + 4)));
- break;
- case 2:
- emit_insn (gen_vec_interleave_highv4sf (dest, operands[1], operands[1]));
- break;
- default:
- /* 0 should be handled by the *vec_extractv4sf_0 pattern above. */
- gcc_unreachable ();
- }
- DONE;
-}
- [(set_attr "isa" "*,noavx,avx")
- (set_attr "type" "sselog,*,*")
- (set_attr "prefix_data16" "1,*,*")
- (set_attr "prefix_extra" "1,*,*")
- (set_attr "length_immediate" "1,*,*")
- (set_attr "prefix" "maybe_vex,*,*")
- (set_attr "mode" "V4SF,*,*")])
-
-(define_insn_and_split "*vec_extract_v4sf_mem"
- [(set (match_operand:SF 0 "register_operand" "=x*rf")
- (vec_select:SF
- (match_operand:V4SF 1 "memory_operand" "o")
- (parallel [(match_operand 2 "const_0_to_3_operand" "n")])))]
- "TARGET_SSE"
- "#"
- "&& reload_completed"
- [(const_int 0)]
-{
- int i = INTVAL (operands[2]);
-
- emit_move_insn (operands[0], adjust_address (operands[1], SFmode, i*4));
- DONE;
-})
-
;; Modes handled by vec_extract patterns.
(define_mode_iterator VEC_EXTRACT_MODE
[(V32QI "TARGET_AVX") V16QI
[(set (match_operand:V4DF 0 "register_operand" "=x,x")
(vec_select:V4DF
(vec_concat:V8DF
- (match_operand:V4DF 1 "nonimmediate_operand" "xm,x")
- (match_operand:V4DF 2 "nonimmediate_operand" " 1,xm"))
+ (match_operand:V4DF 1 "nonimmediate_operand" " x,m")
+ (match_operand:V4DF 2 "nonimmediate_operand" "xm,1"))
(parallel [(const_int 0) (const_int 4)
(const_int 2) (const_int 6)])))]
- "TARGET_AVX
- && (!MEM_P (operands[1]) || rtx_equal_p (operands[1], operands[2]))"
+ "TARGET_AVX"
"@
- vmovddup\t{%1, %0|%0, %1}
- vunpcklpd\t{%2, %1, %0|%0, %1, %2}"
+ vunpcklpd\t{%2, %1, %0|%0, %1, %2}
+ vmovddup\t{%1, %0|%0, %1}"
[(set_attr "type" "sselog")
(set_attr "prefix" "vex")
(set_attr "mode" "V4DF")])
DONE;
})
-;; Modes handled by vec_extract_even/odd pattern.
-(define_mode_iterator VEC_EXTRACT_EVENODD_MODE
- [(V32QI "TARGET_AVX2") (V16QI "TARGET_SSE2")
- (V16HI "TARGET_AVX2") (V8HI "TARGET_SSE2")
- (V8SI "TARGET_AVX2") (V4SI "TARGET_SSE2")
- (V4DI "TARGET_AVX2") (V2DI "TARGET_SSE2")
- (V8SF "TARGET_AVX") V4SF
- (V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
-
-(define_expand "vec_extract_even<mode>"
- [(match_operand:VEC_EXTRACT_EVENODD_MODE 0 "register_operand" "")
- (match_operand:VEC_EXTRACT_EVENODD_MODE 1 "register_operand" "")
- (match_operand:VEC_EXTRACT_EVENODD_MODE 2 "register_operand" "")]
- "TARGET_SSE"
-{
- ix86_expand_vec_extract_even_odd (operands[0], operands[1], operands[2], 0);
- DONE;
-})
-
-(define_expand "vec_extract_odd<mode>"
- [(match_operand:VEC_EXTRACT_EVENODD_MODE 0 "register_operand" "")
- (match_operand:VEC_EXTRACT_EVENODD_MODE 1 "register_operand" "")
- (match_operand:VEC_EXTRACT_EVENODD_MODE 2 "register_operand" "")]
- "TARGET_SSE"
-{
- ix86_expand_vec_extract_even_odd (operands[0], operands[1], operands[2], 1);
- DONE;
-})
-
;; punpcklqdq and punpckhqdq are shorter than shufpd.
(define_insn "avx2_interleave_highv4di"
[(set (match_operand:V4DI 0 "register_operand" "=x")
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(define_expand "neg<mode>2"
- [(set (match_operand:VI_128 0 "register_operand" "")
- (minus:VI_128
+ [(set (match_operand:VI_AVX2 0 "register_operand" "")
+ (minus:VI_AVX2
(match_dup 2)
- (match_operand:VI_128 1 "nonimmediate_operand" "")))]
+ (match_operand:VI_AVX2 1 "nonimmediate_operand" "")))]
"TARGET_SSE2"
"operands[2] = force_reg (<MODE>mode, CONST0_RTX (<MODE>mode));")
DONE;
})
-(define_expand "vec_widen_smult_hi_v8hi"
- [(match_operand:V4SI 0 "register_operand" "")
- (match_operand:V8HI 1 "register_operand" "")
- (match_operand:V8HI 2 "register_operand" "")]
+(define_expand "vec_widen_<s>mult_hi_<mode>"
+ [(match_operand:<sseunpackmode> 0 "register_operand" "")
+ (any_extend:<sseunpackmode>
+ (match_operand:VI2_AVX2 1 "register_operand" ""))
+ (match_operand:VI2_AVX2 2 "register_operand" "")]
"TARGET_SSE2"
{
rtx op1, op2, t1, t2, dest;
op1 = operands[1];
op2 = operands[2];
- t1 = gen_reg_rtx (V8HImode);
- t2 = gen_reg_rtx (V8HImode);
- dest = gen_lowpart (V8HImode, operands[0]);
+ t1 = gen_reg_rtx (<MODE>mode);
+ t2 = gen_reg_rtx (<MODE>mode);
+ dest = gen_lowpart (<MODE>mode, operands[0]);
- emit_insn (gen_mulv8hi3 (t1, op1, op2));
- emit_insn (gen_smulv8hi3_highpart (t2, op1, op2));
- emit_insn (gen_vec_interleave_highv8hi (dest, t1, t2));
+ emit_insn (gen_mul<mode>3 (t1, op1, op2));
+ emit_insn (gen_<s>mul<mode>3_highpart (t2, op1, op2));
+ emit_insn (gen_vec_interleave_high<mode> (dest, t1, t2));
DONE;
})
-(define_expand "vec_widen_smult_lo_v8hi"
- [(match_operand:V4SI 0 "register_operand" "")
- (match_operand:V8HI 1 "register_operand" "")
- (match_operand:V8HI 2 "register_operand" "")]
+(define_expand "vec_widen_<s>mult_lo_<mode>"
+ [(match_operand:<sseunpackmode> 0 "register_operand" "")
+ (any_extend:<sseunpackmode>
+ (match_operand:VI2_AVX2 1 "register_operand" ""))
+ (match_operand:VI2_AVX2 2 "register_operand" "")]
"TARGET_SSE2"
{
rtx op1, op2, t1, t2, dest;
op1 = operands[1];
op2 = operands[2];
- t1 = gen_reg_rtx (V8HImode);
- t2 = gen_reg_rtx (V8HImode);
- dest = gen_lowpart (V8HImode, operands[0]);
+ t1 = gen_reg_rtx (<MODE>mode);
+ t2 = gen_reg_rtx (<MODE>mode);
+ dest = gen_lowpart (<MODE>mode, operands[0]);
- emit_insn (gen_mulv8hi3 (t1, op1, op2));
- emit_insn (gen_smulv8hi3_highpart (t2, op1, op2));
- emit_insn (gen_vec_interleave_lowv8hi (dest, t1, t2));
+ emit_insn (gen_mul<mode>3 (t1, op1, op2));
+ emit_insn (gen_<s>mul<mode>3_highpart (t2, op1, op2));
+ emit_insn (gen_vec_interleave_low<mode> (dest, t1, t2));
DONE;
})
-(define_expand "vec_widen_umult_hi_v8hi"
- [(match_operand:V4SI 0 "register_operand" "")
- (match_operand:V8HI 1 "register_operand" "")
- (match_operand:V8HI 2 "register_operand" "")]
- "TARGET_SSE2"
+(define_expand "vec_widen_<s>mult_hi_v8si"
+ [(match_operand:V4DI 0 "register_operand" "")
+ (any_extend:V4DI (match_operand:V8SI 1 "nonimmediate_operand" ""))
+ (match_operand:V8SI 2 "nonimmediate_operand" "")]
+ "TARGET_AVX2"
{
- rtx op1, op2, t1, t2, dest;
-
- op1 = operands[1];
- op2 = operands[2];
- t1 = gen_reg_rtx (V8HImode);
- t2 = gen_reg_rtx (V8HImode);
- dest = gen_lowpart (V8HImode, operands[0]);
+ rtx t1, t2, t3, t4;
- emit_insn (gen_mulv8hi3 (t1, op1, op2));
- emit_insn (gen_umulv8hi3_highpart (t2, op1, op2));
- emit_insn (gen_vec_interleave_highv8hi (dest, t1, t2));
+ t1 = gen_reg_rtx (V4DImode);
+ t2 = gen_reg_rtx (V4DImode);
+ t3 = gen_reg_rtx (V8SImode);
+ t4 = gen_reg_rtx (V8SImode);
+ emit_insn (gen_avx2_permv4di_1 (t1, gen_lowpart (V4DImode, operands[1]),
+ const0_rtx, const2_rtx,
+ const1_rtx, GEN_INT (3)));
+ emit_insn (gen_avx2_permv4di_1 (t2, gen_lowpart (V4DImode, operands[2]),
+ const0_rtx, const2_rtx,
+ const1_rtx, GEN_INT (3)));
+ emit_insn (gen_avx2_pshufdv3 (t3, gen_lowpart (V8SImode, t1),
+ GEN_INT (2 + (2 << 2) + (3 << 4) + (3 << 6))));
+ emit_insn (gen_avx2_pshufdv3 (t4, gen_lowpart (V8SImode, t2),
+ GEN_INT (2 + (2 << 2) + (3 << 4) + (3 << 6))));
+ emit_insn (gen_avx2_<u>mulv4siv4di3 (operands[0], t3, t4));
DONE;
})
-(define_expand "vec_widen_umult_lo_v8hi"
- [(match_operand:V4SI 0 "register_operand" "")
- (match_operand:V8HI 1 "register_operand" "")
- (match_operand:V8HI 2 "register_operand" "")]
- "TARGET_SSE2"
+(define_expand "vec_widen_<s>mult_lo_v8si"
+ [(match_operand:V4DI 0 "register_operand" "")
+ (any_extend:V4DI (match_operand:V8SI 1 "nonimmediate_operand" ""))
+ (match_operand:V8SI 2 "nonimmediate_operand" "")]
+ "TARGET_AVX2"
{
- rtx op1, op2, t1, t2, dest;
-
- op1 = operands[1];
- op2 = operands[2];
- t1 = gen_reg_rtx (V8HImode);
- t2 = gen_reg_rtx (V8HImode);
- dest = gen_lowpart (V8HImode, operands[0]);
+ rtx t1, t2, t3, t4;
- emit_insn (gen_mulv8hi3 (t1, op1, op2));
- emit_insn (gen_umulv8hi3_highpart (t2, op1, op2));
- emit_insn (gen_vec_interleave_lowv8hi (dest, t1, t2));
+ t1 = gen_reg_rtx (V4DImode);
+ t2 = gen_reg_rtx (V4DImode);
+ t3 = gen_reg_rtx (V8SImode);
+ t4 = gen_reg_rtx (V8SImode);
+ emit_insn (gen_avx2_permv4di_1 (t1, gen_lowpart (V4DImode, operands[1]),
+ const0_rtx, const2_rtx,
+ const1_rtx, GEN_INT (3)));
+ emit_insn (gen_avx2_permv4di_1 (t2, gen_lowpart (V4DImode, operands[2]),
+ const0_rtx, const2_rtx,
+ const1_rtx, GEN_INT (3)));
+ emit_insn (gen_avx2_pshufdv3 (t3, gen_lowpart (V8SImode, t1),
+ GEN_INT (0 + (0 << 2) + (1 << 4) + (1 << 6))));
+ emit_insn (gen_avx2_pshufdv3 (t4, gen_lowpart (V8SImode, t2),
+ GEN_INT (0 + (0 << 2) + (1 << 4) + (1 << 6))));
+ emit_insn (gen_avx2_<u>mulv4siv4di3 (operands[0], t3, t4));
DONE;
})
[(match_operand:V2DI 0 "register_operand" "")
(match_operand:V4SI 1 "register_operand" "")
(match_operand:V4SI 2 "register_operand" "")]
- "TARGET_XOP"
+ "TARGET_SSE4_1"
{
- rtx t1, t2;
+ rtx op1, op2, t1, t2;
+ op1 = operands[1];
+ op2 = operands[2];
t1 = gen_reg_rtx (V4SImode);
t2 = gen_reg_rtx (V4SImode);
- emit_insn (gen_sse2_pshufd_1 (t1, operands[1],
- GEN_INT (0),
- GEN_INT (2),
- GEN_INT (1),
- GEN_INT (3)));
- emit_insn (gen_sse2_pshufd_1 (t2, operands[2],
- GEN_INT (0),
- GEN_INT (2),
- GEN_INT (1),
- GEN_INT (3)));
- emit_insn (gen_xop_mulv2div2di3_high (operands[0], t1, t2));
+ if (TARGET_XOP)
+ {
+ emit_insn (gen_sse2_pshufd_1 (t1, op1, GEN_INT (0), GEN_INT (2),
+ GEN_INT (1), GEN_INT (3)));
+ emit_insn (gen_sse2_pshufd_1 (t2, op2, GEN_INT (0), GEN_INT (2),
+ GEN_INT (1), GEN_INT (3)));
+ emit_insn (gen_xop_mulv2div2di3_high (operands[0], t1, t2));
+ DONE;
+ }
+
+ emit_insn (gen_vec_interleave_highv4si (t1, op1, op1));
+ emit_insn (gen_vec_interleave_highv4si (t2, op2, op2));
+ emit_insn (gen_sse4_1_mulv2siv2di3 (operands[0], t1, t2));
DONE;
})
[(match_operand:V2DI 0 "register_operand" "")
(match_operand:V4SI 1 "register_operand" "")
(match_operand:V4SI 2 "register_operand" "")]
- "TARGET_XOP"
+ "TARGET_SSE4_1"
{
- rtx t1, t2;
+ rtx op1, op2, t1, t2;
+ op1 = operands[1];
+ op2 = operands[2];
t1 = gen_reg_rtx (V4SImode);
t2 = gen_reg_rtx (V4SImode);
- emit_insn (gen_sse2_pshufd_1 (t1, operands[1],
- GEN_INT (0),
- GEN_INT (2),
- GEN_INT (1),
- GEN_INT (3)));
- emit_insn (gen_sse2_pshufd_1 (t2, operands[2],
- GEN_INT (0),
- GEN_INT (2),
- GEN_INT (1),
- GEN_INT (3)));
- emit_insn (gen_xop_mulv2div2di3_low (operands[0], t1, t2));
+ if (TARGET_XOP)
+ {
+ emit_insn (gen_sse2_pshufd_1 (t1, op1, GEN_INT (0), GEN_INT (2),
+ GEN_INT (1), GEN_INT (3)));
+ emit_insn (gen_sse2_pshufd_1 (t2, op2, GEN_INT (0), GEN_INT (2),
+ GEN_INT (1), GEN_INT (3)));
+ emit_insn (gen_xop_mulv2div2di3_low (operands[0], t1, t2));
+ DONE;
+ }
+
+ emit_insn (gen_vec_interleave_lowv4si (t1, op1, op1));
+ emit_insn (gen_vec_interleave_lowv4si (t2, op2, op2));
+ emit_insn (gen_sse4_1_mulv2siv2di3 (operands[0], t1, t2));
DONE;
})
DONE;
})
-(define_expand "sdot_prodv8hi"
- [(match_operand:V4SI 0 "register_operand" "")
- (match_operand:V8HI 1 "register_operand" "")
- (match_operand:V8HI 2 "register_operand" "")
- (match_operand:V4SI 3 "register_operand" "")]
+(define_expand "sdot_prod<mode>"
+ [(match_operand:<sseunpackmode> 0 "register_operand" "")
+ (match_operand:VI2_AVX2 1 "register_operand" "")
+ (match_operand:VI2_AVX2 2 "register_operand" "")
+ (match_operand:<sseunpackmode> 3 "register_operand" "")]
"TARGET_SSE2"
{
- rtx t = gen_reg_rtx (V4SImode);
- emit_insn (gen_sse2_pmaddwd (t, operands[1], operands[2]));
- emit_insn (gen_addv4si3 (operands[0], operands[3], t));
+ rtx t = gen_reg_rtx (<sseunpackmode>mode);
+ emit_insn (gen_<sse2_avx2>_pmaddwd (t, operands[1], operands[2]));
+ emit_insn (gen_rtx_SET (VOIDmode, operands[0],
+ gen_rtx_PLUS (<sseunpackmode>mode,
+ operands[3], t)));
DONE;
})
-(define_expand "udot_prodv4si"
+(define_code_attr sse2_sse4_1
+ [(zero_extend "sse2") (sign_extend "sse4_1")])
+
+(define_expand "<s>dot_prodv4si"
[(match_operand:V2DI 0 "register_operand" "")
- (match_operand:V4SI 1 "register_operand" "")
+ (any_extend:V2DI (match_operand:V4SI 1 "register_operand" ""))
(match_operand:V4SI 2 "register_operand" "")
(match_operand:V2DI 3 "register_operand" "")]
- "TARGET_SSE2"
+ "<CODE> == ZERO_EXTEND ? TARGET_SSE2 : TARGET_SSE4_1"
{
rtx t1, t2, t3, t4;
t1 = gen_reg_rtx (V2DImode);
- emit_insn (gen_sse2_umulv2siv2di3 (t1, operands[1], operands[2]));
+ emit_insn (gen_<sse2_sse4_1>_<u>mulv2siv2di3 (t1, operands[1], operands[2]));
emit_insn (gen_addv2di3 (t1, t1, operands[3]));
t2 = gen_reg_rtx (V4SImode);
GEN_INT (32)));
t4 = gen_reg_rtx (V2DImode);
- emit_insn (gen_sse2_umulv2siv2di3 (t4, t2, t3));
+ emit_insn (gen_<sse2_sse4_1>_<u>mulv2siv2di3 (t4, t2, t3));
emit_insn (gen_addv2di3 (operands[0], t1, t4));
DONE;
})
+(define_expand "<s>dot_prodv8si"
+ [(match_operand:V4DI 0 "register_operand" "")
+ (any_extend:V4DI (match_operand:V8SI 1 "register_operand" ""))
+ (match_operand:V8SI 2 "register_operand" "")
+ (match_operand:V4DI 3 "register_operand" "")]
+ "TARGET_AVX2"
+{
+ rtx t1, t2, t3, t4;
+
+ t1 = gen_reg_rtx (V4DImode);
+ emit_insn (gen_avx2_<u>mulv4siv4di3 (t1, operands[1], operands[2]));
+ emit_insn (gen_addv4di3 (t1, t1, operands[3]));
+
+ t2 = gen_reg_rtx (V8SImode);
+ t3 = gen_reg_rtx (V8SImode);
+ emit_insn (gen_avx2_lshrv2ti3 (gen_lowpart (V2TImode, t2),
+ gen_lowpart (V2TImode, operands[1]),
+ GEN_INT (32)));
+ emit_insn (gen_avx2_lshrv2ti3 (gen_lowpart (V2TImode, t3),
+ gen_lowpart (V2TImode, operands[2]),
+ GEN_INT (32)));
+
+ t4 = gen_reg_rtx (V4DImode);
+ emit_insn (gen_avx2_<u>mulv4siv4di3 (t4, t2, t3));
+
+ emit_insn (gen_addv4di3 (operands[0], t1, t4));
+ DONE;
+})
+
(define_insn "ashr<mode>3"
[(set (match_operand:VI24_AVX2 0 "register_operand" "=x,x")
(ashiftrt:VI24_AVX2
(set_attr "prefix" "orig,vex")
(set_attr "mode" "<sseinsnmode>")])
-(define_insn "avx2_lshl<mode>3"
- [(set (match_operand:VI248_256 0 "register_operand" "=x")
- (ashift:VI248_256
- (match_operand:VI248_256 1 "register_operand" "x")
- (match_operand:SI 2 "nonmemory_operand" "xN")))]
- "TARGET_AVX2"
- "vpsll<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
- [(set_attr "type" "sseishft")
- (set_attr "prefix" "vex")
- (set (attr "length_immediate")
- (if_then_else (match_operand 2 "const_int_operand" "")
- (const_string "1")
- (const_string "0")))
- (set_attr "mode" "OI")])
-
(define_insn "ashl<mode>3"
[(set (match_operand:VI248_AVX2 0 "register_operand" "=x,x")
(ashift:VI248_AVX2
(set_attr "prefix" "orig,vex")
(set_attr "mode" "TI")])
+(define_expand "vec_interleave_high<mode>"
+ [(match_operand:VI_256 0 "register_operand" "=x")
+ (match_operand:VI_256 1 "register_operand" "x")
+ (match_operand:VI_256 2 "nonimmediate_operand" "xm")]
+ "TARGET_AVX2"
+{
+ rtx t1 = gen_reg_rtx (<MODE>mode);
+ rtx t2 = gen_reg_rtx (<MODE>mode);
+ emit_insn (gen_avx2_interleave_low<mode> (t1, operands[1], operands[2]));
+ emit_insn (gen_avx2_interleave_high<mode> (t2, operands[1], operands[2]));
+ emit_insn (gen_avx2_permv2ti (gen_lowpart (V4DImode, operands[0]),
+ gen_lowpart (V4DImode, t1),
+ gen_lowpart (V4DImode, t2), GEN_INT (1 + (3 << 4))));
+ DONE;
+})
+
+(define_expand "vec_interleave_low<mode>"
+ [(match_operand:VI_256 0 "register_operand" "=x")
+ (match_operand:VI_256 1 "register_operand" "x")
+ (match_operand:VI_256 2 "nonimmediate_operand" "xm")]
+ "TARGET_AVX2"
+{
+ rtx t1 = gen_reg_rtx (<MODE>mode);
+ rtx t2 = gen_reg_rtx (<MODE>mode);
+ emit_insn (gen_avx2_interleave_low<mode> (t1, operands[1], operands[2]));
+ emit_insn (gen_avx2_interleave_high<mode> (t2, operands[1], operands[2]));
+ emit_insn (gen_avx2_permv2ti (gen_lowpart (V4DImode, operands[0]),
+ gen_lowpart (V4DImode, t1),
+ gen_lowpart (V4DImode, t2), GEN_INT (0 + (2 << 4))));
+ DONE;
+})
+
;; Modes handled by pinsr patterns.
(define_mode_iterator PINSR_MODE
[(V16QI "TARGET_SSE4_1") V8HI
(const_int 6)
(const_int 7)]))))]
"TARGET_SSE4_1"
- "%vpmov<extsuffix>bw\t{%1, %0|%0, %1}"
+ "%vpmov<extsuffix>bw\t{%1, %0|%0, %q1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "maybe_vex")
(const_int 6)
(const_int 7)]))))]
"TARGET_AVX2"
- "vpmov<extsuffix>bd\t{%1, %0|%0, %1}"
+ "vpmov<extsuffix>bd\t{%1, %0|%0, %q1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "vex")
(const_int 2)
(const_int 3)]))))]
"TARGET_SSE4_1"
- "%vpmov<extsuffix>bd\t{%1, %0|%0, %1}"
+ "%vpmov<extsuffix>bd\t{%1, %0|%0, %k1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "maybe_vex")
(const_int 2)
(const_int 3)]))))]
"TARGET_SSE4_1"
- "%vpmov<extsuffix>wd\t{%1, %0|%0, %1}"
+ "%vpmov<extsuffix>wd\t{%1, %0|%0, %q1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "maybe_vex")
(const_int 2)
(const_int 3)]))))]
"TARGET_AVX2"
- "vpmov<extsuffix>bq\t{%1, %0|%0, %1}"
+ "vpmov<extsuffix>bq\t{%1, %0|%0, %k1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "vex")
(parallel [(const_int 0)
(const_int 1)]))))]
"TARGET_SSE4_1"
- "%vpmov<extsuffix>bq\t{%1, %0|%0, %1}"
+ "%vpmov<extsuffix>bq\t{%1, %0|%0, %w1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "maybe_vex")
(const_int 2)
(const_int 3)]))))]
"TARGET_AVX2"
- "vpmov<extsuffix>wq\t{%1, %0|%0, %1}"
+ "vpmov<extsuffix>wq\t{%1, %0|%0, %q1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "vex")
(parallel [(const_int 0)
(const_int 1)]))))]
"TARGET_SSE4_1"
- "%vpmov<extsuffix>wq\t{%1, %0|%0, %1}"
+ "%vpmov<extsuffix>wq\t{%1, %0|%0, %k1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "maybe_vex")
(parallel [(const_int 0)
(const_int 1)]))))]
"TARGET_SSE4_1"
- "%vpmov<extsuffix>dq\t{%1, %0|%0, %1}"
+ "%vpmov<extsuffix>dq\t{%1, %0|%0, %q1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "maybe_vex")
(set_attr "prefix" "vex")
(set_attr "mode" "OI")])
-(define_expand "<avx_avx2>_maskload<ssemodesuffix><avxsizesuffix>"
- [(set (match_operand:V48_AVX2 0 "register_operand" "")
- (unspec:V48_AVX2
- [(match_operand:<sseintvecmode> 2 "register_operand" "")
- (match_operand:V48_AVX2 1 "memory_operand" "")
- (match_dup 0)]
- UNSPEC_MASKMOV))]
- "TARGET_AVX")
-
-(define_expand "<avx_avx2>_maskstore<ssemodesuffix><avxsizesuffix>"
- [(set (match_operand:V48_AVX2 0 "memory_operand" "")
+(define_insn "<avx_avx2>_maskload<ssemodesuffix><avxsizesuffix>"
+ [(set (match_operand:V48_AVX2 0 "register_operand" "=x")
(unspec:V48_AVX2
- [(match_operand:<sseintvecmode> 1 "register_operand" "")
- (match_operand:V48_AVX2 2 "register_operand" "")
- (match_dup 0)]
+ [(match_operand:<sseintvecmode> 2 "register_operand" "x")
+ (match_operand:V48_AVX2 1 "memory_operand" "m")]
UNSPEC_MASKMOV))]
- "TARGET_AVX")
-
-(define_insn "*avx2_maskmov<ssemodesuffix><avxsizesuffix>"
- [(set (match_operand:VI48_AVX2 0 "nonimmediate_operand" "=x,m")
- (unspec:VI48_AVX2
- [(match_operand:<sseintvecmode> 1 "register_operand" "x,x")
- (match_operand:VI48_AVX2 2 "nonimmediate_operand" "m,x")
- (match_dup 0)]
- UNSPEC_MASKMOV))]
- "TARGET_AVX2
- && (REG_P (operands[0]) == MEM_P (operands[2]))"
- "vpmaskmov<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ "TARGET_AVX"
+ "v<sseintprefix>maskmov<ssemodesuffix>\t{%1, %2, %0|%0, %2, %1}"
[(set_attr "type" "sselog1")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "vex")
(set_attr "mode" "<sseinsnmode>")])
-(define_insn "
*avx_maskmov<ssemodesuffix><avxsizesuffix>"
- [(set (match_operand:VF 0 "nonimmediate_operand" "=x,m")
- (unspec:VF
- [(match_operand:<sseintvecmode> 1 "register_operand" "x,x")
- (match_operand:VF 2 "nonimmediate_operand" "m,x")
+(define_insn "
<avx_avx2>_maskstore<ssemodesuffix><avxsizesuffix>"
+ [(set (match_operand:V48_AVX2 0 "memory_operand" "=m")
+ (unspec:V48_AVX2
+ [(match_operand:<sseintvecmode> 1 "register_operand" "x")
+ (match_operand:V48_AVX2 2 "register_operand" "x")
(match_dup 0)]
UNSPEC_MASKMOV))]
- "TARGET_AVX
- && (REG_P (operands[0]) == MEM_P (operands[2]))"
- "vmaskmov<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ "TARGET_AVX"
+ "v<sseintprefix>maskmov<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
[(set_attr "type" "sselog1")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "vex")
- (set_attr "mode" "<MODE>")])
+ (set_attr "mode" "<sseinsnmode>")])
(define_insn_and_split "avx_<castmode><avxsizesuffix>_<castmode>"
[(set (match_operand:AVX256MODE2P 0 "nonimmediate_operand" "=x,m")
[(parallel [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "")
(unspec:VEC_GATHER_MODE
[(match_operand:VEC_GATHER_MODE 1 "register_operand" "")
- (match_operand 2 "register_operand" "")
+ (mem:<ssescalarmode>
+ (match_par_dup 7
+ [(match_operand 2 "vsib_address_operand" "")
+ (match_operand:<VEC_GATHER_MODE> 3 "register_operand" "")
+ (match_operand:SI 5 "const1248_operand " "")]))
(mem:BLK (scratch))
- (match_operand:<VEC_GATHER_MODE> 3 "register_operand" "")
- (match_operand:VEC_GATHER_MODE 4 "register_operand" "")
- (match_operand:SI 5 "const1248_operand " "")]
+ (match_operand:VEC_GATHER_MODE 4 "register_operand" "")]
UNSPEC_GATHER))
(clobber (match_scratch:VEC_GATHER_MODE 6 ""))])]
- "TARGET_AVX2")
+ "TARGET_AVX2"
+{
+ operands[7]
+ = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
+ operands[5]), UNSPEC_VSIBADDR);
+})
(define_insn "*avx2_gathersi<mode>"
[(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
(unspec:VEC_GATHER_MODE
[(match_operand:VEC_GATHER_MODE 2 "register_operand" "0")
- (match_operand:P 3 "register_operand" "r")
+ (match_operator:<ssescalarmode> 7 "vsib_mem_operator"
+ [(unspec:P
+ [(match_operand:P 3 "vsib_address_operand" "p")
+ (match_operand:<VEC_GATHER_MODE> 4 "register_operand" "x")
+ (match_operand:SI 6 "const1248_operand" "n")]
+ UNSPEC_VSIBADDR)])
(mem:BLK (scratch))
- (match_operand:<VEC_GATHER_MODE> 4 "register_operand" "x")
- (match_operand:VEC_GATHER_MODE 5 "register_operand" "1")
- (match_operand:SI 6 "const1248_operand" "n")]
+ (match_operand:VEC_GATHER_MODE 5 "register_operand" "1")]
UNSPEC_GATHER))
(clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
"TARGET_AVX2"
- "v<gthrfirstp>gatherd<gthrlastp>\t{%1, (%3, %4, %p6), %0|%0, (%3, %4, %p6), %1}"
+ "v<sseintprefix>gatherd<ssemodesuffix>\t{%1, %7, %0|%0, %7, %1}"
[(set_attr "type" "ssemov")
(set_attr "prefix" "vex")
(set_attr "mode" "<sseinsnmode>")])
[(parallel [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "")
(unspec:VEC_GATHER_MODE
[(match_operand:VEC_GATHER_MODE 1 "register_operand" "")
- (match_operand 2 "register_operand" "")
+ (mem:<ssescalarmode>
+ (match_par_dup 7
+ [(match_operand 2 "vsib_address_operand" "")
+ (match_operand:<AVXMODE48P_DI> 3 "register_operand" "")
+ (match_operand:SI 5 "const1248_operand " "")]))
(mem:BLK (scratch))
- (match_operand:<AVXMODE48P_DI> 3 "register_operand" "")
- (match_operand:VEC_GATHER_MODE 4 "register_operand" "")
- (match_operand:SI 5 "const1248_operand " "")]
+ (match_operand:VEC_GATHER_MODE 4 "register_operand" "")]
UNSPEC_GATHER))
(clobber (match_scratch:VEC_GATHER_MODE 6 ""))])]
- "TARGET_AVX2")
+ "TARGET_AVX2"
+{
+ operands[7]
+ = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
+ operands[5]), UNSPEC_VSIBADDR);
+})
(define_insn "*avx2_gatherdi<mode>"
[(set (match_operand:AVXMODE48P_DI 0 "register_operand" "=&x")
(unspec:AVXMODE48P_DI
[(match_operand:AVXMODE48P_DI 2 "register_operand" "0")
- (match_operand:P 3 "register_operand" "r")
+ (match_operator:<ssescalarmode> 7 "vsib_mem_operator"
+ [(unspec:P
+ [(match_operand:P 3 "vsib_address_operand" "p")
+ (match_operand:<AVXMODE48P_DI> 4 "register_operand" "x")
+ (match_operand:SI 6 "const1248_operand" "n")]
+ UNSPEC_VSIBADDR)])
(mem:BLK (scratch))
- (match_operand:<AVXMODE48P_DI> 4 "register_operand" "x")
- (match_operand:AVXMODE48P_DI 5 "register_operand" "1")
- (match_operand:SI 6 "const1248_operand" "n")]
+ (match_operand:AVXMODE48P_DI 5 "register_operand" "1")]
UNSPEC_GATHER))
(clobber (match_scratch:AVXMODE48P_DI 1 "=&x"))]
"TARGET_AVX2"
- "v<gthrfirstp>gatherq<gthrlastp>\t{%1, (%3, %4, %p6), %0|%0, (%3, %4, %p6), %1}"
+ "v<sseintprefix>gatherq<ssemodesuffix>\t{%1, %7, %0|%0, %7, %1}"
[(set_attr "type" "ssemov")
(set_attr "prefix" "vex")
(set_attr "mode" "<sseinsnmode>")])
[(parallel [(set (match_operand:VI4F_128 0 "register_operand" "")
(unspec:VI4F_128
[(match_operand:VI4F_128 1 "register_operand" "")
- (match_operand 2 "register_operand" "")
+ (mem:<ssescalarmode>
+ (match_par_dup 7
+ [(match_operand 2 "vsib_address_operand" "")
+ (match_operand:V4DI 3 "register_operand" "")
+ (match_operand:SI 5 "const1248_operand " "")]))
(mem:BLK (scratch))
- (match_operand:V4DI 3 "register_operand" "")
- (match_operand:VI4F_128 4 "register_operand" "")
- (match_operand:SI 5 "const1248_operand " "")]
+ (match_operand:VI4F_128 4 "register_operand" "")]
UNSPEC_GATHER))
(clobber (match_scratch:VI4F_128 6 ""))])]
- "TARGET_AVX2")
+ "TARGET_AVX2"
+{
+ operands[7]
+ = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
+ operands[5]), UNSPEC_VSIBADDR);
+})
(define_insn "*avx2_gatherdi<mode>256"
[(set (match_operand:VI4F_128 0 "register_operand" "=x")
(unspec:VI4F_128
[(match_operand:VI4F_128 2 "register_operand" "0")
- (match_operand:P 3 "register_operand" "r")
+ (match_operator:<ssescalarmode> 7 "vsib_mem_operator"
+ [(unspec:P
+ [(match_operand:P 3 "vsib_address_operand" "p")
+ (match_operand:V4DI 4 "register_operand" "x")
+ (match_operand:SI 6 "const1248_operand" "n")]
+ UNSPEC_VSIBADDR)])
(mem:BLK (scratch))
- (match_operand:V4DI 4 "register_operand" "x")
- (match_operand:VI4F_128 5 "register_operand" "1")
- (match_operand:SI 6 "const1248_operand" "n")]
+ (match_operand:VI4F_128 5 "register_operand" "1")]
UNSPEC_GATHER))
(clobber (match_scratch:VI4F_128 1 "=&x"))]
"TARGET_AVX2"
- "v<gthrfirstp>gatherq<gthrlastp>\t{%1, (%3, %4, %p6), %0|%0, (%3, %4, %p6), %1}"
+ "v<sseintprefix>gatherq<ssemodesuffix>\t{%1, %7, %0|%0, %7, %1}"
[(set_attr "type" "ssemov")
(set_attr "prefix" "vex")
(set_attr "mode" "<sseinsnmode>")])
OSDN Git Service
