;; GCC machine description for SSE instructions
-;; Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011
+;; Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012
;; Free Software Foundation, Inc.
;;
;; This file is part of GCC.
;; along with GCC; see the file COPYING3. If not see
;; <http://www.gnu.org/licenses/>.
+(define_c_enum "unspec" [
+ ;; SSE
+ UNSPEC_MOVNT
+ UNSPEC_LOADU
+ UNSPEC_STOREU
+
+ ;; SSE3
+ UNSPEC_LDDQU
+
+ ;; SSSE3
+ UNSPEC_PSHUFB
+ UNSPEC_PSIGN
+ UNSPEC_PALIGNR
+
+ ;; For SSE4A support
+ UNSPEC_EXTRQI
+ UNSPEC_EXTRQ
+ UNSPEC_INSERTQI
+ UNSPEC_INSERTQ
+
+ ;; For SSE4.1 support
+ UNSPEC_BLENDV
+ UNSPEC_INSERTPS
+ UNSPEC_DP
+ UNSPEC_MOVNTDQA
+ UNSPEC_MPSADBW
+ UNSPEC_PHMINPOSUW
+ UNSPEC_PTEST
+
+ ;; For SSE4.2 support
+ UNSPEC_PCMPESTR
+ UNSPEC_PCMPISTR
+
+ ;; For FMA4 support
+ UNSPEC_FMADDSUB
+ UNSPEC_XOP_UNSIGNED_CMP
+ UNSPEC_XOP_TRUEFALSE
+ UNSPEC_XOP_PERMUTE
+ UNSPEC_FRCZ
+
+ ;; For AES support
+ UNSPEC_AESENC
+ UNSPEC_AESENCLAST
+ UNSPEC_AESDEC
+ UNSPEC_AESDECLAST
+ UNSPEC_AESIMC
+ UNSPEC_AESKEYGENASSIST
+
+ ;; For PCLMUL support
+ UNSPEC_PCLMUL
+
+ ;; For AVX support
+ UNSPEC_PCMP
+ UNSPEC_VPERMIL
+ UNSPEC_VPERMIL2
+ UNSPEC_VPERMIL2F128
+ UNSPEC_CAST
+ UNSPEC_VTESTP
+ UNSPEC_VCVTPH2PS
+ UNSPEC_VCVTPS2PH
+
+ ;; For AVX2 support
+ UNSPEC_VPERMSI
+ UNSPEC_VPERMDF
+ UNSPEC_VPERMSF
+ UNSPEC_VPERMTI
+ UNSPEC_GATHER
+ UNSPEC_VSIBADDR
+])
+
+(define_c_enum "unspecv" [
+ UNSPECV_LDMXCSR
+ UNSPECV_STMXCSR
+ UNSPECV_CLFLUSH
+ UNSPECV_MONITOR
+ UNSPECV_MWAIT
+ UNSPECV_VZEROALL
+ UNSPECV_VZEROUPPER
+])
+
;; All vector modes including V?TImode, used in move patterns.
(define_mode_iterator V16
[(V32QI "TARGET_AVX") V16QI
(V8SI "TARGET_AVX2") V4SI
(V4DI "TARGET_AVX2") V2DI])
-(define_mode_iterator VI4SD_AVX2
- [V4SI V4DI])
+(define_mode_iterator VI48_AVX2
+ [(V8SI "TARGET_AVX2") V4SI
+ (V4DI "TARGET_AVX2") V2DI])
(define_mode_iterator V48_AVX2
[V4SF V2DF
(V4SI "vec") (V8SI "avx2")
(V2DI "vec") (V4DI "avx2")])
-;; Mapping of logic-shift operators
-(define_code_iterator lshift [lshiftrt ashift])
-
-;; Base name for define_insn
-(define_code_attr lshift_insn [(lshiftrt "srl") (ashift "sll")])
-
-;; Base name for insn mnemonic
-(define_code_attr lshift [(lshiftrt "lshr") (ashift "lshl")])
-
(define_mode_attr ssedoublemode
[(V16HI "V16SI") (V8HI "V8SI")])
(define_mode_iterator VI12_128 [V16QI V8HI])
(define_mode_iterator VI14_128 [V16QI V4SI])
(define_mode_iterator VI124_128 [V16QI V8HI V4SI])
+(define_mode_iterator VI128_128 [V16QI V8HI V2DI])
(define_mode_iterator VI24_128 [V8HI V4SI])
(define_mode_iterator VI248_128 [V8HI V4SI V2DI])
+(define_mode_iterator VI48_128 [V4SI V2DI])
;; Random 256bit vector integer mode combinations
(define_mode_iterator VI124_256 [V32QI V16HI V8SI])
+(define_mode_iterator VI48_256 [V8SI V4DI])
;; Int-float size matches
(define_mode_iterator VI4F_128 [V4SI V4SF])
(define_mode_attr sseintvecmode
[(V8SF "V8SI") (V4DF "V4DI")
(V4SF "V4SI") (V2DF "V2DI")
- (V4DF "V4DI") (V8SF "V8SI")
(V8SI "V8SI") (V4DI "V4DI")
(V4SI "V4SI") (V2DI "V2DI")
(V16HI "V16HI") (V8HI "V8HI")
(V32QI "V32QI") (V16QI "V16QI")])
+(define_mode_attr sseintvecmodelower
+ [(V8SF "v8si") (V4DF "v4di")
+ (V4SF "v4si") (V2DF "v2di")
+ (V8SI "v8si") (V4DI "v4di")
+ (V4SI "v4si") (V2DI "v2di")
+ (V16HI "v16hi") (V8HI "v8hi")
+ (V32QI "v32qi") (V16QI "v16qi")])
+
;; Mapping of vector modes to a vector mode of double size
(define_mode_attr ssedoublevecmode
[(V32QI "V64QI") (V16HI "V32HI") (V8SI "V16SI") (V4DI "V8DI")
;; Mix-n-match
(define_mode_iterator AVX256MODE2P [V8SI V8SF V4DF])
-(define_mode_iterator AVXMODE48P_DI
- [V2DI V2DF V4DI V4DF V4SF V4SI])
-(define_mode_attr AVXMODE48P_DI
- [(V2DI "V2DI") (V2DF "V2DI")
- (V4DI "V4DI") (V4DF "V4DI")
- (V4SI "V2DI") (V4SF "V2DI")
- (V8SI "V4DI") (V8SF "V4DI")])
-
-(define_mode_iterator FMAMODE [SF DF V4SF V2DF V8SF V4DF])
-
;; Mapping of immediate bits for blend instructions
(define_mode_attr blendbits
[(V8SF "255") (V4SF "15") (V4DF "15") (V2DF "3")])
DONE;
})
-(define_expand "<sse>_movu<ssemodesuffix><avxsizesuffix>"
- [(set (match_operand:VF 0 "nonimmediate_operand" "")
+(define_insn "<sse>_loadu<ssemodesuffix><avxsizesuffix>"
+ [(set (match_operand:VF 0 "register_operand" "=x")
(unspec:VF
- [(match_operand:VF 1 "nonimmediate_operand" "")]
- UNSPEC_MOVU))]
+ [(match_operand:VF 1 "memory_operand" "m")]
+ UNSPEC_LOADU))]
"TARGET_SSE"
-{
- if (MEM_P (operands[0]) && MEM_P (operands[1]))
- operands[1] = force_reg (<MODE>mode, operands[1]);
-})
+ "%vmovu<ssemodesuffix>\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssemov")
+ (set_attr "movu" "1")
+ (set_attr "prefix" "maybe_vex")
+ (set_attr "mode" "<MODE>")])
-(define_insn "*<sse>_movu<ssemodesuffix><avxsizesuffix>"
- [(set (match_operand:VF 0 "nonimmediate_operand" "=x,m")
+(define_insn "<sse>_storeu<ssemodesuffix><avxsizesuffix>"
+ [(set (match_operand:VF 0 "memory_operand" "=m")
(unspec:VF
- [(match_operand:VF 1 "nonimmediate_operand" "xm,x")]
- UNSPEC_MOVU))]
- "TARGET_SSE && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
+ [(match_operand:VF 1 "register_operand" "x")]
+ UNSPEC_STOREU))]
+ "TARGET_SSE"
"%vmovu<ssemodesuffix>\t{%1, %0|%0, %1}"
[(set_attr "type" "ssemov")
(set_attr "movu" "1")
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "<MODE>")])
-(define_expand "<sse2>_movdqu<avxsizesuffix>"
- [(set (match_operand:VI1 0 "nonimmediate_operand" "")
- (unspec:VI1 [(match_operand:VI1 1 "nonimmediate_operand" "")]
- UNSPEC_MOVU))]
+(define_insn "<sse2>_loaddqu<avxsizesuffix>"
+ [(set (match_operand:VI1 0 "register_operand" "=x")
+ (unspec:VI1 [(match_operand:VI1 1 "memory_operand" "m")]
+ UNSPEC_LOADU))]
"TARGET_SSE2"
-{
- if (MEM_P (operands[0]) && MEM_P (operands[1]))
- operands[1] = force_reg (<MODE>mode, operands[1]);
-})
+ "%vmovdqu\t{%1, %0|%0, %1}"
+ [(set_attr "type" "ssemov")
+ (set_attr "movu" "1")
+ (set (attr "prefix_data16")
+ (if_then_else
+ (match_test "TARGET_AVX")
+ (const_string "*")
+ (const_string "1")))
+ (set_attr "prefix" "maybe_vex")
+ (set_attr "mode" "<sseinsnmode>")])
-(define_insn "*<sse2>_movdqu<avxsizesuffix>"
- [(set (match_operand:VI1 0 "nonimmediate_operand" "=x,m")
- (unspec:VI1 [(match_operand:VI1 1 "nonimmediate_operand" "xm,x")]
- UNSPEC_MOVU))]
- "TARGET_SSE2 && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
+(define_insn "<sse2>_storedqu<avxsizesuffix>"
+ [(set (match_operand:VI1 0 "memory_operand" "=m")
+ (unspec:VI1 [(match_operand:VI1 1 "register_operand" "x")]
+ UNSPEC_STOREU))]
+ "TARGET_SSE2"
"%vmovdqu\t{%1, %0|%0, %1}"
[(set_attr "type" "ssemov")
(set_attr "movu" "1")
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "<sseinsnmode>")])
-(define_insn "sse2_movntsi"
- [(set (match_operand:SI 0 "memory_operand" "=m")
- (unspec:SI [(match_operand:SI 1 "register_operand" "r")]
- UNSPEC_MOVNT))]
+(define_insn "sse2_movnti<mode>"
+ [(set (match_operand:SWI48 0 "memory_operand" "=m")
+ (unspec:SWI48 [(match_operand:SWI48 1 "register_operand" "r")]
+ UNSPEC_MOVNT))]
"TARGET_SSE2"
"movnti\t{%1, %0|%0, %1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_data16" "0")
- (set_attr "mode" "V2DF")])
+ (set_attr "mode" "<MODE>")])
(define_insn "<sse>_movnt<mode>"
[(set (match_operand:VF 0 "memory_operand" "=m")
;; Modes handled by storent patterns.
(define_mode_iterator STORENT_MODE
- [(SI "TARGET_SSE2") (SF "TARGET_SSE4A") (DF "TARGET_SSE4A")
- (V2DI "TARGET_SSE2")
+ [(DI "TARGET_SSE2 && TARGET_64BIT") (SI "TARGET_SSE2")
+ (SF "TARGET_SSE4A") (DF "TARGET_SSE4A")
+ (V4DI "TARGET_AVX") (V2DI "TARGET_SSE2")
(V8SF "TARGET_AVX") V4SF
(V4DF "TARGET_AVX") (V2DF "TARGET_SSE2")])
(parallel [(const_int 0)]))
(vec_select:DF (match_dup 1) (parallel [(const_int 1)])))
(plusminus:DF
- (vec_select:DF (match_dup 1) (parallel [(const_int 2)]))
- (vec_select:DF (match_dup 1) (parallel [(const_int 3)]))))
- (vec_concat:V2DF
- (plusminus:DF
(vec_select:DF
(match_operand:V4DF 2 "nonimmediate_operand" "xm")
(parallel [(const_int 0)]))
- (vec_select:DF (match_dup 2) (parallel [(const_int 1)])))
+ (vec_select:DF (match_dup 2) (parallel [(const_int 1)]))))
+ (vec_concat:V2DF
+ (plusminus:DF
+ (vec_select:DF (match_dup 1) (parallel [(const_int 2)]))
+ (vec_select:DF (match_dup 1) (parallel [(const_int 3)])))
(plusminus:DF
(vec_select:DF (match_dup 2) (parallel [(const_int 2)]))
(vec_select:DF (match_dup 2) (parallel [(const_int 3)]))))))]
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;
-;; FMA4 floating point multiply/accumulate instructions. This
-;; includes the scalar version of the instructions as well as the
-;; vector.
+;; FMA floating point multiply/accumulate instructions. These include
+;; scalar versions of the instructions as well as vector versions.
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; In order to match (*a * *b) + *c, particularly when vectorizing, allow
-;; combine to generate a multiply/add with two memory references. We then
-;; split this insn, into loading up the destination register with one of the
-;; memory operations. If we don't manage to split the insn, reload will
-;; generate the appropriate moves. The reason this is needed, is that combine
-;; has already folded one of the memory references into both the multiply and
-;; add insns, and it can't generate a new pseudo. I.e.:
-;; (set (reg1) (mem (addr1)))
-;; (set (reg2) (mult (reg1) (mem (addr2))))
-;; (set (reg3) (plus (reg2) (mem (addr3))))
-;;
-;; ??? This is historic, pre-dating the gimple fma transformation.
-;; We could now properly represent that only one memory operand is
-;; allowed and not be penalized during optimization.
-
-;; Intrinsic FMA operations.
+;; The standard names for scalar FMA are only available with SSE math enabled.
+(define_mode_iterator FMAMODEM [(SF "TARGET_SSE_MATH")
+ (DF "TARGET_SSE_MATH")
+ V4SF V2DF V8SF V4DF])
-;; The standard names for fma is only available with SSE math enabled.
(define_expand "fma<mode>4"
- [(set (match_operand:FMAMODE 0 "register_operand")
- (fma:FMAMODE
- (match_operand:FMAMODE 1 "nonimmediate_operand")
- (match_operand:FMAMODE 2 "nonimmediate_operand")
- (match_operand:FMAMODE 3 "nonimmediate_operand")))]
- "(TARGET_FMA || TARGET_FMA4) && TARGET_SSE_MATH")
+ [(set (match_operand:FMAMODEM 0 "register_operand")
+ (fma:FMAMODEM
+ (match_operand:FMAMODEM 1 "nonimmediate_operand")
+ (match_operand:FMAMODEM 2 "nonimmediate_operand")
+ (match_operand:FMAMODEM 3 "nonimmediate_operand")))]
+ "TARGET_FMA || TARGET_FMA4")
(define_expand "fms<mode>4"
- [(set (match_operand:FMAMODE 0 "register_operand")
- (fma:FMAMODE
- (match_operand:FMAMODE 1 "nonimmediate_operand")
- (match_operand:FMAMODE 2 "nonimmediate_operand")
- (neg:FMAMODE (match_operand:FMAMODE 3 "nonimmediate_operand"))))]
- "(TARGET_FMA || TARGET_FMA4) && TARGET_SSE_MATH")
+ [(set (match_operand:FMAMODEM 0 "register_operand")
+ (fma:FMAMODEM
+ (match_operand:FMAMODEM 1 "nonimmediate_operand")
+ (match_operand:FMAMODEM 2 "nonimmediate_operand")
+ (neg:FMAMODEM (match_operand:FMAMODEM 3 "nonimmediate_operand"))))]
+ "TARGET_FMA || TARGET_FMA4")
(define_expand "fnma<mode>4"
- [(set (match_operand:FMAMODE 0 "register_operand")
- (fma:FMAMODE
- (neg:FMAMODE (match_operand:FMAMODE 1 "nonimmediate_operand"))
- (match_operand:FMAMODE 2 "nonimmediate_operand")
- (match_operand:FMAMODE 3 "nonimmediate_operand")))]
- "(TARGET_FMA || TARGET_FMA4) && TARGET_SSE_MATH")
+ [(set (match_operand:FMAMODEM 0 "register_operand")
+ (fma:FMAMODEM
+ (neg:FMAMODEM (match_operand:FMAMODEM 1 "nonimmediate_operand"))
+ (match_operand:FMAMODEM 2 "nonimmediate_operand")
+ (match_operand:FMAMODEM 3 "nonimmediate_operand")))]
+ "TARGET_FMA || TARGET_FMA4")
(define_expand "fnms<mode>4"
- [(set (match_operand:FMAMODE 0 "register_operand")
- (fma:FMAMODE
- (neg:FMAMODE (match_operand:FMAMODE 1 "nonimmediate_operand"))
- (match_operand:FMAMODE 2 "nonimmediate_operand")
- (neg:FMAMODE (match_operand:FMAMODE 3 "nonimmediate_operand"))))]
- "(TARGET_FMA || TARGET_FMA4) && TARGET_SSE_MATH")
+ [(set (match_operand:FMAMODEM 0 "register_operand")
+ (fma:FMAMODEM
+ (neg:FMAMODEM (match_operand:FMAMODEM 1 "nonimmediate_operand"))
+ (match_operand:FMAMODEM 2 "nonimmediate_operand")
+ (neg:FMAMODEM (match_operand:FMAMODEM 3 "nonimmediate_operand"))))]
+ "TARGET_FMA || TARGET_FMA4")
+
+;; The builtins for intrinsics are not constrained by SSE math enabled.
+(define_mode_iterator FMAMODE [SF DF V4SF V2DF V8SF V4DF])
-;; The builtin for fma4intrin.h is not constrained by SSE math enabled.
(define_expand "fma4i_fmadd_<mode>"
[(set (match_operand:FMAMODE 0 "register_operand")
(fma:FMAMODE
(match_operand:FMAMODE 3 "nonimmediate_operand")))]
"TARGET_FMA || TARGET_FMA4")
-(define_insn "*fma4i_fmadd_<mode>"
- [(set (match_operand:FMAMODE 0 "register_operand" "=x,x")
+(define_insn "*fma_fmadd_<mode>"
+ [(set (match_operand:FMAMODE 0 "register_operand" "=x,x,x,x,x")
(fma:FMAMODE
- (match_operand:FMAMODE 1 "nonimmediate_operand" "%x,x")
- (match_operand:FMAMODE 2 "nonimmediate_operand" " x,m")
- (match_operand:FMAMODE 3 "nonimmediate_operand" "xm,x")))]
- "TARGET_FMA4"
- "vfmadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
- [(set_attr "type" "ssemuladd")
+ (match_operand:FMAMODE 1 "nonimmediate_operand" "%0, 0,x, x,x")
+ (match_operand:FMAMODE 2 "nonimmediate_operand" "xm, x,xm,x,m")
+ (match_operand:FMAMODE 3 "nonimmediate_operand" " x,xm,0,xm,x")))]
+ "TARGET_FMA || TARGET_FMA4"
+ "@
+ vfmadd132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
+ vfmadd213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
+ vfmadd231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
+ vfmadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
+ vfmadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+ [(set_attr "isa" "fma,fma,fma,fma4,fma4")
+ (set_attr "type" "ssemuladd")
(set_attr "mode" "<MODE>")])
-(define_insn "*fma4i_fmsub_<mode>"
- [(set (match_operand:FMAMODE 0 "register_operand" "=x,x")
+(define_insn "*fma_fmsub_<mode>"
+ [(set (match_operand:FMAMODE 0 "register_operand" "=x,x,x,x,x")
(fma:FMAMODE
- (match_operand:FMAMODE 1 "nonimmediate_operand" "%x,x")
- (match_operand:FMAMODE 2 "nonimmediate_operand" " x,m")
+ (match_operand:FMAMODE 1 "nonimmediate_operand" "%0, 0,x, x,x")
+ (match_operand:FMAMODE 2 "nonimmediate_operand" "xm, x,xm,x,m")
(neg:FMAMODE
- (match_operand:FMAMODE 3 "nonimmediate_operand" "xm,x"))))]
- "TARGET_FMA4"
- "vfmsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
- [(set_attr "type" "ssemuladd")
+ (match_operand:FMAMODE 3 "nonimmediate_operand" " x,xm,0,xm,x"))))]
+ "TARGET_FMA || TARGET_FMA4"
+ "@
+ vfmsub132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
+ vfmsub213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
+ vfmsub231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
+ vfmsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
+ vfmsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+ [(set_attr "isa" "fma,fma,fma,fma4,fma4")
+ (set_attr "type" "ssemuladd")
(set_attr "mode" "<MODE>")])
-(define_insn "*fma4i_fnmadd_<mode>"
- [(set (match_operand:FMAMODE 0 "register_operand" "=x,x")
+(define_insn "*fma_fnmadd_<mode>"
+ [(set (match_operand:FMAMODE 0 "register_operand" "=x,x,x,x,x")
(fma:FMAMODE
(neg:FMAMODE
- (match_operand:FMAMODE 1 "nonimmediate_operand" "%x,x"))
- (match_operand:FMAMODE 2 "nonimmediate_operand" " x,m")
- (match_operand:FMAMODE 3 "nonimmediate_operand" "xm,x")))]
- "TARGET_FMA4"
- "vfnmadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
- [(set_attr "type" "ssemuladd")
+ (match_operand:FMAMODE 1 "nonimmediate_operand" "%0, 0,x, x,x"))
+ (match_operand:FMAMODE 2 "nonimmediate_operand" "xm, x,xm,x,m")
+ (match_operand:FMAMODE 3 "nonimmediate_operand" " x,xm,0,xm,x")))]
+ "TARGET_FMA || TARGET_FMA4"
+ "@
+ vfnmadd132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
+ vfnmadd213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
+ vfnmadd231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
+ vfnmadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
+ vfnmadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+ [(set_attr "isa" "fma,fma,fma,fma4,fma4")
+ (set_attr "type" "ssemuladd")
(set_attr "mode" "<MODE>")])
-(define_insn "*fma4i_fnmsub_<mode>"
- [(set (match_operand:FMAMODE 0 "register_operand" "=x,x")
+(define_insn "*fma_fnmsub_<mode>"
+ [(set (match_operand:FMAMODE 0 "register_operand" "=x,x,x,x,x")
(fma:FMAMODE
(neg:FMAMODE
- (match_operand:FMAMODE 1 "nonimmediate_operand" "%x,x"))
- (match_operand:FMAMODE 2 "nonimmediate_operand" " x,m")
+ (match_operand:FMAMODE 1 "nonimmediate_operand" "%0, 0,x, x,x"))
+ (match_operand:FMAMODE 2 "nonimmediate_operand" "xm, x,xm,x,m")
(neg:FMAMODE
- (match_operand:FMAMODE 3 "nonimmediate_operand" "xm,x"))))]
- "TARGET_FMA4"
- "vfnmsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
- [(set_attr "type" "ssemuladd")
+ (match_operand:FMAMODE 3 "nonimmediate_operand" " x,xm,0,xm,x"))))]
+ "TARGET_FMA || TARGET_FMA4"
+ "@
+ vfnmsub132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
+ vfnmsub213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
+ vfnmsub231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
+ vfnmsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
+ vfnmsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+ [(set_attr "isa" "fma,fma,fma,fma4,fma4")
+ (set_attr "type" "ssemuladd")
(set_attr "mode" "<MODE>")])
-;; Scalar versions of the above. Unlike ADDSS et al, these write the
-;; entire destination register, with the high-order elements zeroed.
+;; FMA parallel floating point multiply addsub and subadd operations.
-(define_expand "fma4i_vmfmadd_<mode>"
- [(set (match_operand:VF_128 0 "register_operand")
- (vec_merge:VF_128
- (fma:VF_128
- (match_operand:VF_128 1 "nonimmediate_operand")
- (match_operand:VF_128 2 "nonimmediate_operand")
- (match_operand:VF_128 3 "nonimmediate_operand"))
- (match_dup 4)
- (const_int 1)))]
- "TARGET_FMA4"
-{
- operands[4] = CONST0_RTX (<MODE>mode);
-})
+;; It would be possible to represent these without the UNSPEC as
+;;
+;; (vec_merge
+;; (fma op1 op2 op3)
+;; (fma op1 op2 (neg op3))
+;; (merge-const))
+;;
+;; But this doesn't seem useful in practice.
+
+(define_expand "fmaddsub_<mode>"
+ [(set (match_operand:VF 0 "register_operand")
+ (unspec:VF
+ [(match_operand:VF 1 "nonimmediate_operand")
+ (match_operand:VF 2 "nonimmediate_operand")
+ (match_operand:VF 3 "nonimmediate_operand")]
+ UNSPEC_FMADDSUB))]
+ "TARGET_FMA || TARGET_FMA4")
+
+(define_insn "*fma_fmaddsub_<mode>"
+ [(set (match_operand:VF 0 "register_operand" "=x,x,x,x,x")
+ (unspec:VF
+ [(match_operand:VF 1 "nonimmediate_operand" "%0, 0,x, x,x")
+ (match_operand:VF 2 "nonimmediate_operand" "xm, x,xm,x,m")
+ (match_operand:VF 3 "nonimmediate_operand" " x,xm,0,xm,x")]
+ UNSPEC_FMADDSUB))]
+ "TARGET_FMA || TARGET_FMA4"
+ "@
+ vfmaddsub132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
+ vfmaddsub213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
+ vfmaddsub231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
+ vfmaddsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
+ vfmaddsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+ [(set_attr "isa" "fma,fma,fma,fma4,fma4")
+ (set_attr "type" "ssemuladd")
+ (set_attr "mode" "<MODE>")])
+
+(define_insn "*fma_fmsubadd_<mode>"
+ [(set (match_operand:VF 0 "register_operand" "=x,x,x,x,x")
+ (unspec:VF
+ [(match_operand:VF 1 "nonimmediate_operand" "%0, 0,x, x,x")
+ (match_operand:VF 2 "nonimmediate_operand" "xm, x,xm,x,m")
+ (neg:VF
+ (match_operand:VF 3 "nonimmediate_operand" " x,xm,0,xm,x"))]
+ UNSPEC_FMADDSUB))]
+ "TARGET_FMA || TARGET_FMA4"
+ "@
+ vfmsubadd132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
+ vfmsubadd213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
+ vfmsubadd231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}
+ vfmsubadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}
+ vfmsubadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+ [(set_attr "isa" "fma,fma,fma,fma4,fma4")
+ (set_attr "type" "ssemuladd")
+ (set_attr "mode" "<MODE>")])
+
+;; FMA3 floating point scalar intrinsics. These merge result with
+;; high-order elements from the destination register.
(define_expand "fmai_vmfmadd_<mode>"
[(set (match_operand:VF_128 0 "register_operand")
(match_operand:VF_128 1 "nonimmediate_operand")
(match_operand:VF_128 2 "nonimmediate_operand")
(match_operand:VF_128 3 "nonimmediate_operand"))
- (match_dup 0)
+ (match_dup 1)
(const_int 1)))]
"TARGET_FMA")
(define_insn "*fmai_fmadd_<mode>"
- [(set (match_operand:VF_128 0 "register_operand" "=x,x,x")
+ [(set (match_operand:VF_128 0 "register_operand" "=x,x")
(vec_merge:VF_128
(fma:VF_128
- (match_operand:VF_128 1 "nonimmediate_operand" "%0, 0,x")
- (match_operand:VF_128 2 "nonimmediate_operand" "xm, x,xm")
- (match_operand:VF_128 3 "nonimmediate_operand" " x,xm,0"))
- (match_dup 0)
+ (match_operand:VF_128 1 "nonimmediate_operand" " 0, 0")
+ (match_operand:VF_128 2 "nonimmediate_operand" "xm, x")
+ (match_operand:VF_128 3 "nonimmediate_operand" " x,xm"))
+ (match_dup 1)
(const_int 1)))]
"TARGET_FMA"
"@
vfmadd132<ssescalarmodesuffix>\t{%2, %3, %0|%0, %3, %2}
- vfmadd213<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3}
- vfmadd231<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ vfmadd213<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3}"
[(set_attr "type" "ssemuladd")
(set_attr "mode" "<MODE>")])
(define_insn "*fmai_fmsub_<mode>"
- [(set (match_operand:VF_128 0 "register_operand" "=x,x,x")
+ [(set (match_operand:VF_128 0 "register_operand" "=x,x")
(vec_merge:VF_128
(fma:VF_128
- (match_operand:VF_128 1 "nonimmediate_operand" "%0, 0,x")
- (match_operand:VF_128 2 "nonimmediate_operand" "xm, x,xm")
+ (match_operand:VF_128 1 "nonimmediate_operand" " 0, 0")
+ (match_operand:VF_128 2 "nonimmediate_operand" "xm, x")
(neg:VF_128
- (match_operand:VF_128 3 "nonimmediate_operand" " x,xm,0")))
- (match_dup 0)
+ (match_operand:VF_128 3 "nonimmediate_operand" " x,xm")))
+ (match_dup 1)
(const_int 1)))]
"TARGET_FMA"
"@
vfmsub132<ssescalarmodesuffix>\t{%2, %3, %0|%0, %3, %2}
- vfmsub213<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3}
- vfmsub231<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ vfmsub213<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3}"
[(set_attr "type" "ssemuladd")
(set_attr "mode" "<MODE>")])
(define_insn "*fmai_fnmadd_<mode>"
- [(set (match_operand:VF_128 0 "register_operand" "=x,x,x")
+ [(set (match_operand:VF_128 0 "register_operand" "=x,x")
(vec_merge:VF_128
(fma:VF_128
(neg:VF_128
- (match_operand:VF_128 1 "nonimmediate_operand" "%0, 0,x"))
- (match_operand:VF_128 2 "nonimmediate_operand" "xm, x,xm")
- (match_operand:VF_128 3 "nonimmediate_operand" " x,xm,0"))
- (match_dup 0)
+ (match_operand:VF_128 2 "nonimmediate_operand" "xm, x"))
+ (match_operand:VF_128 1 "nonimmediate_operand" " 0, 0")
+ (match_operand:VF_128 3 "nonimmediate_operand" " x,xm"))
+ (match_dup 1)
(const_int 1)))]
"TARGET_FMA"
"@
vfnmadd132<ssescalarmodesuffix>\t{%2, %3, %0|%0, %3, %2}
- vfnmadd213<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3}
- vfnmadd231<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ vfnmadd213<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3}"
[(set_attr "type" "ssemuladd")
(set_attr "mode" "<MODE>")])
(define_insn "*fmai_fnmsub_<mode>"
- [(set (match_operand:VF_128 0 "register_operand" "=x,x,x")
+ [(set (match_operand:VF_128 0 "register_operand" "=x,x")
(vec_merge:VF_128
(fma:VF_128
(neg:VF_128
- (match_operand:VF_128 1 "nonimmediate_operand" "%0, 0,x"))
- (match_operand:VF_128 2 "nonimmediate_operand" "xm, x,xm")
+ (match_operand:VF_128 2 "nonimmediate_operand" "xm, x"))
+ (match_operand:VF_128 1 "nonimmediate_operand" " 0, 0")
(neg:VF_128
- (match_operand:VF_128 3 "nonimmediate_operand" " x,xm,0")))
- (match_dup 0)
+ (match_operand:VF_128 3 "nonimmediate_operand" " x,xm")))
+ (match_dup 1)
(const_int 1)))]
"TARGET_FMA"
"@
vfnmsub132<ssescalarmodesuffix>\t{%2, %3, %0|%0, %3, %2}
- vfnmsub213<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3}
- vfnmsub231<ssescalarmodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ vfnmsub213<ssescalarmodesuffix>\t{%3, %2, %0|%0, %2, %3}"
[(set_attr "type" "ssemuladd")
(set_attr "mode" "<MODE>")])
+;; FMA4 floating point scalar intrinsics. These write the
+;; entire destination register, with the high-order elements zeroed.
+
+(define_expand "fma4i_vmfmadd_<mode>"
+ [(set (match_operand:VF_128 0 "register_operand")
+ (vec_merge:VF_128
+ (fma:VF_128
+ (match_operand:VF_128 1 "nonimmediate_operand")
+ (match_operand:VF_128 2 "nonimmediate_operand")
+ (match_operand:VF_128 3 "nonimmediate_operand"))
+ (match_dup 4)
+ (const_int 1)))]
+ "TARGET_FMA4"
+{
+ operands[4] = CONST0_RTX (<MODE>mode);
+})
+
(define_insn "*fma4i_vmfmadd_<mode>"
[(set (match_operand:VF_128 0 "register_operand" "=x,x")
(vec_merge:VF_128
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;
-;; FMA4 Parallel floating point multiply addsub and subadd operations.
-;;
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; It would be possible to represent these without the UNSPEC as
-;;
-;; (vec_merge
-;; (fma op1 op2 op3)
-;; (fma op1 op2 (neg op3))
-;; (merge-const))
-;;
-;; But this doesn't seem useful in practice.
-
-(define_expand "fmaddsub_<mode>"
- [(set (match_operand:VF 0 "register_operand")
- (unspec:VF
- [(match_operand:VF 1 "nonimmediate_operand")
- (match_operand:VF 2 "nonimmediate_operand")
- (match_operand:VF 3 "nonimmediate_operand")]
- UNSPEC_FMADDSUB))]
- "TARGET_FMA || TARGET_FMA4")
-
-(define_insn "*fma4_fmaddsub_<mode>"
- [(set (match_operand:VF 0 "register_operand" "=x,x")
- (unspec:VF
- [(match_operand:VF 1 "nonimmediate_operand" "%x,x")
- (match_operand:VF 2 "nonimmediate_operand" " x,m")
- (match_operand:VF 3 "nonimmediate_operand" "xm,x")]
- UNSPEC_FMADDSUB))]
- "TARGET_FMA4"
- "vfmaddsub<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
- [(set_attr "type" "ssemuladd")
- (set_attr "mode" "<MODE>")])
-
-(define_insn "*fma4_fmsubadd_<mode>"
- [(set (match_operand:VF 0 "register_operand" "=x,x")
- (unspec:VF
- [(match_operand:VF 1 "nonimmediate_operand" "%x,x")
- (match_operand:VF 2 "nonimmediate_operand" " x,m")
- (neg:VF
- (match_operand:VF 3 "nonimmediate_operand" "xm,x"))]
- UNSPEC_FMADDSUB))]
- "TARGET_FMA4"
- "vfmsubadd<ssemodesuffix>\t{%3, %2, %1, %0|%0, %1, %2, %3}"
- [(set_attr "type" "ssemuladd")
- (set_attr "mode" "<MODE>")])
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;;
-;; FMA3 floating point multiply/accumulate instructions.
-;;
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-(define_insn "*fma_fmadd_<mode>"
- [(set (match_operand:FMAMODE 0 "register_operand" "=x,x,x")
- (fma:FMAMODE
- (match_operand:FMAMODE 1 "nonimmediate_operand" "%0, 0,x")
- (match_operand:FMAMODE 2 "nonimmediate_operand" "xm, x,xm")
- (match_operand:FMAMODE 3 "nonimmediate_operand" " x,xm,0")))]
- "TARGET_FMA"
- "@
- vfmadd132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
- vfmadd213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
- vfmadd231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
- [(set_attr "type" "ssemuladd")
- (set_attr "mode" "<MODE>")])
-
-(define_insn "*fma_fmsub_<mode>"
- [(set (match_operand:FMAMODE 0 "register_operand" "=x,x,x")
- (fma:FMAMODE
- (match_operand:FMAMODE 1 "nonimmediate_operand" "%0, 0,x")
- (match_operand:FMAMODE 2 "nonimmediate_operand" "xm, x,xm")
- (neg:FMAMODE
- (match_operand:FMAMODE 3 "nonimmediate_operand" " x,xm,0"))))]
- "TARGET_FMA"
- "@
- vfmsub132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
- vfmsub213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
- vfmsub231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
- [(set_attr "type" "ssemuladd")
- (set_attr "mode" "<MODE>")])
-
-(define_insn "*fma_fnmadd_<mode>"
- [(set (match_operand:FMAMODE 0 "register_operand" "=x,x,x")
- (fma:FMAMODE
- (neg:FMAMODE
- (match_operand:FMAMODE 1 "nonimmediate_operand" "%0, 0,x"))
- (match_operand:FMAMODE 2 "nonimmediate_operand" "xm, x,xm")
- (match_operand:FMAMODE 3 "nonimmediate_operand" " x,xm,0")))]
- "TARGET_FMA"
- "@
- vfnmadd132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
- vfnmadd213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
- vfnmadd231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
- [(set_attr "type" "ssemuladd")
- (set_attr "mode" "<MODE>")])
-
-(define_insn "*fma_fnmsub_<mode>"
- [(set (match_operand:FMAMODE 0 "register_operand" "=x,x,x")
- (fma:FMAMODE
- (neg:FMAMODE
- (match_operand:FMAMODE 1 "nonimmediate_operand" "%0, 0,x"))
- (match_operand:FMAMODE 2 "nonimmediate_operand" "xm, x,xm")
- (neg:FMAMODE
- (match_operand:FMAMODE 3 "nonimmediate_operand" " x,xm,0"))))]
- "TARGET_FMA"
- "@
- vfnmsub132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
- vfnmsub213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
- vfnmsub231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
- [(set_attr "type" "ssemuladd")
- (set_attr "mode" "<MODE>")])
-
-(define_insn "*fma_fmaddsub_<mode>"
- [(set (match_operand:VF 0 "register_operand" "=x,x,x")
- (unspec:VF
- [(match_operand:VF 1 "nonimmediate_operand" "%0, 0,x")
- (match_operand:VF 2 "nonimmediate_operand" "xm, x,xm")
- (match_operand:VF 3 "nonimmediate_operand" " x,xm,0")]
- UNSPEC_FMADDSUB))]
- "TARGET_FMA"
- "@
- vfmaddsub132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
- vfmaddsub213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
- vfmaddsub231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
- [(set_attr "type" "ssemuladd")
- (set_attr "mode" "<MODE>")])
-
-(define_insn "*fma_fmsubadd_<mode>"
- [(set (match_operand:VF 0 "register_operand" "=x,x,x")
- (unspec:VF
- [(match_operand:VF 1 "nonimmediate_operand" "%0, 0,x")
- (match_operand:VF 2 "nonimmediate_operand" "xm, x,xm")
- (neg:VF
- (match_operand:VF 3 "nonimmediate_operand" " x,xm,0"))]
- UNSPEC_FMADDSUB))]
- "TARGET_FMA"
- "@
- vfmsubadd132<ssemodesuffix>\t{%2, %3, %0|%0, %3, %2}
- vfmsubadd213<ssemodesuffix>\t{%3, %2, %0|%0, %2, %3}
- vfmsubadd231<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
- [(set_attr "type" "ssemuladd")
- (set_attr "mode" "<MODE>")])
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;;
;; Parallel single-precision floating point conversion operations
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "DI")])
-(define_insn "avx_cvtdq2ps256"
- [(set (match_operand:V8SF 0 "register_operand" "=x")
- (float:V8SF (match_operand:V8SI 1 "nonimmediate_operand" "xm")))]
- "TARGET_AVX"
- "vcvtdq2ps\t{%1, %0|%0, %1}"
- [(set_attr "type" "ssecvt")
- (set_attr "prefix" "vex")
- (set_attr "mode" "V8SF")])
-
-(define_insn "sse2_cvtdq2ps"
- [(set (match_operand:V4SF 0 "register_operand" "=x")
- (float:V4SF (match_operand:V4SI 1 "nonimmediate_operand" "xm")))]
+(define_insn "float<sseintvecmodelower><mode>2"
+ [(set (match_operand:VF1 0 "register_operand" "=x")
+ (float:VF1
+ (match_operand:<sseintvecmode> 1 "nonimmediate_operand" "xm")))]
"TARGET_SSE2"
"%vcvtdq2ps\t{%1, %0|%0, %1}"
[(set_attr "type" "ssecvt")
(set_attr "prefix" "maybe_vex")
- (set_attr "mode" "V4SF")])
+ (set_attr "mode" "<sseinsnmode>")])
-(define_expand "sse2_cvtudq2ps"
- [(set (match_dup 5)
- (float:V4SF (match_operand:V4SI 1 "nonimmediate_operand" "")))
- (set (match_dup 6)
- (lt:V4SF (match_dup 5) (match_dup 3)))
- (set (match_dup 7)
- (and:V4SF (match_dup 6) (match_dup 4)))
- (set (match_operand:V4SF 0 "register_operand" "")
- (plus:V4SF (match_dup 5) (match_dup 7)))]
- "TARGET_SSE2"
+(define_expand "floatuns<sseintvecmodelower><mode>2"
+ [(match_operand:VF1 0 "register_operand" "")
+ (match_operand:<sseintvecmode> 1 "register_operand" "")]
+ "TARGET_SSE2 && (<MODE>mode == V4SFmode || TARGET_AVX2)"
{
- REAL_VALUE_TYPE TWO32r;
- rtx x;
- int i;
-
- real_ldexp (&TWO32r, &dconst1, 32);
- x = const_double_from_real_value (TWO32r, SFmode);
-
- operands[3] = force_reg (V4SFmode, CONST0_RTX (V4SFmode));
- operands[4] = force_reg (V4SFmode,
- ix86_build_const_vector (V4SFmode, 1, x));
-
- for (i = 5; i < 8; i++)
- operands[i] = gen_reg_rtx (V4SFmode);
+ ix86_expand_vector_convert_uns_vsivsf (operands[0], operands[1]);
+ DONE;
})
(define_insn "avx_cvtps2dq256"
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "TI")])
-(define_insn "avx_cvttps2dq256"
+(define_insn "fix_truncv8sfv8si2"
[(set (match_operand:V8SI 0 "register_operand" "=x")
(fix:V8SI (match_operand:V8SF 1 "nonimmediate_operand" "xm")))]
"TARGET_AVX"
(set_attr "prefix" "vex")
(set_attr "mode" "OI")])
-(define_insn "sse2_cvttps2dq"
+(define_insn "fix_truncv4sfv4si2"
[(set (match_operand:V4SI 0 "register_operand" "=x")
(fix:V4SI (match_operand:V4SF 1 "nonimmediate_operand" "xm")))]
"TARGET_SSE2"
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "TI")])
+(define_expand "fixuns_trunc<mode><sseintvecmodelower>2"
+ [(match_operand:<sseintvecmode> 0 "register_operand" "")
+ (match_operand:VF1 1 "register_operand" "")]
+ "TARGET_SSE2"
+{
+ rtx tmp[3];
+ tmp[0] = ix86_expand_adjust_ufix_to_sfix_si (operands[1], &tmp[2]);
+ tmp[1] = gen_reg_rtx (<sseintvecmode>mode);
+ emit_insn (gen_fix_trunc<mode><sseintvecmodelower>2 (tmp[1], tmp[0]));
+ emit_insn (gen_xor<sseintvecmodelower>3 (operands[0], tmp[1], tmp[2]));
+ DONE;
+})
+
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;
;; Parallel double-precision floating point conversion operations
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "DI")])
-(define_insn "avx_cvtdq2pd256"
+(define_insn "floatv4siv4df2"
[(set (match_operand:V4DF 0 "register_operand" "=x")
(float:V4DF (match_operand:V4SI 1 "nonimmediate_operand" "xm")))]
"TARGET_AVX"
(match_operand:V4SI 1 "nonimmediate_operand" "xm")
(parallel [(const_int 0) (const_int 1)]))))]
"TARGET_SSE2"
- "%vcvtdq2pd\t{%1, %0|%0, %1}"
+ "%vcvtdq2pd\t{%1, %0|%0, %q1}"
[(set_attr "type" "ssecvt")
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "V2DF")])
(set_attr "prefix" "vex")
(set_attr "mode" "OI")])
+(define_expand "avx_cvtpd2dq256_2"
+ [(set (match_operand:V8SI 0 "register_operand" "")
+ (vec_concat:V8SI
+ (unspec:V4SI [(match_operand:V4DF 1 "nonimmediate_operand" "")]
+ UNSPEC_FIX_NOTRUNC)
+ (match_dup 2)))]
+ "TARGET_AVX"
+ "operands[2] = CONST0_RTX (V4SImode);")
+
+(define_insn "*avx_cvtpd2dq256_2"
+ [(set (match_operand:V8SI 0 "register_operand" "=x")
+ (vec_concat:V8SI
+ (unspec:V4SI [(match_operand:V4DF 1 "nonimmediate_operand" "xm")]
+ UNSPEC_FIX_NOTRUNC)
+ (match_operand:V4SI 2 "const0_operand" "")))]
+ "TARGET_AVX"
+ "vcvtpd2dq{y}\t{%1, %x0|%x0, %1}"
+ [(set_attr "type" "ssecvt")
+ (set_attr "prefix" "vex")
+ (set_attr "mode" "OI")])
+
(define_expand "sse2_cvtpd2dq"
[(set (match_operand:V4SI 0 "register_operand" "")
(vec_concat:V4SI
(set_attr "athlon_decode" "vector")
(set_attr "bdver1_decode" "double")])
-(define_insn "avx_cvttpd2dq256"
+(define_insn "fix_truncv4dfv4si2"
[(set (match_operand:V4SI 0 "register_operand" "=x")
(fix:V4SI (match_operand:V4DF 1 "nonimmediate_operand" "xm")))]
"TARGET_AVX"
(set_attr "prefix" "vex")
(set_attr "mode" "OI")])
+(define_expand "avx_cvttpd2dq256_2"
+ [(set (match_operand:V8SI 0 "register_operand" "")
+ (vec_concat:V8SI
+ (fix:V4SI (match_operand:V4DF 1 "nonimmediate_operand" ""))
+ (match_dup 2)))]
+ "TARGET_AVX"
+ "operands[2] = CONST0_RTX (V4SImode);")
+
+(define_insn "*avx_cvttpd2dq256_2"
+ [(set (match_operand:V8SI 0 "register_operand" "=x")
+ (vec_concat:V8SI
+ (fix:V4SI (match_operand:V4DF 1 "nonimmediate_operand" "xm"))
+ (match_operand:V4SI 2 "const0_operand" "")))]
+ "TARGET_AVX"
+ "vcvttpd2dq{y}\t{%1, %x0|%x0, %1}"
+ [(set_attr "type" "ssecvt")
+ (set_attr "prefix" "vex")
+ (set_attr "mode" "OI")])
+
(define_expand "sse2_cvttpd2dq"
[(set (match_operand:V4SI 0 "register_operand" "")
(vec_concat:V4SI
(match_operand:V4SF 1 "nonimmediate_operand" "xm")
(parallel [(const_int 0) (const_int 1)]))))]
"TARGET_SSE2"
- "%vcvtps2pd\t{%1, %0|%0, %1}"
+ "%vcvtps2pd\t{%1, %0|%0, %q1}"
[(set_attr "type" "ssecvt")
(set_attr "amdfam10_decode" "direct")
(set_attr "athlon_decode" "double")
for (i = 2; i < 5; i++)
tmp[i] = gen_reg_rtx (V4DFmode);
emit_insn (gen_vec_extract_hi_v8si (tmp[5], operands[1]));
- emit_insn (gen_avx_cvtdq2pd256 (tmp[2], tmp[5]));
+ emit_insn (gen_floatv4siv4df2 (tmp[2], tmp[5]));
emit_insn (gen_rtx_SET (VOIDmode, tmp[3],
gen_rtx_LT (V4DFmode, tmp[2], tmp[0])));
emit_insn (gen_andv4df3 (tmp[4], tmp[3], tmp[1]));
(match_operand:V2DF 2 "nonimmediate_operand" "")]
"TARGET_SSE2"
{
+ rtx tmp0, tmp1;
+
+ if (TARGET_AVX && !TARGET_PREFER_AVX128)
+ {
+ tmp0 = gen_reg_rtx (V4DFmode);
+ tmp1 = force_reg (V2DFmode, operands[1]);
+
+ emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
+ emit_insn (gen_avx_cvtpd2ps256 (operands[0], tmp0));
+ }
+ else
+ {
+ tmp0 = gen_reg_rtx (V4SFmode);
+ tmp1 = gen_reg_rtx (V4SFmode);
+
+ emit_insn (gen_sse2_cvtpd2ps (tmp0, operands[1]));
+ emit_insn (gen_sse2_cvtpd2ps (tmp1, operands[2]));
+ emit_insn (gen_sse_movlhps (operands[0], tmp0, tmp1));
+ }
+ DONE;
+})
+
+(define_expand "vec_pack_sfix_trunc_v4df"
+ [(match_operand:V8SI 0 "register_operand" "")
+ (match_operand:V4DF 1 "nonimmediate_operand" "")
+ (match_operand:V4DF 2 "nonimmediate_operand" "")]
+ "TARGET_AVX"
+{
rtx r1, r2;
- r1 = gen_reg_rtx (V4SFmode);
- r2 = gen_reg_rtx (V4SFmode);
+ r1 = gen_reg_rtx (V4SImode);
+ r2 = gen_reg_rtx (V4SImode);
- emit_insn (gen_sse2_cvtpd2ps (r1, operands[1]));
- emit_insn (gen_sse2_cvtpd2ps (r2, operands[2]));
- emit_insn (gen_sse_movlhps (operands[0], r1, r2));
+ emit_insn (gen_fix_truncv4dfv4si2 (r1, operands[1]));
+ emit_insn (gen_fix_truncv4dfv4si2 (r2, operands[2]));
+ emit_insn (gen_avx_vec_concatv8si (operands[0], r1, r2));
DONE;
})
(match_operand:V2DF 2 "nonimmediate_operand" "")]
"TARGET_SSE2"
{
+ rtx tmp0, tmp1;
+
+ if (TARGET_AVX && !TARGET_PREFER_AVX128)
+ {
+ tmp0 = gen_reg_rtx (V4DFmode);
+ tmp1 = force_reg (V2DFmode, operands[1]);
+
+ emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
+ emit_insn (gen_fix_truncv4dfv4si2 (operands[0], tmp0));
+ }
+ else
+ {
+ tmp0 = gen_reg_rtx (V4SImode);
+ tmp1 = gen_reg_rtx (V4SImode);
+
+ emit_insn (gen_sse2_cvttpd2dq (tmp0, operands[1]));
+ emit_insn (gen_sse2_cvttpd2dq (tmp1, operands[2]));
+ emit_insn
+ (gen_vec_interleave_lowv2di (gen_lowpart (V2DImode, operands[0]),
+ gen_lowpart (V2DImode, tmp0),
+ gen_lowpart (V2DImode, tmp1)));
+ }
+ DONE;
+})
+
+(define_mode_attr ssepackfltmode
+ [(V4DF "V8SI") (V2DF "V4SI")])
+
+(define_expand "vec_pack_ufix_trunc_<mode>"
+ [(match_operand:<ssepackfltmode> 0 "register_operand" "")
+ (match_operand:VF2 1 "register_operand" "")
+ (match_operand:VF2 2 "register_operand" "")]
+ "TARGET_SSE2"
+{
+ rtx tmp[7];
+ tmp[0] = ix86_expand_adjust_ufix_to_sfix_si (operands[1], &tmp[2]);
+ tmp[1] = ix86_expand_adjust_ufix_to_sfix_si (operands[2], &tmp[3]);
+ tmp[4] = gen_reg_rtx (<ssepackfltmode>mode);
+ emit_insn (gen_vec_pack_sfix_trunc_<mode> (tmp[4], tmp[0], tmp[1]));
+ if (<ssepackfltmode>mode == V4SImode || TARGET_AVX2)
+ {
+ tmp[5] = gen_reg_rtx (<ssepackfltmode>mode);
+ ix86_expand_vec_extract_even_odd (tmp[5], tmp[2], tmp[3], 0);
+ }
+ else
+ {
+ tmp[5] = gen_reg_rtx (V8SFmode);
+ ix86_expand_vec_extract_even_odd (tmp[5], gen_lowpart (V8SFmode, tmp[2]),
+ gen_lowpart (V8SFmode, tmp[3]), 0);
+ tmp[5] = gen_lowpart (V8SImode, tmp[5]);
+ }
+ tmp[6] = expand_simple_binop (<ssepackfltmode>mode, XOR, tmp[4], tmp[5],
+ operands[0], 0, OPTAB_DIRECT);
+ if (tmp[6] != operands[0])
+ emit_move_insn (operands[0], tmp[6]);
+ DONE;
+})
+
+(define_expand "vec_pack_sfix_v4df"
+ [(match_operand:V8SI 0 "register_operand" "")
+ (match_operand:V4DF 1 "nonimmediate_operand" "")
+ (match_operand:V4DF 2 "nonimmediate_operand" "")]
+ "TARGET_AVX"
+{
rtx r1, r2;
r1 = gen_reg_rtx (V4SImode);
r2 = gen_reg_rtx (V4SImode);
- emit_insn (gen_sse2_cvttpd2dq (r1, operands[1]));
- emit_insn (gen_sse2_cvttpd2dq (r2, operands[2]));
- emit_insn (gen_vec_interleave_lowv2di (gen_lowpart (V2DImode, operands[0]),
- gen_lowpart (V2DImode, r1),
- gen_lowpart (V2DImode, r2)));
+ emit_insn (gen_avx_cvtpd2dq256 (r1, operands[1]));
+ emit_insn (gen_avx_cvtpd2dq256 (r2, operands[2]));
+ emit_insn (gen_avx_vec_concatv8si (operands[0], r1, r2));
DONE;
})
(match_operand:V2DF 2 "nonimmediate_operand" "")]
"TARGET_SSE2"
{
- rtx r1, r2;
+ rtx tmp0, tmp1;
- r1 = gen_reg_rtx (V4SImode);
- r2 = gen_reg_rtx (V4SImode);
+ if (TARGET_AVX && !TARGET_PREFER_AVX128)
+ {
+ tmp0 = gen_reg_rtx (V4DFmode);
+ tmp1 = force_reg (V2DFmode, operands[1]);
- emit_insn (gen_sse2_cvtpd2dq (r1, operands[1]));
- emit_insn (gen_sse2_cvtpd2dq (r2, operands[2]));
- emit_insn (gen_vec_interleave_lowv2di (gen_lowpart (V2DImode, operands[0]),
- gen_lowpart (V2DImode, r1),
- gen_lowpart (V2DImode, r2)));
+ emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
+ emit_insn (gen_avx_cvtpd2dq256 (operands[0], tmp0));
+ }
+ else
+ {
+ tmp0 = gen_reg_rtx (V4SImode);
+ tmp1 = gen_reg_rtx (V4SImode);
+
+ emit_insn (gen_sse2_cvtpd2dq (tmp0, operands[1]));
+ emit_insn (gen_sse2_cvtpd2dq (tmp1, operands[2]));
+ emit_insn
+ (gen_vec_interleave_lowv2di (gen_lowpart (V2DImode, operands[0]),
+ gen_lowpart (V2DImode, tmp0),
+ gen_lowpart (V2DImode, tmp1)));
+ }
DONE;
})
(vec_select:V4SF
(vec_concat:V8SF
(match_operand:V4SF 1 "nonimmediate_operand" " 0,x,0,x,0")
- (match_operand:V4SF 2 "nonimmediate_operand" " x,x,m,x,x"))
+ (match_operand:V4SF 2 "nonimmediate_operand" " x,x,m,m,x"))
(parallel [(const_int 0)
(const_int 1)
(const_int 4)
(define_insn "sse_loadlps"
[(set (match_operand:V4SF 0 "nonimmediate_operand" "=x,x,x,x,m")
(vec_concat:V4SF
- (match_operand:V2SF 2 "nonimmediate_operand" " 0,x,m,x,x")
+ (match_operand:V2SF 2 "nonimmediate_operand" " 0,x,m,m,x")
(vec_select:V2SF
(match_operand:V4SF 1 "nonimmediate_operand" " x,x,0,x,0")
(parallel [(const_int 2) (const_int 3)]))))]
(set_attr "prefix" "orig,vex")
(set_attr "mode" "SF")])
-(define_expand "vec_dupv4sf"
- [(set (match_operand:V4SF 0 "register_operand" "")
- (vec_duplicate:V4SF
- (match_operand:SF 1 "nonimmediate_operand" "")))]
- "TARGET_SSE"
-{
- if (!TARGET_AVX)
- operands[1] = force_reg (SFmode, operands[1]);
-})
-
-(define_insn "avx2_vec_dupv4sf"
- [(set (match_operand:V4SF 0 "register_operand" "=x")
- (vec_duplicate:V4SF
+(define_insn "avx2_vec_dup<mode>"
+ [(set (match_operand:VF1 0 "register_operand" "=x")
+ (vec_duplicate:VF1
(vec_select:SF
(match_operand:V4SF 1 "register_operand" "x")
(parallel [(const_int 0)]))))]
"vbroadcastss\t{%1, %0|%0, %1}"
[(set_attr "type" "sselog1")
(set_attr "prefix" "vex")
- (set_attr "mode" "V4SF")])
+ (set_attr "mode" "<MODE>")])
-(define_insn "*vec_dupv4sf_avx"
- [(set (match_operand:V4SF 0 "register_operand" "=x,x")
+(define_insn "vec_dupv4sf"
+ [(set (match_operand:V4SF 0 "register_operand" "=x,x,x")
(vec_duplicate:V4SF
- (match_operand:SF 1 "nonimmediate_operand" "x,m")))]
- "TARGET_AVX"
+ (match_operand:SF 1 "nonimmediate_operand" "x,m,0")))]
+ "TARGET_SSE"
"@
vshufps\t{$0, %1, %1, %0|%0, %1, %1, 0}
- vbroadcastss\t{%1, %0|%0, %1}"
- [(set_attr "type" "sselog1,ssemov")
- (set_attr "length_immediate" "1,0")
- (set_attr "prefix_extra" "0,1")
- (set_attr "prefix" "vex")
- (set_attr "mode" "V4SF")])
-
-(define_insn "avx2_vec_dupv8sf"
- [(set (match_operand:V8SF 0 "register_operand" "=x")
- (vec_duplicate:V8SF
- (vec_select:SF
- (match_operand:V4SF 1 "register_operand" "x")
- (parallel [(const_int 0)]))))]
- "TARGET_AVX2"
- "vbroadcastss\t{%1, %0|%0, %1}"
- [(set_attr "type" "sselog1")
- (set_attr "prefix" "vex")
- (set_attr "mode" "V8SF")])
-
-(define_insn "*vec_dupv4sf"
- [(set (match_operand:V4SF 0 "register_operand" "=x")
- (vec_duplicate:V4SF
- (match_operand:SF 1 "register_operand" "0")))]
- "TARGET_SSE"
- "shufps\t{$0, %0, %0|%0, %0, 0}"
- [(set_attr "type" "sselog1")
- (set_attr "length_immediate" "1")
+ vbroadcastss\t{%1, %0|%0, %1}
+ shufps\t{$0, %0, %0|%0, %0, 0}"
+ [(set_attr "isa" "avx,avx,noavx")
+ (set_attr "type" "sselog1,ssemov,sselog1")
+ (set_attr "length_immediate" "1,0,1")
+ (set_attr "prefix_extra" "0,1,*")
+ (set_attr "prefix" "vex,vex,orig")
(set_attr "mode" "V4SF")])
;; Although insertps takes register source, we prefer
;; see comment above inline_secondary_memory_needed function in i386.c
(define_insn "vec_set<mode>_0"
[(set (match_operand:VI4F_128 0 "nonimmediate_operand"
- "=x,x,x ,x,x,x,x ,x ,m,m ,m")
+ "=x,x,x ,x,x,x,x ,x ,m ,m ,m")
(vec_merge:VI4F_128
(vec_duplicate:VI4F_128
(match_operand:<ssescalarmode> 2 "general_operand"
- " x,m,*r,m,x,x,*rm,*rm,x,fF,*r"))
+ " x,m,*r,m,x,x,*rm,*rm,!x,!*re,!*fF"))
(match_operand:VI4F_128 1 "vector_move_operand"
- " C,C,C ,C,0,x,0 ,x ,0,0 ,0")
+ " C,C,C ,C,0,x,0 ,x ,0 ,0 ,0")
(const_int 1)))]
"TARGET_SSE"
"@
(cond [(eq_attr "alternative" "0,6,7")
(const_string "sselog")
(eq_attr "alternative" "9")
- (const_string "fmov")
- (eq_attr "alternative" "10")
(const_string "imov")
+ (eq_attr "alternative" "10")
+ (const_string "fmov")
]
(const_string "ssemov")))
(set_attr "prefix_extra" "*,*,*,*,*,*,1,1,*,*,*")
(set_attr "type" "sselog,sselog,sselog,ssemov,ssemov,ssemov")
(set_attr "prefix_data16" "*,*,*,1,*,1")
(set_attr "prefix" "orig,vex,maybe_vex,orig,vex,maybe_vex")
- (set_attr "mode" "V2DF,V2DF,V2DF,V1DF,V1DF,V1DF")])
+ (set_attr "mode" "V2DF,V2DF,DF,V1DF,V1DF,V1DF")])
;; Recall that the 256-bit unpck insns only shuffle within their lanes.
(define_expand "avx_movddup256"
(set_attr "type" "sselog,sselog,sselog,ssemov,ssemov,ssemov")
(set_attr "prefix_data16" "*,*,*,1,*,1")
(set_attr "prefix" "orig,vex,maybe_vex,orig,vex,maybe_vex")
- (set_attr "mode" "V2DF,V2DF,V2DF,V1DF,V1DF,V1DF")])
+ (set_attr "mode" "V2DF,V2DF,DF,V1DF,V1DF,V1DF")])
(define_split
[(set (match_operand:V2DF 0 "memory_operand" "")
(vec_select:DF (match_dup 0) (parallel [(const_int 1)]))))]
"TARGET_SSE2 && reload_completed"
[(set (match_dup 0) (match_dup 1))]
- "operands[0] = adjust_address (operands[0], DFmode, 8);")
+ "operands[0] = adjust_address (operands[0], DFmode, 0);")
(define_insn "sse2_movsd"
[(set (match_operand:V2DF 0 "nonimmediate_operand" "=x,x,x,x,m,x,x,x,o")
(set_attr "prefix" "orig,vex,orig,vex,maybe_vex,orig,orig,vex,maybe_vex")
(set_attr "mode" "DF,DF,V1DF,V1DF,V1DF,V2DF,V1DF,V1DF,V1DF")])
-(define_expand "vec_dupv2df"
- [(set (match_operand:V2DF 0 "register_operand" "")
- (vec_duplicate:V2DF
- (match_operand:DF 1 "nonimmediate_operand" "")))]
- "TARGET_SSE2"
-{
- if (!TARGET_SSE3)
- operands[1] = force_reg (DFmode, operands[1]);
-})
-
-(define_insn "*vec_dupv2df_sse3"
- [(set (match_operand:V2DF 0 "register_operand" "=x")
- (vec_duplicate:V2DF
- (match_operand:DF 1 "nonimmediate_operand" "xm")))]
- "TARGET_SSE3"
- "%vmovddup\t{%1, %0|%0, %1}"
- [(set_attr "type" "sselog1")
- (set_attr "prefix" "maybe_vex")
- (set_attr "mode" "DF")])
-
-(define_insn "*vec_dupv2df"
- [(set (match_operand:V2DF 0 "register_operand" "=x")
+(define_insn "vec_dupv2df"
+ [(set (match_operand:V2DF 0 "register_operand" "=x,x")
(vec_duplicate:V2DF
- (match_operand:DF 1 "register_operand" "0")))]
+ (match_operand:DF 1 "nonimmediate_operand" " 0,xm")))]
"TARGET_SSE2"
- "unpcklpd\t%0, %0"
- [(set_attr "type" "sselog1")
- (set_attr "mode" "V2DF")])
-
-(define_insn "*vec_concatv2df_sse3"
- [(set (match_operand:V2DF 0 "register_operand" "=x")
- (vec_concat:V2DF
- (match_operand:DF 1 "nonimmediate_operand" "xm")
- (match_dup 1)))]
- "TARGET_SSE3"
- "%vmovddup\t{%1, %0|%0, %1}"
- [(set_attr "type" "sselog1")
- (set_attr "prefix" "maybe_vex")
- (set_attr "mode" "DF")])
+ "@
+ unpcklpd\t%0, %0
+ %vmovddup\t{%1, %0|%0, %1}"
+ [(set_attr "isa" "noavx,sse3")
+ (set_attr "type" "sselog1")
+ (set_attr "prefix" "orig,maybe_vex")
+ (set_attr "mode" "V2DF,DF")])
(define_insn "*vec_concatv2df"
- [(set (match_operand:V2DF 0 "register_operand" "=x,x,x,x,x,x,x")
+ [(set (match_operand:V2DF 0 "register_operand" "=x,x,x,x,x,x,x,x")
(vec_concat:V2DF
- (match_operand:DF 1 "nonimmediate_operand" " 0,x,0,x,m,0,0")
- (match_operand:DF 2 "vector_move_operand" " x,x,m,m,C,x,m")))]
+ (match_operand:DF 1 "nonimmediate_operand" " 0,x,m,0,x,m,0,0")
+ (match_operand:DF 2 "vector_move_operand" " x,x,1,m,m,C,x,m")))]
"TARGET_SSE"
"@
unpcklpd\t{%2, %0|%0, %2}
vunpcklpd\t{%2, %1, %0|%0, %1, %2}
+ %vmovddup\t{%1, %0|%0, %1}
movhpd\t{%2, %0|%0, %2}
vmovhpd\t{%2, %1, %0|%0, %1, %2}
%vmovsd\t{%1, %0|%0, %1}
movlhps\t{%2, %0|%0, %2}
movhps\t{%2, %0|%0, %2}"
- [(set_attr "isa" "sse2_noavx,avx,sse2_noavx,avx,sse2,noavx,noavx")
+ [(set_attr "isa" "sse2_noavx,avx,sse3,sse2_noavx,avx,sse2,noavx,noavx")
(set (attr "type")
(if_then_else
- (eq_attr "alternative" "0,1")
+ (eq_attr "alternative" "0,1,2")
(const_string "sselog")
(const_string "ssemov")))
- (set_attr "prefix_data16" "*,*,1,*,*,*,*")
- (set_attr "prefix" "orig,vex,orig,vex,maybe_vex,orig,orig")
- (set_attr "mode" "V2DF,V2DF,V1DF,V1DF,DF,V4SF,V2SF")])
+ (set_attr "prefix_data16" "*,*,*,1,*,*,*,*")
+ (set_attr "prefix" "orig,vex,maybe_vex,orig,vex,maybe_vex,orig,orig")
+ (set_attr "mode" "V2DF,V2DF,DF,V1DF,V1DF,DF,V4SF,V2SF")])
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;
gen_lowpart (mulmode, t[3]))));
/* Extract the even bytes and merge them back together. */
- ix86_expand_vec_extract_even_odd (operands[0], t[5], t[4], 0);
+ if (<MODE>mode == V16QImode)
+ ix86_expand_vec_extract_even_odd (operands[0], t[5], t[4], 0);
+ else
+ {
+ /* Since avx2_interleave_{low,high}v32qi used above aren't cross-lane,
+ this can't be normal even extraction, but one where additionally
+ the second and third quarter are swapped. That is even one insn
+ shorter than even extraction. */
+ rtvec v = rtvec_alloc (32);
+ for (i = 0; i < 32; ++i)
+ RTVEC_ELT (v, i)
+ = GEN_INT (i * 2 + ((i & 24) == 8 ? 16 : (i & 24) == 16 ? -16 : 0));
+ t[0] = operands[0];
+ t[1] = t[5];
+ t[2] = t[4];
+ t[3] = gen_rtx_CONST_VECTOR (<MODE>mode, v);
+ ix86_expand_vec_perm_const (t);
+ }
set_unique_reg_note (get_last_insn (), REG_EQUAL,
gen_rtx_MULT (<MODE>mode, operands[1], operands[2]));
if (TARGET_XOP)
{
+ rtx t3 = gen_reg_rtx (V2DImode);
+
emit_insn (gen_sse2_pshufd_1 (t1, op1, GEN_INT (0), GEN_INT (2),
GEN_INT (1), GEN_INT (3)));
emit_insn (gen_sse2_pshufd_1 (t2, op2, GEN_INT (0), GEN_INT (2),
GEN_INT (1), GEN_INT (3)));
- emit_insn (gen_xop_mulv2div2di3_high (operands[0], t1, t2));
+ emit_move_insn (t3, CONST0_RTX (V2DImode));
+
+ emit_insn (gen_xop_pmacsdqh (operands[0], t1, t2, t3));
DONE;
}
if (TARGET_XOP)
{
+ rtx t3 = gen_reg_rtx (V2DImode);
+
emit_insn (gen_sse2_pshufd_1 (t1, op1, GEN_INT (0), GEN_INT (2),
GEN_INT (1), GEN_INT (3)));
emit_insn (gen_sse2_pshufd_1 (t2, op2, GEN_INT (0), GEN_INT (2),
GEN_INT (1), GEN_INT (3)));
- emit_insn (gen_xop_mulv2div2di3_low (operands[0], t1, t2));
+ emit_move_insn (t3, CONST0_RTX (V2DImode));
+
+ emit_insn (gen_xop_pmacsdql (operands[0], t1, t2, t3));
DONE;
}
(set_attr "prefix" "orig,vex")
(set_attr "mode" "<sseinsnmode>")])
-(define_insn "lshr<mode>3"
- [(set (match_operand:VI248_AVX2 0 "register_operand" "=x,x")
- (lshiftrt:VI248_AVX2
- (match_operand:VI248_AVX2 1 "register_operand" "0,x")
- (match_operand:SI 2 "nonmemory_operand" "xN,xN")))]
- "TARGET_SSE2"
- "@
- psrl<ssemodesuffix>\t{%2, %0|%0, %2}
- vpsrl<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
- [(set_attr "isa" "noavx,avx")
- (set_attr "type" "sseishft")
- (set (attr "length_immediate")
- (if_then_else (match_operand 2 "const_int_operand" "")
- (const_string "1")
- (const_string "0")))
- (set_attr "prefix_data16" "1,*")
- (set_attr "prefix" "orig,vex")
- (set_attr "mode" "<sseinsnmode>")])
-
-(define_insn "ashl<mode>3"
+(define_insn "<shift_insn><mode>3"
[(set (match_operand:VI248_AVX2 0 "register_operand" "=x,x")
- (ashift:VI248_AVX2
+ (any_lshift:VI248_AVX2
(match_operand:VI248_AVX2 1 "register_operand" "0,x")
(match_operand:SI 2 "nonmemory_operand" "xN,xN")))]
"TARGET_SSE2"
"@
- psll<ssemodesuffix>\t{%2, %0|%0, %2}
- vpsll<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
+ p<vshift><ssemodesuffix>\t{%2, %0|%0, %2}
+ vp<vshift><ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
[(set_attr "isa" "noavx,avx")
(set_attr "type" "sseishft")
(set (attr "length_immediate")
(define_expand "<code><mode>3"
[(set (match_operand:VI8_AVX2 0 "register_operand" "")
- (maxmin:VI8_AVX2 (match_operand:VI8_AVX2 1 "register_operand" "")
- (match_operand:VI8_AVX2 2 "register_operand" "")))]
+ (maxmin:VI8_AVX2
+ (match_operand:VI8_AVX2 1 "register_operand" "")
+ (match_operand:VI8_AVX2 2 "register_operand" "")))]
"TARGET_SSE4_2"
{
enum rtx_code code;
(define_expand "<code><mode>3"
[(set (match_operand:VI124_128 0 "register_operand" "")
- (smaxmin:VI124_128 (match_operand:VI124_128 1 "nonimmediate_operand" "")
- (match_operand:VI124_128 2 "nonimmediate_operand" "")))]
+ (smaxmin:VI124_128
+ (match_operand:VI124_128 1 "nonimmediate_operand" "")
+ (match_operand:VI124_128 2 "nonimmediate_operand" "")))]
"TARGET_SSE2"
{
if (TARGET_SSE4_1 || <MODE>mode == V8HImode)
(define_expand "<code><mode>3"
[(set (match_operand:VI124_128 0 "register_operand" "")
- (umaxmin:VI124_128 (match_operand:VI124_128 1 "nonimmediate_operand" "")
- (match_operand:VI124_128 2 "nonimmediate_operand" "")))]
+ (umaxmin:VI124_128
+ (match_operand:VI124_128 1 "nonimmediate_operand" "")
+ (match_operand:VI124_128 2 "nonimmediate_operand" "")))]
"TARGET_SSE2"
{
if (TARGET_SSE4_1 || <MODE>mode == V16QImode)
(if_then_else:V_256
(match_operator 3 ""
[(match_operand:VI_256 4 "nonimmediate_operand" "")
- (match_operand:VI_256 5 "nonimmediate_operand" "")])
- (match_operand:V_256 1 "general_operand" "")
- (match_operand:V_256 2 "general_operand" "")))]
+ (match_operand:VI_256 5 "general_operand" "")])
+ (match_operand:V_256 1 "" "")
+ (match_operand:V_256 2 "" "")))]
"TARGET_AVX2
&& (GET_MODE_NUNITS (<V_256:MODE>mode)
== GET_MODE_NUNITS (<VI_256:MODE>mode))"
(if_then_else:V_128
(match_operator 3 ""
[(match_operand:VI124_128 4 "nonimmediate_operand" "")
- (match_operand:VI124_128 5 "nonimmediate_operand" "")])
- (match_operand:V_128 1 "general_operand" "")
- (match_operand:V_128 2 "general_operand" "")))]
+ (match_operand:VI124_128 5 "general_operand" "")])
+ (match_operand:V_128 1 "" "")
+ (match_operand:V_128 2 "" "")))]
"TARGET_SSE2
&& (GET_MODE_NUNITS (<V_128:MODE>mode)
== GET_MODE_NUNITS (<VI124_128:MODE>mode))"
(if_then_else:VI8F_128
(match_operator 3 ""
[(match_operand:V2DI 4 "nonimmediate_operand" "")
- (match_operand:V2DI 5 "nonimmediate_operand" "")])
- (match_operand:VI8F_128 1 "general_operand" "")
- (match_operand:VI8F_128 2 "general_operand" "")))]
+ (match_operand:V2DI 5 "general_operand" "")])
+ (match_operand:VI8F_128 1 "" "")
+ (match_operand:VI8F_128 2 "" "")))]
"TARGET_SSE4_2"
{
bool ok = ix86_expand_int_vcond (operands);
(set_attr "prefix" "maybe_vex,orig,vex,maybe_vex,orig,orig")
(set_attr "mode" "V2SF,TI,TI,TI,V4SF,V2SF")])
-(define_insn "*vec_dupv4si_avx"
- [(set (match_operand:V4SI 0 "register_operand" "=x,x")
- (vec_duplicate:V4SI
- (match_operand:SI 1 "nonimmediate_operand" " x,m")))]
- "TARGET_AVX"
- "@
- vpshufd\t{$0, %1, %0|%0, %1, 0}
- vbroadcastss\t{%1, %0|%0, %1}"
- [(set_attr "type" "sselog1,ssemov")
- (set_attr "length_immediate" "1,0")
- (set_attr "prefix_extra" "0,1")
- (set_attr "prefix" "vex")
- (set_attr "mode" "TI,V4SF")])
-
(define_insn "*vec_dupv4si"
- [(set (match_operand:V4SI 0 "register_operand" "=x,x")
+ [(set (match_operand:V4SI 0 "register_operand" "=x,x,x")
(vec_duplicate:V4SI
- (match_operand:SI 1 "register_operand" " x,0")))]
+ (match_operand:SI 1 "nonimmediate_operand" " x,m,0")))]
"TARGET_SSE"
"@
- pshufd\t{$0, %1, %0|%0, %1, 0}
+ %vpshufd\t{$0, %1, %0|%0, %1, 0}
+ vbroadcastss\t{%1, %0|%0, %1}
shufps\t{$0, %0, %0|%0, %0, 0}"
- [(set_attr "isa" "sse2,*")
- (set_attr "type" "sselog1")
- (set_attr "length_immediate" "1")
- (set_attr "mode" "TI,V4SF")])
-
-(define_insn "*vec_dupv2di_sse3"
- [(set (match_operand:V2DI 0 "register_operand" "=x,x,x")
- (vec_duplicate:V2DI
- (match_operand:DI 1 "nonimmediate_operand" " 0,x,m")))]
- "TARGET_SSE3"
- "@
- punpcklqdq\t%0, %0
- vpunpcklqdq\t{%d1, %0|%0, %d1}
- %vmovddup\t{%1, %0|%0, %1}"
- [(set_attr "isa" "noavx,avx,*")
- (set_attr "type" "sselog1")
- (set_attr "prefix" "orig,vex,maybe_vex")
- (set_attr "mode" "TI,TI,DF")])
+ [(set_attr "isa" "sse2,avx,noavx")
+ (set_attr "type" "sselog1,ssemov,sselog1")
+ (set_attr "length_immediate" "1,0,1")
+ (set_attr "prefix_extra" "0,1,*")
+ (set_attr "prefix" "maybe_vex,vex,orig")
+ (set_attr "mode" "TI,V4SF,V4SF")])
(define_insn "*vec_dupv2di"
- [(set (match_operand:V2DI 0 "register_operand" "=x,x")
+ [(set (match_operand:V2DI 0 "register_operand" "=x,x,x,x")
(vec_duplicate:V2DI
- (match_operand:DI 1 "register_operand" " 0,0")))]
+ (match_operand:DI 1 "nonimmediate_operand" " 0,x,m,0")))]
"TARGET_SSE"
"@
punpcklqdq\t%0, %0
+ vpunpcklqdq\t{%d1, %0|%0, %d1}
+ %vmovddup\t{%1, %0|%0, %1}
movlhps\t%0, %0"
- [(set_attr "isa" "sse2,*")
- (set_attr "type" "sselog1,ssemov")
- (set_attr "mode" "TI,V4SF")])
+ [(set_attr "isa" "sse2_noavx,avx,sse3,noavx")
+ (set_attr "type" "sselog1,sselog1,sselog1,ssemov")
+ (set_attr "prefix" "orig,vex,maybe_vex,orig")
+ (set_attr "mode" "TI,TI,DF,V4SF")])
(define_insn "*vec_concatv2si_sse4_1"
[(set (match_operand:V2SI 0 "register_operand" "=x, x,x,x, x, *y,*y")
(set_attr "prefix" "maybe_vex")
(set_attr "memory" "store")])
-(define_expand "sse_sfence"
- [(set (match_dup 0)
- (unspec:BLK [(match_dup 0)] UNSPEC_SFENCE))]
- "TARGET_SSE || TARGET_3DNOW_A"
-{
- operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
- MEM_VOLATILE_P (operands[0]) = 1;
-})
-
-(define_insn "*sse_sfence"
- [(set (match_operand:BLK 0 "" "")
- (unspec:BLK [(match_dup 0)] UNSPEC_SFENCE))]
- "TARGET_SSE || TARGET_3DNOW_A"
- "sfence"
- [(set_attr "type" "sse")
- (set_attr "length_address" "0")
- (set_attr "atom_sse_attr" "fence")
- (set_attr "memory" "unknown")])
-
(define_insn "sse2_clflush"
[(unspec_volatile [(match_operand 0 "address_operand" "p")]
UNSPECV_CLFLUSH)]
(set_attr "atom_sse_attr" "fence")
(set_attr "memory" "unknown")])
-(define_expand "sse2_mfence"
- [(set (match_dup 0)
- (unspec:BLK [(match_dup 0)] UNSPEC_MFENCE))]
- "TARGET_SSE2"
-{
- operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
- MEM_VOLATILE_P (operands[0]) = 1;
-})
-
-(define_insn "*sse2_mfence"
- [(set (match_operand:BLK 0 "" "")
- (unspec:BLK [(match_dup 0)] UNSPEC_MFENCE))]
- "TARGET_64BIT || TARGET_SSE2"
- "mfence"
- [(set_attr "type" "sse")
- (set_attr "length_address" "0")
- (set_attr "atom_sse_attr" "fence")
- (set_attr "memory" "unknown")])
-
-(define_expand "sse2_lfence"
- [(set (match_dup 0)
- (unspec:BLK [(match_dup 0)] UNSPEC_LFENCE))]
- "TARGET_SSE2"
-{
- operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (Pmode));
- MEM_VOLATILE_P (operands[0]) = 1;
-})
-
-(define_insn "*sse2_lfence"
- [(set (match_operand:BLK 0 "" "")
- (unspec:BLK [(match_dup 0)] UNSPEC_LFENCE))]
- "TARGET_SSE2"
- "lfence"
- [(set_attr "type" "sse")
- (set_attr "length_address" "0")
- (set_attr "atom_sse_attr" "lfence")
- (set_attr "memory" "unknown")])
(define_insn "sse3_mwait"
[(unspec_volatile [(match_operand:SI 0 "register_operand" "a")
(set_attr "mode" "TI")])
(define_insn "<sse4_1_avx2>_pblendvb"
- [(set (match_operand:VI1_AVX2 0 "reg_not_xmm0_operand" "=x,x")
+ [(set (match_operand:VI1_AVX2 0 "reg_not_xmm0_operand_maybe_avx" "=x,x")
(unspec:VI1_AVX2
[(match_operand:VI1_AVX2 1 "reg_not_xmm0_operand_maybe_avx" "0,x")
(match_operand:VI1_AVX2 2 "nonimm_not_xmm0_operand_maybe_avx" "xm,xm")
(const_int 6)
(const_int 7)]))))]
"TARGET_SSE4_1"
- "%vpmov<extsuffix>bw\t{%1, %0|%0, %1}"
+ "%vpmov<extsuffix>bw\t{%1, %0|%0, %q1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "maybe_vex")
(const_int 6)
(const_int 7)]))))]
"TARGET_AVX2"
- "vpmov<extsuffix>bd\t{%1, %0|%0, %1}"
+ "vpmov<extsuffix>bd\t{%1, %0|%0, %q1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "vex")
(const_int 2)
(const_int 3)]))))]
"TARGET_SSE4_1"
- "%vpmov<extsuffix>bd\t{%1, %0|%0, %1}"
+ "%vpmov<extsuffix>bd\t{%1, %0|%0, %k1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "maybe_vex")
(const_int 2)
(const_int 3)]))))]
"TARGET_SSE4_1"
- "%vpmov<extsuffix>wd\t{%1, %0|%0, %1}"
+ "%vpmov<extsuffix>wd\t{%1, %0|%0, %q1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "maybe_vex")
(const_int 2)
(const_int 3)]))))]
"TARGET_AVX2"
- "vpmov<extsuffix>bq\t{%1, %0|%0, %1}"
+ "vpmov<extsuffix>bq\t{%1, %0|%0, %k1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "vex")
(parallel [(const_int 0)
(const_int 1)]))))]
"TARGET_SSE4_1"
- "%vpmov<extsuffix>bq\t{%1, %0|%0, %1}"
+ "%vpmov<extsuffix>bq\t{%1, %0|%0, %w1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "maybe_vex")
(const_int 2)
(const_int 3)]))))]
"TARGET_AVX2"
- "vpmov<extsuffix>wq\t{%1, %0|%0, %1}"
+ "vpmov<extsuffix>wq\t{%1, %0|%0, %q1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "vex")
(parallel [(const_int 0)
(const_int 1)]))))]
"TARGET_SSE4_1"
- "%vpmov<extsuffix>wq\t{%1, %0|%0, %1}"
+ "%vpmov<extsuffix>wq\t{%1, %0|%0, %k1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "maybe_vex")
(parallel [(const_int 0)
(const_int 1)]))))]
"TARGET_SSE4_1"
- "%vpmov<extsuffix>dq\t{%1, %0|%0, %1}"
+ "%vpmov<extsuffix>dq\t{%1, %0|%0, %q1}"
[(set_attr "type" "ssemov")
(set_attr "prefix_extra" "1")
(set_attr "prefix" "maybe_vex")
(set_attr "prefix" "maybe_vex")
(set_attr "mode" "<MODE>")])
+(define_expand "<sse4_1>_round<ssemodesuffix>_sfix<avxsizesuffix>"
+ [(match_operand:<sseintvecmode> 0 "register_operand" "")
+ (match_operand:VF1 1 "nonimmediate_operand" "")
+ (match_operand:SI 2 "const_0_to_15_operand" "")]
+ "TARGET_ROUND"
+{
+ rtx tmp = gen_reg_rtx (<MODE>mode);
+
+ emit_insn
+ (gen_<sse4_1>_round<ssemodesuffix><avxsizesuffix> (tmp, operands[1],
+ operands[2]));
+ emit_insn
+ (gen_fix_trunc<mode><sseintvecmodelower>2 (operands[0], tmp));
+ DONE;
+})
+
+(define_expand "<sse4_1>_round<ssemodesuffix>_vec_pack_sfix<avxsizesuffix>"
+ [(match_operand:<ssepackfltmode> 0 "register_operand" "")
+ (match_operand:VF2 1 "nonimmediate_operand" "")
+ (match_operand:VF2 2 "nonimmediate_operand" "")
+ (match_operand:SI 3 "const_0_to_15_operand" "")]
+ "TARGET_ROUND"
+{
+ rtx tmp0, tmp1;
+
+ if (<MODE>mode == V2DFmode
+ && TARGET_AVX && !TARGET_PREFER_AVX128)
+ {
+ rtx tmp2 = gen_reg_rtx (V4DFmode);
+
+ tmp0 = gen_reg_rtx (V4DFmode);
+ tmp1 = force_reg (V2DFmode, operands[1]);
+
+ emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
+ emit_insn (gen_avx_roundpd256 (tmp2, tmp0, operands[3]));
+ emit_insn (gen_fix_truncv4dfv4si2 (operands[0], tmp2));
+ }
+ else
+ {
+ tmp0 = gen_reg_rtx (<MODE>mode);
+ tmp1 = gen_reg_rtx (<MODE>mode);
+
+ emit_insn
+ (gen_<sse4_1>_round<ssemodesuffix><avxsizesuffix> (tmp0, operands[1],
+ operands[3]));
+ emit_insn
+ (gen_<sse4_1>_round<ssemodesuffix><avxsizesuffix> (tmp1, operands[2],
+ operands[3]));
+ emit_insn
+ (gen_vec_pack_sfix_trunc_<mode> (operands[0], tmp0, tmp1));
+ }
+ DONE;
+})
+
(define_insn "sse4_1_round<ssescalarmodesuffix>"
[(set (match_operand:VF_128 0 "register_operand" "=x,x")
(vec_merge:VF_128
(define_expand "round<mode>2"
[(set (match_dup 4)
(plus:VF
- (match_operand:VF 1 "nonimmediate_operand" "")
+ (match_operand:VF 1 "register_operand" "")
(match_dup 3)))
(set (match_operand:VF 0 "register_operand" "")
(unspec:VF
operands[5] = GEN_INT (ROUND_TRUNC);
})
+(define_expand "round<mode>2_sfix"
+ [(match_operand:<sseintvecmode> 0 "register_operand" "")
+ (match_operand:VF1 1 "register_operand" "")]
+ "TARGET_ROUND && !flag_trapping_math"
+{
+ rtx tmp = gen_reg_rtx (<MODE>mode);
+
+ emit_insn (gen_round<mode>2 (tmp, operands[1]));
+
+ emit_insn
+ (gen_fix_trunc<mode><sseintvecmodelower>2 (operands[0], tmp));
+ DONE;
+})
+
+(define_expand "round<mode>2_vec_pack_sfix"
+ [(match_operand:<ssepackfltmode> 0 "register_operand" "")
+ (match_operand:VF2 1 "register_operand" "")
+ (match_operand:VF2 2 "register_operand" "")]
+ "TARGET_ROUND && !flag_trapping_math"
+{
+ rtx tmp0, tmp1;
+
+ if (<MODE>mode == V2DFmode
+ && TARGET_AVX && !TARGET_PREFER_AVX128)
+ {
+ rtx tmp2 = gen_reg_rtx (V4DFmode);
+
+ tmp0 = gen_reg_rtx (V4DFmode);
+ tmp1 = force_reg (V2DFmode, operands[1]);
+
+ emit_insn (gen_avx_vec_concatv4df (tmp0, tmp1, operands[2]));
+ emit_insn (gen_roundv4df2 (tmp2, tmp0));
+ emit_insn (gen_fix_truncv4dfv4si2 (operands[0], tmp2));
+ }
+ else
+ {
+ tmp0 = gen_reg_rtx (<MODE>mode);
+ tmp1 = gen_reg_rtx (<MODE>mode);
+
+ emit_insn (gen_round<mode>2 (tmp0, operands[1]));
+ emit_insn (gen_round<mode>2 (tmp1, operands[2]));
+
+ emit_insn
+ (gen_vec_pack_sfix_trunc_<mode> (operands[0], tmp0, tmp1));
+ }
+ DONE;
+})
+
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;
;; Intel SSE4.2 string/text processing instructions
(sign_extend:V2DI
(vec_select:V2SI
(match_operand:V4SI 1 "nonimmediate_operand" "%x")
- (parallel [(const_int 1)
- (const_int 3)])))
- (vec_select:V2SI
+ (parallel [(const_int 0)
+ (const_int 2)])))
+ (vec_select:V2SI
(match_operand:V4SI 2 "nonimmediate_operand" "xm")
- (parallel [(const_int 1)
- (const_int 3)])))
+ (parallel [(const_int 0)
+ (const_int 2)])))
(match_operand:V2DI 3 "nonimmediate_operand" "x")))]
"TARGET_XOP"
"vpmacssdql\t{%3, %2, %1, %0|%0, %1, %2, %3}"
(set_attr "mode" "TI")])
(define_insn "xop_pmacssdqh"
- [(set (match_operand:V2DI 0 "register_operand" "=x")
- (ss_plus:V2DI
- (mult:V2DI
- (sign_extend:V2DI
- (vec_select:V2SI
- (match_operand:V4SI 1 "nonimmediate_operand" "%x")
- (parallel [(const_int 0)
- (const_int 2)])))
- (sign_extend:V2DI
- (vec_select:V2SI
- (match_operand:V4SI 2 "nonimmediate_operand" "xm")
- (parallel [(const_int 0)
- (const_int 2)]))))
- (match_operand:V2DI 3 "nonimmediate_operand" "x")))]
- "TARGET_XOP"
- "vpmacssdqh\t{%3, %2, %1, %0|%0, %1, %2, %3}"
- [(set_attr "type" "ssemuladd")
- (set_attr "mode" "TI")])
-
-(define_insn "xop_pmacsdql"
- [(set (match_operand:V2DI 0 "register_operand" "=x")
- (plus:V2DI
- (mult:V2DI
- (sign_extend:V2DI
- (vec_select:V2SI
- (match_operand:V4SI 1 "nonimmediate_operand" "%x")
- (parallel [(const_int 1)
- (const_int 3)])))
- (sign_extend:V2DI
- (vec_select:V2SI
- (match_operand:V4SI 2 "nonimmediate_operand" "xm")
- (parallel [(const_int 1)
- (const_int 3)]))))
- (match_operand:V2DI 3 "nonimmediate_operand" "x")))]
- "TARGET_XOP"
- "vpmacsdql\t{%3, %2, %1, %0|%0, %1, %2, %3}"
- [(set_attr "type" "ssemuladd")
- (set_attr "mode" "TI")])
-
-;; We don't have a straight 32-bit parallel multiply and extend on XOP, so
-;; fake it with a multiply/add. In general, we expect the define_split to
-;; occur before register allocation, so we have to handle the corner case where
-;; the target is the same as operands 1/2
-(define_insn_and_split "xop_mulv2div2di3_low"
- [(set (match_operand:V2DI 0 "register_operand" "=&x")
- (mult:V2DI
- (sign_extend:V2DI
- (vec_select:V2SI
- (match_operand:V4SI 1 "register_operand" "%x")
- (parallel [(const_int 1)
- (const_int 3)])))
- (sign_extend:V2DI
- (vec_select:V2SI
- (match_operand:V4SI 2 "nonimmediate_operand" "xm")
- (parallel [(const_int 1)
- (const_int 3)])))))]
- "TARGET_XOP"
- "#"
- "&& reload_completed"
- [(set (match_dup 0)
- (match_dup 3))
- (set (match_dup 0)
- (plus:V2DI
+ [(set (match_operand:V2DI 0 "register_operand" "=x")
+ (ss_plus:V2DI
(mult:V2DI
(sign_extend:V2DI
(vec_select:V2SI
- (match_dup 1)
+ (match_operand:V4SI 1 "nonimmediate_operand" "%x")
(parallel [(const_int 1)
(const_int 3)])))
(sign_extend:V2DI
(vec_select:V2SI
- (match_dup 2)
+ (match_operand:V4SI 2 "nonimmediate_operand" "xm")
(parallel [(const_int 1)
(const_int 3)]))))
- (match_dup 0)))]
-{
- operands[3] = CONST0_RTX (V2DImode);
-}
- [(set_attr "type" "ssemul")
+ (match_operand:V2DI 3 "nonimmediate_operand" "x")))]
+ "TARGET_XOP"
+ "vpmacssdqh\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+ [(set_attr "type" "ssemuladd")
(set_attr "mode" "TI")])
-(define_insn "xop_pmacsdqh"
+(define_insn "xop_pmacsdql"
[(set (match_operand:V2DI 0 "register_operand" "=x")
(plus:V2DI
(mult:V2DI
(const_int 2)]))))
(match_operand:V2DI 3 "nonimmediate_operand" "x")))]
"TARGET_XOP"
- "vpmacsdqh\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+ "vpmacsdql\t{%3, %2, %1, %0|%0, %1, %2, %3}"
[(set_attr "type" "ssemuladd")
(set_attr "mode" "TI")])
-;; We don't have a straight 32-bit parallel multiply and extend on XOP, so
-;; fake it with a multiply/add. In general, we expect the define_split to
-;; occur before register allocation, so we have to handle the corner case where
-;; the target is the same as either operands[1] or operands[2]
-(define_insn_and_split "xop_mulv2div2di3_high"
- [(set (match_operand:V2DI 0 "register_operand" "=&x")
- (mult:V2DI
- (sign_extend:V2DI
- (vec_select:V2SI
- (match_operand:V4SI 1 "register_operand" "%x")
- (parallel [(const_int 0)
- (const_int 2)])))
- (sign_extend:V2DI
- (vec_select:V2SI
- (match_operand:V4SI 2 "nonimmediate_operand" "xm")
- (parallel [(const_int 0)
- (const_int 2)])))))]
- "TARGET_XOP"
- "#"
- "&& reload_completed"
- [(set (match_dup 0)
- (match_dup 3))
- (set (match_dup 0)
+(define_insn "xop_pmacsdqh"
+ [(set (match_operand:V2DI 0 "register_operand" "=x")
(plus:V2DI
(mult:V2DI
(sign_extend:V2DI
(vec_select:V2SI
- (match_dup 1)
- (parallel [(const_int 0)
- (const_int 2)])))
+ (match_operand:V4SI 1 "nonimmediate_operand" "%x")
+ (parallel [(const_int 1)
+ (const_int 3)])))
(sign_extend:V2DI
(vec_select:V2SI
- (match_dup 2)
- (parallel [(const_int 0)
- (const_int 2)]))))
- (match_dup 0)))]
-{
- operands[3] = CONST0_RTX (V2DImode);
-}
- [(set_attr "type" "ssemul")
+ (match_operand:V4SI 2 "nonimmediate_operand" "xm")
+ (parallel [(const_int 1)
+ (const_int 3)]))))
+ (match_operand:V2DI 3 "nonimmediate_operand" "x")))]
+ "TARGET_XOP"
+ "vpmacsdqh\t{%3, %2, %1, %0|%0, %1, %2, %3}"
+ [(set_attr "type" "ssemuladd")
(set_attr "mode" "TI")])
;; XOP parallel integer multiply/add instructions for the intrinisics
(vec_select:V2QI
(match_operand:V16QI 1 "nonimmediate_operand" "xm")
(parallel [(const_int 0)
- (const_int 4)])))
+ (const_int 8)])))
(sign_extend:V2DI
(vec_select:V2QI
(match_dup 1)
(parallel [(const_int 1)
- (const_int 5)]))))
+ (const_int 9)]))))
(plus:V2DI
(sign_extend:V2DI
(vec_select:V2QI
(match_dup 1)
(parallel [(const_int 2)
- (const_int 6)])))
+ (const_int 10)])))
(sign_extend:V2DI
(vec_select:V2QI
(match_dup 1)
(parallel [(const_int 3)
- (const_int 7)])))))
+ (const_int 11)])))))
(plus:V2DI
(plus:V2DI
(sign_extend:V2DI
(vec_select:V2QI
(match_dup 1)
- (parallel [(const_int 8)
+ (parallel [(const_int 4)
(const_int 12)])))
(sign_extend:V2DI
(vec_select:V2QI
(match_dup 1)
- (parallel [(const_int 9)
+ (parallel [(const_int 5)
(const_int 13)]))))
(plus:V2DI
(sign_extend:V2DI
(vec_select:V2QI
(match_dup 1)
- (parallel [(const_int 10)
+ (parallel [(const_int 6)
(const_int 14)])))
(sign_extend:V2DI
(vec_select:V2QI
(match_dup 1)
- (parallel [(const_int 11)
+ (parallel [(const_int 7)
(const_int 15)])))))))]
"TARGET_XOP"
"vphaddbq\t{%1, %0|%0, %1}"
(vec_select:V2QI
(match_operand:V16QI 1 "nonimmediate_operand" "xm")
(parallel [(const_int 0)
- (const_int 4)])))
+ (const_int 8)])))
(sign_extend:V2DI
(vec_select:V2QI
(match_dup 1)
(parallel [(const_int 1)
- (const_int 5)]))))
+ (const_int 9)]))))
(plus:V2DI
(zero_extend:V2DI
(vec_select:V2QI
(match_dup 1)
(parallel [(const_int 2)
- (const_int 6)])))
+ (const_int 10)])))
(zero_extend:V2DI
(vec_select:V2QI
(match_dup 1)
(parallel [(const_int 3)
- (const_int 7)])))))
+ (const_int 11)])))))
(plus:V2DI
(plus:V2DI
(zero_extend:V2DI
(vec_select:V2QI
(match_dup 1)
- (parallel [(const_int 8)
+ (parallel [(const_int 4)
(const_int 12)])))
(sign_extend:V2DI
(vec_select:V2QI
(match_dup 1)
- (parallel [(const_int 9)
+ (parallel [(const_int 5)
(const_int 13)]))))
(plus:V2DI
(zero_extend:V2DI
(vec_select:V2QI
(match_dup 1)
- (parallel [(const_int 10)
+ (parallel [(const_int 6)
(const_int 14)])))
(zero_extend:V2DI
(vec_select:V2QI
(match_dup 1)
- (parallel [(const_int 11)
+ (parallel [(const_int 7)
(const_int 15)])))))))]
"TARGET_XOP"
"vphaddubq\t{%1, %0|%0, %1}"
(match_operand:SI 2 "const_0_to_<sserotatemax>_operand" "n")))]
"TARGET_XOP"
{
- operands[3] = GEN_INT ((<ssescalarnum> * 8) - INTVAL (operands[2]));
+ operands[3]
+ = GEN_INT (GET_MODE_BITSIZE (<ssescalarmode>mode) - INTVAL (operands[2]));
return \"vprot<ssemodesuffix>\t{%3, %1, %0|%0, %1, %3}\";
}
[(set_attr "type" "sseishft")
(set_attr "mode" "TI")])
;; XOP packed shift instructions.
-;; FIXME: add V2DI back in
(define_expand "vlshr<mode>3"
- [(match_operand:VI124_128 0 "register_operand" "")
- (match_operand:VI124_128 1 "register_operand" "")
- (match_operand:VI124_128 2 "register_operand" "")]
+ [(set (match_operand:VI12_128 0 "register_operand" "")
+ (lshiftrt:VI12_128
+ (match_operand:VI12_128 1 "register_operand" "")
+ (match_operand:VI12_128 2 "nonimmediate_operand" "")))]
"TARGET_XOP"
{
rtx neg = gen_reg_rtx (<MODE>mode);
emit_insn (gen_neg<mode>2 (neg, operands[2]));
- emit_insn (gen_xop_lshl<mode>3 (operands[0], operands[1], neg));
+ emit_insn (gen_xop_shl<mode>3 (operands[0], operands[1], neg));
DONE;
})
+(define_expand "vlshr<mode>3"
+ [(set (match_operand:VI48_128 0 "register_operand" "")
+ (lshiftrt:VI48_128
+ (match_operand:VI48_128 1 "register_operand" "")
+ (match_operand:VI48_128 2 "nonimmediate_operand" "")))]
+ "TARGET_AVX2 || TARGET_XOP"
+{
+ if (!TARGET_AVX2)
+ {
+ rtx neg = gen_reg_rtx (<MODE>mode);
+ emit_insn (gen_neg<mode>2 (neg, operands[2]));
+ emit_insn (gen_xop_shl<mode>3 (operands[0], operands[1], neg));
+ DONE;
+ }
+})
+
+(define_expand "vlshr<mode>3"
+ [(set (match_operand:VI48_256 0 "register_operand" "")
+ (lshiftrt:VI48_256
+ (match_operand:VI48_256 1 "register_operand" "")
+ (match_operand:VI48_256 2 "nonimmediate_operand" "")))]
+ "TARGET_AVX2")
+
(define_expand "vashr<mode>3"
- [(match_operand:VI124_128 0 "register_operand" "")
- (match_operand:VI124_128 1 "register_operand" "")
- (match_operand:VI124_128 2 "register_operand" "")]
+ [(set (match_operand:VI128_128 0 "register_operand" "")
+ (ashiftrt:VI128_128
+ (match_operand:VI128_128 1 "register_operand" "")
+ (match_operand:VI128_128 2 "nonimmediate_operand" "")))]
"TARGET_XOP"
{
rtx neg = gen_reg_rtx (<MODE>mode);
emit_insn (gen_neg<mode>2 (neg, operands[2]));
- emit_insn (gen_xop_ashl<mode>3 (operands[0], operands[1], neg));
+ emit_insn (gen_xop_sha<mode>3 (operands[0], operands[1], neg));
DONE;
})
+(define_expand "vashrv4si3"
+ [(set (match_operand:V4SI 0 "register_operand" "")
+ (ashiftrt:V4SI (match_operand:V4SI 1 "register_operand" "")
+ (match_operand:V4SI 2 "nonimmediate_operand" "")))]
+ "TARGET_AVX2 || TARGET_XOP"
+{
+ if (!TARGET_AVX2)
+ {
+ rtx neg = gen_reg_rtx (V4SImode);
+ emit_insn (gen_negv4si2 (neg, operands[2]));
+ emit_insn (gen_xop_shav4si3 (operands[0], operands[1], neg));
+ DONE;
+ }
+})
+
+(define_expand "vashrv8si3"
+ [(set (match_operand:V8SI 0 "register_operand" "")
+ (ashiftrt:V8SI (match_operand:V8SI 1 "register_operand" "")
+ (match_operand:V8SI 2 "nonimmediate_operand" "")))]
+ "TARGET_AVX2")
+
(define_expand "vashl<mode>3"
- [(match_operand:VI124_128 0 "register_operand" "")
- (match_operand:VI124_128 1 "register_operand" "")
- (match_operand:VI124_128 2 "register_operand" "")]
+ [(set (match_operand:VI12_128 0 "register_operand" "")
+ (ashift:VI12_128
+ (match_operand:VI12_128 1 "register_operand" "")
+ (match_operand:VI12_128 2 "nonimmediate_operand" "")))]
"TARGET_XOP"
{
- emit_insn (gen_xop_ashl<mode>3 (operands[0], operands[1], operands[2]));
+ emit_insn (gen_xop_sha<mode>3 (operands[0], operands[1], operands[2]));
DONE;
})
-(define_insn "xop_ashl<mode>3"
+(define_expand "vashl<mode>3"
+ [(set (match_operand:VI48_128 0 "register_operand" "")
+ (ashift:VI48_128
+ (match_operand:VI48_128 1 "register_operand" "")
+ (match_operand:VI48_128 2 "nonimmediate_operand" "")))]
+ "TARGET_AVX2 || TARGET_XOP"
+{
+ if (!TARGET_AVX2)
+ {
+ operands[2] = force_reg (<MODE>mode, operands[2]);
+ emit_insn (gen_xop_sha<mode>3 (operands[0], operands[1], operands[2]));
+ DONE;
+ }
+})
+
+(define_expand "vashl<mode>3"
+ [(set (match_operand:VI48_256 0 "register_operand" "")
+ (ashift:VI48_256
+ (match_operand:VI48_256 1 "register_operand" "")
+ (match_operand:VI48_256 2 "nonimmediate_operand" "")))]
+ "TARGET_AVX2")
+
+(define_insn "xop_sha<mode>3"
[(set (match_operand:VI_128 0 "register_operand" "=x,x")
(if_then_else:VI_128
(ge:VI_128
(set_attr "prefix_extra" "2")
(set_attr "mode" "TI")])
-(define_insn "xop_lshl<mode>3"
+(define_insn "xop_shl<mode>3"
[(set (match_operand:VI_128 0 "register_operand" "=x,x")
(if_then_else:VI_128
(ge:VI_128
(set_attr "prefix_extra" "2")
(set_attr "mode" "TI")])
-;; SSE2 doesn't have some shift varients, so define versions for XOP
+;; SSE2 doesn't have some shift variants, so define versions for XOP
(define_expand "ashlv16qi3"
- [(match_operand:V16QI 0 "register_operand" "")
- (match_operand:V16QI 1 "register_operand" "")
- (match_operand:SI 2 "nonmemory_operand" "")]
+ [(set (match_operand:V16QI 0 "register_operand" "")
+ (ashift:V16QI
+ (match_operand:V16QI 1 "register_operand" "")
+ (match_operand:SI 2 "nonmemory_operand" "")))]
"TARGET_XOP"
{
- rtvec vs = rtvec_alloc (16);
- rtx par = gen_rtx_PARALLEL (V16QImode, vs);
rtx reg = gen_reg_rtx (V16QImode);
+ rtx par;
int i;
- for (i = 0; i < 16; i++)
- RTVEC_ELT (vs, i) = operands[2];
-
- emit_insn (gen_vec_initv16qi (reg, par));
- emit_insn (gen_xop_ashlv16qi3 (operands[0], operands[1], reg));
- DONE;
-})
-(define_expand "lshlv16qi3"
- [(match_operand:V16QI 0 "register_operand" "")
- (match_operand:V16QI 1 "register_operand" "")
- (match_operand:SI 2 "nonmemory_operand" "")]
- "TARGET_XOP"
-{
- rtvec vs = rtvec_alloc (16);
- rtx par = gen_rtx_PARALLEL (V16QImode, vs);
- rtx reg = gen_reg_rtx (V16QImode);
- int i;
+ par = gen_rtx_PARALLEL (V16QImode, rtvec_alloc (16));
for (i = 0; i < 16; i++)
- RTVEC_ELT (vs, i) = operands[2];
+ XVECEXP (par, 0, i) = operands[2];
emit_insn (gen_vec_initv16qi (reg, par));
- emit_insn (gen_xop_lshlv16qi3 (operands[0], operands[1], reg));
+ emit_insn (gen_xop_shav16qi3 (operands[0], operands[1], reg));
DONE;
})
-(define_expand "ashrv16qi3"
- [(match_operand:V16QI 0 "register_operand" "")
- (match_operand:V16QI 1 "register_operand" "")
- (match_operand:SI 2 "nonmemory_operand" "")]
+(define_expand "<shift_insn>v16qi3"
+ [(set (match_operand:V16QI 0 "register_operand" "")
+ (any_shiftrt:V16QI
+ (match_operand:V16QI 1 "register_operand" "")
+ (match_operand:SI 2 "nonmemory_operand" "")))]
"TARGET_XOP"
{
- rtvec vs = rtvec_alloc (16);
- rtx par = gen_rtx_PARALLEL (V16QImode, vs);
rtx reg = gen_reg_rtx (V16QImode);
+ rtx par;
+ bool negate = false;
+ rtx (*shift_insn)(rtx, rtx, rtx);
int i;
- rtx ele = ((CONST_INT_P (operands[2]))
- ? GEN_INT (- INTVAL (operands[2]))
- : operands[2]);
+ if (CONST_INT_P (operands[2]))
+ operands[2] = GEN_INT (-INTVAL (operands[2]));
+ else
+ negate = true;
+
+ par = gen_rtx_PARALLEL (V16QImode, rtvec_alloc (16));
for (i = 0; i < 16; i++)
- RTVEC_ELT (vs, i) = ele;
+ XVECEXP (par, 0, i) = operands[2];
emit_insn (gen_vec_initv16qi (reg, par));
- if (!CONST_INT_P (operands[2]))
- {
- rtx neg = gen_reg_rtx (V16QImode);
- emit_insn (gen_negv16qi2 (neg, reg));
- emit_insn (gen_xop_ashlv16qi3 (operands[0], operands[1], neg));
- }
+ if (negate)
+ emit_insn (gen_negv16qi2 (reg, reg));
+
+ if (<CODE> == LSHIFTRT)
+ shift_insn = gen_xop_shlv16qi3;
else
- emit_insn (gen_xop_ashlv16qi3 (operands[0], operands[1], reg));
+ shift_insn = gen_xop_shav16qi3;
+ emit_insn (shift_insn (operands[0], operands[1], reg));
DONE;
})
(define_expand "ashrv2di3"
- [(match_operand:V2DI 0 "register_operand" "")
- (match_operand:V2DI 1 "register_operand" "")
- (match_operand:DI 2 "nonmemory_operand" "")]
+ [(set (match_operand:V2DI 0 "register_operand" "")
+ (ashiftrt:V2DI
+ (match_operand:V2DI 1 "register_operand" "")
+ (match_operand:DI 2 "nonmemory_operand" "")))]
"TARGET_XOP"
{
- rtvec vs = rtvec_alloc (2);
- rtx par = gen_rtx_PARALLEL (V2DImode, vs);
rtx reg = gen_reg_rtx (V2DImode);
- rtx ele;
+ rtx par;
+ bool negate = false;
+ int i;
if (CONST_INT_P (operands[2]))
- ele = GEN_INT (- INTVAL (operands[2]));
- else if (GET_MODE (operands[2]) != DImode)
- {
- rtx move = gen_reg_rtx (DImode);
- ele = gen_reg_rtx (DImode);
- convert_move (move, operands[2], false);
- emit_insn (gen_negdi2 (ele, move));
- }
+ operands[2] = GEN_INT (-INTVAL (operands[2]));
else
- {
- ele = gen_reg_rtx (DImode);
- emit_insn (gen_negdi2 (ele, operands[2]));
- }
+ negate = true;
+
+ par = gen_rtx_PARALLEL (V2DImode, rtvec_alloc (2));
+ for (i = 0; i < 2; i++)
+ XVECEXP (par, 0, i) = operands[2];
- RTVEC_ELT (vs, 0) = ele;
- RTVEC_ELT (vs, 1) = ele;
emit_insn (gen_vec_initv2di (reg, par));
- emit_insn (gen_xop_ashlv2di3 (operands[0], operands[1], reg));
+
+ if (negate)
+ emit_insn (gen_negv2di2 (reg, reg));
+
+ emit_insn (gen_xop_shav2di3 (operands[0], operands[1], reg));
DONE;
})
(define_insn "avx2_permvarv8si"
[(set (match_operand:V8SI 0 "register_operand" "=x")
(unspec:V8SI
- [(match_operand:V8SI 1 "register_operand" "x")
- (match_operand:V8SI 2 "nonimmediate_operand" "xm")]
+ [(match_operand:V8SI 1 "nonimmediate_operand" "xm")
+ (match_operand:V8SI 2 "register_operand" "x")]
UNSPEC_VPERMSI))]
"TARGET_AVX2"
- "vpermd\t{%2, %1, %0|%0, %1, %2}"
+ "vpermd\t{%1, %2, %0|%0, %2, %1}"
[(set_attr "type" "sselog")
(set_attr "prefix" "vex")
(set_attr "mode" "OI")])
(define_insn "avx2_permvarv8sf"
[(set (match_operand:V8SF 0 "register_operand" "=x")
(unspec:V8SF
- [(match_operand:V8SF 1 "register_operand" "x")
- (match_operand:V8SF 2 "nonimmediate_operand" "xm")]
+ [(match_operand:V8SF 1 "nonimmediate_operand" "xm")
+ (match_operand:V8SI 2 "register_operand" "x")]
UNSPEC_VPERMSF))]
"TARGET_AVX2"
- "vpermps\t{%2, %1, %0|%0, %1, %2}"
+ "vpermps\t{%1, %2, %0|%0, %2, %1}"
[(set_attr "type" "sselog")
(set_attr "prefix" "vex")
(set_attr "mode" "OI")])
&& avx_vperm2f128_parallel (operands[3], <MODE>mode)"
{
int mask = avx_vperm2f128_parallel (operands[3], <MODE>mode) - 1;
+ if (mask == 0x12)
+ return "vinsert<i128>\t{$0, %x2, %1, %0|%0, %1, %x2, 0}";
+ if (mask == 0x20)
+ return "vinsert<i128>\t{$1, %x2, %1, %0|%0, %1, %x2, 1}";
operands[3] = GEN_INT (mask);
return "vperm2<i128>\t{%3, %2, %1, %0|%0, %1, %2, %3}";
}
(set_attr "mode" "<sseinsnmode>")])
(define_insn "<avx_avx2>_maskstore<ssemodesuffix><avxsizesuffix>"
- [(set (match_operand:V48_AVX2 0 "memory_operand" "=m")
+ [(set (match_operand:V48_AVX2 0 "memory_operand" "+m")
(unspec:V48_AVX2
[(match_operand:<sseintvecmode> 1 "register_operand" "x")
(match_operand:V48_AVX2 2 "register_operand" "x")
DONE;
})
-(define_insn "avx2_ashrvv8si"
- [(set (match_operand:V8SI 0 "register_operand" "=x")
- (vec_concat:V8SI
- (vec_concat:V4SI
- (vec_concat:V2SI
- (ashiftrt:SI
- (vec_select:SI
- (match_operand:V8SI 1 "register_operand" "x")
- (parallel [(const_int 0)]))
- (vec_select:SI
- (match_operand:V8SI 2 "nonimmediate_operand" "xm")
- (parallel [(const_int 0)])))
- (ashiftrt:SI
- (vec_select:SI
- (match_dup 1)
- (parallel [(const_int 1)]))
- (vec_select:SI
- (match_dup 2)
- (parallel [(const_int 1)]))))
- (vec_concat:V2SI
- (ashiftrt:SI
- (vec_select:SI
- (match_dup 1)
- (parallel [(const_int 2)]))
- (vec_select:SI
- (match_dup 2)
- (parallel [(const_int 2)])))
- (ashiftrt:SI
- (vec_select:SI
- (match_dup 1)
- (parallel [(const_int 3)]))
- (vec_select:SI
- (match_dup 2)
- (parallel [(const_int 3)])))))
- (vec_concat:V4SI
- (vec_concat:V2SI
- (ashiftrt:SI
- (vec_select:SI
- (match_dup 1)
- (parallel [(const_int 0)]))
- (vec_select:SI
- (match_dup 2)
- (parallel [(const_int 0)])))
- (ashiftrt:SI
- (vec_select:SI
- (match_dup 1)
- (parallel [(const_int 1)]))
- (vec_select:SI
- (match_dup 2)
- (parallel [(const_int 1)]))))
- (vec_concat:V2SI
- (ashiftrt:SI
- (vec_select:SI
- (match_dup 1)
- (parallel [(const_int 2)]))
- (vec_select:SI
- (match_dup 2)
- (parallel [(const_int 2)])))
- (ashiftrt:SI
- (vec_select:SI
- (match_dup 1)
- (parallel [(const_int 3)]))
- (vec_select:SI
- (match_dup 2)
- (parallel [(const_int 3)])))))))]
- "TARGET_AVX2"
- "vpsravd\t{%2, %1, %0|%0, %1, %2}"
- [(set_attr "type" "sseishft")
- (set_attr "prefix" "vex")
- (set_attr "mode" "OI")])
-
-(define_insn "avx2_ashrvv4si"
- [(set (match_operand:V4SI 0 "register_operand" "=x")
- (vec_concat:V4SI
- (vec_concat:V2SI
- (ashiftrt:SI
- (vec_select:SI
- (match_operand:V4SI 1 "register_operand" "x")
- (parallel [(const_int 0)]))
- (vec_select:SI
- (match_operand:V4SI 2 "nonimmediate_operand" "xm")
- (parallel [(const_int 0)])))
- (ashiftrt:SI
- (vec_select:SI
- (match_dup 1)
- (parallel [(const_int 1)]))
- (vec_select:SI
- (match_dup 2)
- (parallel [(const_int 1)]))))
- (vec_concat:V2SI
- (ashiftrt:SI
- (vec_select:SI
- (match_dup 1)
- (parallel [(const_int 2)]))
- (vec_select:SI
- (match_dup 2)
- (parallel [(const_int 2)])))
- (ashiftrt:SI
- (vec_select:SI
- (match_dup 1)
- (parallel [(const_int 3)]))
- (vec_select:SI
- (match_dup 2)
- (parallel [(const_int 3)]))))))]
+(define_insn "avx2_ashrv<mode>"
+ [(set (match_operand:VI4_AVX2 0 "register_operand" "=x")
+ (ashiftrt:VI4_AVX2
+ (match_operand:VI4_AVX2 1 "register_operand" "x")
+ (match_operand:VI4_AVX2 2 "nonimmediate_operand" "xm")))]
"TARGET_AVX2"
"vpsravd\t{%2, %1, %0|%0, %1, %2}"
[(set_attr "type" "sseishft")
(set_attr "prefix" "vex")
- (set_attr "mode" "TI")])
-
-(define_insn "avx2_<lshift>vv8si"
- [(set (match_operand:V8SI 0 "register_operand" "=x")
- (vec_concat:V8SI
- (vec_concat:V4SI
- (vec_concat:V2SI
- (lshift:SI
- (vec_select:SI
- (match_operand:V8SI 1 "register_operand" "x")
- (parallel [(const_int 0)]))
- (vec_select:SI
- (match_operand:V8SI 2 "nonimmediate_operand" "xm")
- (parallel [(const_int 0)])))
- (lshift:SI
- (vec_select:SI
- (match_dup 1)
- (parallel [(const_int 1)]))
- (vec_select:SI
- (match_dup 2)
- (parallel [(const_int 1)]))))
- (vec_concat:V2SI
- (lshift:SI
- (vec_select:SI
- (match_dup 1)
- (parallel [(const_int 2)]))
- (vec_select:SI
- (match_dup 2)
- (parallel [(const_int 2)])))
- (lshift:SI
- (vec_select:SI
- (match_dup 1)
- (parallel [(const_int 3)]))
- (vec_select:SI
- (match_dup 2)
- (parallel [(const_int 3)])))))
- (vec_concat:V4SI
- (vec_concat:V2SI
- (lshift:SI
- (vec_select:SI
- (match_dup 1)
- (parallel [(const_int 0)]))
- (vec_select:SI
- (match_dup 2)
- (parallel [(const_int 0)])))
- (lshift:SI
- (vec_select:SI
- (match_dup 1)
- (parallel [(const_int 1)]))
- (vec_select:SI
- (match_dup 2)
- (parallel [(const_int 1)]))))
- (vec_concat:V2SI
- (lshift:SI
- (vec_select:SI
- (match_dup 1)
- (parallel [(const_int 2)]))
- (vec_select:SI
- (match_dup 2)
- (parallel [(const_int 2)])))
- (lshift:SI
- (vec_select:SI
- (match_dup 1)
- (parallel [(const_int 3)]))
- (vec_select:SI
- (match_dup 2)
- (parallel [(const_int 3)])))))))]
- "TARGET_AVX2"
- "vp<lshift_insn>vd\t{%2, %1, %0|%0, %1, %2}"
- [(set_attr "type" "sseishft")
- (set_attr "prefix" "vex")
- (set_attr "mode" "OI")])
-
-(define_insn "avx2_<lshift>v<mode>"
- [(set (match_operand:VI4SD_AVX2 0 "register_operand" "=x")
- (vec_concat:VI4SD_AVX2
- (vec_concat:<ssehalfvecmode>
- (lshift:<ssescalarmode>
- (vec_select:<ssescalarmode>
- (match_operand:VI4SD_AVX2 1 "register_operand" "x")
- (parallel [(const_int 0)]))
- (vec_select:<ssescalarmode>
- (match_operand:VI4SD_AVX2 2 "nonimmediate_operand" "xm")
- (parallel [(const_int 0)])))
- (lshift:<ssescalarmode>
- (vec_select:<ssescalarmode>
- (match_dup 1)
- (parallel [(const_int 1)]))
- (vec_select:<ssescalarmode>
- (match_dup 2)
- (parallel [(const_int 1)]))))
- (vec_concat:<ssehalfvecmode>
- (lshift:<ssescalarmode>
- (vec_select:<ssescalarmode>
- (match_dup 1)
- (parallel [(const_int 2)]))
- (vec_select:<ssescalarmode>
- (match_dup 2)
- (parallel [(const_int 2)])))
- (lshift:<ssescalarmode>
- (vec_select:<ssescalarmode>
- (match_dup 1)
- (parallel [(const_int 3)]))
- (vec_select:<ssescalarmode>
- (match_dup 2)
- (parallel [(const_int 3)]))))))]
- "TARGET_AVX2"
- "vp<lshift_insn>v<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
- [(set_attr "type" "sseishft")
- (set_attr "prefix" "vex")
(set_attr "mode" "<sseinsnmode>")])
-(define_insn "avx2_<lshift>vv2di"
- [(set (match_operand:V2DI 0 "register_operand" "=x")
- (vec_concat:V2DI
- (lshift:DI
- (vec_select:DI
- (match_operand:V2DI 1 "register_operand" "x")
- (parallel [(const_int 0)]))
- (vec_select:DI
- (match_operand:V2DI 2 "nonimmediate_operand" "xm")
- (parallel [(const_int 0)])))
- (lshift:DI
- (vec_select:DI
- (match_dup 1)
- (parallel [(const_int 1)]))
- (vec_select:DI
- (match_dup 2)
- (parallel [(const_int 1)])))))]
+(define_insn "avx2_<shift_insn>v<mode>"
+ [(set (match_operand:VI48_AVX2 0 "register_operand" "=x")
+ (any_lshift:VI48_AVX2
+ (match_operand:VI48_AVX2 1 "register_operand" "x")
+ (match_operand:VI48_AVX2 2 "nonimmediate_operand" "xm")))]
"TARGET_AVX2"
- "vp<lshift_insn>vq\t{%2, %1, %0|%0, %1, %2}"
+ "vp<vshift>v<ssemodesuffix>\t{%2, %1, %0|%0, %1, %2}"
[(set_attr "type" "sseishft")
(set_attr "prefix" "vex")
- (set_attr "mode" "TI")])
+ (set_attr "mode" "<sseinsnmode>")])
(define_insn "avx_vec_concat<mode>"
[(set (match_operand:V_256 0 "register_operand" "=x,x")
(unspec:V8SF [(match_operand:V8HI 1 "register_operand" "x")]
UNSPEC_VCVTPH2PS)
(parallel [(const_int 0) (const_int 1)
- (const_int 1) (const_int 2)])))]
+ (const_int 2) (const_int 3)])))]
"TARGET_F16C"
"vcvtph2ps\t{%1, %0|%0, %1}"
[(set_attr "type" "ssecvt")
;; For gather* insn patterns
(define_mode_iterator VEC_GATHER_MODE
[V2DI V2DF V4DI V4DF V4SI V4SF V8SI V8SF])
-(define_mode_attr VEC_GATHER_MODE
+(define_mode_attr VEC_GATHER_IDXSI
[(V2DI "V4SI") (V2DF "V4SI")
(V4DI "V4SI") (V4DF "V4SI")
(V4SI "V4SI") (V4SF "V4SI")
(V8SI "V8SI") (V8SF "V8SI")])
+(define_mode_attr VEC_GATHER_IDXDI
+ [(V2DI "V2DI") (V2DF "V2DI")
+ (V4DI "V4DI") (V4DF "V4DI")
+ (V4SI "V2DI") (V4SF "V2DI")
+ (V8SI "V4DI") (V8SF "V4DI")])
+(define_mode_attr VEC_GATHER_SRCDI
+ [(V2DI "V2DI") (V2DF "V2DF")
+ (V4DI "V4DI") (V4DF "V4DF")
+ (V4SI "V4SI") (V4SF "V4SF")
+ (V8SI "V4SI") (V8SF "V4SF")])
(define_expand "avx2_gathersi<mode>"
[(parallel [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "")
(mem:<ssescalarmode>
(match_par_dup 7
[(match_operand 2 "vsib_address_operand" "")
- (match_operand:<VEC_GATHER_MODE> 3 "register_operand" "")
+ (match_operand:<VEC_GATHER_IDXSI>
+ 3 "register_operand" "")
(match_operand:SI 5 "const1248_operand " "")]))
(mem:BLK (scratch))
(match_operand:VEC_GATHER_MODE 4 "register_operand" "")]
(match_operator:<ssescalarmode> 7 "vsib_mem_operator"
[(unspec:P
[(match_operand:P 3 "vsib_address_operand" "p")
- (match_operand:<VEC_GATHER_MODE> 4 "register_operand" "x")
+ (match_operand:<VEC_GATHER_IDXSI> 4 "register_operand" "x")
(match_operand:SI 6 "const1248_operand" "n")]
UNSPEC_VSIBADDR)])
(mem:BLK (scratch))
(set_attr "prefix" "vex")
(set_attr "mode" "<sseinsnmode>")])
+(define_insn "*avx2_gathersi<mode>_2"
+ [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
+ (unspec:VEC_GATHER_MODE
+ [(pc)
+ (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
+ [(unspec:P
+ [(match_operand:P 2 "vsib_address_operand" "p")
+ (match_operand:<VEC_GATHER_IDXSI> 3 "register_operand" "x")
+ (match_operand:SI 5 "const1248_operand" "n")]
+ UNSPEC_VSIBADDR)])
+ (mem:BLK (scratch))
+ (match_operand:VEC_GATHER_MODE 4 "register_operand" "1")]
+ UNSPEC_GATHER))
+ (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
+ "TARGET_AVX2"
+ "v<sseintprefix>gatherd<ssemodesuffix>\t{%1, %6, %0|%0, %6, %1}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "vex")
+ (set_attr "mode" "<sseinsnmode>")])
+
(define_expand "avx2_gatherdi<mode>"
[(parallel [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "")
(unspec:VEC_GATHER_MODE
- [(match_operand:VEC_GATHER_MODE 1 "register_operand" "")
+ [(match_operand:<VEC_GATHER_SRCDI> 1 "register_operand" "")
(mem:<ssescalarmode>
(match_par_dup 7
[(match_operand 2 "vsib_address_operand" "")
- (match_operand:<AVXMODE48P_DI> 3 "register_operand" "")
+ (match_operand:<VEC_GATHER_IDXDI>
+ 3 "register_operand" "")
(match_operand:SI 5 "const1248_operand " "")]))
(mem:BLK (scratch))
- (match_operand:VEC_GATHER_MODE 4 "register_operand" "")]
+ (match_operand:<VEC_GATHER_SRCDI>
+ 4 "register_operand" "")]
UNSPEC_GATHER))
(clobber (match_scratch:VEC_GATHER_MODE 6 ""))])]
"TARGET_AVX2"
})
(define_insn "*avx2_gatherdi<mode>"
- [(set (match_operand:AVXMODE48P_DI 0 "register_operand" "=&x")
- (unspec:AVXMODE48P_DI
- [(match_operand:AVXMODE48P_DI 2 "register_operand" "0")
+ [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
+ (unspec:VEC_GATHER_MODE
+ [(match_operand:<VEC_GATHER_SRCDI> 2 "register_operand" "0")
(match_operator:<ssescalarmode> 7 "vsib_mem_operator"
[(unspec:P
[(match_operand:P 3 "vsib_address_operand" "p")
- (match_operand:<AVXMODE48P_DI> 4 "register_operand" "x")
+ (match_operand:<VEC_GATHER_IDXDI> 4 "register_operand" "x")
(match_operand:SI 6 "const1248_operand" "n")]
UNSPEC_VSIBADDR)])
(mem:BLK (scratch))
- (match_operand:AVXMODE48P_DI 5 "register_operand" "1")]
+ (match_operand:<VEC_GATHER_SRCDI> 5 "register_operand" "1")]
UNSPEC_GATHER))
- (clobber (match_scratch:AVXMODE48P_DI 1 "=&x"))]
+ (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
"TARGET_AVX2"
- "v<sseintprefix>gatherq<ssemodesuffix>\t{%1, %7, %0|%0, %7, %1}"
+ "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %7, %2|%2, %7, %5}"
[(set_attr "type" "ssemov")
(set_attr "prefix" "vex")
(set_attr "mode" "<sseinsnmode>")])
-;; Special handling for VEX.256 with float arguments
-;; since there're still xmms as operands
-(define_expand "avx2_gatherdi<mode>256"
- [(parallel [(set (match_operand:VI4F_128 0 "register_operand" "")
- (unspec:VI4F_128
- [(match_operand:VI4F_128 1 "register_operand" "")
- (mem:<ssescalarmode>
- (match_par_dup 7
- [(match_operand 2 "vsib_address_operand" "")
- (match_operand:V4DI 3 "register_operand" "")
- (match_operand:SI 5 "const1248_operand " "")]))
- (mem:BLK (scratch))
- (match_operand:VI4F_128 4 "register_operand" "")]
- UNSPEC_GATHER))
- (clobber (match_scratch:VI4F_128 6 ""))])]
- "TARGET_AVX2"
-{
- operands[7]
- = gen_rtx_UNSPEC (Pmode, gen_rtvec (3, operands[2], operands[3],
- operands[5]), UNSPEC_VSIBADDR);
-})
-
-(define_insn "*avx2_gatherdi<mode>256"
- [(set (match_operand:VI4F_128 0 "register_operand" "=x")
- (unspec:VI4F_128
- [(match_operand:VI4F_128 2 "register_operand" "0")
- (match_operator:<ssescalarmode> 7 "vsib_mem_operator"
+(define_insn "*avx2_gatherdi<mode>_2"
+ [(set (match_operand:VEC_GATHER_MODE 0 "register_operand" "=&x")
+ (unspec:VEC_GATHER_MODE
+ [(pc)
+ (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
[(unspec:P
- [(match_operand:P 3 "vsib_address_operand" "p")
- (match_operand:V4DI 4 "register_operand" "x")
- (match_operand:SI 6 "const1248_operand" "n")]
+ [(match_operand:P 2 "vsib_address_operand" "p")
+ (match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "x")
+ (match_operand:SI 5 "const1248_operand" "n")]
UNSPEC_VSIBADDR)])
(mem:BLK (scratch))
- (match_operand:VI4F_128 5 "register_operand" "1")]
- UNSPEC_GATHER))
- (clobber (match_scratch:VI4F_128 1 "=&x"))]
+ (match_operand:<VEC_GATHER_SRCDI> 4 "register_operand" "1")]
+ UNSPEC_GATHER))
+ (clobber (match_scratch:VEC_GATHER_MODE 1 "=&x"))]
+ "TARGET_AVX2"
+{
+ if (<MODE>mode != <VEC_GATHER_SRCDI>mode)
+ return "v<sseintprefix>gatherq<ssemodesuffix>\t{%4, %6, %x0|%x0, %6, %4}";
+ return "v<sseintprefix>gatherq<ssemodesuffix>\t{%4, %6, %0|%0, %6, %4}";
+}
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "vex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "*avx2_gatherdi<mode>_3"
+ [(set (match_operand:<VEC_GATHER_SRCDI> 0 "register_operand" "=&x")
+ (vec_select:<VEC_GATHER_SRCDI>
+ (unspec:VI4F_256
+ [(match_operand:<VEC_GATHER_SRCDI> 2 "register_operand" "0")
+ (match_operator:<ssescalarmode> 7 "vsib_mem_operator"
+ [(unspec:P
+ [(match_operand:P 3 "vsib_address_operand" "p")
+ (match_operand:<VEC_GATHER_IDXDI> 4 "register_operand" "x")
+ (match_operand:SI 6 "const1248_operand" "n")]
+ UNSPEC_VSIBADDR)])
+ (mem:BLK (scratch))
+ (match_operand:<VEC_GATHER_SRCDI> 5 "register_operand" "1")]
+ UNSPEC_GATHER)
+ (parallel [(const_int 0) (const_int 1)
+ (const_int 2) (const_int 3)])))
+ (clobber (match_scratch:VI4F_256 1 "=&x"))]
+ "TARGET_AVX2"
+ "v<sseintprefix>gatherq<ssemodesuffix>\t{%5, %7, %0|%0, %7, %5}"
+ [(set_attr "type" "ssemov")
+ (set_attr "prefix" "vex")
+ (set_attr "mode" "<sseinsnmode>")])
+
+(define_insn "*avx2_gatherdi<mode>_4"
+ [(set (match_operand:<VEC_GATHER_SRCDI> 0 "register_operand" "=&x")
+ (vec_select:<VEC_GATHER_SRCDI>
+ (unspec:VI4F_256
+ [(pc)
+ (match_operator:<ssescalarmode> 6 "vsib_mem_operator"
+ [(unspec:P
+ [(match_operand:P 2 "vsib_address_operand" "p")
+ (match_operand:<VEC_GATHER_IDXDI> 3 "register_operand" "x")
+ (match_operand:SI 5 "const1248_operand" "n")]
+ UNSPEC_VSIBADDR)])
+ (mem:BLK (scratch))
+ (match_operand:<VEC_GATHER_SRCDI> 4 "register_operand" "1")]
+ UNSPEC_GATHER)
+ (parallel [(const_int 0) (const_int 1)
+ (const_int 2) (const_int 3)])))
+ (clobber (match_scratch:VI4F_256 1 "=&x"))]
"TARGET_AVX2"
- "v<sseintprefix>gatherq<ssemodesuffix>\t{%1, %7, %0|%0, %7, %1}"
+ "v<sseintprefix>gatherq<ssemodesuffix>\t{%4, %6, %0|%0, %6, %4}"
[(set_attr "type" "ssemov")
(set_attr "prefix" "vex")
(set_attr "mode" "<sseinsnmode>")])