X-Git-Url: http://git.sourceforge.jp/view?p=pf3gnuchains%2Fgcc-fork.git;a=blobdiff_plain;f=gcc%2Fconfig%2Fi386%2Fi386.md;h=d428c3b58d91d48ba93c211e5895796652e1f86c;hp=751d46b9ac67919dd5908138b959c3cd51330aa0;hb=f9162d6cdf59a6a8fef6467f58406634e6aa7fee;hpb=662e4881030345c1e1e37a422ca3e3af928fc931

diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
index 751d46b9ac6..d428c3b58d9 100644
--- a/gcc/config/i386/i386.md
+++ b/gcc/config/i386/i386.md
@@ -1,6 +1,6 @@
 ;; GCC machine description for IA-32 and x86-64.
 ;; Copyright (C) 1988, 1994, 1995, 1996, 1997, 1998, 1999, 2000,
-;; 2001, 2002, 2003, 2004, 2005, 2006, 2007
+;; 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008
 ;; Free Software Foundation, Inc.
 ;; Mostly by William Schelter.
 ;; x86_64 support added by Jan Hubicka
@@ -95,7 +95,6 @@
    (UNSPEC_RCP			45)
    (UNSPEC_RSQRT		46)
    (UNSPEC_SFENCE		47)
-   (UNSPEC_NOP			48)	; prevents combiner cleverness
    (UNSPEC_PFRCP		49)
    (UNSPEC_PFRCPIT1		40)
    (UNSPEC_PFRCPIT2		41)
@@ -158,7 +157,7 @@
 
    ; For SSE4A support
    (UNSPEC_EXTRQI               130)
-   (UNSPEC_EXTRQ                131)   
+   (UNSPEC_EXTRQ                131)
    (UNSPEC_INSERTQI             132)
    (UNSPEC_INSERTQ              133)
 
@@ -176,6 +175,28 @@
    (UNSPEC_CRC32		143)
    (UNSPEC_PCMPESTR		144)
    (UNSPEC_PCMPISTR		145)
+
+   ;; For SSE5
+   (UNSPEC_SSE5_INTRINSIC	150)
+   (UNSPEC_SSE5_UNSIGNED_CMP	151)
+   (UNSPEC_SSE5_TRUEFALSE	152)
+   (UNSPEC_SSE5_PERMUTE		153)
+   (UNSPEC_SSE5_ASHIFT		154)
+   (UNSPEC_SSE5_LSHIFT		155)
+   (UNSPEC_FRCZ			156)
+   (UNSPEC_CVTPH2PS		157)
+   (UNSPEC_CVTPS2PH		158)
+
+   ; For AES support
+   (UNSPEC_AESENC		159)
+   (UNSPEC_AESENCLAST		160)
+   (UNSPEC_AESDEC		161)
+   (UNSPEC_AESDECLAST		162)
+   (UNSPEC_AESIMC		163)
+   (UNSPEC_AESKEYGENASSIST	164)
+
+   ; For PCLMUL support
+   (UNSPEC_PCLMUL		165)
   ])
 
 (define_constants
@@ -196,9 +217,24 @@
    (UNSPECV_PROLOGUE_USE	14)
   ])
 
+;; Constants to represent pcomtrue/pcomfalse variants
+(define_constants
+  [(PCOM_FALSE			0)
+   (PCOM_TRUE			1)
+   (COM_FALSE_S			2)
+   (COM_FALSE_P			3)
+   (COM_TRUE_S			4)
+   (COM_TRUE_P			5)
+  ])
+
 ;; Registers by name.
 (define_constants
-  [(BP_REG			 6)
+  [(AX_REG			 0)
+   (DX_REG			 1)
+   (CX_REG			 2)
+   (SI_REG			 4)
+   (DI_REG			 5)
+   (BP_REG			 6)
    (SP_REG			 7)
    (FLAGS_REG			17)
    (FPSR_REG			18)
@@ -232,8 +268,9 @@
    push,pop,call,callv,leave,
    str,bitmanip,
    fmov,fop,fsgn,fmul,fdiv,fpspc,fcmov,fcmp,fxch,fistp,fisttp,frndint,
-   sselog,sselog1,sseiadd,sseishft,sseimul,
-   sse,ssemov,sseadd,ssemul,ssecmp,ssecomi,ssecvt,sseicvt,ssediv,sseins,
+   sselog,sselog1,sseiadd,sseiadd1,sseishft,sseimul,
+   sse,ssemov,sseadd,ssemul,ssecmp,ssecomi,ssecvt,ssecvt1,sseicvt,ssediv,sseins,
+   ssemuladd,sse4arg,
    mmx,mmxmov,mmxadd,mmxmul,mmxcmp,mmxcvt,mmxshft"
   (const_string "other"))
 
@@ -246,8 +283,9 @@
 (define_attr "unit" "integer,i387,sse,mmx,unknown"
   (cond [(eq_attr "type" "fmov,fop,fsgn,fmul,fdiv,fpspc,fcmov,fcmp,fxch,fistp,fisttp,frndint")
 	   (const_string "i387")
-	 (eq_attr "type" "sselog,sselog1,sseiadd,sseishft,sseimul,
-			  sse,ssemov,sseadd,ssemul,ssecmp,ssecomi,ssecvt,sseicvt,ssediv,sseins")
+	 (eq_attr "type" "sselog,sselog1,sseiadd,sseiadd1,sseishft,sseimul,
+			  sse,ssemov,sseadd,ssemul,ssecmp,ssecomi,ssecvt,
+			  ssecvt1,sseicvt,ssediv,sseins,ssemuladd,sse4arg")
 	   (const_string "sse")
 	 (eq_attr "type" "mmx,mmxmov,mmxadd,mmxmul,mmxcmp,mmxcvt,mmxshft")
 	   (const_string "mmx")
@@ -447,11 +485,11 @@
 		 "!alu1,negnot,ishift1,
 		   imov,imovx,icmp,test,bitmanip,
 		   fmov,fcmp,fsgn,
-		   sse,ssemov,ssecmp,ssecomi,ssecvt,sseicvt,sselog1,
-		   mmx,mmxmov,mmxcmp,mmxcvt")
+		   sse,ssemov,ssecmp,ssecomi,ssecvt,ssecvt1,sseicvt,sselog1,
+		   sseiadd1,mmx,mmxmov,mmxcmp,mmxcvt")
 	      (match_operand 2 "memory_operand" ""))
 	   (const_string "load")
-	 (and (eq_attr "type" "icmov")
+	 (and (eq_attr "type" "icmov,ssemuladd,sse4arg")
 	      (match_operand 3 "memory_operand" ""))
 	   (const_string "load")
 	]
@@ -488,7 +526,14 @@
   [(set_attr "length" "128")
    (set_attr "type" "multi")])
 
-(define_code_macro plusminus [plus minus])
+;; All integer comparison codes.
+(define_code_iterator int_cond [ne eq ge gt le lt geu gtu leu ltu ])
+
+;; All floating-point comparison codes.
+(define_code_iterator fp_cond [unordered ordered
+			       uneq unge ungt unle unlt ltgt ])
+
+(define_code_iterator plusminus [plus minus])
 
 ;; Base name for define_insn and insn mnemonic.
 (define_code_attr addsub [(plus "add") (minus "sub")])
@@ -496,8 +541,31 @@
 ;; Mark commutative operators as such in constraints.
 (define_code_attr comm [(plus "%") (minus "")])
 
+;; Mapping of signed max and min
+(define_code_iterator smaxmin [smax smin])
+
+;; Mapping of unsigned max and min
+(define_code_iterator umaxmin [umax umin])
+
+;; Base name for integer and FP insn mnemonic
+(define_code_attr maxminiprefix [(smax "maxs") (smin "mins")
+				 (umax "maxu") (umin "minu")])
+(define_code_attr maxminfprefix [(smax "max") (smin "min")])
+
+;; Mapping of parallel logic operators
+(define_code_iterator plogic [and ior xor])
+
+;; Base name for insn mnemonic.
+(define_code_attr plogicprefix [(and "and") (ior "or") (xor "xor")])
+
+;; Mapping of abs neg operators
+(define_code_iterator absneg [abs neg])
+
+;; Base name for x87 insn mnemonic.
+(define_code_attr absnegprefix [(abs "abs") (neg "chs")])
+
 ;; All single word integer modes.
-(define_mode_macro SWI [QI HI SI (DI "TARGET_64BIT")])
+(define_mode_iterator SWI [QI HI SI (DI "TARGET_64BIT")])
 
 ;; Instruction suffix for integer modes.
 (define_mode_attr imodesuffix [(QI "b") (HI "w") (SI "l") (DI "q")])
@@ -515,23 +583,20 @@
 	 (SI "general_operand")
 	 (DI "x86_64_general_operand")])
 
-;; All x87 floating point modes
-(define_mode_macro X87MODEF [SF DF XF])
+;; SSE and x87 SFmode and DFmode floating point modes
+(define_mode_iterator MODEF [SF DF])
 
-;; x87 SFmode and DFMode floating point modes
-(define_mode_macro X87MODEF12 [SF DF])
+;; All x87 floating point modes
+(define_mode_iterator X87MODEF [SF DF XF])
 
 ;; All integer modes handled by x87 fisttp operator.
-(define_mode_macro X87MODEI [HI SI DI])
+(define_mode_iterator X87MODEI [HI SI DI])
 
 ;; All integer modes handled by integer x87 operators.
-(define_mode_macro X87MODEI12 [HI SI])
-
-;; All SSE floating point modes
-(define_mode_macro SSEMODEF [SF DF])
+(define_mode_iterator X87MODEI12 [HI SI])
 
 ;; All integer modes handled by SSE cvtts?2si* operators.
-(define_mode_macro SSEMODEI24 [SI DI])
+(define_mode_iterator SSEMODEI24 [SI DI])
 
 ;; SSE asm suffix for floating point modes
 (define_mode_attr ssemodefsuffix [(SF "s") (DF "d")])
@@ -539,6 +604,9 @@
 ;; SSE vector mode corresponding to a scalar mode
 (define_mode_attr ssevecmode
   [(QI "V16QI") (HI "V8HI") (SI "V4SI") (DI "V2DI") (SF "V4SF") (DF "V2DF")])
+
+;; Instruction suffix for REX 64bit operators.
+(define_mode_attr rex64suffix [(SI "") (DI "{q}")])
 
 ;; Scheduling descriptions
 
@@ -689,8 +757,8 @@
 
 (define_expand "cmpsi_1"
   [(set (reg:CC FLAGS_REG)
-	(compare:CC (match_operand:SI 0 "nonimmediate_operand" "rm,r")
-		    (match_operand:SI 1 "general_operand" "ri,mr")))]
+	(compare:CC (match_operand:SI 0 "nonimmediate_operand" "")
+		    (match_operand:SI 1 "general_operand" "")))]
   ""
   "")
 
@@ -887,8 +955,8 @@
 
 (define_expand "cmp<mode>"
   [(set (reg:CC FLAGS_REG)
-	(compare:CC (match_operand:SSEMODEF 0 "cmp_fp_expander_operand" "")
-		    (match_operand:SSEMODEF 1 "cmp_fp_expander_operand" "")))]
+	(compare:CC (match_operand:MODEF 0 "cmp_fp_expander_operand" "")
+		    (match_operand:MODEF 1 "cmp_fp_expander_operand" "")))]
   "TARGET_80387 || (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)"
 {
   ix86_compare_op0 = operands[0];
@@ -925,6 +993,34 @@
 	   ]
 	   (const_string "XF")))])
 
+(define_insn_and_split "*cmpfp_0_cc"
+  [(set (reg:CCFP FLAGS_REG)
+	(compare:CCFP
+	  (match_operand 1 "register_operand" "f")
+	  (match_operand 2 "const0_operand" "X")))
+   (clobber (match_operand:HI 0 "register_operand" "=a"))]
+  "X87_FLOAT_MODE_P (GET_MODE (operands[1]))
+   && TARGET_SAHF && !TARGET_CMOVE
+   && GET_MODE (operands[1]) == GET_MODE (operands[2])"
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 0)
+	(unspec:HI
+	  [(compare:CCFP (match_dup 1)(match_dup 2))]
+	UNSPEC_FNSTSW))
+   (set (reg:CC FLAGS_REG)
+	(unspec:CC [(match_dup 0)] UNSPEC_SAHF))]
+  ""
+  [(set_attr "type" "multi")
+   (set_attr "unit" "i387")
+   (set (attr "mode")
+     (cond [(match_operand:SF 1 "" "")
+	      (const_string "SF")
+	    (match_operand:DF 1 "" "")
+	      (const_string "DF")
+	   ]
+	   (const_string "XF")))])
+
 (define_insn "*cmpfp_xf"
   [(set (match_operand:HI 0 "register_operand" "=a")
 	(unspec:HI
@@ -938,12 +1034,33 @@
    (set_attr "unit" "i387")
    (set_attr "mode" "XF")])
 
+(define_insn_and_split "*cmpfp_xf_cc"
+  [(set (reg:CCFP FLAGS_REG)
+	(compare:CCFP
+	  (match_operand:XF 1 "register_operand" "f")
+	  (match_operand:XF 2 "register_operand" "f")))
+   (clobber (match_operand:HI 0 "register_operand" "=a"))]
+  "TARGET_80387
+   && TARGET_SAHF && !TARGET_CMOVE"
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 0)
+	(unspec:HI
+	  [(compare:CCFP (match_dup 1)(match_dup 2))]
+	UNSPEC_FNSTSW))
+   (set (reg:CC FLAGS_REG)
+	(unspec:CC [(match_dup 0)] UNSPEC_SAHF))]
+  ""
+  [(set_attr "type" "multi")
+   (set_attr "unit" "i387")
+   (set_attr "mode" "XF")])
+
 (define_insn "*cmpfp_<mode>"
   [(set (match_operand:HI 0 "register_operand" "=a")
 	(unspec:HI
 	  [(compare:CCFP
-	     (match_operand:X87MODEF12 1 "register_operand" "f")
-	     (match_operand:X87MODEF12 2 "nonimmediate_operand" "fm"))]
+	     (match_operand:MODEF 1 "register_operand" "f")
+	     (match_operand:MODEF 2 "nonimmediate_operand" "fm"))]
 	  UNSPEC_FNSTSW))]
   "TARGET_80387"
   "* return output_fp_compare (insn, operands, 0, 0);"
@@ -951,6 +1068,27 @@
    (set_attr "unit" "i387")
    (set_attr "mode" "<MODE>")])
 
+(define_insn_and_split "*cmpfp_<mode>_cc"
+  [(set (reg:CCFP FLAGS_REG)
+	(compare:CCFP
+	  (match_operand:MODEF 1 "register_operand" "f")
+	  (match_operand:MODEF 2 "nonimmediate_operand" "fm")))
+   (clobber (match_operand:HI 0 "register_operand" "=a"))]
+  "TARGET_80387
+   && TARGET_SAHF && !TARGET_CMOVE"
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 0)
+	(unspec:HI
+	  [(compare:CCFP (match_dup 1)(match_dup 2))]
+	UNSPEC_FNSTSW))
+   (set (reg:CC FLAGS_REG)
+	(unspec:CC [(match_dup 0)] UNSPEC_SAHF))]
+  ""
+  [(set_attr "type" "multi")
+   (set_attr "unit" "i387")
+   (set_attr "mode" "<MODE>")])
+
 (define_insn "*cmpfp_u"
   [(set (match_operand:HI 0 "register_operand" "=a")
 	(unspec:HI
@@ -971,6 +1109,34 @@
 	   ]
 	   (const_string "XF")))])
 
+(define_insn_and_split "*cmpfp_u_cc"
+  [(set (reg:CCFPU FLAGS_REG)
+	(compare:CCFPU
+	  (match_operand 1 "register_operand" "f")
+	  (match_operand 2 "register_operand" "f")))
+   (clobber (match_operand:HI 0 "register_operand" "=a"))]
+  "X87_FLOAT_MODE_P (GET_MODE (operands[1]))
+   && TARGET_SAHF && !TARGET_CMOVE
+   && GET_MODE (operands[1]) == GET_MODE (operands[2])"
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 0)
+	(unspec:HI
+	  [(compare:CCFPU (match_dup 1)(match_dup 2))]
+	UNSPEC_FNSTSW))
+   (set (reg:CC FLAGS_REG)
+	(unspec:CC [(match_dup 0)] UNSPEC_SAHF))]
+  ""
+  [(set_attr "type" "multi")
+   (set_attr "unit" "i387")
+   (set (attr "mode")
+     (cond [(match_operand:SF 1 "" "")
+	      (const_string "SF")
+	    (match_operand:DF 1 "" "")
+	      (const_string "DF")
+	   ]
+	   (const_string "XF")))])
+
 (define_insn "*cmpfp_<mode>"
   [(set (match_operand:HI 0 "register_operand" "=a")
 	(unspec:HI
@@ -988,6 +1154,33 @@
    (set_attr "fp_int_src" "true")
    (set_attr "mode" "<MODE>")])
 
+(define_insn_and_split "*cmpfp_<mode>_cc"
+  [(set (reg:CCFP FLAGS_REG)
+	(compare:CCFP
+	  (match_operand 1 "register_operand" "f")
+	  (match_operator 3 "float_operator"
+	    [(match_operand:X87MODEI12 2 "memory_operand" "m")])))
+   (clobber (match_operand:HI 0 "register_operand" "=a"))]
+  "X87_FLOAT_MODE_P (GET_MODE (operands[1]))
+   && TARGET_SAHF && !TARGET_CMOVE
+   && TARGET_USE_<MODE>MODE_FIOP
+   && (GET_MODE (operands [3]) == GET_MODE (operands[1]))"
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 0)
+	(unspec:HI
+	  [(compare:CCFP
+	     (match_dup 1)
+	     (match_op_dup 3 [(match_dup 2)]))]
+	UNSPEC_FNSTSW))
+   (set (reg:CC FLAGS_REG)
+	(unspec:CC [(match_dup 0)] UNSPEC_SAHF))]
+  ""
+  [(set_attr "type" "multi")
+   (set_attr "unit" "i387")
+   (set_attr "fp_int_src" "true")
+   (set_attr "mode" "<MODE>")])
+
 ;; FP compares, step 2
 ;; Move the fpsw to ax.
 
@@ -1021,7 +1214,7 @@
    (set_attr "mode" "SI")])
 
 ;; Pentium Pro can do steps 1 through 3 in one go.
-;; comi*, ucomi*, fcomi*, ficomi*,fucomi* (i387 instructions set condition codes) 
+;; comi*, ucomi*, fcomi*, ficomi*,fucomi* (i387 instructions set condition codes)
 (define_insn "*cmpfp_i_mixed"
   [(set (reg:CCFP FLAGS_REG)
 	(compare:CCFP (match_operand 0 "register_operand" "f,x")
@@ -1060,7 +1253,7 @@
 		      (match_operand 1 "register_operand" "f")))]
   "X87_FLOAT_MODE_P (GET_MODE (operands[0]))
    && TARGET_CMOVE
-   && (!TARGET_SSE_MATH || !SSE_FLOAT_MODE_P (GET_MODE (operands[0])))
+   && !(SSE_FLOAT_MODE_P (GET_MODE (operands[0])) && TARGET_SSE_MATH)
    && GET_MODE (operands[0]) == GET_MODE (operands[1])"
   "* return output_fp_compare (insn, operands, 1, 0);"
   [(set_attr "type" "fcmp")
@@ -1112,7 +1305,7 @@
 		       (match_operand 1 "register_operand" "f")))]
   "X87_FLOAT_MODE_P (GET_MODE (operands[0]))
    && TARGET_CMOVE
-   && (!TARGET_SSE_MATH || !SSE_FLOAT_MODE_P (GET_MODE (operands[0])))
+   && !(SSE_FLOAT_MODE_P (GET_MODE (operands[0])) && TARGET_SSE_MATH)
    && GET_MODE (operands[0]) == GET_MODE (operands[1])"
   "* return output_fp_compare (insn, operands, 1, 1);"
   [(set_attr "type" "fcmp")
@@ -1331,7 +1524,7 @@
    (set_attr "mode" "SI")
    (set_attr "pent_pair" "np")
    (set_attr "athlon_decode" "vector")
-   (set_attr "amdfam10_decode" "double")])   
+   (set_attr "amdfam10_decode" "double")])
 
 (define_expand "movhi"
   [(set (match_operand:HI 0 "nonimmediate_operand" "")
@@ -1449,7 +1642,7 @@
    (set_attr "mode" "SI")
    (set_attr "pent_pair" "np")
    (set_attr "athlon_decode" "vector")
-   (set_attr "amdfam10_decode" "double")])   
+   (set_attr "amdfam10_decode" "double")])
 
 ;; Not added amdfam10_decode since TARGET_PARTIAL_REG_STALL is disabled for AMDFAM10
 (define_insn "*swaphi_2"
@@ -1625,7 +1818,7 @@
    (set_attr "mode" "SI")
    (set_attr "pent_pair" "np")
    (set_attr "athlon_decode" "vector")
-   (set_attr "amdfam10_decode" "vector")])   
+   (set_attr "amdfam10_decode" "vector")])
 
 ;; Not added amdfam10_decode since TARGET_PARTIAL_REG_STALL is disabled for AMDFAM10
 (define_insn "*swapqi_2"
@@ -2013,9 +2206,9 @@
 
 (define_insn "*movdi_2"
   [(set (match_operand:DI 0 "nonimmediate_operand"
-			"=r  ,o  ,*y,m*y,*y,*Yt,m  ,*Yt,*Yt,*x,m ,*x,*x")
+			"=r  ,o  ,*y,m*y,*y,*Y2,m  ,*Y2,*Y2,*x,m ,*x,*x")
 	(match_operand:DI 1 "general_operand"
-			"riFo,riF,C ,*y ,m ,C  ,*Yt,*Yt,m  ,C ,*x,*x,m "))]
+			"riFo,riF,C ,*y ,m ,C  ,*Y2,*Y2,m  ,C ,*x,*x,m "))]
   "!TARGET_64BIT && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
   "@
    #
@@ -2198,7 +2391,7 @@
    (set_attr "mode" "DI")
    (set_attr "pent_pair" "np")
    (set_attr "athlon_decode" "vector")
-   (set_attr "amdfam10_decode" "double")])   
+   (set_attr "amdfam10_decode" "double")])
 
 (define_expand "movti"
   [(set (match_operand:TI 0 "nonimmediate_operand" "")
@@ -2229,10 +2422,23 @@
 	return "pxor\t%0, %0";
     case 1:
     case 2:
-      if (get_attr_mode (insn) == MODE_V4SF)
-	return "movaps\t{%1, %0|%0, %1}";
+      /* TDmode values are passed as TImode on the stack.  Moving them
+	 to stack may result in unaligned memory access.  */
+      if (misaligned_operand (operands[0], TImode)
+	  || misaligned_operand (operands[1], TImode))
+	{ 
+	  if (get_attr_mode (insn) == MODE_V4SF)
+	    return "movups\t{%1, %0|%0, %1}";
+	 else
+	   return "movdqu\t{%1, %0|%0, %1}";
+	}
       else
-	return "movdqa\t{%1, %0|%0, %1}";
+	{ 
+	  if (get_attr_mode (insn) == MODE_V4SF)
+	    return "movaps\t{%1, %0|%0, %1}";
+	 else
+	   return "movdqa\t{%1, %0|%0, %1}";
+	}
     default:
       gcc_unreachable ();
     }
@@ -2266,10 +2472,23 @@
 	return "pxor\t%0, %0";
     case 3:
     case 4:
-      if (get_attr_mode (insn) == MODE_V4SF)
-	return "movaps\t{%1, %0|%0, %1}";
+      /* TDmode values are passed as TImode on the stack.  Moving them
+	 to stack may result in unaligned memory access.  */
+      if (misaligned_operand (operands[0], TImode)
+	  || misaligned_operand (operands[1], TImode))
+	{ 
+	  if (get_attr_mode (insn) == MODE_V4SF)
+	    return "movups\t{%1, %0|%0, %1}";
+	 else
+	   return "movdqu\t{%1, %0|%0, %1}";
+	}
       else
-	return "movdqa\t{%1, %0|%0, %1}";
+	{ 
+	  if (get_attr_mode (insn) == MODE_V4SF)
+	    return "movaps\t{%1, %0|%0, %1}";
+	 else
+	   return "movdqa\t{%1, %0|%0, %1}";
+	}
     default:
       gcc_unreachable ();
     }
@@ -2336,7 +2555,7 @@
    (set_attr "mode" "SF,SI,SF")])
 
 (define_insn "*pushsf_rex64"
-  [(set (match_operand:SF 0 "push_operand" "=<,<,<")
+  [(set (match_operand:SF 0 "push_operand" "=X,X,X")
 	(match_operand:SF 1 "nonmemory_no_elim_operand" "f,rF,x"))]
   "TARGET_64BIT"
 {
@@ -2486,7 +2705,7 @@
 
 (define_insn "*pushdf_nointeger"
   [(set (match_operand:DF 0 "push_operand" "=<,<,<,<")
-	(match_operand:DF 1 "general_no_elim_operand" "f,Fo,*r,Yt"))]
+	(match_operand:DF 1 "general_no_elim_operand" "f,Fo,*r,Y2"))]
   "!TARGET_64BIT && !TARGET_INTEGER_DFMODE_MOVES"
 {
   /* This insn should be already split before reg-stack.  */
@@ -2498,7 +2717,7 @@
 
 (define_insn "*pushdf_integer"
   [(set (match_operand:DF 0 "push_operand" "=<,<,<")
-	(match_operand:DF 1 "general_no_elim_operand" "f,rFo,Yt"))]
+	(match_operand:DF 1 "general_no_elim_operand" "f,rFo,Y2"))]
   "TARGET_64BIT || TARGET_INTEGER_DFMODE_MOVES"
 {
   /* This insn should be already split before reg-stack.  */
@@ -2538,17 +2757,21 @@
 
 (define_insn "*movdf_nointeger"
   [(set (match_operand:DF 0 "nonimmediate_operand"
-			"=f,m,f,*r  ,o  ,Yt*x,Yt*x,Yt*x ,m  ")
+			"=f,m,f,*r  ,o  ,Y2*x,Y2*x,Y2*x ,m  ")
 	(match_operand:DF 1 "general_operand"
-			"fm,f,G,*roF,F*r,C   ,Yt*x,mYt*x,Yt*x"))]
+			"fm,f,G,*roF,*Fr,C   ,Y2*x,mY2*x,Y2*x"))]
   "!(MEM_P (operands[0]) && MEM_P (operands[1]))
    && ((optimize_size || !TARGET_INTEGER_DFMODE_MOVES) && !TARGET_64BIT)
    && (reload_in_progress || reload_completed
        || (ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_LARGE)
        || (!(TARGET_SSE2 && TARGET_SSE_MATH) && optimize_size
+           && !memory_operand (operands[0], DFmode)
 	   && standard_80387_constant_p (operands[1]))
        || GET_CODE (operands[1]) != CONST_DOUBLE
-       || memory_operand (operands[0], DFmode))"
+       || ((optimize_size
+            || !TARGET_MEMORY_MISMATCH_STALL
+	    || reload_in_progress || reload_completed)
+ 	   && memory_operand (operands[0], DFmode)))"
 {
   switch (which_alternative)
     {
@@ -2655,9 +2878,9 @@
 
 (define_insn "*movdf_integer_rex64"
   [(set (match_operand:DF 0 "nonimmediate_operand"
-		"=f,m,f,r  ,m ,Yt*x,Yt*x,Yt*x,m   ,Yi,r ")
+		"=f,m,f,r  ,m ,Y2*x,Y2*x,Y2*x,m   ,Yi,r ")
 	(match_operand:DF 1 "general_operand"
-		"fm,f,G,rmF,Fr,C   ,Yt*x,m   ,Yt*x,r ,Yi"))]
+		"fm,f,G,rmF,Fr,C   ,Y2*x,m   ,Y2*x,r ,Yi"))]
   "TARGET_64BIT && !(MEM_P (operands[0]) && MEM_P (operands[1]))
    && (reload_in_progress || reload_completed
        || (ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_LARGE)
@@ -2776,9 +2999,9 @@
 
 (define_insn "*movdf_integer"
   [(set (match_operand:DF 0 "nonimmediate_operand"
-		"=f,m,f,r  ,o ,Yt*x,Yt*x,Yt*x,m   ")
+		"=f,m,f,r  ,o ,Y2*x,Y2*x,Y2*x,m   ")
 	(match_operand:DF 1 "general_operand"
-		"fm,f,G,roF,Fr,C   ,Yt*x,m   ,Yt*x"))]
+		"fm,f,G,roF,Fr,C   ,Y2*x,m   ,Y2*x"))]
   "!(MEM_P (operands[0]) && MEM_P (operands[1]))
    && !optimize_size && TARGET_INTEGER_DFMODE_MOVES
    && (reload_in_progress || reload_completed
@@ -3289,7 +3512,7 @@
   [(set (match_operand:HI 0 "register_operand" "=r")
      (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "qm")))]
   "(!TARGET_ZERO_EXTEND_WITH_AND || optimize_size) && reload_completed"
-  "movz{bl|x}\t{%1, %k0|%k0, %k1}"
+  "movz{bl|x}\t{%1, %k0|%k0, %1}"
   [(set_attr "type" "imovx")
    (set_attr "mode" "SI")])
 
@@ -3402,8 +3625,8 @@
 
 ;; %%% Kill me once multi-word ops are sane.
 (define_expand "zero_extendsidi2"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-     (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "rm")))]
+  [(set (match_operand:DI 0 "register_operand" "")
+     (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "")))]
   ""
 {
   if (!TARGET_64BIT)
@@ -3414,7 +3637,7 @@
 })
 
 (define_insn "zero_extendsidi2_32"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "=r,?r,?o,?*Ym,?*y,?*Yi,*Yt")
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=r,?r,?o,?*Ym,?*y,?*Yi,*Y2")
 	(zero_extend:DI
 	 (match_operand:SI 1 "nonimmediate_operand" "0,rm,r ,r   ,m  ,r   ,m")))
    (clobber (reg:CC FLAGS_REG))]
@@ -3431,7 +3654,7 @@
    (set_attr "type" "multi,multi,multi,mmxmov,mmxmov,ssemov,ssemov")])
 
 (define_insn "zero_extendsidi2_rex64"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "=r,o,?*Ym,?*y,?*Yi,*Yt")
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=r,o,?*Ym,?*y,?*Yi,*Y2")
      (zero_extend:DI
        (match_operand:SI 1 "nonimmediate_operand"  "rm,0,r   ,m  ,r   ,m")))]
   "TARGET_64BIT"
@@ -3569,8 +3792,8 @@
 
   /* Generate a cltd if possible and doing so it profitable.  */
   if ((optimize_size || TARGET_USE_CLTD)
-      && true_regnum (operands[1]) == 0
-      && true_regnum (operands[2]) == 1)
+      && true_regnum (operands[1]) == AX_REG
+      && true_regnum (operands[2]) == DX_REG)
     {
       emit_insn (gen_ashrsi3_31 (operands[2], operands[1], GEN_INT (31)));
     }
@@ -3600,7 +3823,7 @@
 
   /* Generate a cltd if possible and doing so it profitable.  */
   if ((optimize_size || TARGET_USE_CLTD)
-      && true_regnum (operands[3]) == 0)
+      && true_regnum (operands[3]) == AX_REG)
     {
       emit_insn (gen_ashrsi3_31 (operands[4], operands[3], GEN_INT (31)));
       DONE;
@@ -3717,7 +3940,7 @@
 ;; %%% Kill these when call knows how to work out a DFmode push earlier.
 (define_insn "*dummy_extendsfdf2"
   [(set (match_operand:DF 0 "push_operand" "=<")
-	(float_extend:DF (match_operand:SF 1 "nonimmediate_operand" "fY")))]
+	(float_extend:DF (match_operand:SF 1 "nonimmediate_operand" "fY2")))]
   "0"
   "#")
 
@@ -3794,6 +4017,49 @@
     }
 })
 
+/* For converting SF(xmm2) to DF(xmm1), use the following code instead of
+   cvtss2sd:
+      unpcklps xmm2,xmm2   ; packed conversion might crash on signaling NaNs
+      cvtps2pd xmm2,xmm1
+   We do the conversion post reload to avoid producing of 128bit spills
+   that might lead to ICE on 32bit target.  The sequence unlikely combine
+   anyway.  */
+(define_split
+  [(set (match_operand:DF 0 "register_operand" "")
+        (float_extend:DF
+	  (match_operand:SF 1 "nonimmediate_operand" "")))]
+  "(TARGET_USE_VECTOR_CONVERTS || TARGET_GENERIC) && !optimize_size
+   && reload_completed && SSE_REG_P (operands[0])"
+   [(set (match_dup 2)
+	 (float_extend:V2DF
+	   (vec_select:V2SF
+	     (match_dup 3)
+	     (parallel [(const_int 0) (const_int 1)]))))]
+{
+  operands[2] = simplify_gen_subreg (V2DFmode, operands[0], DFmode, 0);
+  operands[3] = simplify_gen_subreg (V4SFmode, operands[0], DFmode, 0);
+  /* Use movss for loading from memory, unpcklps reg, reg for registers.
+     Try to avoid move when unpacking can be done in source.  */
+  if (REG_P (operands[1]))
+    {
+      /* If it is unsafe to overwrite upper half of source, we need
+	 to move to destination and unpack there.  */
+      if ((ORIGINAL_REGNO (operands[1]) < FIRST_PSEUDO_REGISTER
+	   || PSEUDO_REGNO_BYTES (ORIGINAL_REGNO (operands[1])) > 4)
+	  && true_regnum (operands[0]) != true_regnum (operands[1]))
+	{
+	  rtx tmp = gen_rtx_REG (SFmode, true_regnum (operands[0]));
+	  emit_move_insn (tmp, operands[1]);
+	}
+      else
+	operands[3] = simplify_gen_subreg (V4SFmode, operands[1], SFmode, 0);
+      emit_insn (gen_sse_unpcklps (operands[3], operands[3], operands[3]));
+    }
+  else
+    emit_insn (gen_vec_setv4sf_0 (operands[3],
+				  CONST0_RTX (V4SFmode), operands[1]));
+})
+
 (define_insn "*extendsfdf2_mixed"
   [(set (match_operand:DF 0 "nonimmediate_operand" "=f,m,x")
         (float_extend:DF
@@ -3834,7 +4100,7 @@
 
 (define_expand "extend<mode>xf2"
   [(set (match_operand:XF 0 "nonimmediate_operand" "")
-        (float_extend:XF (match_operand:X87MODEF12 1 "general_operand" "")))]
+        (float_extend:XF (match_operand:MODEF 1 "general_operand" "")))]
   "TARGET_80387"
 {
   /* ??? Needed for compress_float_constant since all fp constants
@@ -3855,7 +4121,7 @@
 (define_insn "*extend<mode>xf2_i387"
   [(set (match_operand:XF 0 "nonimmediate_operand" "=f,m")
         (float_extend:XF
-	  (match_operand:X87MODEF12 1 "nonimmediate_operand" "fm,f")))]
+	  (match_operand:MODEF 1 "nonimmediate_operand" "fm,f")))]
   "TARGET_80387"
   "* return output_387_reg_move (insn, operands);"
   [(set_attr "type" "fmov")
@@ -3881,12 +4147,58 @@
     ;
   else
     {
-      rtx temp = assign_386_stack_local (SFmode, SLOT_VIRTUAL);
+      int slot = virtuals_instantiated ? SLOT_TEMP : SLOT_VIRTUAL;
+      rtx temp = assign_386_stack_local (SFmode, slot);
       emit_insn (gen_truncdfsf2_with_temp (operands[0], operands[1], temp));
       DONE;
     }
 })
 
+/* For converting DF(xmm2) to SF(xmm1), use the following code instead of
+   cvtsd2ss:
+      unpcklpd xmm2,xmm2   ; packed conversion might crash on signaling NaNs
+      cvtpd2ps xmm2,xmm1
+   We do the conversion post reload to avoid producing of 128bit spills
+   that might lead to ICE on 32bit target.  The sequence unlikely combine
+   anyway.  */
+(define_split
+  [(set (match_operand:SF 0 "register_operand" "")
+        (float_truncate:SF
+	  (match_operand:DF 1 "nonimmediate_operand" "")))]
+  "(TARGET_USE_VECTOR_CONVERTS || TARGET_GENERIC) && !optimize_size
+   && reload_completed && SSE_REG_P (operands[0])"
+   [(set (match_dup 2)
+	 (vec_concat:V4SF
+	   (float_truncate:V2SF
+	     (match_dup 4))
+	   (match_dup 3)))]
+{
+  operands[2] = simplify_gen_subreg (V4SFmode, operands[0], SFmode, 0);
+  operands[3] = CONST0_RTX (V2SFmode);
+  operands[4] = simplify_gen_subreg (V2DFmode, operands[0], SFmode, 0);
+  /* Use movsd for loading from memory, unpcklpd for registers.
+     Try to avoid move when unpacking can be done in source, or SSE3
+     movddup is available.  */
+  if (REG_P (operands[1]))
+    {
+      if (!TARGET_SSE3
+	  && true_regnum (operands[0]) != true_regnum (operands[1])
+	  && (ORIGINAL_REGNO (operands[1]) < FIRST_PSEUDO_REGISTER
+	      || PSEUDO_REGNO_BYTES (ORIGINAL_REGNO (operands[1])) > 8))
+	{
+	  rtx tmp = simplify_gen_subreg (DFmode, operands[0], SFmode, 0);
+	  emit_move_insn (tmp, operands[1]);
+	  operands[1] = tmp;
+	}
+      else if (!TARGET_SSE3)
+	operands[4] = simplify_gen_subreg (V2DFmode, operands[1], DFmode, 0);
+      emit_insn (gen_vec_dupv2df (operands[4], operands[1]));
+    }
+  else
+    emit_insn (gen_sse2_loadlpd (operands[4],
+				 CONST0_RTX (V2DFmode), operands[1]));
+})
+
 (define_expand "truncdfsf2_with_temp"
   [(parallel [(set (match_operand:SF 0 "" "")
 		   (float_truncate:SF (match_operand:DF 1 "" "")))
@@ -3934,9 +4246,9 @@
    (set_attr "mode" "SF")])
 
 (define_insn "*truncdfsf_mixed"
-  [(set (match_operand:SF 0 "nonimmediate_operand"   "=m,?fx*r,Yt")
+  [(set (match_operand:SF 0 "nonimmediate_operand"   "=m,?fx*r,Y2")
 	(float_truncate:SF
-	  (match_operand:DF 1 "nonimmediate_operand" "f ,f    ,Ytm")))
+	  (match_operand:DF 1 "nonimmediate_operand" "f ,f    ,Y2m")))
    (clobber (match_operand:SF 2 "memory_operand"     "=X,m    ,X"))]
   "TARGET_MIX_SSE_I387"
 {
@@ -4005,8 +4317,8 @@
 ;; Conversion from XFmode to {SF,DF}mode
 
 (define_expand "truncxf<mode>2"
-  [(parallel [(set (match_operand:X87MODEF12 0 "nonimmediate_operand" "")
-		   (float_truncate:X87MODEF12
+  [(parallel [(set (match_operand:MODEF 0 "nonimmediate_operand" "")
+		   (float_truncate:MODEF
 		     (match_operand:XF 1 "register_operand" "")))
 	      (clobber (match_dup 2))])]
   "TARGET_80387"
@@ -4020,7 +4332,10 @@
       DONE;
     }
   else
-    operands[2] = assign_386_stack_local (<MODE>mode, SLOT_VIRTUAL);
+    {
+      int slot = virtuals_instantiated ? SLOT_TEMP : SLOT_VIRTUAL;
+      operands[2] = assign_386_stack_local (<MODE>mode, slot);
+    }
 })
 
 (define_insn "*truncxfsf2_mixed"
@@ -4038,7 +4353,7 @@
    (set_attr "mode" "SF")])
 
 (define_insn "*truncxfdf2_mixed"
-  [(set (match_operand:DF 0 "nonimmediate_operand" "=m,?fYt*r")
+  [(set (match_operand:DF 0 "nonimmediate_operand" "=m,?fY2*r")
 	(float_truncate:DF
 	  (match_operand:XF 1 "register_operand" "f,f")))
    (clobber (match_operand:DF 2 "memory_operand" "=X,m"))]
@@ -4052,8 +4367,8 @@
    (set_attr "mode" "DF")])
 
 (define_insn "truncxf<mode>2_i387_noop"
-  [(set (match_operand:X87MODEF12 0 "register_operand" "=f")
-	(float_truncate:X87MODEF12
+  [(set (match_operand:MODEF 0 "register_operand" "=f")
+	(float_truncate:MODEF
 	  (match_operand:XF 1 "register_operand" "f")))]
   "TARGET_80387 && flag_unsafe_math_optimizations"
   "* return output_387_reg_move (insn, operands);"
@@ -4061,8 +4376,8 @@
    (set_attr "mode" "<MODE>")])
 
 (define_insn "*truncxf<mode>2_i387"
-  [(set (match_operand:X87MODEF12 0 "memory_operand" "=m")
-	(float_truncate:X87MODEF12
+  [(set (match_operand:MODEF 0 "memory_operand" "=m")
+	(float_truncate:MODEF
 	  (match_operand:XF 1 "register_operand" "f")))]
   "TARGET_80387"
   "* return output_387_reg_move (insn, operands);"
@@ -4070,22 +4385,22 @@
    (set_attr "mode" "<MODE>")])
 
 (define_split
-  [(set (match_operand:X87MODEF12 0 "register_operand" "")
-	(float_truncate:X87MODEF12
+  [(set (match_operand:MODEF 0 "register_operand" "")
+	(float_truncate:MODEF
 	  (match_operand:XF 1 "register_operand" "")))
-   (clobber (match_operand:X87MODEF12 2 "memory_operand" ""))]
+   (clobber (match_operand:MODEF 2 "memory_operand" ""))]
   "TARGET_80387 && reload_completed"
-  [(set (match_dup 2) (float_truncate:X87MODEF12 (match_dup 1)))
+  [(set (match_dup 2) (float_truncate:MODEF (match_dup 1)))
    (set (match_dup 0) (match_dup 2))]
   "")
 
 (define_split
-  [(set (match_operand:X87MODEF12 0 "memory_operand" "")
-	(float_truncate:X87MODEF12
+  [(set (match_operand:MODEF 0 "memory_operand" "")
+	(float_truncate:MODEF
 	  (match_operand:XF 1 "register_operand" "")))
-   (clobber (match_operand:X87MODEF12 2 "memory_operand" ""))]
+   (clobber (match_operand:MODEF 2 "memory_operand" ""))]
   "TARGET_80387"
-  [(set (match_dup 0) (float_truncate:X87MODEF12 (match_dup 1)))]
+  [(set (match_dup 0) (float_truncate:MODEF (match_dup 1)))]
   "")
 
 ;; Signed conversion to DImode.
@@ -4105,7 +4420,7 @@
 
 (define_expand "fix_trunc<mode>di2"
   [(parallel [(set (match_operand:DI 0 "nonimmediate_operand" "")
-                   (fix:DI (match_operand:SSEMODEF 1 "register_operand" "")))
+                   (fix:DI (match_operand:MODEF 1 "register_operand" "")))
               (clobber (reg:CC FLAGS_REG))])]
   "TARGET_80387 || (TARGET_64BIT && SSE_FLOAT_MODE_P (<MODE>mode))"
 {
@@ -4142,7 +4457,7 @@
 
 (define_expand "fix_trunc<mode>si2"
   [(parallel [(set (match_operand:SI 0 "nonimmediate_operand" "")
-	           (fix:SI (match_operand:SSEMODEF 1 "register_operand" "")))
+	           (fix:SI (match_operand:MODEF 1 "register_operand" "")))
 	      (clobber (reg:CC FLAGS_REG))])]
   "TARGET_80387 || SSE_FLOAT_MODE_P (<MODE>mode)"
 {
@@ -4184,7 +4499,7 @@
   [(parallel
     [(set (match_operand:SI 0 "register_operand" "")
 	  (unsigned_fix:SI
-	    (match_operand:SSEMODEF 1 "nonimmediate_operand" "")))
+	    (match_operand:MODEF 1 "nonimmediate_operand" "")))
      (use (match_dup 2))
      (clobber (match_scratch:<ssevecmode> 3 ""))
      (clobber (match_scratch:<ssevecmode> 4 ""))])]
@@ -4204,7 +4519,7 @@
 (define_insn_and_split "*fixuns_trunc<mode>_1"
   [(set (match_operand:SI 0 "register_operand" "=&x,&x")
 	(unsigned_fix:SI
-	  (match_operand:SSEMODEF 3 "nonimmediate_operand" "xm,xm")))
+	  (match_operand:MODEF 3 "nonimmediate_operand" "xm,xm")))
    (use (match_operand:<ssevecmode> 4  "nonimmediate_operand" "m,x"))
    (clobber (match_scratch:<ssevecmode> 1 "=x,&x"))
    (clobber (match_scratch:<ssevecmode> 2 "=x,x"))]
@@ -4223,7 +4538,7 @@
 
 (define_expand "fixuns_trunc<mode>hi2"
   [(set (match_dup 2)
-	(fix:SI (match_operand:SSEMODEF 1 "nonimmediate_operand" "")))
+	(fix:SI (match_operand:MODEF 1 "nonimmediate_operand" "")))
    (set (match_operand:HI 0 "nonimmediate_operand" "")
 	(subreg:HI (match_dup 2) 0))]
   "SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH"
@@ -4232,7 +4547,7 @@
 ;; When SSE is available, it is always faster to use it!
 (define_insn "fix_trunc<mode>di_sse"
   [(set (match_operand:DI 0 "register_operand" "=r,r")
-	(fix:DI (match_operand:SSEMODEF 1 "nonimmediate_operand" "x,m")))]
+	(fix:DI (match_operand:MODEF 1 "nonimmediate_operand" "x,m")))]
   "TARGET_64BIT && SSE_FLOAT_MODE_P (<MODE>mode)
    && (!TARGET_FISTTP || TARGET_SSE_MATH)"
   "cvtts<ssemodefsuffix>2si{q}\t{%1, %0|%0, %1}"
@@ -4243,7 +4558,7 @@
 
 (define_insn "fix_trunc<mode>si_sse"
   [(set (match_operand:SI 0 "register_operand" "=r,r")
-	(fix:SI (match_operand:SSEMODEF 1 "nonimmediate_operand" "x,m")))]
+	(fix:SI (match_operand:MODEF 1 "nonimmediate_operand" "x,m")))]
   "SSE_FLOAT_MODE_P (<MODE>mode)
    && (!TARGET_FISTTP || TARGET_SSE_MATH)"
   "cvtts<ssemodefsuffix>2si\t{%1, %0|%0, %1}"
@@ -4254,8 +4569,8 @@
 
 ;; Shorten x87->SSE reload sequences of fix_trunc?f?i_sse patterns.
 (define_peephole2
-  [(set (match_operand:SSEMODEF 0 "register_operand" "")
-	(match_operand:SSEMODEF 1 "memory_operand" ""))
+  [(set (match_operand:MODEF 0 "register_operand" "")
+	(match_operand:MODEF 1 "memory_operand" ""))
    (set (match_operand:SSEMODEI24 2 "register_operand" "")
 	(fix:SSEMODEI24 (match_dup 0)))]
   "TARGET_SHORTEN_X87_SSE
@@ -4265,7 +4580,7 @@
 
 ;; Avoid vector decoded forms of the instruction.
 (define_peephole2
-  [(match_scratch:DF 2 "Yt")
+  [(match_scratch:DF 2 "Y2")
    (set (match_operand:SSEMODEI24 0 "register_operand" "")
 	(fix:SSEMODEI24 (match_operand:DF 1 "memory_operand" "")))]
   "TARGET_AVOID_VECTOR_DECODE && !optimize_size"
@@ -4283,8 +4598,8 @@
   "")
 
 (define_insn_and_split "fix_trunc<mode>_fisttp_i387_1"
-  [(set (match_operand:X87MODEI 0 "nonimmediate_operand" "=m,?r")
-	(fix:X87MODEI (match_operand 1 "register_operand" "f,f")))]
+  [(set (match_operand:X87MODEI 0 "nonimmediate_operand" "")
+	(fix:X87MODEI (match_operand 1 "register_operand" "")))]
   "X87_FLOAT_MODE_P (GET_MODE (operands[1]))
    && TARGET_FISTTP
    && !((SSE_FLOAT_MODE_P (GET_MODE (operands[1]))
@@ -4363,8 +4678,8 @@
 ;; clobbering insns can be used. Look at emit_i387_cw_initialization ()
 ;; function in i386.c.
 (define_insn_and_split "*fix_trunc<mode>_i387_1"
-  [(set (match_operand:X87MODEI 0 "nonimmediate_operand" "=m,?r")
-	(fix:X87MODEI (match_operand 1 "register_operand" "f,f")))
+  [(set (match_operand:X87MODEI 0 "nonimmediate_operand" "")
+	(fix:X87MODEI (match_operand 1 "register_operand" "")))
    (clobber (reg:CC FLAGS_REG))]
   "X87_FLOAT_MODE_P (GET_MODE (operands[1]))
    && !TARGET_FISTTP
@@ -4523,7 +4838,7 @@
    (set_attr "mode" "HI")
    (set_attr "unit" "i387")
    (set_attr "athlon_decode" "vector")
-   (set_attr "amdfam10_decode" "vector")])   
+   (set_attr "amdfam10_decode" "vector")])
 
 ;; Conversion between fixed point and floating point.
 
@@ -4531,237 +4846,629 @@
 ;; wants to be able to do this between registers.
 
 (define_expand "floathi<mode>2"
-  [(set (match_operand:SSEMODEF 0 "register_operand" "")
-	(float:SSEMODEF (match_operand:HI 1 "nonimmediate_operand" "")))]
-  "TARGET_80387 || (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)"
-{
-  if (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
-    {
-      emit_insn
-	(gen_floatsi<mode>2 (operands[0],
-			     convert_to_mode (SImode, operands[1], 0)));
-      DONE;
-    }
-})
+  [(set (match_operand:X87MODEF 0 "register_operand" "")
+	(float:X87MODEF (match_operand:HI 1 "nonimmediate_operand" "")))]
+  "TARGET_80387
+   && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
+       || TARGET_MIX_SSE_I387)"
+  "")
 
-(define_insn "*floathi<mode>2_i387"
-  [(set (match_operand:X87MODEF12 0 "register_operand" "=f,f")
-	(float:X87MODEF12
-	  (match_operand:HI 1 "nonimmediate_operand" "m,?r")))]
+;; Pre-reload splitter to add memory clobber to the pattern.
+(define_insn_and_split "*floathi<mode>2_1"
+  [(set (match_operand:X87MODEF 0 "register_operand" "")
+	(float:X87MODEF (match_operand:HI 1 "register_operand" "")))]
+  "TARGET_80387
+   && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
+       || TARGET_MIX_SSE_I387)
+   && !(reload_completed || reload_in_progress)"
+  "#"
+  "&& 1"
+  [(parallel [(set (match_dup 0)
+	      (float:X87MODEF (match_dup 1)))
+   (clobber (match_dup 2))])]
+  "operands[2] = assign_386_stack_local (HImode, SLOT_TEMP);")
+
+(define_insn "*floathi<mode>2_i387_with_temp"
+  [(set (match_operand:X87MODEF 0 "register_operand" "=f,f")
+	(float:X87MODEF (match_operand:HI 1 "nonimmediate_operand" "m,?r")))
+  (clobber (match_operand:HI 2 "memory_operand" "=m,m"))]
   "TARGET_80387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)"
-  "@
-   fild%z1\t%1
-   #"
+  "#"
   [(set_attr "type" "fmov,multi")
    (set_attr "mode" "<MODE>")
    (set_attr "unit" "*,i387")
    (set_attr "fp_int_src" "true")])
 
-(define_expand "floatsi<mode>2"
-  [(set (match_operand:SSEMODEF 0 "register_operand" "")
-	(float:SSEMODEF (match_operand:SI 1 "nonimmediate_operand" "")))]
-  "TARGET_80387 || (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)"
+(define_insn "*floathi<mode>2_i387"
+  [(set (match_operand:X87MODEF 0 "register_operand" "=f")
+	(float:X87MODEF (match_operand:HI 1 "memory_operand" "m")))]
+  "TARGET_80387
+   && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
+       || TARGET_MIX_SSE_I387)"
+  "fild%z1\t%1"
+  [(set_attr "type" "fmov")
+   (set_attr "mode" "<MODE>")
+   (set_attr "fp_int_src" "true")])
+
+(define_split
+  [(set (match_operand:X87MODEF 0 "register_operand" "")
+	(float:X87MODEF (match_operand:HI 1 "register_operand" "")))
+   (clobber (match_operand:HI 2 "memory_operand" ""))]
+  "TARGET_80387
+   && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
+       || TARGET_MIX_SSE_I387)
+   && reload_completed"
+  [(set (match_dup 2) (match_dup 1))
+   (set (match_dup 0) (float:X87MODEF (match_dup 2)))]
   "")
 
-(define_insn "*floatsisf2_mixed"
-  [(set (match_operand:SF 0 "register_operand" "=f,?f,x,x")
-	(float:SF (match_operand:SI 1 "nonimmediate_operand" "m,r,r,m")))]
-  "TARGET_MIX_SSE_I387"
+(define_split
+  [(set (match_operand:X87MODEF 0 "register_operand" "")
+	(float:X87MODEF (match_operand:HI 1 "memory_operand" "")))
+   (clobber (match_operand:HI 2 "memory_operand" ""))]
+   "TARGET_80387
+    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
+        || TARGET_MIX_SSE_I387)
+    && reload_completed"
+  [(set (match_dup 0) (float:X87MODEF (match_dup 1)))]
+  "")
+
+(define_expand "float<SSEMODEI24:mode><X87MODEF:mode>2"
+  [(set (match_operand:X87MODEF 0 "register_operand" "")
+	(float:X87MODEF
+	  (match_operand:SSEMODEI24 1 "nonimmediate_operand" "")))]
+  "TARGET_80387
+   || ((<SSEMODEI24:MODE>mode != DImode || TARGET_64BIT)
+       && SSE_FLOAT_MODE_P (<X87MODEF:MODE>mode) && TARGET_SSE_MATH)"
+  "")
+
+;; Pre-reload splitter to add memory clobber to the pattern.
+(define_insn_and_split "*float<SSEMODEI24:mode><X87MODEF:mode>2_1"
+  [(set (match_operand:X87MODEF 0 "register_operand" "")
+	(float:X87MODEF (match_operand:SSEMODEI24 1 "register_operand" "")))]
+  "((TARGET_80387
+     && (!((<SSEMODEI24:MODE>mode != DImode || TARGET_64BIT)
+	   && SSE_FLOAT_MODE_P (<X87MODEF:MODE>mode) && TARGET_SSE_MATH)
+	 || TARGET_MIX_SSE_I387))
+    || ((<SSEMODEI24:MODE>mode != DImode || TARGET_64BIT)
+	&& SSE_FLOAT_MODE_P (<X87MODEF:MODE>mode) && TARGET_SSE_MATH
+	&& ((<SSEMODEI24:MODE>mode == SImode
+	     && TARGET_SSE2 && TARGET_USE_VECTOR_CONVERTS && !optimize_size
+	     && flag_trapping_math)
+	    || !(TARGET_INTER_UNIT_CONVERSIONS || optimize_size))))
+   && !(reload_completed || reload_in_progress)"
+  "#"
+  "&& 1"
+  [(parallel [(set (match_dup 0) (float:X87MODEF (match_dup 1)))
+	      (clobber (match_dup 2))])]
+{
+  operands[2] = assign_386_stack_local (<SSEMODEI24:MODE>mode, SLOT_TEMP);
+
+  /* Avoid store forwarding (partial memory) stall penalty
+     by passing DImode value through XMM registers.  */
+  if (<SSEMODEI24:MODE>mode == DImode && !TARGET_64BIT 
+      && TARGET_80387 && TARGET_SSE2 && TARGET_INTER_UNIT_MOVES 
+      && !optimize_size)
+    {
+      emit_insn (gen_floatdi<X87MODEF:mode>2_i387_with_xmm (operands[0],
+							    operands[1],
+							    operands[2]));
+      DONE;
+    }
+})
+
+(define_insn "*floatsi<mode>2_vector_mixed_with_temp"
+  [(set (match_operand:MODEF 0 "register_operand" "=f,f,x,x,x")
+	(float:MODEF
+	  (match_operand:SI 1 "nonimmediate_operand" "m,?r,r,m,!x")))
+   (clobber (match_operand:SI 2 "memory_operand" "=m,m,m,m,m"))]
+  "TARGET_SSE2 && TARGET_MIX_SSE_I387
+   && TARGET_USE_VECTOR_CONVERTS && !optimize_size"
+  "#"
+  [(set_attr "type" "fmov,multi,sseicvt,sseicvt,sseicvt")
+   (set_attr "mode" "<MODE>,<MODE>,<MODE>,<MODE>,<ssevecmode>")
+   (set_attr "unit" "*,i387,*,*,*")
+   (set_attr "athlon_decode" "*,*,double,direct,double")
+   (set_attr "amdfam10_decode" "*,*,vector,double,double")
+   (set_attr "fp_int_src" "true")])
+
+(define_insn "*floatsi<mode>2_vector_mixed"
+  [(set (match_operand:MODEF 0 "register_operand" "=f,x")
+	(float:MODEF (match_operand:SI 1 "memory_operand" "m,m")))]
+  "TARGET_SSE2 && TARGET_MIX_SSE_I387
+   && TARGET_USE_VECTOR_CONVERTS && !optimize_size"
   "@
    fild%z1\t%1
-   #
-   cvtsi2ss\t{%1, %0|%0, %1}
-   cvtsi2ss\t{%1, %0|%0, %1}"
+   #"
+  [(set_attr "type" "fmov,sseicvt")
+   (set_attr "mode" "<MODE>,<ssevecmode>")
+   (set_attr "unit" "i387,*")
+   (set_attr "athlon_decode" "*,direct")
+   (set_attr "amdfam10_decode" "*,double")
+   (set_attr "fp_int_src" "true")])
+
+(define_insn "*float<SSEMODEI24:mode><MODEF:mode>2_mixed_with_temp"
+  [(set (match_operand:MODEF 0 "register_operand" "=f,f,x,x")
+	(float:MODEF
+	  (match_operand:SSEMODEI24 1 "nonimmediate_operand" "m,?r,r,m")))
+  (clobber (match_operand:SSEMODEI24 2 "memory_operand" "=m,m,m,m"))]
+  "(<SSEMODEI24:MODE>mode != DImode || TARGET_64BIT)
+   && SSE_FLOAT_MODE_P (<MODEF:MODE>mode) && TARGET_MIX_SSE_I387"
+  "#"
   [(set_attr "type" "fmov,multi,sseicvt,sseicvt")
-   (set_attr "mode" "SF")
+   (set_attr "mode" "<MODEF:MODE>")
    (set_attr "unit" "*,i387,*,*")
-   (set_attr "athlon_decode" "*,*,vector,double")
+   (set_attr "athlon_decode" "*,*,double,direct")
    (set_attr "amdfam10_decode" "*,*,vector,double")
    (set_attr "fp_int_src" "true")])
 
-(define_insn "*floatsisf2_sse"
-  [(set (match_operand:SF 0 "register_operand" "=x,x")
-	(float:SF (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
-  "TARGET_SSE_MATH"
-  "cvtsi2ss\t{%1, %0|%0, %1}"
-  [(set_attr "type" "sseicvt")
-   (set_attr "mode" "SF")
-   (set_attr "athlon_decode" "vector,double")
-   (set_attr "amdfam10_decode" "vector,double")
+(define_split
+  [(set (match_operand:MODEF 0 "register_operand" "")
+	(float:MODEF (match_operand:SSEMODEI24 1 "register_operand" "")))
+   (clobber (match_operand:SSEMODEI24 2 "memory_operand" ""))]
+  "(<SSEMODEI24:MODE>mode != DImode || TARGET_64BIT)
+   && SSE_FLOAT_MODE_P (<MODEF:MODE>mode) && TARGET_MIX_SSE_I387
+   && TARGET_INTER_UNIT_CONVERSIONS
+   && reload_completed
+   && (SSE_REG_P (operands[0])
+       || (GET_CODE (operands[0]) == SUBREG
+	   && SSE_REG_P (operands[0])))"
+  [(set (match_dup 0) (float:MODEF (match_dup 1)))]
+  "")
+
+(define_split
+  [(set (match_operand:MODEF 0 "register_operand" "")
+	(float:MODEF (match_operand:SSEMODEI24 1 "register_operand" "")))
+   (clobber (match_operand:SSEMODEI24 2 "memory_operand" ""))]
+  "(<SSEMODEI24:MODE>mode != DImode || TARGET_64BIT)
+   && SSE_FLOAT_MODE_P (<MODEF:MODE>mode) && TARGET_MIX_SSE_I387
+   && !(TARGET_INTER_UNIT_CONVERSIONS || optimize_size)
+   && reload_completed
+   && (SSE_REG_P (operands[0])
+       || (GET_CODE (operands[0]) == SUBREG
+	   && SSE_REG_P (operands[0])))"
+  [(set (match_dup 2) (match_dup 1))
+   (set (match_dup 0) (float:MODEF (match_dup 2)))]
+  "")
+
+(define_insn "*float<SSEMODEI24:mode><MODEF:mode>2_mixed_interunit"
+  [(set (match_operand:MODEF 0 "register_operand" "=f,x,x")
+	(float:MODEF
+	  (match_operand:SSEMODEI24 1 "register_operand" "m,r,m")))]
+  "(<SSEMODEI24:MODE>mode != DImode || TARGET_64BIT)
+   && SSE_FLOAT_MODE_P (<MODEF:MODE>mode) && TARGET_MIX_SSE_I387
+   && (TARGET_INTER_UNIT_CONVERSIONS || optimize_size)"
+  "@
+   fild%z1\t%1
+   cvtsi2s<MODEF:ssemodefsuffix><SSEMODEI24:rex64suffix>\t{%1, %0|%0, %1}
+   cvtsi2s<MODEF:ssemodefsuffix><SSEMODEI24:rex64suffix>\t{%1, %0|%0, %1}"
+  [(set_attr "type" "fmov,sseicvt,sseicvt")
+   (set_attr "mode" "<MODEF:MODE>")
+   (set_attr "unit" "i387,*,*")
+   (set_attr "athlon_decode" "*,double,direct")
+   (set_attr "amdfam10_decode" "*,vector,double")
    (set_attr "fp_int_src" "true")])
 
-(define_insn "*floatsidf2_mixed"
-  [(set (match_operand:DF 0 "register_operand" "=f,?f,x,x")
-	(float:DF (match_operand:SI 1 "nonimmediate_operand" "m,r,r,m")))]
-  "TARGET_SSE2 && TARGET_MIX_SSE_I387"
+(define_insn "*float<SSEMODEI24:mode><MODEF:mode>2_mixed_nointerunit"
+  [(set (match_operand:MODEF 0 "register_operand" "=f,x")
+	(float:MODEF
+	  (match_operand:SSEMODEI24 1 "memory_operand" "m,m")))]
+  "(<SSEMODEI24:MODE>mode != DImode || TARGET_64BIT)
+   && SSE_FLOAT_MODE_P (<MODEF:MODE>mode) && TARGET_MIX_SSE_I387
+   && !(TARGET_INTER_UNIT_CONVERSIONS || optimize_size)"
   "@
    fild%z1\t%1
-   #
-   cvtsi2sd\t{%1, %0|%0, %1}
-   cvtsi2sd\t{%1, %0|%0, %1}"
-  [(set_attr "type" "fmov,multi,sseicvt,sseicvt")
-   (set_attr "mode" "DF")
-   (set_attr "unit" "*,i387,*,*")
-   (set_attr "athlon_decode" "*,*,double,direct")
-   (set_attr "amdfam10_decode" "*,*,vector,double")
+   cvtsi2s<MODEF:ssemodefsuffix><SSEMODEI24:rex64suffix>\t{%1, %0|%0, %1}"
+  [(set_attr "type" "fmov,sseicvt")
+   (set_attr "mode" "<MODEF:MODE>")
+   (set_attr "athlon_decode" "*,direct")
+   (set_attr "amdfam10_decode" "*,double")
    (set_attr "fp_int_src" "true")])
 
-(define_insn "*floatsidf2_sse"
-  [(set (match_operand:DF 0 "register_operand" "=x,x")
-	(float:DF (match_operand:SI 1 "nonimmediate_operand" "r,m")))]
-  "TARGET_SSE2 && TARGET_SSE_MATH"
-  "cvtsi2sd\t{%1, %0|%0, %1}"
+(define_insn "*floatsi<mode>2_vector_sse_with_temp"
+  [(set (match_operand:MODEF 0 "register_operand" "=x,x,x")
+	(float:MODEF
+	  (match_operand:SI 1 "nonimmediate_operand" "r,m,!x")))
+   (clobber (match_operand:SI 2 "memory_operand" "=m,m,m"))]
+  "TARGET_SSE2 && TARGET_SSE_MATH
+   && TARGET_USE_VECTOR_CONVERTS && !optimize_size"
+  "#"
+  [(set_attr "type" "sseicvt")
+   (set_attr "mode" "<MODE>,<MODE>,<ssevecmode>")
+   (set_attr "athlon_decode" "double,direct,double")
+   (set_attr "amdfam10_decode" "vector,double,double")
+   (set_attr "fp_int_src" "true")])
+
+(define_insn "*floatsi<mode>2_vector_sse"
+  [(set (match_operand:MODEF 0 "register_operand" "=x")
+	(float:MODEF (match_operand:SI 1 "memory_operand" "m")))]
+  "TARGET_SSE2 && TARGET_SSE_MATH
+   && TARGET_USE_VECTOR_CONVERTS && !optimize_size"
+  "#"
   [(set_attr "type" "sseicvt")
-   (set_attr "mode" "DF")
+   (set_attr "mode" "<MODE>")
+   (set_attr "athlon_decode" "direct")
+   (set_attr "amdfam10_decode" "double")
+   (set_attr "fp_int_src" "true")])
+
+(define_split
+  [(set (match_operand:MODEF 0 "register_operand" "")
+	(float:MODEF (match_operand:SI 1 "register_operand" "")))
+   (clobber (match_operand:SI 2 "memory_operand" ""))]
+  "TARGET_SSE2 && TARGET_SSE_MATH
+   && TARGET_USE_VECTOR_CONVERTS && !optimize_size
+   && reload_completed
+   && (SSE_REG_P (operands[0])
+       || (GET_CODE (operands[0]) == SUBREG
+	   && SSE_REG_P (operands[0])))"
+  [(const_int 0)]
+{
+  rtx op1 = operands[1];
+
+  operands[3] = simplify_gen_subreg (<ssevecmode>mode, operands[0],
+				     <MODE>mode, 0);
+  if (GET_CODE (op1) == SUBREG)
+    op1 = SUBREG_REG (op1);
+
+  if (GENERAL_REG_P (op1) && TARGET_INTER_UNIT_MOVES)
+    {
+      operands[4] = simplify_gen_subreg (V4SImode, operands[0], <MODE>mode, 0);
+      emit_insn (gen_sse2_loadld (operands[4],
+				  CONST0_RTX (V4SImode), operands[1]));
+    }
+  /* We can ignore possible trapping value in the
+     high part of SSE register for non-trapping math. */
+  else if (SSE_REG_P (op1) && !flag_trapping_math)
+    operands[4] = simplify_gen_subreg (V4SImode, operands[1], SImode, 0);
+  else
+    {
+      operands[4] = simplify_gen_subreg (V4SImode, operands[0], <MODE>mode, 0);
+      emit_move_insn (operands[2], operands[1]);
+      emit_insn (gen_sse2_loadld (operands[4],
+				  CONST0_RTX (V4SImode), operands[2]));
+    }
+  emit_insn
+    (gen_sse2_cvtdq2p<ssemodefsuffix> (operands[3], operands[4]));
+  DONE;
+})
+
+(define_split
+  [(set (match_operand:MODEF 0 "register_operand" "")
+	(float:MODEF (match_operand:SI 1 "memory_operand" "")))
+   (clobber (match_operand:SI 2 "memory_operand" ""))]
+  "TARGET_SSE2 && TARGET_SSE_MATH
+   && TARGET_USE_VECTOR_CONVERTS && !optimize_size
+   && reload_completed
+   && (SSE_REG_P (operands[0])
+       || (GET_CODE (operands[0]) == SUBREG
+	   && SSE_REG_P (operands[0])))"
+  [(const_int 0)]
+{
+  operands[3] = simplify_gen_subreg (<ssevecmode>mode, operands[0],
+				     <MODE>mode, 0);
+  operands[4] = simplify_gen_subreg (V4SImode, operands[0], <MODE>mode, 0);
+
+  emit_insn (gen_sse2_loadld (operands[4],
+			      CONST0_RTX (V4SImode), operands[1]));
+  emit_insn
+    (gen_sse2_cvtdq2p<ssemodefsuffix> (operands[3], operands[4]));
+  DONE;
+})
+
+(define_split
+  [(set (match_operand:MODEF 0 "register_operand" "")
+	(float:MODEF (match_operand:SI 1 "register_operand" "")))]
+  "TARGET_SSE2 && TARGET_SSE_MATH
+   && TARGET_USE_VECTOR_CONVERTS && !optimize_size
+   && reload_completed
+   && (SSE_REG_P (operands[0])
+       || (GET_CODE (operands[0]) == SUBREG
+	   && SSE_REG_P (operands[0])))"
+  [(const_int 0)]
+{
+  rtx op1 = operands[1];
+
+  operands[3] = simplify_gen_subreg (<ssevecmode>mode, operands[0],
+				     <MODE>mode, 0);
+  if (GET_CODE (op1) == SUBREG)
+    op1 = SUBREG_REG (op1);
+
+  if (GENERAL_REG_P (op1) && TARGET_INTER_UNIT_MOVES)
+    {
+      operands[4] = simplify_gen_subreg (V4SImode, operands[0], <MODE>mode, 0);
+      emit_insn (gen_sse2_loadld (operands[4],
+				  CONST0_RTX (V4SImode), operands[1]));
+    }
+  /* We can ignore possible trapping value in the
+     high part of SSE register for non-trapping math. */
+  else if (SSE_REG_P (op1) && !flag_trapping_math)
+    operands[4] = simplify_gen_subreg (V4SImode, operands[1], SImode, 0);
+  else
+    gcc_unreachable ();
+})
+
+(define_split
+  [(set (match_operand:MODEF 0 "register_operand" "")
+	(float:MODEF (match_operand:SI 1 "memory_operand" "")))]
+  "TARGET_SSE2 && TARGET_SSE_MATH
+   && TARGET_USE_VECTOR_CONVERTS && !optimize_size
+   && reload_completed
+   && (SSE_REG_P (operands[0])
+       || (GET_CODE (operands[0]) == SUBREG
+	   && SSE_REG_P (operands[0])))"
+  [(const_int 0)]
+{
+  operands[3] = simplify_gen_subreg (<ssevecmode>mode, operands[0],
+				     <MODE>mode, 0);
+  operands[4] = simplify_gen_subreg (V4SImode, operands[0], <MODE>mode, 0);
+
+  emit_insn (gen_sse2_loadld (operands[4],
+			      CONST0_RTX (V4SImode), operands[1]));
+  emit_insn
+    (gen_sse2_cvtdq2p<ssemodefsuffix> (operands[3], operands[4]));
+  DONE;
+})
+
+(define_insn "*float<SSEMODEI24:mode><MODEF:mode>2_sse_with_temp"
+  [(set (match_operand:MODEF 0 "register_operand" "=x,x")
+	(float:MODEF
+	  (match_operand:SSEMODEI24 1 "nonimmediate_operand" "r,m")))
+  (clobber (match_operand:SSEMODEI24 2 "memory_operand" "=m,m"))]
+  "(<SSEMODEI24:MODE>mode != DImode || TARGET_64BIT)
+   && SSE_FLOAT_MODE_P (<MODEF:MODE>mode) && TARGET_SSE_MATH"
+  "#"
+  [(set_attr "type" "sseicvt")
+   (set_attr "mode" "<MODEF:MODE>")
+   (set_attr "athlon_decode" "double,direct")
+   (set_attr "amdfam10_decode" "vector,double")
+   (set_attr "fp_int_src" "true")])
+
+(define_insn "*float<SSEMODEI24:mode><MODEF:mode>2_sse_interunit"
+  [(set (match_operand:MODEF 0 "register_operand" "=x,x")
+	(float:MODEF
+	  (match_operand:SSEMODEI24 1 "nonimmediate_operand" "r,m")))]
+  "(<SSEMODEI24:MODE>mode != DImode || TARGET_64BIT)
+   && SSE_FLOAT_MODE_P (<MODEF:MODE>mode) && TARGET_SSE_MATH
+   && (TARGET_INTER_UNIT_CONVERSIONS || optimize_size)"
+  "@
+   cvtsi2s<MODEF:ssemodefsuffix><SSEMODEI24:rex64suffix>\t{%1, %0|%0, %1}
+   cvtsi2s<MODEF:ssemodefsuffix><SSEMODEI24:rex64suffix>\t{%1, %0|%0, %1}"
+  [(set_attr "type" "sseicvt")
+   (set_attr "mode" "<MODEF:MODE>")
    (set_attr "athlon_decode" "double,direct")
    (set_attr "amdfam10_decode" "vector,double")
    (set_attr "fp_int_src" "true")])
 
-(define_insn "*floatsi<mode>2_i387"
-  [(set (match_operand:X87MODEF12 0 "register_operand" "=f,f")
-	(float:X87MODEF12
-	  (match_operand:SI 1 "nonimmediate_operand" "m,?r")))]
+(define_split
+  [(set (match_operand:MODEF 0 "register_operand" "")
+	(float:MODEF (match_operand:SSEMODEI24 1 "nonimmediate_operand" "")))
+   (clobber (match_operand:SSEMODEI24 2 "memory_operand" ""))]
+  "(<SSEMODEI24:MODE>mode != DImode || TARGET_64BIT)
+   && SSE_FLOAT_MODE_P (<MODEF:MODE>mode) && TARGET_SSE_MATH
+   && (TARGET_INTER_UNIT_CONVERSIONS || optimize_size)
+   && reload_completed
+   && (SSE_REG_P (operands[0])
+       || (GET_CODE (operands[0]) == SUBREG
+	   && SSE_REG_P (operands[0])))"
+  [(set (match_dup 0) (float:MODEF (match_dup 1)))]
+  "")
+
+(define_insn "*float<SSEMODEI24:mode><MODEF:mode>2_sse_nointerunit"
+  [(set (match_operand:MODEF 0 "register_operand" "=x")
+	(float:MODEF
+	  (match_operand:SSEMODEI24 1 "memory_operand" "m")))]
+  "(<SSEMODEI24:MODE>mode != DImode || TARGET_64BIT)
+   && SSE_FLOAT_MODE_P (<MODEF:MODE>mode) && TARGET_SSE_MATH
+   && !(TARGET_INTER_UNIT_CONVERSIONS || optimize_size)"
+  "cvtsi2s<MODEF:ssemodefsuffix><SSEMODEI24:rex64suffix>\t{%1, %0|%0, %1}"
+  [(set_attr "type" "sseicvt")
+   (set_attr "mode" "<MODEF:MODE>")
+   (set_attr "athlon_decode" "direct")
+   (set_attr "amdfam10_decode" "double")
+   (set_attr "fp_int_src" "true")])
+
+(define_split
+  [(set (match_operand:MODEF 0 "register_operand" "")
+	(float:MODEF (match_operand:SSEMODEI24 1 "register_operand" "")))
+   (clobber (match_operand:SSEMODEI24 2 "memory_operand" ""))]
+  "(<SSEMODEI24:MODE>mode != DImode || TARGET_64BIT)
+   && SSE_FLOAT_MODE_P (<MODEF:MODE>mode) && TARGET_SSE_MATH
+   && !(TARGET_INTER_UNIT_CONVERSIONS || optimize_size)
+   && reload_completed
+   && (SSE_REG_P (operands[0])
+       || (GET_CODE (operands[0]) == SUBREG
+	   && SSE_REG_P (operands[0])))"
+  [(set (match_dup 2) (match_dup 1))
+   (set (match_dup 0) (float:MODEF (match_dup 2)))]
+  "")
+
+(define_split
+  [(set (match_operand:MODEF 0 "register_operand" "")
+	(float:MODEF (match_operand:SSEMODEI24 1 "memory_operand" "")))
+   (clobber (match_operand:SSEMODEI24 2 "memory_operand" ""))]
+  "(<SSEMODEI24:MODE>mode != DImode || TARGET_64BIT)
+   && SSE_FLOAT_MODE_P (<MODEF:MODE>mode) && TARGET_SSE_MATH
+   && reload_completed
+   && (SSE_REG_P (operands[0])
+       || (GET_CODE (operands[0]) == SUBREG
+	   && SSE_REG_P (operands[0])))"
+  [(set (match_dup 0) (float:MODEF (match_dup 1)))]
+  "")
+
+(define_insn "*float<SSEMODEI24:mode><X87MODEF:mode>2_i387_with_temp"
+  [(set (match_operand:X87MODEF 0 "register_operand" "=f,f")
+	(float:X87MODEF
+	  (match_operand:SSEMODEI24 1 "nonimmediate_operand" "m,?r")))
+  (clobber (match_operand:SSEMODEI24 2 "memory_operand" "=m,m"))]
   "TARGET_80387"
   "@
    fild%z1\t%1
    #"
   [(set_attr "type" "fmov,multi")
-   (set_attr "mode" "<MODE>")
+   (set_attr "mode" "<X87MODEF:MODE>")
    (set_attr "unit" "*,i387")
    (set_attr "fp_int_src" "true")])
 
-(define_expand "floatdisf2"
-  [(set (match_operand:SF 0 "register_operand" "")
-	(float:SF (match_operand:DI 1 "nonimmediate_operand" "")))]
-  "TARGET_80387 || (TARGET_64BIT && TARGET_SSE_MATH)"
+(define_insn "*float<SSEMODEI24:mode><X87MODEF:mode>2_i387"
+  [(set (match_operand:X87MODEF 0 "register_operand" "=f")
+	(float:X87MODEF
+	  (match_operand:SSEMODEI24 1 "memory_operand" "m")))]
+  "TARGET_80387"
+  "fild%z1\t%1"
+  [(set_attr "type" "fmov")
+   (set_attr "mode" "<X87MODEF:MODE>")
+   (set_attr "fp_int_src" "true")])
+
+(define_split
+  [(set (match_operand:X87MODEF 0 "register_operand" "")
+	(float:X87MODEF (match_operand:SSEMODEI24 1 "register_operand" "")))
+   (clobber (match_operand:SSEMODEI24 2 "memory_operand" ""))]
+  "TARGET_80387
+   && reload_completed
+   && FP_REG_P (operands[0])"
+  [(set (match_dup 2) (match_dup 1))
+   (set (match_dup 0) (float:X87MODEF (match_dup 2)))]
   "")
 
-(define_insn "*floatdisf2_mixed"
-  [(set (match_operand:SF 0 "register_operand" "=f,?f,x,x")
-	(float:SF (match_operand:DI 1 "nonimmediate_operand" "m,r,r,m")))]
-  "TARGET_64BIT && TARGET_MIX_SSE_I387"
-  "@
-   fild%z1\t%1
-   #
-   cvtsi2ss{q}\t{%1, %0|%0, %1}
-   cvtsi2ss{q}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "fmov,multi,sseicvt,sseicvt")
-   (set_attr "mode" "SF")
-   (set_attr "unit" "*,i387,*,*")
-   (set_attr "athlon_decode" "*,*,vector,double")
-   (set_attr "amdfam10_decode" "*,*,vector,double")
-   (set_attr "fp_int_src" "true")])
+(define_split
+  [(set (match_operand:X87MODEF 0 "register_operand" "")
+	(float:X87MODEF (match_operand:SSEMODEI24 1 "memory_operand" "")))
+   (clobber (match_operand:SSEMODEI24 2 "memory_operand" ""))]
+  "TARGET_80387
+   && reload_completed
+   && FP_REG_P (operands[0])"
+  [(set (match_dup 0) (float:X87MODEF (match_dup 1)))]
+  "")
 
-(define_insn "*floatdisf2_sse"
-  [(set (match_operand:SF 0 "register_operand" "=x,x")
-	(float:SF (match_operand:DI 1 "nonimmediate_operand" "r,m")))]
-  "TARGET_64BIT && TARGET_SSE_MATH"
-  "cvtsi2ss{q}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "sseicvt")
-   (set_attr "mode" "SF")
-   (set_attr "athlon_decode" "vector,double")
-   (set_attr "amdfam10_decode" "vector,double")
+;; Avoid store forwarding (partial memory) stall penalty
+;; by passing DImode value through XMM registers.  */
+
+(define_insn "floatdi<X87MODEF:mode>2_i387_with_xmm"
+  [(set (match_operand:X87MODEF 0 "register_operand" "=f,f")
+	(float:X87MODEF
+	  (match_operand:DI 1 "nonimmediate_operand" "m,?r")))
+   (clobber (match_scratch:V4SI 3 "=&x,x"))
+   (clobber (match_scratch:V4SI 4 "=&x,x"))
+   (clobber (match_operand:DI 2 "memory_operand" "=m,m"))]
+  "TARGET_80387 && TARGET_SSE2 && TARGET_INTER_UNIT_MOVES
+   && !TARGET_64BIT && !optimize_size"
+  "#"
+  [(set_attr "type" "multi")
+   (set_attr "mode" "<X87MODEF:MODE>")
+   (set_attr "unit" "i387")
    (set_attr "fp_int_src" "true")])
 
-(define_expand "floatdidf2"
-  [(set (match_operand:DF 0 "register_operand" "")
-	(float:DF (match_operand:DI 1 "nonimmediate_operand" "")))]
-  "TARGET_80387 || (TARGET_SSE2 && TARGET_SSE_MATH)"
+(define_split
+  [(set (match_operand:X87MODEF 0 "register_operand" "")
+	(float:X87MODEF (match_operand:DI 1 "register_operand" "")))
+   (clobber (match_operand:V4SI 3 "register_operand" ""))
+   (clobber (match_operand:V4SI 4 "register_operand" ""))
+   (clobber (match_operand:DI 2 "memory_operand" ""))]
+  "TARGET_80387 && TARGET_SSE2 && TARGET_INTER_UNIT_MOVES
+   && !TARGET_64BIT && !optimize_size
+   && reload_completed
+   && FP_REG_P (operands[0])"
+  [(set (match_dup 2) (match_dup 3))
+   (set (match_dup 0) (float:X87MODEF (match_dup 2)))]
 {
-  if (!TARGET_64BIT && TARGET_SSE2 && TARGET_SSE_MATH)
-    {
-      ix86_expand_convert_sign_didf_sse (operands[0], operands[1]);
-      DONE;
-    }
+  /* The DImode arrived in a pair of integral registers (e.g. %edx:%eax).
+     Assemble the 64-bit DImode value in an xmm register.  */
+  emit_insn (gen_sse2_loadld (operands[3], CONST0_RTX (V4SImode),
+			      gen_rtx_SUBREG (SImode, operands[1], 0)));
+  emit_insn (gen_sse2_loadld (operands[4], CONST0_RTX (V4SImode),
+			      gen_rtx_SUBREG (SImode, operands[1], 4)));
+  emit_insn (gen_sse2_punpckldq (operands[3], operands[3], operands[4]));
+
+  operands[3] = gen_rtx_REG (DImode, REGNO (operands[3]));
 })
 
-(define_insn "*floatdidf2_mixed"
-  [(set (match_operand:DF 0 "register_operand" "=f,?f,x,x")
-	(float:DF (match_operand:DI 1 "nonimmediate_operand" "m,r,r,m")))]
-  "TARGET_64BIT && TARGET_SSE2 && TARGET_MIX_SSE_I387"
-  "@
-   fild%z1\t%1
-   #
-   cvtsi2sd{q}\t{%1, %0|%0, %1}
-   cvtsi2sd{q}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "fmov,multi,sseicvt,sseicvt")
-   (set_attr "mode" "DF")
-   (set_attr "unit" "*,i387,*,*")
-   (set_attr "athlon_decode" "*,*,double,direct")
-   (set_attr "amdfam10_decode" "*,*,vector,double")
-   (set_attr "fp_int_src" "true")])
+(define_split
+  [(set (match_operand:X87MODEF 0 "register_operand" "")
+	(float:X87MODEF (match_operand:DI 1 "memory_operand" "")))
+   (clobber (match_operand:V4SI 2 "register_operand" ""))
+   (clobber (match_operand:V4SI 3 "register_operand" ""))
+   (clobber (match_operand:DI 4 "memory_operand" ""))]
+  "TARGET_80387 && TARGET_SSE2 && TARGET_INTER_UNIT_MOVES
+   && !TARGET_64BIT && !optimize_size
+   && reload_completed
+   && FP_REG_P (operands[0])"
+  [(set (match_dup 0) (float:X87MODEF (match_dup 1)))]
+  "")
 
-(define_insn "*floatdidf2_sse"
-  [(set (match_operand:DF 0 "register_operand" "=x,x")
-	(float:DF (match_operand:DI 1 "nonimmediate_operand" "r,m")))]
-  "TARGET_64BIT && TARGET_SSE2 && TARGET_SSE_MATH"
-  "cvtsi2sd{q}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "sseicvt")
-   (set_attr "mode" "DF")
-   (set_attr "athlon_decode" "double,direct")
-   (set_attr "amdfam10_decode" "vector,double")
-   (set_attr "fp_int_src" "true")])
+;; Avoid store forwarding (partial memory) stall penalty by extending
+;; SImode value to DImode through XMM register instead of pushing two
+;; SImode values to stack. Note that even !TARGET_INTER_UNIT_MOVES
+;; targets benefit from this optimization. Also note that fild
+;; loads from memory only.
 
-(define_insn "*floatdi<mode>2_i387"
-  [(set (match_operand:X87MODEF12 0 "register_operand" "=f,f")
-	(float:X87MODEF12
-	  (match_operand:DI 1 "nonimmediate_operand" "m,?r")))]
-  "TARGET_80387"
-  "@
-   fild%z1\t%1
-   #"
-  [(set_attr "type" "fmov,multi")
-   (set_attr "mode" "<MODE>")
-   (set_attr "unit" "*,i387")
-   (set_attr "fp_int_src" "true")])
+(define_insn "*floatunssi<mode>2_1"
+  [(set (match_operand:X87MODEF 0 "register_operand" "=f,f")
+	(unsigned_float:X87MODEF
+	  (match_operand:SI 1 "nonimmediate_operand" "x,m")))
+   (clobber (match_operand:DI 2 "memory_operand" "=m,m"))
+   (clobber (match_scratch:SI 3 "=X,x"))]
+  "!TARGET_64BIT
+   && TARGET_80387 && TARGET_SSE"
+  "#"
+  [(set_attr "type" "multi")
+   (set_attr "mode" "<MODE>")])
 
-(define_insn "float<mode>xf2"
-  [(set (match_operand:XF 0 "register_operand" "=f,f")
-	(float:XF (match_operand:X87MODEI 1 "nonimmediate_operand" "m,?r")))]
-  "TARGET_80387"
-  "@
-   fild%z1\t%1
-   #"
-  [(set_attr "type" "fmov,multi")
-   (set_attr "mode" "XF")
-   (set_attr "unit" "*,i387")
-   (set_attr "fp_int_src" "true")])
+(define_split
+  [(set (match_operand:X87MODEF 0 "register_operand" "")
+	(unsigned_float:X87MODEF
+	  (match_operand:SI 1 "register_operand" "")))
+   (clobber (match_operand:DI 2 "memory_operand" ""))
+   (clobber (match_scratch:SI 3 ""))]
+  "!TARGET_64BIT
+   && TARGET_80387 && TARGET_SSE
+   && reload_completed"
+  [(set (match_dup 2) (match_dup 1))
+   (set (match_dup 0)
+	(float:X87MODEF (match_dup 2)))]
+  "operands[1] = simplify_gen_subreg (DImode, operands[1], SImode, 0);")
 
-;; %%% Kill these when reload knows how to do it.
 (define_split
-  [(set (match_operand 0 "fp_register_operand" "")
-	(float (match_operand 1 "register_operand" "")))]
-  "reload_completed
-   && X87_FLOAT_MODE_P (GET_MODE (operands[0]))"
-  [(const_int 0)]
+  [(set (match_operand:X87MODEF 0 "register_operand" "")
+	(unsigned_float:X87MODEF
+	  (match_operand:SI 1 "memory_operand" "")))
+   (clobber (match_operand:DI 2 "memory_operand" ""))
+   (clobber (match_scratch:SI 3 ""))]
+  "!TARGET_64BIT
+   && TARGET_80387 && TARGET_SSE
+   && reload_completed"
+  [(set (match_dup 2) (match_dup 3))
+   (set (match_dup 0)
+	(float:X87MODEF (match_dup 2)))]
 {
-  operands[2] = ix86_force_to_memory (GET_MODE (operands[1]), operands[1]);
-  operands[2] = gen_rtx_FLOAT (GET_MODE (operands[0]), operands[2]);
-  emit_insn (gen_rtx_SET (VOIDmode, operands[0], operands[2]));
-  ix86_free_from_memory (GET_MODE (operands[1]));
-  DONE;
+  emit_move_insn (operands[3], operands[1]);
+  operands[3] = simplify_gen_subreg (DImode, operands[3], SImode, 0);
 })
 
-(define_expand "floatunssisf2"
-  [(use (match_operand:SF 0 "register_operand" ""))
-   (use (match_operand:SI 1 "nonimmediate_operand" ""))]
-  "!TARGET_64BIT"
+(define_expand "floatunssi<mode>2"
+  [(parallel
+     [(set (match_operand:X87MODEF 0 "register_operand" "")
+	   (unsigned_float:X87MODEF
+	     (match_operand:SI 1 "nonimmediate_operand" "")))
+      (clobber (match_dup 2))
+      (clobber (match_scratch:SI 3 ""))])]
+  "!TARGET_64BIT
+   && ((TARGET_80387 && TARGET_SSE)
+       || (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH))"
 {
-  if (TARGET_SSE_MATH && TARGET_SSE2)
-    ix86_expand_convert_uns_sisf_sse (operands[0], operands[1]);
+  if (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
+    {
+      ix86_expand_convert_uns_si<mode>_sse (operands[0], operands[1]);
+      DONE;
+    }
   else
-    x86_emit_floatuns (operands);
-  DONE;
+    {
+      int slot = virtuals_instantiated ? SLOT_TEMP : SLOT_VIRTUAL;
+      operands[2] = assign_386_stack_local (DImode, slot);
+    }
 })
 
-(define_expand "floatunssidf2"
-  [(use (match_operand:DF 0 "register_operand" ""))
-   (use (match_operand:SI 1 "nonimmediate_operand" ""))]
-  "!TARGET_64BIT && TARGET_SSE_MATH && TARGET_SSE2"
-  "ix86_expand_convert_uns_sidf_sse (operands[0], operands[1]); DONE;")
-
 (define_expand "floatunsdisf2"
   [(use (match_operand:SF 0 "register_operand" ""))
    (use (match_operand:DI 1 "nonimmediate_operand" ""))]
@@ -4771,8 +5478,8 @@
 (define_expand "floatunsdidf2"
   [(use (match_operand:DF 0 "register_operand" ""))
    (use (match_operand:DI 1 "nonimmediate_operand" ""))]
-  "TARGET_SSE_MATH && TARGET_SSE2
-   && (TARGET_64BIT || TARGET_KEEPS_VECTOR_ALIGNED_STACK)"
+  "(TARGET_64BIT || TARGET_KEEPS_VECTOR_ALIGNED_STACK)
+   && TARGET_SSE2 && TARGET_SSE_MATH"
 {
   if (TARGET_64BIT)
     x86_emit_floatuns (operands);
@@ -6551,18 +7258,11 @@
   "TARGET_80387"
   "")
 
-(define_expand "adddf3"
-  [(set (match_operand:DF 0 "register_operand" "")
-	(plus:DF (match_operand:DF 1 "register_operand" "")
-		 (match_operand:DF 2 "nonimmediate_operand" "")))]
-  "TARGET_80387 || (TARGET_SSE2 && TARGET_SSE_MATH)"
-  "")
-
-(define_expand "addsf3"
-  [(set (match_operand:SF 0 "register_operand" "")
-	(plus:SF (match_operand:SF 1 "register_operand" "")
-		 (match_operand:SF 2 "nonimmediate_operand" "")))]
-  "TARGET_80387 || TARGET_SSE_MATH"
+(define_expand "add<mode>3"
+  [(set (match_operand:MODEF 0 "register_operand" "")
+	(plus:MODEF (match_operand:MODEF 1 "register_operand" "")
+		    (match_operand:MODEF 2 "nonimmediate_operand" "")))]
+  "TARGET_80387 || (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)"
   "")
 
 ;; Subtract instructions
@@ -6927,18 +7627,11 @@
   "TARGET_80387"
   "")
 
-(define_expand "subdf3"
-  [(set (match_operand:DF 0 "register_operand" "")
-	(minus:DF (match_operand:DF 1 "register_operand" "")
-		  (match_operand:DF 2 "nonimmediate_operand" "")))]
-  "TARGET_80387 || (TARGET_SSE2 && TARGET_SSE_MATH)"
-  "")
-
-(define_expand "subsf3"
-  [(set (match_operand:SF 0 "register_operand" "")
-	(minus:SF (match_operand:SF 1 "register_operand" "")
-		  (match_operand:SF 2 "nonimmediate_operand" "")))]
-  "TARGET_80387 || TARGET_SSE_MATH"
+(define_expand "sub<mode>3"
+  [(set (match_operand:MODEF 0 "register_operand" "")
+	(minus:MODEF (match_operand:MODEF 1 "register_operand" "")
+		     (match_operand:MODEF 2 "nonimmediate_operand" "")))]
+  "TARGET_80387 || (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)"
   "")
 
 ;; Multiply instructions
@@ -6951,11 +7644,11 @@
   "TARGET_64BIT"
   "")
 
-;; On AMDFAM10 
+;; On AMDFAM10
 ;; IMUL reg64, reg64, imm8 	Direct
 ;; IMUL reg64, mem64, imm8 	VectorPath
 ;; IMUL reg64, reg64, imm32 	Direct
-;; IMUL reg64, mem64, imm32 	VectorPath 
+;; IMUL reg64, mem64, imm32 	VectorPath
 ;; IMUL reg64, reg64 		Direct
 ;; IMUL reg64, mem64 		Direct
 
@@ -6985,7 +7678,7 @@
 	(cond [(and (eq_attr "alternative" "0,1")
 		    (match_operand 1 "memory_operand" ""))
 		  (const_string "vector")]
-	      (const_string "direct")))	      
+	      (const_string "direct")))
    (set_attr "mode" "DI")])
 
 (define_expand "mulsi3"
@@ -6996,7 +7689,7 @@
   ""
   "")
 
-;; On AMDFAM10 
+;; On AMDFAM10
 ;; IMUL reg32, reg32, imm8 	Direct
 ;; IMUL reg32, mem32, imm8 	VectorPath
 ;; IMUL reg32, reg32, imm32 	Direct
@@ -7029,7 +7722,7 @@
 	(cond [(and (eq_attr "alternative" "0,1")
 		    (match_operand 1 "memory_operand" ""))
 		  (const_string "vector")]
-	      (const_string "direct")))	      
+	      (const_string "direct")))
    (set_attr "mode" "SI")])
 
 (define_insn "*mulsi3_1_zext"
@@ -7059,7 +7752,7 @@
 	(cond [(and (eq_attr "alternative" "0,1")
 		    (match_operand 1 "memory_operand" ""))
 		  (const_string "vector")]
-	      (const_string "direct")))	      
+	      (const_string "direct")))
    (set_attr "mode" "SI")])
 
 (define_expand "mulhi3"
@@ -7127,7 +7820,7 @@
      (if_then_else (eq_attr "cpu" "athlon")
         (const_string "vector")
         (const_string "direct")))
-   (set_attr "amdfam10_decode" "direct")        
+   (set_attr "amdfam10_decode" "direct")
    (set_attr "mode" "QI")])
 
 (define_expand "umulqihi3"
@@ -7154,7 +7847,7 @@
      (if_then_else (eq_attr "cpu" "athlon")
         (const_string "vector")
         (const_string "direct")))
-   (set_attr "amdfam10_decode" "direct")        
+   (set_attr "amdfam10_decode" "direct")
    (set_attr "mode" "QI")])
 
 (define_expand "mulqihi3"
@@ -7179,7 +7872,7 @@
      (if_then_else (eq_attr "cpu" "athlon")
         (const_string "vector")
         (const_string "direct")))
-   (set_attr "amdfam10_decode" "direct")        
+   (set_attr "amdfam10_decode" "direct")
    (set_attr "mode" "QI")])
 
 (define_expand "umulditi3"
@@ -7206,7 +7899,7 @@
      (if_then_else (eq_attr "cpu" "athlon")
         (const_string "vector")
         (const_string "double")))
-   (set_attr "amdfam10_decode" "double")        
+   (set_attr "amdfam10_decode" "double")
    (set_attr "mode" "DI")])
 
 ;; We can't use this pattern in 64bit mode, since it results in two separate 32bit registers
@@ -7234,7 +7927,7 @@
      (if_then_else (eq_attr "cpu" "athlon")
         (const_string "vector")
         (const_string "double")))
-   (set_attr "amdfam10_decode" "double")        
+   (set_attr "amdfam10_decode" "double")
    (set_attr "mode" "SI")])
 
 (define_expand "mulditi3"
@@ -7288,7 +7981,7 @@
      (if_then_else (eq_attr "cpu" "athlon")
         (const_string "vector")
         (const_string "double")))
-   (set_attr "amdfam10_decode" "double")        
+   (set_attr "amdfam10_decode" "double")
    (set_attr "mode" "SI")])
 
 (define_expand "umuldi3_highpart"
@@ -7325,7 +8018,7 @@
      (if_then_else (eq_attr "cpu" "athlon")
         (const_string "vector")
         (const_string "double")))
-   (set_attr "amdfam10_decode" "double")        
+   (set_attr "amdfam10_decode" "double")
    (set_attr "mode" "DI")])
 
 (define_expand "umulsi3_highpart"
@@ -7388,7 +8081,7 @@
    (set_attr "mode" "SI")])
 
 (define_expand "smuldi3_highpart"
-  [(parallel [(set (match_operand:DI 0 "register_operand" "=d")
+  [(parallel [(set (match_operand:DI 0 "register_operand" "")
 		   (truncate:DI
 		     (lshiftrt:TI
 		       (mult:TI (sign_extend:TI
@@ -7489,19 +8182,15 @@
   "TARGET_80387"
   "")
 
-(define_expand "muldf3"
-  [(set (match_operand:DF 0 "register_operand" "")
-	(mult:DF (match_operand:DF 1 "register_operand" "")
-		 (match_operand:DF 2 "nonimmediate_operand" "")))]
-  "TARGET_80387 || (TARGET_SSE2 && TARGET_SSE_MATH)"
+(define_expand "mul<mode>3"
+  [(set (match_operand:MODEF 0 "register_operand" "")
+	(mult:MODEF (match_operand:MODEF 1 "register_operand" "")
+		    (match_operand:MODEF 2 "nonimmediate_operand" "")))]
+  "TARGET_80387 || (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)"
   "")
 
-(define_expand "mulsf3"
-  [(set (match_operand:SF 0 "register_operand" "")
-	(mult:SF (match_operand:SF 1 "register_operand" "")
-		 (match_operand:SF 2 "nonimmediate_operand" "")))]
-  "TARGET_80387 || TARGET_SSE_MATH"
-  "")
+;; SSE5 scalar multiply/add instructions are defined in sse.md.
+
 
 ;; Divide instructions
 
@@ -9697,132 +10386,52 @@
 
 ;; Changing of sign for FP values is doable using integer unit too.
 
-(define_expand "negsf2"
-  [(set (match_operand:SF 0 "nonimmediate_operand" "")
-	(neg:SF (match_operand:SF 1 "nonimmediate_operand" "")))]
-  "TARGET_80387 || TARGET_SSE_MATH"
-  "ix86_expand_fp_absneg_operator (NEG, SFmode, operands); DONE;")
-
-(define_expand "abssf2"
-  [(set (match_operand:SF 0 "nonimmediate_operand" "")
-	(abs:SF (match_operand:SF 1 "nonimmediate_operand" "")))]
-  "TARGET_80387 || TARGET_SSE_MATH"
-  "ix86_expand_fp_absneg_operator (ABS, SFmode, operands); DONE;")
-
-(define_insn "*absnegsf2_mixed"
-  [(set (match_operand:SF 0 "nonimmediate_operand"    "=x  ,x,f,rm")
-	(match_operator:SF 3 "absneg_operator"
-	  [(match_operand:SF 1 "nonimmediate_operand" "0   ,x,0,0 ")]))
-   (use (match_operand:V4SF 2 "nonimmediate_operand"  "xm  ,0,X,X "))
-   (clobber (reg:CC FLAGS_REG))]
-  "TARGET_SSE_MATH && TARGET_MIX_SSE_I387
-   && ix86_unary_operator_ok (GET_CODE (operands[3]), SFmode, operands)"
-  "#")
-
-(define_insn "*absnegsf2_sse"
-  [(set (match_operand:SF 0 "nonimmediate_operand"    "=x,x,rm")
-	(match_operator:SF 3 "absneg_operator"
-	  [(match_operand:SF 1 "nonimmediate_operand" "0 ,x,0")]))
-   (use (match_operand:V4SF 2 "nonimmediate_operand"  "xm,0,X"))
-   (clobber (reg:CC FLAGS_REG))]
-  "TARGET_SSE_MATH
-   && ix86_unary_operator_ok (GET_CODE (operands[3]), SFmode, operands)"
-  "#")
-
-(define_insn "*absnegsf2_i387"
-  [(set (match_operand:SF 0 "nonimmediate_operand" "=f,rm")
-	(match_operator:SF 3 "absneg_operator"
-	  [(match_operand:SF 1 "nonimmediate_operand" "0,0")]))
-   (use (match_operand 2 "" ""))
-   (clobber (reg:CC FLAGS_REG))]
-  "TARGET_80387 && !TARGET_SSE_MATH
-   && ix86_unary_operator_ok (GET_CODE (operands[3]), SFmode, operands)"
-  "#")
-
-(define_expand "negdf2"
-  [(set (match_operand:DF 0 "nonimmediate_operand" "")
-	(neg:DF (match_operand:DF 1 "nonimmediate_operand" "")))]
-  "TARGET_80387 || (TARGET_SSE2 && TARGET_SSE_MATH)"
-  "ix86_expand_fp_absneg_operator (NEG, DFmode, operands); DONE;")
-
-(define_expand "absdf2"
-  [(set (match_operand:DF 0 "nonimmediate_operand" "")
-	(abs:DF (match_operand:DF 1 "nonimmediate_operand" "")))]
-  "TARGET_80387 || (TARGET_SSE2 && TARGET_SSE_MATH)"
-  "ix86_expand_fp_absneg_operator (ABS, DFmode, operands); DONE;")
-
-(define_insn "*absnegdf2_mixed"
-  [(set (match_operand:DF 0 "nonimmediate_operand"    "=x,x,f,rm")
-	(match_operator:DF 3 "absneg_operator"
-	  [(match_operand:DF 1 "nonimmediate_operand" "0 ,x,0,0")]))
-   (use (match_operand:V2DF 2 "nonimmediate_operand"  "xm,0,X,X"))
-   (clobber (reg:CC FLAGS_REG))]
-  "TARGET_SSE2 && TARGET_SSE_MATH && TARGET_MIX_SSE_I387
-   && ix86_unary_operator_ok (GET_CODE (operands[3]), DFmode, operands)"
-  "#")
+(define_expand "<code><mode>2"
+  [(set (match_operand:X87MODEF 0 "register_operand" "")
+	(absneg:X87MODEF (match_operand:X87MODEF 1 "register_operand" "")))]
+  "TARGET_80387 || (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)"
+  "ix86_expand_fp_absneg_operator (<CODE>, <MODE>mode, operands); DONE;")
 
-(define_insn "*absnegdf2_sse"
-  [(set (match_operand:DF 0 "nonimmediate_operand"    "=x,x,rm")
-	(match_operator:DF 3 "absneg_operator"
-	  [(match_operand:DF 1 "nonimmediate_operand" "0 ,x,0 ")]))
-   (use (match_operand:V2DF 2 "nonimmediate_operand"  "xm,0,X "))
+(define_insn "*absneg<mode>2_mixed"
+  [(set (match_operand:MODEF 0 "register_operand" "=x,x,f,!r")
+	(match_operator:MODEF 3 "absneg_operator"
+	  [(match_operand:MODEF 1 "register_operand" "0,x,0,0")]))
+   (use (match_operand:<ssevecmode> 2 "nonimmediate_operand" "xm,0,X,X"))
    (clobber (reg:CC FLAGS_REG))]
-  "TARGET_SSE2 && TARGET_SSE_MATH
-   && ix86_unary_operator_ok (GET_CODE (operands[3]), DFmode, operands)"
+  "TARGET_MIX_SSE_I387 && SSE_FLOAT_MODE_P (<MODE>mode)"
   "#")
 
-(define_insn "*absnegdf2_i387"
-  [(set (match_operand:DF 0 "nonimmediate_operand" "=f,rm")
-	(match_operator:DF 3 "absneg_operator"
-	  [(match_operand:DF 1 "nonimmediate_operand" "0,0")]))
-   (use (match_operand 2 "" ""))
+(define_insn "*absneg<mode>2_sse"
+  [(set (match_operand:MODEF 0 "register_operand" "=x,x,!r")
+	(match_operator:MODEF 3 "absneg_operator"
+	  [(match_operand:MODEF 1 "register_operand" "0 ,x,0")]))
+   (use (match_operand:<ssevecmode> 2 "register_operand" "xm,0,X"))
    (clobber (reg:CC FLAGS_REG))]
-  "TARGET_80387 && !(TARGET_SSE2 && TARGET_SSE_MATH)
-   && ix86_unary_operator_ok (GET_CODE (operands[3]), DFmode, operands)"
+  "SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH"
   "#")
 
-(define_expand "negxf2"
-  [(set (match_operand:XF 0 "nonimmediate_operand" "")
-	(neg:XF (match_operand:XF 1 "nonimmediate_operand" "")))]
-  "TARGET_80387"
-  "ix86_expand_fp_absneg_operator (NEG, XFmode, operands); DONE;")
-
-(define_expand "absxf2"
-  [(set (match_operand:XF 0 "nonimmediate_operand" "")
-	(abs:XF (match_operand:XF 1 "nonimmediate_operand" "")))]
-  "TARGET_80387"
-  "ix86_expand_fp_absneg_operator (ABS, XFmode, operands); DONE;")
-
-(define_insn "*absnegxf2_i387"
-  [(set (match_operand:XF 0 "nonimmediate_operand" "=f,?rm")
-	(match_operator:XF 3 "absneg_operator"
-	  [(match_operand:XF 1 "nonimmediate_operand" "0,0")]))
+(define_insn "*absneg<mode>2_i387"
+  [(set (match_operand:X87MODEF 0 "register_operand" "=f,!r")
+	(match_operator:X87MODEF 3 "absneg_operator"
+	  [(match_operand:X87MODEF 1 "register_operand" "0,0")]))
    (use (match_operand 2 "" ""))
    (clobber (reg:CC FLAGS_REG))]
-  "TARGET_80387
-   && ix86_unary_operator_ok (GET_CODE (operands[3]), XFmode, operands)"
+  "TARGET_80387 && !(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)"
   "#")
 
-(define_expand "negtf2"
-  [(set (match_operand:TF 0 "nonimmediate_operand" "")
-	(neg:TF (match_operand:TF 1 "nonimmediate_operand" "")))]
-  "TARGET_64BIT"
-  "ix86_expand_fp_absneg_operator (NEG, TFmode, operands); DONE;")
-
-(define_expand "abstf2"
-  [(set (match_operand:TF 0 "nonimmediate_operand" "")
-	(abs:TF (match_operand:TF 1 "nonimmediate_operand" "")))]
+(define_expand "<code>tf2"
+  [(set (match_operand:TF 0 "register_operand" "")
+	(absneg:TF (match_operand:TF 1 "register_operand" "")))]
   "TARGET_64BIT"
-  "ix86_expand_fp_absneg_operator (ABS, TFmode, operands); DONE;")
+  "ix86_expand_fp_absneg_operator (<CODE>, TFmode, operands); DONE;")
 
 (define_insn "*absnegtf2_sse"
-  [(set (match_operand:TF 0 "nonimmediate_operand"    "=x,x,m")
+  [(set (match_operand:TF 0 "register_operand" "=x,x")
 	(match_operator:TF 3 "absneg_operator"
-	  [(match_operand:TF 1 "nonimmediate_operand" "0, x,0")]))
-   (use (match_operand:TF 2 "nonimmediate_operand"    "xm,0,X"))
+	  [(match_operand:TF 1 "register_operand" "0,x")]))
+   (use (match_operand:TF 2 "nonimmediate_operand" "xm,0"))
    (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT
-   && ix86_unary_operator_ok (GET_CODE (operands[3]), TFmode, operands)"
+  "TARGET_64BIT"
   "#")
 
 ;; Splitters for fp abs and neg.
@@ -9951,141 +10560,49 @@
   operands[1] = tmp;
 })
 
-(define_split
-  [(set (match_operand 0 "memory_operand" "")
-	(match_operator 1 "absneg_operator" [(match_dup 0)]))
-   (use (match_operand 2 "" ""))
-   (clobber (reg:CC FLAGS_REG))]
-  "reload_completed"
-  [(parallel [(set (match_dup 0) (match_dup 1))
-	      (clobber (reg:CC FLAGS_REG))])]
-{
-  enum machine_mode mode = GET_MODE (operands[0]);
-  int size = mode == XFmode ? 10 : GET_MODE_SIZE (mode);
-  rtx tmp;
-
-  operands[0] = adjust_address (operands[0], QImode, size - 1);
-  if (GET_CODE (operands[1]) == ABS)
-    {
-      tmp = gen_int_mode (0x7f, QImode);
-      tmp = gen_rtx_AND (QImode, operands[0], tmp);
-    }
-  else
-    {
-      tmp = gen_int_mode (0x80, QImode);
-      tmp = gen_rtx_XOR (QImode, operands[0], tmp);
-    }
-  operands[1] = tmp;
-})
-
 ;; Conditionalize these after reload. If they match before reload, we
 ;; lose the clobber and ability to use integer instructions.
 
-(define_insn "*negsf2_1"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-	(neg:SF (match_operand:SF 1 "register_operand" "0")))]
-  "TARGET_80387 && (reload_completed || !TARGET_SSE_MATH)"
-  "fchs"
-  [(set_attr "type" "fsgn")
-   (set_attr "mode" "SF")])
-
-(define_insn "*negdf2_1"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-	(neg:DF (match_operand:DF 1 "register_operand" "0")))]
-  "TARGET_80387 && (reload_completed || !(TARGET_SSE2 && TARGET_SSE_MATH))"
-  "fchs"
-  [(set_attr "type" "fsgn")
-   (set_attr "mode" "DF")])
-
-(define_insn "*negxf2_1"
-  [(set (match_operand:XF 0 "register_operand" "=f")
-	(neg:XF (match_operand:XF 1 "register_operand" "0")))]
-  "TARGET_80387"
-  "fchs"
-  [(set_attr "type" "fsgn")
-   (set_attr "mode" "XF")])
-
-(define_insn "*abssf2_1"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-	(abs:SF (match_operand:SF 1 "register_operand" "0")))]
-  "TARGET_80387 && (reload_completed || !TARGET_SSE_MATH)"
-  "fabs"
-  [(set_attr "type" "fsgn")
-   (set_attr "mode" "SF")])
-
-(define_insn "*absdf2_1"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-	(abs:DF (match_operand:DF 1 "register_operand" "0")))]
-  "TARGET_80387 && (reload_completed || !(TARGET_SSE2 && TARGET_SSE_MATH))"
-  "fabs"
-  [(set_attr "type" "fsgn")
-   (set_attr "mode" "DF")])
-
-(define_insn "*absxf2_1"
-  [(set (match_operand:XF 0 "register_operand" "=f")
-	(abs:XF (match_operand:XF 1 "register_operand" "0")))]
-  "TARGET_80387"
-  "fabs"
-  [(set_attr "type" "fsgn")
-   (set_attr "mode" "DF")])
-
-(define_insn "*negextendsfdf2"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-	(neg:DF (float_extend:DF
-		  (match_operand:SF 1 "register_operand" "0"))))]
-  "TARGET_80387 && (!TARGET_SSE_MATH || TARGET_MIX_SSE_I387)"
-  "fchs"
-  [(set_attr "type" "fsgn")
-   (set_attr "mode" "DF")])
-
-(define_insn "*negextenddfxf2"
-  [(set (match_operand:XF 0 "register_operand" "=f")
-	(neg:XF (float_extend:XF
-		  (match_operand:DF 1 "register_operand" "0"))))]
-  "TARGET_80387"
-  "fchs"
-  [(set_attr "type" "fsgn")
-   (set_attr "mode" "XF")])
-
-(define_insn "*negextendsfxf2"
-  [(set (match_operand:XF 0 "register_operand" "=f")
-	(neg:XF (float_extend:XF
-		  (match_operand:SF 1 "register_operand" "0"))))]
-  "TARGET_80387"
-  "fchs"
+(define_insn "*<code><mode>2_1"
+  [(set (match_operand:X87MODEF 0 "register_operand" "=f")
+	(absneg:X87MODEF (match_operand:X87MODEF 1 "register_operand" "0")))]
+  "TARGET_80387
+   && (reload_completed
+       || !(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH))"
+  "f<absnegprefix>"
   [(set_attr "type" "fsgn")
-   (set_attr "mode" "XF")])
+   (set_attr "mode" "<MODE>")])
 
-(define_insn "*absextendsfdf2"
+(define_insn "*<code>extendsfdf2"
   [(set (match_operand:DF 0 "register_operand" "=f")
-	(abs:DF (float_extend:DF
-		  (match_operand:SF 1 "register_operand" "0"))))]
+	(absneg:DF (float_extend:DF
+		     (match_operand:SF 1 "register_operand" "0"))))]
   "TARGET_80387 && (!TARGET_SSE_MATH || TARGET_MIX_SSE_I387)"
-  "fabs"
+  "f<absnegprefix>"
   [(set_attr "type" "fsgn")
    (set_attr "mode" "DF")])
 
-(define_insn "*absextenddfxf2"
+(define_insn "*<code>extendsfxf2"
   [(set (match_operand:XF 0 "register_operand" "=f")
-	(abs:XF (float_extend:XF
-	  (match_operand:DF 1 "register_operand" "0"))))]
+	(absneg:XF (float_extend:XF
+		     (match_operand:SF 1 "register_operand" "0"))))]
   "TARGET_80387"
-  "fabs"
+  "f<absnegprefix>"
   [(set_attr "type" "fsgn")
    (set_attr "mode" "XF")])
 
-(define_insn "*absextendsfxf2"
+(define_insn "*<code>extenddfxf2"
   [(set (match_operand:XF 0 "register_operand" "=f")
-	(abs:XF (float_extend:XF
-	  (match_operand:SF 1 "register_operand" "0"))))]
+	(absneg:XF (float_extend:XF
+		      (match_operand:DF 1 "register_operand" "0"))))]
   "TARGET_80387"
-  "fabs"
+  "f<absnegprefix>"
   [(set_attr "type" "fsgn")
    (set_attr "mode" "XF")])
 
 ;; Copysign instructions
 
-(define_mode_macro CSGNMODE [SF DF TF])
+(define_mode_iterator CSGNMODE [SF DF TF])
 (define_mode_attr CSGNVMODE [(SF "V4SF") (DF "V2DF") (TF "TF")])
 
 (define_expand "copysign<mode>3"
@@ -10463,7 +10980,7 @@
    (set_attr "prefix_0f" "1")
    (set_attr "mode" "DI")
    (set_attr "athlon_decode" "vector")
-   (set_attr "amdfam10_decode" "vector")])   
+   (set_attr "amdfam10_decode" "vector")])
 
 (define_expand "x86_64_shift_adj"
   [(set (reg:CCZ FLAGS_REG)
@@ -10681,7 +11198,7 @@
    (set_attr "mode" "SI")
    (set_attr "pent_pair" "np")
    (set_attr "athlon_decode" "vector")
-   (set_attr "amdfam10_decode" "vector")])   
+   (set_attr "amdfam10_decode" "vector")])
 
 (define_expand "x86_shift_adj_1"
   [(set (reg:CCZ FLAGS_REG)
@@ -11443,7 +11960,7 @@
    (set_attr "prefix_0f" "1")
    (set_attr "mode" "DI")
    (set_attr "athlon_decode" "vector")
-   (set_attr "amdfam10_decode" "vector")])   
+   (set_attr "amdfam10_decode" "vector")])
 
 (define_expand "ashrdi3"
   [(set (match_operand:DI 0 "shiftdi_operand" "")
@@ -13345,120 +13862,24 @@
 
 ;; Store-flag instructions.
 
-;; For all sCOND expanders, also expand the compare or test insn that
-;; generates cc0.  Generate an equality comparison if `seq' or `sne'.
-
-;; %%% Do the expansion to SImode.  If PII, do things the xor+setcc way
-;; to avoid partial register stalls.  Otherwise do things the setcc+movzx
-;; way, which can later delete the movzx if only QImode is needed.
-
-(define_expand "seq"
-  [(set (match_operand:QI 0 "register_operand" "")
-        (eq:QI (reg:CC FLAGS_REG) (const_int 0)))]
-  ""
-  "if (ix86_expand_setcc (EQ, operands[0])) DONE; else FAIL;")
-
-(define_expand "sne"
-  [(set (match_operand:QI 0 "register_operand" "")
-        (ne:QI (reg:CC FLAGS_REG) (const_int 0)))]
-  ""
-  "if (ix86_expand_setcc (NE, operands[0])) DONE; else FAIL;")
-
-(define_expand "sgt"
-  [(set (match_operand:QI 0 "register_operand" "")
-        (gt:QI (reg:CC FLAGS_REG) (const_int 0)))]
-  ""
-  "if (ix86_expand_setcc (GT, operands[0])) DONE; else FAIL;")
-
-(define_expand "sgtu"
-  [(set (match_operand:QI 0 "register_operand" "")
-        (gtu:QI (reg:CC FLAGS_REG) (const_int 0)))]
-  ""
-  "if (ix86_expand_setcc (GTU, operands[0])) DONE; else FAIL;")
-
-(define_expand "slt"
-  [(set (match_operand:QI 0 "register_operand" "")
-        (lt:QI (reg:CC FLAGS_REG) (const_int 0)))]
-  ""
-  "if (ix86_expand_setcc (LT, operands[0])) DONE; else FAIL;")
-
-(define_expand "sltu"
-  [(set (match_operand:QI 0 "register_operand" "")
-        (ltu:QI (reg:CC FLAGS_REG) (const_int 0)))]
-  ""
-  "if (ix86_expand_setcc (LTU, operands[0])) DONE; else FAIL;")
-
-(define_expand "sge"
-  [(set (match_operand:QI 0 "register_operand" "")
-        (ge:QI (reg:CC FLAGS_REG) (const_int 0)))]
-  ""
-  "if (ix86_expand_setcc (GE, operands[0])) DONE; else FAIL;")
-
-(define_expand "sgeu"
-  [(set (match_operand:QI 0 "register_operand" "")
-        (geu:QI (reg:CC FLAGS_REG) (const_int 0)))]
-  ""
-  "if (ix86_expand_setcc (GEU, operands[0])) DONE; else FAIL;")
-
-(define_expand "sle"
-  [(set (match_operand:QI 0 "register_operand" "")
-        (le:QI (reg:CC FLAGS_REG) (const_int 0)))]
-  ""
-  "if (ix86_expand_setcc (LE, operands[0])) DONE; else FAIL;")
-
-(define_expand "sleu"
-  [(set (match_operand:QI 0 "register_operand" "")
-        (leu:QI (reg:CC FLAGS_REG) (const_int 0)))]
-  ""
-  "if (ix86_expand_setcc (LEU, operands[0])) DONE; else FAIL;")
-
-(define_expand "sunordered"
-  [(set (match_operand:QI 0 "register_operand" "")
-        (unordered:QI (reg:CC FLAGS_REG) (const_int 0)))]
-  "TARGET_80387 || TARGET_SSE"
-  "if (ix86_expand_setcc (UNORDERED, operands[0])) DONE; else FAIL;")
-
-(define_expand "sordered"
-  [(set (match_operand:QI 0 "register_operand" "")
-        (ordered:QI (reg:CC FLAGS_REG) (const_int 0)))]
-  "TARGET_80387"
-  "if (ix86_expand_setcc (ORDERED, operands[0])) DONE; else FAIL;")
-
-(define_expand "suneq"
-  [(set (match_operand:QI 0 "register_operand" "")
-        (uneq:QI (reg:CC FLAGS_REG) (const_int 0)))]
-  "TARGET_80387 || TARGET_SSE"
-  "if (ix86_expand_setcc (UNEQ, operands[0])) DONE; else FAIL;")
-
-(define_expand "sunge"
-  [(set (match_operand:QI 0 "register_operand" "")
-        (unge:QI (reg:CC FLAGS_REG) (const_int 0)))]
-  "TARGET_80387 || TARGET_SSE"
-  "if (ix86_expand_setcc (UNGE, operands[0])) DONE; else FAIL;")
-
-(define_expand "sungt"
-  [(set (match_operand:QI 0 "register_operand" "")
-        (ungt:QI (reg:CC FLAGS_REG) (const_int 0)))]
-  "TARGET_80387 || TARGET_SSE"
-  "if (ix86_expand_setcc (UNGT, operands[0])) DONE; else FAIL;")
-
-(define_expand "sunle"
-  [(set (match_operand:QI 0 "register_operand" "")
-        (unle:QI (reg:CC FLAGS_REG) (const_int 0)))]
-  "TARGET_80387 || TARGET_SSE"
-  "if (ix86_expand_setcc (UNLE, operands[0])) DONE; else FAIL;")
+;; For all sCOND expanders, also expand the compare or test insn that
+;; generates cc0.  Generate an equality comparison if `seq' or `sne'.
+
+;; %%% Do the expansion to SImode.  If PII, do things the xor+setcc way
+;; to avoid partial register stalls.  Otherwise do things the setcc+movzx
+;; way, which can later delete the movzx if only QImode is needed.
 
-(define_expand "sunlt"
+(define_expand "s<code>"
   [(set (match_operand:QI 0 "register_operand" "")
-        (unlt:QI (reg:CC FLAGS_REG) (const_int 0)))]
-  "TARGET_80387 || TARGET_SSE"
-  "if (ix86_expand_setcc (UNLT, operands[0])) DONE; else FAIL;")
+        (int_cond:QI (reg:CC FLAGS_REG) (const_int 0)))]
+  ""
+  "if (ix86_expand_setcc (<CODE>, operands[0])) DONE; else FAIL;")
 
-(define_expand "sltgt"
+(define_expand "s<code>"
   [(set (match_operand:QI 0 "register_operand" "")
-        (ltgt:QI (reg:CC FLAGS_REG) (const_int 0)))]
+        (fp_cond:QI (reg:CC FLAGS_REG) (const_int 0)))]
   "TARGET_80387 || TARGET_SSE"
-  "if (ix86_expand_setcc (LTGT, operands[0])) DONE; else FAIL;")
+  "if (ix86_expand_setcc (<CODE>, operands[0])) DONE; else FAIL;")
 
 (define_insn "*setcc_1"
   [(set (match_operand:QI 0 "nonimmediate_operand" "=qm")
@@ -13553,25 +13974,26 @@
 ;; 0xffffffff is NaN, but not in normalized form, so we can't represent
 ;; it directly.
 
-(define_insn "*sse_setccsf"
-  [(set (match_operand:SF 0 "register_operand" "=x")
-	(match_operator:SF 1 "sse_comparison_operator"
-	  [(match_operand:SF 2 "register_operand" "0")
-	   (match_operand:SF 3 "nonimmediate_operand" "xm")]))]
-  "TARGET_SSE"
-  "cmp%D1ss\t{%3, %0|%0, %3}"
+(define_insn "*sse_setcc<mode>"
+  [(set (match_operand:MODEF 0 "register_operand" "=x")
+	(match_operator:MODEF 1 "sse_comparison_operator"
+	  [(match_operand:MODEF 2 "register_operand" "0")
+	   (match_operand:MODEF 3 "nonimmediate_operand" "xm")]))]
+  "SSE_FLOAT_MODE_P (<MODE>mode) && !TARGET_SSE5"
+  "cmp%D1s<ssemodefsuffix>\t{%3, %0|%0, %3}"
   [(set_attr "type" "ssecmp")
-   (set_attr "mode" "SF")])
+   (set_attr "mode" "<MODE>")])
+
+(define_insn "*sse5_setcc<mode>"
+  [(set (match_operand:MODEF 0 "register_operand" "=x")
+	(match_operator:MODEF 1 "sse5_comparison_float_operator"
+	  [(match_operand:MODEF 2 "register_operand" "x")
+	   (match_operand:MODEF 3 "nonimmediate_operand" "xm")]))]
+  "TARGET_SSE5"
+  "com%Y1s<ssemodefsuffix>\t{%3, %2, %0|%0, %2, %3}"
+  [(set_attr "type" "sse4arg")
+   (set_attr "mode" "<MODE>")])
 
-(define_insn "*sse_setccdf"
-  [(set (match_operand:DF 0 "register_operand" "=x")
-	(match_operator:DF 1 "sse_comparison_operator"
-	  [(match_operand:DF 2 "register_operand" "0")
-	   (match_operand:DF 3 "nonimmediate_operand" "xm")]))]
-  "TARGET_SSE2"
-  "cmp%D1sd\t{%3, %0|%0, %3}"
-  [(set_attr "type" "ssecmp")
-   (set_attr "mode" "DF")])
 
 ;; Basic conditional jump instructions.
 ;; We ignore the overflow flag for signed branch instructions.
@@ -13579,149 +14001,23 @@
 ;; For all bCOND expanders, also expand the compare or test insn that
 ;; generates reg FLAGS_REG.  Generate an equality comparison if `beq' or `bne'.
 
-(define_expand "beq"
-  [(set (pc)
-	(if_then_else (match_dup 1)
-		      (label_ref (match_operand 0 "" ""))
-		      (pc)))]
-  ""
-  "ix86_expand_branch (EQ, operands[0]); DONE;")
-
-(define_expand "bne"
-  [(set (pc)
-	(if_then_else (match_dup 1)
-		      (label_ref (match_operand 0 "" ""))
-		      (pc)))]
-  ""
-  "ix86_expand_branch (NE, operands[0]); DONE;")
-
-(define_expand "bgt"
-  [(set (pc)
-	(if_then_else (match_dup 1)
-		      (label_ref (match_operand 0 "" ""))
-		      (pc)))]
-  ""
-  "ix86_expand_branch (GT, operands[0]); DONE;")
-
-(define_expand "bgtu"
-  [(set (pc)
-	(if_then_else (match_dup 1)
-		      (label_ref (match_operand 0 "" ""))
-		      (pc)))]
-  ""
-  "ix86_expand_branch (GTU, operands[0]); DONE;")
-
-(define_expand "blt"
-  [(set (pc)
-	(if_then_else (match_dup 1)
-		      (label_ref (match_operand 0 "" ""))
-		      (pc)))]
-  ""
-  "ix86_expand_branch (LT, operands[0]); DONE;")
-
-(define_expand "bltu"
-  [(set (pc)
-	(if_then_else (match_dup 1)
-		      (label_ref (match_operand 0 "" ""))
-		      (pc)))]
-  ""
-  "ix86_expand_branch (LTU, operands[0]); DONE;")
-
-(define_expand "bge"
-  [(set (pc)
-	(if_then_else (match_dup 1)
-		      (label_ref (match_operand 0 "" ""))
-		      (pc)))]
-  ""
-  "ix86_expand_branch (GE, operands[0]); DONE;")
-
-(define_expand "bgeu"
-  [(set (pc)
-	(if_then_else (match_dup 1)
-		      (label_ref (match_operand 0 "" ""))
-		      (pc)))]
-  ""
-  "ix86_expand_branch (GEU, operands[0]); DONE;")
-
-(define_expand "ble"
-  [(set (pc)
-	(if_then_else (match_dup 1)
-		      (label_ref (match_operand 0 "" ""))
-		      (pc)))]
-  ""
-  "ix86_expand_branch (LE, operands[0]); DONE;")
-
-(define_expand "bleu"
+(define_expand "b<code>"
   [(set (pc)
-	(if_then_else (match_dup 1)
-		      (label_ref (match_operand 0 "" ""))
+	(if_then_else (int_cond:CC (reg:CC FLAGS_REG)
+				   (const_int 0))
+		      (label_ref (match_operand 0 ""))
 		      (pc)))]
   ""
-  "ix86_expand_branch (LEU, operands[0]); DONE;")
-
-(define_expand "bunordered"
-  [(set (pc)
-	(if_then_else (match_dup 1)
-		      (label_ref (match_operand 0 "" ""))
-		      (pc)))]
-  "TARGET_80387 || TARGET_SSE_MATH"
-  "ix86_expand_branch (UNORDERED, operands[0]); DONE;")
-
-(define_expand "bordered"
-  [(set (pc)
-	(if_then_else (match_dup 1)
-		      (label_ref (match_operand 0 "" ""))
-		      (pc)))]
-  "TARGET_80387 || TARGET_SSE_MATH"
-  "ix86_expand_branch (ORDERED, operands[0]); DONE;")
-
-(define_expand "buneq"
-  [(set (pc)
-	(if_then_else (match_dup 1)
-		      (label_ref (match_operand 0 "" ""))
-		      (pc)))]
-  "TARGET_80387 || TARGET_SSE_MATH"
-  "ix86_expand_branch (UNEQ, operands[0]); DONE;")
-
-(define_expand "bunge"
-  [(set (pc)
-	(if_then_else (match_dup 1)
-		      (label_ref (match_operand 0 "" ""))
-		      (pc)))]
-  "TARGET_80387 || TARGET_SSE_MATH"
-  "ix86_expand_branch (UNGE, operands[0]); DONE;")
-
-(define_expand "bungt"
-  [(set (pc)
-	(if_then_else (match_dup 1)
-		      (label_ref (match_operand 0 "" ""))
-		      (pc)))]
-  "TARGET_80387 || TARGET_SSE_MATH"
-  "ix86_expand_branch (UNGT, operands[0]); DONE;")
-
-(define_expand "bunle"
-  [(set (pc)
-	(if_then_else (match_dup 1)
-		      (label_ref (match_operand 0 "" ""))
-		      (pc)))]
-  "TARGET_80387 || TARGET_SSE_MATH"
-  "ix86_expand_branch (UNLE, operands[0]); DONE;")
-
-(define_expand "bunlt"
-  [(set (pc)
-	(if_then_else (match_dup 1)
-		      (label_ref (match_operand 0 "" ""))
-		      (pc)))]
-  "TARGET_80387 || TARGET_SSE_MATH"
-  "ix86_expand_branch (UNLT, operands[0]); DONE;")
+  "ix86_expand_branch (<CODE>, operands[0]); DONE;")
 
-(define_expand "bltgt"
+(define_expand "b<code>"
   [(set (pc)
-	(if_then_else (match_dup 1)
-		      (label_ref (match_operand 0 "" ""))
+	(if_then_else (fp_cond:CC (reg:CC FLAGS_REG)
+				  (const_int 0))
+		      (label_ref (match_operand 0 ""))
 		      (pc)))]
   "TARGET_80387 || TARGET_SSE_MATH"
-  "ix86_expand_branch (LTGT, operands[0]); DONE;")
+  "ix86_expand_branch (<CODE>, operands[0]); DONE;")
 
 (define_insn "*jcc_1"
   [(set (pc)
@@ -14109,7 +14405,7 @@
    (set_attr "modrm" "0")])
 
 (define_expand "indirect_jump"
-  [(set (pc) (match_operand 0 "nonimmediate_operand" "rm"))]
+  [(set (pc) (match_operand 0 "nonimmediate_operand" ""))]
   ""
   "")
 
@@ -14128,7 +14424,7 @@
    (set_attr "length_immediate" "0")])
 
 (define_expand "tablejump"
-  [(parallel [(set (pc) (match_operand 0 "nonimmediate_operand" "rm"))
+  [(parallel [(set (pc) (match_operand 0 "nonimmediate_operand" ""))
 	      (use (label_ref (match_operand 1 "" "")))])]
   ""
 {
@@ -14363,7 +14659,7 @@
   [(call (mem:QI (reg:DI R11_REG))
 	 (match_operand 0 "" ""))]
   "SIBLING_CALL_P (insn) && TARGET_64BIT"
-  "jmp\t*%%r11"
+  "jmp\t{*%%}r11"
   [(set_attr "type" "call")])
 
 
@@ -14470,9 +14766,9 @@
   [(return)]
   "ix86_can_use_return_insn_p ()"
 {
-  if (current_function_pops_args)
+  if (crtl->args.pops_args)
     {
-      rtx popc = GEN_INT (current_function_pops_args);
+      rtx popc = GEN_INT (crtl->args.pops_args);
       emit_jump_insn (gen_return_pop_internal (popc));
       DONE;
     }
@@ -14493,7 +14789,7 @@
   [(return)
    (unspec [(const_int 0)] UNSPEC_REP)]
   "reload_completed"
-  "rep{\;| }ret"
+  "rep\;ret"
   [(set_attr "length" "1")
    (set_attr "length_immediate" "0")
    (set_attr "prefix_rep" "1")
@@ -14545,7 +14841,7 @@
   [(set_attr "length" "16")])
 
 (define_expand "prologue"
-  [(const_int 1)]
+  [(const_int 0)]
   ""
   "ix86_expand_prologue (); DONE;")
 
@@ -14572,7 +14868,7 @@
   [(set (match_operand:DI 0 "register_operand" "=r")
 	(unspec:DI [(const_int 0)] UNSPEC_SET_GOT))]
   "TARGET_64BIT"
-  "lea{q}\t_GLOBAL_OFFSET_TABLE_(%%rip), %0"
+  "lea{q}\t{_GLOBAL_OFFSET_TABLE_(%%rip), %0|%0, _GLOBAL_OFFSET_TABLE_[rip]}"
   [(set_attr "type" "lea")
    (set_attr "length" "6")])
 
@@ -14580,7 +14876,7 @@
   [(set (match_operand:DI 0 "register_operand" "=r")
 	(unspec:DI [(match_operand:DI 1 "" "")] UNSPEC_SET_RIP))]
   "TARGET_64BIT"
-  "lea{q}\t%l1(%%rip), %0"
+  "lea{q}\t{%l1(%%rip), %0|%0, %l1[rip]}"
   [(set_attr "type" "lea")
    (set_attr "length" "6")])
 
@@ -14588,17 +14884,17 @@
   [(set (match_operand:DI 0 "register_operand" "=r")
 	(unspec:DI [(match_operand:DI 1 "" "")] UNSPEC_SET_GOT_OFFSET))]
   "TARGET_64BIT"
-  "movabs{q}\t$_GLOBAL_OFFSET_TABLE_-%l1, %0"
+  "movabs{q}\t{$_GLOBAL_OFFSET_TABLE_-%l1, %0|%0, OFFSET FLAT:_GLOBAL_OFFSET_TABLE_-%l1}"
   [(set_attr "type" "imov")
    (set_attr "length" "11")])
 
 (define_expand "epilogue"
-  [(const_int 1)]
+  [(const_int 0)]
   ""
   "ix86_expand_epilogue (1); DONE;")
 
 (define_expand "sibcall_epilogue"
-  [(const_int 1)]
+  [(const_int 0)]
   ""
   "ix86_expand_epilogue (0); DONE;")
 
@@ -14631,7 +14927,7 @@
   "!TARGET_64BIT"
   "#"
   "reload_completed"
-  [(const_int 1)]
+  [(const_int 0)]
   "ix86_expand_epilogue (2); DONE;")
 
 (define_insn_and_split "eh_return_di"
@@ -14641,7 +14937,7 @@
   "TARGET_64BIT"
   "#"
   "reload_completed"
-  [(const_int 1)]
+  [(const_int 0)]
   "ix86_expand_epilogue (2); DONE;")
 
 (define_insn "leave"
@@ -15018,7 +15314,7 @@
 
 (define_expand "paritydi2"
   [(set (match_operand:DI 0 "register_operand" "")
-	(parity:DI (match_operand:DI 1 "nonimmediate_operand" "")))]
+	(parity:DI (match_operand:DI 1 "register_operand" "")))]
   "! TARGET_POPCNT"
 {
   rtx scratch = gen_reg_rtx (QImode);
@@ -15046,10 +15342,10 @@
 
 (define_insn_and_split "paritydi2_cmp"
   [(set (reg:CC FLAGS_REG)
-	(parity:CC (match_operand:DI 3 "nonimmediate_operand" "0,m")))
-   (clobber (match_scratch:DI 0 "=r,X"))
-   (clobber (match_scratch:SI 1 "=r,r"))
-   (clobber (match_scratch:HI 2 "=Q,Q"))]
+	(parity:CC (match_operand:DI 3 "register_operand" "0")))
+   (clobber (match_scratch:DI 0 "=r"))
+   (clobber (match_scratch:SI 1 "=&r"))
+   (clobber (match_scratch:HI 2 "=Q"))]
   "! TARGET_POPCNT"
   "#"
   "&& reload_completed"
@@ -15065,20 +15361,18 @@
 {
   operands[4] = gen_lowpart (SImode, operands[3]);
 
-  if (MEM_P (operands[3]))
-    emit_move_insn (operands[1], gen_highpart (SImode, operands[3]));
-  else if (! TARGET_64BIT)
-    operands[1] = gen_highpart (SImode, operands[3]);
-  else
+  if (TARGET_64BIT)
     {
       emit_move_insn (operands[1], gen_lowpart (SImode, operands[3]));
       emit_insn (gen_lshrdi3 (operands[3], operands[3], GEN_INT (32)));
     }
+  else
+    operands[1] = gen_highpart (SImode, operands[3]);
 })
 
 (define_expand "paritysi2"
   [(set (match_operand:SI 0 "register_operand" "")
-	(parity:SI (match_operand:SI 1 "nonimmediate_operand" "")))]
+	(parity:SI (match_operand:SI 1 "register_operand" "")))]
   "! TARGET_POPCNT"
 {
   rtx scratch = gen_reg_rtx (QImode);
@@ -15097,9 +15391,9 @@
 
 (define_insn_and_split "paritysi2_cmp"
   [(set (reg:CC FLAGS_REG)
-	(parity:CC (match_operand:SI 2 "nonimmediate_operand" "0,m")))
-   (clobber (match_scratch:SI 0 "=r,X"))
-   (clobber (match_scratch:HI 1 "=Q,Q"))]
+	(parity:CC (match_operand:SI 2 "register_operand" "0")))
+   (clobber (match_scratch:SI 0 "=r"))
+   (clobber (match_scratch:HI 1 "=&Q"))]
   "! TARGET_POPCNT"
   "#"
   "&& reload_completed"
@@ -15114,13 +15408,8 @@
 {
   operands[3] = gen_lowpart (HImode, operands[2]);
 
-  if (MEM_P (operands[2]))
-    emit_move_insn (operands[1], gen_highpart (HImode, operands[2]));
-  else
-    {
-      emit_move_insn (operands[1], gen_lowpart (HImode, operands[2]));
-      emit_insn (gen_lshrsi3 (operands[2], operands[2], GEN_INT (16)));
-    }
+  emit_move_insn (operands[1], gen_lowpart (HImode, operands[2]));
+  emit_insn (gen_lshrsi3 (operands[2], operands[2], GEN_INT (16)));
 })
 
 (define_insn "*parityhi2_cmp"
@@ -15209,7 +15498,7 @@
    (unspec:DI [(match_operand:DI 1 "tls_symbolic_operand" "")]
 	      UNSPEC_TLS_GD)]
   "TARGET_64BIT"
-  ".byte\t0x66\;lea{q}\t{%a1@TLSGD(%%rip), %%rdi|%%rdi, %a1@TLSGD[%%rip]}\;.word\t0x6666\;rex64\;call\t%P2"
+  ".byte\t0x66\;lea{q}\t{%a1@TLSGD(%%rip), %%rdi|rdi, %a1@TLSGD[rip]}\;.word\t0x6666\;rex64\;call\t%P2"
   [(set_attr "type" "multi")
    (set_attr "length" "16")])
 
@@ -15287,7 +15576,7 @@
 		 (match_operand:DI 2 "" "")))
    (unspec:DI [(const_int 0)] UNSPEC_TLS_LD_BASE)]
   "TARGET_64BIT"
-  "lea{q}\t{%&@TLSLD(%%rip), %%rdi|%%rdi, %&@TLSLD[%%rip]}\;call\t%P1"
+  "lea{q}\t{%&@TLSLD(%%rip), %%rdi|rdi, %&@TLSLD[rip]}\;call\t%P1"
   [(set_attr "type" "multi")
    (set_attr "length" "12")])
 
@@ -15337,7 +15626,7 @@
   [(set (match_operand:SI 0 "register_operand" "=r")
 	(unspec:SI [(const_int 0)] UNSPEC_TP))]
   "!TARGET_64BIT"
-  "mov{l}\t{%%gs:0, %0|%0, DWORD PTR %%gs:0}"
+  "mov{l}\t{%%gs:0, %0|%0, DWORD PTR gs:0}"
   [(set_attr "type" "imov")
    (set_attr "modrm" "0")
    (set_attr "length" "7")
@@ -15350,7 +15639,7 @@
 		 (match_operand:SI 1 "register_operand" "0")))
    (clobber (reg:CC FLAGS_REG))]
   "!TARGET_64BIT"
-  "add{l}\t{%%gs:0, %0|%0, DWORD PTR %%gs:0}"
+  "add{l}\t{%%gs:0, %0|%0, DWORD PTR gs:0}"
   [(set_attr "type" "alu")
    (set_attr "modrm" "0")
    (set_attr "length" "7")
@@ -15361,7 +15650,7 @@
   [(set (match_operand:DI 0 "register_operand" "=r")
 	(unspec:DI [(const_int 0)] UNSPEC_TP))]
   "TARGET_64BIT"
-  "mov{q}\t{%%fs:0, %0|%0, QWORD PTR %%fs:0}"
+  "mov{q}\t{%%fs:0, %0|%0, QWORD PTR fs:0}"
   [(set_attr "type" "imov")
    (set_attr "modrm" "0")
    (set_attr "length" "7")
@@ -15374,7 +15663,7 @@
 		 (match_operand:DI 1 "register_operand" "0")))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_64BIT"
-  "add{q}\t{%%fs:0, %0|%0, QWORD PTR %%fs:0}"
+  "add{q}\t{%%fs:0, %0|%0, QWORD PTR fs:0}"
   [(set_attr "type" "alu")
    (set_attr "modrm" "0")
    (set_attr "length" "7")
@@ -15470,7 +15759,7 @@
 	(unspec:DI [(match_operand:DI 1 "tls_symbolic_operand" "")]
 		   UNSPEC_TLSDESC))]
   "TARGET_64BIT && TARGET_GNU2_TLS"
-  "lea{q}\t{%a1@TLSDESC(%%rip), %0|%0, %a1@TLSDESC[%%rip]}"
+  "lea{q}\t{%a1@TLSDESC(%%rip), %0|%0, %a1@TLSDESC[rip]}"
   [(set_attr "type" "lea")
    (set_attr "mode" "DI")
    (set_attr "length" "7")
@@ -15946,7 +16235,7 @@
   [(set (match_operand:XF 0 "register_operand" "=f,f")
 	(match_operator:XF 3 "binary_fp_operator"
 	   [(float_extend:XF
-	      (match_operand:X87MODEF12 1 "nonimmediate_operand" "fm,0"))
+	      (match_operand:MODEF 1 "nonimmediate_operand" "fm,0"))
 	    (match_operand:XF 2 "register_operand" "0,f")]))]
   "TARGET_80387"
   "* return output_387_binary_op (insn, operands);"
@@ -15964,7 +16253,7 @@
 	(match_operator:XF 3 "binary_fp_operator"
 	  [(match_operand:XF 1 "register_operand" "0,f")
 	   (float_extend:XF
-	     (match_operand:X87MODEF12 2 "nonimmediate_operand" "fm,0"))]))]
+	     (match_operand:MODEF 2 "nonimmediate_operand" "fm,0"))]))]
   "TARGET_80387"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type")
@@ -15980,9 +16269,9 @@
   [(set (match_operand:XF 0 "register_operand" "=f,f")
 	(match_operator:XF 3 "binary_fp_operator"
 	  [(float_extend:XF
-	     (match_operand:X87MODEF12 1 "register_operand" "0,f"))
+	     (match_operand:MODEF 1 "register_operand" "0,f"))
 	   (float_extend:XF
-	     (match_operand:X87MODEF12 2 "nonimmediate_operand" "fm,0"))]))]
+	     (match_operand:MODEF 2 "nonimmediate_operand" "fm,0"))]))]
   "TARGET_80387"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type")
@@ -16040,8 +16329,8 @@
 ;; all fancy i386 XFmode math functions.
 
 (define_insn "truncxf<mode>2_i387_noop_unspec"
-  [(set (match_operand:X87MODEF12 0 "register_operand" "=f")
-	(unspec:X87MODEF12 [(match_operand:XF 1 "register_operand" "f")]
+  [(set (match_operand:MODEF 0 "register_operand" "=f")
+	(unspec:MODEF [(match_operand:XF 1 "register_operand" "f")]
 	UNSPEC_TRUNC_NOOP))]
   "TARGET_USE_FANCY_MATH_387"
   "* return output_387_reg_move (insn, operands);"
@@ -16062,12 +16351,12 @@
   [(set (match_operand:XF 0 "register_operand" "=f")
 	(sqrt:XF
 	  (float_extend:XF
-	    (match_operand:X87MODEF12 1 "register_operand" "0"))))]
+	    (match_operand:MODEF 1 "register_operand" "0"))))]
   "TARGET_USE_FANCY_MATH_387"
   "fsqrt"
   [(set_attr "type" "fpspc")
    (set_attr "mode" "XF")
-   (set_attr "athlon_decode" "direct")   
+   (set_attr "athlon_decode" "direct")
    (set_attr "amdfam10_decode" "direct")])
 
 (define_insn "*rsqrtsf2_sse"
@@ -16080,21 +16369,19 @@
    (set_attr "mode" "SF")])
 
 (define_expand "rsqrtsf2"
-  [(set (match_operand:SF 0 "register_operand" "=x")
-	(unspec:SF [(match_operand:SF 1 "nonimmediate_operand" "xm")]
+  [(set (match_operand:SF 0 "register_operand" "")
+	(unspec:SF [(match_operand:SF 1 "nonimmediate_operand" "")]
 		   UNSPEC_RSQRT))]
-  "TARGET_SSE_MATH && TARGET_RECIP && !optimize_size
-   && flag_finite_math_only && !flag_trapping_math
-   && flag_unsafe_math_optimizations"
+  "TARGET_SSE_MATH"
 {
   ix86_emit_swsqrtsf (operands[0], operands[1], SFmode, 1);
   DONE;
 })
 
 (define_insn "*sqrt<mode>2_sse"
-  [(set (match_operand:SSEMODEF 0 "register_operand" "=x")
-	(sqrt:SSEMODEF
-	  (match_operand:SSEMODEF 1 "nonimmediate_operand" "xm")))]
+  [(set (match_operand:MODEF 0 "register_operand" "=x")
+	(sqrt:MODEF
+	  (match_operand:MODEF 1 "nonimmediate_operand" "xm")))]
   "SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH"
   "sqrts<ssemodefsuffix>\t{%1, %0|%0, %1}"
   [(set_attr "type" "sse")
@@ -16103,9 +16390,9 @@
    (set_attr "amdfam10_decode" "*")])
 
 (define_expand "sqrt<mode>2"
-  [(set (match_operand:X87MODEF12 0 "register_operand" "")
-	(sqrt:X87MODEF12
-	  (match_operand:X87MODEF12 1 "nonimmediate_operand" "")))]
+  [(set (match_operand:MODEF 0 "register_operand" "")
+	(sqrt:MODEF
+	  (match_operand:MODEF 1 "nonimmediate_operand" "")))]
   "TARGET_USE_FANCY_MATH_387
    || (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)"
 {
@@ -16153,10 +16440,18 @@
 {
   rtx label = gen_label_rtx ();
 
-  emit_label (label);
+  rtx op2;
+
+  if (rtx_equal_p (operands[1], operands[2]))
+    {
+      op2 = gen_reg_rtx (XFmode);
+      emit_move_insn (op2, operands[2]);
+    }
+  else
+    op2 = operands[2];
 
-  emit_insn (gen_fpremxf4_i387 (operands[1], operands[2],
-				operands[1], operands[2]));
+  emit_label (label);
+  emit_insn (gen_fpremxf4_i387 (operands[1], op2, operands[1], op2));
   ix86_emit_fp_unordered_jump (label);
   LABEL_NUSES (label) = 1;
 
@@ -16165,9 +16460,9 @@
 })
 
 (define_expand "fmod<mode>3"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "general_operand" ""))
-   (use (match_operand:X87MODEF12 2 "general_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "general_operand" ""))
+   (use (match_operand:MODEF 2 "general_operand" ""))]
   "TARGET_USE_FANCY_MATH_387"
 {
   rtx label = gen_label_rtx ();
@@ -16217,10 +16512,18 @@
 {
   rtx label = gen_label_rtx ();
 
-  emit_label (label);
+  rtx op2;
+
+  if (rtx_equal_p (operands[1], operands[2]))
+    {
+      op2 = gen_reg_rtx (XFmode);
+      emit_move_insn (op2, operands[2]);
+    }
+  else
+    op2 = operands[2];
 
-  emit_insn (gen_fprem1xf4_i387 (operands[1], operands[2],
-				 operands[1], operands[2]));
+  emit_label (label);
+  emit_insn (gen_fprem1xf4_i387 (operands[1], op2, operands[1], op2));
   ix86_emit_fp_unordered_jump (label);
   LABEL_NUSES (label) = 1;
 
@@ -16229,9 +16532,9 @@
 })
 
 (define_expand "remainder<mode>3"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "general_operand" ""))
-   (use (match_operand:X87MODEF12 2 "general_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "general_operand" ""))
+   (use (match_operand:MODEF 2 "general_operand" ""))]
   "TARGET_USE_FANCY_MATH_387"
 {
   rtx label = gen_label_rtx ();
@@ -16270,7 +16573,7 @@
 (define_insn "*sin_extend<mode>xf2_i387"
   [(set (match_operand:XF 0 "register_operand" "=f")
 	(unspec:XF [(float_extend:XF
-		      (match_operand:X87MODEF12 1 "register_operand" "0"))]
+		      (match_operand:MODEF 1 "register_operand" "0"))]
 		   UNSPEC_SIN))]
   "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
@@ -16292,7 +16595,7 @@
 (define_insn "*cos_extend<mode>xf2_i387"
   [(set (match_operand:XF 0 "register_operand" "=f")
 	(unspec:XF [(float_extend:XF
-		      (match_operand:X87MODEF12 1 "register_operand" "0"))]
+		      (match_operand:MODEF 1 "register_operand" "0"))]
 		   UNSPEC_COS))]
   "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
@@ -16327,7 +16630,7 @@
    (set (match_operand:XF 1 "register_operand" "")
 	(unspec:XF [(match_dup 2)] UNSPEC_SINCOS_SIN))]
   "find_regno_note (insn, REG_UNUSED, REGNO (operands[0]))
-   && !reload_completed && !reload_in_progress"
+   && !(reload_completed || reload_in_progress)"
   [(set (match_dup 1) (unspec:XF [(match_dup 2)] UNSPEC_SIN))]
   "")
 
@@ -16338,14 +16641,14 @@
    (set (match_operand:XF 1 "register_operand" "")
 	(unspec:XF [(match_dup 2)] UNSPEC_SINCOS_SIN))]
   "find_regno_note (insn, REG_UNUSED, REGNO (operands[1]))
-   && !reload_completed && !reload_in_progress"
+   && !(reload_completed || reload_in_progress)"
   [(set (match_dup 0) (unspec:XF [(match_dup 2)] UNSPEC_COS))]
   "")
 
 (define_insn "sincos_extend<mode>xf3_i387"
   [(set (match_operand:XF 0 "register_operand" "=f")
 	(unspec:XF [(float_extend:XF
-		      (match_operand:X87MODEF12 2 "register_operand" "0"))]
+		      (match_operand:MODEF 2 "register_operand" "0"))]
 		   UNSPEC_SINCOS_COS))
    (set (match_operand:XF 1 "register_operand" "=u")
         (unspec:XF [(float_extend:XF (match_dup 2))] UNSPEC_SINCOS_SIN))]
@@ -16360,31 +16663,31 @@
 (define_split
   [(set (match_operand:XF 0 "register_operand" "")
 	(unspec:XF [(float_extend:XF
-		      (match_operand:X87MODEF12 2 "register_operand" ""))]
+		      (match_operand:MODEF 2 "register_operand" ""))]
 		   UNSPEC_SINCOS_COS))
    (set (match_operand:XF 1 "register_operand" "")
 	(unspec:XF [(float_extend:XF (match_dup 2))] UNSPEC_SINCOS_SIN))]
   "find_regno_note (insn, REG_UNUSED, REGNO (operands[0]))
-   && !reload_completed && !reload_in_progress"
+   && !(reload_completed || reload_in_progress)"
   [(set (match_dup 1) (unspec:XF [(float_extend:XF (match_dup 2))] UNSPEC_SIN))]
   "")
 
 (define_split
   [(set (match_operand:XF 0 "register_operand" "")
 	(unspec:XF [(float_extend:XF
-		      (match_operand:X87MODEF12 2 "register_operand" ""))]
+		      (match_operand:MODEF 2 "register_operand" ""))]
 		   UNSPEC_SINCOS_COS))
    (set (match_operand:XF 1 "register_operand" "")
 	(unspec:XF [(float_extend:XF (match_dup 2))] UNSPEC_SINCOS_SIN))]
   "find_regno_note (insn, REG_UNUSED, REGNO (operands[1]))
-   && !reload_completed && !reload_in_progress"
+   && !(reload_completed || reload_in_progress)"
   [(set (match_dup 0) (unspec:XF [(float_extend:XF (match_dup 2))] UNSPEC_COS))]
   "")
 
 (define_expand "sincos<mode>3"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "register_operand" ""))
-   (use (match_operand:X87MODEF12 2 "register_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "register_operand" ""))
+   (use (match_operand:MODEF 2 "register_operand" ""))]
   "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)
@@ -16413,11 +16716,11 @@
    (set_attr "mode" "XF")])
 
 (define_insn "fptan_extend<mode>xf4_i387"
-  [(set (match_operand:X87MODEF12 0 "register_operand" "=f")
-	(match_operand:X87MODEF12 3 "const_double_operand" "F"))
+  [(set (match_operand:MODEF 0 "register_operand" "=f")
+	(match_operand:MODEF 3 "const_double_operand" "F"))
    (set (match_operand:XF 1 "register_operand" "=u")
         (unspec:XF [(float_extend:XF
-		      (match_operand:X87MODEF12 2 "register_operand" "0"))]
+		      (match_operand:MODEF 2 "register_operand" "0"))]
 		   UNSPEC_TAN))]
   "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
@@ -16442,8 +16745,8 @@
 })
 
 (define_expand "tan<mode>2"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "register_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "register_operand" ""))]
   "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)
@@ -16475,9 +16778,9 @@
 (define_insn "fpatan_extend<mode>xf3_i387"
   [(set (match_operand:XF 0 "register_operand" "=f")
         (unspec:XF [(float_extend:XF
-		      (match_operand:X87MODEF12 1 "register_operand" "0"))
+		      (match_operand:MODEF 1 "register_operand" "0"))
 		    (float_extend:XF
-		      (match_operand:X87MODEF12 2 "register_operand" "u"))]
+		      (match_operand:MODEF 2 "register_operand" "u"))]
 	           UNSPEC_FPATAN))
    (clobber (match_scratch:XF 3 "=2"))]
   "TARGET_USE_FANCY_MATH_387
@@ -16499,9 +16802,9 @@
   "")
 
 (define_expand "atan2<mode>3"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "register_operand" ""))
-   (use (match_operand:X87MODEF12 2 "register_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "register_operand" ""))
+   (use (match_operand:MODEF 2 "register_operand" ""))]
   "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)
@@ -16528,8 +16831,8 @@
 })
 
 (define_expand "atan<mode>2"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "register_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "register_operand" ""))]
   "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)
@@ -16567,8 +16870,8 @@
 })
 
 (define_expand "asin<mode>2"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "general_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "general_operand" ""))]
  "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)
@@ -16605,8 +16908,8 @@
 })
 
 (define_expand "acos<mode>2"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "general_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "general_operand" ""))]
  "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)
@@ -16636,7 +16939,7 @@
 (define_insn "fyl2x_extend<mode>xf3_i387"
   [(set (match_operand:XF 0 "register_operand" "=f")
         (unspec:XF [(float_extend:XF
-		      (match_operand:X87MODEF12 1 "register_operand" "0"))
+		      (match_operand:MODEF 1 "register_operand" "0"))
 		    (match_operand:XF 2 "register_operand" "u")]
 	           UNSPEC_FYL2X))
    (clobber (match_scratch:XF 3 "=2"))]
@@ -16661,8 +16964,8 @@
 })
 
 (define_expand "log<mode>2"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "register_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "register_operand" ""))]
   "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)
@@ -16691,8 +16994,8 @@
 })
 
 (define_expand "log10<mode>2"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "register_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "register_operand" ""))]
   "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)
@@ -16721,8 +17024,8 @@
 })
 
 (define_expand "log2<mode>2"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "register_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "register_operand" ""))]
   "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)
@@ -16753,7 +17056,7 @@
 (define_insn "fyl2xp1_extend<mode>xf3_i387"
   [(set (match_operand:XF 0 "register_operand" "=f")
         (unspec:XF [(float_extend:XF
-		      (match_operand:X87MODEF12 1 "register_operand" "0"))
+		      (match_operand:MODEF 1 "register_operand" "0"))
 		    (match_operand:XF 2 "register_operand" "u")]
 	           UNSPEC_FYL2XP1))
    (clobber (match_scratch:XF 3 "=2"))]
@@ -16776,8 +17079,8 @@
 })
 
 (define_expand "log1p<mode>2"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "register_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "register_operand" ""))]
   "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)
@@ -16807,7 +17110,7 @@
 (define_insn "fxtract_extend<mode>xf3_i387"
   [(set (match_operand:XF 0 "register_operand" "=f")
 	(unspec:XF [(float_extend:XF
-		      (match_operand:X87MODEF12 2 "register_operand" "0"))]
+		      (match_operand:MODEF 2 "register_operand" "0"))]
 		   UNSPEC_XTRACT_FRACT))
    (set (match_operand:XF 1 "register_operand" "=u")
         (unspec:XF [(float_extend:XF (match_dup 2))] UNSPEC_XTRACT_EXP))]
@@ -16832,8 +17135,8 @@
 })
 
 (define_expand "logb<mode>2"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "register_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "register_operand" ""))]
   "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)
@@ -16863,7 +17166,7 @@
 
 (define_expand "ilogb<mode>2"
   [(use (match_operand:SI 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "register_operand" ""))]
+   (use (match_operand:MODEF 1 "register_operand" ""))]
   "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)
@@ -16939,8 +17242,8 @@
 })
 
 (define_expand "exp<mode>2"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "general_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "general_operand" ""))]
  "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)
@@ -16969,8 +17272,8 @@
 })
 
 (define_expand "exp10<mode>2"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "general_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "general_operand" ""))]
  "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)
@@ -16999,8 +17302,8 @@
 })
 
 (define_expand "exp2<mode>2"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "general_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "general_operand" ""))]
  "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)
@@ -17053,8 +17356,8 @@
 })
 
 (define_expand "expm1<mode>2"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "general_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "general_operand" ""))]
  "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)
@@ -17087,8 +17390,8 @@
 })
 
 (define_expand "ldexp<mode>3"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "general_operand" ""))
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "general_operand" ""))
    (use (match_operand:SI 2 "register_operand" ""))]
  "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
@@ -17119,9 +17422,9 @@
 })
 
 (define_expand "scalb<mode>3"
-  [(use (match_operand:X87MODEF12 0 "register_operand" ""))
-   (use (match_operand:X87MODEF12 1 "general_operand" ""))
-   (use (match_operand:X87MODEF12 2 "register_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "general_operand" ""))
+   (use (match_operand:MODEF 2 "register_operand" ""))]
  "TARGET_USE_FANCY_MATH_387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)
@@ -17140,11 +17443,11 @@
 
 
 (define_insn "sse4_1_round<mode>2"
-  [(set (match_operand:SSEMODEF 0 "register_operand" "=x")
-	(unspec:SSEMODEF [(match_operand:SSEMODEF 1 "register_operand" "x")
-			  (match_operand:SI 2 "const_0_to_15_operand" "n")]
-		         UNSPEC_ROUND))]
-  "TARGET_SSE4_1"
+  [(set (match_operand:MODEF 0 "register_operand" "=x")
+	(unspec:MODEF [(match_operand:MODEF 1 "register_operand" "x")
+		       (match_operand:SI 2 "const_0_to_15_operand" "n")]
+		      UNSPEC_ROUND))]
+  "TARGET_ROUND"
   "rounds<ssemodefsuffix>\t{%2, %1, %0|%0, %1, %2}"
   [(set_attr "type" "ssecvt")
    (set_attr "prefix_extra" "1")
@@ -17161,21 +17464,21 @@
    (set_attr "mode" "XF")])
 
 (define_expand "rint<mode>2"
-  [(use (match_operand:SSEMODEF 0 "register_operand" ""))
-   (use (match_operand:SSEMODEF 1 "register_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "register_operand" ""))]
   "(TARGET_USE_FANCY_MATH_387
     && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
 	|| TARGET_MIX_SSE_I387)
     && flag_unsafe_math_optimizations)
    || (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH
        && !flag_trapping_math
-       && (TARGET_SSE4_1 || !optimize_size))"
+       && (TARGET_ROUND || !optimize_size))"
 {
   if (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH
       && !flag_trapping_math
-      && (TARGET_SSE4_1 || !optimize_size))
+      && (TARGET_ROUND || !optimize_size))
     {
-      if (TARGET_SSE4_1)
+      if (TARGET_ROUND)
 	emit_insn (gen_sse4_1_round<mode>2
 		   (operands[0], operands[1], GEN_INT (0x04)));
       else
@@ -17195,8 +17498,8 @@
 })
 
 (define_expand "round<mode>2"
-  [(match_operand:SSEMODEF 0 "register_operand" "")
-   (match_operand:SSEMODEF 1 "nonimmediate_operand" "")]
+  [(match_operand:MODEF 0 "register_operand" "")
+   (match_operand:MODEF 1 "nonimmediate_operand" "")]
   "SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH
    && !flag_trapping_math && !flag_rounding_math
    && !optimize_size"
@@ -17209,8 +17512,8 @@
 })
 
 (define_insn_and_split "*fistdi2_1"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "=m,?r")
-	(unspec:DI [(match_operand:XF 1 "register_operand" "f,f")]
+  [(set (match_operand:DI 0 "nonimmediate_operand" "")
+	(unspec:DI [(match_operand:XF 1 "register_operand" "")]
 		   UNSPEC_FIST))]
   "TARGET_USE_FANCY_MATH_387
    && !(reload_completed || reload_in_progress)"
@@ -17276,8 +17579,8 @@
   "")
 
 (define_insn_and_split "*fist<mode>2_1"
-  [(set (match_operand:X87MODEI12 0 "register_operand" "=r")
-	(unspec:X87MODEI12 [(match_operand:XF 1 "register_operand" "f")]
+  [(set (match_operand:X87MODEI12 0 "register_operand" "")
+	(unspec:X87MODEI12 [(match_operand:XF 1 "register_operand" "")]
 			   UNSPEC_FIST))]
   "TARGET_USE_FANCY_MATH_387
    && !(reload_completed || reload_in_progress)"
@@ -17338,18 +17641,18 @@
   "TARGET_USE_FANCY_MATH_387"
   "")
 
-(define_expand "lrint<SSEMODEF:mode><SSEMODEI24:mode>2"
+(define_expand "lrint<MODEF:mode><SSEMODEI24:mode>2"
   [(set (match_operand:SSEMODEI24 0 "nonimmediate_operand" "")
-     (unspec:SSEMODEI24 [(match_operand:SSEMODEF 1 "register_operand" "")]
+     (unspec:SSEMODEI24 [(match_operand:MODEF 1 "register_operand" "")]
 			UNSPEC_FIX_NOTRUNC))]
-  "SSE_FLOAT_MODE_P (<SSEMODEF:MODE>mode) && TARGET_SSE_MATH
+  "SSE_FLOAT_MODE_P (<MODEF:MODE>mode) && TARGET_SSE_MATH
    && ((<SSEMODEI24:MODE>mode != DImode) || TARGET_64BIT)"
   "")
 
-(define_expand "lround<SSEMODEF:mode><SSEMODEI24:mode>2"
+(define_expand "lround<MODEF:mode><SSEMODEI24:mode>2"
   [(match_operand:SSEMODEI24 0 "nonimmediate_operand" "")
-   (match_operand:SSEMODEF 1 "register_operand" "")]
-  "SSE_FLOAT_MODE_P (<SSEMODEF:MODE>mode) && TARGET_SSE_MATH
+   (match_operand:MODEF 1 "register_operand" "")]
+  "SSE_FLOAT_MODE_P (<MODEF:MODE>mode) && TARGET_SSE_MATH
    && ((<SSEMODEI24:MODE>mode != DImode) || TARGET_64BIT)
    && !flag_trapping_math && !flag_rounding_math
    && !optimize_size"
@@ -17360,8 +17663,8 @@
 
 ;; Rounding mode control word calculation could clobber FLAGS_REG.
 (define_insn_and_split "frndintxf2_floor"
-  [(set (match_operand:XF 0 "register_operand" "=f")
-	(unspec:XF [(match_operand:XF 1 "register_operand" "0")]
+  [(set (match_operand:XF 0 "register_operand" "")
+	(unspec:XF [(match_operand:XF 1 "register_operand" "")]
 	 UNSPEC_FRNDINT_FLOOR))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_USE_FANCY_MATH_387
@@ -17408,21 +17711,21 @@
 })
 
 (define_expand "floor<mode>2"
-  [(use (match_operand:SSEMODEF 0 "register_operand" ""))
-   (use (match_operand:SSEMODEF 1 "register_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "register_operand" ""))]
   "(TARGET_USE_FANCY_MATH_387
     && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
 	|| TARGET_MIX_SSE_I387)
     && flag_unsafe_math_optimizations && !optimize_size)
    || (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH
        && !flag_trapping_math
-       && (TARGET_SSE4_1 || !optimize_size))"
+       && (TARGET_ROUND || !optimize_size))"
 {
   if (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH
       && !flag_trapping_math
-      && (TARGET_SSE4_1 || !optimize_size))
+      && (TARGET_ROUND || !optimize_size))
     {
-      if (TARGET_SSE4_1)
+      if (TARGET_ROUND)
 	emit_insn (gen_sse4_1_round<mode>2
 		   (operands[0], operands[1], GEN_INT (0x01)));
       else if (TARGET_64BIT || (<MODE>mode != DFmode))
@@ -17444,8 +17747,8 @@
 })
 
 (define_insn_and_split "*fist<mode>2_floor_1"
-  [(set (match_operand:X87MODEI 0 "nonimmediate_operand" "=m,?r")
-	(unspec:X87MODEI [(match_operand:XF 1 "register_operand" "f,f")]
+  [(set (match_operand:X87MODEI 0 "nonimmediate_operand" "")
+	(unspec:X87MODEI [(match_operand:XF 1 "register_operand" "")]
 	 UNSPEC_FIST_FLOOR))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_USE_FANCY_MATH_387
@@ -17603,7 +17906,7 @@
 
 (define_expand "lfloor<mode>di2"
   [(match_operand:DI 0 "nonimmediate_operand" "")
-   (match_operand:SSEMODEF 1 "register_operand" "")]
+   (match_operand:MODEF 1 "register_operand" "")]
   "SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH && TARGET_64BIT
    && !flag_trapping_math
    && !optimize_size"
@@ -17614,7 +17917,7 @@
 
 (define_expand "lfloor<mode>si2"
   [(match_operand:SI 0 "nonimmediate_operand" "")
-   (match_operand:SSEMODEF 1 "register_operand" "")]
+   (match_operand:MODEF 1 "register_operand" "")]
   "SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH
    && !flag_trapping_math
    && (!optimize_size || !TARGET_64BIT)"
@@ -17625,8 +17928,8 @@
 
 ;; Rounding mode control word calculation could clobber FLAGS_REG.
 (define_insn_and_split "frndintxf2_ceil"
-  [(set (match_operand:XF 0 "register_operand" "=f")
-	(unspec:XF [(match_operand:XF 1 "register_operand" "0")]
+  [(set (match_operand:XF 0 "register_operand" "")
+	(unspec:XF [(match_operand:XF 1 "register_operand" "")]
 	 UNSPEC_FRNDINT_CEIL))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_USE_FANCY_MATH_387
@@ -17673,21 +17976,21 @@
 })
 
 (define_expand "ceil<mode>2"
-  [(use (match_operand:SSEMODEF 0 "register_operand" ""))
-   (use (match_operand:SSEMODEF 1 "register_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "register_operand" ""))]
   "(TARGET_USE_FANCY_MATH_387
     && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
 	|| TARGET_MIX_SSE_I387)
     && flag_unsafe_math_optimizations && !optimize_size)
    || (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH
        && !flag_trapping_math
-       && (TARGET_SSE4_1 || !optimize_size))"
+       && (TARGET_ROUND || !optimize_size))"
 {
   if (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH
       && !flag_trapping_math
-      && (TARGET_SSE4_1 || !optimize_size))
+      && (TARGET_ROUND || !optimize_size))
     {
-      if (TARGET_SSE4_1)
+      if (TARGET_ROUND)
 	emit_insn (gen_sse4_1_round<mode>2
 		   (operands[0], operands[1], GEN_INT (0x02)));
       else if (TARGET_64BIT || (<MODE>mode != DFmode))
@@ -17709,8 +18012,8 @@
 })
 
 (define_insn_and_split "*fist<mode>2_ceil_1"
-  [(set (match_operand:X87MODEI 0 "nonimmediate_operand" "=m,?r")
-	(unspec:X87MODEI [(match_operand:XF 1 "register_operand" "f,f")]
+  [(set (match_operand:X87MODEI 0 "nonimmediate_operand" "")
+	(unspec:X87MODEI [(match_operand:XF 1 "register_operand" "")]
 	 UNSPEC_FIST_CEIL))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_USE_FANCY_MATH_387
@@ -17868,7 +18171,7 @@
 
 (define_expand "lceil<mode>di2"
   [(match_operand:DI 0 "nonimmediate_operand" "")
-   (match_operand:SSEMODEF 1 "register_operand" "")]
+   (match_operand:MODEF 1 "register_operand" "")]
   "SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH && TARGET_64BIT
    && !flag_trapping_math"
 {
@@ -17878,7 +18181,7 @@
 
 (define_expand "lceil<mode>si2"
   [(match_operand:SI 0 "nonimmediate_operand" "")
-   (match_operand:SSEMODEF 1 "register_operand" "")]
+   (match_operand:MODEF 1 "register_operand" "")]
   "SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH
    && !flag_trapping_math"
 {
@@ -17888,8 +18191,8 @@
 
 ;; Rounding mode control word calculation could clobber FLAGS_REG.
 (define_insn_and_split "frndintxf2_trunc"
-  [(set (match_operand:XF 0 "register_operand" "=f")
-	(unspec:XF [(match_operand:XF 1 "register_operand" "0")]
+  [(set (match_operand:XF 0 "register_operand" "")
+	(unspec:XF [(match_operand:XF 1 "register_operand" "")]
 	 UNSPEC_FRNDINT_TRUNC))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_USE_FANCY_MATH_387
@@ -17936,21 +18239,21 @@
 })
 
 (define_expand "btrunc<mode>2"
-  [(use (match_operand:SSEMODEF 0 "register_operand" ""))
-   (use (match_operand:SSEMODEF 1 "register_operand" ""))]
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "register_operand" ""))]
   "(TARGET_USE_FANCY_MATH_387
     && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
 	|| TARGET_MIX_SSE_I387)
     && flag_unsafe_math_optimizations && !optimize_size)
    || (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH
        && !flag_trapping_math
-       && (TARGET_SSE4_1 || !optimize_size))"
+       && (TARGET_ROUND || !optimize_size))"
 {
   if (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH
       && !flag_trapping_math
-      && (TARGET_SSE4_1 || !optimize_size))
+      && (TARGET_ROUND || !optimize_size))
     {
-      if (TARGET_SSE4_1)
+      if (TARGET_ROUND)
 	emit_insn (gen_sse4_1_round<mode>2
 		   (operands[0], operands[1], GEN_INT (0x03)));
       else if (TARGET_64BIT || (<MODE>mode != DFmode))
@@ -17973,8 +18276,8 @@
 
 ;; Rounding mode control word calculation could clobber FLAGS_REG.
 (define_insn_and_split "frndintxf2_mask_pm"
-  [(set (match_operand:XF 0 "register_operand" "=f")
-	(unspec:XF [(match_operand:XF 1 "register_operand" "0")]
+  [(set (match_operand:XF 0 "register_operand" "")
+	(unspec:XF [(match_operand:XF 1 "register_operand" "")]
 	 UNSPEC_FRNDINT_MASK_PM))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_USE_FANCY_MATH_387
@@ -18021,37 +18324,21 @@
   DONE;
 })
 
-(define_expand "nearbyintdf2"
-  [(use (match_operand:DF 0 "register_operand" ""))
-   (use (match_operand:DF 1 "register_operand" ""))]
-  "TARGET_USE_FANCY_MATH_387
-   && (!(TARGET_SSE2 && TARGET_SSE_MATH) || TARGET_MIX_SSE_I387)
-   && flag_unsafe_math_optimizations"
-{
-  rtx op0 = gen_reg_rtx (XFmode);
-  rtx op1 = gen_reg_rtx (XFmode);
-
-  emit_insn (gen_extenddfxf2 (op1, operands[1]));
-  emit_insn (gen_frndintxf2_mask_pm (op0, op1));
-
-  emit_insn (gen_truncxfdf2_i387_noop (operands[0], op0));
-  DONE;
-})
-
-(define_expand "nearbyintsf2"
-  [(use (match_operand:SF 0 "register_operand" ""))
-   (use (match_operand:SF 1 "register_operand" ""))]
+(define_expand "nearbyint<mode>2"
+  [(use (match_operand:MODEF 0 "register_operand" ""))
+   (use (match_operand:MODEF 1 "register_operand" ""))]
   "TARGET_USE_FANCY_MATH_387
-   && (!TARGET_SSE_MATH || TARGET_MIX_SSE_I387)
+   && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
+       || TARGET_MIX_SSE_I387)
    && flag_unsafe_math_optimizations"
 {
   rtx op0 = gen_reg_rtx (XFmode);
   rtx op1 = gen_reg_rtx (XFmode);
 
-  emit_insn (gen_extendsfxf2 (op1, operands[1]));
+  emit_insn (gen_extend<mode>xf2 (op1, operands[1]));
   emit_insn (gen_frndintxf2_mask_pm (op0, op1));
 
-  emit_insn (gen_truncxfsf2_i387_noop (operands[0], op0));
+  emit_insn (gen_truncxf<mode>2_i387_noop (operands[0], op0));
   DONE;
 })
 
@@ -18160,7 +18447,9 @@
   operands[5] = gen_rtx_PLUS (Pmode, operands[0], adjust);
   operands[6] = gen_rtx_PLUS (Pmode, operands[2], adjust);
 
-  if (TARGET_SINGLE_STRINGOP || optimize_size)
+  /* Can't use this if the user has appropriated esi or edi.  */
+  if ((TARGET_SINGLE_STRINGOP || optimize_size)
+      && !(fixed_regs[SI_REG] || fixed_regs[DI_REG]))
     {
       emit_insn (gen_strmov_singleop (operands[0], operands[1],
 				      operands[2], operands[3],
@@ -18646,7 +18935,7 @@
   rtx addr1, addr2, out, outlow, count, countreg, align;
 
   /* Can't use this if the user has appropriated esi or edi.  */
-  if (global_regs[4] || global_regs[5])
+  if (fixed_regs[SI_REG] || fixed_regs[DI_REG])
     FAIL;
 
   out = operands[0];
@@ -18961,7 +19250,7 @@
 			 (match_operand:DI 2 "general_operand" "")
 			 (match_operand:DI 3 "general_operand" "")))]
   "TARGET_64BIT"
-  "if (!ix86_expand_int_movcc (operands)) FAIL; DONE;")
+  "if (ix86_expand_int_movcc (operands)) DONE; else FAIL;")
 
 (define_insn "x86_movdicc_0_m1_rex64"
   [(set (match_operand:DI 0 "register_operand" "=r")
@@ -18980,6 +19269,21 @@
    (set_attr "mode" "DI")
    (set_attr "length_immediate" "0")])
 
+(define_insn "*x86_movdicc_0_m1_se"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	(sign_extract:DI (match_operand 1 "ix86_carry_flag_operator" "")
+			 (const_int 1)
+			 (const_int 0)))
+   (clobber (reg:CC FLAGS_REG))]
+  ""
+  "sbb{q}\t%0, %0"
+  [(set_attr "type" "alu")
+   (set_attr "pent_pair" "pu")
+   (set_attr "memory" "none")
+   (set_attr "imm_disp" "false")
+   (set_attr "mode" "DI")
+   (set_attr "length_immediate" "0")])
+
 (define_insn "*movdicc_c_rex64"
   [(set (match_operand:DI 0 "register_operand" "=r,r")
 	(if_then_else:DI (match_operator 1 "ix86_comparison_operator"
@@ -19000,7 +19304,7 @@
 			 (match_operand:SI 2 "general_operand" "")
 			 (match_operand:SI 3 "general_operand" "")))]
   ""
-  "if (!ix86_expand_int_movcc (operands)) FAIL; DONE;")
+  "if (ix86_expand_int_movcc (operands)) DONE; else FAIL;")
 
 ;; Data flow gets confused by our desire for `sbbl reg,reg', and clearing
 ;; the register first winds up with `sbbl $0,reg', which is also weird.
@@ -19023,6 +19327,21 @@
    (set_attr "mode" "SI")
    (set_attr "length_immediate" "0")])
 
+(define_insn "*x86_movsicc_0_m1_se"
+  [(set (match_operand:SI 0 "register_operand" "=r")
+	(sign_extract:SI (match_operand 1 "ix86_carry_flag_operator" "")
+			 (const_int 1)
+			 (const_int 0)))
+   (clobber (reg:CC FLAGS_REG))]
+  ""
+  "sbb{l}\t%0, %0"
+  [(set_attr "type" "alu")
+   (set_attr "pent_pair" "pu")
+   (set_attr "memory" "none")
+   (set_attr "imm_disp" "false")
+   (set_attr "mode" "SI")
+   (set_attr "length_immediate" "0")])
+
 (define_insn "*movsicc_noc"
   [(set (match_operand:SI 0 "register_operand" "=r,r")
 	(if_then_else:SI (match_operator 1 "ix86_comparison_operator"
@@ -19043,7 +19362,7 @@
 			 (match_operand:HI 2 "general_operand" "")
 			 (match_operand:HI 3 "general_operand" "")))]
   "TARGET_HIMODE_MATH"
-  "if (!ix86_expand_int_movcc (operands)) FAIL; DONE;")
+  "if (ix86_expand_int_movcc (operands)) DONE; else FAIL;")
 
 (define_insn "*movhicc_noc"
   [(set (match_operand:HI 0 "register_operand" "=r,r")
@@ -19065,7 +19384,7 @@
 			 (match_operand:QI 2 "general_operand" "")
 			 (match_operand:QI 3 "general_operand" "")))]
   "TARGET_QIMODE_MATH"
-  "if (!ix86_expand_int_movcc (operands)) FAIL; DONE;")
+  "if (ix86_expand_int_movcc (operands)) DONE; else FAIL;")
 
 (define_insn_and_split "*movqicc_noc"
   [(set (match_operand:QI 0 "register_operand" "=r,r")
@@ -19087,13 +19406,15 @@
   [(set_attr "type" "icmov")
    (set_attr "mode" "SI")])
 
-(define_expand "movsfcc"
-  [(set (match_operand:SF 0 "register_operand" "")
-	(if_then_else:SF (match_operand 1 "comparison_operator" "")
-			 (match_operand:SF 2 "register_operand" "")
-			 (match_operand:SF 3 "register_operand" "")))]
-  "(TARGET_80387 && TARGET_CMOVE) || TARGET_SSE_MATH"
-  "if (! ix86_expand_fp_movcc (operands)) FAIL; DONE;")
+(define_expand "mov<mode>cc"
+  [(set (match_operand:X87MODEF 0 "register_operand" "")
+	(if_then_else:X87MODEF
+	  (match_operand 1 "comparison_operator" "")
+	  (match_operand:X87MODEF 2 "register_operand" "")
+	  (match_operand:X87MODEF 3 "register_operand" "")))]
+  "(TARGET_80387 && TARGET_CMOVE)
+   || (SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)"
+  "if (ix86_expand_fp_movcc (operands)) DONE; else FAIL;")
 
 (define_insn "*movsfcc_1_387"
   [(set (match_operand:SF 0 "register_operand" "=f,f,r,r")
@@ -19111,14 +19432,6 @@
   [(set_attr "type" "fcmov,fcmov,icmov,icmov")
    (set_attr "mode" "SF,SF,SI,SI")])
 
-(define_expand "movdfcc"
-  [(set (match_operand:DF 0 "register_operand" "")
-	(if_then_else:DF (match_operand 1 "comparison_operator" "")
-			 (match_operand:DF 2 "register_operand" "")
-			 (match_operand:DF 3 "register_operand" "")))]
-  "(TARGET_80387 && TARGET_CMOVE) || (TARGET_SSE2 && TARGET_SSE_MATH)"
-  "if (! ix86_expand_fp_movcc (operands)) FAIL; DONE;")
-
 (define_insn "*movdfcc_1"
   [(set (match_operand:DF 0 "register_operand" "=f,f,&r,&r")
 	(if_then_else:DF (match_operator 1 "fcmov_comparison_operator"
@@ -19171,14 +19484,6 @@
    split_di (operands+3, 1, operands+7, operands+8);
    split_di (operands, 1, operands+2, operands+3);")
 
-(define_expand "movxfcc"
-  [(set (match_operand:XF 0 "register_operand" "")
-	(if_then_else:XF (match_operand 1 "comparison_operator" "")
-			 (match_operand:XF 2 "register_operand" "")
-			 (match_operand:XF 3 "register_operand" "")))]
-  "TARGET_80387 && TARGET_CMOVE"
-  "if (! ix86_expand_fp_movcc (operands)) FAIL; DONE;")
-
 (define_insn "*movxfcc_1"
   [(set (match_operand:XF 0 "register_operand" "=f,f")
 	(if_then_else:XF (match_operator 1 "fcmov_comparison_operator"
@@ -19192,46 +19497,34 @@
   [(set_attr "type" "fcmov")
    (set_attr "mode" "XF")])
 
+;; All moves in SSE5 pcmov instructions are 128 bits and hence we restrict
+;; the scalar versions to have only XMM registers as operands.
+
+;; SSE5 conditional move
+(define_insn "*sse5_pcmov_<mode>"
+  [(set (match_operand:MODEF 0 "register_operand" "=x,x")
+	(if_then_else:MODEF
+	  (match_operand:MODEF 1 "register_operand" "x,0")
+	  (match_operand:MODEF 2 "register_operand" "0,x")
+	  (match_operand:MODEF 3 "register_operand" "x,x")))]
+  "TARGET_SSE5 && ix86_sse5_valid_op_p (operands, insn, 4, true, 1)"
+  "pcmov\t{%1, %3, %2, %0|%0, %2, %3, %1}"
+  [(set_attr "type" "sse4arg")])
+
 ;; These versions of the min/max patterns are intentionally ignorant of
 ;; their behavior wrt -0.0 and NaN (via the commutative operand mark).
 ;; Since both the tree-level MAX_EXPR and the rtl-level SMAX operator
 ;; are undefined in this condition, we're certain this is correct.
 
-(define_insn "sminsf3"
-  [(set (match_operand:SF 0 "register_operand" "=x")
-	(smin:SF (match_operand:SF 1 "nonimmediate_operand" "%0")
-		 (match_operand:SF 2 "nonimmediate_operand" "xm")))]
-  "TARGET_SSE_MATH"
-  "minss\t{%2, %0|%0, %2}"
-  [(set_attr "type" "sseadd")
-   (set_attr "mode" "SF")])
-
-(define_insn "smaxsf3"
-  [(set (match_operand:SF 0 "register_operand" "=x")
-	(smax:SF (match_operand:SF 1 "nonimmediate_operand" "%0")
-		 (match_operand:SF 2 "nonimmediate_operand" "xm")))]
-  "TARGET_SSE_MATH"
-  "maxss\t{%2, %0|%0, %2}"
-  [(set_attr "type" "sseadd")
-   (set_attr "mode" "SF")])
-
-(define_insn "smindf3"
-  [(set (match_operand:DF 0 "register_operand" "=x")
-	(smin:DF (match_operand:DF 1 "nonimmediate_operand" "%0")
-		 (match_operand:DF 2 "nonimmediate_operand" "xm")))]
-  "TARGET_SSE2 && TARGET_SSE_MATH"
-  "minsd\t{%2, %0|%0, %2}"
-  [(set_attr "type" "sseadd")
-   (set_attr "mode" "DF")])
-
-(define_insn "smaxdf3"
-  [(set (match_operand:DF 0 "register_operand" "=x")
-	(smax:DF (match_operand:DF 1 "nonimmediate_operand" "%0")
-		 (match_operand:DF 2 "nonimmediate_operand" "xm")))]
-  "TARGET_SSE2 && TARGET_SSE_MATH"
-  "maxsd\t{%2, %0|%0, %2}"
+(define_insn "<code><mode>3"
+  [(set (match_operand:MODEF 0 "register_operand" "=x")
+	(smaxmin:MODEF
+	  (match_operand:MODEF 1 "nonimmediate_operand" "%0")
+	  (match_operand:MODEF 2 "nonimmediate_operand" "xm")))]
+  "SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH"
+  "<maxminfprefix>s<ssemodefsuffix>\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseadd")
-   (set_attr "mode" "DF")])
+   (set_attr "mode" "<MODE>")])
 
 ;; These versions of the min/max patterns implement exactly the operations
 ;;   min = (op1 < op2 ? op1 : op2)
@@ -19239,45 +19532,27 @@
 ;; Their operands are not commutative, and thus they may be used in the
 ;; presence of -0.0 and NaN.
 
-(define_insn "*ieee_sminsf3"
-  [(set (match_operand:SF 0 "register_operand" "=x")
-	(unspec:SF [(match_operand:SF 1 "register_operand" "0")
-		    (match_operand:SF 2 "nonimmediate_operand" "xm")]
-		   UNSPEC_IEEE_MIN))]
-  "TARGET_SSE_MATH"
-  "minss\t{%2, %0|%0, %2}"
-  [(set_attr "type" "sseadd")
-   (set_attr "mode" "SF")])
-
-(define_insn "*ieee_smaxsf3"
-  [(set (match_operand:SF 0 "register_operand" "=x")
-	(unspec:SF [(match_operand:SF 1 "register_operand" "0")
-		    (match_operand:SF 2 "nonimmediate_operand" "xm")]
-		   UNSPEC_IEEE_MAX))]
-  "TARGET_SSE_MATH"
-  "maxss\t{%2, %0|%0, %2}"
-  [(set_attr "type" "sseadd")
-   (set_attr "mode" "SF")])
-
-(define_insn "*ieee_smindf3"
-  [(set (match_operand:DF 0 "register_operand" "=x")
-	(unspec:DF [(match_operand:DF 1 "register_operand" "0")
-		    (match_operand:DF 2 "nonimmediate_operand" "xm")]
-		   UNSPEC_IEEE_MIN))]
-  "TARGET_SSE2 && TARGET_SSE_MATH"
-  "minsd\t{%2, %0|%0, %2}"
+(define_insn "*ieee_smin<mode>3"
+  [(set (match_operand:MODEF 0 "register_operand" "=x")
+	(unspec:MODEF
+	  [(match_operand:MODEF 1 "register_operand" "0")
+	   (match_operand:MODEF 2 "nonimmediate_operand" "xm")]
+	 UNSPEC_IEEE_MIN))]
+  "SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH"
+  "mins<ssemodefsuffix>\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseadd")
-   (set_attr "mode" "DF")])
+   (set_attr "mode" "<MODE>")])
 
-(define_insn "*ieee_smaxdf3"
-  [(set (match_operand:DF 0 "register_operand" "=x")
-	(unspec:DF [(match_operand:DF 1 "register_operand" "0")
-		    (match_operand:DF 2 "nonimmediate_operand" "xm")]
-		   UNSPEC_IEEE_MAX))]
-  "TARGET_SSE2 && TARGET_SSE_MATH"
-  "maxsd\t{%2, %0|%0, %2}"
+(define_insn "*ieee_smax<mode>3"
+  [(set (match_operand:MODEF 0 "register_operand" "=x")
+	(unspec:MODEF
+	  [(match_operand:MODEF 1 "register_operand" "0")
+	   (match_operand:MODEF 2 "nonimmediate_operand" "xm")]
+	 UNSPEC_IEEE_MAX))]
+  "SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH"
+  "maxs<ssemodefsuffix>\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseadd")
-   (set_attr "mode" "DF")])
+   (set_attr "mode" "<MODE>")])
 
 ;; Make two stack loads independent:
 ;;   fld aa              fld aa
@@ -19312,7 +19587,7 @@
    (match_operand:QI 2 "register_operand" "")
    (match_operand:QI 3 "const_int_operand" "")]
   ""
-  "if (!ix86_expand_int_addcc (operands)) FAIL; DONE;")
+  "if (ix86_expand_int_addcc (operands)) DONE; else FAIL;")
 
 (define_expand "addhicc"
   [(match_operand:HI 0 "register_operand" "")
@@ -19320,7 +19595,7 @@
    (match_operand:HI 2 "register_operand" "")
    (match_operand:HI 3 "const_int_operand" "")]
   ""
-  "if (!ix86_expand_int_addcc (operands)) FAIL; DONE;")
+  "if (ix86_expand_int_addcc (operands)) DONE; else FAIL;")
 
 (define_expand "addsicc"
   [(match_operand:SI 0 "register_operand" "")
@@ -19328,7 +19603,7 @@
    (match_operand:SI 2 "register_operand" "")
    (match_operand:SI 3 "const_int_operand" "")]
   ""
-  "if (!ix86_expand_int_addcc (operands)) FAIL; DONE;")
+  "if (ix86_expand_int_addcc (operands)) DONE; else FAIL;")
 
 (define_expand "adddicc"
   [(match_operand:DI 0 "register_operand" "")
@@ -19336,7 +19611,7 @@
    (match_operand:DI 2 "register_operand" "")
    (match_operand:DI 3 "const_int_operand" "")]
   "TARGET_64BIT"
-  "if (!ix86_expand_int_addcc (operands)) FAIL; DONE;")
+  "if (ix86_expand_int_addcc (operands)) DONE; else FAIL;")
 
 
 ;; Misc patterns (?)
@@ -19465,7 +19740,7 @@
    (set (reg:SI SP_REG) (minus:SI (reg:SI SP_REG) (match_dup 0)))
    (clobber (reg:CC FLAGS_REG))]
   "!TARGET_64BIT && TARGET_STACK_PROBE"
-  "call\t__alloca"
+  "call\t___chkstk"
   [(set_attr "type" "multi")
    (set_attr "length" "5")])
 
@@ -19872,7 +20147,7 @@
 		   (match_operand:SI 3 "immediate_operand" ""))
 	   (const_int 0)]))]
   "ix86_match_ccmode (insn, CCNOmode)
-   && (true_regnum (operands[2]) != 0
+   && (true_regnum (operands[2]) != AX_REG
        || satisfies_constraint_K (operands[3]))
    && peep2_reg_dead_p (1, operands[2])"
   [(parallel
@@ -19894,7 +20169,7 @@
 	   (const_int 0)]))]
   "! TARGET_PARTIAL_REG_STALL
    && ix86_match_ccmode (insn, CCNOmode)
-   && true_regnum (operands[2]) != 0
+   && true_regnum (operands[2]) != AX_REG
    && peep2_reg_dead_p (1, operands[2])"
   [(parallel
      [(set (match_dup 0)
@@ -19916,7 +20191,7 @@
 	   (const_int 0)]))]
   "! TARGET_PARTIAL_REG_STALL
    && ix86_match_ccmode (insn, CCNOmode)
-   && true_regnum (operands[2]) != 0
+   && true_regnum (operands[2]) != AX_REG
    && peep2_reg_dead_p (1, operands[2])"
   [(parallel [(set (match_dup 0)
 		   (match_op_dup 1
@@ -20705,7 +20980,7 @@
 	(call (mem:QI (reg:DI R11_REG))
 	      (match_operand:DI 1 "" "")))]
   "SIBLING_CALL_P (insn) && TARGET_64BIT"
-  "jmp\t*%%r11"
+  "jmp\t{*%%}r11"
   [(set_attr "type" "callv")])
 
 ;; We used to use "int $5", in honor of #BR which maps to interrupt vector 5.
@@ -20752,26 +21027,24 @@
   "TARGET_64BIT
    && INTVAL (operands[4]) + SSE_REGPARM_MAX * 16 - 16 < 128
    && INTVAL (operands[4]) + INTVAL (operands[2]) * 16 >= -128"
-  "*
 {
   int i;
   operands[0] = gen_rtx_MEM (Pmode,
 			     gen_rtx_PLUS (Pmode, operands[0], operands[4]));
-  output_asm_insn (\"jmp\\t%A1\", operands);
+  output_asm_insn ("jmp\t%A1", operands);
   for (i = SSE_REGPARM_MAX - 1; i >= INTVAL (operands[2]); i--)
     {
       operands[4] = adjust_address (operands[0], DImode, i*16);
       operands[5] = gen_rtx_REG (TImode, SSE_REGNO (i));
       PUT_MODE (operands[4], TImode);
       if (GET_CODE (XEXP (operands[0], 0)) != PLUS)
-        output_asm_insn (\"rex\", operands);
-      output_asm_insn (\"movaps\\t{%5, %4|%4, %5}\", operands);
+        output_asm_insn ("rex", operands);
+      output_asm_insn ("movaps\t{%5, %4|%4, %5}", operands);
     }
-  (*targetm.asm_out.internal_label) (asm_out_file, \"L\",
-			     CODE_LABEL_NUMBER (operands[3]));
-  return \"\";
+  (*targetm.asm_out.internal_label) (asm_out_file, "L",
+				     CODE_LABEL_NUMBER (operands[3]));
+  return "";
 }
-  "
   [(set_attr "type" "other")
    (set_attr "length_immediate" "0")
    (set_attr "length_address" "0")
@@ -20913,7 +21186,7 @@
    (set (match_scratch:SI 2 "=&r") (const_int 0))
    (clobber (reg:CC FLAGS_REG))]
   ""
-  "mov{l}\t{%%gs:%P1, %2|%2, DWORD PTR %%gs:%P1}\;mov{l}\t{%2, %0|%0, %2}\;xor{l}\t%2, %2"
+  "mov{l}\t{%%gs:%P1, %2|%2, DWORD PTR gs:%P1}\;mov{l}\t{%2, %0|%0, %2}\;xor{l}\t%2, %2"
   [(set_attr "type" "multi")])
 
 (define_insn "stack_tls_protect_set_di"
@@ -20927,9 +21200,9 @@
         system call would not have to trash the userspace segment register,
         which would be expensive */
      if (ix86_cmodel != CM_KERNEL)
-        return "mov{q}\t{%%fs:%P1, %2|%2, QWORD PTR %%fs:%P1}\;mov{q}\t{%2, %0|%0, %2}\;xor{l}\t%k2, %k2";
+        return "mov{q}\t{%%fs:%P1, %2|%2, QWORD PTR fs:%P1}\;mov{q}\t{%2, %0|%0, %2}\;xor{l}\t%k2, %k2";
      else
-        return "mov{q}\t{%%gs:%P1, %2|%2, QWORD PTR %%gs:%P1}\;mov{q}\t{%2, %0|%0, %2}\;xor{l}\t%k2, %k2";
+        return "mov{q}\t{%%gs:%P1, %2|%2, QWORD PTR gs:%P1}\;mov{q}\t{%2, %0|%0, %2}\;xor{l}\t%k2, %k2";
   }
   [(set_attr "type" "multi")])
 
@@ -20988,7 +21261,7 @@
 		    UNSPEC_SP_TLS_TEST))
    (clobber (match_scratch:SI 3 "=r"))]
   ""
-  "mov{l}\t{%1, %3|%3, %1}\;xor{l}\t{%%gs:%P2, %3|%3, DWORD PTR %%gs:%P2}"
+  "mov{l}\t{%1, %3|%3, %1}\;xor{l}\t{%%gs:%P2, %3|%3, DWORD PTR gs:%P2}"
   [(set_attr "type" "multi")])
 
 (define_insn "stack_tls_protect_test_di"
@@ -21003,13 +21276,13 @@
         system call would not have to trash the userspace segment register,
         which would be expensive */
      if (ix86_cmodel != CM_KERNEL)
-        return "mov{q}\t{%1, %3|%3, %1}\;xor{q}\t{%%fs:%P2, %3|%3, QWORD PTR %%fs:%P2}";
+        return "mov{q}\t{%1, %3|%3, %1}\;xor{q}\t{%%fs:%P2, %3|%3, QWORD PTR fs:%P2}";
      else
-        return "mov{q}\t{%1, %3|%3, %1}\;xor{q}\t{%%gs:%P2, %3|%3, QWORD PTR %%gs:%P2}";
+        return "mov{q}\t{%1, %3|%3, %1}\;xor{q}\t{%%gs:%P2, %3|%3, QWORD PTR gs:%P2}";
   }
   [(set_attr "type" "multi")])
 
-(define_mode_macro CRC32MODE [QI HI SI])
+(define_mode_iterator CRC32MODE [QI HI SI])
 (define_mode_attr crc32modesuffix [(QI "b") (HI "w") (SI "l")])
 (define_mode_attr crc32modeconstraint [(QI "qm") (HI "rm") (SI "rm")])