X-Git-Url: http://git.sourceforge.jp/view?a=blobdiff_plain;f=gcc%2Fconfig%2Fi386%2Fi386.md;h=f9be68f9e75521317c94337b5344859763977173;hb=062db9f39ed31bb2c570996cf3ee884dfaf7df21;hp=f8059adb7ea9a48e30cad1558f67e8b34d57c079;hpb=f27642b4e24ba896205504a6968166c3d860e361;p=pf3gnuchains%2Fgcc-fork.git

diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
index f8059adb7ea..f9be68f9e75 100644
--- a/gcc/config/i386/i386.md
+++ b/gcc/config/i386/i386.md
@@ -1,6 +1,6 @@
 ;; GCC machine description for IA-32 and x86-64.
 ;; Copyright (C) 1988, 1994, 1995, 1996, 1997, 1998, 1999, 2000,
-;; 2001, 2002, 2003, 2004, 2005
+;; 2001, 2002, 2003, 2004, 2005, 2006
 ;; Free Software Foundation, Inc.
 ;; Mostly by William Schelter.
 ;; x86_64 support added by Jan Hubicka
@@ -66,11 +66,14 @@
    (UNSPEC_STACK_ALLOC		11)
    (UNSPEC_SET_GOT		12)
    (UNSPEC_SSE_PROLOGUE_SAVE	13)
+   (UNSPEC_REG_SAVE		14)
+   (UNSPEC_DEF_CFA		15)
 
    ; TLS support
-   (UNSPEC_TP			15)
-   (UNSPEC_TLS_GD		16)
-   (UNSPEC_TLS_LD_BASE		17)
+   (UNSPEC_TP			16)
+   (UNSPEC_TLS_GD		17)
+   (UNSPEC_TLS_LD_BASE		18)
+   (UNSPEC_TLSDESC		19)
 
    ; Other random patterns
    (UNSPEC_SCAS			20)
@@ -81,6 +84,7 @@
    (UNSPEC_FLDCW		25)
    (UNSPEC_REP			26)
    (UNSPEC_EH_RETURN		27)
+   (UNSPEC_LD_MPIC		28)	; load_macho_picbase
 
    ; For SSE/MMX support:
    (UNSPEC_FIX_NOTRUNC		30)
@@ -183,7 +187,7 @@
 
 ;; Processor type.  This attribute must exactly match the processor_type
 ;; enumeration in i386.h.
-(define_attr "cpu" "i386,i486,pentium,pentiumpro,k6,athlon,pentium4,k8,nocona"
+(define_attr "cpu" "i386,i486,pentium,pentiumpro,k6,athlon,pentium4,k8,nocona,generic32,generic64"
   (const (symbol_ref "ix86_tune")))
 
 ;; A basic instruction type.  Refinements due to arguments to be
@@ -312,8 +316,12 @@
 	      (not (match_operand 0 "memory_operand" "")))
 	   (const_int 0)
 	 (and (eq_attr "type" "imov")
-	      (and (match_operand 0 "register_operand" "")
-	           (match_operand 1 "immediate_operand" "")))
+	      (ior (and (match_operand 0 "register_operand" "")
+			(match_operand 1 "immediate_operand" ""))
+		   (ior (and (match_operand 0 "ax_reg_operand" "")
+			     (match_operand 1 "memory_displacement_only_operand" ""))
+			(and (match_operand 0 "memory_displacement_only_operand" "")
+			     (match_operand 1 "ax_reg_operand" "")))))
 	   (const_int 0)
 	 (and (eq_attr "type" "call")
 	      (match_operand 0 "constant_call_address_operand" ""))
@@ -467,9 +475,10 @@
 (include "athlon.md")
 
 
-;; Operand and operator predicates
+;; Operand and operator predicates and constraints
 
 (include "predicates.md")
+(include "constraints.md")
 
 
 ;; Compare instructions.
@@ -959,8 +968,8 @@
 
 (define_insn "*cmpfp_i_mixed"
   [(set (reg:CCFP FLAGS_REG)
-	(compare:CCFP (match_operand 0 "register_operand" "f#x,x#f")
-		      (match_operand 1 "nonimmediate_operand" "f#x,xm#f")))]
+	(compare:CCFP (match_operand 0 "register_operand" "f,x")
+		      (match_operand 1 "nonimmediate_operand" "f,xm")))]
   "TARGET_MIX_SSE_I387
    && SSE_FLOAT_MODE_P (GET_MODE (operands[0]))
    && GET_MODE (operands[0]) == GET_MODE (operands[1])"
@@ -1008,8 +1017,8 @@
 
 (define_insn "*cmpfp_iu_mixed"
   [(set (reg:CCFPU FLAGS_REG)
-	(compare:CCFPU (match_operand 0 "register_operand" "f#x,x#f")
-		       (match_operand 1 "nonimmediate_operand" "f#x,xm#f")))]
+	(compare:CCFPU (match_operand 0 "register_operand" "f,x")
+		       (match_operand 1 "nonimmediate_operand" "f,xm")))]
   "TARGET_MIX_SSE_I387
    && SSE_FLOAT_MODE_P (GET_MODE (operands[0]))
    && GET_MODE (operands[0]) == GET_MODE (operands[1])"
@@ -1193,7 +1202,7 @@
 }
   [(set (attr "type")
      (cond [(eq_attr "alternative" "2")
-	      (const_string "mmx")
+	      (const_string "mmxadd")
 	    (eq_attr "alternative" "3,4,5")
 	      (const_string "mmxmov")
 	    (eq_attr "alternative" "6")
@@ -1268,14 +1277,12 @@
   "ix86_expand_move (HImode, operands); DONE;")
 
 (define_insn "*pushhi2"
-  [(set (match_operand:HI 0 "push_operand" "=<,<")
-	(match_operand:HI 1 "general_no_elim_operand" "n,r*m"))]
+  [(set (match_operand:HI 0 "push_operand" "=X")
+	(match_operand:HI 1 "nonmemory_no_elim_operand" "rn"))]
   "!TARGET_64BIT"
-  "@
-   push{w}\t{|WORD PTR }%1
-   push{w}\t%1"
+  "push{l}\t%k1"
   [(set_attr "type" "push")
-   (set_attr "mode" "HI")])
+   (set_attr "mode" "SI")])
 
 ;; For 64BIT abi we always round up to 8 bytes.
 (define_insn "*pushhi2_rex64"
@@ -1284,7 +1291,7 @@
   "TARGET_64BIT"
   "push{q}\t%q1"
   [(set_attr "type" "push")
-   (set_attr "mode" "QI")])
+   (set_attr "mode" "DI")])
 
 (define_insn "*movhi_1"
   [(set (match_operand:HI 0 "nonimmediate_operand" "=r,r,r,m")
@@ -1429,18 +1436,16 @@
   "ix86_expand_move (QImode, operands); DONE;")
 
 ;; emit_push_insn when it calls move_by_pieces requires an insn to
-;; "push a byte".  But actually we use pushw, which has the effect
-;; of rounding the amount pushed up to a halfword.
+;; "push a byte".  But actually we use pushl, which has the effect
+;; of rounding the amount pushed up to a word.
 
 (define_insn "*pushqi2"
-  [(set (match_operand:QI 0 "push_operand" "=X,X")
-	(match_operand:QI 1 "nonmemory_no_elim_operand" "n,r"))]
+  [(set (match_operand:QI 0 "push_operand" "=X")
+	(match_operand:QI 1 "nonmemory_no_elim_operand" "rn"))]
   "!TARGET_64BIT"
-  "@
-   push{w}\t{|word ptr }%1
-   push{w}\t%w1"
+  "push{l}\t%k1"
   [(set_attr "type" "push")
-   (set_attr "mode" "HI")])
+   (set_attr "mode" "SI")])
 
 ;; For 64BIT abi we always round up to 8 bytes.
 (define_insn "*pushqi2_rex64"
@@ -1449,7 +1454,7 @@
   "TARGET_64BIT"
   "push{q}\t%q1"
   [(set_attr "type" "push")
-   (set_attr "mode" "QI")])
+   (set_attr "mode" "DI")])
 
 ;; Situation is quite tricky about when to choose full sized (SImode) move
 ;; over QImode moves.  For Q_REG -> Q_REG move we use full size only for
@@ -1463,7 +1468,7 @@
 ;; partial register stall can be caused there.  Then we use movzx.
 (define_insn "*movqi_1"
   [(set (match_operand:QI 0 "nonimmediate_operand" "=q,q ,q ,r,r ,?r,m")
-	(match_operand:QI 1 "general_operand"      " q,qn,qm,q,rn,m ,qn"))]
+	(match_operand:QI 1 "general_operand"      " q,qn,qm,q,rn,qm,qn"))]
   "GET_CODE (operands[0]) != MEM || GET_CODE (operands[1]) != MEM"
 {
   switch (get_attr_type (insn))
@@ -1479,7 +1484,8 @@
     }
 }
   [(set (attr "type")
-     (cond [(eq_attr "alternative" "5")
+     (cond [(and (eq_attr "alternative" "5")
+		 (not (match_operand:QI 1 "aligned_operand" "")))
 	      (const_string "imovx")
 	    (ne (symbol_ref "optimize_size") (const_int 0))
 	      (const_string "imov")
@@ -1489,7 +1495,7 @@
 		      (eq (symbol_ref "TARGET_QIMODE_MATH")
 			  (const_int 0))))
 	      (const_string "imov")
-	    (eq_attr "alternative" "3")
+	    (eq_attr "alternative" "3,5")
 	      (const_string "imovx")
 	    (and (ne (symbol_ref "TARGET_MOVX")
 		     (const_int 0))
@@ -1506,8 +1512,12 @@
 	       (const_string "SI")
 	     (and (eq_attr "type" "imov")
 		  (and (eq_attr "alternative" "0,1")
-		       (ne (symbol_ref "TARGET_PARTIAL_REG_DEPENDENCY")
-			   (const_int 0))))
+		       (and (ne (symbol_ref "TARGET_PARTIAL_REG_DEPENDENCY")
+				(const_int 0))
+			    (and (eq (symbol_ref "optimize_size")
+				     (const_int 0))
+			    	 (eq (symbol_ref "TARGET_PARTIAL_REG_STALL")
+				     (const_int 0))))))
 	       (const_string "SI")
 	     ;; Avoid partial register stalls when not using QImode arithmetic
 	     (and (eq_attr "type" "imov")
@@ -1856,7 +1866,8 @@
 (define_split
   [(set (match_operand:DI 0 "push_operand" "")
         (match_operand:DI 1 "immediate_operand" ""))]
-  "TARGET_64BIT && (flag_peephole2 ? flow2_completed : reload_completed)
+  "TARGET_64BIT && ((optimize > 0 && flag_peephole2)
+		    ? flow2_completed : reload_completed)
    && !symbolic_operand (operands[1], DImode)
    && !x86_64_immediate_operand (operands[1], DImode)"
   [(set (match_dup 0) (match_dup 1))
@@ -2007,7 +2018,7 @@
 }
   [(set (attr "type")
      (cond [(eq_attr "alternative" "5")
-	      (const_string "mmx")
+	      (const_string "mmxadd")
 	    (eq_attr "alternative" "6,7,8")
 	      (const_string "mmxmov")
 	    (eq_attr "alternative" "9")
@@ -2085,7 +2096,8 @@
 (define_split
   [(set (match_operand:DI 0 "memory_operand" "")
         (match_operand:DI 1 "immediate_operand" ""))]
-  "TARGET_64BIT && (flag_peephole2 ? flow2_completed : reload_completed)
+  "TARGET_64BIT && ((optimize > 0 && flag_peephole2)
+		    ? flow2_completed : reload_completed)
    && !symbolic_operand (operands[1], DImode)
    && !x86_64_immediate_operand (operands[1], DImode)"
   [(set (match_dup 2) (match_dup 3))
@@ -2139,24 +2151,16 @@
       gcc_unreachable ();
     }
 }
-  [(set_attr "type" "ssemov,ssemov,ssemov")
+  [(set_attr "type" "sselog1,ssemov,ssemov")
    (set (attr "mode")
-        (cond [(eq (symbol_ref "TARGET_SSE2") (const_int 0))
+	(cond [(ior (eq (symbol_ref "TARGET_SSE2") (const_int 0))
+		    (ne (symbol_ref "optimize_size") (const_int 0)))
 		 (const_string "V4SF")
-
-	       (eq_attr "alternative" "0,1")
-		 (if_then_else
-		   (ne (symbol_ref "optimize_size")
-		       (const_int 0))
-		   (const_string "V4SF")
-		   (const_string "TI"))
-	       (eq_attr "alternative" "2")
-		 (if_then_else
-		   (ne (symbol_ref "optimize_size")
-		       (const_int 0))
-		   (const_string "V4SF")
-		   (const_string "TI"))]
-	       (const_string "TI")))])
+	       (and (eq_attr "alternative" "2")
+		    (ne (symbol_ref "TARGET_SSE_TYPELESS_STORES")
+			(const_int 0)))
+		 (const_string "V4SF")]
+	      (const_string "TI")))])
 
 (define_insn "*movti_rex64"
   [(set (match_operand:TI 0 "nonimmediate_operand" "=r,o,x,x,xm")
@@ -2184,7 +2188,7 @@
       gcc_unreachable ();
     }
 }
-  [(set_attr "type" "*,*,ssemov,ssemov,ssemov")
+  [(set_attr "type" "*,*,sselog1,ssemov,ssemov")
    (set (attr "mode")
         (cond [(eq_attr "alternative" "2,3")
 		 (if_then_else
@@ -2218,7 +2222,7 @@
 
 (define_insn "*pushsf"
   [(set (match_operand:SF 0 "push_operand" "=<,<,<")
-	(match_operand:SF 1 "general_no_elim_operand" "f#rx,rFm#fx,x#rf"))]
+	(match_operand:SF 1 "general_no_elim_operand" "f,rFm,x"))]
   "!TARGET_64BIT"
 {
   /* Anything else should be already split before reg-stack.  */
@@ -2231,7 +2235,7 @@
 
 (define_insn "*pushsf_rex64"
   [(set (match_operand:SF 0 "push_operand" "=X,X,X")
-	(match_operand:SF 1 "nonmemory_no_elim_operand" "f#rx,rF#fx,x#rf"))]
+	(match_operand:SF 1 "nonmemory_no_elim_operand" "f,rF,x"))]
   "TARGET_64BIT"
 {
   /* Anything else should be already split before reg-stack.  */
@@ -2270,9 +2274,9 @@
 
 (define_insn "*movsf_1"
   [(set (match_operand:SF 0 "nonimmediate_operand"
-	  "=f#xr,m   ,f#xr,r#xf  ,m    ,x#rf,x#rf,x#rf ,m   ,!*y,!rm,!*y")
+	  "=f,m   ,f,r  ,m    ,x,x,x ,m   ,!*y,!rm,!*y")
 	(match_operand:SF 1 "general_operand"
-	  "fm#rx,f#rx,G   ,rmF#fx,Fr#fx,C   ,x   ,xm#rf,x#rf,rm ,*y ,*y"))]
+	  "fm,f,G   ,rmF,Fr,C   ,x   ,xm,x,rm ,*y ,*y"))]
   "!(MEM_P (operands[0]) && MEM_P (operands[1]))
    && (reload_in_progress || reload_completed
        || (ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_LARGE)
@@ -2321,7 +2325,7 @@
       gcc_unreachable ();
     }
 }
-  [(set_attr "type" "fmov,fmov,fmov,imov,imov,ssemov,ssemov,ssemov,ssemov,mmxmov,mmxmov,mmxmov")
+  [(set_attr "type" "fmov,fmov,fmov,imov,imov,sselog1,ssemov,ssemov,ssemov,mmxmov,mmxmov,mmxmov")
    (set (attr "mode")
         (cond [(eq_attr "alternative" "3,4,9,10")
 		 (const_string "SI")
@@ -2385,7 +2389,7 @@
 
 (define_insn "*pushdf_nointeger"
   [(set (match_operand:DF 0 "push_operand" "=<,<,<,<")
-	(match_operand:DF 1 "general_no_elim_operand" "f#Y,Fo#fY,*r#fY,Y#f"))]
+	(match_operand:DF 1 "general_no_elim_operand" "f,Fo,*r,Y"))]
   "!TARGET_64BIT && !TARGET_INTEGER_DFMODE_MOVES"
 {
   /* This insn should be already split before reg-stack.  */
@@ -2397,7 +2401,7 @@
 
 (define_insn "*pushdf_integer"
   [(set (match_operand:DF 0 "push_operand" "=<,<,<")
-	(match_operand:DF 1 "general_no_elim_operand" "f#rY,rFo#fY,Y#rf"))]
+	(match_operand:DF 1 "general_no_elim_operand" "f,rFo,Y"))]
   "TARGET_64BIT || TARGET_INTEGER_DFMODE_MOVES"
 {
   /* This insn should be already split before reg-stack.  */
@@ -2437,9 +2441,9 @@
 
 (define_insn "*movdf_nointeger"
   [(set (match_operand:DF 0 "nonimmediate_operand"
-			"=f#Y,m  ,f#Y,*r  ,o  ,Y*x#f,Y*x#f,Y*x#f  ,m    ")
+			"=f,m,f,*r  ,o  ,Y*x,Y*x,Y*x ,m  ")
 	(match_operand:DF 1 "general_operand"
-			"fm#Y,f#Y,G  ,*roF,F*r,C    ,Y*x#f,HmY*x#f,Y*x#f"))]
+			"fm,f,G,*roF,F*r,C  ,Y*x,mY*x,Y*x"))]
   "(GET_CODE (operands[0]) != MEM || GET_CODE (operands[1]) != MEM)
    && ((optimize_size || !TARGET_INTEGER_DFMODE_MOVES) && !TARGET_64BIT)
    && (reload_in_progress || reload_completed
@@ -2503,7 +2507,7 @@
       gcc_unreachable ();
     }
 }
-  [(set_attr "type" "fmov,fmov,fmov,multi,multi,ssemov,ssemov,ssemov,ssemov")
+  [(set_attr "type" "fmov,fmov,fmov,multi,multi,sselog1,ssemov,ssemov,ssemov")
    (set (attr "mode")
         (cond [(eq_attr "alternative" "0,1,2")
 		 (const_string "DF")
@@ -2557,9 +2561,9 @@
 
 (define_insn "*movdf_integer"
   [(set (match_operand:DF 0 "nonimmediate_operand"
-		"=f#Yr,m   ,f#Yr,r#Yf  ,o    ,Y*x#rf,Y*x#rf,Y*x#rf,m")
+		"=f,m,f,r  ,o ,Y*x,Y*x,Y*x,m  ")
 	(match_operand:DF 1 "general_operand"
-		"fm#Yr,f#Yr,G   ,roF#Yf,Fr#Yf,C     ,Y*x#rf,m     ,Y*x#rf"))]
+		"fm,f,G,roF,Fr,C  ,Y*x,m  ,Y*x"))]
   "(GET_CODE (operands[0]) != MEM || GET_CODE (operands[1]) != MEM)
    && ((!optimize_size && TARGET_INTEGER_DFMODE_MOVES) || TARGET_64BIT)
    && (reload_in_progress || reload_completed
@@ -2624,7 +2628,7 @@
       gcc_unreachable();
     }
 }
-  [(set_attr "type" "fmov,fmov,fmov,multi,multi,ssemov,ssemov,ssemov,ssemov")
+  [(set_attr "type" "fmov,fmov,fmov,multi,multi,sselog1,ssemov,ssemov,ssemov")
    (set (attr "mode")
         (cond [(eq_attr "alternative" "0,1,2")
 		 (const_string "DF")
@@ -2732,7 +2736,7 @@
 
 (define_insn "*pushxf_integer"
   [(set (match_operand:XF 0 "push_operand" "=<,<")
-	(match_operand:XF 1 "general_no_elim_operand" "f#r,ro#f"))]
+	(match_operand:XF 1 "general_no_elim_operand" "f,ro"))]
   "!optimize_size"
 {
   /* This insn should be already split before reg-stack.  */
@@ -2804,8 +2808,8 @@
    (set_attr "mode" "XF,XF,XF,SI,SI")])
 
 (define_insn "*movxf_integer"
-  [(set (match_operand:XF 0 "nonimmediate_operand" "=f#r,m,f#r,r#f,o")
-	(match_operand:XF 1 "general_operand" "fm#r,f#r,G,roF#f,Fr#f"))]
+  [(set (match_operand:XF 0 "nonimmediate_operand" "=f,m,f,r,o")
+	(match_operand:XF 1 "general_operand" "fm,f,G,roF,Fr"))]
   "!optimize_size
    && (GET_CODE (operands[0]) != MEM || GET_CODE (operands[1]) != MEM)
    && (reload_in_progress || reload_completed
@@ -2935,7 +2939,7 @@
       gcc_unreachable ();
     }
 }
-  [(set_attr "type" "*,*,ssemov,ssemov,ssemov")
+  [(set_attr "type" "*,*,sselog1,ssemov,ssemov")
    (set (attr "mode")
         (cond [(eq_attr "alternative" "2,3")
 		 (if_then_else
@@ -3028,13 +3032,14 @@
   [(set_attr "type" "imovx,alu1")
    (set_attr "mode" "HI")])
 
-(define_insn "*zero_extendqihi2_movzbw"
+; zero extend to SImode here to avoid partial register stalls
+(define_insn "*zero_extendqihi2_movzbl"
   [(set (match_operand:HI 0 "register_operand" "=r")
      (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "qm")))]
   "(!TARGET_ZERO_EXTEND_WITH_AND || optimize_size) && reload_completed"
-  "movz{bw|x}\t{%1, %0|%0, %1}"
+  "movz{bl|x}\t{%1, %k0|%k0, %k1}"
   [(set_attr "type" "imovx")
-   (set_attr "mode" "HI")])
+   (set_attr "mode" "SI")])
 
 ;; For the movzbw case strip only the clobber
 (define_split
@@ -3517,14 +3522,24 @@
   /* ??? Needed for compress_float_constant since all fp constants
      are LEGITIMATE_CONSTANT_P.  */
   if (GET_CODE (operands[1]) == CONST_DOUBLE)
-    operands[1] = validize_mem (force_const_mem (SFmode, operands[1]));
+    {
+      if ((!TARGET_SSE2 || TARGET_MIX_SSE_I387)
+	  && standard_80387_constant_p (operands[1]) > 0)
+	{
+	  operands[1] = simplify_const_unary_operation
+	    (FLOAT_EXTEND, DFmode, operands[1], SFmode);
+	  emit_move_insn_1 (operands[0], operands[1]);
+	  DONE;
+	}
+      operands[1] = validize_mem (force_const_mem (SFmode, operands[1]));
+    }
   if (GET_CODE (operands[0]) == MEM && GET_CODE (operands[1]) == MEM)
     operands[1] = force_reg (SFmode, operands[1]);
 })
 
 (define_insn "*extendsfdf2_mixed"
-  [(set (match_operand:DF 0 "nonimmediate_operand" "=f#Y,m#fY,Y#f")
-        (float_extend:DF (match_operand:SF 1 "nonimmediate_operand" "fm#Y,f#Y,mY#f")))]
+  [(set (match_operand:DF 0 "nonimmediate_operand" "=f,m,Y")
+        (float_extend:DF (match_operand:SF 1 "nonimmediate_operand" "fm,f,mY")))]
   "TARGET_SSE2 && TARGET_MIX_SSE_I387
    && (GET_CODE (operands[0]) != MEM || GET_CODE (operands[1]) != MEM)"
 {
@@ -3590,7 +3605,16 @@
   /* ??? Needed for compress_float_constant since all fp constants
      are LEGITIMATE_CONSTANT_P.  */
   if (GET_CODE (operands[1]) == CONST_DOUBLE)
-    operands[1] = validize_mem (force_const_mem (SFmode, operands[1]));
+    {
+      if (standard_80387_constant_p (operands[1]) > 0)
+	{
+	  operands[1] = simplify_const_unary_operation
+	    (FLOAT_EXTEND, XFmode, operands[1], SFmode);
+	  emit_move_insn_1 (operands[0], operands[1]);
+	  DONE;
+	}
+      operands[1] = validize_mem (force_const_mem (SFmode, operands[1]));
+    }
   if (GET_CODE (operands[0]) == MEM && GET_CODE (operands[1]) == MEM)
     operands[1] = force_reg (SFmode, operands[1]);
 })
@@ -3629,7 +3653,16 @@
   /* ??? Needed for compress_float_constant since all fp constants
      are LEGITIMATE_CONSTANT_P.  */
   if (GET_CODE (operands[1]) == CONST_DOUBLE)
-    operands[1] = validize_mem (force_const_mem (DFmode, operands[1]));
+    {
+      if (standard_80387_constant_p (operands[1]) > 0)
+	{
+	  operands[1] = simplify_const_unary_operation
+	    (FLOAT_EXTEND, XFmode, operands[1], DFmode);
+	  emit_move_insn_1 (operands[0], operands[1]);
+	  DONE;
+	}
+      operands[1] = validize_mem (force_const_mem (DFmode, operands[1]));
+    }
   if (GET_CODE (operands[0]) == MEM && GET_CODE (operands[1]) == MEM)
     operands[1] = force_reg (DFmode, operands[1]);
 })
@@ -3839,7 +3872,7 @@
 })
 
 (define_insn "*truncxfsf2_mixed"
-  [(set (match_operand:SF 0 "nonimmediate_operand" "=m,?f#rx,?r#fx,?x#rf")
+  [(set (match_operand:SF 0 "nonimmediate_operand" "=m,?f,?r,?x")
 	(float_truncate:SF
 	 (match_operand:XF 1 "register_operand" "f,f,f,f")))
    (clobber (match_operand:SF 2 "memory_operand" "=X,m,m,m"))]
@@ -3866,7 +3899,7 @@
    (set_attr "mode" "SF")])
 
 (define_insn "*truncxfsf2_i387"
-  [(set (match_operand:SF 0 "nonimmediate_operand" "=m,?f#r,?r#f")
+  [(set (match_operand:SF 0 "nonimmediate_operand" "=m,?f,?r")
 	(float_truncate:SF
 	 (match_operand:XF 1 "register_operand" "f,f,f")))
    (clobber (match_operand:SF 2 "memory_operand" "=X,m,m"))]
@@ -3937,7 +3970,7 @@
 })
 
 (define_insn "*truncxfdf2_mixed"
-  [(set (match_operand:DF 0 "nonimmediate_operand" "=m,?f#rY,?r#fY,?Y#rf")
+  [(set (match_operand:DF 0 "nonimmediate_operand" "=m,?f,?r,?Y")
 	(float_truncate:DF
 	 (match_operand:XF 1 "register_operand" "f,f,f,f")))
    (clobber (match_operand:DF 2 "memory_operand" "=X,m,m,m"))]
@@ -3964,7 +3997,7 @@
    (set_attr "mode" "DF")])
 
 (define_insn "*truncxfdf2_i387"
-  [(set (match_operand:DF 0 "nonimmediate_operand" "=m,?f#r,?r#f")
+  [(set (match_operand:DF 0 "nonimmediate_operand" "=m,?f,?r")
 	(float_truncate:DF
 	 (match_operand:XF 1 "register_operand" "f,f,f")))
    (clobber (match_operand:DF 2 "memory_operand" "=X,m,m"))]
@@ -4145,7 +4178,7 @@
   [(match_scratch:DF 2 "Y")
    (set (match_operand:SSEMODEI24 0 "register_operand" "")
 	(fix:SSEMODEI24 (match_operand:DF 1 "memory_operand" "")))]
-  "TARGET_K8 && !optimize_size"
+  "(TARGET_K8 || TARGET_GENERIC64) && !optimize_size"
   [(set (match_dup 2) (match_dup 1))
    (set (match_dup 0) (fix:SSEMODEI24 (match_dup 2)))]
   "")
@@ -4154,7 +4187,7 @@
   [(match_scratch:SF 2 "x")
    (set (match_operand:SSEMODEI24 0 "register_operand" "")
 	(fix:SSEMODEI24 (match_operand:SF 1 "memory_operand" "")))]
-  "TARGET_K8 && !optimize_size"
+  "(TARGET_K8 || TARGET_GENERIC64) && !optimize_size"
   [(set (match_dup 2) (match_dup 1))
    (set (match_dup 0) (fix:SSEMODEI24 (match_dup 2)))]
   "")
@@ -4438,7 +4471,7 @@
   "")
 
 (define_insn "*floatsisf2_mixed"
-  [(set (match_operand:SF 0 "register_operand" "=f#x,?f#x,x#f,x#f")
+  [(set (match_operand:SF 0 "register_operand" "=f,?f,x,x")
 	(float:SF (match_operand:SI 1 "nonimmediate_operand" "m,r,r,mr")))]
   "TARGET_MIX_SSE_I387"
   "@
@@ -4481,7 +4514,7 @@
   "")
 
 (define_insn "*floatdisf2_mixed"
-  [(set (match_operand:SF 0 "register_operand" "=f#x,?f#x,x#f,x#f")
+  [(set (match_operand:SF 0 "register_operand" "=f,?f,x,x")
 	(float:SF (match_operand:DI 1 "nonimmediate_operand" "m,r,r,mr")))]
   "TARGET_64BIT && TARGET_MIX_SSE_I387"
   "@
@@ -4549,7 +4582,7 @@
   "")
 
 (define_insn "*floatsidf2_mixed"
-  [(set (match_operand:DF 0 "register_operand" "=f#Y,?f#Y,Y#f,Y#f")
+  [(set (match_operand:DF 0 "register_operand" "=f,?f,Y,Y")
 	(float:DF (match_operand:SI 1 "nonimmediate_operand" "m,r,r,mr")))]
   "TARGET_SSE2 && TARGET_MIX_SSE_I387"
   "@
@@ -4592,7 +4625,7 @@
   "")
 
 (define_insn "*floatdidf2_mixed"
-  [(set (match_operand:DF 0 "register_operand" "=f#Y,?f#Y,Y#f,Y#f")
+  [(set (match_operand:DF 0 "register_operand" "=f,?f,Y,Y")
 	(float:DF (match_operand:DI 1 "nonimmediate_operand" "m,r,r,mr")))]
   "TARGET_64BIT && TARGET_SSE2 && TARGET_MIX_SSE_I387"
   "@
@@ -6685,7 +6718,7 @@
 		    (match_dup 2))))]
   "TARGET_64BIT && ix86_match_ccmode (insn, CCmode)
    && ix86_binary_operator_ok (MINUS, SImode, operands)"
-  "sub{q}\t{%2, %0|%0, %2}"
+  "sub{l}\t{%2, %1|%1, %2}"
   [(set_attr "type" "alu")
    (set_attr "mode" "DI")])
 
@@ -7867,6 +7900,9 @@
 		   (match_operand:SI 2 "const_int_operand" ""))
 		 (const_int 0)))]
   "ix86_match_ccmode (insn, CCNOmode)
+   && INTVAL (operands[1]) > 0
+   && INTVAL (operands[2]) >= 0
+   && INTVAL (operands[1]) + INTVAL (operands[2]) <= 32
    && (GET_MODE (operands[0]) == SImode
        || (TARGET_64BIT && GET_MODE (operands[0]) == DImode)
        || GET_MODE (operands[0]) == HImode
@@ -7882,8 +7918,8 @@
 		 (const_int 0)))]
   "TARGET_64BIT
    && ix86_match_ccmode (insn, CCNOmode)
-   /* The code below cannot deal with constants outside HOST_WIDE_INT.  */
-   && INTVAL (operands[1]) + INTVAL (operands[2]) < HOST_BITS_PER_WIDE_INT
+   && INTVAL (operands[1]) > 0
+   && INTVAL (operands[2]) >= 0
    /* Ensure that resulting mask is zero or sign extended operand.  */
    && (INTVAL (operands[1]) + INTVAL (operands[2]) <= 32
        || (INTVAL (operands[1]) + INTVAL (operands[2]) == 64
@@ -7938,8 +7974,11 @@
       val = gen_lowpart (QImode, val);
     }
 
-  mask  = ((HOST_WIDE_INT)1 << (pos + len)) - 1;
-  mask &= ~(((HOST_WIDE_INT)1 << pos) - 1);
+  if (len == HOST_BITS_PER_WIDE_INT)
+    mask = -1;
+  else
+    mask = ((HOST_WIDE_INT)1 << len) - 1;
+  mask <<= pos;
 
   operands[2] = gen_rtx_AND (mode, val, gen_int_mode (mask, mode));
 })
@@ -9506,10 +9545,10 @@
   "ix86_expand_fp_absneg_operator (ABS, SFmode, operands); DONE;")
 
 (define_insn "*absnegsf2_mixed"
-  [(set (match_operand:SF 0 "nonimmediate_operand"    "=x#f,x#f,f#x,rm")
+  [(set (match_operand:SF 0 "nonimmediate_operand"    "=x  ,x,f,rm")
 	(match_operator:SF 3 "absneg_operator"
-	  [(match_operand:SF 1 "nonimmediate_operand" "0   ,x#f,0  ,0")]))
-   (use (match_operand:V4SF 2 "nonimmediate_operand"  "xm  ,0  ,X  ,X"))
+	  [(match_operand:SF 1 "nonimmediate_operand" "0   ,x,0,0 ")]))
+   (use (match_operand:V4SF 2 "nonimmediate_operand"  "xm  ,0,X,X "))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_SSE_MATH && TARGET_MIX_SSE_I387
    && ix86_unary_operator_ok (GET_CODE (operands[3]), SFmode, operands)"
@@ -9602,10 +9641,10 @@
   "ix86_expand_fp_absneg_operator (ABS, DFmode, operands); DONE;")
 
 (define_insn "*absnegdf2_mixed"
-  [(set (match_operand:DF 0 "nonimmediate_operand"    "=Y#f,Y#f,f#Y,rm")
+  [(set (match_operand:DF 0 "nonimmediate_operand"    "=Y,Y,f,rm")
 	(match_operator:DF 3 "absneg_operator"
-	  [(match_operand:DF 1 "nonimmediate_operand" "0   ,Y#f,0  ,0")]))
-   (use (match_operand:V2DF 2 "nonimmediate_operand"  "Ym  ,0  ,X  ,X"))
+	  [(match_operand:DF 1 "nonimmediate_operand" "0 ,Y,0,0")]))
+   (use (match_operand:V2DF 2 "nonimmediate_operand"  "Ym,0,X,X"))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_SSE2 && TARGET_SSE_MATH && TARGET_MIX_SSE_I387
    && ix86_unary_operator_ok (GET_CODE (operands[3]), DFmode, operands)"
@@ -9614,8 +9653,8 @@
 (define_insn "*absnegdf2_sse"
   [(set (match_operand:DF 0 "nonimmediate_operand"    "=Y,Y,rm")
 	(match_operator:DF 3 "absneg_operator"
-	  [(match_operand:DF 1 "nonimmediate_operand" "0 ,Y,0")]))
-   (use (match_operand:V2DF 2 "nonimmediate_operand"  "Ym,0,X"))
+	  [(match_operand:DF 1 "nonimmediate_operand" "0 ,Y,0 ")]))
+   (use (match_operand:V2DF 2 "nonimmediate_operand"  "Ym,0,X "))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_SSE2 && TARGET_SSE_MATH
    && ix86_unary_operator_ok (GET_CODE (operands[3]), DFmode, operands)"
@@ -9693,7 +9732,7 @@
 
 (define_expand "absxf2"
   [(set (match_operand:XF 0 "nonimmediate_operand" "")
-	(neg:XF (match_operand:XF 1 "nonimmediate_operand" "")))]
+	(abs:XF (match_operand:XF 1 "nonimmediate_operand" "")))]
   "TARGET_80387"
   "ix86_expand_fp_absneg_operator (ABS, XFmode, operands); DONE;")
 
@@ -9866,7 +9905,7 @@
 (define_insn "*negsf2_1"
   [(set (match_operand:SF 0 "register_operand" "=f")
 	(neg:SF (match_operand:SF 1 "register_operand" "0")))]
-  "TARGET_80387 && reload_completed"
+  "TARGET_80387 && (reload_completed || !TARGET_SSE_MATH)"
   "fchs"
   [(set_attr "type" "fsgn")
    (set_attr "mode" "SF")])
@@ -9874,7 +9913,7 @@
 (define_insn "*negdf2_1"
   [(set (match_operand:DF 0 "register_operand" "=f")
 	(neg:DF (match_operand:DF 1 "register_operand" "0")))]
-  "TARGET_80387 && reload_completed"
+  "TARGET_80387 && (reload_completed || !(TARGET_SSE2 && TARGET_SSE_MATH))"
   "fchs"
   [(set_attr "type" "fsgn")
    (set_attr "mode" "DF")])
@@ -9882,7 +9921,7 @@
 (define_insn "*negxf2_1"
   [(set (match_operand:XF 0 "register_operand" "=f")
 	(neg:XF (match_operand:XF 1 "register_operand" "0")))]
-  "TARGET_80387 && reload_completed"
+  "TARGET_80387"
   "fchs"
   [(set_attr "type" "fsgn")
    (set_attr "mode" "XF")])
@@ -9890,7 +9929,7 @@
 (define_insn "*abssf2_1"
   [(set (match_operand:SF 0 "register_operand" "=f")
 	(abs:SF (match_operand:SF 1 "register_operand" "0")))]
-  "TARGET_80387 && reload_completed"
+  "TARGET_80387 && (reload_completed || !TARGET_SSE_MATH)"
   "fabs"
   [(set_attr "type" "fsgn")
    (set_attr "mode" "SF")])
@@ -9898,7 +9937,7 @@
 (define_insn "*absdf2_1"
   [(set (match_operand:DF 0 "register_operand" "=f")
 	(abs:DF (match_operand:DF 1 "register_operand" "0")))]
-  "TARGET_80387 && reload_completed"
+  "TARGET_80387 && (reload_completed || !(TARGET_SSE2 && TARGET_SSE_MATH))"
   "fabs"
   [(set_attr "type" "fsgn")
    (set_attr "mode" "DF")])
@@ -9906,7 +9945,7 @@
 (define_insn "*absxf2_1"
   [(set (match_operand:XF 0 "register_operand" "=f")
 	(abs:XF (match_operand:XF 1 "register_operand" "0")))]
-  "TARGET_80387 && reload_completed"
+  "TARGET_80387"
   "fabs"
   [(set_attr "type" "fsgn")
    (set_attr "mode" "DF")])
@@ -10385,6 +10424,42 @@
 	   (const_string "ishift")))
    (set_attr "mode" "DI")])
 
+(define_insn "*ashldi3_cconly_rex64"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (ashift:DI (match_operand:DI 1 "nonimmediate_operand" "0")
+		     (match_operand:QI 2 "immediate_operand" "e"))
+	  (const_int 0)))
+   (clobber (match_scratch:DI 0 "=r"))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCGOCmode)
+   && ix86_binary_operator_ok (ASHIFT, DImode, operands)"
+{
+  switch (get_attr_type (insn))
+    {
+    case TYPE_ALU:
+      gcc_assert (operands[2] == const1_rtx);
+      return "add{q}\t{%0, %0|%0, %0}";
+
+    default:
+      if (REG_P (operands[2]))
+	return "sal{q}\t{%b2, %0|%0, %b2}";
+      else if (operands[2] == const1_rtx
+	       && (TARGET_SHIFT1 || optimize_size))
+	return "sal{q}\t%0";
+      else
+	return "sal{q}\t{%2, %0|%0, %2}";
+    }
+}
+  [(set (attr "type")
+     (cond [(and (and (ne (symbol_ref "TARGET_DOUBLE_WITH_ADD")
+		          (const_int 0))
+		      (match_operand 0 "register_operand" ""))
+		 (match_operand 2 "const1_operand" ""))
+	      (const_string "alu")
+	   ]
+	   (const_string "ishift")))
+   (set_attr "mode" "DI")])
+
 (define_insn "*ashldi3_1"
   [(set (match_operand:DI 0 "register_operand" "=&r,r")
 	(ashift:DI (match_operand:DI 1 "reg_or_pm1_operand" "n,0")
@@ -10413,7 +10488,8 @@
 	(ashift:DI (match_operand:DI 1 "nonmemory_operand" "")
 		   (match_operand:QI 2 "nonmemory_operand" "")))
    (clobber (reg:CC FLAGS_REG))]
-  "!TARGET_64BIT && (flag_peephole2 ? flow2_completed : reload_completed)"
+  "!TARGET_64BIT && ((optimize > 0 && flag_peephole2)
+		     ? flow2_completed : reload_completed)"
   [(const_int 0)]
   "ix86_split_ashl (operands, NULL_RTX, DImode); DONE;")
 
@@ -10563,7 +10639,7 @@
   [(const_int 0)]
 {
   rtx pat, clob;
-  emit_move_insn (operands[1], operands[0]);
+  emit_move_insn (operands[0], operands[1]);
   pat = gen_rtx_SET (VOIDmode, operands[0],
 		     gen_rtx_ASHIFT (GET_MODE (operands[0]),
 				     operands[0], operands[2]));
@@ -10665,6 +10741,42 @@
 	   (const_string "ishift")))
    (set_attr "mode" "SI")])
 
+(define_insn "*ashlsi3_cconly"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (ashift:SI (match_operand:SI 1 "nonimmediate_operand" "0")
+		     (match_operand:QI 2 "const_1_to_31_operand" "I"))
+	  (const_int 0)))
+   (clobber (match_scratch:SI 0 "=r"))]
+  "ix86_match_ccmode (insn, CCGOCmode)
+   && ix86_binary_operator_ok (ASHIFT, SImode, operands)"
+{
+  switch (get_attr_type (insn))
+    {
+    case TYPE_ALU:
+      gcc_assert (operands[2] == const1_rtx);
+      return "add{l}\t{%0, %0|%0, %0}";
+
+    default:
+      if (REG_P (operands[2]))
+	return "sal{l}\t{%b2, %0|%0, %b2}";
+      else if (operands[2] == const1_rtx
+	       && (TARGET_SHIFT1 || optimize_size))
+	return "sal{l}\t%0";
+      else
+	return "sal{l}\t{%2, %0|%0, %2}";
+    }
+}
+  [(set (attr "type")
+     (cond [(and (and (ne (symbol_ref "TARGET_DOUBLE_WITH_ADD")
+		          (const_int 0))
+		      (match_operand 0 "register_operand" ""))
+		 (match_operand 2 "const1_operand" ""))
+	      (const_string "alu")
+	   ]
+	   (const_string "ishift")))
+   (set_attr "mode" "SI")])
+
 (define_insn "*ashlsi3_cmp_zext"
   [(set (reg FLAGS_REG)
 	(compare
@@ -10821,6 +10933,42 @@
 	   (const_string "ishift")))
    (set_attr "mode" "HI")])
 
+(define_insn "*ashlhi3_cconly"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (ashift:HI (match_operand:HI 1 "nonimmediate_operand" "0")
+		     (match_operand:QI 2 "const_1_to_31_operand" "I"))
+	  (const_int 0)))
+   (clobber (match_scratch:HI 0 "=r"))]
+  "ix86_match_ccmode (insn, CCGOCmode)
+   && ix86_binary_operator_ok (ASHIFT, HImode, operands)"
+{
+  switch (get_attr_type (insn))
+    {
+    case TYPE_ALU:
+      gcc_assert (operands[2] == const1_rtx);
+      return "add{w}\t{%0, %0|%0, %0}";
+
+    default:
+      if (REG_P (operands[2]))
+	return "sal{w}\t{%b2, %0|%0, %b2}";
+      else if (operands[2] == const1_rtx
+	       && (TARGET_SHIFT1 || optimize_size))
+	return "sal{w}\t%0";
+      else
+	return "sal{w}\t{%2, %0|%0, %2}";
+    }
+}
+  [(set (attr "type")
+     (cond [(and (and (ne (symbol_ref "TARGET_DOUBLE_WITH_ADD")
+		          (const_int 0))
+		      (match_operand 0 "register_operand" ""))
+		 (match_operand 2 "const1_operand" ""))
+	      (const_string "alu")
+	   ]
+	   (const_string "ishift")))
+   (set_attr "mode" "HI")])
+
 (define_expand "ashlqi3"
   [(set (match_operand:QI 0 "nonimmediate_operand" "")
 	(ashift:QI (match_operand:QI 1 "nonimmediate_operand" "")
@@ -10979,6 +11127,42 @@
 	   (const_string "ishift")))
    (set_attr "mode" "QI")])
 
+(define_insn "*ashlqi3_cconly"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (ashift:QI (match_operand:QI 1 "nonimmediate_operand" "0")
+		     (match_operand:QI 2 "const_1_to_31_operand" "I"))
+	  (const_int 0)))
+   (clobber (match_scratch:QI 0 "=q"))]
+  "ix86_match_ccmode (insn, CCGOCmode)
+   && ix86_binary_operator_ok (ASHIFT, QImode, operands)"
+{
+  switch (get_attr_type (insn))
+    {
+    case TYPE_ALU:
+      gcc_assert (operands[2] == const1_rtx);
+      return "add{b}\t{%0, %0|%0, %0}";
+
+    default:
+      if (REG_P (operands[2]))
+	return "sal{b}\t{%b2, %0|%0, %b2}";
+      else if (operands[2] == const1_rtx
+	       && (TARGET_SHIFT1 || optimize_size))
+	return "sal{b}\t%0";
+      else
+	return "sal{b}\t{%2, %0|%0, %2}";
+    }
+}
+  [(set (attr "type")
+     (cond [(and (and (ne (symbol_ref "TARGET_DOUBLE_WITH_ADD")
+		          (const_int 0))
+		      (match_operand 0 "register_operand" ""))
+		 (match_operand 2 "const1_operand" ""))
+	      (const_string "alu")
+	   ]
+	   (const_string "ishift")))
+   (set_attr "mode" "QI")])
+
 ;; See comment above `ashldi3' about how this works.
 
 (define_expand "ashrti3"
@@ -11122,6 +11306,20 @@
 	(const_string "2")
 	(const_string "*")))])
 
+(define_insn "*ashrdi3_one_bit_cconly_rex64"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (ashiftrt:DI (match_operand:DI 1 "nonimmediate_operand" "0")
+		       (match_operand:QI 2 "const1_operand" ""))
+	  (const_int 0)))
+   (clobber (match_scratch:DI 0 "=r"))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCGOCmode)
+   && (TARGET_SHIFT1 || optimize_size)
+   && ix86_binary_operator_ok (ASHIFTRT, DImode, operands)"
+  "sar{q}\t%0"
+  [(set_attr "type" "ishift")
+   (set_attr "length" "2")])
+
 ;; This pattern can't accept a variable shift count, since shifts by
 ;; zero don't affect the flags.  We assume that shifts by constant
 ;; zero are optimized away.
@@ -11139,6 +11337,19 @@
   [(set_attr "type" "ishift")
    (set_attr "mode" "DI")])
 
+(define_insn "*ashrdi3_cconly_rex64"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (ashiftrt:DI (match_operand:DI 1 "nonimmediate_operand" "0")
+		       (match_operand:QI 2 "const_int_operand" "n"))
+	  (const_int 0)))
+   (clobber (match_scratch:DI 0 "=r"))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCGOCmode)
+   && ix86_binary_operator_ok (ASHIFTRT, DImode, operands)"
+  "sar{q}\t{%2, %0|%0, %2}"
+  [(set_attr "type" "ishift")
+   (set_attr "mode" "DI")])
+
 (define_insn "*ashrdi3_1"
   [(set (match_operand:DI 0 "register_operand" "=r")
 	(ashiftrt:DI (match_operand:DI 1 "register_operand" "0")
@@ -11167,7 +11378,8 @@
 	(ashiftrt:DI (match_operand:DI 1 "register_operand" "")
 		     (match_operand:QI 2 "nonmemory_operand" "")))
    (clobber (reg:CC FLAGS_REG))]
-  "!TARGET_64BIT && (flag_peephole2 ? flow2_completed : reload_completed)"
+  "!TARGET_64BIT && ((optimize > 0 && flag_peephole2)
+		     ? flow2_completed : reload_completed)"
   [(const_int 0)]
   "ix86_split_ashr (operands, NULL_RTX, DImode); DONE;")
 
@@ -11326,6 +11538,20 @@
 	(const_string "2")
 	(const_string "*")))])
 
+(define_insn "*ashrsi3_one_bit_cconly"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (ashiftrt:SI (match_operand:SI 1 "nonimmediate_operand" "0")
+		       (match_operand:QI 2 "const1_operand" ""))
+	  (const_int 0)))
+   (clobber (match_scratch:SI 0 "=r"))]
+  "ix86_match_ccmode (insn, CCGOCmode)
+   && (TARGET_SHIFT1 || optimize_size)
+   && ix86_binary_operator_ok (ASHIFTRT, SImode, operands)"
+  "sar{l}\t%0"
+  [(set_attr "type" "ishift")
+   (set_attr "length" "2")])
+
 (define_insn "*ashrsi3_one_bit_cmp_zext"
   [(set (reg FLAGS_REG)
 	(compare
@@ -11358,6 +11584,19 @@
   [(set_attr "type" "ishift")
    (set_attr "mode" "SI")])
 
+(define_insn "*ashrsi3_cconly"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (ashiftrt:SI (match_operand:SI 1 "nonimmediate_operand" "0")
+		       (match_operand:QI 2 "const_1_to_31_operand" "I"))
+	  (const_int 0)))
+   (clobber (match_scratch:SI 0 "=r"))]
+  "ix86_match_ccmode (insn, CCGOCmode)
+   && ix86_binary_operator_ok (ASHIFTRT, SImode, operands)"
+  "sar{l}\t{%2, %0|%0, %2}"
+  [(set_attr "type" "ishift")
+   (set_attr "mode" "SI")])
+
 (define_insn "*ashrsi3_cmp_zext"
   [(set (reg FLAGS_REG)
 	(compare
@@ -11427,6 +11666,20 @@
 	(const_string "2")
 	(const_string "*")))])
 
+(define_insn "*ashrhi3_one_bit_cconly"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (ashiftrt:HI (match_operand:HI 1 "nonimmediate_operand" "0")
+		       (match_operand:QI 2 "const1_operand" ""))
+	  (const_int 0)))
+   (clobber (match_scratch:HI 0 "=r"))]
+  "ix86_match_ccmode (insn, CCGOCmode)
+   && (TARGET_SHIFT1 || optimize_size)
+   && ix86_binary_operator_ok (ASHIFTRT, HImode, operands)"
+  "sar{w}\t%0"
+  [(set_attr "type" "ishift")
+   (set_attr "length" "2")])
+
 ;; This pattern can't accept a variable shift count, since shifts by
 ;; zero don't affect the flags.  We assume that shifts by constant
 ;; zero are optimized away.
@@ -11444,6 +11697,19 @@
   [(set_attr "type" "ishift")
    (set_attr "mode" "HI")])
 
+(define_insn "*ashrhi3_cconly"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (ashiftrt:HI (match_operand:HI 1 "nonimmediate_operand" "0")
+		       (match_operand:QI 2 "const_1_to_31_operand" "I"))
+	  (const_int 0)))
+   (clobber (match_scratch:HI 0 "=r"))]
+  "ix86_match_ccmode (insn, CCGOCmode)
+   && ix86_binary_operator_ok (ASHIFTRT, HImode, operands)"
+  "sar{w}\t{%2, %0|%0, %2}"
+  [(set_attr "type" "ishift")
+   (set_attr "mode" "HI")])
+
 (define_expand "ashrqi3"
   [(set (match_operand:QI 0 "nonimmediate_operand" "")
 	(ashiftrt:QI (match_operand:QI 1 "nonimmediate_operand" "")
@@ -11527,6 +11793,20 @@
 	(const_string "2")
 	(const_string "*")))])
 
+(define_insn "*ashrqi3_one_bit_cconly"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (ashiftrt:QI (match_operand:QI 1 "nonimmediate_operand" "0")
+		       (match_operand:QI 2 "const1_operand" "I"))
+	  (const_int 0)))
+   (clobber (match_scratch:QI 0 "=q"))]
+  "ix86_match_ccmode (insn, CCGOCmode)
+   && (TARGET_SHIFT1 || optimize_size)
+   && ix86_binary_operator_ok (ASHIFTRT, QImode, operands)"
+  "sar{b}\t%0"
+  [(set_attr "type" "ishift")
+   (set_attr "length" "2")])
+
 ;; This pattern can't accept a variable shift count, since shifts by
 ;; zero don't affect the flags.  We assume that shifts by constant
 ;; zero are optimized away.
@@ -11543,6 +11823,20 @@
   "sar{b}\t{%2, %0|%0, %2}"
   [(set_attr "type" "ishift")
    (set_attr "mode" "QI")])
+
+(define_insn "*ashrqi3_cconly"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (ashiftrt:QI (match_operand:QI 1 "nonimmediate_operand" "0")
+		       (match_operand:QI 2 "const_1_to_31_operand" "I"))
+	  (const_int 0)))
+   (clobber (match_scratch:QI 0 "=q"))]
+  "ix86_match_ccmode (insn, CCGOCmode)
+   && ix86_binary_operator_ok (ASHIFTRT, QImode, operands)"
+  "sar{b}\t{%2, %0|%0, %2}"
+  [(set_attr "type" "ishift")
+   (set_attr "mode" "QI")])
+
 
 ;; Logical shift instructions
 
@@ -11656,6 +11950,20 @@
 	(const_string "2")
 	(const_string "*")))])
 
+(define_insn "*lshrdi3_cconly_one_bit_rex64"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (lshiftrt:DI (match_operand:DI 1 "nonimmediate_operand" "0")
+		       (match_operand:QI 2 "const1_operand" ""))
+	  (const_int 0)))
+   (clobber (match_scratch:DI 0 "=r"))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCGOCmode)
+   && (TARGET_SHIFT1 || optimize_size)
+   && ix86_binary_operator_ok (LSHIFTRT, HImode, operands)"
+  "shr{q}\t%0"
+  [(set_attr "type" "ishift")
+   (set_attr "length" "2")])
+
 ;; This pattern can't accept a variable shift count, since shifts by
 ;; zero don't affect the flags.  We assume that shifts by constant
 ;; zero are optimized away.
@@ -11673,6 +11981,19 @@
   [(set_attr "type" "ishift")
    (set_attr "mode" "DI")])
 
+(define_insn "*lshrdi3_cconly_rex64"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (lshiftrt:DI (match_operand:DI 1 "nonimmediate_operand" "0")
+		       (match_operand:QI 2 "const_int_operand" "e"))
+	  (const_int 0)))
+   (clobber (match_scratch:DI 0 "=r"))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCGOCmode)
+   && ix86_binary_operator_ok (LSHIFTRT, HImode, operands)"
+  "shr{q}\t{%2, %0|%0, %2}"
+  [(set_attr "type" "ishift")
+   (set_attr "mode" "DI")])
+
 (define_insn "*lshrdi3_1"
   [(set (match_operand:DI 0 "register_operand" "=r")
 	(lshiftrt:DI (match_operand:DI 1 "register_operand" "0")
@@ -11701,7 +12022,8 @@
 	(lshiftrt:DI (match_operand:DI 1 "register_operand" "")
 		     (match_operand:QI 2 "nonmemory_operand" "")))
    (clobber (reg:CC FLAGS_REG))]
-  "!TARGET_64BIT && (flag_peephole2 ? flow2_completed : reload_completed)"
+  "!TARGET_64BIT && ((optimize > 0 && flag_peephole2)
+		     ? flow2_completed : reload_completed)"
   [(const_int 0)]
   "ix86_split_lshr (operands, NULL_RTX, DImode); DONE;")
 
@@ -11784,6 +12106,20 @@
 	(const_string "2")
 	(const_string "*")))])
 
+(define_insn "*lshrsi3_one_bit_cconly"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (lshiftrt:SI (match_operand:SI 1 "nonimmediate_operand" "0")
+		       (match_operand:QI 2 "const1_operand" ""))
+	  (const_int 0)))
+   (clobber (match_scratch:SI 0 "=r"))]
+  "ix86_match_ccmode (insn, CCGOCmode)
+   && (TARGET_SHIFT1 || optimize_size)
+   && ix86_binary_operator_ok (LSHIFTRT, HImode, operands)"
+  "shr{l}\t%0"
+  [(set_attr "type" "ishift")
+   (set_attr "length" "2")])
+
 (define_insn "*lshrsi3_cmp_one_bit_zext"
   [(set (reg FLAGS_REG)
 	(compare
@@ -11816,6 +12152,19 @@
   [(set_attr "type" "ishift")
    (set_attr "mode" "SI")])
 
+(define_insn "*lshrsi3_cconly"
+  [(set (reg FLAGS_REG)
+      (compare
+	(lshiftrt:SI (match_operand:SI 1 "nonimmediate_operand" "0")
+		     (match_operand:QI 2 "const_1_to_31_operand" "I"))
+        (const_int 0)))
+   (clobber (match_scratch:SI 0 "=r"))]
+  "ix86_match_ccmode (insn, CCGOCmode)
+   && ix86_binary_operator_ok (LSHIFTRT, HImode, operands)"
+  "shr{l}\t{%2, %0|%0, %2}"
+  [(set_attr "type" "ishift")
+   (set_attr "mode" "SI")])
+
 (define_insn "*lshrsi3_cmp_zext"
   [(set (reg FLAGS_REG)
 	(compare
@@ -11885,6 +12234,20 @@
 	(const_string "2")
 	(const_string "*")))])
 
+(define_insn "*lshrhi3_one_bit_cconly"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (lshiftrt:HI (match_operand:HI 1 "nonimmediate_operand" "0")
+		       (match_operand:QI 2 "const1_operand" ""))
+	  (const_int 0)))
+   (clobber (match_scratch:HI 0 "=r"))]
+  "ix86_match_ccmode (insn, CCGOCmode)
+   && (TARGET_SHIFT1 || optimize_size)
+   && ix86_binary_operator_ok (LSHIFTRT, HImode, operands)"
+  "shr{w}\t%0"
+  [(set_attr "type" "ishift")
+   (set_attr "length" "2")])
+
 ;; This pattern can't accept a variable shift count, since shifts by
 ;; zero don't affect the flags.  We assume that shifts by constant
 ;; zero are optimized away.
@@ -11902,6 +12265,19 @@
   [(set_attr "type" "ishift")
    (set_attr "mode" "HI")])
 
+(define_insn "*lshrhi3_cconly"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (lshiftrt:HI (match_operand:HI 1 "nonimmediate_operand" "0")
+		       (match_operand:QI 2 "const_1_to_31_operand" "I"))
+	  (const_int 0)))
+   (clobber (match_scratch:HI 0 "=r"))]
+  "ix86_match_ccmode (insn, CCGOCmode)
+   && ix86_binary_operator_ok (LSHIFTRT, HImode, operands)"
+  "shr{w}\t{%2, %0|%0, %2}"
+  [(set_attr "type" "ishift")
+   (set_attr "mode" "HI")])
+
 (define_expand "lshrqi3"
   [(set (match_operand:QI 0 "nonimmediate_operand" "")
 	(lshiftrt:QI (match_operand:QI 1 "nonimmediate_operand" "")
@@ -11984,6 +12360,20 @@
 	(const_string "2")
 	(const_string "*")))])
 
+(define_insn "*lshrqi2_one_bit_cconly"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (lshiftrt:QI (match_operand:QI 1 "nonimmediate_operand" "0")
+		       (match_operand:QI 2 "const1_operand" ""))
+	  (const_int 0)))
+   (clobber (match_scratch:QI 0 "=q"))]
+  "ix86_match_ccmode (insn, CCGOCmode)
+   && (TARGET_SHIFT1 || optimize_size)
+   && ix86_binary_operator_ok (LSHIFTRT, QImode, operands)"
+  "shr{b}\t%0"
+  [(set_attr "type" "ishift")
+   (set_attr "length" "2")])
+
 ;; This pattern can't accept a variable shift count, since shifts by
 ;; zero don't affect the flags.  We assume that shifts by constant
 ;; zero are optimized away.
@@ -12000,6 +12390,19 @@
   "shr{b}\t{%2, %0|%0, %2}"
   [(set_attr "type" "ishift")
    (set_attr "mode" "QI")])
+
+(define_insn "*lshrqi2_cconly"
+  [(set (reg FLAGS_REG)
+	(compare
+	  (lshiftrt:QI (match_operand:QI 1 "nonimmediate_operand" "0")
+		       (match_operand:QI 2 "const_1_to_31_operand" "I"))
+	  (const_int 0)))
+   (clobber (match_scratch:QI 0 "=q"))]
+  "ix86_match_ccmode (insn, CCGOCmode)
+   && ix86_binary_operator_ok (LSHIFTRT, QImode, operands)"
+  "shr{b}\t{%2, %0|%0, %2}"
+  [(set_attr "type" "ishift")
+   (set_attr "mode" "QI")])
 
 ;; Rotate instructions
 
@@ -12459,8 +12862,8 @@
 (define_expand "extv"
   [(set (match_operand:SI 0 "register_operand" "")
 	(sign_extract:SI (match_operand:SI 1 "register_operand" "")
-			 (match_operand:SI 2 "immediate_operand" "")
-			 (match_operand:SI 3 "immediate_operand" "")))]
+			 (match_operand:SI 2 "const8_operand" "")
+			 (match_operand:SI 3 "const8_operand" "")))]
   ""
 {
   /* Handle extractions from %ah et al.  */
@@ -12476,8 +12879,8 @@
 (define_expand "extzv"
   [(set (match_operand:SI 0 "register_operand" "")
 	(zero_extract:SI (match_operand 1 "ext_register_operand" "")
-			 (match_operand:SI 2 "immediate_operand" "")
-			 (match_operand:SI 3 "immediate_operand" "")))]
+			 (match_operand:SI 2 "const8_operand" "")
+			 (match_operand:SI 3 "const8_operand" "")))]
   ""
 {
   /* Handle extractions from %ah et al.  */
@@ -12492,12 +12895,12 @@
 
 (define_expand "insv"
   [(set (zero_extract (match_operand 0 "ext_register_operand" "")
-		      (match_operand 1 "immediate_operand" "")
-		      (match_operand 2 "immediate_operand" ""))
+		      (match_operand 1 "const8_operand" "")
+		      (match_operand 2 "const8_operand" ""))
         (match_operand 3 "register_operand" ""))]
   ""
 {
-  /* Handle extractions from %ah et al.  */
+  /* Handle insertions to %ah et al.  */
   if (INTVAL (operands[1]) != 8 || INTVAL (operands[2]) != 8)
     FAIL;
 
@@ -13128,8 +13531,8 @@
 (define_insn "*fp_jcc_1_mixed"
   [(set (pc)
 	(if_then_else (match_operator 0 "comparison_operator"
-			[(match_operand 1 "register_operand" "f#x,x#f")
-			 (match_operand 2 "nonimmediate_operand" "f#x,xm#f")])
+			[(match_operand 1 "register_operand" "f,x")
+			 (match_operand 2 "nonimmediate_operand" "f,xm")])
 	  (label_ref (match_operand 3 "" ""))
 	  (pc)))
    (clobber (reg:CCFP FPSR_REG))
@@ -13173,8 +13576,8 @@
 (define_insn "*fp_jcc_2_mixed"
   [(set (pc)
 	(if_then_else (match_operator 0 "comparison_operator"
-			[(match_operand 1 "register_operand" "f#x,x#f")
-			 (match_operand 2 "nonimmediate_operand" "f#x,xm#f")])
+			[(match_operand 1 "register_operand" "f,x")
+			 (match_operand 2 "nonimmediate_operand" "f,xm")])
 	  (pc)
 	  (label_ref (match_operand 3 "" ""))))
    (clobber (reg:CCFP FPSR_REG))
@@ -13835,7 +14238,7 @@
 #else
   /* It is tempting to use ASM_OUTPUT_ALIGN here, but we don't want to do that.
      The align insn is used to avoid 3 jump instructions in the row to improve
-     branch prediction and the benefits hardly outweight the cost of extra 8
+     branch prediction and the benefits hardly outweigh the cost of extra 8
      nops on the average inserted by full alignment pseudo operation.  */
 #endif
   return "";
@@ -13852,7 +14255,17 @@
 	(unspec:SI [(const_int 0)] UNSPEC_SET_GOT))
    (clobber (reg:CC FLAGS_REG))]
   "!TARGET_64BIT"
-  { return output_set_got (operands[0]); }
+  { return output_set_got (operands[0], NULL_RTX); }
+  [(set_attr "type" "multi")
+   (set_attr "length" "12")])
+
+(define_insn "set_got_labelled"
+  [(set (match_operand:SI 0 "register_operand" "=r")
+	(unspec:SI [(label_ref (match_operand 1 "" ""))]
+	 UNSPEC_SET_GOT))
+   (clobber (reg:CC FLAGS_REG))]
+  "!TARGET_64BIT"
+  { return output_set_got (operands[0], operands[1]); }
   [(set_attr "type" "multi")
    (set_attr "length" "12")])
 
@@ -14143,6 +14556,12 @@
       operands[2] = gen_reg_rtx (Pmode);
       emit_insn (gen_set_got (operands[2]));
     }
+  if (TARGET_GNU2_TLS)
+    {
+       emit_insn (gen_tls_dynamic_gnu2_32
+		  (operands[0], operands[1], operands[2]));
+       DONE;
+    }
   operands[3] = ix86_tls_get_addr ();
 })
 
@@ -14164,6 +14583,12 @@
 			 UNSPEC_TLS_GD)])]
   ""
 {
+  if (TARGET_GNU2_TLS)
+    {
+       emit_insn (gen_tls_dynamic_gnu2_64
+		  (operands[0], operands[1]));
+       DONE;
+    }
   operands[2] = ix86_tls_get_addr ();
 })
 
@@ -14210,6 +14635,12 @@
       operands[1] = gen_reg_rtx (Pmode);
       emit_insn (gen_set_got (operands[1]));
     }
+  if (TARGET_GNU2_TLS)
+    {
+       emit_insn (gen_tls_dynamic_gnu2_32
+		  (operands[0], ix86_tls_module_base (), operands[1]));
+       DONE;
+    }
   operands[2] = ix86_tls_get_addr ();
 })
 
@@ -14229,6 +14660,12 @@
 	      (unspec:DI [(const_int 0)] UNSPEC_TLS_LD_BASE)])]
   ""
 {
+  if (TARGET_GNU2_TLS)
+    {
+       emit_insn (gen_tls_dynamic_gnu2_64
+		  (operands[0], ix86_tls_module_base ()));
+       DONE;
+    }
   operands[1] = ix86_tls_get_addr ();
 })
 
@@ -14306,6 +14743,136 @@
    (set_attr "length" "7")
    (set_attr "memory" "load")
    (set_attr "imm_disp" "false")])
+
+;; GNU2 TLS patterns can be split.
+
+(define_expand "tls_dynamic_gnu2_32"
+  [(set (match_dup 3)
+	(plus:SI (match_operand:SI 2 "register_operand" "")
+		 (const:SI
+		  (unspec:SI [(match_operand:SI 1 "tls_symbolic_operand" "")]
+			     UNSPEC_TLSDESC))))
+   (parallel
+    [(set (match_operand:SI 0 "register_operand" "")
+	  (unspec:SI [(match_dup 1) (match_dup 3)
+		      (match_dup 2) (reg:SI SP_REG)]
+		      UNSPEC_TLSDESC))
+     (clobber (reg:CC FLAGS_REG))])]
+  "!TARGET_64BIT && TARGET_GNU2_TLS"
+{
+  operands[3] = no_new_pseudos ? operands[0] : gen_reg_rtx (Pmode);
+  ix86_tls_descriptor_calls_expanded_in_cfun = true;
+})
+
+(define_insn "*tls_dynamic_lea_32"
+  [(set (match_operand:SI 0 "register_operand" "=r")
+	(plus:SI (match_operand:SI 1 "register_operand" "b")
+		 (const:SI
+		  (unspec:SI [(match_operand:SI 2 "tls_symbolic_operand" "")]
+			      UNSPEC_TLSDESC))))]
+  "!TARGET_64BIT && TARGET_GNU2_TLS"
+  "lea{l}\t{%a2@TLSDESC(%1), %0|%0, %a2@TLSDESC[%1]}"
+  [(set_attr "type" "lea")
+   (set_attr "mode" "SI")
+   (set_attr "length" "6")
+   (set_attr "length_address" "4")])
+
+(define_insn "*tls_dynamic_call_32"
+  [(set (match_operand:SI 0 "register_operand" "=a")
+	(unspec:SI [(match_operand:SI 1 "tls_symbolic_operand" "")
+		    (match_operand:SI 2 "register_operand" "0")
+		    ;; we have to make sure %ebx still points to the GOT
+		    (match_operand:SI 3 "register_operand" "b")
+		    (reg:SI SP_REG)]
+		   UNSPEC_TLSDESC))
+   (clobber (reg:CC FLAGS_REG))]
+  "!TARGET_64BIT && TARGET_GNU2_TLS"
+  "call\t{*%a1@TLSCALL(%2)|[DWORD PTR [%2+%a1@TLSCALL]]}"
+  [(set_attr "type" "call")
+   (set_attr "length" "2")
+   (set_attr "length_address" "0")])
+
+(define_insn_and_split "*tls_dynamic_gnu2_combine_32"
+  [(set (match_operand:SI 0 "register_operand" "=&a")
+	(plus:SI
+	 (unspec:SI [(match_operand:SI 3 "tls_modbase_operand" "")
+		     (match_operand:SI 4 "" "")
+		     (match_operand:SI 2 "register_operand" "b")
+		     (reg:SI SP_REG)]
+		    UNSPEC_TLSDESC)
+	 (const:SI (unspec:SI
+		    [(match_operand:SI 1 "tls_symbolic_operand" "")]
+		    UNSPEC_DTPOFF))))
+   (clobber (reg:CC FLAGS_REG))]
+  "!TARGET_64BIT && TARGET_GNU2_TLS"
+  "#"
+  ""
+  [(set (match_dup 0) (match_dup 5))]
+{
+  operands[5] = no_new_pseudos ? operands[0] : gen_reg_rtx (Pmode);
+  emit_insn (gen_tls_dynamic_gnu2_32 (operands[5], operands[1], operands[2]));
+})
+
+(define_expand "tls_dynamic_gnu2_64"
+  [(set (match_dup 2)
+	(unspec:DI [(match_operand:DI 1 "tls_symbolic_operand" "")]
+		   UNSPEC_TLSDESC))
+   (parallel
+    [(set (match_operand:DI 0 "register_operand" "")
+	  (unspec:DI [(match_dup 1) (match_dup 2) (reg:DI SP_REG)]
+		     UNSPEC_TLSDESC))
+     (clobber (reg:CC FLAGS_REG))])]
+  "TARGET_64BIT && TARGET_GNU2_TLS"
+{
+  operands[2] = no_new_pseudos ? operands[0] : gen_reg_rtx (Pmode);
+  ix86_tls_descriptor_calls_expanded_in_cfun = true;
+})
+
+(define_insn "*tls_dynamic_lea_64"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+	(unspec:DI [(match_operand:DI 1 "tls_symbolic_operand" "")]
+		   UNSPEC_TLSDESC))]
+  "TARGET_64BIT && TARGET_GNU2_TLS"
+  "lea{q}\t{%a1@TLSDESC(%%rip), %0|%0, %a1@TLSDESC[%%rip]}"
+  [(set_attr "type" "lea")
+   (set_attr "mode" "DI")
+   (set_attr "length" "7")
+   (set_attr "length_address" "4")])
+
+(define_insn "*tls_dynamic_call_64"
+  [(set (match_operand:DI 0 "register_operand" "=a")
+	(unspec:DI [(match_operand:DI 1 "tls_symbolic_operand" "")
+		    (match_operand:DI 2 "register_operand" "0")
+		    (reg:DI SP_REG)]
+		   UNSPEC_TLSDESC))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT && TARGET_GNU2_TLS"
+  "call\t{*%a1@TLSCALL(%2)|[QWORD PTR [%2+%a1@TLSCALL]]}"
+  [(set_attr "type" "call")
+   (set_attr "length" "2")
+   (set_attr "length_address" "0")])
+
+(define_insn_and_split "*tls_dynamic_gnu2_combine_64"
+  [(set (match_operand:DI 0 "register_operand" "=&a")
+	(plus:DI
+	 (unspec:DI [(match_operand:DI 2 "tls_modbase_operand" "")
+		     (match_operand:DI 3 "" "")
+		     (reg:DI SP_REG)]
+		    UNSPEC_TLSDESC)
+	 (const:DI (unspec:DI
+		    [(match_operand:DI 1 "tls_symbolic_operand" "")]
+		    UNSPEC_DTPOFF))))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT && TARGET_GNU2_TLS"
+  "#"
+  ""
+  [(set (match_dup 0) (match_dup 4))]
+{
+  operands[4] = no_new_pseudos ? operands[0] : gen_reg_rtx (Pmode);
+  emit_insn (gen_tls_dynamic_gnu2_64 (operands[4], operands[1]));
+})
+
+;;
 
 ;; These patterns match the binary 387 instructions for addM3, subM3,
 ;; mulM3 and divM3.  There are three patterns for each of DFmode and
@@ -14319,10 +14886,10 @@
 ;; so use special patterns for add and mull.
 
 (define_insn "*fop_sf_comm_mixed"
-  [(set (match_operand:SF 0 "register_operand" "=f#x,x#f")
+  [(set (match_operand:SF 0 "register_operand" "=f,x")
 	(match_operator:SF 3 "binary_fp_operator"
 			[(match_operand:SF 1 "nonimmediate_operand" "%0,0")
-			 (match_operand:SF 2 "nonimmediate_operand" "fm#x,xm#f")]))]
+			 (match_operand:SF 2 "nonimmediate_operand" "fm,xm")]))]
   "TARGET_MIX_SSE_I387
    && COMMUTATIVE_ARITH_P (operands[3])
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
@@ -14371,7 +14938,7 @@
   [(set (match_operand:SF 0 "register_operand" "=f,f,x")
 	(match_operator:SF 3 "binary_fp_operator"
 			[(match_operand:SF 1 "nonimmediate_operand" "0,fm,0")
-			 (match_operand:SF 2 "nonimmediate_operand" "fm,0,xm#f")]))]
+			 (match_operand:SF 2 "nonimmediate_operand" "fm,0,xm")]))]
   "TARGET_MIX_SSE_I387
    && !COMMUTATIVE_ARITH_P (operands[3])
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
@@ -14465,20 +15032,20 @@
    (set_attr "mode" "<MODE>")])
 
 (define_insn "*fop_df_comm_mixed"
-  [(set (match_operand:DF 0 "register_operand" "=f#Y,Y#f")
+  [(set (match_operand:DF 0 "register_operand" "=f,Y")
 	(match_operator:DF 3 "binary_fp_operator"
 			[(match_operand:DF 1 "nonimmediate_operand" "%0,0")
-			 (match_operand:DF 2 "nonimmediate_operand" "fm#Y,Ym#f")]))]
+			 (match_operand:DF 2 "nonimmediate_operand" "fm,Ym")]))]
   "TARGET_SSE2 && TARGET_MIX_SSE_I387
    && COMMUTATIVE_ARITH_P (operands[3])
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
 	(if_then_else (eq_attr "alternative" "1")
-	   (if_then_else (match_operand:SF 3 "mult_operator" "") 
+	   (if_then_else (match_operand:DF 3 "mult_operator" "") 
 	      (const_string "ssemul")
 	      (const_string "sseadd"))
-	   (if_then_else (match_operand:SF 3 "mult_operator" "") 
+	   (if_then_else (match_operand:DF 3 "mult_operator" "") 
 	      (const_string "fmul")
 	      (const_string "fop"))))
    (set_attr "mode" "DF")])
@@ -14493,7 +15060,7 @@
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
-        (if_then_else (match_operand:SF 3 "mult_operator" "") 
+        (if_then_else (match_operand:DF 3 "mult_operator" "") 
 	   (const_string "ssemul")
 	   (const_string "sseadd")))
    (set_attr "mode" "DF")])
@@ -14508,26 +15075,26 @@
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
-	(if_then_else (match_operand:SF 3 "mult_operator" "") 
+	(if_then_else (match_operand:DF 3 "mult_operator" "") 
 	   (const_string "fmul")
 	   (const_string "fop")))
    (set_attr "mode" "DF")])
 
 (define_insn "*fop_df_1_mixed"
-  [(set (match_operand:DF 0 "register_operand" "=f#Y,f#Y,Y#f")
+  [(set (match_operand:DF 0 "register_operand" "=f,f,Y")
 	(match_operator:DF 3 "binary_fp_operator"
 			[(match_operand:DF 1 "nonimmediate_operand" "0,fm,0")
-			 (match_operand:DF 2 "nonimmediate_operand" "fm,0,Ym#f")]))]
+			 (match_operand:DF 2 "nonimmediate_operand" "fm,0,Ym")]))]
   "TARGET_SSE2 && TARGET_SSE_MATH && TARGET_MIX_SSE_I387
    && !COMMUTATIVE_ARITH_P (operands[3])
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
         (cond [(and (eq_attr "alternative" "2")
-	            (match_operand:SF 3 "mult_operator" ""))
+	            (match_operand:DF 3 "mult_operator" ""))
                  (const_string "ssemul")
 	       (and (eq_attr "alternative" "2")
-	            (match_operand:SF 3 "div_operator" ""))
+	            (match_operand:DF 3 "div_operator" ""))
                  (const_string "ssediv")
 	       (eq_attr "alternative" "2")
                  (const_string "sseadd")
@@ -14549,9 +15116,9 @@
   "* return output_387_binary_op (insn, operands);"
   [(set_attr "mode" "DF")
    (set (attr "type") 
-        (cond [(match_operand:SF 3 "mult_operator" "")
+        (cond [(match_operand:DF 3 "mult_operator" "")
                  (const_string "ssemul")
-	       (match_operand:SF 3 "div_operator" "")
+	       (match_operand:DF 3 "div_operator" "")
                  (const_string "ssediv")
               ]
               (const_string "sseadd")))])
@@ -14832,8 +15399,8 @@
 })
 
 (define_insn "*sqrtsf2_mixed"
-  [(set (match_operand:SF 0 "register_operand" "=f#x,x#f")
-	(sqrt:SF (match_operand:SF 1 "nonimmediate_operand" "0#x,xm#f")))]
+  [(set (match_operand:SF 0 "register_operand" "=f,x")
+	(sqrt:SF (match_operand:SF 1 "nonimmediate_operand" "0,xm")))]
   "TARGET_USE_FANCY_MATH_387 && TARGET_MIX_SSE_I387"
   "@
    fsqrt
@@ -14870,8 +15437,8 @@
 })
 
 (define_insn "*sqrtdf2_mixed"
-  [(set (match_operand:DF 0 "register_operand" "=f#Y,Y#f")
-	(sqrt:DF (match_operand:DF 1 "nonimmediate_operand" "0#Y,Ym#f")))]
+  [(set (match_operand:DF 0 "register_operand" "=f,Y")
+	(sqrt:DF (match_operand:DF 1 "nonimmediate_operand" "0,Ym")))]
   "TARGET_USE_FANCY_MATH_387 && TARGET_SSE2 && TARGET_MIX_SSE_I387"
   "@
    fsqrt
@@ -18368,11 +18935,11 @@
   "if (! ix86_expand_fp_movcc (operands)) FAIL; DONE;")
 
 (define_insn "*movsfcc_1_387"
-  [(set (match_operand:SF 0 "register_operand" "=f#r,f#r,r#f,r#f")
+  [(set (match_operand:SF 0 "register_operand" "=f,f,r,r")
 	(if_then_else:SF (match_operator 1 "fcmov_comparison_operator" 
 				[(reg FLAGS_REG) (const_int 0)])
-		      (match_operand:SF 2 "nonimmediate_operand" "f#r,0,rm#f,0")
-		      (match_operand:SF 3 "nonimmediate_operand" "0,f#r,0,rm#f")))]
+		      (match_operand:SF 2 "nonimmediate_operand" "f,0,rm,0")
+		      (match_operand:SF 3 "nonimmediate_operand" "0,f,0,rm")))]
   "TARGET_80387 && TARGET_CMOVE
    && (GET_CODE (operands[2]) != MEM || GET_CODE (operands[3]) != MEM)"
   "@
@@ -18392,11 +18959,11 @@
   "if (! ix86_expand_fp_movcc (operands)) FAIL; DONE;")
 
 (define_insn "*movdfcc_1"
-  [(set (match_operand:DF 0 "register_operand" "=f#r,f#r,&r#f,&r#f")
+  [(set (match_operand:DF 0 "register_operand" "=f,f,&r,&r")
 	(if_then_else:DF (match_operator 1 "fcmov_comparison_operator" 
 				[(reg FLAGS_REG) (const_int 0)])
-		      (match_operand:DF 2 "nonimmediate_operand" "f#r,0,rm#f,0")
-		      (match_operand:DF 3 "nonimmediate_operand" "0,f#r,0,rm#f")))]
+		      (match_operand:DF 2 "nonimmediate_operand" "f,0,rm,0")
+		      (match_operand:DF 3 "nonimmediate_operand" "0,f,0,rm")))]
   "!TARGET_64BIT && TARGET_80387 && TARGET_CMOVE
    && (GET_CODE (operands[2]) != MEM || GET_CODE (operands[3]) != MEM)"
   "@
@@ -18408,11 +18975,11 @@
    (set_attr "mode" "DF")])
 
 (define_insn "*movdfcc_1_rex64"
-  [(set (match_operand:DF 0 "register_operand" "=f#r,f#r,r#f,r#f")
+  [(set (match_operand:DF 0 "register_operand" "=f,f,r,r")
 	(if_then_else:DF (match_operator 1 "fcmov_comparison_operator" 
 				[(reg FLAGS_REG) (const_int 0)])
-		      (match_operand:DF 2 "nonimmediate_operand" "f#r,0#r,rm#f,0#f")
-		      (match_operand:DF 3 "nonimmediate_operand" "0#r,f#r,0#f,rm#f")))]
+		      (match_operand:DF 2 "nonimmediate_operand" "f,0,rm,0")
+		      (match_operand:DF 3 "nonimmediate_operand" "0,f,0,rm")))]
   "TARGET_64BIT && TARGET_80387 && TARGET_CMOVE
    && (GET_CODE (operands[2]) != MEM || GET_CODE (operands[3]) != MEM)"
   "@
@@ -18551,6 +19118,32 @@
   [(set_attr "type" "sseadd")
    (set_attr "mode" "DF")])
 
+;; Make two stack loads independent:
+;;   fld aa              fld aa
+;;   fld %st(0)     ->   fld bb
+;;   fmul bb             fmul %st(1), %st
+;;
+;; Actually we only match the last two instructions for simplicity.
+(define_peephole2
+  [(set (match_operand 0 "fp_register_operand" "")
+	(match_operand 1 "fp_register_operand" ""))
+   (set (match_dup 0)
+	(match_operator 2 "binary_fp_operator"
+	   [(match_dup 0)
+	    (match_operand 3 "memory_operand" "")]))]
+  "REGNO (operands[0]) != REGNO (operands[1])"
+  [(set (match_dup 0) (match_dup 3))
+   (set (match_dup 0) (match_dup 4))]
+
+  ;; The % modifier is not operational anymore in peephole2's, so we have to
+  ;; swap the operands manually in the case of addition and multiplication.
+  "if (COMMUTATIVE_ARITH_P (operands[2]))
+     operands[4] = gen_rtx_fmt_ee (GET_CODE (operands[2]), GET_MODE (operands[2]),
+				 operands[0], operands[1]);
+   else
+     operands[4] = gen_rtx_fmt_ee (GET_CODE (operands[2]), GET_MODE (operands[2]),
+				 operands[1], operands[0]);")
+
 ;; Conditional addition patterns
 (define_expand "addqicc"
   [(match_operand:QI 0 "register_operand" "")
@@ -18794,7 +19387,21 @@
   [(label_ref (match_operand 0 "" ""))]
   "!TARGET_64BIT && flag_pic"
 {
-  emit_insn (gen_set_got (pic_offset_table_rtx));
+  if (TARGET_MACHO)
+    {
+      rtx xops[3];
+      rtx picreg = gen_rtx_REG (Pmode, PIC_OFFSET_TABLE_REGNUM);
+      rtx label_rtx = gen_label_rtx ();
+      emit_insn (gen_set_got_labelled (pic_offset_table_rtx, label_rtx));
+      xops[0] = xops[1] = picreg;
+      xops[2] = gen_rtx_CONST (SImode,
+	          gen_rtx_MINUS (SImode,
+		    gen_rtx_LABEL_REF (SImode, label_rtx),
+		    gen_rtx_SYMBOL_REF (SImode, GOT_SYMBOL_NAME)));
+      ix86_expand_binary_operator (MINUS, SImode, xops);
+    }
+  else
+    emit_insn (gen_set_got (pic_offset_table_rtx));
   DONE;
 })
 
@@ -18809,8 +19416,9 @@
   "! TARGET_PARTIAL_REG_STALL && reload_completed
    && ((GET_MODE (operands[0]) == HImode 
 	&& ((!optimize_size && !TARGET_FAST_PREFIX)
+            /* ??? next two lines just !satisfies_constraint_K (...) */
 	    || GET_CODE (operands[2]) != CONST_INT
-	    || CONST_OK_FOR_LETTER_P (INTVAL (operands[2]), 'K')))
+	    || satisfies_constraint_K (operands[2])))
        || (GET_MODE (operands[0]) == QImode 
 	   && (TARGET_PROMOTE_QImode || optimize_size)))"
   [(parallel [(set (match_dup 0)
@@ -18929,7 +19537,8 @@
   [(set (match_operand:SI 0 "push_operand" "")
 	(match_operand:SI 1 "memory_operand" ""))
    (match_scratch:SI 2 "r")]
-  "! optimize_size && ! TARGET_PUSH_MEMORY"
+  "!optimize_size && !TARGET_PUSH_MEMORY
+   && !RTX_FRAME_RELATED_P (peep2_next_insn (0))"
   [(set (match_dup 2) (match_dup 1))
    (set (match_dup 0) (match_dup 2))]
   "")
@@ -18938,7 +19547,8 @@
   [(set (match_operand:DI 0 "push_operand" "")
 	(match_operand:DI 1 "memory_operand" ""))
    (match_scratch:DI 2 "r")]
-  "! optimize_size && ! TARGET_PUSH_MEMORY"
+  "!optimize_size && !TARGET_PUSH_MEMORY
+   && !RTX_FRAME_RELATED_P (peep2_next_insn (0))"
   [(set (match_dup 2) (match_dup 1))
    (set (match_dup 0) (match_dup 2))]
   "")
@@ -18949,7 +19559,8 @@
   [(set (match_operand:SF 0 "push_operand" "")
 	(match_operand:SF 1 "memory_operand" ""))
    (match_scratch:SF 2 "r")]
-  "! optimize_size && ! TARGET_PUSH_MEMORY"
+  "!optimize_size && !TARGET_PUSH_MEMORY
+   && !RTX_FRAME_RELATED_P (peep2_next_insn (0))"
   [(set (match_dup 2) (match_dup 1))
    (set (match_dup 0) (match_dup 2))]
   "")
@@ -18958,7 +19569,8 @@
   [(set (match_operand:HI 0 "push_operand" "")
 	(match_operand:HI 1 "memory_operand" ""))
    (match_scratch:HI 2 "r")]
-  "! optimize_size && ! TARGET_PUSH_MEMORY"
+  "!optimize_size && !TARGET_PUSH_MEMORY
+   && !RTX_FRAME_RELATED_P (peep2_next_insn (0))"
   [(set (match_dup 2) (match_dup 1))
    (set (match_dup 0) (match_dup 2))]
   "")
@@ -18967,7 +19579,8 @@
   [(set (match_operand:QI 0 "push_operand" "")
 	(match_operand:QI 1 "memory_operand" ""))
    (match_scratch:QI 2 "q")]
-  "! optimize_size && ! TARGET_PUSH_MEMORY"
+  "!optimize_size && !TARGET_PUSH_MEMORY
+   && !RTX_FRAME_RELATED_P (peep2_next_insn (0))"
   [(set (match_dup 2) (match_dup 1))
    (set (match_dup 0) (match_dup 2))]
   "")
@@ -19127,8 +19740,7 @@
 	   (const_int 0)]))]
   "ix86_match_ccmode (insn, CCNOmode)
    && (true_regnum (operands[2]) != 0
-       || (GET_CODE (operands[3]) == CONST_INT
-	   && CONST_OK_FOR_LETTER_P (INTVAL (operands[3]), 'K')))
+       || satisfies_constraint_K (operands[3]))
    && peep2_reg_dead_p (1, operands[2])"
   [(parallel
      [(set (match_dup 0)
@@ -19693,9 +20305,8 @@
 		   (mult:DI (match_operand:DI 1 "memory_operand" "")
 			    (match_operand:DI 2 "immediate_operand" "")))
 	      (clobber (reg:CC FLAGS_REG))])]
-  "TARGET_K8 && !optimize_size
-   && (GET_CODE (operands[2]) != CONST_INT
-       || !CONST_OK_FOR_LETTER_P (INTVAL (operands[2]), 'K'))"
+  "(TARGET_K8 || TARGET_GENERIC64) && !optimize_size
+   && !satisfies_constraint_K (operands[2])"
   [(set (match_dup 3) (match_dup 1))
    (parallel [(set (match_dup 0) (mult:DI (match_dup 3) (match_dup 2)))
 	      (clobber (reg:CC FLAGS_REG))])]
@@ -19707,9 +20318,8 @@
 		   (mult:SI (match_operand:SI 1 "memory_operand" "")
 			    (match_operand:SI 2 "immediate_operand" "")))
 	      (clobber (reg:CC FLAGS_REG))])]
-  "TARGET_K8 && !optimize_size
-   && (GET_CODE (operands[2]) != CONST_INT
-       || !CONST_OK_FOR_LETTER_P (INTVAL (operands[2]), 'K'))"
+  "(TARGET_K8 || TARGET_GENERIC64) && !optimize_size
+   && !satisfies_constraint_K (operands[2])"
   [(set (match_dup 3) (match_dup 1))
    (parallel [(set (match_dup 0) (mult:SI (match_dup 3) (match_dup 2)))
 	      (clobber (reg:CC FLAGS_REG))])]
@@ -19722,9 +20332,8 @@
 		     (mult:SI (match_operand:SI 1 "memory_operand" "")
 			      (match_operand:SI 2 "immediate_operand" ""))))
 	      (clobber (reg:CC FLAGS_REG))])]
-  "TARGET_K8 && !optimize_size
-   && (GET_CODE (operands[2]) != CONST_INT
-       || !CONST_OK_FOR_LETTER_P (INTVAL (operands[2]), 'K'))"
+  "(TARGET_K8 || TARGET_GENERIC64) && !optimize_size
+   && !satisfies_constraint_K (operands[2])"
   [(set (match_dup 3) (match_dup 1))
    (parallel [(set (match_dup 0) (zero_extend:DI (mult:SI (match_dup 3) (match_dup 2))))
 	      (clobber (reg:CC FLAGS_REG))])]
@@ -19740,8 +20349,8 @@
 			    (match_operand:DI 2 "const_int_operand" "")))
 	      (clobber (reg:CC FLAGS_REG))])
    (match_scratch:DI 3 "r")]
-  "TARGET_K8 && !optimize_size
-   && CONST_OK_FOR_LETTER_P (INTVAL (operands[2]), 'K')"
+  "(TARGET_K8 || TARGET_GENERIC64) && !optimize_size
+   && satisfies_constraint_K (operands[2])"
   [(set (match_dup 3) (match_dup 2))
    (parallel [(set (match_dup 0) (mult:DI (match_dup 0) (match_dup 3)))
 	      (clobber (reg:CC FLAGS_REG))])]
@@ -19756,8 +20365,8 @@
 			    (match_operand:SI 2 "const_int_operand" "")))
 	      (clobber (reg:CC FLAGS_REG))])
    (match_scratch:SI 3 "r")]
-  "TARGET_K8 && !optimize_size
-   && CONST_OK_FOR_LETTER_P (INTVAL (operands[2]), 'K')"
+  "(TARGET_K8 || TARGET_GENERIC64) && !optimize_size
+   && satisfies_constraint_K (operands[2])"
   [(set (match_dup 3) (match_dup 2))
    (parallel [(set (match_dup 0) (mult:SI (match_dup 0) (match_dup 3)))
 	      (clobber (reg:CC FLAGS_REG))])]
@@ -19772,7 +20381,7 @@
 			    (match_operand:HI 2 "immediate_operand" "")))
 	      (clobber (reg:CC FLAGS_REG))])
    (match_scratch:HI 3 "r")]
-  "TARGET_K8 && !optimize_size"
+  "(TARGET_K8 || TARGET_GENERIC64) && !optimize_size"
   [(set (match_dup 3) (match_dup 2))
    (parallel [(set (match_dup 0) (mult:HI (match_dup 0) (match_dup 3)))
 	      (clobber (reg:CC FLAGS_REG))])]
@@ -19780,6 +20389,64 @@
   if (!rtx_equal_p (operands[0], operands[1]))
     emit_move_insn (operands[0], operands[1]);
 })
+
+;; After splitting up read-modify operations, array accesses with memory
+;; operands might end up in form:
+;;  sall    $2, %eax
+;;  movl    4(%esp), %edx
+;;  addl    %edx, %eax
+;; instead of pre-splitting:
+;;  sall    $2, %eax
+;;  addl    4(%esp), %eax
+;; Turn it into:
+;;  movl    4(%esp), %edx
+;;  leal    (%edx,%eax,4), %eax
+
+(define_peephole2
+  [(parallel [(set (match_operand 0 "register_operand" "")
+		   (ashift (match_operand 1 "register_operand" "")
+			   (match_operand 2 "const_int_operand" "")))
+	       (clobber (reg:CC FLAGS_REG))])
+   (set (match_operand 3 "register_operand")
+        (match_operand 4 "x86_64_general_operand" ""))
+   (parallel [(set (match_operand 5 "register_operand" "")
+		   (plus (match_operand 6 "register_operand" "")
+			 (match_operand 7 "register_operand" "")))
+		   (clobber (reg:CC FLAGS_REG))])]
+  "INTVAL (operands[2]) >= 0 && INTVAL (operands[2]) <= 3
+   /* Validate MODE for lea.  */
+   && ((!TARGET_PARTIAL_REG_STALL
+	&& (GET_MODE (operands[0]) == QImode
+	    || GET_MODE (operands[0]) == HImode))
+       || GET_MODE (operands[0]) == SImode 
+       || (TARGET_64BIT && GET_MODE (operands[0]) == DImode))
+   /* We reorder load and the shift.  */
+   && !rtx_equal_p (operands[1], operands[3])
+   && !reg_overlap_mentioned_p (operands[0], operands[4])
+   /* Last PLUS must consist of operand 0 and 3.  */
+   && !rtx_equal_p (operands[0], operands[3])
+   && (rtx_equal_p (operands[3], operands[6])
+       || rtx_equal_p (operands[3], operands[7]))
+   && (rtx_equal_p (operands[0], operands[6])
+       || rtx_equal_p (operands[0], operands[7]))
+   /* The intermediate operand 0 must die or be same as output.  */
+   && (rtx_equal_p (operands[0], operands[5])
+       || peep2_reg_dead_p (3, operands[0]))"
+  [(set (match_dup 3) (match_dup 4))
+   (set (match_dup 0) (match_dup 1))]
+{
+  enum machine_mode mode = GET_MODE (operands[5]) == DImode ? DImode : SImode;
+  int scale = 1 << INTVAL (operands[2]);
+  rtx index = gen_lowpart (Pmode, operands[1]);
+  rtx base = gen_lowpart (Pmode, operands[3]);
+  rtx dest = gen_lowpart (mode, operands[5]);
+
+  operands[1] = gen_rtx_PLUS (Pmode, base,
+  			      gen_rtx_MULT (Pmode, index, GEN_INT (scale)));
+  if (mode != Pmode)
+    operands[1] = gen_rtx_SUBREG (mode, operands[1], 0);
+  operands[0] = dest;
+})
 
 ;; Call-value patterns last so that the wildcard operand does not
 ;; disrupt insn-recog's switch tables.
@@ -19907,7 +20574,7 @@
 (define_insn "trap"
   [(trap_if (const_int 1) (const_int 6))]
   ""
-  ".word\t0x0b0f"
+  { return ASM_SHORT "0x0b0f"; }
   [(set_attr "length" "2")])
 
 (define_expand "sse_prologue_save"
@@ -20113,7 +20780,15 @@
    (set (match_scratch:DI 2 "=&r") (const_int 0))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_64BIT"
-  "mov{q}\t{%%fs:%P1, %2|%2, QWORD PTR %%fs:%P1}\;mov{q}\t{%2, %0|%0, %2}\;xor{l}\t%k2, %k2"
+  {
+     /* The kernel uses a different segment register for performance reasons; a
+        system call would not have to trash the userspace segment register,
+        which would be expensive */
+     if (ix86_cmodel != CM_KERNEL)
+        return "mov{q}\t{%%fs:%P1, %2|%2, QWORD PTR %%fs:%P1}\;mov{q}\t{%2, %0|%0, %2}\;xor{l}\t%k2, %k2";
+     else
+        return "mov{q}\t{%%gs:%P1, %2|%2, QWORD PTR %%gs:%P1}\;mov{q}\t{%2, %0|%0, %2}\;xor{l}\t%k2, %k2";
+  }
   [(set_attr "type" "multi")])
 
 (define_expand "stack_protect_test"
@@ -20181,7 +20856,15 @@
 		    UNSPEC_SP_TLS_TEST))
    (clobber (match_scratch:DI 3 "=r"))]
   "TARGET_64BIT"
-  "mov{q}\t{%1, %3|%3, %1}\;xor{q}\t{%%fs:%P2, %3|%3, QWORD PTR %%fs:%P2}"
+  {
+     /* The kernel uses a different segment register for performance reasons; a
+        system call would not have to trash the userspace segment register,
+        which would be expensive */
+     if (ix86_cmodel != CM_KERNEL)
+        return "mov{q}\t{%1, %3|%3, %1}\;xor{q}\t{%%fs:%P2, %3|%3, QWORD PTR %%fs:%P2}";
+     else
+        return "mov{q}\t{%1, %3|%3, %1}\;xor{q}\t{%%gs:%P2, %3|%3, QWORD PTR %%gs:%P2}";
+  }
   [(set_attr "type" "multi")])
 
 (include "sse.md")