* config/i386/i386.md (ashlti3, ashrti3, lshrti3): Expand using

[pf3gnuchains/gcc-fork.git] / gcc / config / i386 / i386.md

diff --git a/gcc/config/i386/i386.md b/gcc/config/i386/i386.md
index a021e7c..36e812f 100644 (file)
--- a/gcc/config/i386/i386.md
+++ b/gcc/config/i386/i386.md
@@ -213,6 +213,7 @@
    (UNSPECV_XCHG               12)
    (UNSPECV_LOCK               13)
    (UNSPECV_PROLOGUE_USE       14)
+   (UNSPECV_CLD                        15)
   ])
 
 ;; Constants to represent pcomtrue/pcomfalse variants
@@ -596,7 +597,7 @@
 (define_mode_attr r [(QI "q") (HI "r") (SI "r") (DI "r")])
 
 ;; Immediate operand constraint for integer modes.
-(define_mode_attr i [(QI "i") (HI "i") (SI "i") (DI "e")])
+(define_mode_attr i [(QI "n") (HI "n") (SI "i") (DI "e")])
 
 ;; General operand predicate for integer modes.
 (define_mode_attr general_operand
@@ -629,6 +630,11 @@
 
 ;; Instruction suffix for REX 64bit operators.
 (define_mode_attr rex64suffix [(SI "") (DI "{q}")])
+
+;; This mode iterator allows :P to be used for patterns that operate on
+;; pointer-sized quantities.  Exactly one of the two alternatives will match.
+(define_mode_iterator P [(SI "Pmode == SImode") (DI "Pmode == DImode")])
+
 \f
 ;; Scheduling descriptions
 
@@ -719,7 +725,7 @@
 (define_insn "cmpdi_ccno_1_rex64"
   [(set (reg FLAGS_REG)
        (compare (match_operand:DI 0 "nonimmediate_operand" "r,?mr")
-                (match_operand:DI 1 "const0_operand" "n,n")))]
+                (match_operand:DI 1 "const0_operand" "")))]
   "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)"
   "@
    test{q}\t%0, %0
@@ -758,7 +764,7 @@
 (define_insn "*cmpsi_ccno_1"
   [(set (reg FLAGS_REG)
        (compare (match_operand:SI 0 "nonimmediate_operand" "r,?mr")
-                (match_operand:SI 1 "const0_operand" "n,n")))]
+                (match_operand:SI 1 "const0_operand" "")))]
   "ix86_match_ccmode (insn, CCNOmode)"
   "@
    test{l}\t%0, %0
@@ -797,7 +803,7 @@
 (define_insn "*cmphi_ccno_1"
   [(set (reg FLAGS_REG)
        (compare (match_operand:HI 0 "nonimmediate_operand" "r,?mr")
-                (match_operand:HI 1 "const0_operand" "n,n")))]
+                (match_operand:HI 1 "const0_operand" "")))]
   "ix86_match_ccmode (insn, CCNOmode)"
   "@
    test{w}\t%0, %0
@@ -809,7 +815,7 @@
 (define_insn "*cmphi_minus_1"
   [(set (reg FLAGS_REG)
        (compare (minus:HI (match_operand:HI 0 "nonimmediate_operand" "rm,r")
-                          (match_operand:HI 1 "general_operand" "ri,mr"))
+                          (match_operand:HI 1 "general_operand" "rn,mr"))
                 (const_int 0)))]
   "ix86_match_ccmode (insn, CCGOCmode)"
   "cmp{w}\t{%1, %0|%0, %1}"
@@ -819,7 +825,7 @@
 (define_insn "*cmphi_1"
   [(set (reg FLAGS_REG)
        (compare (match_operand:HI 0 "nonimmediate_operand" "rm,r")
-                (match_operand:HI 1 "general_operand" "ri,mr")))]
+                (match_operand:HI 1 "general_operand" "rn,mr")))]
   "!(MEM_P (operands[0]) && MEM_P (operands[1]))
    && ix86_match_ccmode (insn, CCmode)"
   "cmp{w}\t{%1, %0|%0, %1}"
@@ -829,7 +835,7 @@
 (define_insn "*cmpqi_ccno_1"
   [(set (reg FLAGS_REG)
        (compare (match_operand:QI 0 "nonimmediate_operand" "q,?mq")
-                (match_operand:QI 1 "const0_operand" "n,n")))]
+                (match_operand:QI 1 "const0_operand" "")))]
   "ix86_match_ccmode (insn, CCNOmode)"
   "@
    test{b}\t%0, %0
@@ -841,7 +847,7 @@
 (define_insn "*cmpqi_1"
   [(set (reg FLAGS_REG)
        (compare (match_operand:QI 0 "nonimmediate_operand" "qm,q")
-                (match_operand:QI 1 "general_operand" "qi,mq")))]
+                (match_operand:QI 1 "general_operand" "qn,mq")))]
   "!(MEM_P (operands[0]) && MEM_P (operands[1]))
     && ix86_match_ccmode (insn, CCmode)"
   "cmp{b}\t{%1, %0|%0, %1}"
@@ -851,7 +857,7 @@
 (define_insn "*cmpqi_minus_1"
   [(set (reg FLAGS_REG)
        (compare (minus:QI (match_operand:QI 0 "nonimmediate_operand" "qm,q")
-                          (match_operand:QI 1 "general_operand" "qi,mq"))
+                          (match_operand:QI 1 "general_operand" "qn,mq"))
                 (const_int 0)))]
   "ix86_match_ccmode (insn, CCGOCmode)"
   "cmp{b}\t{%1, %0|%0, %1}"
@@ -894,7 +900,7 @@
              (match_operand 0 "ext_register_operand" "Q")
              (const_int 8)
              (const_int 8)) 0)
-         (match_operand:QI 1 "const0_operand" "n")))]
+         (match_operand:QI 1 "const0_operand" "")))]
   "ix86_match_ccmode (insn, CCNOmode)"
   "test{b}\t%h0, %h0"
   [(set_attr "type" "test")
@@ -1000,7 +1006,7 @@
        (unspec:HI
          [(compare:CCFP
             (match_operand 1 "register_operand" "f")
-            (match_operand 2 "const0_operand" "X"))]
+            (match_operand 2 "const0_operand" ""))]
        UNSPEC_FNSTSW))]
   "X87_FLOAT_MODE_P (GET_MODE (operands[1]))
    && GET_MODE (operands[1]) == GET_MODE (operands[2])"
@@ -1019,7 +1025,7 @@
   [(set (reg:CCFP FLAGS_REG)
        (compare:CCFP
          (match_operand 1 "register_operand" "f")
-         (match_operand 2 "const0_operand" "X")))
+         (match_operand 2 "const0_operand" "")))
    (clobber (match_operand:HI 0 "register_operand" "=a"))]
   "X87_FLOAT_MODE_P (GET_MODE (operands[1]))
    && TARGET_SAHF && !TARGET_CMOVE
@@ -1168,7 +1174,7 @@
               [(match_operand:X87MODEI12 2 "memory_operand" "m")]))]
          UNSPEC_FNSTSW))]
   "X87_FLOAT_MODE_P (GET_MODE (operands[1]))
-   && TARGET_USE_<MODE>MODE_FIOP
+   && (TARGET_USE_<MODE>MODE_FIOP || optimize_size)
    && (GET_MODE (operands [3]) == GET_MODE (operands[1]))"
   "* return output_fp_compare (insn, operands, 0, 0);"
   [(set_attr "type" "multi")
@@ -1185,7 +1191,7 @@
    (clobber (match_operand:HI 0 "register_operand" "=a"))]
   "X87_FLOAT_MODE_P (GET_MODE (operands[1]))
    && TARGET_SAHF && !TARGET_CMOVE
-   && TARGET_USE_<MODE>MODE_FIOP
+   && (TARGET_USE_<MODE>MODE_FIOP || optimize_size)
    && (GET_MODE (operands [3]) == GET_MODE (operands[1]))"
   "#"
   "&& reload_completed"
@@ -1409,7 +1415,7 @@
 
 (define_insn "*movsi_xor"
   [(set (match_operand:SI 0 "register_operand" "=r")
-       (match_operand:SI 1 "const0_operand" "i"))
+       (match_operand:SI 1 "const0_operand" ""))
    (clobber (reg:CC FLAGS_REG))]
   "reload_completed && (!TARGET_USE_MOV0 || optimize_size)"
   "xor{l}\t%0, %0"
@@ -1434,9 +1440,9 @@
 
 (define_insn "*movsi_1"
   [(set (match_operand:SI 0 "nonimmediate_operand"
-                       "=r  ,m  ,*y,*y,?rm,?*y,*x,*x,?r ,m ,?*Yi,*x")
+                       "=r,m ,*y,*y,?rm,?*y,*x,*x,?r ,m ,?*Yi,*x")
        (match_operand:SI 1 "general_operand"
-                       "rinm,rin,C ,*y,*y ,rm ,C ,*x,*Yi,*x,r   ,m "))]
+                       "g ,ri,C ,*y,*y ,rm ,C ,*x,*Yi,*x,r   ,m "))]
   "!(MEM_P (operands[0]) && MEM_P (operands[1]))"
 {
   switch (get_attr_type (insn))
@@ -1565,7 +1571,7 @@
 ;; For 64BIT abi we always round up to 8 bytes.
 (define_insn "*pushhi2_rex64"
   [(set (match_operand:HI 0 "push_operand" "=X")
-       (match_operand:HI 1 "nonmemory_no_elim_operand" "ri"))]
+       (match_operand:HI 1 "nonmemory_no_elim_operand" "rn"))]
   "TARGET_64BIT"
   "push{q}\t%q1"
   [(set_attr "type" "push")
@@ -1700,7 +1706,7 @@
 
 (define_insn "*movstricthi_xor"
   [(set (strict_low_part (match_operand:HI 0 "register_operand" "+r"))
-       (match_operand:HI 1 "const0_operand" "i"))
+       (match_operand:HI 1 "const0_operand" ""))
    (clobber (reg:CC FLAGS_REG))]
   "reload_completed
    && ((!TARGET_USE_MOV0 && !TARGET_PARTIAL_REG_STALL) || optimize_size)"
@@ -1730,7 +1736,7 @@
 ;; For 64BIT abi we always round up to 8 bytes.
 (define_insn "*pushqi2_rex64"
   [(set (match_operand:QI 0 "push_operand" "=X")
-       (match_operand:QI 1 "nonmemory_no_elim_operand" "qi"))]
+       (match_operand:QI 1 "nonmemory_no_elim_operand" "qn"))]
   "TARGET_64BIT"
   "push{q}\t%q1"
   [(set_attr "type" "push")
@@ -1857,7 +1863,7 @@
 
 (define_insn "*movstrictqi_xor"
   [(set (strict_low_part (match_operand:QI 0 "q_regs_operand" "+q"))
-       (match_operand:QI 1 "const0_operand" "i"))
+       (match_operand:QI 1 "const0_operand" ""))
    (clobber (reg:CC FLAGS_REG))]
   "reload_completed && (!TARGET_USE_MOV0 || optimize_size)"
   "xor{b}\t%0, %0"
@@ -2183,7 +2189,7 @@
 
 (define_insn "*movdi_xor_rex64"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (match_operand:DI 1 "const0_operand" "i"))
+       (match_operand:DI 1 "const0_operand" ""))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_64BIT && (!TARGET_USE_MOV0 || optimize_size)
    && reload_completed"
@@ -2734,17 +2740,9 @@
 (define_split
   [(set (match_operand:DF 0 "push_operand" "")
        (match_operand:DF 1 "any_fp_register_operand" ""))]
-  "!TARGET_64BIT && reload_completed"
-  [(set (reg:SI SP_REG) (plus:SI (reg:SI SP_REG) (const_int -8)))
-   (set (mem:DF (reg:SI SP_REG)) (match_dup 1))]
-  "")
-
-(define_split
-  [(set (match_operand:DF 0 "push_operand" "")
-       (match_operand:DF 1 "any_fp_register_operand" ""))]
-  "TARGET_64BIT && reload_completed"
-  [(set (reg:DI SP_REG) (plus:DI (reg:DI SP_REG) (const_int -8)))
-   (set (mem:DF (reg:DI SP_REG)) (match_dup 1))]
+  "reload_completed"
+  [(set (reg:P SP_REG) (plus:P (reg:P SP_REG) (const_int -8)))
+   (set (mem:DF (reg:P SP_REG)) (match_dup 1))]
   "")
 
 (define_split
@@ -3197,17 +3195,9 @@
 (define_split
   [(set (match_operand:XF 0 "push_operand" "")
        (match_operand:XF 1 "any_fp_register_operand" ""))]
-  "!TARGET_64BIT"
-  [(set (reg:SI SP_REG) (plus:SI (reg:SI SP_REG) (match_dup 2)))
-   (set (mem:XF (reg:SI SP_REG)) (match_dup 1))]
-  "operands[2] = GEN_INT (TARGET_128BIT_LONG_DOUBLE ? -16 : -12);")
-
-(define_split
-  [(set (match_operand:XF 0 "push_operand" "")
-       (match_operand:XF 1 "any_fp_register_operand" ""))]
-  "TARGET_64BIT"
-  [(set (reg:DI SP_REG) (plus:DI (reg:DI SP_REG) (match_dup 2)))
-   (set (mem:XF (reg:DI SP_REG)) (match_dup 1))]
+  ""
+  [(set (reg:P SP_REG) (plus:P (reg:P SP_REG) (match_dup 2)))
+   (set (mem:XF (reg:P SP_REG)) (match_dup 1))]
   "operands[2] = GEN_INT (TARGET_128BIT_LONG_DOUBLE ? -16 : -12);")
 
 ;; Do not use integer registers when optimizing for size
@@ -3950,16 +3940,9 @@
 (define_split
   [(set (match_operand:DF 0 "push_operand" "")
        (float_extend:DF (match_operand:SF 1 "fp_register_operand" "")))]
-  "!TARGET_64BIT"
-  [(set (reg:SI SP_REG) (plus:SI (reg:SI SP_REG) (const_int -8)))
-   (set (mem:DF (reg:SI SP_REG)) (float_extend:DF (match_dup 1)))])
-
-(define_split
-  [(set (match_operand:DF 0 "push_operand" "")
-       (float_extend:DF (match_operand:SF 1 "fp_register_operand" "")))]
-  "TARGET_64BIT"
-  [(set (reg:DI SP_REG) (plus:DI (reg:DI SP_REG) (const_int -8)))
-   (set (mem:DF (reg:DI SP_REG)) (float_extend:DF (match_dup 1)))])
+  ""
+  [(set (reg:P SP_REG) (plus:P (reg:P SP_REG) (const_int -8)))
+   (set (mem:DF (reg:P SP_REG)) (float_extend:DF (match_dup 1)))])
 
 (define_insn "*dummy_extendsfxf2"
   [(set (match_operand:XF 0 "push_operand" "=<")
@@ -3971,32 +3954,16 @@
   [(set (match_operand:XF 0 "push_operand" "")
        (float_extend:XF (match_operand:SF 1 "fp_register_operand" "")))]
   ""
-  [(set (reg:SI SP_REG) (plus:SI (reg:SI SP_REG) (match_dup 2)))
-   (set (mem:XF (reg:SI SP_REG)) (float_extend:XF (match_dup 1)))]
-  "operands[2] = GEN_INT (TARGET_128BIT_LONG_DOUBLE ? -16 : -12);")
-
-(define_split
-  [(set (match_operand:XF 0 "push_operand" "")
-       (float_extend:XF (match_operand:SF 1 "fp_register_operand" "")))]
-  "TARGET_64BIT"
-  [(set (reg:DI SP_REG) (plus:DI (reg:DI SP_REG) (match_dup 2)))
-   (set (mem:DF (reg:DI SP_REG)) (float_extend:XF (match_dup 1)))]
+  [(set (reg:P SP_REG) (plus:P (reg:P SP_REG) (match_dup 2)))
+   (set (mem:XF (reg:P SP_REG)) (float_extend:XF (match_dup 1)))]
   "operands[2] = GEN_INT (TARGET_128BIT_LONG_DOUBLE ? -16 : -12);")
 
 (define_split
   [(set (match_operand:XF 0 "push_operand" "")
        (float_extend:XF (match_operand:DF 1 "fp_register_operand" "")))]
   ""
-  [(set (reg:SI SP_REG) (plus:SI (reg:SI SP_REG) (match_dup 2)))
-   (set (mem:DF (reg:SI SP_REG)) (float_extend:XF (match_dup 1)))]
-  "operands[2] = GEN_INT (TARGET_128BIT_LONG_DOUBLE ? -16 : -12);")
-
-(define_split
-  [(set (match_operand:XF 0 "push_operand" "")
-       (float_extend:XF (match_operand:DF 1 "fp_register_operand" "")))]
-  "TARGET_64BIT"
-  [(set (reg:DI SP_REG) (plus:DI (reg:DI SP_REG) (match_dup 2)))
-   (set (mem:XF (reg:DI SP_REG)) (float_extend:XF (match_dup 1)))]
+  [(set (reg:P SP_REG) (plus:P (reg:P SP_REG) (match_dup 2)))
+   (set (mem:DF (reg:P SP_REG)) (float_extend:XF (match_dup 1)))]
   "operands[2] = GEN_INT (TARGET_128BIT_LONG_DOUBLE ? -16 : -12);")
 
 (define_expand "extendsfdf2"
@@ -5639,7 +5606,7 @@
   [(set (match_operand:QI 0 "nonimmediate_operand" "=qm,q")
          (plus:QI (plus:QI (match_operand:QI 3 "ix86_carry_flag_operator" "")
                            (match_operand:QI 1 "nonimmediate_operand" "%0,0"))
-                  (match_operand:QI 2 "general_operand" "qi,qm")))
+                  (match_operand:QI 2 "general_operand" "qn,qm")))
    (clobber (reg:CC FLAGS_REG))]
   "ix86_binary_operator_ok (PLUS, QImode, operands)"
   "adc{b}\t{%2, %0|%0, %2}"
@@ -5651,7 +5618,7 @@
   [(set (match_operand:HI 0 "nonimmediate_operand" "=rm,r")
          (plus:HI (plus:HI (match_operand:HI 3 "ix86_carry_flag_operator" "")
                            (match_operand:HI 1 "nonimmediate_operand" "%0,0"))
-                  (match_operand:HI 2 "general_operand" "ri,rm")))
+                  (match_operand:HI 2 "general_operand" "rn,rm")))
    (clobber (reg:CC FLAGS_REG))]
   "ix86_binary_operator_ok (PLUS, HImode, operands)"
   "adc{w}\t{%2, %0|%0, %2}"
@@ -5699,7 +5666,7 @@
 (define_insn "addqi3_cc"
   [(set (reg:CC FLAGS_REG)
        (unspec:CC [(match_operand:QI 1 "nonimmediate_operand" "%0,0")
-                   (match_operand:QI 2 "general_operand" "qi,qm")]
+                   (match_operand:QI 2 "general_operand" "qn,qm")]
                   UNSPEC_ADD_CARRY))
    (set (match_operand:QI 0 "nonimmediate_operand" "=qm,q")
        (plus:QI (match_dup 1) (match_dup 2)))]
@@ -6177,7 +6144,7 @@
 (define_insn "*addsi_1"
   [(set (match_operand:SI 0 "nonimmediate_operand" "=r,rm,r")
        (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,r")
-                (match_operand:SI 2 "general_operand" "rmni,rni,lni")))
+                (match_operand:SI 2 "general_operand" "g,ri,li")))
    (clobber (reg:CC FLAGS_REG))]
   "ix86_binary_operator_ok (PLUS, SImode, operands)"
 {
@@ -6261,7 +6228,7 @@
   [(set (match_operand:DI 0 "register_operand" "=r,r")
        (zero_extend:DI
          (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,r")
-                  (match_operand:SI 2 "general_operand" "rmni,lni"))))
+                  (match_operand:SI 2 "general_operand" "g,li"))))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_64BIT && ix86_binary_operator_ok (PLUS, SImode, operands)"
 {
@@ -6327,7 +6294,7 @@
   [(set (reg FLAGS_REG)
        (compare
          (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                  (match_operand:SI 2 "general_operand" "rmni,rni"))
+                  (match_operand:SI 2 "general_operand" "g,ri"))
          (const_int 0)))
    (set (match_operand:SI 0 "nonimmediate_operand" "=r,rm")
        (plus:SI (match_dup 1) (match_dup 2)))]
@@ -6375,7 +6342,7 @@
   [(set (reg FLAGS_REG)
        (compare
          (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
-                  (match_operand:SI 2 "general_operand" "rmni"))
+                  (match_operand:SI 2 "general_operand" "g"))
          (const_int 0)))
    (set (match_operand:DI 0 "register_operand" "=r")
        (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
@@ -6418,7 +6385,7 @@
 
 (define_insn "*addsi_3"
   [(set (reg FLAGS_REG)
-       (compare (neg:SI (match_operand:SI 2 "general_operand" "rmni"))
+       (compare (neg:SI (match_operand:SI 2 "general_operand" "g"))
                 (match_operand:SI 1 "nonimmediate_operand" "%0")))
    (clobber (match_scratch:SI 0 "=r"))]
   "ix86_match_ccmode (insn, CCZmode)
@@ -6463,7 +6430,7 @@
 ;; See comment for addsi_1_zext why we do use nonimmediate_operand
 (define_insn "*addsi_3_zext"
   [(set (reg FLAGS_REG)
-       (compare (neg:SI (match_operand:SI 2 "general_operand" "rmni"))
+       (compare (neg:SI (match_operand:SI 2 "general_operand" "g"))
                 (match_operand:SI 1 "nonimmediate_operand" "%0")))
    (set (match_operand:DI 0 "register_operand" "=r")
        (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
@@ -6553,7 +6520,7 @@
   [(set (reg FLAGS_REG)
        (compare
          (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
-                  (match_operand:SI 2 "general_operand" "rmni"))
+                  (match_operand:SI 2 "general_operand" "g"))
          (const_int 0)))
    (clobber (match_scratch:SI 0 "=r"))]
   "ix86_match_ccmode (insn, CCGOCmode)
@@ -6610,7 +6577,7 @@
 (define_insn "*addhi_1_lea"
   [(set (match_operand:HI 0 "nonimmediate_operand" "=rm,r,r")
        (plus:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0,r")
-                (match_operand:HI 2 "general_operand" "ri,rm,lni")))
+                (match_operand:HI 2 "general_operand" "rn,rm,ln")))
    (clobber (reg:CC FLAGS_REG))]
   "!TARGET_PARTIAL_REG_STALL
    && ix86_binary_operator_ok (PLUS, HImode, operands)"
@@ -6653,7 +6620,7 @@
 (define_insn "*addhi_1"
   [(set (match_operand:HI 0 "nonimmediate_operand" "=rm,r")
        (plus:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0")
-                (match_operand:HI 2 "general_operand" "ri,rm")))
+                (match_operand:HI 2 "general_operand" "rn,rm")))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_PARTIAL_REG_STALL
    && ix86_binary_operator_ok (PLUS, HImode, operands)"
@@ -6693,7 +6660,7 @@
   [(set (reg FLAGS_REG)
        (compare
          (plus:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0")
-                  (match_operand:HI 2 "general_operand" "rmni,rni"))
+                  (match_operand:HI 2 "general_operand" "rmn,rn"))
          (const_int 0)))
    (set (match_operand:HI 0 "nonimmediate_operand" "=r,rm")
        (plus:HI (match_dup 1) (match_dup 2)))]
@@ -6733,7 +6700,7 @@
 
 (define_insn "*addhi_3"
   [(set (reg FLAGS_REG)
-       (compare (neg:HI (match_operand:HI 2 "general_operand" "rmni"))
+       (compare (neg:HI (match_operand:HI 2 "general_operand" "rmn"))
                 (match_operand:HI 1 "nonimmediate_operand" "%0")))
    (clobber (match_scratch:HI 0 "=r"))]
   "ix86_match_ccmode (insn, CCZmode)
@@ -6813,7 +6780,7 @@
   [(set (reg FLAGS_REG)
        (compare
          (plus:HI (match_operand:HI 1 "nonimmediate_operand" "%0")
-                  (match_operand:HI 2 "general_operand" "rmni"))
+                  (match_operand:HI 2 "general_operand" "rmn"))
          (const_int 0)))
    (clobber (match_scratch:HI 0 "=r"))]
   "ix86_match_ccmode (insn, CCGOCmode)
@@ -6999,7 +6966,7 @@
   [(set (reg FLAGS_REG)
        (compare
          (plus:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0")
-                  (match_operand:QI 2 "general_operand" "qmni,qni"))
+                  (match_operand:QI 2 "general_operand" "qmn,qn"))
          (const_int 0)))
    (set (match_operand:QI 0 "nonimmediate_operand" "=q,qm")
        (plus:QI (match_dup 1) (match_dup 2)))]
@@ -7038,7 +7005,7 @@
 
 (define_insn "*addqi_3"
   [(set (reg FLAGS_REG)
-       (compare (neg:QI (match_operand:QI 2 "general_operand" "qmni"))
+       (compare (neg:QI (match_operand:QI 2 "general_operand" "qmn"))
                 (match_operand:QI 1 "nonimmediate_operand" "%0")))
    (clobber (match_scratch:QI 0 "=q"))]
   "ix86_match_ccmode (insn, CCZmode)
@@ -7117,7 +7084,7 @@
   [(set (reg FLAGS_REG)
        (compare
          (plus:QI (match_operand:QI 1 "nonimmediate_operand" "%0")
-                  (match_operand:QI 2 "general_operand" "qmni"))
+                  (match_operand:QI 2 "general_operand" "qmn"))
          (const_int 0)))
    (clobber (match_scratch:QI 0 "=q"))]
   "ix86_match_ccmode (insn, CCGOCmode)
@@ -7381,7 +7348,7 @@
   [(set (match_operand:QI 0 "nonimmediate_operand" "=qm,q")
          (minus:QI (match_operand:QI 1 "nonimmediate_operand" "0,0")
            (plus:QI (match_operand:QI 3 "ix86_carry_flag_operator" "")
-              (match_operand:QI 2 "general_operand" "qi,qm"))))
+              (match_operand:QI 2 "general_operand" "qn,qm"))))
    (clobber (reg:CC FLAGS_REG))]
   "ix86_binary_operator_ok (MINUS, QImode, operands)"
   "sbb{b}\t{%2, %0|%0, %2}"
@@ -7393,7 +7360,7 @@
   [(set (match_operand:HI 0 "nonimmediate_operand" "=rm,r")
          (minus:HI (match_operand:HI 1 "nonimmediate_operand" "0,0")
            (plus:HI (match_operand:HI 3 "ix86_carry_flag_operator" "")
-              (match_operand:HI 2 "general_operand" "ri,rm"))))
+              (match_operand:HI 2 "general_operand" "rn,rm"))))
    (clobber (reg:CC FLAGS_REG))]
   "ix86_binary_operator_ok (MINUS, HImode, operands)"
   "sbb{w}\t{%2, %0|%0, %2}"
@@ -7522,7 +7489,7 @@
 (define_insn "*subhi_1"
   [(set (match_operand:HI 0 "nonimmediate_operand" "=rm,r")
        (minus:HI (match_operand:HI 1 "nonimmediate_operand" "0,0")
-                 (match_operand:HI 2 "general_operand" "ri,rm")))
+                 (match_operand:HI 2 "general_operand" "rn,rm")))
    (clobber (reg:CC FLAGS_REG))]
   "ix86_binary_operator_ok (MINUS, HImode, operands)"
   "sub{w}\t{%2, %0|%0, %2}"
@@ -7533,7 +7500,7 @@
   [(set (reg FLAGS_REG)
        (compare
          (minus:HI (match_operand:HI 1 "nonimmediate_operand" "0,0")
-                   (match_operand:HI 2 "general_operand" "ri,rm"))
+                   (match_operand:HI 2 "general_operand" "rn,rm"))
          (const_int 0)))
    (set (match_operand:HI 0 "nonimmediate_operand" "=rm,r")
        (minus:HI (match_dup 1) (match_dup 2)))]
@@ -7546,7 +7513,7 @@
 (define_insn "*subhi_3"
   [(set (reg FLAGS_REG)
        (compare (match_operand:HI 1 "nonimmediate_operand" "0,0")
-                (match_operand:HI 2 "general_operand" "ri,rm")))
+                (match_operand:HI 2 "general_operand" "rn,rm")))
    (set (match_operand:HI 0 "nonimmediate_operand" "=rm,r")
        (minus:HI (match_dup 1) (match_dup 2)))]
   "ix86_match_ccmode (insn, CCmode)
@@ -7566,7 +7533,7 @@
 (define_insn "*subqi_1"
   [(set (match_operand:QI 0 "nonimmediate_operand" "=qm,q")
        (minus:QI (match_operand:QI 1 "nonimmediate_operand" "0,0")
-                 (match_operand:QI 2 "general_operand" "qn,qmn")))
+                 (match_operand:QI 2 "general_operand" "qn,qm")))
    (clobber (reg:CC FLAGS_REG))]
   "ix86_binary_operator_ok (MINUS, QImode, operands)"
   "sub{b}\t{%2, %0|%0, %2}"
@@ -7576,7 +7543,7 @@
 (define_insn "*subqi_1_slp"
   [(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+qm,q"))
        (minus:QI (match_dup 0)
-                 (match_operand:QI 1 "general_operand" "qn,qmn")))
+                 (match_operand:QI 1 "general_operand" "qn,qm")))
    (clobber (reg:CC FLAGS_REG))]
   "(! TARGET_PARTIAL_REG_STALL || optimize_size)
    && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
@@ -7588,10 +7555,10 @@
   [(set (reg FLAGS_REG)
        (compare
          (minus:QI (match_operand:QI 1 "nonimmediate_operand" "0,0")
-                   (match_operand:QI 2 "general_operand" "qi,qm"))
+                   (match_operand:QI 2 "general_operand" "qn,qm"))
          (const_int 0)))
-   (set (match_operand:HI 0 "nonimmediate_operand" "=qm,q")
-       (minus:HI (match_dup 1) (match_dup 2)))]
+   (set (match_operand:QI 0 "nonimmediate_operand" "=qm,q")
+       (minus:QI (match_dup 1) (match_dup 2)))]
   "ix86_match_ccmode (insn, CCGOCmode)
    && ix86_binary_operator_ok (MINUS, QImode, operands)"
   "sub{b}\t{%2, %0|%0, %2}"
@@ -7601,9 +7568,9 @@
 (define_insn "*subqi_3"
   [(set (reg FLAGS_REG)
        (compare (match_operand:QI 1 "nonimmediate_operand" "0,0")
-                (match_operand:QI 2 "general_operand" "qi,qm")))
-   (set (match_operand:HI 0 "nonimmediate_operand" "=qm,q")
-       (minus:HI (match_dup 1) (match_dup 2)))]
+                (match_operand:QI 2 "general_operand" "qn,qm")))
+   (set (match_operand:QI 0 "nonimmediate_operand" "=qm,q")
+       (minus:QI (match_dup 1) (match_dup 2)))]
   "ix86_match_ccmode (insn, CCmode)
    && ix86_binary_operator_ok (MINUS, QImode, operands)"
   "sub{b}\t{%2, %0|%0, %2}"
@@ -7765,7 +7732,7 @@
 (define_insn "*mulhi3_1"
   [(set (match_operand:HI 0 "register_operand" "=r,r,r")
        (mult:HI (match_operand:HI 1 "nonimmediate_operand" "%rm,rm,0")
-                (match_operand:HI 2 "general_operand" "K,i,mr")))
+                (match_operand:HI 2 "general_operand" "K,n,mr")))
    (clobber (reg:CC FLAGS_REG))]
   "!(MEM_P (operands[1]) && MEM_P (operands[2]))"
   "@
@@ -8576,7 +8543,7 @@
   [(set (reg FLAGS_REG)
        (compare
          (and:SI (match_operand:SI 0 "nonimmediate_operand" "%!*a,r,rm")
-                 (match_operand:SI 1 "general_operand" "in,in,rin"))
+                 (match_operand:SI 1 "general_operand" "i,i,ri"))
          (const_int 0)))]
   "ix86_match_ccmode (insn, CCNOmode)
    && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
@@ -9077,7 +9044,7 @@
 (define_insn "*andhi_1"
   [(set (match_operand:HI 0 "nonimmediate_operand" "=rm,r,r")
        (and:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0,qm")
-               (match_operand:HI 2 "general_operand" "ri,rm,L")))
+               (match_operand:HI 2 "general_operand" "rn,rm,L")))
    (clobber (reg:CC FLAGS_REG))]
   "ix86_binary_operator_ok (AND, HImode, operands)"
 {
@@ -9101,7 +9068,7 @@
 (define_insn "*andhi_2"
   [(set (reg FLAGS_REG)
        (compare (and:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0")
-                        (match_operand:HI 2 "general_operand" "g,ri"))
+                        (match_operand:HI 2 "general_operand" "rmn,rn"))
                 (const_int 0)))
    (set (match_operand:HI 0 "nonimmediate_operand" "=r,rm")
        (and:HI (match_dup 1) (match_dup 2)))]
@@ -9123,7 +9090,7 @@
 (define_insn "*andqi_1"
   [(set (match_operand:QI 0 "nonimmediate_operand" "=qm,q,r")
        (and:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0,0")
-               (match_operand:QI 2 "general_operand" "qi,qmi,ri")))
+               (match_operand:QI 2 "general_operand" "qn,qmn,rn")))
    (clobber (reg:CC FLAGS_REG))]
   "ix86_binary_operator_ok (AND, QImode, operands)"
   "@
@@ -9136,7 +9103,7 @@
 (define_insn "*andqi_1_slp"
   [(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+qm,q"))
        (and:QI (match_dup 0)
-               (match_operand:QI 1 "general_operand" "qi,qmi")))
+               (match_operand:QI 1 "general_operand" "qn,qmn")))
    (clobber (reg:CC FLAGS_REG))]
   "(! TARGET_PARTIAL_REG_STALL || optimize_size)
    && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
@@ -9148,7 +9115,7 @@
   [(set (reg FLAGS_REG)
        (compare (and:QI
                      (match_operand:QI 1 "nonimmediate_operand" "%0,0,0")
-                     (match_operand:QI 2 "general_operand" "qim,qi,i"))
+                     (match_operand:QI 2 "general_operand" "qmn,qn,n"))
                 (const_int 0)))
    (set (match_operand:QI 0 "nonimmediate_operand" "=q,qm,*r")
        (and:QI (match_dup 1) (match_dup 2)))]
@@ -9172,7 +9139,7 @@
   [(set (reg FLAGS_REG)
        (compare (and:QI
                   (match_operand:QI 1 "nonimmediate_operand" "%0,0")
-                  (match_operand:QI 2 "general_operand" "qim,qi"))
+                  (match_operand:QI 2 "general_operand" "qmn,qn"))
                 (const_int 0)))
    (set (match_operand:QI 0 "nonimmediate_operand" "=q,qm")
        (and:QI (match_dup 1) (match_dup 2)))]
@@ -9186,7 +9153,7 @@
   [(set (reg FLAGS_REG)
        (compare (and:QI
                   (match_operand:QI 0 "nonimmediate_operand" "+q,qm")
-                  (match_operand:QI 1 "nonimmediate_operand" "qmi,qi"))
+                  (match_operand:QI 1 "nonimmediate_operand" "qmn,qn"))
                 (const_int 0)))
    (set (strict_low_part (match_dup 0))
        (and:QI (match_dup 0) (match_dup 1)))]
@@ -9503,7 +9470,7 @@
 (define_insn "*iorhi_1"
   [(set (match_operand:HI 0 "nonimmediate_operand" "=r,m")
        (ior:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0")
-               (match_operand:HI 2 "general_operand" "g,ri")))
+               (match_operand:HI 2 "general_operand" "rmn,rn")))
    (clobber (reg:CC FLAGS_REG))]
   "ix86_binary_operator_ok (IOR, HImode, operands)"
   "or{w}\t{%2, %0|%0, %2}"
@@ -9513,7 +9480,7 @@
 (define_insn "*iorhi_2"
   [(set (reg FLAGS_REG)
        (compare (ior:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0")
-                        (match_operand:HI 2 "general_operand" "g,ri"))
+                        (match_operand:HI 2 "general_operand" "rmn,rn"))
                 (const_int 0)))
    (set (match_operand:HI 0 "nonimmediate_operand" "=r,rm")
        (ior:HI (match_dup 1) (match_dup 2)))]
@@ -9526,7 +9493,7 @@
 (define_insn "*iorhi_3"
   [(set (reg FLAGS_REG)
        (compare (ior:HI (match_operand:HI 1 "nonimmediate_operand" "%0")
-                        (match_operand:HI 2 "general_operand" "g"))
+                        (match_operand:HI 2 "general_operand" "rmn"))
                 (const_int 0)))
    (clobber (match_scratch:HI 0 "=r"))]
   "ix86_match_ccmode (insn, CCNOmode)
@@ -9547,7 +9514,7 @@
 (define_insn "*iorqi_1"
   [(set (match_operand:QI 0 "nonimmediate_operand" "=q,m,r")
        (ior:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0,0")
-               (match_operand:QI 2 "general_operand" "qmi,qi,ri")))
+               (match_operand:QI 2 "general_operand" "qmn,qn,rn")))
    (clobber (reg:CC FLAGS_REG))]
   "ix86_binary_operator_ok (IOR, QImode, operands)"
   "@
@@ -9560,7 +9527,7 @@
 (define_insn "*iorqi_1_slp"
   [(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+q,m"))
        (ior:QI (match_dup 0)
-               (match_operand:QI 1 "general_operand" "qmi,qi")))
+               (match_operand:QI 1 "general_operand" "qmn,qn")))
    (clobber (reg:CC FLAGS_REG))]
   "(! TARGET_PARTIAL_REG_STALL || optimize_size)
    && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
@@ -9571,7 +9538,7 @@
 (define_insn "*iorqi_2"
   [(set (reg FLAGS_REG)
        (compare (ior:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0")
-                        (match_operand:QI 2 "general_operand" "qim,qi"))
+                        (match_operand:QI 2 "general_operand" "qmn,qn"))
                 (const_int 0)))
    (set (match_operand:QI 0 "nonimmediate_operand" "=q,qm")
        (ior:QI (match_dup 1) (match_dup 2)))]
@@ -9584,7 +9551,7 @@
 (define_insn "*iorqi_2_slp"
   [(set (reg FLAGS_REG)
        (compare (ior:QI (match_operand:QI 0 "nonimmediate_operand" "+q,qm")
-                        (match_operand:QI 1 "general_operand" "qim,qi"))
+                        (match_operand:QI 1 "general_operand" "qmn,qn"))
                 (const_int 0)))
    (set (strict_low_part (match_dup 0))
        (ior:QI (match_dup 0) (match_dup 1)))]
@@ -9598,7 +9565,7 @@
 (define_insn "*iorqi_3"
   [(set (reg FLAGS_REG)
        (compare (ior:QI (match_operand:QI 1 "nonimmediate_operand" "%0")
-                        (match_operand:QI 2 "general_operand" "qim"))
+                        (match_operand:QI 2 "general_operand" "qmn"))
                 (const_int 0)))
    (clobber (match_scratch:QI 0 "=q"))]
   "ix86_match_ccmode (insn, CCNOmode)
@@ -9876,7 +9843,7 @@
 (define_insn "*xorhi_1"
   [(set (match_operand:HI 0 "nonimmediate_operand" "=r,m")
        (xor:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0")
-               (match_operand:HI 2 "general_operand" "g,ri")))
+               (match_operand:HI 2 "general_operand" "rmn,rn")))
    (clobber (reg:CC FLAGS_REG))]
   "ix86_binary_operator_ok (XOR, HImode, operands)"
   "xor{w}\t{%2, %0|%0, %2}"
@@ -9886,7 +9853,7 @@
 (define_insn "*xorhi_2"
   [(set (reg FLAGS_REG)
        (compare (xor:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0")
-                        (match_operand:HI 2 "general_operand" "g,ri"))
+                        (match_operand:HI 2 "general_operand" "rmn,rn"))
                 (const_int 0)))
    (set (match_operand:HI 0 "nonimmediate_operand" "=r,rm")
        (xor:HI (match_dup 1) (match_dup 2)))]
@@ -9899,7 +9866,7 @@
 (define_insn "*xorhi_3"
   [(set (reg FLAGS_REG)
        (compare (xor:HI (match_operand:HI 1 "nonimmediate_operand" "%0")
-                        (match_operand:HI 2 "general_operand" "g"))
+                        (match_operand:HI 2 "general_operand" "rmn"))
                 (const_int 0)))
    (clobber (match_scratch:HI 0 "=r"))]
   "ix86_match_ccmode (insn, CCNOmode)
@@ -9920,7 +9887,7 @@
 (define_insn "*xorqi_1"
   [(set (match_operand:QI 0 "nonimmediate_operand" "=q,m,r")
        (xor:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0,0")
-               (match_operand:QI 2 "general_operand" "qmi,qi,ri")))
+               (match_operand:QI 2 "general_operand" "qmn,qn,rn")))
    (clobber (reg:CC FLAGS_REG))]
   "ix86_binary_operator_ok (XOR, QImode, operands)"
   "@
@@ -9933,7 +9900,7 @@
 (define_insn "*xorqi_1_slp"
   [(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+qm,q"))
        (xor:QI (match_dup 0)
-               (match_operand:QI 1 "general_operand" "qi,qmi")))
+               (match_operand:QI 1 "general_operand" "qn,qmn")))
    (clobber (reg:CC FLAGS_REG))]
   "(! TARGET_PARTIAL_REG_STALL || optimize_size)
    && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
@@ -10018,7 +9985,7 @@
   [(set (reg FLAGS_REG)
        (compare
          (xor:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0")
-                 (match_operand:QI 2 "general_operand" "qim,qi"))
+                 (match_operand:QI 2 "general_operand" "qmn,qn"))
          (const_int 0)))
    (set (match_operand:QI 0 "nonimmediate_operand" "=q,qm")
        (xor:QI (match_dup 1) (match_dup 2)))]
@@ -10031,7 +9998,7 @@
 (define_insn "*xorqi_2_slp"
   [(set (reg FLAGS_REG)
        (compare (xor:QI (match_operand:QI 0 "nonimmediate_operand" "+q,qm")
-                        (match_operand:QI 1 "general_operand" "qim,qi"))
+                        (match_operand:QI 1 "general_operand" "qmn,qn"))
                 (const_int 0)))
    (set (strict_low_part (match_dup 0))
        (xor:QI (match_dup 0) (match_dup 1)))]
@@ -10046,7 +10013,7 @@
   [(set (reg FLAGS_REG)
        (compare
          (xor:QI (match_operand:QI 1 "nonimmediate_operand" "%0")
-                 (match_operand:QI 2 "general_operand" "qim"))
+                 (match_operand:QI 2 "general_operand" "qmn"))
          (const_int 0)))
    (clobber (match_scratch:QI 0 "=q"))]
   "ix86_match_ccmode (insn, CCNOmode)
@@ -10882,33 +10849,14 @@
 ;; than 31.
 
 (define_expand "ashlti3"
-  [(parallel [(set (match_operand:TI 0 "register_operand" "")
-                  (ashift:TI (match_operand:TI 1 "register_operand" "")
-                             (match_operand:QI 2 "nonmemory_operand" "")))
-             (clobber (reg:CC FLAGS_REG))])]
-  "TARGET_64BIT"
-{
-  if (! immediate_operand (operands[2], QImode))
-    {
-      emit_insn (gen_ashlti3_1 (operands[0], operands[1], operands[2]));
-      DONE;
-    }
-  ix86_expand_binary_operator (ASHIFT, TImode, operands);
-  DONE;
-})
-
-(define_insn "ashlti3_1"
-  [(set (match_operand:TI 0 "register_operand" "=r")
-       (ashift:TI (match_operand:TI 1 "register_operand" "0")
-                  (match_operand:QI 2 "register_operand" "c")))
-   (clobber (match_scratch:DI 3 "=&r"))
-   (clobber (reg:CC FLAGS_REG))]
+  [(set (match_operand:TI 0 "register_operand" "")
+       (ashift:TI (match_operand:TI 1 "reg_or_pm1_operand" "")
+                  (match_operand:QI 2 "nonmemory_operand" "")))]
   "TARGET_64BIT"
-  "#"
-  [(set_attr "type" "multi")])
+  "ix86_expand_binary_operator (ASHIFT, TImode, operands); DONE;")
 
-;; This pattern must be defined before *ashlti3_2 to prevent
-;; combine pass from converting sse2_ashlti3 to *ashlti3_2.
+;; This pattern must be defined before *ashlti3_1 to prevent
+;; combine pass from converting sse2_ashlti3 to *ashlti3_1.
 
 (define_insn "sse2_ashlti3"
   [(set (match_operand:TI 0 "register_operand" "=x")
@@ -10923,45 +10871,45 @@
    (set_attr "prefix_data16" "1")
    (set_attr "mode" "TI")])
 
-(define_insn "*ashlti3_2"
-  [(set (match_operand:TI 0 "register_operand" "=r")
-       (ashift:TI (match_operand:TI 1 "register_operand" "0")
-                  (match_operand:QI 2 "immediate_operand" "O")))
+(define_insn "*ashlti3_1"
+  [(set (match_operand:TI 0 "register_operand" "=&r,r")
+       (ashift:TI (match_operand:TI 1 "reg_or_pm1_operand" "n,0")
+                  (match_operand:QI 2 "nonmemory_operand" "Oc,Oc")))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_64BIT"
   "#"
   [(set_attr "type" "multi")])
 
-(define_split
-  [(set (match_operand:TI 0 "register_operand" "")
-       (ashift:TI (match_operand:TI 1 "nonmemory_operand" "")
-                  (match_operand:QI 2 "register_operand" "")))
-   (clobber (match_scratch:DI 3 ""))
-   (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT && reload_completed"
+(define_peephole2
+  [(match_scratch:DI 3 "r")
+   (parallel [(set (match_operand:TI 0 "register_operand" "")
+                  (ashift:TI (match_operand:TI 1 "nonmemory_operand" "")
+                             (match_operand:QI 2 "nonmemory_operand" "")))
+             (clobber (reg:CC FLAGS_REG))])
+   (match_dup 3)]
+  "TARGET_64BIT"
   [(const_int 0)]
   "ix86_split_ashl (operands, operands[3], TImode); DONE;")
 
 (define_split
   [(set (match_operand:TI 0 "register_operand" "")
-       (ashift:TI (match_operand:TI 1 "register_operand" "")
-                  (match_operand:QI 2 "immediate_operand" "")))
+       (ashift:TI (match_operand:TI 1 "nonmemory_operand" "")
+                  (match_operand:QI 2 "nonmemory_operand" "")))
    (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT && reload_completed"
+  "TARGET_64BIT && ((optimize > 0 && flag_peephole2)
+                   ? epilogue_completed : reload_completed)"
   [(const_int 0)]
   "ix86_split_ashl (operands, NULL_RTX, TImode); DONE;")
 
 (define_insn "x86_64_shld"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "+r*m,r*m")
+  [(set (match_operand:DI 0 "nonimmediate_operand" "+r*m")
         (ior:DI (ashift:DI (match_dup 0)
-                 (match_operand:QI 2 "nonmemory_operand" "J,c"))
-               (lshiftrt:DI (match_operand:DI 1 "register_operand" "r,r")
+                 (match_operand:QI 2 "nonmemory_operand" "Jc"))
+               (lshiftrt:DI (match_operand:DI 1 "register_operand" "r")
                  (minus:QI (const_int 64) (match_dup 2)))))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_64BIT"
-  "@
-   shld{q}\t{%2, %1, %0|%0, %1, %2}
-   shld{q}\t{%s2%1, %0|%0, %1, %2}"
+  "shld{q}\t{%s2%1, %0|%0, %1, %2}"
   [(set_attr "type" "ishift")
    (set_attr "prefix_0f" "1")
    (set_attr "mode" "DI")
@@ -11054,7 +11002,7 @@
   [(set (reg FLAGS_REG)
        (compare
          (ashift:DI (match_operand:DI 1 "nonimmediate_operand" "0")
-                    (match_operand:QI 2 "immediate_operand" "e"))
+                    (match_operand:QI 2 "const_1_to_63_operand" "J"))
          (const_int 0)))
    (set (match_operand:DI 0 "nonimmediate_operand" "=rm")
        (ashift:DI (match_dup 1) (match_dup 2)))]
@@ -11097,7 +11045,7 @@
   [(set (reg FLAGS_REG)
        (compare
          (ashift:DI (match_operand:DI 1 "nonimmediate_operand" "0")
-                    (match_operand:QI 2 "immediate_operand" "e"))
+                    (match_operand:QI 2 "const_1_to_63_operand" "J"))
          (const_int 0)))
    (clobber (match_scratch:DI 0 "=r"))]
   "TARGET_64BIT
@@ -11168,17 +11116,15 @@
   [(const_int 0)]
   "ix86_split_ashl (operands, NULL_RTX, DImode); DONE;")
 
-(define_insn "x86_shld_1"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "+r*m,r*m")
+(define_insn "x86_shld"
+  [(set (match_operand:SI 0 "nonimmediate_operand" "+r*m")
         (ior:SI (ashift:SI (match_dup 0)
-                 (match_operand:QI 2 "nonmemory_operand" "I,c"))
-               (lshiftrt:SI (match_operand:SI 1 "register_operand" "r,r")
+                 (match_operand:QI 2 "nonmemory_operand" "Ic"))
+               (lshiftrt:SI (match_operand:SI 1 "register_operand" "r")
                  (minus:QI (const_int 32) (match_dup 2)))))
    (clobber (reg:CC FLAGS_REG))]
   ""
-  "@
-   shld{l}\t{%2, %1, %0|%0, %1, %2}
-   shld{l}\t{%s2%1, %0|%0, %1, %2}"
+  "shld{l}\t{%s2%1, %0|%0, %1, %2}"
   [(set_attr "type" "ishift")
    (set_attr "prefix_0f" "1")
    (set_attr "mode" "SI")
@@ -11878,70 +11824,51 @@
 ;; See comment above `ashldi3' about how this works.
 
 (define_expand "ashrti3"
-  [(parallel [(set (match_operand:TI 0 "register_operand" "")
-                  (ashiftrt:TI (match_operand:TI 1 "register_operand" "")
-                               (match_operand:QI 2 "nonmemory_operand" "")))
-             (clobber (reg:CC FLAGS_REG))])]
+  [(set (match_operand:TI 0 "register_operand" "")
+       (ashiftrt:TI (match_operand:TI 1 "register_operand" "")
+                    (match_operand:QI 2 "nonmemory_operand" "")))]
   "TARGET_64BIT"
-{
-  if (! immediate_operand (operands[2], QImode))
-    {
-      emit_insn (gen_ashrti3_1 (operands[0], operands[1], operands[2]));
-      DONE;
-    }
-  ix86_expand_binary_operator (ASHIFTRT, TImode, operands);
-  DONE;
-})
+  "ix86_expand_binary_operator (ASHIFTRT, TImode, operands); DONE;")
 
-(define_insn "ashrti3_1"
+(define_insn "*ashrti3_1"
   [(set (match_operand:TI 0 "register_operand" "=r")
        (ashiftrt:TI (match_operand:TI 1 "register_operand" "0")
-                    (match_operand:QI 2 "register_operand" "c")))
-   (clobber (match_scratch:DI 3 "=&r"))
+                    (match_operand:QI 2 "nonmemory_operand" "Oc")))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_64BIT"
   "#"
   [(set_attr "type" "multi")])
 
-(define_insn "*ashrti3_2"
-  [(set (match_operand:TI 0 "register_operand" "=r")
-       (ashiftrt:TI (match_operand:TI 1 "register_operand" "0")
-                    (match_operand:QI 2 "immediate_operand" "O")))
-   (clobber (reg:CC FLAGS_REG))]
+(define_peephole2
+  [(match_scratch:DI 3 "r")
+   (parallel [(set (match_operand:TI 0 "register_operand" "")
+                  (ashiftrt:TI (match_operand:TI 1 "register_operand" "")
+                               (match_operand:QI 2 "nonmemory_operand" "")))
+             (clobber (reg:CC FLAGS_REG))])
+   (match_dup 3)]
   "TARGET_64BIT"
-  "#"
-  [(set_attr "type" "multi")])
-
-(define_split
-  [(set (match_operand:TI 0 "register_operand" "")
-       (ashiftrt:TI (match_operand:TI 1 "register_operand" "")
-                    (match_operand:QI 2 "register_operand" "")))
-   (clobber (match_scratch:DI 3 ""))
-   (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT && reload_completed"
   [(const_int 0)]
   "ix86_split_ashr (operands, operands[3], TImode); DONE;")
 
 (define_split
   [(set (match_operand:TI 0 "register_operand" "")
        (ashiftrt:TI (match_operand:TI 1 "register_operand" "")
-                    (match_operand:QI 2 "immediate_operand" "")))
+                    (match_operand:QI 2 "nonmemory_operand" "")))
    (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT && reload_completed"
+  "TARGET_64BIT && ((optimize > 0 && flag_peephole2)
+                   ? epilogue_completed : reload_completed)"
   [(const_int 0)]
   "ix86_split_ashr (operands, NULL_RTX, TImode); DONE;")
 
 (define_insn "x86_64_shrd"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "+r*m,r*m")
+  [(set (match_operand:DI 0 "nonimmediate_operand" "+r*m")
         (ior:DI (ashiftrt:DI (match_dup 0)
-                 (match_operand:QI 2 "nonmemory_operand" "J,c"))
-               (ashift:DI (match_operand:DI 1 "register_operand" "r,r")
+                 (match_operand:QI 2 "nonmemory_operand" "Jc"))
+               (ashift:DI (match_operand:DI 1 "register_operand" "r")
                  (minus:QI (const_int 64) (match_dup 2)))))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_64BIT"
-  "@
-   shrd{q}\t{%2, %1, %0|%0, %1, %2}
-   shrd{q}\t{%s2%1, %0|%0, %1, %2}"
+  "shrd{q}\t{%s2%1, %0|%0, %1, %2}"
   [(set_attr "type" "ishift")
    (set_attr "prefix_0f" "1")
    (set_attr "mode" "DI")
@@ -12043,7 +11970,7 @@
   [(set (reg FLAGS_REG)
        (compare
          (ashiftrt:DI (match_operand:DI 1 "nonimmediate_operand" "0")
-                      (match_operand:QI 2 "const_int_operand" "n"))
+                      (match_operand:QI 2 "const_1_to_63_operand" "J"))
          (const_int 0)))
    (set (match_operand:DI 0 "nonimmediate_operand" "=rm")
        (ashiftrt:DI (match_dup 1) (match_dup 2)))]
@@ -12059,7 +11986,7 @@
   [(set (reg FLAGS_REG)
        (compare
          (ashiftrt:DI (match_operand:DI 1 "nonimmediate_operand" "0")
-                      (match_operand:QI 2 "const_int_operand" "n"))
+                      (match_operand:QI 2 "const_1_to_63_operand" "J"))
          (const_int 0)))
    (clobber (match_scratch:DI 0 "=r"))]
   "TARGET_64BIT
@@ -12103,17 +12030,15 @@
   [(const_int 0)]
   "ix86_split_ashr (operands, NULL_RTX, DImode); DONE;")
 
-(define_insn "x86_shrd_1"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "+r*m,r*m")
+(define_insn "x86_shrd"
+  [(set (match_operand:SI 0 "nonimmediate_operand" "+r*m")
         (ior:SI (ashiftrt:SI (match_dup 0)
-                 (match_operand:QI 2 "nonmemory_operand" "I,c"))
-               (ashift:SI (match_operand:SI 1 "register_operand" "r,r")
+                 (match_operand:QI 2 "nonmemory_operand" "Ic"))
+               (ashift:SI (match_operand:SI 1 "register_operand" "r")
                  (minus:QI (const_int 32) (match_dup 2)))))
    (clobber (reg:CC FLAGS_REG))]
   ""
-  "@
-   shrd{l}\t{%2, %1, %0|%0, %1, %2}
-   shrd{l}\t{%s2%1, %0|%0, %1, %2}"
+  "shrd{l}\t{%s2%1, %0|%0, %1, %2}"
   [(set_attr "type" "ishift")
    (set_attr "prefix_0f" "1")
    (set_attr "pent_pair" "np")
@@ -12525,7 +12450,7 @@
   [(set (reg FLAGS_REG)
        (compare
          (ashiftrt:QI (match_operand:QI 1 "nonimmediate_operand" "0")
-                      (match_operand:QI 2 "const1_operand" "I"))
+                      (match_operand:QI 2 "const1_operand" ""))
          (const_int 0)))
    (clobber (match_scratch:QI 0 "=q"))]
   "(TARGET_SHIFT1 || optimize_size)
@@ -12573,33 +12498,14 @@
 ;; See comment above `ashldi3' about how this works.
 
 (define_expand "lshrti3"
-  [(parallel [(set (match_operand:TI 0 "register_operand" "")
-                  (lshiftrt:TI (match_operand:TI 1 "register_operand" "")
-                               (match_operand:QI 2 "nonmemory_operand" "")))
-             (clobber (reg:CC FLAGS_REG))])]
-  "TARGET_64BIT"
-{
-  if (! immediate_operand (operands[2], QImode))
-    {
-      emit_insn (gen_lshrti3_1 (operands[0], operands[1], operands[2]));
-      DONE;
-    }
-  ix86_expand_binary_operator (LSHIFTRT, TImode, operands);
-  DONE;
-})
-
-(define_insn "lshrti3_1"
-  [(set (match_operand:TI 0 "register_operand" "=r")
-       (lshiftrt:TI (match_operand:TI 1 "register_operand" "0")
-                    (match_operand:QI 2 "register_operand" "c")))
-   (clobber (match_scratch:DI 3 "=&r"))
-   (clobber (reg:CC FLAGS_REG))]
+  [(set (match_operand:TI 0 "register_operand" "")
+       (lshiftrt:TI (match_operand:TI 1 "register_operand" "")
+                    (match_operand:QI 2 "nonmemory_operand" "")))]
   "TARGET_64BIT"
-  "#"
-  [(set_attr "type" "multi")])
+  "ix86_expand_binary_operator (LSHIFTRT, TImode, operands); DONE;")
 
-;; This pattern must be defined before *lshrti3_2 to prevent
-;; combine pass from converting sse2_lshrti3 to *lshrti3_2.
+;; This pattern must be defined before *lshrti3_1 to prevent
+;; combine pass from converting sse2_lshrti3 to *lshrti3_1.
 
 (define_insn "sse2_lshrti3"
   [(set (match_operand:TI 0 "register_operand" "=x")
@@ -12614,31 +12520,33 @@
    (set_attr "prefix_data16" "1")
    (set_attr "mode" "TI")])
 
-(define_insn "*lshrti3_2"
+(define_insn "*lshrti3_1"
   [(set (match_operand:TI 0 "register_operand" "=r")
        (lshiftrt:TI (match_operand:TI 1 "register_operand" "0")
-                    (match_operand:QI 2 "immediate_operand" "O")))
+                    (match_operand:QI 2 "nonmemory_operand" "Oc")))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_64BIT"
   "#"
   [(set_attr "type" "multi")])
 
-(define_split
-  [(set (match_operand:TI 0 "register_operand" "")
-       (lshiftrt:TI (match_operand:TI 1 "register_operand" "")
-                    (match_operand:QI 2 "register_operand" "")))
-   (clobber (match_scratch:DI 3 ""))
-   (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT && reload_completed"
+(define_peephole2
+  [(match_scratch:DI 3 "r")
+   (parallel [(set (match_operand:TI 0 "register_operand" "")
+                  (lshiftrt:TI (match_operand:TI 1 "register_operand" "")
+                               (match_operand:QI 2 "nonmemory_operand" "")))
+             (clobber (reg:CC FLAGS_REG))])
+   (match_dup 3)]
+  "TARGET_64BIT"
   [(const_int 0)]
   "ix86_split_lshr (operands, operands[3], TImode); DONE;")
 
 (define_split
   [(set (match_operand:TI 0 "register_operand" "")
        (lshiftrt:TI (match_operand:TI 1 "register_operand" "")
-                    (match_operand:QI 2 "immediate_operand" "")))
+                    (match_operand:QI 2 "nonmemory_operand" "")))
    (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT && reload_completed"
+  "!TARGET_64BIT && ((optimize > 0 && flag_peephole2)
+                    ? epilogue_completed : reload_completed)"
   [(const_int 0)]
   "ix86_split_lshr (operands, NULL_RTX, TImode); DONE;")
 
@@ -12720,7 +12628,7 @@
   [(set (reg FLAGS_REG)
        (compare
          (lshiftrt:DI (match_operand:DI 1 "nonimmediate_operand" "0")
-                      (match_operand:QI 2 "const_int_operand" "e"))
+                      (match_operand:QI 2 "const_1_to_63_operand" "J"))
          (const_int 0)))
    (set (match_operand:DI 0 "nonimmediate_operand" "=rm")
        (lshiftrt:DI (match_dup 1) (match_dup 2)))]
@@ -12736,7 +12644,7 @@
   [(set (reg FLAGS_REG)
        (compare
          (lshiftrt:DI (match_operand:DI 1 "nonimmediate_operand" "0")
-                      (match_operand:QI 2 "const_int_operand" "e"))
+                      (match_operand:QI 2 "const_1_to_63_operand" "J"))
          (const_int 0)))
    (clobber (match_scratch:DI 0 "=r"))]
   "TARGET_64BIT
@@ -13724,7 +13632,7 @@
        (const_int 1))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_64BIT && (TARGET_USE_BT || reload_completed)"
-  "bts{q} %1,%0"
+  "bts{q}\t{%1, %0|%0, %1}"
   [(set_attr "type" "alu1")])
 
 (define_insn "*btrq"
@@ -13734,7 +13642,7 @@
        (const_int 0))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_64BIT && (TARGET_USE_BT || reload_completed)"
-  "btr{q} %1,%0"
+  "btr{q}\t{%1, %0|%0, %1}"
   [(set_attr "type" "alu1")])
 
 (define_insn "*btcq"
@@ -13744,7 +13652,7 @@
        (not:DI (zero_extract:DI (match_dup 0) (const_int 1) (match_dup 1))))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_64BIT && (TARGET_USE_BT || reload_completed)"
-  "btc{q} %1,%0"
+  "btc{q}\t{%1, %0|%0, %1}"
   [(set_attr "type" "alu1")])
 
 ;; Allow Nocona to avoid these instructions if a register is available.
@@ -13845,6 +13753,30 @@
   emit_insn (gen_xordi3 (operands[0], operands[0], op1));
   DONE;
 })
+
+(define_insn "*btdi_rex64"
+  [(set (reg:CCC FLAGS_REG)
+       (compare:CCC
+         (zero_extract:DI
+           (match_operand:DI 0 "register_operand" "r")
+           (const_int 1)
+           (match_operand:DI 1 "nonmemory_operand" "rN"))
+         (const_int 0)))]
+  "TARGET_64BIT && (TARGET_USE_BT || optimize_size)"
+  "bt{q}\t{%1, %0|%0, %1}"
+  [(set_attr "type" "alu1")])
+
+(define_insn "*btsi"
+  [(set (reg:CCC FLAGS_REG)
+       (compare:CCC
+         (zero_extract:SI
+           (match_operand:SI 0 "register_operand" "r")
+           (const_int 1)
+           (match_operand:SI 1 "nonmemory_operand" "rN"))
+         (const_int 0)))]
+  "TARGET_USE_BT || optimize_size"
+  "bt{l}\t{%1, %0|%0, %1}"
+  [(set_attr "type" "alu1")])
 \f
 ;; Store-flag instructions.
 
@@ -14041,6 +13973,76 @@
             (const_int 2)
             (const_int 6)))])
 
+;; ??? Handle alignment requirements for compare and branch fused macro-op;
+;; the branch instruction does not start at a 16-byte boundary or cross
+;; a 16-byte boundary.
+
+(define_insn "*jcc_fused_1"
+  [(set (pc)
+       (if_then_else (match_operator 1 "comparison_operator"
+                       [(match_operand:SWI 2 "register_operand" "<r>")
+                        (match_operand:SWI 3 "const0_operand" "")])
+        (label_ref (match_operand 0 "" ""))
+        (pc)))]
+  "TARGET_FUSE_CMP_AND_BRANCH && !TARGET_64BIT"
+{
+  return "test{<imodesuffix>}\t%2, %2\n\t"
+        "%+j%E1\t%l0\t" ASM_COMMENT_START " fused";
+}
+  [(set_attr "type" "multi")
+   (set_attr "mode" "<MODE>")])
+
+(define_insn "*jcc_fused_2"
+  [(set (pc)
+       (if_then_else (match_operator 1 "comparison_operator"
+                       [(match_operand:SWI 2 "register_operand" "<r>")
+                        (match_operand:SWI 3 "const0_operand" "")])
+        (pc)
+        (label_ref (match_operand 0 "" ""))))]
+  "TARGET_FUSE_CMP_AND_BRANCH && !TARGET_64BIT"
+{
+  return "test{<imodesuffix>}\t%2, %2\n\t"
+        "%+j%e1\t%l0\t" ASM_COMMENT_START " fused";
+}
+  [(set_attr "type" "multi")
+   (set_attr "mode" "<MODE>")])
+
+(define_insn "*jcc_fused_3"
+  [(set (pc)
+       (if_then_else
+         (match_operator 1 "ix86_comparison_uns_operator"
+           [(match_operand:SWI 2 "nonimmediate_operand" "<r>,m,<r>")
+            (match_operand:SWI 3 "<general_operand>" "<r><i>,<r>,m")])
+        (label_ref (match_operand 0 "" ""))
+        (pc)))]
+  "TARGET_FUSE_CMP_AND_BRANCH && !TARGET_64BIT
+   && !(MEM_P (operands[2])
+       && (MEM_P (operands[3]) || CONST_INT_P (operands[3])))"
+{
+  return "cmp{<imodesuffix>}\t{%3, %2|%2, %3}\n\t"
+        "%+j%E1\t%l0\t" ASM_COMMENT_START " fused";
+}
+  [(set_attr "type" "multi")
+   (set_attr "mode" "<MODE>")])
+
+(define_insn "*jcc_fused_4"
+  [(set (pc)
+       (if_then_else
+         (match_operator 1 "ix86_comparison_uns_operator"
+           [(match_operand:SWI 2 "nonimmediate_operand" "<r>,m,<r>")
+            (match_operand:SWI 3 "<general_operand>" "<r><i>,<r>,m")])
+        (pc)
+        (label_ref (match_operand 0 "" ""))))]
+  "TARGET_FUSE_CMP_AND_BRANCH && !TARGET_64BIT
+   && !(MEM_P (operands[2])
+       && (MEM_P (operands[3]) || CONST_INT_P (operands[3])))"
+{
+  return "cmp{<imodesuffix>}\t{%3, %2|%2, %3}\n\t"
+        "%+j%e1\t%l0\t" ASM_COMMENT_START " fused";
+}
+  [(set_attr "type" "multi")
+   (set_attr "mode" "<MODE>")])
+
 ;; In general it is not safe to assume too much about CCmode registers,
 ;; so simplify-rtx stops when it sees a second one.  Under certain
 ;; conditions this is safe on x86, so help combine not create
@@ -14090,6 +14092,199 @@
     FAIL;
 })
 
+;; zero_extend in SImode is correct, since this is what combine pass
+;; generates from shift insn with QImode operand.  Actually, the mode of
+;; operand 2 (bit offset operand) doesn't matter since bt insn takes
+;; appropriate modulo of the bit offset value.
+
+(define_insn_and_split "*jcc_btdi_rex64"
+  [(set (pc)
+       (if_then_else (match_operator 0 "bt_comparison_operator"
+                       [(zero_extract:DI
+                          (match_operand:DI 1 "register_operand" "r")
+                          (const_int 1)
+                          (zero_extend:SI
+                            (match_operand:QI 2 "register_operand" "r")))
+                        (const_int 0)])
+                     (label_ref (match_operand 3 "" ""))
+                     (pc)))]
+  "TARGET_64BIT && (TARGET_USE_BT || optimize_size)"
+  "#"
+  "&& 1"
+  [(set (reg:CCC FLAGS_REG)
+       (compare:CCC
+         (zero_extract:DI
+           (match_dup 1)
+           (const_int 1)
+           (match_dup 2))
+         (const_int 0)))
+   (set (pc)
+       (if_then_else (match_op_dup 0 [(reg:CCC FLAGS_REG) (const_int 0)])
+                     (label_ref (match_dup 3))
+                     (pc)))]
+{
+  operands[2] = simplify_gen_subreg (DImode, operands[2], QImode, 0);
+
+  PUT_CODE (operands[0], reverse_condition (GET_CODE (operands[0])));
+})
+
+;; avoid useless masking of bit offset operand
+(define_insn_and_split "*jcc_btdi_mask_rex64"
+  [(set (pc)
+       (if_then_else (match_operator 0 "bt_comparison_operator"
+                       [(zero_extract:DI
+                          (match_operand:DI 1 "register_operand" "r")
+                          (const_int 1)
+                          (and:SI
+                            (match_operand:SI 2 "register_operand" "r")
+                            (match_operand:SI 3 "const_int_operand" "n")))])
+                     (label_ref (match_operand 4 "" ""))
+                     (pc)))]
+  "TARGET_64BIT && (TARGET_USE_BT || optimize_size)
+   && (INTVAL (operands[3]) & 0x3f) == 0x3f"
+  "#"
+  "&& 1"
+  [(set (reg:CCC FLAGS_REG)
+       (compare:CCC
+         (zero_extract:DI
+           (match_dup 1)
+           (const_int 1)
+           (match_dup 2))
+         (const_int 0)))
+   (set (pc)
+       (if_then_else (match_op_dup 0 [(reg:CCC FLAGS_REG) (const_int 0)])
+                     (label_ref (match_dup 4))
+                     (pc)))]
+{
+  operands[2] = simplify_gen_subreg (DImode, operands[2], SImode, 0);
+
+  PUT_CODE (operands[0], reverse_condition (GET_CODE (operands[0])));
+})
+
+(define_insn_and_split "*jcc_btsi"
+  [(set (pc)
+       (if_then_else (match_operator 0 "bt_comparison_operator"
+                       [(zero_extract:SI
+                          (match_operand:SI 1 "register_operand" "r")
+                          (const_int 1)
+                          (zero_extend:SI
+                            (match_operand:QI 2 "register_operand" "r")))
+                        (const_int 0)])
+                     (label_ref (match_operand 3 "" ""))
+                     (pc)))]
+  "TARGET_USE_BT || optimize_size"
+  "#"
+  "&& 1"
+  [(set (reg:CCC FLAGS_REG)
+       (compare:CCC
+         (zero_extract:SI
+           (match_dup 1)
+           (const_int 1)
+           (match_dup 2))
+         (const_int 0)))
+   (set (pc)
+       (if_then_else (match_op_dup 0 [(reg:CCC FLAGS_REG) (const_int 0)])
+                     (label_ref (match_dup 3))
+                     (pc)))]
+{
+  operands[2] = simplify_gen_subreg (SImode, operands[2], QImode, 0);
+
+  PUT_CODE (operands[0], reverse_condition (GET_CODE (operands[0])));
+})
+
+;; avoid useless masking of bit offset operand
+(define_insn_and_split "*jcc_btsi_mask"
+  [(set (pc)
+       (if_then_else (match_operator 0 "bt_comparison_operator"
+                       [(zero_extract:SI
+                          (match_operand:SI 1 "register_operand" "r")
+                          (const_int 1)
+                          (and:SI
+                            (match_operand:SI 2 "register_operand" "r")
+                            (match_operand:SI 3 "const_int_operand" "n")))])
+                     (label_ref (match_operand 4 "" ""))
+                     (pc)))]
+  "(TARGET_USE_BT || optimize_size)
+   && (INTVAL (operands[3]) & 0x1f) == 0x1f"
+  "#"
+  "&& 1"
+  [(set (reg:CCC FLAGS_REG)
+       (compare:CCC
+         (zero_extract:SI
+           (match_dup 1)
+           (const_int 1)
+           (match_dup 2))
+         (const_int 0)))
+   (set (pc)
+       (if_then_else (match_op_dup 0 [(reg:CCC FLAGS_REG) (const_int 0)])
+                     (label_ref (match_dup 4))
+                     (pc)))]
+  "PUT_CODE (operands[0], reverse_condition (GET_CODE (operands[0])));")
+
+(define_insn_and_split "*jcc_btsi_1"
+  [(set (pc)
+       (if_then_else (match_operator 0 "bt_comparison_operator"
+                       [(and:SI
+                          (lshiftrt:SI
+                            (match_operand:SI 1 "register_operand" "r")
+                            (match_operand:QI 2 "register_operand" "r"))
+                          (const_int 1))
+                        (const_int 0)])
+                     (label_ref (match_operand 3 "" ""))
+                     (pc)))]
+  "TARGET_USE_BT || optimize_size"
+  "#"
+  "&& 1"
+  [(set (reg:CCC FLAGS_REG)
+       (compare:CCC
+         (zero_extract:SI
+           (match_dup 1)
+           (const_int 1)
+           (match_dup 2))
+         (const_int 0)))
+   (set (pc)
+       (if_then_else (match_op_dup 0 [(reg:CCC FLAGS_REG) (const_int 0)])
+                     (label_ref (match_dup 3))
+                     (pc)))]
+{
+  operands[2] = simplify_gen_subreg (SImode, operands[2], QImode, 0);
+
+  PUT_CODE (operands[0], reverse_condition (GET_CODE (operands[0])));
+})
+
+;; avoid useless masking of bit offset operand
+(define_insn_and_split "*jcc_btsi_mask_1"
+  [(set (pc)
+       (if_then_else
+         (match_operator 0 "bt_comparison_operator"
+           [(and:SI
+              (lshiftrt:SI
+                (match_operand:SI 1 "register_operand" "r")
+                (subreg:QI
+                  (and:SI
+                    (match_operand:SI 2 "register_operand" "r")
+                    (match_operand:SI 3 "const_int_operand" "n")) 0))
+              (const_int 1))
+            (const_int 0)])
+         (label_ref (match_operand 4 "" ""))
+         (pc)))]
+  "(TARGET_USE_BT || optimize_size)
+   && (INTVAL (operands[3]) & 0x1f) == 0x1f"
+  "#"
+  "&& 1"
+  [(set (reg:CCC FLAGS_REG)
+       (compare:CCC
+         (zero_extract:SI
+           (match_dup 1)
+           (const_int 1)
+           (match_dup 2))
+         (const_int 0)))
+   (set (pc)
+       (if_then_else (match_op_dup 0 [(reg:CCC FLAGS_REG) (const_int 0)])
+                     (label_ref (match_dup 4))
+                     (pc)))]
+  "PUT_CODE (operands[0], reverse_condition (GET_CODE (operands[0])));")
+
 ;; Define combination compare-and-branch fp compare instructions to use
 ;; during early optimization.  Splitting the operation apart early makes
 ;; for bad code when we want to reverse the operation.
@@ -14256,7 +14451,7 @@
   [(set (pc)
        (if_then_else (match_operator 0 "comparison_operator"
                        [(match_operand 1 "register_operand" "f")
-                        (match_operand 2 "const0_operand" "X")])
+                        (match_operand 2 "const0_operand" "")])
          (label_ref (match_operand 3 "" ""))
          (pc)))
    (clobber (reg:CCFP FPSR_REG))
@@ -14287,7 +14482,7 @@
    (clobber (reg:CCFP FLAGS_REG))
    (clobber (match_scratch:HI 5 "=a,a"))]
   "X87_FLOAT_MODE_P (GET_MODE (operands[3]))
-   && TARGET_USE_<MODE>MODE_FIOP
+   && (TARGET_USE_<MODE>MODE_FIOP || optimize_size)
    && GET_MODE (operands[1]) == GET_MODE (operands[3])
    && !ix86_use_fcomi_compare (swap_condition (GET_CODE (operands[0])))
    && ix86_fp_compare_mode (swap_condition (GET_CODE (operands[0]))) == CCFPmode
@@ -14396,15 +14591,8 @@
   "")
 
 (define_insn "*indirect_jump"
-  [(set (pc) (match_operand:SI 0 "nonimmediate_operand" "rm"))]
-  "!TARGET_64BIT"
-  "jmp\t%A0"
-  [(set_attr "type" "ibr")
-   (set_attr "length_immediate" "0")])
-
-(define_insn "*indirect_jump_rtx64"
-  [(set (pc) (match_operand:DI 0 "nonimmediate_operand" "rm"))]
-  "TARGET_64BIT"
+  [(set (pc) (match_operand:P 0 "nonimmediate_operand" "rm"))]
+  ""
   "jmp\t%A0"
   [(set_attr "type" "ibr")
    (set_attr "length_immediate" "0")])
@@ -14448,17 +14636,9 @@
 })
 
 (define_insn "*tablejump_1"
-  [(set (pc) (match_operand:SI 0 "nonimmediate_operand" "rm"))
+  [(set (pc) (match_operand:P 0 "nonimmediate_operand" "rm"))
    (use (label_ref (match_operand 1 "" "")))]
-  "!TARGET_64BIT"
-  "jmp\t%A0"
-  [(set_attr "type" "ibr")
-   (set_attr "length_immediate" "0")])
-
-(define_insn "*tablejump_1_rtx64"
-  [(set (pc) (match_operand:DI 0 "nonimmediate_operand" "rm"))
-   (use (label_ref (match_operand 1 "" "")))]
-  "TARGET_64BIT"
+  ""
   "jmp\t%A0"
   [(set_attr "type" "ibr")
    (set_attr "length_immediate" "0")])
@@ -14707,7 +14887,10 @@
 
   ix86_expand_call ((TARGET_FLOAT_RETURNS_IN_80387
                     ? gen_rtx_REG (XCmode, FIRST_FLOAT_REG) : NULL),
-                   operands[0], const0_rtx, GEN_INT (SSE_REGPARM_MAX - 1),
+                   operands[0], const0_rtx,
+                   GEN_INT ((DEFAULT_ABI == SYSV_ABI ? X86_64_SSE_REGPARM_MAX
+                                                     : X64_SSE_REGPARM_MAX)
+                            - 1),
                    NULL, 0);
 
   for (i = 0; i < XVECLEN (operands[2], 0); i++)
@@ -14906,21 +15089,11 @@
   DONE;
 })
 
-(define_insn_and_split "eh_return_si"
-  [(set (pc)
-        (unspec [(match_operand:SI 0 "register_operand" "c")]
-                UNSPEC_EH_RETURN))]
-  "!TARGET_64BIT"
-  "#"
-  "reload_completed"
-  [(const_int 0)]
-  "ix86_expand_epilogue (2); DONE;")
-
-(define_insn_and_split "eh_return_di"
+(define_insn_and_split "eh_return_<mode>"
   [(set (pc)
-        (unspec [(match_operand:DI 0 "register_operand" "c")]
+        (unspec [(match_operand:P 0 "register_operand" "c")]
                 UNSPEC_EH_RETURN))]
-  "TARGET_64BIT"
+  ""
   "#"
   "reload_completed"
   [(const_int 0)]
@@ -15797,80 +15970,80 @@
 ;; Gcc is slightly more smart about handling normal two address instructions
 ;; so use special patterns for add and mull.
 
-(define_insn "*fop_sf_comm_mixed"
-  [(set (match_operand:SF 0 "register_operand" "=f,x")
-       (match_operator:SF 3 "binary_fp_operator"
-                       [(match_operand:SF 1 "nonimmediate_operand" "%0,0")
-                        (match_operand:SF 2 "nonimmediate_operand" "fm,xm")]))]
-  "TARGET_MIX_SSE_I387
+(define_insn "*fop_<mode>_comm_mixed"
+  [(set (match_operand:MODEF 0 "register_operand" "=f,x")
+       (match_operator:MODEF 3 "binary_fp_operator"
+         [(match_operand:MODEF 1 "nonimmediate_operand" "%0,0")
+          (match_operand:MODEF 2 "nonimmediate_operand" "fm,xm")]))]
+  "SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_MIX_SSE_I387
    && COMMUTATIVE_ARITH_P (operands[3])
    && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type")
        (if_then_else (eq_attr "alternative" "1")
-          (if_then_else (match_operand:SF 3 "mult_operator" "")
+          (if_then_else (match_operand:MODEF 3 "mult_operator" "")
              (const_string "ssemul")
              (const_string "sseadd"))
-          (if_then_else (match_operand:SF 3 "mult_operator" "")
+          (if_then_else (match_operand:MODEF 3 "mult_operator" "")
              (const_string "fmul")
              (const_string "fop"))))
-   (set_attr "mode" "SF")])
+   (set_attr "mode" "<MODE>")])
 
-(define_insn "*fop_sf_comm_sse"
-  [(set (match_operand:SF 0 "register_operand" "=x")
-       (match_operator:SF 3 "binary_fp_operator"
-                       [(match_operand:SF 1 "nonimmediate_operand" "%0")
-                        (match_operand:SF 2 "nonimmediate_operand" "xm")]))]
-  "TARGET_SSE_MATH
+(define_insn "*fop_<mode>_comm_sse"
+  [(set (match_operand:MODEF 0 "register_operand" "=x")
+       (match_operator:MODEF 3 "binary_fp_operator"
+         [(match_operand:MODEF 1 "nonimmediate_operand" "%0")
+          (match_operand:MODEF 2 "nonimmediate_operand" "xm")]))]
+  "SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH
    && COMMUTATIVE_ARITH_P (operands[3])
    && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type")
-        (if_then_else (match_operand:SF 3 "mult_operator" "")
+        (if_then_else (match_operand:MODEF 3 "mult_operator" "")
           (const_string "ssemul")
           (const_string "sseadd")))
-   (set_attr "mode" "SF")])
+   (set_attr "mode" "<MODE>")])
 
-(define_insn "*fop_sf_comm_i387"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-       (match_operator:SF 3 "binary_fp_operator"
-                       [(match_operand:SF 1 "nonimmediate_operand" "%0")
-                        (match_operand:SF 2 "nonimmediate_operand" "fm")]))]
+(define_insn "*fop_<mode>_comm_i387"
+  [(set (match_operand:MODEF 0 "register_operand" "=f")
+       (match_operator:MODEF 3 "binary_fp_operator"
+         [(match_operand:MODEF 1 "nonimmediate_operand" "%0")
+          (match_operand:MODEF 2 "nonimmediate_operand" "fm")]))]
   "TARGET_80387
    && COMMUTATIVE_ARITH_P (operands[3])
    && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type")
-       (if_then_else (match_operand:SF 3 "mult_operator" "")
+       (if_then_else (match_operand:MODEF 3 "mult_operator" "")
           (const_string "fmul")
           (const_string "fop")))
-   (set_attr "mode" "SF")])
+   (set_attr "mode" "<MODE>")])
 
-(define_insn "*fop_sf_1_mixed"
-  [(set (match_operand:SF 0 "register_operand" "=f,f,x")
-       (match_operator:SF 3 "binary_fp_operator"
-                       [(match_operand:SF 1 "nonimmediate_operand" "0,fm,0")
-                        (match_operand:SF 2 "nonimmediate_operand" "fm,0,xm")]))]
-  "TARGET_MIX_SSE_I387
+(define_insn "*fop_<mode>_1_mixed"
+  [(set (match_operand:MODEF 0 "register_operand" "=f,f,x")
+       (match_operator:MODEF 3 "binary_fp_operator"
+         [(match_operand:MODEF 1 "nonimmediate_operand" "0,fm,0")
+          (match_operand:MODEF 2 "nonimmediate_operand" "fm,0,xm")]))]
+  "SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_MIX_SSE_I387
    && !COMMUTATIVE_ARITH_P (operands[3])
    && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type")
         (cond [(and (eq_attr "alternative" "2")
-                   (match_operand:SF 3 "mult_operator" ""))
+                   (match_operand:MODEF 3 "mult_operator" ""))
                  (const_string "ssemul")
               (and (eq_attr "alternative" "2")
-                   (match_operand:SF 3 "div_operator" ""))
+                   (match_operand:MODEF 3 "div_operator" ""))
                  (const_string "ssediv")
               (eq_attr "alternative" "2")
                  (const_string "sseadd")
-              (match_operand:SF 3 "mult_operator" "")
+              (match_operand:MODEF 3 "mult_operator" "")
                  (const_string "fmul")
-               (match_operand:SF 3 "div_operator" "")
+               (match_operand:MODEF 3 "div_operator" "")
                  (const_string "fdiv")
               ]
               (const_string "fop")))
-   (set_attr "mode" "SF")])
+   (set_attr "mode" "<MODE>")])
 
 (define_insn "*rcpsf2_sse"
   [(set (match_operand:SF 0 "register_operand" "=x")
@@ -15881,219 +16054,75 @@
   [(set_attr "type" "sse")
    (set_attr "mode" "SF")])
 
-(define_insn "*fop_sf_1_sse"
-  [(set (match_operand:SF 0 "register_operand" "=x")
-       (match_operator:SF 3 "binary_fp_operator"
-                       [(match_operand:SF 1 "register_operand" "0")
-                        (match_operand:SF 2 "nonimmediate_operand" "xm")]))]
-  "TARGET_SSE_MATH
+(define_insn "*fop_<mode>_1_sse"
+  [(set (match_operand:MODEF 0 "register_operand" "=x")
+       (match_operator:MODEF 3 "binary_fp_operator"
+         [(match_operand:MODEF 1 "register_operand" "0")
+          (match_operand:MODEF 2 "nonimmediate_operand" "xm")]))]
+  "SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH
    && !COMMUTATIVE_ARITH_P (operands[3])"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type")
-        (cond [(match_operand:SF 3 "mult_operator" "")
+        (cond [(match_operand:MODEF 3 "mult_operator" "")
                  (const_string "ssemul")
-              (match_operand:SF 3 "div_operator" "")
+              (match_operand:MODEF 3 "div_operator" "")
                  (const_string "ssediv")
               ]
               (const_string "sseadd")))
-   (set_attr "mode" "SF")])
-
-;; This pattern is not fully shadowed by the pattern above.
-(define_insn "*fop_sf_1_i387"
-  [(set (match_operand:SF 0 "register_operand" "=f,f")
-       (match_operator:SF 3 "binary_fp_operator"
-                       [(match_operand:SF 1 "nonimmediate_operand" "0,fm")
-                        (match_operand:SF 2 "nonimmediate_operand" "fm,0")]))]
-  "TARGET_80387 && !TARGET_SSE_MATH
-   && !COMMUTATIVE_ARITH_P (operands[3])
-   && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
-  "* return output_387_binary_op (insn, operands);"
-  [(set (attr "type")
-        (cond [(match_operand:SF 3 "mult_operator" "")
-                 (const_string "fmul")
-               (match_operand:SF 3 "div_operator" "")
-                 (const_string "fdiv")
-              ]
-              (const_string "fop")))
-   (set_attr "mode" "SF")])
-
-;; ??? Add SSE splitters for these!
-(define_insn "*fop_sf_2<mode>_i387"
-  [(set (match_operand:SF 0 "register_operand" "=f,f")
-       (match_operator:SF 3 "binary_fp_operator"
-         [(float:SF (match_operand:X87MODEI12 1 "nonimmediate_operand" "m,?r"))
-          (match_operand:SF 2 "register_operand" "0,0")]))]
-  "TARGET_80387 && TARGET_USE_<MODE>MODE_FIOP && !TARGET_SSE_MATH"
-  "* return which_alternative ? \"#\" : output_387_binary_op (insn, operands);"
-  [(set (attr "type")
-        (cond [(match_operand:SF 3 "mult_operator" "")
-                 (const_string "fmul")
-               (match_operand:SF 3 "div_operator" "")
-                 (const_string "fdiv")
-              ]
-              (const_string "fop")))
-   (set_attr "fp_int_src" "true")
    (set_attr "mode" "<MODE>")])
 
-(define_insn "*fop_sf_3<mode>_i387"
-  [(set (match_operand:SF 0 "register_operand" "=f,f")
-       (match_operator:SF 3 "binary_fp_operator"
-         [(match_operand:SF 1 "register_operand" "0,0")
-          (float:SF (match_operand:X87MODEI12 2 "nonimmediate_operand" "m,?r"))]))]
-  "TARGET_80387 && TARGET_USE_<MODE>MODE_FIOP && !TARGET_SSE_MATH"
-  "* return which_alternative ? \"#\" : output_387_binary_op (insn, operands);"
-  [(set (attr "type")
-        (cond [(match_operand:SF 3 "mult_operator" "")
-                 (const_string "fmul")
-               (match_operand:SF 3 "div_operator" "")
-                 (const_string "fdiv")
-              ]
-              (const_string "fop")))
-   (set_attr "fp_int_src" "true")
-   (set_attr "mode" "<MODE>")])
-
-(define_insn "*fop_df_comm_mixed"
-  [(set (match_operand:DF 0 "register_operand" "=f,x")
-       (match_operator:DF 3 "binary_fp_operator"
-         [(match_operand:DF 1 "nonimmediate_operand" "%0,0")
-          (match_operand:DF 2 "nonimmediate_operand" "fm,xm")]))]
-  "TARGET_SSE2 && TARGET_MIX_SSE_I387
-   && COMMUTATIVE_ARITH_P (operands[3])
-   && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
-  "* return output_387_binary_op (insn, operands);"
-  [(set (attr "type")
-       (if_then_else (eq_attr "alternative" "1")
-          (if_then_else (match_operand:DF 3 "mult_operator" "")
-             (const_string "ssemul")
-             (const_string "sseadd"))
-          (if_then_else (match_operand:DF 3 "mult_operator" "")
-             (const_string "fmul")
-             (const_string "fop"))))
-   (set_attr "mode" "DF")])
-
-(define_insn "*fop_df_comm_sse"
-  [(set (match_operand:DF 0 "register_operand" "=x")
-       (match_operator:DF 3 "binary_fp_operator"
-         [(match_operand:DF 1 "nonimmediate_operand" "%0")
-          (match_operand:DF 2 "nonimmediate_operand" "xm")]))]
-  "TARGET_SSE2 && TARGET_SSE_MATH
-   && COMMUTATIVE_ARITH_P (operands[3])
-   && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
-  "* return output_387_binary_op (insn, operands);"
-  [(set (attr "type")
-        (if_then_else (match_operand:DF 3 "mult_operator" "")
-          (const_string "ssemul")
-          (const_string "sseadd")))
-   (set_attr "mode" "DF")])
-
-(define_insn "*fop_df_comm_i387"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-       (match_operator:DF 3 "binary_fp_operator"
-                       [(match_operand:DF 1 "nonimmediate_operand" "%0")
-                        (match_operand:DF 2 "nonimmediate_operand" "fm")]))]
-  "TARGET_80387
-   && COMMUTATIVE_ARITH_P (operands[3])
-   && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
-  "* return output_387_binary_op (insn, operands);"
-  [(set (attr "type")
-       (if_then_else (match_operand:DF 3 "mult_operator" "")
-          (const_string "fmul")
-          (const_string "fop")))
-   (set_attr "mode" "DF")])
-
-(define_insn "*fop_df_1_mixed"
-  [(set (match_operand:DF 0 "register_operand" "=f,f,x")
-       (match_operator:DF 3 "binary_fp_operator"
-         [(match_operand:DF 1 "nonimmediate_operand" "0,fm,0")
-          (match_operand:DF 2 "nonimmediate_operand" "fm,0,xm")]))]
-  "TARGET_SSE2 && TARGET_SSE_MATH && TARGET_MIX_SSE_I387
-   && !COMMUTATIVE_ARITH_P (operands[3])
-   && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
-  "* return output_387_binary_op (insn, operands);"
-  [(set (attr "type")
-        (cond [(and (eq_attr "alternative" "2")
-                   (match_operand:DF 3 "mult_operator" ""))
-                 (const_string "ssemul")
-              (and (eq_attr "alternative" "2")
-                   (match_operand:DF 3 "div_operator" ""))
-                 (const_string "ssediv")
-              (eq_attr "alternative" "2")
-                 (const_string "sseadd")
-              (match_operand:DF 3 "mult_operator" "")
-                 (const_string "fmul")
-               (match_operand:DF 3 "div_operator" "")
-                 (const_string "fdiv")
-              ]
-              (const_string "fop")))
-   (set_attr "mode" "DF")])
-
-(define_insn "*fop_df_1_sse"
-  [(set (match_operand:DF 0 "register_operand" "=x")
-       (match_operator:DF 3 "binary_fp_operator"
-         [(match_operand:DF 1 "register_operand" "0")
-          (match_operand:DF 2 "nonimmediate_operand" "xm")]))]
-  "TARGET_SSE2 && TARGET_SSE_MATH
-   && !COMMUTATIVE_ARITH_P (operands[3])"
-  "* return output_387_binary_op (insn, operands);"
-  [(set_attr "mode" "DF")
-   (set (attr "type")
-        (cond [(match_operand:DF 3 "mult_operator" "")
-                 (const_string "ssemul")
-              (match_operand:DF 3 "div_operator" "")
-                 (const_string "ssediv")
-              ]
-              (const_string "sseadd")))])
-
 ;; This pattern is not fully shadowed by the pattern above.
-(define_insn "*fop_df_1_i387"
-  [(set (match_operand:DF 0 "register_operand" "=f,f")
-       (match_operator:DF 3 "binary_fp_operator"
-                       [(match_operand:DF 1 "nonimmediate_operand" "0,fm")
-                        (match_operand:DF 2 "nonimmediate_operand" "fm,0")]))]
-  "TARGET_80387 && !(TARGET_SSE2 && TARGET_SSE_MATH)
+(define_insn "*fop_<mode>_1_i387"
+  [(set (match_operand:MODEF 0 "register_operand" "=f,f")
+       (match_operator:MODEF 3 "binary_fp_operator"
+         [(match_operand:MODEF 1 "nonimmediate_operand" "0,fm")
+          (match_operand:MODEF 2 "nonimmediate_operand" "fm,0")]))]
+  "TARGET_80387 && !(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
    && !COMMUTATIVE_ARITH_P (operands[3])
    && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type")
-        (cond [(match_operand:DF 3 "mult_operator" "")
+        (cond [(match_operand:MODEF 3 "mult_operator" "")
                  (const_string "fmul")
-               (match_operand:DF 3 "div_operator" "")
+               (match_operand:MODEF 3 "div_operator" "")
                  (const_string "fdiv")
               ]
               (const_string "fop")))
-   (set_attr "mode" "DF")])
+   (set_attr "mode" "<MODE>")])
 
 ;; ??? Add SSE splitters for these!
-(define_insn "*fop_df_2<mode>_i387"
-  [(set (match_operand:DF 0 "register_operand" "=f,f")
-       (match_operator:DF 3 "binary_fp_operator"
-          [(float:DF (match_operand:X87MODEI12 1 "nonimmediate_operand" "m,?r"))
-           (match_operand:DF 2 "register_operand" "0,0")]))]
-  "TARGET_80387 && TARGET_USE_<MODE>MODE_FIOP
-   && !(TARGET_SSE2 && TARGET_SSE_MATH)"
+(define_insn "*fop_<MODEF:mode>_2_i387"
+  [(set (match_operand:MODEF 0 "register_operand" "=f,f")
+       (match_operator:MODEF 3 "binary_fp_operator"
+         [(float:MODEF
+            (match_operand:X87MODEI12 1 "nonimmediate_operand" "m,?r"))
+          (match_operand:MODEF 2 "register_operand" "0,0")]))]
+  "TARGET_80387 && !(SSE_FLOAT_MODE_P (<MODEF:MODE>mode) && TARGET_SSE_MATH)
+   && (TARGET_USE_<X87MODEI12:MODE>MODE_FIOP || optimize_size)"
   "* return which_alternative ? \"#\" : output_387_binary_op (insn, operands);"
   [(set (attr "type")
-        (cond [(match_operand:DF 3 "mult_operator" "")
+        (cond [(match_operand:MODEF 3 "mult_operator" "")
                  (const_string "fmul")
-               (match_operand:DF 3 "div_operator" "")
+               (match_operand:MODEF 3 "div_operator" "")
                  (const_string "fdiv")
               ]
               (const_string "fop")))
    (set_attr "fp_int_src" "true")
-   (set_attr "mode" "<MODE>")])
-
-(define_insn "*fop_df_3<mode>_i387"
-  [(set (match_operand:DF 0 "register_operand" "=f,f")
-       (match_operator:DF 3 "binary_fp_operator"
-          [(match_operand:DF 1 "register_operand" "0,0")
-           (float:DF (match_operand:X87MODEI12 2 "nonimmediate_operand" "m,?r"))]))]
-  "TARGET_80387 && TARGET_USE_<MODE>MODE_FIOP
-   && !(TARGET_SSE2 && TARGET_SSE_MATH)"
+   (set_attr "mode" "<X87MODEI12:MODE>")])
+
+(define_insn "*fop_<MODEF:mode>_3_i387"
+  [(set (match_operand:MODEF 0 "register_operand" "=f,f")
+       (match_operator:MODEF 3 "binary_fp_operator"
+         [(match_operand:MODEF 1 "register_operand" "0,0")
+          (float:MODEF
+            (match_operand:X87MODEI12 2 "nonimmediate_operand" "m,?r"))]))]
+  "TARGET_80387 && !(SSE_FLOAT_MODE_P (<MODEF:MODE>mode) && TARGET_SSE_MATH)
+   && (TARGET_USE_<X87MODEI12:MODE>MODE_FIOP || optimize_size)"
   "* return which_alternative ? \"#\" : output_387_binary_op (insn, operands);"
   [(set (attr "type")
-        (cond [(match_operand:DF 3 "mult_operator" "")
+        (cond [(match_operand:MODEF 3 "mult_operator" "")
                  (const_string "fmul")
-               (match_operand:DF 3 "div_operator" "")
+               (match_operand:MODEF 3 "div_operator" "")
                  (const_string "fdiv")
               ]
               (const_string "fop")))
@@ -16103,7 +16132,8 @@
 (define_insn "*fop_df_4_i387"
   [(set (match_operand:DF 0 "register_operand" "=f,f")
        (match_operator:DF 3 "binary_fp_operator"
-          [(float_extend:DF (match_operand:SF 1 "nonimmediate_operand" "fm,0"))
+          [(float_extend:DF
+            (match_operand:SF 1 "nonimmediate_operand" "fm,0"))
            (match_operand:DF 2 "register_operand" "0,f")]))]
   "TARGET_80387 && !(TARGET_SSE2 && TARGET_SSE_MATH)
    && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
@@ -16183,12 +16213,13 @@
               (const_string "fop")))
    (set_attr "mode" "XF")])
 
-(define_insn "*fop_xf_2<mode>_i387"
+(define_insn "*fop_xf_2_i387"
   [(set (match_operand:XF 0 "register_operand" "=f,f")
        (match_operator:XF 3 "binary_fp_operator"
-          [(float:XF (match_operand:X87MODEI12 1 "nonimmediate_operand" "m,?r"))
-           (match_operand:XF 2 "register_operand" "0,0")]))]
-  "TARGET_80387 && TARGET_USE_<MODE>MODE_FIOP"
+         [(float:XF
+            (match_operand:X87MODEI12 1 "nonimmediate_operand" "m,?r"))
+          (match_operand:XF 2 "register_operand" "0,0")]))]
+  "TARGET_80387 && (TARGET_USE_<MODE>MODE_FIOP || optimize_size)"
   "* return which_alternative ? \"#\" : output_387_binary_op (insn, operands);"
   [(set (attr "type")
         (cond [(match_operand:XF 3 "mult_operator" "")
@@ -16200,12 +16231,13 @@
    (set_attr "fp_int_src" "true")
    (set_attr "mode" "<MODE>")])
 
-(define_insn "*fop_xf_3<mode>_i387"
+(define_insn "*fop_xf_3_i387"
   [(set (match_operand:XF 0 "register_operand" "=f,f")
        (match_operator:XF 3 "binary_fp_operator"
          [(match_operand:XF 1 "register_operand" "0,0")
-          (float:XF (match_operand:X87MODEI12 2 "nonimmediate_operand" "m,?r"))]))]
-  "TARGET_80387 && TARGET_USE_<MODE>MODE_FIOP"
+          (float:XF
+            (match_operand:X87MODEI12 2 "nonimmediate_operand" "m,?r"))]))]
+  "TARGET_80387 && (TARGET_USE_<MODE>MODE_FIOP || optimize_size)"
   "* return which_alternative ? \"#\" : output_387_binary_op (insn, operands);"
   [(set (attr "type")
         (cond [(match_operand:XF 3 "mult_operator" "")
@@ -16232,7 +16264,7 @@
                  (const_string "fdiv")
               ]
               (const_string "fop")))
-   (set_attr "mode" "SF")])
+   (set_attr "mode" "<MODE>")])
 
 (define_insn "*fop_xf_5_i387"
   [(set (match_operand:XF 0 "register_operand" "=f,f")
@@ -16249,7 +16281,7 @@
                  (const_string "fdiv")
               ]
               (const_string "fop")))
-   (set_attr "mode" "SF")])
+   (set_attr "mode" "<MODE>")])
 
 (define_insn "*fop_xf_6_i387"
   [(set (match_operand:XF 0 "register_operand" "=f,f")
@@ -16267,7 +16299,7 @@
                  (const_string "fdiv")
               ]
               (const_string "fop")))
-   (set_attr "mode" "SF")])
+   (set_attr "mode" "<MODE>")])
 
 (define_split
   [(set (match_operand 0 "register_operand" "")
@@ -16429,8 +16461,8 @@
   rtx op1 = gen_reg_rtx (XFmode);
   rtx op2 = gen_reg_rtx (XFmode);
 
-  emit_move_insn (op1, operands[1]);
   emit_move_insn (op2, operands[2]);
+  emit_move_insn (op1, operands[1]);
 
   emit_label (label);
   emit_insn (gen_fpremxf4_i387 (op1, op2, op1, op2));
@@ -16452,8 +16484,8 @@
   rtx op1 = gen_reg_rtx (XFmode);
   rtx op2 = gen_reg_rtx (XFmode);
 
-  emit_insn (gen_extend<mode>xf2 (op1, operands[1]));
   emit_insn (gen_extend<mode>xf2 (op2, operands[2]));
+  emit_insn (gen_extend<mode>xf2 (op1, operands[1]));
 
   emit_label (label);
   emit_insn (gen_fpremxf4_i387 (op1, op2, op1, op2));
@@ -16497,8 +16529,8 @@
   rtx op1 = gen_reg_rtx (XFmode);
   rtx op2 = gen_reg_rtx (XFmode);
 
-  emit_move_insn (op1, operands[1]);
   emit_move_insn (op2, operands[2]);
+  emit_move_insn (op1, operands[1]);
 
   emit_label (label);
   emit_insn (gen_fprem1xf4_i387 (op1, op2, op1, op2));
@@ -16520,8 +16552,8 @@
   rtx op1 = gen_reg_rtx (XFmode);
   rtx op2 = gen_reg_rtx (XFmode);
 
-  emit_insn (gen_extend<mode>xf2 (op1, operands[1]));
   emit_insn (gen_extend<mode>xf2 (op2, operands[2]));
+  emit_insn (gen_extend<mode>xf2 (op1, operands[1]));
 
   emit_label (label);
 
@@ -18374,6 +18406,14 @@
 \f
 ;; Block operation instructions
 
+(define_insn "cld"
+  [(unspec_volatile [(const_int 0)] UNSPECV_CLD)]
+  ""
+  "cld"
+  [(set_attr "length" "1")
+   (set_attr "length_immediate" "0")
+   (set_attr "modrm" "0")])
+
 (define_expand "movmemsi"
   [(use (match_operand:BLK 0 "memory_operand" ""))
    (use (match_operand:BLK 1 "memory_operand" ""))
@@ -18446,7 +18486,7 @@
              (set (match_operand 2 "register_operand" "")
                   (match_operand 5 "" ""))])]
   "TARGET_SINGLE_STRINGOP || optimize_size"
-  "")
+  "ix86_current_function_needs_cld = 1;")
 
 (define_insn "*strmovdi_rex_1"
   [(set (mem:DI (match_operand:DI 2 "register_operand" "0"))
@@ -18563,7 +18603,7 @@
                   (match_operand 3 "memory_operand" ""))
              (use (match_dup 4))])]
   ""
-  "")
+  "ix86_current_function_needs_cld = 1;")
 
 (define_insn "*rep_movdi_rex64"
   [(set (match_operand:DI 2 "register_operand" "=c") (const_int 0))
@@ -18723,7 +18763,7 @@
              (set (match_operand 0 "register_operand" "")
                   (match_operand 3 "" ""))])]
   "TARGET_SINGLE_STRINGOP || optimize_size"
-  "")
+  "ix86_current_function_needs_cld = 1;")
 
 (define_insn "*strsetdi_rex_1"
   [(set (mem:DI (match_operand:DI 1 "register_operand" "0"))
@@ -18817,7 +18857,7 @@
              (use (match_operand 3 "register_operand" ""))
              (use (match_dup 1))])]
   ""
-  "")
+  "ix86_current_function_needs_cld = 1;")
 
 (define_insn "*rep_stosdi_rex64"
   [(set (match_operand:DI 1 "register_operand" "=c") (const_int 0))
@@ -18993,7 +19033,7 @@
              (clobber (match_operand 1 "register_operand" ""))
              (clobber (match_dup 2))])]
   ""
-  "")
+  "ix86_current_function_needs_cld = 1;")
 
 (define_insn "*cmpstrnqi_nz_1"
   [(set (reg:CC FLAGS_REG)
@@ -19040,7 +19080,7 @@
              (clobber (match_operand 1 "register_operand" ""))
              (clobber (match_dup 2))])]
   ""
-  "")
+  "ix86_current_function_needs_cld = 1;")
 
 (define_insn "*cmpstrnqi_1"
   [(set (reg:CC FLAGS_REG)
@@ -19109,7 +19149,7 @@
              (clobber (match_operand 1 "register_operand" ""))
              (clobber (reg:CC FLAGS_REG))])]
   ""
-  "")
+  "ix86_current_function_needs_cld = 1;")
 
 (define_insn "*strlenqi_1"
   [(set (match_operand:SI 0 "register_operand" "=&c")
@@ -19698,7 +19738,7 @@
    (set_attr "length" "5")])
 
 (define_insn "allocate_stack_worker_64"
-  [(set (match_operand:DI 0 "register_operand" "=a")
+  [(set (match_operand:DI 0 "register_operand" "+a")
        (unspec_volatile:DI [(match_dup 0)] UNSPECV_STACK_PROBE))
    (set (reg:DI SP_REG) (minus:DI (reg:DI SP_REG) (match_dup 0)))
    (clobber (reg:DI R10_REG))
@@ -20978,14 +21018,14 @@
    (use (match_operand:DI 2 "const_int_operand" "i"))
    (use (label_ref:DI (match_operand 3 "" "X")))]
   "TARGET_64BIT
-   && INTVAL (operands[4]) + SSE_REGPARM_MAX * 16 - 16 < 128
+   && INTVAL (operands[4]) + X86_64_SSE_REGPARM_MAX * 16 - 16 < 128
    && INTVAL (operands[4]) + INTVAL (operands[2]) * 16 >= -128"
 {
   int i;
   operands[0] = gen_rtx_MEM (Pmode,
                             gen_rtx_PLUS (Pmode, operands[0], operands[4]));
   output_asm_insn ("jmp\t%A1", operands);
-  for (i = SSE_REGPARM_MAX - 1; i >= INTVAL (operands[2]); i--)
+  for (i = X86_64_SSE_REGPARM_MAX - 1; i >= INTVAL (operands[2]); i--)
     {
       operands[4] = adjust_address (operands[0], DImode, i*16);
       operands[5] = gen_rtx_REG (TImode, SSE_REGNO (i));
@@ -21001,7 +21041,7 @@
   [(set_attr "type" "other")
    (set_attr "length_immediate" "0")
    (set_attr "length_address" "0")
-   (set_attr "length" "135")
+   (set_attr "length" "34")
    (set_attr "memory" "store")
    (set_attr "modrm" "0")
    (set_attr "mode" "DI")])