OSDN Git Service

2004-04-27 Wu Yongwei <adah@sh163.net>
[pf3gnuchains/gcc-fork.git] / gcc / config / i386 / i386.md
index eb03cf2..c0a3ff4 100644 (file)
@@ -1,24 +1,24 @@
 ;; GCC machine description for IA-32 and x86-64.
 ;; Copyright (C) 1988, 1994, 1995, 1996, 1997, 1998, 1999, 2000,
-;; 2001, 2002, 2003
+;; 2001, 2002, 2003, 2004
 ;; Free Software Foundation, Inc.
 ;; Mostly by William Schelter.
 ;; x86_64 support added by Jan Hubicka
 ;;
-;; This file is part of GNU CC.
+;; This file is part of GCC.
 ;;
-;; GNU CC is free software; you can redistribute it and/or modify
+;; GCC is free software; you can redistribute it and/or modify
 ;; it under the terms of the GNU General Public License as published by
 ;; the Free Software Foundation; either version 2, or (at your option)
 ;; any later version.
 ;;
-;; GNU CC is distributed in the hope that it will be useful,
+;; GCC is distributed in the hope that it will be useful,
 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ;; GNU General Public License for more details.
 ;;
 ;; You should have received a copy of the GNU General Public License
-;; along with GNU CC; see the file COPYING.  If not, write to
+;; along with GCC; see the file COPYING.  If not, write to
 ;; the Free Software Foundation, 59 Temple Place - Suite 330,
 ;; Boston, MA 02111-1307, USA.  */
 ;;
@@ -27,9 +27,6 @@
 ;;
 ;; See file "rtl.def" for documentation on define_insn, match_*, et. al.
 ;;
-;; Macro #define NOTICE_UPDATE_CC in file i386.h handles condition code
-;; updates for most instructions.
-;;
 ;; Macro REG_CLASS_FROM_LETTER in file i386.h defines the register
 ;; constraint letters.
 ;;
    (UNSPEC_MFENCE              59)
    (UNSPEC_LFENCE              60)
    (UNSPEC_PSADBW              61)
+   (UNSPEC_ADDSUB              71)
+   (UNSPEC_HADD                        72)
+   (UNSPEC_HSUB                        73)
+   (UNSPEC_MOVSHDUP            74)
+   (UNSPEC_MOVSLDUP            75)
+   (UNSPEC_LDQQU               76)
+   (UNSPEC_MOVDDUP             77)
 
    ; x87 Floating point
    (UNSPEC_FPATAN              65)
    (UNSPEC_FYL2X               66)
+   (UNSPEC_FSCALE              67)
+   (UNSPEC_FRNDINT             68)
+   (UNSPEC_F2XM1               69)
+
+   ; x87 Double output FP
+   (UNSPEC_SINCOS_COS          80)
+   (UNSPEC_SINCOS_SIN          81)
+   (UNSPEC_TAN_ONE             82)
+   (UNSPEC_TAN_TAN             83)
+   (UNSPEC_XTRACT_FRACT                84)
+   (UNSPEC_XTRACT_EXP          85)
+
+   ; REP instruction
+   (UNSPEC_REP                 75)
   ])
 
 (define_constants
    (UNSPECV_STMXCSR            40)
    (UNSPECV_FEMMS              46)
    (UNSPECV_CLFLUSH            57)
+   (UNSPECV_ALIGN              68)
+   (UNSPECV_MONITOR            69)
+   (UNSPECV_MWAIT              70)
   ])
 
 ;; Insns whose names begin with "x86_" are emitted by gen_FOO calls
 \f
 ;; Processor type.  This attribute must exactly match the processor_type
 ;; enumeration in i386.h.
-(define_attr "cpu" "i386,i486,pentium,pentiumpro,k6,athlon,pentium4,k8"
+(define_attr "cpu" "i386,i486,pentium,pentiumpro,k6,athlon,pentium4,k8,nocona"
   (const (symbol_ref "ix86_tune")))
 
 ;; A basic instruction type.  Refinements due to arguments to be
     (const_int 1)
     (const_int 0)))
 
-;; Set when 0f opcode prefix is used.
+;; Set when REX opcode prefix is used.
 (define_attr "prefix_rex" ""
   (cond [(and (eq_attr "mode" "DI")
              (eq_attr "type" "!push,pop,call,callv,leave,ibr"))
 (define_attr "length" ""
   (cond [(eq_attr "type" "other,multi,fistp")
           (const_int 16)
+        (eq_attr "type" "fcmp")
+          (const_int 4)
         (eq_attr "unit" "i387")
           (plus (const_int 2)
                 (plus (attr "prefix_data16")
           (if_then_else (match_operand 1 "constant_call_address_operand" "")
             (const_string "none")
             (const_string "load"))
-        (and (eq_attr "type" "alu1,negnot")
+        (and (eq_attr "type" "alu1,negnot,ishift1")
              (match_operand 1 "memory_operand" ""))
           (const_string "both")
         (and (match_operand 0 "memory_operand" "")
         (match_operand 1 "memory_operand" "")
           (const_string "load")
         (and (eq_attr "type"
-                "!alu1,negnot,
+                "!alu1,negnot,ishift1,
                   imov,imovx,icmp,test,
                   fmov,fcmp,fsgn,
                   sse,ssemov,ssecmp,ssecomi,ssecvt,sseicvt,
   [(set (reg:CC 17)
        (compare:CC (match_operand:XF 0 "cmp_fp_expander_operand" "")
                    (match_operand:XF 1 "cmp_fp_expander_operand" "")))]
-  "!TARGET_64BIT && TARGET_80387"
-{
-  ix86_compare_op0 = operands[0];
-  ix86_compare_op1 = operands[1];
-  DONE;
-})
-
-(define_expand "cmptf"
-  [(set (reg:CC 17)
-       (compare:CC (match_operand:TF 0 "cmp_fp_expander_operand" "")
-                   (match_operand:TF 1 "cmp_fp_expander_operand" "")))]
   "TARGET_80387"
 {
   ix86_compare_op0 = operands[0];
        (compare:CCFP
          (match_operand:XF 0 "register_operand" "f")
          (match_operand:XF 1 "register_operand" "f")))]
-  "!TARGET_64BIT && TARGET_80387"
-  "* return output_fp_compare (insn, operands, 0, 0);"
-  [(set_attr "type" "fcmp")
-   (set_attr "mode" "XF")])
-
-(define_insn "*cmpfp_2_tf"
-  [(set (reg:CCFP 18)
-       (compare:CCFP
-         (match_operand:TF 0 "register_operand" "f")
-         (match_operand:TF 1 "register_operand" "f")))]
   "TARGET_80387"
   "* return output_fp_compare (insn, operands, 0, 0);"
   [(set_attr "type" "fcmp")
             (match_operand:XF 1 "register_operand" "f")
             (match_operand:XF 2 "register_operand" "f"))]
          UNSPEC_FNSTSW))]
-  "!TARGET_64BIT && TARGET_80387"
-  "* return output_fp_compare (insn, operands, 2, 0);"
-  [(set_attr "type" "multi")
-   (set_attr "mode" "XF")])
-
-(define_insn "*cmpfp_2_tf_1"
-  [(set (match_operand:HI 0 "register_operand" "=a")
-       (unspec:HI
-         [(compare:CCFP
-            (match_operand:TF 1 "register_operand" "f")
-            (match_operand:TF 2 "register_operand" "f"))]
-         UNSPEC_FNSTSW))]
   "TARGET_80387"
   "* return output_fp_compare (insn, operands, 2, 0);"
   [(set_attr "type" "multi")
   "fnstsw\t%0"
   [(set_attr "length" "2")
    (set_attr "mode" "SI")
-   (set_attr "unit" "i387")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "unit" "i387")])
 
 ;; FP compares, step 3
 ;; Get ax into flags, general case.
   "sahf"
   [(set_attr "length" "1")
    (set_attr "athlon_decode" "vector")
-   (set_attr "mode" "SI")
-   (set_attr "ppro_uops" "one")])
+   (set_attr "mode" "SI")])
 
 ;; Pentium Pro can do steps 1 through 3 in one go.
 
   [(set_attr "type" "alu1")
    (set_attr "mode" "SI")
    (set_attr "length_immediate" "0")])
-
 (define_insn "*movsi_or"
   [(set (match_operand:SI 0 "register_operand" "=r")
        (match_operand:SI 1 "immediate_operand" "i"))
    (clobber (reg:CC 17))]
-  "reload_completed && GET_CODE (operands[1]) == CONST_INT
-   && INTVAL (operands[1]) == -1
+  "reload_completed
+   && operands[1] == constm1_rtx
    && (TARGET_PENTIUM || optimize_size)"
 {
   operands[1] = constm1_rtx;
 ;; We fake an second form of instruction to force reload to load address
 ;; into register when rax is not available
 (define_insn "*movabssi_1_rex64"
-  [(set (mem:SI (match_operand:DI 0 "x86_64_movabs_operand" "i,r,r"))
-       (match_operand:SI 1 "nonmemory_operand" "a,er,i"))]
-  "TARGET_64BIT"
+  [(set (mem:SI (match_operand:DI 0 "x86_64_movabs_operand" "i,r"))
+       (match_operand:SI 1 "nonmemory_operand" "a,er"))]
+  "TARGET_64BIT && ix86_check_movabs (insn, 0)"
   "@
    movabs{l}\t{%1, %P0|%P0, %1}
-   mov{l}\t{%1, %a0|%a0, %1}
-   movabs{l}\t{%1, %a0|%a0, %1}"
+   mov{l}\t{%1, %a0|%a0, %1}"
   [(set_attr "type" "imov")
-   (set_attr "modrm" "0,*,*")
-   (set_attr "length_address" "8,0,0")
-   (set_attr "length_immediate" "0,*,*")
+   (set_attr "modrm" "0,*")
+   (set_attr "length_address" "8,0")
+   (set_attr "length_immediate" "0,*")
    (set_attr "memory" "store")
    (set_attr "mode" "SI")])
 
 (define_insn "*movabssi_2_rex64"
   [(set (match_operand:SI 0 "register_operand" "=a,r")
         (mem:SI (match_operand:DI 1 "x86_64_movabs_operand" "i,r")))]
-  "TARGET_64BIT"
+  "TARGET_64BIT && ix86_check_movabs (insn, 1)"
   "@
    movabs{l}\t{%P1, %0|%0, %P1}
    mov{l}\t{%a1, %0|%0, %a1}"
    (set_attr "pent_pair" "np")
    (set_attr "athlon_decode" "vector")
    (set_attr "mode" "SI")
-   (set_attr "modrm" "0")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "modrm" "0")])
 
 (define_expand "movhi"
   [(set (match_operand:HI 0 "nonimmediate_operand" "")
     }
 }
   [(set (attr "type")
-     (cond [(and (eq_attr "alternative" "0")
+     (cond [(ne (symbol_ref "optimize_size") (const_int 0))
+             (const_string "imov")
+           (and (eq_attr "alternative" "0")
                 (ior (eq (symbol_ref "TARGET_PARTIAL_REG_STALL")
                          (const_int 0))
                      (eq (symbol_ref "TARGET_HIMODE_MATH")
 ;; We fake an second form of instruction to force reload to load address
 ;; into register when rax is not available
 (define_insn "*movabshi_1_rex64"
-  [(set (mem:HI (match_operand:DI 0 "x86_64_movabs_operand" "i,r,r"))
-       (match_operand:HI 1 "nonmemory_operand" "a,er,i"))]
-  "TARGET_64BIT"
+  [(set (mem:HI (match_operand:DI 0 "x86_64_movabs_operand" "i,r"))
+       (match_operand:HI 1 "nonmemory_operand" "a,er"))]
+  "TARGET_64BIT && ix86_check_movabs (insn, 0)"
   "@
    movabs{w}\t{%1, %P0|%P0, %1}
-   mov{w}\t{%1, %a0|%a0, %1}
-   movabs{w}\t{%1, %a0|%a0, %1}"
+   mov{w}\t{%1, %a0|%a0, %1}"
   [(set_attr "type" "imov")
-   (set_attr "modrm" "0,*,*")
-   (set_attr "length_address" "8,0,0")
-   (set_attr "length_immediate" "0,*,*")
+   (set_attr "modrm" "0,*")
+   (set_attr "length_address" "8,0")
+   (set_attr "length_immediate" "0,*")
    (set_attr "memory" "store")
    (set_attr "mode" "HI")])
 
 (define_insn "*movabshi_2_rex64"
   [(set (match_operand:HI 0 "register_operand" "=a,r")
         (mem:HI (match_operand:DI 1 "x86_64_movabs_operand" "i,r")))]
-  "TARGET_64BIT"
+  "TARGET_64BIT && ix86_check_movabs (insn, 1)"
   "@
    movabs{w}\t{%P1, %0|%0, %P1}
    mov{w}\t{%a1, %0|%0, %a1}"
   [(set_attr "type" "imov")
    (set_attr "pent_pair" "np")
    (set_attr "mode" "HI")
-   (set_attr "modrm" "0")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "modrm" "0")])
 
 (define_insn "*swaphi_2"
   [(set (match_operand:HI 0 "register_operand" "+r")
   [(set_attr "type" "imov")
    (set_attr "pent_pair" "np")
    (set_attr "mode" "SI")
-   (set_attr "modrm" "0")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "modrm" "0")])
 
 (define_expand "movstricthi"
   [(set (strict_low_part (match_operand:HI 0 "nonimmediate_operand" ""))
     }
 }
   [(set (attr "type")
-     (cond [(and (eq_attr "alternative" "3")
+     (cond [(ne (symbol_ref "optimize_size") (const_int 0))
+             (const_string "imov")
+           (and (eq_attr "alternative" "3")
                 (ior (eq (symbol_ref "TARGET_PARTIAL_REG_STALL")
                          (const_int 0))
                      (eq (symbol_ref "TARGET_QIMODE_MATH")
   [(set_attr "type" "imov")
    (set_attr "pent_pair" "np")
    (set_attr "mode" "QI")
-   (set_attr "modrm" "0")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "modrm" "0")])
 
 (define_expand "movstrictqi"
   [(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" ""))
 ;; We fake an second form of instruction to force reload to load address
 ;; into register when rax is not available
 (define_insn "*movabsqi_1_rex64"
-  [(set (mem:QI (match_operand:DI 0 "x86_64_movabs_operand" "i,r,r"))
-       (match_operand:QI 1 "nonmemory_operand" "a,er,i"))]
-  "TARGET_64BIT"
+  [(set (mem:QI (match_operand:DI 0 "x86_64_movabs_operand" "i,r"))
+       (match_operand:QI 1 "nonmemory_operand" "a,er"))]
+  "TARGET_64BIT && ix86_check_movabs (insn, 0)"
   "@
    movabs{b}\t{%1, %P0|%P0, %1}
-   mov{b}\t{%1, %a0|%a0, %1}
-   movabs{b}\t{%1, %a0|%a0, %1}"
+   mov{b}\t{%1, %a0|%a0, %1}"
   [(set_attr "type" "imov")
-   (set_attr "modrm" "0,*,*")
-   (set_attr "length_address" "8,0,0")
-   (set_attr "length_immediate" "0,*,*")
+   (set_attr "modrm" "0,*")
+   (set_attr "length_address" "8,0")
+   (set_attr "length_immediate" "0,*")
    (set_attr "memory" "store")
    (set_attr "mode" "QI")])
 
 (define_insn "*movabsqi_2_rex64"
   [(set (match_operand:QI 0 "register_operand" "=a,r")
         (mem:QI (match_operand:DI 1 "x86_64_movabs_operand" "i,r")))]
-  "TARGET_64BIT"
+  "TARGET_64BIT && ix86_check_movabs (insn, 1)"
   "@
    movabs{b}\t{%P1, %0|%0, %P1}
    mov{b}\t{%a1, %0|%0, %a1}"
   [(set (zero_extract:SI (match_operand 0 "ext_register_operand" "+Q")
                         (const_int 8)
                         (const_int 8))
-       (and:SI (lshiftrt:SI (match_operand:SI 1 "register_operand" "Q")
-                            (const_int 8))
-               (const_int 255)))]
+       (lshiftrt:SI (match_operand:SI 1 "register_operand" "Q")
+                    (const_int 8)))]
   ""
   "mov{b}\t{%h1, %h0|%h0, %h1}"
   [(set_attr "type" "imov")
    (clobber (reg:CC 17))]
   "TARGET_64BIT && (TARGET_PENTIUM || optimize_size)
    && reload_completed
-   && GET_CODE (operands[1]) == CONST_INT
-   && INTVAL (operands[1]) == -1"
+   && operands[1] == constm1_rtx"
 {
   operands[1] = constm1_rtx;
   return "or{q}\t{%1, %0|%0, %1}";
 ;; We fake an second form of instruction to force reload to load address
 ;; into register when rax is not available
 (define_insn "*movabsdi_1_rex64"
-  [(set (mem:DI (match_operand:DI 0 "x86_64_movabs_operand" "i,r,r"))
-       (match_operand:DI 1 "nonmemory_operand" "a,er,i"))]
-  "TARGET_64BIT"
+  [(set (mem:DI (match_operand:DI 0 "x86_64_movabs_operand" "i,r"))
+       (match_operand:DI 1 "nonmemory_operand" "a,er"))]
+  "TARGET_64BIT && ix86_check_movabs (insn, 0)"
   "@
    movabs{q}\t{%1, %P0|%P0, %1}
-   mov{q}\t{%1, %a0|%a0, %1}
-   movabs{q}\t{%1, %a0|%a0, %1}"
+   mov{q}\t{%1, %a0|%a0, %1}"
   [(set_attr "type" "imov")
-   (set_attr "modrm" "0,*,*")
-   (set_attr "length_address" "8,0,0")
-   (set_attr "length_immediate" "0,*,*")
+   (set_attr "modrm" "0,*")
+   (set_attr "length_address" "8,0")
+   (set_attr "length_immediate" "0,*")
    (set_attr "memory" "store")
    (set_attr "mode" "DI")])
 
 (define_insn "*movabsdi_2_rex64"
   [(set (match_operand:DI 0 "register_operand" "=a,r")
         (mem:DI (match_operand:DI 1 "x86_64_movabs_operand" "i,r")))]
-  "TARGET_64BIT"
+  "TARGET_64BIT && ix86_check_movabs (insn, 1)"
   "@
    movabs{q}\t{%P1, %0|%0, %P1}
    mov{q}\t{%a1, %0|%0, %a1}"
    (set_attr "pent_pair" "np")
    (set_attr "athlon_decode" "vector")
    (set_attr "mode" "DI")
-   (set_attr "modrm" "0")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "modrm" "0")])
 
   
 (define_expand "movsf"
       return "push{l}\t%1";
 
     default:
-      /* This insn should be already splitted before reg-stack.  */
+      /* This insn should be already split before reg-stack.  */
       abort ();
     }
 }
       return "push{q}\t%q1";
 
     default:
-      /* This insn should be already splitted before reg-stack.  */
+      /* This insn should be already split before reg-stack.  */
       abort ();
     }
 }
   switch (which_alternative)
     {
     case 0:
-      if (REG_P (operands[1])
-          && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-        return "fstp\t%y0";
-      else if (STACK_TOP_P (operands[0]))
-        return "fld%z1\t%y1";
-      else
-        return "fst\t%y0";
+      return output_387_reg_move (insn, operands);
 
     case 1:
       if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
   switch (which_alternative)
     {
     case 0:
-      if (REG_P (operands[1])
-          && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-       {
-         if (REGNO (operands[0]) == FIRST_STACK_REG
-             && TARGET_USE_FFREEP)
-           return "ffreep\t%y0";
-          return "fstp\t%y0";
-       }
-      else if (STACK_TOP_P (operands[0]))
-        return "fld%z1\t%y1";
-      else
-        return "fst\t%y0";
+      return output_387_reg_move (insn, operands);
 
     case 1:
       if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
        (match_operand:DF 1 "general_no_elim_operand" "f#Y,Fo#fY,*r#fY,Y#f"))]
   "!TARGET_64BIT && !TARGET_INTEGER_DFMODE_MOVES"
 {
-  /* This insn should be already splitted before reg-stack.  */
+  /* This insn should be already split before reg-stack.  */
   abort ();
 }
   [(set_attr "type" "multi")
        (match_operand:DF 1 "general_no_elim_operand" "f#rY,rFo#fY,Y#rf"))]
   "TARGET_64BIT || TARGET_INTEGER_DFMODE_MOVES"
 {
-  /* This insn should be already splitted before reg-stack.  */
+  /* This insn should be already split before reg-stack.  */
   abort ();
 }
   [(set_attr "type" "multi")
   switch (which_alternative)
     {
     case 0:
-      if (REG_P (operands[1])
-          && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-       {
-         if (REGNO (operands[0]) == FIRST_STACK_REG
-             && TARGET_USE_FFREEP)
-           return "ffreep\t%y0";
-          return "fstp\t%y0";
-       }
-      else if (STACK_TOP_P (operands[0]))
-        return "fld%z1\t%y1";
-      else
-        return "fst\t%y0";
+      return output_387_reg_move (insn, operands);
 
     case 1:
       if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
   switch (which_alternative)
     {
     case 0:
-      if (REG_P (operands[1])
-          && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-       {
-         if (REGNO (operands[0]) == FIRST_STACK_REG
-             && TARGET_USE_FFREEP)
-           return "ffreep\t%y0";
-          return "fstp\t%y0";
-       }
-      else if (STACK_TOP_P (operands[0]))
-        return "fld%z1\t%y1";
-      else
-        return "fst\t%y0";
+      return output_387_reg_move (insn, operands);
 
     case 1:
       if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
 (define_expand "movxf"
   [(set (match_operand:XF 0 "nonimmediate_operand" "")
        (match_operand:XF 1 "general_operand" ""))]
-  "!TARGET_64BIT"
-  "ix86_expand_move (XFmode, operands); DONE;")
-
-(define_expand "movtf"
-  [(set (match_operand:TF 0 "nonimmediate_operand" "")
-       (match_operand:TF 1 "general_operand" ""))]
   ""
-  "ix86_expand_move (TFmode, operands); DONE;")
+  "ix86_expand_move (XFmode, operands); DONE;")
 
 ;; Size of pushdf is 3 (for sub) + 2 (for fstp) + memory operand size.
 ;; Size of pushdf using integer instructions is 3+3*memory operand size
 (define_insn "*pushxf_nointeger"
   [(set (match_operand:XF 0 "push_operand" "=X,X,X")
        (match_operand:XF 1 "general_no_elim_operand" "f,Fo,*r"))]
-  "!TARGET_64BIT && optimize_size"
-{
-  /* This insn should be already splitted before reg-stack.  */
-  abort ();
-}
-  [(set_attr "type" "multi")
-   (set_attr "mode" "XF,SI,SI")])
-
-(define_insn "*pushtf_nointeger"
-  [(set (match_operand:TF 0 "push_operand" "=<,<,<")
-       (match_operand:TF 1 "general_no_elim_operand" "f,Fo,*r"))]
   "optimize_size"
 {
-  /* This insn should be already splitted before reg-stack.  */
+  /* This insn should be already split before reg-stack.  */
   abort ();
 }
   [(set_attr "type" "multi")
 (define_insn "*pushxf_integer"
   [(set (match_operand:XF 0 "push_operand" "=<,<")
        (match_operand:XF 1 "general_no_elim_operand" "f#r,ro#f"))]
-  "!TARGET_64BIT && !optimize_size"
-{
-  /* This insn should be already splitted before reg-stack.  */
-  abort ();
-}
-  [(set_attr "type" "multi")
-   (set_attr "mode" "XF,SI")])
-
-(define_insn "*pushtf_integer"
-  [(set (match_operand:TF 0 "push_operand" "=<,<")
-       (match_operand:TF 1 "general_no_elim_operand" "f#r,rFo#f"))]
   "!optimize_size"
 {
-  /* This insn should be already splitted before reg-stack.  */
+  /* This insn should be already split before reg-stack.  */
   abort ();
 }
   [(set_attr "type" "multi")
        (match_operand 1 "general_operand" ""))]
   "reload_completed
    && (GET_MODE (operands[0]) == XFmode
-       || GET_MODE (operands[0]) == TFmode
        || GET_MODE (operands[0]) == DFmode)
    && !ANY_FP_REG_P (operands[1])"
   [(const_int 0)]
   [(set (match_operand:XF 0 "push_operand" "")
        (match_operand:XF 1 "any_fp_register_operand" ""))]
   "!TARGET_64BIT"
-  [(set (reg:SI 7) (plus:SI (reg:SI 7) (const_int -12)))
-   (set (mem:XF (reg:SI 7)) (match_dup 1))])
-
-(define_split
-  [(set (match_operand:TF 0 "push_operand" "")
-       (match_operand:TF 1 "any_fp_register_operand" ""))]
-  "!TARGET_64BIT"
-  [(set (reg:SI 7) (plus:SI (reg:SI 7) (const_int -16)))
-   (set (mem:TF (reg:SI 7)) (match_dup 1))])
+  [(set (reg:SI 7) (plus:SI (reg:SI 7) (match_dup 2)))
+   (set (mem:XF (reg:SI 7)) (match_dup 1))]
+  "operands[2] = GEN_INT (TARGET_128BIT_LONG_DOUBLE ? -16 : -12);")
 
 (define_split
-  [(set (match_operand:TF 0 "push_operand" "")
-       (match_operand:TF 1 "any_fp_register_operand" ""))]
+  [(set (match_operand:XF 0 "push_operand" "")
+       (match_operand:XF 1 "any_fp_register_operand" ""))]
   "TARGET_64BIT"
-  [(set (reg:DI 7) (plus:DI (reg:DI 7) (const_int -16)))
-   (set (mem:TF (reg:DI 7)) (match_dup 1))])
+  [(set (reg:DI 7) (plus:DI (reg:DI 7) (match_dup 2)))
+   (set (mem:XF (reg:DI 7)) (match_dup 1))]
+  "operands[2] = GEN_INT (TARGET_128BIT_LONG_DOUBLE ? -16 : -12);")
 
 ;; Do not use integer registers when optimizing for size
 (define_insn "*movxf_nointeger"
   [(set (match_operand:XF 0 "nonimmediate_operand" "=f,m,f,*r,o")
        (match_operand:XF 1 "general_operand" "fm,f,G,*roF,F*r"))]
-  "!TARGET_64BIT
-   && optimize_size
+  "optimize_size
    && (GET_CODE (operands[0]) != MEM || GET_CODE (operands[1]) != MEM)
    && (reload_in_progress || reload_completed
        || GET_CODE (operands[1]) != CONST_DOUBLE
   switch (which_alternative)
     {
     case 0:
-      if (REG_P (operands[1])
-          && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-       {
-         if (REGNO (operands[0]) == FIRST_STACK_REG
-             && TARGET_USE_FFREEP)
-           return "ffreep\t%y0";
-          return "fstp\t%y0";
-       }
-      else if (STACK_TOP_P (operands[0]))
-        return "fld%z1\t%y1";
-      else
-        return "fst\t%y0";
-
-    case 1:
-      /* There is no non-popping store to memory for XFmode.  So if
-        we need one, follow the store with a load.  */
-      if (! find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-        return "fstp%z0\t%y0\;fld%z0\t%y0";
-      else
-        return "fstp%z0\t%y0";
-
-    case 2:
-      return standard_80387_constant_opcode (operands[1]);
-
-    case 3: case 4:
-      return "#";
-    }
-  abort();
-}
-  [(set_attr "type" "fmov,fmov,fmov,multi,multi")
-   (set_attr "mode" "XF,XF,XF,SI,SI")])
-
-(define_insn "*movtf_nointeger"
-  [(set (match_operand:TF 0 "nonimmediate_operand" "=f,m,f,*r,o")
-       (match_operand:TF 1 "general_operand" "fm,f,G,*roF,F*r"))]
-  "(GET_CODE (operands[0]) != MEM || GET_CODE (operands[1]) != MEM)
-   && optimize_size
-   && (reload_in_progress || reload_completed
-       || GET_CODE (operands[1]) != CONST_DOUBLE
-       || (ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_LARGE)
-       || memory_operand (operands[0], TFmode))" 
-{
-  switch (which_alternative)
-    {
-    case 0:
-      if (REG_P (operands[1])
-          && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-       {
-         if (REGNO (operands[0]) == FIRST_STACK_REG
-             && TARGET_USE_FFREEP)
-           return "ffreep\t%y0";
-          return "fstp\t%y0";
-       }
-      else if (STACK_TOP_P (operands[0]))
-        return "fld%z1\t%y1";
-      else
-        return "fst\t%y0";
+      return output_387_reg_move (insn, operands);
 
     case 1:
       /* There is no non-popping store to memory for XFmode.  So if
 (define_insn "*movxf_integer"
   [(set (match_operand:XF 0 "nonimmediate_operand" "=f#r,m,f#r,r#f,o")
        (match_operand:XF 1 "general_operand" "fm#r,f#r,G,roF#f,Fr#f"))]
-  "!TARGET_64BIT
-   && !optimize_size
+  "!optimize_size
    && (GET_CODE (operands[0]) != MEM || GET_CODE (operands[1]) != MEM)
    && (reload_in_progress || reload_completed
        || GET_CODE (operands[1]) != CONST_DOUBLE
   switch (which_alternative)
     {
     case 0:
-      if (REG_P (operands[1])
-          && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-       {
-         if (REGNO (operands[0]) == FIRST_STACK_REG
-             && TARGET_USE_FFREEP)
-           return "ffreep\t%y0";
-          return "fstp\t%y0";
-       }
-      else if (STACK_TOP_P (operands[0]))
-        return "fld%z1\t%y1";
-      else
-        return "fst\t%y0";
-
-    case 1:
-      /* There is no non-popping store to memory for XFmode.  So if
-        we need one, follow the store with a load.  */
-      if (! find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-        return "fstp%z0\t%y0\;fld%z0\t%y0";
-      else
-        return "fstp%z0\t%y0";
-
-    case 2:
-      return standard_80387_constant_opcode (operands[1]);
-
-    case 3: case 4:
-      return "#";
-    }
-  abort();
-}
-  [(set_attr "type" "fmov,fmov,fmov,multi,multi")
-   (set_attr "mode" "XF,XF,XF,SI,SI")])
-
-(define_insn "*movtf_integer"
-  [(set (match_operand:TF 0 "nonimmediate_operand" "=f#r,m,f#r,r#f,o")
-       (match_operand:TF 1 "general_operand" "fm#r,f#r,G,roF#f,Fr#f"))]
-  "(GET_CODE (operands[0]) != MEM || GET_CODE (operands[1]) != MEM)
-   && !optimize_size
-   && (reload_in_progress || reload_completed
-       || GET_CODE (operands[1]) != CONST_DOUBLE
-       || (ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_LARGE)
-       || memory_operand (operands[0], TFmode))" 
-{
-  switch (which_alternative)
-    {
-    case 0:
-      if (REG_P (operands[1])
-          && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-       {
-         if (REGNO (operands[0]) == FIRST_STACK_REG
-             && TARGET_USE_FFREEP)
-           return "ffreep\t%y0";
-          return "fstp\t%y0";
-       }
-      else if (STACK_TOP_P (operands[0]))
-        return "fld%z1\t%y1";
-      else
-        return "fst\t%y0";
+      return output_387_reg_move (insn, operands);
 
     case 1:
       /* There is no non-popping store to memory for XFmode.  So if
        (match_operand 1 "general_operand" ""))]
   "reload_completed
    && (GET_CODE (operands[0]) != MEM || GET_CODE (operands[1]) != MEM)
-   && (GET_MODE (operands[0]) == XFmode || GET_MODE (operands[0]) == TFmode)
+   && GET_MODE (operands[0]) == XFmode
    && ! (ANY_FP_REG_P (operands[0]) || 
         (GET_CODE (operands[0]) == SUBREG
          && ANY_FP_REG_P (SUBREG_REG (operands[0]))))
        (match_operand 1 "memory_operand" ""))]
   "reload_completed
    && GET_CODE (operands[1]) == MEM
-   && (GET_MODE (operands[0]) == XFmode || GET_MODE (operands[0]) == TFmode
+   && (GET_MODE (operands[0]) == XFmode
        || GET_MODE (operands[0]) == SFmode || GET_MODE (operands[0]) == DFmode)
    && GET_CODE (XEXP (operands[1], 0)) == SYMBOL_REF
-   && CONSTANT_POOL_ADDRESS_P (XEXP (operands[1], 0))
-   && (!(SSE_REG_P (operands[0]) || 
-        (GET_CODE (operands[0]) == SUBREG
-         && SSE_REG_P (SUBREG_REG (operands[0]))))
-       || standard_sse_constant_p (get_pool_constant (XEXP (operands[1], 0))))
-   && (!(FP_REG_P (operands[0]) || 
-        (GET_CODE (operands[0]) == SUBREG
-         && FP_REG_P (SUBREG_REG (operands[0]))))
-       || standard_80387_constant_p (get_pool_constant (XEXP (operands[1], 0))))"
-  [(set (match_dup 0)
-       (match_dup 1))]
-  "operands[1] = get_pool_constant (XEXP (operands[1], 0));")
+   && CONSTANT_POOL_ADDRESS_P (XEXP (operands[1], 0))"
+  [(set (match_dup 0) (match_dup 1))]
+{
+  rtx c = get_pool_constant (XEXP (operands[1], 0));
+  rtx r = operands[0];
+
+  if (GET_CODE (r) == SUBREG)
+    r = SUBREG_REG (r);
+
+  if (SSE_REG_P (r))
+    {
+      if (!standard_sse_constant_p (c))
+       FAIL;
+    }
+  else if (FP_REG_P (r))
+    {
+      if (!standard_80387_constant_p (c))
+       FAIL;
+    }
+  else if (MMX_REG_P (r))
+    FAIL;
+
+  operands[1] = c;
+})
 
 (define_insn "swapxf"
   [(set (match_operand:XF 0 "register_operand" "+f")
 }
   [(set_attr "type" "fxch")
    (set_attr "mode" "XF")])
-
-(define_insn "swaptf"
-  [(set (match_operand:TF 0 "register_operand" "+f")
-       (match_operand:TF 1 "register_operand" "+f"))
-   (set (match_dup 1)
-       (match_dup 0))]
-  ""
-{
-  if (STACK_TOP_P (operands[0]))
-    return "fxch\t%1";
-  else
-    return "fxch\t%0";
-}
-  [(set_attr "type" "fxch")
-   (set_attr "mode" "XF")])
 \f
 ;; Zero extension instructions
 
   ")
 
 (define_insn "zero_extendsidi2_32"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "=r,?r,?*o")
-       (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "0,rm,r")))
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=r,?r,?*o,!?y,!?Y")
+       (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "0,rm,r,m,m")))
    (clobber (reg:CC 17))]
-  "!TARGET_64BIT"
-  "#"
-  [(set_attr "mode" "SI")])
+  "!TARGET_64BIT && !TARGET_INTER_UNIT_MOVES"
+  "@
+   #
+   #
+   #
+   movd\t{%1, %0|%0, %1}
+   movd\t{%1, %0|%0, %1}"
+  [(set_attr "mode" "SI,SI,SI,DI,TI")
+   (set_attr "type" "multi,multi,multi,mmxmov,ssemov")])
+
+(define_insn "*zero_extendsidi2_32_1"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=r,?r,?*o,!?y,!?Y")
+       (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "0,rm,r,rm,rm")))
+   (clobber (reg:CC 17))]
+  "!TARGET_64BIT && TARGET_INTER_UNIT_MOVES"
+  "@
+   #
+   #
+   #
+   movd\t{%1, %0|%0, %1}
+   movd\t{%1, %0|%0, %1}"
+  [(set_attr "mode" "SI,SI,SI,DI,TI")
+   (set_attr "type" "multi,multi,multi,mmxmov,ssemov")])
 
 (define_insn "zero_extendsidi2_rex64"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "=r,o")
-     (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "rm,0")))]
-  "TARGET_64BIT"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=r,o,!?y,!?Y")
+     (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "rm,0,m,m")))]
+  "TARGET_64BIT && !TARGET_INTER_UNIT_MOVES"
   "@
    mov\t{%k1, %k0|%k0, %k1}
-   #"
-  [(set_attr "type" "imovx,imov")
-   (set_attr "mode" "SI,DI")])
+   #
+   movd\t{%1, %0|%0, %1}
+   movd\t{%1, %0|%0, %1}"
+  [(set_attr "type" "imovx,imov,mmxmov,ssemov")
+   (set_attr "mode" "SI,DI,DI,TI")])
+
+(define_insn "*zero_extendsidi2_rex64_1"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=r,o,!?y,!*?")
+     (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "rm,0,rm,rm")))]
+  "TARGET_64BIT && TARGET_INTER_UNIT_MOVES"
+  "@
+   mov\t{%k1, %k0|%k0, %k1}
+   #
+   movd\t{%1, %0|%0, %1}
+   movd\t{%1, %0|%0, %1}"
+  [(set_attr "type" "imovx,imov,mmxmov,ssemov")
+   (set_attr "mode" "SI,DI,SI,SI")])
 
 (define_split
   [(set (match_operand:DI 0 "memory_operand" "")
   [(set (match_operand:DI 0 "nonimmediate_operand" "")
        (zero_extend:DI (match_operand:SI 1 "general_operand" "")))
    (clobber (reg:CC 17))]
-  "!TARGET_64BIT && reload_completed"
+  "!TARGET_64BIT && reload_completed
+   && !SSE_REG_P (operands[0]) && !MMX_REG_P (operands[0])"
   [(set (match_dup 3) (match_dup 1))
    (set (match_dup 4) (const_int 0))]
   "split_di (&operands[0], 1, &operands[3], &operands[4]);")
      (zero_extend:DI (match_operand:HI 1 "nonimmediate_operand" "r,m")))]
   "TARGET_64BIT"
   "@
-   movz{wl|x}\t{%1, %k0|%k0, %1} 
+   movz{wl|x}\t{%1, %k0|%k0, %1}
    movz{wq|x}\t{%1, %0|%0, %1}"
   [(set_attr "type" "imovx")
    (set_attr "mode" "SI,DI")])
      (zero_extend:DI (match_operand:QI 1 "nonimmediate_operand" "Q,m")))]
   "TARGET_64BIT"
   "@
-   movz{bl|x}\t{%1, %k0|%k0, %1} 
+   movz{bl|x}\t{%1, %k0|%k0, %1}
    movz{bq|x}\t{%1, %0|%0, %1}"
   [(set_attr "type" "imovx")
    (set_attr "mode" "SI,DI")])
 (define_split
   [(set (match_operand:XF 0 "push_operand" "")
        (float_extend:XF (match_operand:SF 1 "fp_register_operand" "")))]
-  "!TARGET_64BIT"
-  [(set (reg:SI 7) (plus:SI (reg:SI 7) (const_int -12)))
-   (set (mem:XF (reg:SI 7)) (float_extend:XF (match_dup 1)))])
-
-(define_insn "*dummy_extendsftf2"
-  [(set (match_operand:TF 0 "push_operand" "=<")
-       (float_extend:TF (match_operand:SF 1 "nonimmediate_operand" "f")))]
-  "0"
-  "#")
-
-(define_split
-  [(set (match_operand:TF 0 "push_operand" "")
-       (float_extend:TF (match_operand:SF 1 "fp_register_operand" "")))]
-  "!TARGET_64BIT"
-  [(set (reg:SI 7) (plus:SI (reg:SI 7) (const_int -16)))
-   (set (mem:TF (reg:SI 7)) (float_extend:TF (match_dup 1)))])
+  ""
+  [(set (reg:SI 7) (plus:SI (reg:SI 7) (match_dup 2)))
+   (set (mem:XF (reg:SI 7)) (float_extend:XF (match_dup 1)))]
+  "operands[2] = GEN_INT (TARGET_128BIT_LONG_DOUBLE ? -16 : -12);")
 
 (define_split
-  [(set (match_operand:TF 0 "push_operand" "")
-       (float_extend:TF (match_operand:SF 1 "fp_register_operand" "")))]
+  [(set (match_operand:XF 0 "push_operand" "")
+       (float_extend:XF (match_operand:SF 1 "fp_register_operand" "")))]
   "TARGET_64BIT"
-  [(set (reg:DI 7) (plus:DI (reg:DI 7) (const_int -16)))
-   (set (mem:DF (reg:DI 7)) (float_extend:TF (match_dup 1)))])
-
-(define_insn "*dummy_extenddfxf2"
-  [(set (match_operand:XF 0 "push_operand" "=<")
-       (float_extend:XF (match_operand:DF 1 "nonimmediate_operand" "f")))]
-  "0"
-  "#")
+  [(set (reg:DI 7) (plus:DI (reg:DI 7) (match_dup 2)))
+   (set (mem:DF (reg:DI 7)) (float_extend:XF (match_dup 1)))]
+  "operands[2] = GEN_INT (TARGET_128BIT_LONG_DOUBLE ? -16 : -12);")
 
 (define_split
   [(set (match_operand:XF 0 "push_operand" "")
        (float_extend:XF (match_operand:DF 1 "fp_register_operand" "")))]
-  "!TARGET_64BIT"
-  [(set (reg:SI 7) (plus:SI (reg:SI 7) (const_int -12)))
-   (set (mem:DF (reg:SI 7)) (float_extend:XF (match_dup 1)))])
-
-(define_insn "*dummy_extenddftf2"
-  [(set (match_operand:TF 0 "push_operand" "=<")
-       (float_extend:TF (match_operand:DF 1 "nonimmediate_operand" "f")))]
-  "0"
-  "#")
-
-(define_split
-  [(set (match_operand:TF 0 "push_operand" "")
-       (float_extend:TF (match_operand:DF 1 "fp_register_operand" "")))]
-  "!TARGET_64BIT"
-  [(set (reg:SI 7) (plus:SI (reg:SI 7) (const_int -16)))
-   (set (mem:TF (reg:SI 7)) (float_extend:XF (match_dup 1)))])
+  ""
+  [(set (reg:SI 7) (plus:SI (reg:SI 7) (match_dup 2)))
+   (set (mem:DF (reg:SI 7)) (float_extend:XF (match_dup 1)))]
+  "operands[2] = GEN_INT (TARGET_128BIT_LONG_DOUBLE ? -16 : -12);")
 
 (define_split
-  [(set (match_operand:TF 0 "push_operand" "")
-       (float_extend:TF (match_operand:DF 1 "fp_register_operand" "")))]
+  [(set (match_operand:XF 0 "push_operand" "")
+       (float_extend:XF (match_operand:DF 1 "fp_register_operand" "")))]
   "TARGET_64BIT"
-  [(set (reg:DI 7) (plus:DI (reg:DI 7) (const_int -16)))
-   (set (mem:TF (reg:DI 7)) (float_extend:TF (match_dup 1)))])
+  [(set (reg:DI 7) (plus:DI (reg:DI 7) (match_dup 2)))
+   (set (mem:XF (reg:DI 7)) (float_extend:XF (match_dup 1)))]
+  "operands[2] = GEN_INT (TARGET_128BIT_LONG_DOUBLE ? -16 : -12);")
 
 (define_expand "extendsfdf2"
   [(set (match_operand:DF 0 "nonimmediate_operand" "")
   switch (which_alternative)
     {
     case 0:
-      if (REG_P (operands[1])
-          && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-        return "fstp\t%y0";
-      else if (STACK_TOP_P (operands[0]))
-        return "fld%z1\t%y1";
-      else
-        return "fst\t%y0";
+      return output_387_reg_move (insn, operands);
 
     case 1:
       if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
         return "fstp%z0\t%y0";
-
       else
         return "fst%z0\t%y0";
+
     case 2:
       return "cvtss2sd\t{%1, %0|%0, %1}";
 
 (define_expand "extendsfxf2"
   [(set (match_operand:XF 0 "nonimmediate_operand" "")
         (float_extend:XF (match_operand:SF 1 "general_operand" "")))]
-  "!TARGET_64BIT && TARGET_80387"
+  "TARGET_80387"
 {
   /* ??? Needed for compress_float_constant since all fp constants
      are LEGITIMATE_CONSTANT_P.  */
 (define_insn "*extendsfxf2_1"
   [(set (match_operand:XF 0 "nonimmediate_operand" "=f,m")
         (float_extend:XF (match_operand:SF 1 "nonimmediate_operand" "fm,f")))]
-  "!TARGET_64BIT && TARGET_80387
+  "TARGET_80387
    && (GET_CODE (operands[0]) != MEM || GET_CODE (operands[1]) != MEM)"
 {
   switch (which_alternative)
     {
     case 0:
-      if (REG_P (operands[1])
-          && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-        return "fstp\t%y0";
-      else if (STACK_TOP_P (operands[0]))
-        return "fld%z1\t%y1";
-      else
-        return "fst\t%y0";
-
-    case 1:
-      /* There is no non-popping store to memory for XFmode.  So if
-        we need one, follow the store with a load.  */
-      if (! find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-        return "fstp%z0\t%y0\n\tfld%z0\t%y0";
-      else
-        return "fstp%z0\t%y0";
-
-    default:
-      abort ();
-    }
-}
-  [(set_attr "type" "fmov")
-   (set_attr "mode" "SF,XF")])
-
-(define_expand "extendsftf2"
-  [(set (match_operand:TF 0 "nonimmediate_operand" "")
-        (float_extend:TF (match_operand:SF 1 "general_operand" "")))]
-  "TARGET_80387"
-{
-  /* ??? Needed for compress_float_constant since all fp constants
-     are LEGITIMATE_CONSTANT_P.  */
-  if (GET_CODE (operands[1]) == CONST_DOUBLE)
-    operands[1] = validize_mem (force_const_mem (SFmode, operands[1]));
-  if (GET_CODE (operands[0]) == MEM && GET_CODE (operands[1]) == MEM)
-    operands[1] = force_reg (SFmode, operands[1]);
-})
-
-(define_insn "*extendsftf2_1"
-  [(set (match_operand:TF 0 "nonimmediate_operand" "=f,m")
-        (float_extend:TF (match_operand:SF 1 "nonimmediate_operand" "fm,f")))]
-  "TARGET_80387
-   && (GET_CODE (operands[0]) != MEM || GET_CODE (operands[1]) != MEM)"
-{
-  switch (which_alternative)
-    {
-    case 0:
-      if (REG_P (operands[1])
-          && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-        return "fstp\t%y0";
-      else if (STACK_TOP_P (operands[0]))
-        return "fld%z1\t%y1";
-      else
-        return "fst\t%y0";
+      return output_387_reg_move (insn, operands);
 
     case 1:
       /* There is no non-popping store to memory for XFmode.  So if
 (define_expand "extenddfxf2"
   [(set (match_operand:XF 0 "nonimmediate_operand" "")
         (float_extend:XF (match_operand:DF 1 "general_operand" "")))]
-  "!TARGET_64BIT && TARGET_80387"
+  "TARGET_80387"
 {
   /* ??? Needed for compress_float_constant since all fp constants
      are LEGITIMATE_CONSTANT_P.  */
 (define_insn "*extenddfxf2_1"
   [(set (match_operand:XF 0 "nonimmediate_operand" "=f,m")
         (float_extend:XF (match_operand:DF 1 "nonimmediate_operand" "fm,f")))]
-  "!TARGET_64BIT && TARGET_80387
-   && (GET_CODE (operands[0]) != MEM || GET_CODE (operands[1]) != MEM)"
-{
-  switch (which_alternative)
-    {
-    case 0:
-      if (REG_P (operands[1])
-          && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-        return "fstp\t%y0";
-      else if (STACK_TOP_P (operands[0]))
-        return "fld%z1\t%y1";
-      else
-        return "fst\t%y0";
-
-    case 1:
-      /* There is no non-popping store to memory for XFmode.  So if
-        we need one, follow the store with a load.  */
-      if (! find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-        return "fstp%z0\t%y0\n\tfld%z0\t%y0";
-      else
-        return "fstp%z0\t%y0";
-
-    default:
-      abort ();
-    }
-}
-  [(set_attr "type" "fmov")
-   (set_attr "mode" "DF,XF")])
-
-(define_expand "extenddftf2"
-  [(set (match_operand:TF 0 "nonimmediate_operand" "")
-        (float_extend:TF (match_operand:DF 1 "general_operand" "")))]
-  "TARGET_80387"
-{
-  /* ??? Needed for compress_float_constant since all fp constants
-     are LEGITIMATE_CONSTANT_P.  */
-  if (GET_CODE (operands[1]) == CONST_DOUBLE)
-    operands[1] = validize_mem (force_const_mem (DFmode, operands[1]));
-  if (GET_CODE (operands[0]) == MEM && GET_CODE (operands[1]) == MEM)
-    operands[1] = force_reg (DFmode, operands[1]);
-})
-
-(define_insn "*extenddftf2_1"
-  [(set (match_operand:TF 0 "nonimmediate_operand" "=f,m")
-        (float_extend:TF (match_operand:DF 1 "nonimmediate_operand" "fm,f")))]
   "TARGET_80387
    && (GET_CODE (operands[0]) != MEM || GET_CODE (operands[1]) != MEM)"
 {
   switch (which_alternative)
     {
     case 0:
-      if (REG_P (operands[1])
-          && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-        return "fstp\t%y0";
-      else if (STACK_TOP_P (operands[0]))
-        return "fld%z1\t%y1";
-      else
-        return "fst\t%y0";
+      return output_387_reg_move (insn, operands);
 
     case 1:
       /* There is no non-popping store to memory for XFmode.  So if
              (clobber (match_dup 2))])]
   "TARGET_80387 || TARGET_SSE2"
   "
-   if (TARGET_80387)
-     operands[2] = assign_386_stack_local (SFmode, 0);
-   else
+   if (!TARGET_80387)
      {
        emit_insn (gen_truncdfsf2_sse_only (operands[0], operands[1]));
        DONE;
      }
+   else if (flag_unsafe_math_optimizations)
+     {
+       rtx reg = REG_P (operands[0]) ? operands[0] : gen_reg_rtx (SFmode);
+       emit_insn (gen_truncdfsf2_noop (reg, operands[1]));
+       if (reg != operands[0])
+         emit_move_insn (operands[0], reg);
+       DONE;
+     }
+   else
+     operands[2] = assign_386_stack_local (SFmode, 0);
 ")
 
+(define_insn "truncdfsf2_noop"
+  [(set (match_operand:SF 0 "register_operand" "=f")
+       (float_truncate:SF (match_operand:DF 1 "register_operand" "f")))]
+  "TARGET_80387 && flag_unsafe_math_optimizations"
+{
+  return output_387_reg_move (insn, operands);
+}
+  [(set_attr "type" "fmov")
+   (set_attr "mode" "SF")])
+
 (define_insn "*truncdfsf2_1"
   [(set (match_operand:SF 0 "nonimmediate_operand" "=m,?f#rx,?r#fx,?x#rf")
        (float_truncate:SF
                   (float_truncate:SF
                    (match_operand:XF 1 "register_operand" "")))
              (clobber (match_dup 2))])]
-  "!TARGET_64BIT && TARGET_80387"
-  "operands[2] = assign_386_stack_local (SFmode, 0);")
-
-(define_insn "*truncxfsf2_1"
-  [(set (match_operand:SF 0 "nonimmediate_operand" "=m,?f#rx,?r#fx,?x#rf")
-       (float_truncate:SF
-        (match_operand:XF 1 "register_operand" "f,f,f,f")))
-   (clobber (match_operand:SF 2 "memory_operand" "=X,m,m,m"))]
-  "!TARGET_64BIT && TARGET_80387"
-{
-  switch (which_alternative)
+  "TARGET_80387"
+  "
+  if (flag_unsafe_math_optimizations)
     {
-    case 0:
-      if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-       return "fstp%z0\t%y0";
-      else
-       return "fst%z0\t%y0";
-    default:
-      abort();
+      rtx reg = REG_P (operands[0]) ? operands[0] : gen_reg_rtx (SFmode);
+      emit_insn (gen_truncxfsf2_noop (reg, operands[1]));
+      if (reg != operands[0])
+       emit_move_insn (operands[0], reg);
+      DONE;
     }
-}
-  [(set_attr "type" "fmov,multi,multi,multi")
-   (set_attr "mode" "SF")])
+  else
+    operands[2] = assign_386_stack_local (SFmode, 0);
+  ")
 
-(define_insn "*truncxfsf2_2"
-  [(set (match_operand:SF 0 "memory_operand" "=m")
-       (float_truncate:SF
-        (match_operand:XF 1 "register_operand" "f")))]
-  "!TARGET_64BIT && TARGET_80387"
+(define_insn "truncxfsf2_noop"
+  [(set (match_operand:SF 0 "register_operand" "=f")
+       (float_truncate:SF (match_operand:XF 1 "register_operand" "f")))]
+  "TARGET_80387 && flag_unsafe_math_optimizations"
 {
-  if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-    return "fstp%z0\t%y0";
-  else
-    return "fst%z0\t%y0";
+  return output_387_reg_move (insn, operands);
 }
   [(set_attr "type" "fmov")
    (set_attr "mode" "SF")])
 
-(define_split
-  [(set (match_operand:SF 0 "memory_operand" "")
-       (float_truncate:SF
-        (match_operand:XF 1 "register_operand" "")))
-   (clobber (match_operand:SF 2 "memory_operand" ""))]
-  "TARGET_80387"
-  [(set (match_dup 0) (float_truncate:SF (match_dup 1)))]
-  "")
-
-(define_split
-  [(set (match_operand:SF 0 "register_operand" "")
-       (float_truncate:SF
-        (match_operand:XF 1 "register_operand" "")))
-   (clobber (match_operand:SF 2 "memory_operand" ""))]
-  "TARGET_80387 && reload_completed"
-  [(set (match_dup 2) (float_truncate:SF (match_dup 1)))
-   (set (match_dup 0) (match_dup 2))]
-  "")
-
-(define_expand "trunctfsf2"
-  [(parallel [(set (match_operand:SF 0 "nonimmediate_operand" "")
-                  (float_truncate:SF
-                   (match_operand:TF 1 "register_operand" "")))
-             (clobber (match_dup 2))])]
-  "TARGET_80387"
-  "operands[2] = assign_386_stack_local (SFmode, 0);")
-
-(define_insn "*trunctfsf2_1"
+(define_insn "*truncxfsf2_1"
   [(set (match_operand:SF 0 "nonimmediate_operand" "=m,?f#rx,?r#fx,?x#rf")
        (float_truncate:SF
-        (match_operand:TF 1 "register_operand" "f,f,f,f")))
+        (match_operand:XF 1 "register_operand" "f,f,f,f")))
    (clobber (match_operand:SF 2 "memory_operand" "=X,m,m,m"))]
   "TARGET_80387"
 {
   [(set_attr "type" "fmov,multi,multi,multi")
    (set_attr "mode" "SF")])
 
-(define_insn "*trunctfsf2_2"
+(define_insn "*truncxfsf2_2"
   [(set (match_operand:SF 0 "memory_operand" "=m")
        (float_truncate:SF
-        (match_operand:TF 1 "register_operand" "f")))]
+        (match_operand:XF 1 "register_operand" "f")))]
   "TARGET_80387"
 {
   if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
 (define_split
   [(set (match_operand:SF 0 "memory_operand" "")
        (float_truncate:SF
-        (match_operand:TF 1 "register_operand" "")))
+        (match_operand:XF 1 "register_operand" "")))
    (clobber (match_operand:SF 2 "memory_operand" ""))]
   "TARGET_80387"
   [(set (match_dup 0) (float_truncate:SF (match_dup 1)))]
 (define_split
   [(set (match_operand:SF 0 "register_operand" "")
        (float_truncate:SF
-        (match_operand:TF 1 "register_operand" "")))
+        (match_operand:XF 1 "register_operand" "")))
    (clobber (match_operand:SF 2 "memory_operand" ""))]
   "TARGET_80387 && reload_completed"
   [(set (match_dup 2) (float_truncate:SF (match_dup 1)))
    (set (match_dup 0) (match_dup 2))]
   "")
 
-
 (define_expand "truncxfdf2"
   [(parallel [(set (match_operand:DF 0 "nonimmediate_operand" "")
                   (float_truncate:DF
                    (match_operand:XF 1 "register_operand" "")))
              (clobber (match_dup 2))])]
-  "!TARGET_64BIT && TARGET_80387"
-  "operands[2] = assign_386_stack_local (DFmode, 0);")
-
-(define_insn "*truncxfdf2_1"
-  [(set (match_operand:DF 0 "nonimmediate_operand" "=m,?f#rY,?r#fY,?Y#rf")
-       (float_truncate:DF
-        (match_operand:XF 1 "register_operand" "f,f,f,f")))
-   (clobber (match_operand:DF 2 "memory_operand" "=X,m,m,m"))]
-  "!TARGET_64BIT && TARGET_80387"
-{
-  switch (which_alternative)
+  "TARGET_80387"
+  "
+  if (flag_unsafe_math_optimizations)
     {
-    case 0:
-      if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-       return "fstp%z0\t%y0";
-      else
-       return "fst%z0\t%y0";
-    default:
-      abort();
+      rtx reg = REG_P (operands[0]) ? operands[0] : gen_reg_rtx (DFmode);
+      emit_insn (gen_truncxfdf2_noop (reg, operands[1]));
+      if (reg != operands[0])
+       emit_move_insn (operands[0], reg);
+      DONE;
     }
-  abort ();
-}
-  [(set_attr "type" "fmov,multi,multi,multi")
-   (set_attr "mode" "DF")])
+  else
+    operands[2] = assign_386_stack_local (DFmode, 0);
+  ")
 
-(define_insn "*truncxfdf2_2"
-  [(set (match_operand:DF 0 "memory_operand" "=m")
-       (float_truncate:DF
-         (match_operand:XF 1 "register_operand" "f")))]
-  "!TARGET_64BIT && TARGET_80387"
+(define_insn "truncxfdf2_noop"
+  [(set (match_operand:DF 0 "register_operand" "=f")
+       (float_truncate:DF (match_operand:XF 1 "register_operand" "f")))]
+  "TARGET_80387 && flag_unsafe_math_optimizations"
 {
-  if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
-    return "fstp%z0\t%y0";
-  else
-    return "fst%z0\t%y0";
+  return output_387_reg_move (insn, operands);
 }
   [(set_attr "type" "fmov")
    (set_attr "mode" "DF")])
 
-(define_split
-  [(set (match_operand:DF 0 "memory_operand" "")
-       (float_truncate:DF
-        (match_operand:XF 1 "register_operand" "")))
-   (clobber (match_operand:DF 2 "memory_operand" ""))]
-  "TARGET_80387"
-  [(set (match_dup 0) (float_truncate:DF (match_dup 1)))]
-  "")
-
-(define_split
-  [(set (match_operand:DF 0 "register_operand" "")
-       (float_truncate:DF
-        (match_operand:XF 1 "register_operand" "")))
-   (clobber (match_operand:DF 2 "memory_operand" ""))]
-  "TARGET_80387 && reload_completed"
-  [(set (match_dup 2) (float_truncate:DF (match_dup 1)))
-   (set (match_dup 0) (match_dup 2))]
-  "")
-
-(define_expand "trunctfdf2"
-  [(parallel [(set (match_operand:DF 0 "nonimmediate_operand" "")
-                  (float_truncate:DF
-                   (match_operand:TF 1 "register_operand" "")))
-             (clobber (match_dup 2))])]
-  "TARGET_80387"
-  "operands[2] = assign_386_stack_local (DFmode, 0);")
-
-(define_insn "*trunctfdf2_1"
+(define_insn "*truncxfdf2_1"
   [(set (match_operand:DF 0 "nonimmediate_operand" "=m,?f#rY,?r#fY,?Y#rf")
        (float_truncate:DF
-        (match_operand:TF 1 "register_operand" "f,f,f,f")))
+        (match_operand:XF 1 "register_operand" "f,f,f,f")))
    (clobber (match_operand:DF 2 "memory_operand" "=X,m,m,m"))]
   "TARGET_80387"
 {
   [(set_attr "type" "fmov,multi,multi,multi")
    (set_attr "mode" "DF")])
 
-       (define_insn "*trunctfdf2_2"
+(define_insn "*truncxfdf2_2"
   [(set (match_operand:DF 0 "memory_operand" "=m")
        (float_truncate:DF
-         (match_operand:TF 1 "register_operand" "f")))]
+         (match_operand:XF 1 "register_operand" "f")))]
   "TARGET_80387"
 {
   if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
 (define_split
   [(set (match_operand:DF 0 "memory_operand" "")
        (float_truncate:DF
-        (match_operand:TF 1 "register_operand" "")))
+        (match_operand:XF 1 "register_operand" "")))
    (clobber (match_operand:DF 2 "memory_operand" ""))]
   "TARGET_80387"
   [(set (match_dup 0) (float_truncate:DF (match_dup 1)))]
 (define_split
   [(set (match_operand:DF 0 "register_operand" "")
        (float_truncate:DF
-        (match_operand:TF 1 "register_operand" "")))
+        (match_operand:XF 1 "register_operand" "")))
    (clobber (match_operand:DF 2 "memory_operand" ""))]
   "TARGET_80387 && reload_completed"
   [(set (match_dup 2) (float_truncate:DF (match_dup 1)))
 ;; Signed conversion to DImode.
 
 (define_expand "fix_truncxfdi2"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "")
-        (fix:DI (match_operand:XF 1 "register_operand" "")))]
-  "!TARGET_64BIT && TARGET_80387"
-  "")
-
-(define_expand "fix_trunctfdi2"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "")
-       (fix:DI (match_operand:TF 1 "register_operand" "")))]
+  [(parallel [(set (match_operand:DI 0 "nonimmediate_operand" "")
+                   (fix:DI (match_operand:XF 1 "register_operand" "")))
+             (clobber (reg:CC 17))])]
   "TARGET_80387"
   "")
 
 (define_expand "fix_truncdfdi2"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "")
-        (fix:DI (match_operand:DF 1 "register_operand" "")))]
+  [(parallel [(set (match_operand:DI 0 "nonimmediate_operand" "")
+                   (fix:DI (match_operand:DF 1 "register_operand" "")))
+              (clobber (reg:CC 17))])]
   "TARGET_80387 || (TARGET_SSE2 && TARGET_64BIT)"
 {
   if (TARGET_64BIT && TARGET_SSE2)
 })
 
 (define_expand "fix_truncsfdi2"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "")
-       (fix:DI (match_operand:SF 1 "register_operand" "")))]
+  [(parallel [(set (match_operand:DI 0 "nonimmediate_operand" "")
+                  (fix:DI (match_operand:SF 1 "register_operand" "")))
+              (clobber (reg:CC 17))])] 
   "TARGET_80387 || (TARGET_SSE && TARGET_64BIT)"
 {
   if (TARGET_SSE && TARGET_64BIT)
 ;; of the machinery.
 (define_insn_and_split "*fix_truncdi_1"
   [(set (match_operand:DI 0 "nonimmediate_operand" "=m,?r")
-       (fix:DI (match_operand 1 "register_operand" "f,f")))]
+       (fix:DI (match_operand 1 "register_operand" "f,f")))
+   (clobber (reg:CC 17))]
   "TARGET_80387 && FLOAT_MODE_P (GET_MODE (operands[1]))
    && !reload_completed && !reload_in_progress
    && (!SSE_FLOAT_MODE_P (GET_MODE (operands[1])) || !TARGET_64BIT)"
 ;; Signed conversion to SImode.
 
 (define_expand "fix_truncxfsi2"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "")
-       (fix:SI (match_operand:XF 1 "register_operand" "")))]
-  "!TARGET_64BIT && TARGET_80387"
-  "")
-
-(define_expand "fix_trunctfsi2"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "")
-       (fix:SI (match_operand:TF 1 "register_operand" "")))]
+  [(parallel [(set (match_operand:SI 0 "nonimmediate_operand" "")
+                  (fix:SI (match_operand:XF 1 "register_operand" "")))
+             (clobber (reg:CC 17))])]
   "TARGET_80387"
   "")
 
 (define_expand "fix_truncdfsi2"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "")
-       (fix:SI (match_operand:DF 1 "register_operand" "")))]
+  [(parallel [(set (match_operand:SI 0 "nonimmediate_operand" "")
+                  (fix:SI (match_operand:DF 1 "register_operand" "")))
+             (clobber (reg:CC 17))])]
   "TARGET_80387 || TARGET_SSE2"
 {
   if (TARGET_SSE2)
 })
 
 (define_expand "fix_truncsfsi2"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "")
-       (fix:SI (match_operand:SF 1 "register_operand" "")))]
+  [(parallel [(set (match_operand:SI 0 "nonimmediate_operand" "")
+                  (fix:SI (match_operand:SF 1 "register_operand" "")))
+             (clobber (reg:CC 17))])] 
   "TARGET_80387 || TARGET_SSE"
 {
   if (TARGET_SSE)
 ;; of the machinery.
 (define_insn_and_split "*fix_truncsi_1"
   [(set (match_operand:SI 0 "nonimmediate_operand" "=m,?r")
-       (fix:SI (match_operand 1 "register_operand" "f,f")))]
+       (fix:SI (match_operand 1 "register_operand" "f,f")))
+   (clobber (reg:CC 17))]
   "TARGET_80387 && FLOAT_MODE_P (GET_MODE (operands[1]))
    && !reload_completed && !reload_in_progress
    && !SSE_FLOAT_MODE_P (GET_MODE (operands[1]))"
 ;; Signed conversion to HImode.
 
 (define_expand "fix_truncxfhi2"
-  [(set (match_operand:HI 0 "nonimmediate_operand" "")
-        (fix:HI (match_operand:XF 1 "register_operand" "")))]
-  "!TARGET_64BIT && TARGET_80387"
-  "")
-
-(define_expand "fix_trunctfhi2"
-  [(set (match_operand:HI 0 "nonimmediate_operand" "")
-       (fix:HI (match_operand:TF 1 "register_operand" "")))]
+  [(parallel [(set (match_operand:HI 0 "nonimmediate_operand" "")
+                   (fix:HI (match_operand:XF 1 "register_operand" "")))
+              (clobber (reg:CC 17))])] 
   "TARGET_80387"
   "")
 
 (define_expand "fix_truncdfhi2"
-  [(set (match_operand:HI 0 "nonimmediate_operand" "")
-       (fix:HI (match_operand:DF 1 "register_operand" "")))]
+  [(parallel [(set (match_operand:HI 0 "nonimmediate_operand" "")
+                  (fix:HI (match_operand:DF 1 "register_operand" "")))
+              (clobber (reg:CC 17))])]
   "TARGET_80387 && !TARGET_SSE2"
   "")
 
 (define_expand "fix_truncsfhi2"
-  [(set (match_operand:HI 0 "nonimmediate_operand" "")
-       (fix:HI (match_operand:SF 1 "register_operand" "")))]
+  [(parallel [(set (match_operand:HI 0 "nonimmediate_operand" "")
+                  (fix:HI (match_operand:SF 1 "register_operand" "")))
+               (clobber (reg:CC 17))])]
   "TARGET_80387 && !TARGET_SSE"
   "")
 
 ;; of the machinery.
 (define_insn_and_split "*fix_trunchi_1"
   [(set (match_operand:HI 0 "nonimmediate_operand" "=m,?r")
-       (fix:HI (match_operand 1 "register_operand" "f,f")))]
+       (fix:HI (match_operand 1 "register_operand" "f,f")))
+   (clobber (reg:CC 17))]
   "TARGET_80387 && FLOAT_MODE_P (GET_MODE (operands[1]))
    && !reload_completed && !reload_in_progress
    && !SSE_FLOAT_MODE_P (GET_MODE (operands[1]))"
   "fnstcw\t%0"
   [(set_attr "length" "2")
    (set_attr "mode" "HI")
-   (set_attr "unit" "i387")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "unit" "i387")])
 
 (define_insn "x86_fldcw_1"
   [(set (reg:HI 18)
   [(set_attr "length" "2")
    (set_attr "mode" "HI")
    (set_attr "unit" "i387")
-   (set_attr "athlon_decode" "vector")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "athlon_decode" "vector")])
 \f
 ;; Conversion between fixed point and floating point.
 
 (define_insn "floathixf2"
   [(set (match_operand:XF 0 "register_operand" "=f,f")
        (float:XF (match_operand:HI 1 "nonimmediate_operand" "m,r")))]
-  "!TARGET_64BIT && TARGET_80387"
+  "TARGET_80387"
   "@
    fild%z1\t%1
    #"
    (set_attr "mode" "XF")
    (set_attr "fp_int_src" "true")])
 
-(define_insn "floathitf2"
-  [(set (match_operand:TF 0 "register_operand" "=f,f")
-       (float:TF (match_operand:HI 1 "nonimmediate_operand" "m,r")))]
+(define_insn "floatsixf2"
+  [(set (match_operand:XF 0 "register_operand" "=f,f")
+       (float:XF (match_operand:SI 1 "nonimmediate_operand" "m,r")))]
   "TARGET_80387"
   "@
    fild%z1\t%1
    (set_attr "mode" "XF")
    (set_attr "fp_int_src" "true")])
 
-(define_insn "floatsixf2"
+(define_insn "floatdixf2"
   [(set (match_operand:XF 0 "register_operand" "=f,f")
-       (float:XF (match_operand:SI 1 "nonimmediate_operand" "m,r")))]
-  "!TARGET_64BIT && TARGET_80387"
-  "@
-   fild%z1\t%1
-   #"
-  [(set_attr "type" "fmov,multi")
-   (set_attr "mode" "XF")
-   (set_attr "fp_int_src" "true")])
-
-(define_insn "floatsitf2"
-  [(set (match_operand:TF 0 "register_operand" "=f,f")
-       (float:TF (match_operand:SI 1 "nonimmediate_operand" "m,r")))]
-  "TARGET_80387"
-  "@
-   fild%z1\t%1
-   #"
-  [(set_attr "type" "fmov,multi")
-   (set_attr "mode" "XF")
-   (set_attr "fp_int_src" "true")])
-
-(define_insn "floatdixf2"
-  [(set (match_operand:XF 0 "register_operand" "=f,f")
-       (float:XF (match_operand:DI 1 "nonimmediate_operand" "m,r")))]
-  "!TARGET_64BIT && TARGET_80387"
-  "@
-   fild%z1\t%1
-   #"
-  [(set_attr "type" "fmov,multi")
-   (set_attr "mode" "XF")
-   (set_attr "fp_int_src" "true")])
-
-(define_insn "floatditf2"
-  [(set (match_operand:TF 0 "register_operand" "=f,f")
-       (float:TF (match_operand:DI 1 "nonimmediate_operand" "m,r")))]
-  "TARGET_80387"
+       (float:XF (match_operand:DI 1 "nonimmediate_operand" "m,r")))]
+  "TARGET_80387"
   "@
    fild%z1\t%1
    #"
   "TARGET_SSE2 && TARGET_SSE_MATH && TARGET_64BIT"
   "x86_emit_floatuns (operands); DONE;")
 \f
+;; SSE extract/set expanders
+
+(define_expand "vec_setv2df"
+  [(match_operand:V2DF 0 "register_operand" "")
+   (match_operand:DF 1 "register_operand" "")
+   (match_operand 2 "const_int_operand" "")]
+  "TARGET_SSE2"
+{
+  switch (INTVAL (operands[2]))
+    {
+    case 0:
+      emit_insn (gen_sse2_movsd (operands[0], operands[0],
+                                simplify_gen_subreg (V2DFmode, operands[1],
+                                                     DFmode, 0)));
+      break;
+    case 1:
+      {
+       rtx op1 = simplify_gen_subreg (V2DFmode, operands[1], DFmode, 0);
+
+       emit_insn (gen_sse2_unpcklpd (operands[0], operands[0], op1));
+      }
+      break;
+    default:
+      abort ();
+    }
+  DONE;
+})
+
+(define_expand "vec_extractv2df"
+  [(match_operand:DF 0 "register_operand" "")
+   (match_operand:V2DF 1 "register_operand" "")
+   (match_operand 2 "const_int_operand" "")]
+  "TARGET_SSE2"
+{
+  switch (INTVAL (operands[2]))
+    {
+    case 0:
+      emit_move_insn (operands[0], gen_lowpart (DFmode, operands[1]));
+      break;
+    case 1:
+      {
+       rtx dest = simplify_gen_subreg (V2DFmode, operands[0], DFmode, 0);
+
+       emit_insn (gen_sse2_unpckhpd (dest, operands[1], operands[1]));
+      }
+      break;
+    default:
+      abort ();
+    }
+  DONE;
+})
+
+(define_expand "vec_initv2df"
+  [(match_operand:V2DF 0 "register_operand" "")
+   (match_operand 1 "" "")]
+  "TARGET_SSE2"
+{
+  ix86_expand_vector_init (operands[0], operands[1]);
+  DONE;
+})
+
+(define_expand "vec_setv4sf"
+  [(match_operand:V4SF 0 "register_operand" "")
+   (match_operand:SF 1 "register_operand" "")
+   (match_operand 2 "const_int_operand" "")]
+  "TARGET_SSE"
+{
+  switch (INTVAL (operands[2]))
+    {
+    case 0:
+      emit_insn (gen_sse_movss (operands[0], operands[0],
+                               simplify_gen_subreg (V4SFmode, operands[1],
+                                                    SFmode, 0)));
+      break;
+    case 1:
+      {
+       rtx op1 = simplify_gen_subreg (V4SFmode, operands[1], SFmode, 0);
+       rtx tmp = gen_reg_rtx (V4SFmode);
+        emit_move_insn (tmp, operands[0]);
+       emit_insn (gen_sse_unpcklps (operands[0], operands[0], operands[0]));
+       emit_insn (gen_sse_movss (operands[0], operands[0], op1));
+        emit_insn (gen_sse_shufps (operands[0], operands[0], tmp,
+                                   GEN_INT (1 + (0<<2) + (2<<4) + (3<<6))));
+      }
+    case 2:
+      {
+        rtx op1 = simplify_gen_subreg (V4SFmode, operands[1], SFmode, 0);
+        rtx tmp = gen_reg_rtx (V4SFmode);
+
+        emit_move_insn (tmp, operands[0]);
+        emit_insn (gen_sse_movss (tmp, tmp, op1));
+        emit_insn (gen_sse_shufps (operands[0], operands[0], tmp,
+                                   GEN_INT (0 + (1<<2) + (0<<4) + (3<<6))));
+      }
+      break;
+    case 3:
+      {
+        rtx op1 = simplify_gen_subreg (V4SFmode, operands[1], SFmode, 0);
+        rtx tmp = gen_reg_rtx (V4SFmode);
+
+        emit_move_insn (tmp, operands[0]);
+        emit_insn (gen_sse_movss (tmp, tmp, op1));
+        emit_insn (gen_sse_shufps (operands[0], operands[0], tmp,
+                                   GEN_INT (0 + (1<<2) + (2<<4) + (0<<6))));
+      }
+      break;
+    default:
+      abort ();
+    }
+  DONE;
+})
+
+(define_expand "vec_extractv4sf"
+  [(match_operand:SF 0 "register_operand" "")
+   (match_operand:V4SF 1 "register_operand" "")
+   (match_operand 2 "const_int_operand" "")]
+  "TARGET_SSE"
+{
+  switch (INTVAL (operands[2]))
+    {
+    case 0:
+      emit_move_insn (operands[0], gen_lowpart (SFmode, operands[1]));
+      break;
+    case 1:
+      {
+       rtx op0 = simplify_gen_subreg (V4SFmode, operands[1], SFmode, 0);
+       rtx tmp = gen_reg_rtx (V4SFmode);
+        emit_move_insn (tmp, operands[1]);
+        emit_insn (gen_sse_shufps (op0, tmp, tmp,
+                                   const1_rtx));
+      }
+    case 2:
+      {
+       rtx op0 = simplify_gen_subreg (V4SFmode, operands[1], SFmode, 0);
+       rtx tmp = gen_reg_rtx (V4SFmode);
+        emit_move_insn (tmp, operands[1]);
+        emit_insn (gen_sse_unpckhps (op0, tmp, tmp));
+      }
+    case 3:
+      {
+       rtx op0 = simplify_gen_subreg (V4SFmode, operands[1], SFmode, 0);
+       rtx tmp = gen_reg_rtx (V4SFmode);
+        emit_move_insn (tmp, operands[1]);
+        emit_insn (gen_sse_shufps (op0, tmp, tmp,
+                                   GEN_INT (3)));
+      }
+    default:
+      abort ();
+    }
+  DONE;
+})
+
+(define_expand "vec_initv4sf"
+  [(match_operand:V4SF 0 "register_operand" "")
+   (match_operand 1 "" "")]
+  "TARGET_SSE"
+{
+  ix86_expand_vector_init (operands[0], operands[1]);
+  DONE;
+})
+\f
 ;; Add instructions
 
 ;; %%% splits for addsidi3
   "adc{q}\t{%2, %0|%0, %2}"
   [(set_attr "type" "alu")
    (set_attr "pent_pair" "pu")
-   (set_attr "mode" "DI")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "mode" "DI")])
 
 (define_insn "*adddi3_cc_rex64"
   [(set (reg:CC 17)
    (set_attr "mode" "DI")])
 
 (define_insn "addqi3_carry"
-  [(set (match_operand:QI 0 "nonimmediate_operand" "=rm,r")
+  [(set (match_operand:QI 0 "nonimmediate_operand" "=qm,q")
          (plus:QI (plus:QI (match_operand:QI 3 "ix86_carry_flag_operator" "")
                            (match_operand:QI 1 "nonimmediate_operand" "%0,0"))
-                  (match_operand:QI 2 "general_operand" "ri,rm")))
+                  (match_operand:QI 2 "general_operand" "qi,qm")))
    (clobber (reg:CC 17))]
   "ix86_binary_operator_ok (PLUS, QImode, operands)"
   "adc{b}\t{%2, %0|%0, %2}"
   [(set_attr "type" "alu")
    (set_attr "pent_pair" "pu")
-   (set_attr "mode" "QI")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "mode" "QI")])
 
 (define_insn "addhi3_carry"
   [(set (match_operand:HI 0 "nonimmediate_operand" "=rm,r")
   "adc{w}\t{%2, %0|%0, %2}"
   [(set_attr "type" "alu")
    (set_attr "pent_pair" "pu")
-   (set_attr "mode" "HI")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "mode" "HI")])
 
 (define_insn "addsi3_carry"
   [(set (match_operand:SI 0 "nonimmediate_operand" "=rm,r")
   "adc{l}\t{%2, %0|%0, %2}"
   [(set_attr "type" "alu")
    (set_attr "pent_pair" "pu")
-   (set_attr "mode" "SI")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "mode" "SI")])
 
 (define_insn "*addsi3_carry_zext"
   [(set (match_operand:DI 0 "register_operand" "=r")
   "adc{l}\t{%2, %k0|%k0, %2}"
   [(set_attr "type" "alu")
    (set_attr "pent_pair" "pu")
-   (set_attr "mode" "SI")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "mode" "SI")])
 
 (define_insn "*addsi3_cc"
   [(set (reg:CC 17)
 
 (define_insn "*lea_1"
   [(set (match_operand:SI 0 "register_operand" "=r")
-       (match_operand:SI 1 "address_operand" "p"))]
+       (match_operand:SI 1 "no_seg_address_operand" "p"))]
   "!TARGET_64BIT"
   "lea{l}\t{%a1, %0|%0, %a1}"
   [(set_attr "type" "lea")
 
 (define_insn "*lea_1_rex64"
   [(set (match_operand:SI 0 "register_operand" "=r")
-       (subreg:SI (match_operand:DI 1 "address_operand" "p") 0))]
+       (subreg:SI (match_operand:DI 1 "no_seg_address_operand" "p") 0))]
   "TARGET_64BIT"
   "lea{l}\t{%a1, %0|%0, %a1}"
   [(set_attr "type" "lea")
 
 (define_insn "*lea_1_zext"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (zero_extend:DI (subreg:SI (match_operand:DI 1 "address_operand" "p") 0)))]
+       (zero_extend:DI
+        (subreg:SI (match_operand:DI 1 "no_seg_address_operand" "p") 0)))]
   "TARGET_64BIT"
   "lea{l}\t{%a1, %k0|%k0, %a1}"
   [(set_attr "type" "lea")
 
 (define_insn "*lea_2_rex64"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (match_operand:DI 1 "address_operand" "p"))]
+       (match_operand:DI 1 "no_seg_address_operand" "p"))]
   "TARGET_64BIT"
   "lea{q}\t{%a1, %0|%0, %a1}"
   [(set_attr "type" "lea")
       if (GET_CODE (operands[1]) == CONST_INT
          && INTVAL (operands[1]) < 0)
        {
-         operands[2] = GEN_INT (-INTVAL (operands[2]));
+         operands[1] = GEN_INT (-INTVAL (operands[1]));
          return "sub{b}\t{%1, %0|%0, %1}";
        }
       return "add{b}\t{%1, %0|%0, %1}";
   [(set (match_operand:XF 0 "register_operand" "")
        (plus:XF (match_operand:XF 1 "register_operand" "")
                 (match_operand:XF 2 "register_operand" "")))]
-  "!TARGET_64BIT && TARGET_80387"
-  "")
-
-(define_expand "addtf3"
-  [(set (match_operand:TF 0 "register_operand" "")
-       (plus:TF (match_operand:TF 1 "register_operand" "")
-                (match_operand:TF 2 "register_operand" "")))]
   "TARGET_80387"
   "")
 
   "sbb{q}\t{%2, %0|%0, %2}"
   [(set_attr "type" "alu")
    (set_attr "pent_pair" "pu")
-   (set_attr "ppro_uops" "few")
    (set_attr "mode" "DI")])
 
 (define_insn "*subdi_1_rex64"
    (set_attr "mode" "DI")])
 
 (define_insn "subqi3_carry"
-  [(set (match_operand:QI 0 "nonimmediate_operand" "=rm,r")
+  [(set (match_operand:QI 0 "nonimmediate_operand" "=qm,q")
          (minus:QI (match_operand:QI 1 "nonimmediate_operand" "0,0")
            (plus:QI (match_operand:QI 3 "ix86_carry_flag_operator" "")
-              (match_operand:QI 2 "general_operand" "ri,rm"))))
+              (match_operand:QI 2 "general_operand" "qi,qm"))))
    (clobber (reg:CC 17))]
   "ix86_binary_operator_ok (MINUS, QImode, operands)"
   "sbb{b}\t{%2, %0|%0, %2}"
   [(set_attr "type" "alu")
    (set_attr "pent_pair" "pu")
-   (set_attr "ppro_uops" "few")
    (set_attr "mode" "QI")])
 
 (define_insn "subhi3_carry"
   "sbb{w}\t{%2, %0|%0, %2}"
   [(set_attr "type" "alu")
    (set_attr "pent_pair" "pu")
-   (set_attr "ppro_uops" "few")
    (set_attr "mode" "HI")])
 
 (define_insn "subsi3_carry"
   "sbb{l}\t{%2, %0|%0, %2}"
   [(set_attr "type" "alu")
    (set_attr "pent_pair" "pu")
-   (set_attr "ppro_uops" "few")
    (set_attr "mode" "SI")])
 
 (define_insn "subsi3_carry_zext"
   "sbb{l}\t{%2, %k0|%k0, %2}"
   [(set_attr "type" "alu")
    (set_attr "pent_pair" "pu")
-   (set_attr "ppro_uops" "few")
    (set_attr "mode" "SI")])
 
 (define_expand "subsi3"
 
 (define_insn "*subsi_3_zext"
   [(set (reg 17)
-       (compare (match_operand:SI 1 "nonimmediate_operand" "0")
+       (compare (match_operand:SI 1 "register_operand" "0")
                 (match_operand:SI 2 "general_operand" "rim")))
    (set (match_operand:DI 0 "register_operand" "=r")
        (zero_extend:DI
   [(set (match_operand:XF 0 "register_operand" "")
        (minus:XF (match_operand:XF 1 "register_operand" "")
                  (match_operand:XF 2 "register_operand" "")))]
-  "!TARGET_64BIT && TARGET_80387"
-  "")
-
-(define_expand "subtf3"
-  [(set (match_operand:TF 0 "register_operand" "")
-       (minus:TF (match_operand:TF 1 "register_operand" "")
-                 (match_operand:TF 2 "register_operand" "")))]
   "TARGET_80387"
   "")
 
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "mul{q}\t%2"
   [(set_attr "type" "imul")
-   (set_attr "ppro_uops" "few")
    (set_attr "length_immediate" "0")
    (set (attr "athlon_decode")
      (if_then_else (eq_attr "cpu" "athlon")
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "mul{l}\t%2"
   [(set_attr "type" "imul")
-   (set_attr "ppro_uops" "few")
    (set_attr "length_immediate" "0")
    (set (attr "athlon_decode")
      (if_then_else (eq_attr "cpu" "athlon")
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "mul{q}\t%2"
   [(set_attr "type" "imul")
-   (set_attr "ppro_uops" "few")
    (set_attr "length_immediate" "0")
    (set (attr "athlon_decode")
      (if_then_else (eq_attr "cpu" "athlon")
   "GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM"
   "mul{l}\t%2"
   [(set_attr "type" "imul")
-   (set_attr "ppro_uops" "few")
    (set_attr "length_immediate" "0")
    (set (attr "athlon_decode")
      (if_then_else (eq_attr "cpu" "athlon")
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "mul{l}\t%2"
   [(set_attr "type" "imul")
-   (set_attr "ppro_uops" "few")
    (set_attr "length_immediate" "0")
    (set (attr "athlon_decode")
      (if_then_else (eq_attr "cpu" "athlon")
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "imul{q}\t%2"
   [(set_attr "type" "imul")
-   (set_attr "ppro_uops" "few")
    (set (attr "athlon_decode")
      (if_then_else (eq_attr "cpu" "athlon")
         (const_string "vector")
   "GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM"
   "imul{l}\t%2"
   [(set_attr "type" "imul")
-   (set_attr "ppro_uops" "few")
    (set (attr "athlon_decode")
      (if_then_else (eq_attr "cpu" "athlon")
         (const_string "vector")
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "imul{l}\t%2"
   [(set_attr "type" "imul")
-   (set_attr "ppro_uops" "few")
    (set (attr "athlon_decode")
      (if_then_else (eq_attr "cpu" "athlon")
         (const_string "vector")
   [(set (match_operand:XF 0 "register_operand" "")
        (mult:XF (match_operand:XF 1 "register_operand" "")
                 (match_operand:XF 2 "register_operand" "")))]
-  "!TARGET_64BIT && TARGET_80387"
-  "")
-
-(define_expand "multf3"
-  [(set (match_operand:TF 0 "register_operand" "")
-       (mult:TF (match_operand:TF 1 "register_operand" "")
-                (match_operand:TF 2 "register_operand" "")))]
   "TARGET_80387"
   "")
 
   "TARGET_QIMODE_MATH"
   "idiv{b}\t%2"
   [(set_attr "type" "idiv")
-   (set_attr "mode" "QI")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "mode" "QI")])
 
 (define_insn "udivqi3"
   [(set (match_operand:QI 0 "register_operand" "=a")
   "TARGET_QIMODE_MATH"
   "div{b}\t%2"
   [(set_attr "type" "idiv")
-   (set_attr "mode" "QI")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "mode" "QI")])
 
 ;; The patterns that match these are at the end of this file.
 
   [(set (match_operand:XF 0 "register_operand" "")
        (div:XF (match_operand:XF 1 "register_operand" "")
                (match_operand:XF 2 "register_operand" "")))]
-  "!TARGET_64BIT && TARGET_80387"
-  "")
-
-(define_expand "divtf3"
-  [(set (match_operand:TF 0 "register_operand" "")
-       (div:TF (match_operand:TF 1 "register_operand" "")
-               (match_operand:TF 2 "register_operand" "")))]
   "TARGET_80387"
   "")
 
   "TARGET_64BIT"
   "idiv{q}\t%2"
   [(set_attr "type" "idiv")
-   (set_attr "mode" "DI")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "mode" "DI")])
 
 (define_split
   [(set (match_operand:DI 0 "register_operand" "")
   ""
   "idiv{l}\t%2"
   [(set_attr "type" "idiv")
-   (set_attr "mode" "SI")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "mode" "SI")])
 
 (define_split
   [(set (match_operand:SI 0 "register_operand" "")
   "TARGET_64BIT"
   "div{q}\t%2"
   [(set_attr "type" "idiv")
-   (set_attr "ppro_uops" "few")
    (set_attr "mode" "DI")])
 
 (define_split
   ""
   "div{l}\t%2"
   [(set_attr "type" "idiv")
-   (set_attr "ppro_uops" "few")
    (set_attr "mode" "SI")])
 
 (define_split
   ""
   "div{w}\t%2"
   [(set_attr "type" "idiv")
-   (set_attr "mode" "HI")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "mode" "HI")])
 
 ;; We can not use div/idiv for double division, because it causes
 ;; "division by zero" on the overflow and that's not what we expect
 ;   (clobber (reg:CC 17))]
 ;  ""
 ;  "div{l}\t{%2, %0|%0, %2}"
-;  [(set_attr "type" "idiv")
-;   (set_attr "ppro_uops" "few")])
+;  [(set_attr "type" "idiv")])
 \f
 ;;- Logical AND instructions
 
   "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
    && (GET_CODE (operands[0]) != MEM || GET_CODE (operands[1]) != MEM)"
   "@
-   test{l}\t{%k1, %k0|%k0, %k1} 
-   test{l}\t{%k1, %k0|%k0, %k1} 
-   test{q}\t{%1, %0|%0, %1} 
-   test{q}\t{%1, %0|%0, %1} 
+   test{l}\t{%k1, %k0|%k0, %k1}
+   test{l}\t{%k1, %k0|%k0, %k1}
+   test{q}\t{%1, %0|%0, %1}
+   test{q}\t{%1, %0|%0, %1}
    test{q}\t{%1, %0|%0, %1}"
   [(set_attr "type" "test")
    (set_attr "modrm" "0,1,0,1,1")
   "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
    && ix86_binary_operator_ok (AND, DImode, operands)"
   "@
-   and{l}\t{%k2, %k0|%k0, %k2} 
-   and{q}\t{%2, %0|%0, %2} 
+   and{l}\t{%k2, %k0|%k0, %k2}
+   and{q}\t{%2, %0|%0, %2}
    and{q}\t{%2, %0|%0, %2}"
   [(set_attr "type" "alu")
    (set_attr "mode" "SI,DI,DI")])
   "TARGET_64BIT
    && ix86_binary_operator_ok (XOR, DImode, operands)"
   "@
-   xor{q}\t{%2, %0|%0, %2} 
+   xor{q}\t{%2, %0|%0, %2}
    xor{q}\t{%2, %0|%0, %2}"
   [(set_attr "type" "alu")
    (set_attr "mode" "DI,DI")])
    && ix86_match_ccmode (insn, CCNOmode)
    && ix86_binary_operator_ok (XOR, DImode, operands)"
   "@
-   xor{q}\t{%2, %0|%0, %2} 
+   xor{q}\t{%2, %0|%0, %2}
    xor{q}\t{%2, %0|%0, %2}"
   [(set_attr "type" "alu")
    (set_attr "mode" "DI,DI")])
    (use (match_operand:V4SF 2 "nonimmediate_operand" ""))
    (clobber (reg:CC 17))]
   "reload_completed && SSE_REG_P (operands[0])"
-  [(set (subreg:TI (match_dup 0) 0)
-       (xor:TI (match_dup 1)
-               (match_dup 2)))]
+  [(set (match_dup 0)
+       (xor:V4SF (match_dup 1)
+                 (match_dup 2)))]
 {
-  operands[1] = simplify_gen_subreg (TImode, operands[1], SFmode, 0);
-  operands[2] = simplify_gen_subreg (TImode, operands[2], V4SFmode, 0);
+  operands[0] = simplify_gen_subreg (V4SFmode, operands[0], SFmode, 0);
+  operands[1] = simplify_gen_subreg (V4SFmode, operands[1], SFmode, 0);
   if (operands_match_p (operands[0], operands[2]))
     {
       rtx tmp;
 {
   int size = GET_MODE_SIZE (GET_MODE (operands[1]));
 
-  /* XFmode's size is 12, TFmode 16, but only 10 bytes are used.  */
-  if (size >= 12)
+  if (GET_MODE (operands[1]) == XFmode)
     size = 10;
   operands[0] = adjust_address (operands[0], QImode, size - 1);
   operands[1] = gen_int_mode (0x80, QImode);
    (use (match_operand:V2DF 2 "nonimmediate_operand" ""))
    (clobber (reg:CC 17))]
   "reload_completed && SSE_REG_P (operands[0])"
-  [(set (subreg:TI (match_dup 0) 0)
-       (xor:TI (match_dup 1)
-               (match_dup 2)))]
+  [(set (match_dup 0)
+       (xor:V2DF (match_dup 1)
+                 (match_dup 2)))]
 {
   operands[0] = simplify_gen_subreg (V2DFmode, operands[0], DFmode, 0);
-  operands[1] = simplify_gen_subreg (TImode, operands[1], DFmode, 0);
-  operands[2] = simplify_gen_subreg (TImode, operands[2], V2DFmode, 0);
+  operands[1] = simplify_gen_subreg (V2DFmode, operands[1], DFmode, 0);
   /* Avoid possible reformatting on the operands.  */
   if (TARGET_SSE_PARTIAL_REGS && !optimize_size)
     emit_insn (gen_sse2_unpcklpd (operands[0], operands[0], operands[0]));
   [(parallel [(set (match_operand:XF 0 "nonimmediate_operand" "")
                   (neg:XF (match_operand:XF 1 "nonimmediate_operand" "")))
              (clobber (reg:CC 17))])]
-  "!TARGET_64BIT && TARGET_80387"
-  "ix86_expand_unary_operator (NEG, XFmode, operands); DONE;")
-
-(define_expand "negtf2"
-  [(parallel [(set (match_operand:TF 0 "nonimmediate_operand" "")
-                  (neg:TF (match_operand:TF 1 "nonimmediate_operand" "")))
-             (clobber (reg:CC 17))])]
   "TARGET_80387"
-  "ix86_expand_unary_operator (NEG, TFmode, operands); DONE;")
+  "ix86_expand_unary_operator (NEG, XFmode, operands); DONE;")
 
 ;; Keep 'f' and 'r' in separate alternatives to avoid reload problems
 ;; because of secondary memory needed to reload from class FLOAT_INT_REGS
   [(set (match_operand:XF 0 "nonimmediate_operand" "=f#r,rm#f")
        (neg:XF (match_operand:XF 1 "nonimmediate_operand" "0,0")))
    (clobber (reg:CC 17))]
-  "!TARGET_64BIT && TARGET_80387
+  "TARGET_80387
    && ix86_unary_operator_ok (NEG, XFmode, operands)"
   "#")
 
    operands[0] = gen_rtx_REG (SImode,
                              true_regnum (operands[0]) + (TARGET_64BIT ? 1 : 2));")
 
-;; Keep 'f' and 'r' in separate alternatives to avoid reload problems
-;; because of secondary memory needed to reload from class FLOAT_INT_REGS
-;; to itself.
-(define_insn "*negtf2_if"
-  [(set (match_operand:TF 0 "nonimmediate_operand" "=f#r,rm#f")
-       (neg:TF (match_operand:TF 1 "nonimmediate_operand" "0,0")))
-   (clobber (reg:CC 17))]
-  "TARGET_80387 && ix86_unary_operator_ok (NEG, TFmode, operands)"
-  "#")
-
-(define_split
-  [(set (match_operand:TF 0 "fp_register_operand" "")
-       (neg:TF (match_operand:TF 1 "register_operand" "")))
-   (clobber (reg:CC 17))]
-  "TARGET_80387 && reload_completed"
-  [(set (match_dup 0)
-       (neg:TF (match_dup 1)))]
-  "")
-
-(define_split
-  [(set (match_operand:TF 0 "register_and_not_fp_reg_operand" "")
-       (neg:TF (match_operand:TF 1 "register_operand" "")))
-   (clobber (reg:CC 17))]
-  "TARGET_80387 && reload_completed"
-  [(parallel [(set (match_dup 0) (xor:SI (match_dup 0) (match_dup 1)))
-             (clobber (reg:CC 17))])]
-  "operands[1] = GEN_INT (0x8000);
-   operands[0] = gen_rtx_REG (SImode,
-                             true_regnum (operands[0]) + (TARGET_64BIT ? 1 : 2));")
-
 ;; Conditionalize these after reload. If they matches before reload, we 
 ;; lose the clobber and ability to use integer instructions.
 
   "TARGET_80387 && reload_completed"
   "fchs"
   [(set_attr "type" "fsgn")
-   (set_attr "mode" "SF")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "mode" "SF")])
 
 (define_insn "*negdf2_1"
   [(set (match_operand:DF 0 "register_operand" "=f")
   "TARGET_80387 && reload_completed"
   "fchs"
   [(set_attr "type" "fsgn")
-   (set_attr "mode" "DF")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "mode" "DF")])
 
 (define_insn "*negextendsfdf2"
   [(set (match_operand:DF 0 "register_operand" "=f")
   "TARGET_80387"
   "fchs"
   [(set_attr "type" "fsgn")
-   (set_attr "mode" "DF")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "mode" "DF")])
 
 (define_insn "*negxf2_1"
   [(set (match_operand:XF 0 "register_operand" "=f")
        (neg:XF (match_operand:XF 1 "register_operand" "0")))]
-  "!TARGET_64BIT && TARGET_80387 && reload_completed"
+  "TARGET_80387 && reload_completed"
   "fchs"
   [(set_attr "type" "fsgn")
-   (set_attr "mode" "XF")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "mode" "XF")])
 
 (define_insn "*negextenddfxf2"
   [(set (match_operand:XF 0 "register_operand" "=f")
        (neg:XF (float_extend:XF
                  (match_operand:DF 1 "register_operand" "0"))))]
-  "!TARGET_64BIT && TARGET_80387"
+  "TARGET_80387"
   "fchs"
   [(set_attr "type" "fsgn")
-   (set_attr "mode" "XF")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "mode" "XF")])
 
 (define_insn "*negextendsfxf2"
   [(set (match_operand:XF 0 "register_operand" "=f")
        (neg:XF (float_extend:XF
                  (match_operand:SF 1 "register_operand" "0"))))]
-  "!TARGET_64BIT && TARGET_80387"
-  "fchs"
-  [(set_attr "type" "fsgn")
-   (set_attr "mode" "XF")
-   (set_attr "ppro_uops" "few")])
-
-(define_insn "*negtf2_1"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (neg:TF (match_operand:TF 1 "register_operand" "0")))]
-  "TARGET_80387 && reload_completed"
-  "fchs"
-  [(set_attr "type" "fsgn")
-   (set_attr "mode" "XF")
-   (set_attr "ppro_uops" "few")])
-
-(define_insn "*negextenddftf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (neg:TF (float_extend:TF
-                 (match_operand:DF 1 "register_operand" "0"))))]
-  "TARGET_80387"
-  "fchs"
-  [(set_attr "type" "fsgn")
-   (set_attr "mode" "XF")
-   (set_attr "ppro_uops" "few")])
-
-(define_insn "*negextendsftf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (neg:TF (float_extend:TF
-                 (match_operand:SF 1 "register_operand" "0"))))]
   "TARGET_80387"
   "fchs"
   [(set_attr "type" "fsgn")
-   (set_attr "mode" "XF")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "mode" "XF")])
 \f
 ;; Absolute value instructions
 
    (use (match_operand:V4SF 2 "nonimmediate_operand" ""))
    (clobber (reg:CC 17))]
   "reload_completed && SSE_REG_P (operands[0])"
-  [(set (subreg:TI (match_dup 0) 0)
-       (and:TI (match_dup 1)
-               (match_dup 2)))]
+  [(set (match_dup 0)
+       (and:V4SF (match_dup 1)
+                 (match_dup 2)))]
 {
-  operands[1] = simplify_gen_subreg (TImode, operands[1], SFmode, 0);
-  operands[2] = simplify_gen_subreg (TImode, operands[2], V4SFmode, 0);
+  operands[0] = simplify_gen_subreg (V4SFmode, operands[0], SFmode, 0);
+  operands[1] = simplify_gen_subreg (V4SFmode, operands[1], SFmode, 0);
   if (operands_match_p (operands[0], operands[2]))
     {
       rtx tmp;
 {
   int size = GET_MODE_SIZE (GET_MODE (operands[1]));
 
-  /* XFmode's size is 12, TFmode 16, but only 10 bytes are used.  */
-  if (size >= 12)
+  if (GET_MODE (operands[1]) == XFmode)
     size = 10;
   operands[0] = adjust_address (operands[0], QImode, size - 1);
   operands[1] = gen_int_mode (~0x80, QImode);
    (use (match_operand:V2DF 2 "nonimmediate_operand" ""))
    (clobber (reg:CC 17))]
   "reload_completed && SSE_REG_P (operands[0])"
-  [(set (subreg:TI (match_dup 0) 0)
-       (and:TI (match_dup 1)
-               (match_dup 2)))]
+  [(set (match_dup 0)
+       (and:V2DF (match_dup 1)
+                 (match_dup 2)))]
 {
   operands[0] = simplify_gen_subreg (V2DFmode, operands[0], DFmode, 0);
-  operands[1] = simplify_gen_subreg (TImode, operands[1], DFmode, 0);
-  operands[2] = simplify_gen_subreg (TImode, operands[2], V2DFmode, 0);
+  operands[1] = simplify_gen_subreg (V2DFmode, operands[1], DFmode, 0);
   /* Avoid possible reformatting on the operands.  */
   if (TARGET_SSE_PARTIAL_REGS && !optimize_size)
     emit_insn (gen_sse2_unpcklpd (operands[0], operands[0], operands[0]));
   [(parallel [(set (match_operand:XF 0 "nonimmediate_operand" "")
                   (neg:XF (match_operand:XF 1 "nonimmediate_operand" "")))
              (clobber (reg:CC 17))])]
-  "!TARGET_64BIT && TARGET_80387"
-  "ix86_expand_unary_operator (ABS, XFmode, operands); DONE;")
-
-(define_expand "abstf2"
-  [(parallel [(set (match_operand:TF 0 "nonimmediate_operand" "")
-                  (neg:TF (match_operand:TF 1 "nonimmediate_operand" "")))
-             (clobber (reg:CC 17))])]
   "TARGET_80387"
-  "ix86_expand_unary_operator (ABS, TFmode, operands); DONE;")
+  "ix86_expand_unary_operator (ABS, XFmode, operands); DONE;")
 
 ;; Keep 'f' and 'r' in separate alternatives to avoid reload problems
 ;; because of secondary memory needed to reload from class FLOAT_INT_REGS
   [(set (match_operand:XF 0 "nonimmediate_operand" "=f#r,rm#f")
        (abs:XF (match_operand:XF 1 "nonimmediate_operand" "0,0")))
    (clobber (reg:CC 17))]
-  "!TARGET_64BIT && TARGET_80387
+  "TARGET_80387
    && ix86_unary_operator_ok (ABS, XFmode, operands)"
   "#")
 
    operands[0] = gen_rtx_REG (SImode,
                              true_regnum (operands[0]) + (TARGET_64BIT ? 1 : 2));")
 
-(define_insn "*abstf2_if"
-  [(set (match_operand:TF 0 "nonimmediate_operand" "=f#r,rm#f")
-       (abs:TF (match_operand:TF 1 "nonimmediate_operand" "0,0")))
-   (clobber (reg:CC 17))]
-  "TARGET_80387 && ix86_unary_operator_ok (ABS, TFmode, operands)"
-  "#")
-
-(define_split
-  [(set (match_operand:TF 0 "fp_register_operand" "")
-       (abs:TF (match_operand:TF 1 "register_operand" "")))
-   (clobber (reg:CC 17))]
-  "TARGET_80387 && reload_completed"
-  [(set (match_dup 0)
-       (abs:TF (match_dup 1)))]
-  "")
-
-(define_split
-  [(set (match_operand:TF 0 "register_and_not_any_fp_reg_operand" "")
-       (abs:TF (match_operand:TF 1 "register_operand" "")))
-   (clobber (reg:CC 17))]
-  "TARGET_80387 && reload_completed"
-  [(parallel [(set (match_dup 0) (and:SI (match_dup 0) (match_dup 1)))
-             (clobber (reg:CC 17))])]
-  "operands[1] = GEN_INT (~0x8000);
-   operands[0] = gen_rtx_REG (SImode,
-                             true_regnum (operands[0]) + (TARGET_64BIT ? 1 : 2));")
-
 (define_insn "*abssf2_1"
   [(set (match_operand:SF 0 "register_operand" "=f")
        (abs:SF (match_operand:SF 1 "register_operand" "0")))]
 (define_insn "*absxf2_1"
   [(set (match_operand:XF 0 "register_operand" "=f")
        (abs:XF (match_operand:XF 1 "register_operand" "0")))]
-  "!TARGET_64BIT && TARGET_80387 && reload_completed"
+  "TARGET_80387 && reload_completed"
   "fabs"
   [(set_attr "type" "fsgn")
    (set_attr "mode" "DF")])
   [(set (match_operand:XF 0 "register_operand" "=f")
        (abs:XF (float_extend:XF
          (match_operand:DF 1 "register_operand" "0"))))]
-  "!TARGET_64BIT && TARGET_80387"
+  "TARGET_80387"
   "fabs"
   [(set_attr "type" "fsgn")
    (set_attr "mode" "XF")])
   [(set (match_operand:XF 0 "register_operand" "=f")
        (abs:XF (float_extend:XF
          (match_operand:SF 1 "register_operand" "0"))))]
-  "!TARGET_64BIT && TARGET_80387"
-  "fabs"
-  [(set_attr "type" "fsgn")
-   (set_attr "mode" "XF")])
-
-(define_insn "*abstf2_1"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (abs:TF (match_operand:TF 1 "register_operand" "0")))]
-  "TARGET_80387 && reload_completed"
-  "fabs"
-  [(set_attr "type" "fsgn")
-   (set_attr "mode" "DF")])
-
-(define_insn "*absextenddftf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (abs:TF (float_extend:TF
-         (match_operand:DF 1 "register_operand" "0"))))]
-  "TARGET_80387"
-  "fabs"
-  [(set_attr "type" "fsgn")
-   (set_attr "mode" "XF")])
-
-(define_insn "*absextendsftf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (abs:TF (float_extend:TF
-         (match_operand:SF 1 "register_operand" "0"))))]
   "TARGET_80387"
   "fabs"
   [(set_attr "type" "fsgn")
     default:
       if (REG_P (operands[2]))
        return "sal{q}\t{%b2, %0|%0, %b2}";
-      else if (GET_CODE (operands[2]) == CONST_INT
-              && INTVAL (operands[2]) == 1
+      else if (operands[2] == const1_rtx
               && (TARGET_SHIFT1 || optimize_size))
        return "sal{q}\t%0";
       else
     default:
       if (REG_P (operands[2]))
        return "sal{q}\t{%b2, %0|%0, %b2}";
-      else if (GET_CODE (operands[2]) == CONST_INT
-              && INTVAL (operands[2]) == 1
+      else if (operands[2] == const1_rtx
               && (TARGET_SHIFT1 || optimize_size))
        return "sal{q}\t%0";
       else
    (set_attr "prefix_0f" "1")
    (set_attr "mode" "SI")
    (set_attr "pent_pair" "np")
-   (set_attr "athlon_decode" "vector")
-   (set_attr "ppro_uops" "few")])
+   (set_attr "athlon_decode" "vector")])
 
 (define_expand "x86_shift_adj_1"
   [(set (reg:CCZ 17)
     default:
       if (REG_P (operands[2]))
        return "sal{l}\t{%b2, %0|%0, %b2}";
-      else if (GET_CODE (operands[2]) == CONST_INT
-              && INTVAL (operands[2]) == 1
+      else if (operands[2] == const1_rtx
               && (TARGET_SHIFT1 || optimize_size))
        return "sal{l}\t%0";
       else
     default:
       if (REG_P (operands[2]))
        return "sal{l}\t{%b2, %k0|%k0, %b2}";
-      else if (GET_CODE (operands[2]) == CONST_INT
-              && INTVAL (operands[2]) == 1
+      else if (operands[2] == const1_rtx
               && (TARGET_SHIFT1 || optimize_size))
        return "sal{l}\t%k0";
       else
     default:
       if (REG_P (operands[2]))
        return "sal{l}\t{%b2, %0|%0, %b2}";
-      else if (GET_CODE (operands[2]) == CONST_INT
-              && INTVAL (operands[2]) == 1
+      else if (operands[2] == const1_rtx
               && (TARGET_SHIFT1 || optimize_size))
        return "sal{l}\t%0";
       else
     default:
       if (REG_P (operands[2]))
        return "sal{l}\t{%b2, %k0|%k0, %b2}";
-      else if (GET_CODE (operands[2]) == CONST_INT
-              && INTVAL (operands[2]) == 1
+      else if (operands[2] == const1_rtx
               && (TARGET_SHIFT1 || optimize_size))
        return "sal{l}\t%k0";
       else
     default:
       if (REG_P (operands[2]))
        return "sal{w}\t{%b2, %0|%0, %b2}";
-      else if (GET_CODE (operands[2]) == CONST_INT
-              && INTVAL (operands[2]) == 1
+      else if (operands[2] == const1_rtx
               && (TARGET_SHIFT1 || optimize_size))
        return "sal{w}\t%0";
       else
     default:
       if (REG_P (operands[2]))
        return "sal{w}\t{%b2, %0|%0, %b2}";
-      else if (GET_CODE (operands[2]) == CONST_INT
-              && INTVAL (operands[2]) == 1
+      else if (operands[2] == const1_rtx
               && (TARGET_SHIFT1 || optimize_size))
        return "sal{w}\t%0";
       else
     default:
       if (REG_P (operands[2]))
        return "sal{w}\t{%b2, %0|%0, %b2}";
-      else if (GET_CODE (operands[2]) == CONST_INT
-              && INTVAL (operands[2]) == 1
+      else if (operands[2] == const1_rtx
               && (TARGET_SHIFT1 || optimize_size))
        return "sal{w}\t%0";
       else
          else
            return "sal{b}\t{%b2, %0|%0, %b2}";
        }
-      else if (GET_CODE (operands[2]) == CONST_INT
-              && INTVAL (operands[2]) == 1
+      else if (operands[2] == const1_rtx
               && (TARGET_SHIFT1 || optimize_size))
        {
          if (get_attr_mode (insn) == MODE_SI)
          else
            return "sal{b}\t{%b2, %0|%0, %b2}";
        }
-      else if (GET_CODE (operands[2]) == CONST_INT
-              && INTVAL (operands[2]) == 1
+      else if (operands[2] == const1_rtx
               && (TARGET_SHIFT1 || optimize_size))
        {
          if (get_attr_mode (insn) == MODE_SI)
     default:
       if (REG_P (operands[2]))
        return "sal{b}\t{%b2, %0|%0, %b2}";
-      else if (GET_CODE (operands[2]) == CONST_INT
-              && INTVAL (operands[2]) == 1
+      else if (operands[2] == const1_rtx
               && (TARGET_SHIFT1 || optimize_size))
        return "sal{b}\t%0";
       else
 (define_insn "*ashrdi3_1_one_bit_rex64"
   [(set (match_operand:DI 0 "nonimmediate_operand" "=rm")
        (ashiftrt:DI (match_operand:DI 1 "nonimmediate_operand" "0")
-                    (match_operand:QI 2 "const_int_1_operand" "")))
+                    (match_operand:QI 2 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "TARGET_64BIT && ix86_binary_operator_ok (ASHIFTRT, DImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
   [(set (reg 17)
        (compare
          (ashiftrt:DI (match_operand:DI 1 "nonimmediate_operand" "0")
-                      (match_operand:QI 2 "const_int_1_operand" ""))
+                      (match_operand:QI 2 "const1_operand" ""))
          (const_int 0)))
    (set (match_operand:DI 0 "nonimmediate_operand" "=rm")
        (ashiftrt:DI (match_dup 1) (match_dup 2)))]
   [(set_attr "type" "ishift")
    (set_attr "prefix_0f" "1")
    (set_attr "pent_pair" "np")
-   (set_attr "ppro_uops" "few")
    (set_attr "mode" "SI")])
 
 (define_expand "x86_shift_adj_3"
 (define_insn "*ashrsi3_1_one_bit"
   [(set (match_operand:SI 0 "nonimmediate_operand" "=rm")
        (ashiftrt:SI (match_operand:SI 1 "nonimmediate_operand" "0")
-                    (match_operand:QI 2 "const_int_1_operand" "")))
+                    (match_operand:QI 2 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "ix86_binary_operator_ok (ASHIFTRT, SImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
 (define_insn "*ashrsi3_1_one_bit_zext"
   [(set (match_operand:DI 0 "register_operand" "=r")
        (zero_extend:DI (ashiftrt:SI (match_operand:SI 1 "register_operand" "0")
-                                    (match_operand:QI 2 "const_int_1_operand" ""))))
+                                    (match_operand:QI 2 "const1_operand" ""))))
    (clobber (reg:CC 17))]
   "TARGET_64BIT && ix86_binary_operator_ok (ASHIFTRT, SImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
   [(set (reg 17)
        (compare
          (ashiftrt:SI (match_operand:SI 1 "nonimmediate_operand" "0")
-                      (match_operand:QI 2 "const_int_1_operand" ""))
+                      (match_operand:QI 2 "const1_operand" ""))
          (const_int 0)))
    (set (match_operand:SI 0 "nonimmediate_operand" "=rm")
        (ashiftrt:SI (match_dup 1) (match_dup 2)))]
   [(set (reg 17)
        (compare
          (ashiftrt:SI (match_operand:SI 1 "register_operand" "0")
-                      (match_operand:QI 2 "const_int_1_operand" ""))
+                      (match_operand:QI 2 "const1_operand" ""))
          (const_int 0)))
    (set (match_operand:DI 0 "register_operand" "=r")
        (zero_extend:DI (ashiftrt:SI (match_dup 1) (match_dup 2))))]
 (define_insn "*ashrhi3_1_one_bit"
   [(set (match_operand:HI 0 "nonimmediate_operand" "=rm")
        (ashiftrt:HI (match_operand:HI 1 "nonimmediate_operand" "0")
-                    (match_operand:QI 2 "const_int_1_operand" "")))
+                    (match_operand:QI 2 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "ix86_binary_operator_ok (ASHIFTRT, HImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
   [(set (reg 17)
        (compare
          (ashiftrt:HI (match_operand:HI 1 "nonimmediate_operand" "0")
-                      (match_operand:QI 2 "const_int_1_operand" ""))
+                      (match_operand:QI 2 "const1_operand" ""))
          (const_int 0)))
    (set (match_operand:HI 0 "nonimmediate_operand" "=rm")
        (ashiftrt:HI (match_dup 1) (match_dup 2)))]
 (define_insn "*ashrqi3_1_one_bit"
   [(set (match_operand:QI 0 "nonimmediate_operand" "=qm")
        (ashiftrt:QI (match_operand:QI 1 "nonimmediate_operand" "0")
-                    (match_operand:QI 2 "const_int_1_operand" "")))
+                    (match_operand:QI 2 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "ix86_binary_operator_ok (ASHIFTRT, QImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
 (define_insn "*ashrqi3_1_one_bit_slp"
   [(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+qm"))
        (ashiftrt:QI (match_dup 0)
-                    (match_operand:QI 1 "const_int_1_operand" "")))
+                    (match_operand:QI 1 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "ix86_binary_operator_ok (ASHIFTRT, QImode, operands)
    && (! TARGET_PARTIAL_REG_STALL || optimize_size)
   [(set (reg 17)
        (compare
          (ashiftrt:QI (match_operand:QI 1 "nonimmediate_operand" "0")
-                      (match_operand:QI 2 "const_int_1_operand" "I"))
+                      (match_operand:QI 2 "const1_operand" "I"))
          (const_int 0)))
    (set (match_operand:QI 0 "nonimmediate_operand" "=qm")
        (ashiftrt:QI (match_dup 1) (match_dup 2)))]
 (define_insn "*lshrdi3_1_one_bit_rex64"
   [(set (match_operand:DI 0 "nonimmediate_operand" "=rm")
        (lshiftrt:DI (match_operand:DI 1 "nonimmediate_operand" "0")
-                    (match_operand:QI 2 "const_int_1_operand" "")))
+                    (match_operand:QI 2 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "TARGET_64BIT && ix86_binary_operator_ok (LSHIFTRT, HImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
   [(set (reg 17)
        (compare
          (lshiftrt:DI (match_operand:DI 1 "nonimmediate_operand" "0")
-                      (match_operand:QI 2 "const_int_1_operand" ""))
+                      (match_operand:QI 2 "const1_operand" ""))
          (const_int 0)))
    (set (match_operand:DI 0 "nonimmediate_operand" "=rm")
        (lshiftrt:DI (match_dup 1) (match_dup 2)))]
 (define_insn "*lshrsi3_1_one_bit"
   [(set (match_operand:SI 0 "nonimmediate_operand" "=rm")
        (lshiftrt:SI (match_operand:SI 1 "nonimmediate_operand" "0")
-                    (match_operand:QI 2 "const_int_1_operand" "")))
+                    (match_operand:QI 2 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "ix86_binary_operator_ok (LSHIFTRT, HImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
 (define_insn "*lshrsi3_1_one_bit_zext"
   [(set (match_operand:DI 0 "register_operand" "=r")
        (lshiftrt:DI (zero_extend:DI (match_operand:SI 1 "register_operand" "0"))
-                    (match_operand:QI 2 "const_int_1_operand" "")))
+                    (match_operand:QI 2 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "TARGET_64BIT && ix86_binary_operator_ok (LSHIFTRT, HImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
   [(set (reg 17)
        (compare
          (lshiftrt:SI (match_operand:SI 1 "nonimmediate_operand" "0")
-                      (match_operand:QI 2 "const_int_1_operand" ""))
+                      (match_operand:QI 2 "const1_operand" ""))
          (const_int 0)))
    (set (match_operand:SI 0 "nonimmediate_operand" "=rm")
        (lshiftrt:SI (match_dup 1) (match_dup 2)))]
   [(set (reg 17)
        (compare
          (lshiftrt:SI (match_operand:SI 1 "register_operand" "0")
-                      (match_operand:QI 2 "const_int_1_operand" ""))
+                      (match_operand:QI 2 "const1_operand" ""))
          (const_int 0)))
    (set (match_operand:DI 0 "register_operand" "=r")
        (lshiftrt:DI (zero_extend:DI (match_dup 1)) (match_dup 2)))]
 (define_insn "*lshrhi3_1_one_bit"
   [(set (match_operand:HI 0 "nonimmediate_operand" "=rm")
        (lshiftrt:HI (match_operand:HI 1 "nonimmediate_operand" "0")
-                    (match_operand:QI 2 "const_int_1_operand" "")))
+                    (match_operand:QI 2 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "ix86_binary_operator_ok (LSHIFTRT, HImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
   [(set (reg 17)
        (compare
          (lshiftrt:HI (match_operand:HI 1 "nonimmediate_operand" "0")
-                      (match_operand:QI 2 "const_int_1_operand" ""))
+                      (match_operand:QI 2 "const1_operand" ""))
          (const_int 0)))
    (set (match_operand:HI 0 "nonimmediate_operand" "=rm")
        (lshiftrt:HI (match_dup 1) (match_dup 2)))]
 (define_insn "*lshrqi3_1_one_bit"
   [(set (match_operand:QI 0 "nonimmediate_operand" "=qm")
        (lshiftrt:QI (match_operand:QI 1 "nonimmediate_operand" "0")
-                    (match_operand:QI 2 "const_int_1_operand" "")))
+                    (match_operand:QI 2 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "ix86_binary_operator_ok (LSHIFTRT, QImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
 (define_insn "*lshrqi3_1_one_bit_slp"
   [(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+qm"))
        (lshiftrt:QI (match_dup 0)
-                    (match_operand:QI 1 "const_int_1_operand" "")))
+                    (match_operand:QI 1 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "(! TARGET_PARTIAL_REG_STALL || optimize_size)
    && (TARGET_SHIFT1 || optimize_size)"
   [(set (reg 17)
        (compare
          (lshiftrt:QI (match_operand:QI 1 "nonimmediate_operand" "0")
-                      (match_operand:QI 2 "const_int_1_operand" ""))
+                      (match_operand:QI 2 "const1_operand" ""))
          (const_int 0)))
    (set (match_operand:QI 0 "nonimmediate_operand" "=qm")
        (lshiftrt:QI (match_dup 1) (match_dup 2)))]
 (define_insn "*rotlsi3_1_one_bit_rex64"
   [(set (match_operand:DI 0 "nonimmediate_operand" "=rm")
        (rotate:DI (match_operand:DI 1 "nonimmediate_operand" "0")
-                  (match_operand:QI 2 "const_int_1_operand" "")))
+                  (match_operand:QI 2 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "TARGET_64BIT && ix86_binary_operator_ok (ROTATE, DImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
 (define_insn "*rotlsi3_1_one_bit"
   [(set (match_operand:SI 0 "nonimmediate_operand" "=rm")
        (rotate:SI (match_operand:SI 1 "nonimmediate_operand" "0")
-                  (match_operand:QI 2 "const_int_1_operand" "")))
+                  (match_operand:QI 2 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "ix86_binary_operator_ok (ROTATE, SImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
   [(set (match_operand:DI 0 "register_operand" "=r")
        (zero_extend:DI
          (rotate:SI (match_operand:SI 1 "register_operand" "0")
-                    (match_operand:QI 2 "const_int_1_operand" ""))))
+                    (match_operand:QI 2 "const1_operand" ""))))
    (clobber (reg:CC 17))]
   "TARGET_64BIT && ix86_binary_operator_ok (ROTATE, SImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
 (define_insn "*rotlhi3_1_one_bit"
   [(set (match_operand:HI 0 "nonimmediate_operand" "=rm")
        (rotate:HI (match_operand:HI 1 "nonimmediate_operand" "0")
-                  (match_operand:QI 2 "const_int_1_operand" "")))
+                  (match_operand:QI 2 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "ix86_binary_operator_ok (ROTATE, HImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
 (define_insn "*rotlqi3_1_one_bit_slp"
   [(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+qm"))
        (rotate:QI (match_dup 0)
-                  (match_operand:QI 1 "const_int_1_operand" "")))
+                  (match_operand:QI 1 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "(! TARGET_PARTIAL_REG_STALL || optimize_size)
    && (TARGET_SHIFT1 || optimize_size)"
 (define_insn "*rotlqi3_1_one_bit"
   [(set (match_operand:QI 0 "nonimmediate_operand" "=qm")
        (rotate:QI (match_operand:QI 1 "nonimmediate_operand" "0")
-                  (match_operand:QI 2 "const_int_1_operand" "")))
+                  (match_operand:QI 2 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "ix86_binary_operator_ok (ROTATE, QImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
 (define_insn "*rotrdi3_1_one_bit_rex64"
   [(set (match_operand:DI 0 "nonimmediate_operand" "=rm")
        (rotatert:DI (match_operand:DI 1 "nonimmediate_operand" "0")
-                    (match_operand:QI 2 "const_int_1_operand" "")))
+                    (match_operand:QI 2 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "TARGET_64BIT && ix86_binary_operator_ok (ROTATERT, DImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
 (define_insn "*rotrsi3_1_one_bit"
   [(set (match_operand:SI 0 "nonimmediate_operand" "=rm")
        (rotatert:SI (match_operand:SI 1 "nonimmediate_operand" "0")
-                    (match_operand:QI 2 "const_int_1_operand" "")))
+                    (match_operand:QI 2 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "ix86_binary_operator_ok (ROTATERT, SImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
   [(set (match_operand:DI 0 "register_operand" "=r")
        (zero_extend:DI
          (rotatert:SI (match_operand:SI 1 "register_operand" "0")
-                      (match_operand:QI 2 "const_int_1_operand" ""))))
+                      (match_operand:QI 2 "const1_operand" ""))))
    (clobber (reg:CC 17))]
   "TARGET_64BIT && ix86_binary_operator_ok (ROTATERT, SImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
 (define_insn "*rotrhi3_one_bit"
   [(set (match_operand:HI 0 "nonimmediate_operand" "=rm")
        (rotatert:HI (match_operand:HI 1 "nonimmediate_operand" "0")
-                    (match_operand:QI 2 "const_int_1_operand" "")))
+                    (match_operand:QI 2 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "ix86_binary_operator_ok (ROTATERT, HImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
 (define_insn "*rotrqi3_1_one_bit"
   [(set (match_operand:QI 0 "nonimmediate_operand" "=qm")
        (rotatert:QI (match_operand:QI 1 "nonimmediate_operand" "0")
-                    (match_operand:QI 2 "const_int_1_operand" "")))
+                    (match_operand:QI 2 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "ix86_binary_operator_ok (ROTATERT, QImode, operands)
    && (TARGET_SHIFT1 || optimize_size)"
 (define_insn "*rotrqi3_1_one_bit_slp"
   [(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+qm"))
        (rotatert:QI (match_dup 0)
-                    (match_operand:QI 1 "const_int_1_operand" "")))
+                    (match_operand:QI 1 "const1_operand" "")))
    (clobber (reg:CC 17))]
   "(! TARGET_PARTIAL_REG_STALL || optimize_size)
    && (TARGET_SHIFT1 || optimize_size)"
   rtx new_op1 = copy_rtx (operands[1]);
   operands[1] = new_op1;
   PUT_MODE (new_op1, QImode);
-  PUT_CODE (new_op1, REVERSE_CONDITION (GET_CODE (new_op1),
-                                       GET_MODE (XEXP (new_op1, 0))));
+  PUT_CODE (new_op1, ix86_reverse_condition (GET_CODE (new_op1),
+                                            GET_MODE (XEXP (new_op1, 0))));
 
   /* Make sure that (a) the CCmode we have for the flags is strong
      enough for the reversed compare or (b) we have a valid FP compare.  */
   rtx new_op1 = copy_rtx (operands[1]);
   operands[1] = new_op1;
   PUT_MODE (new_op1, QImode);
-  PUT_CODE (new_op1, REVERSE_CONDITION (GET_CODE (new_op1),
-                                       GET_MODE (XEXP (new_op1, 0))));
+  PUT_CODE (new_op1, ix86_reverse_condition (GET_CODE (new_op1),
+                                            GET_MODE (XEXP (new_op1, 0))));
 
   /* Make sure that (a) the CCmode we have for the flags is strong
      enough for the reversed compare or (b) we have a valid FP compare.  */
   rtx new_op0 = copy_rtx (operands[0]);
   operands[0] = new_op0;
   PUT_MODE (new_op0, VOIDmode);
-  PUT_CODE (new_op0, REVERSE_CONDITION (GET_CODE (new_op0),
-                                       GET_MODE (XEXP (new_op0, 0))));
+  PUT_CODE (new_op0, ix86_reverse_condition (GET_CODE (new_op0),
+                                            GET_MODE (XEXP (new_op0, 0))));
 
   /* Make sure that (a) the CCmode we have for the flags is strong
      enough for the reversed compare or (b) we have a valid FP compare.  */
   [(const_int 0)]
 {
   ix86_split_fp_branch (GET_CODE (operands[0]), operands[1], operands[2],
-                       operands[3], operands[4], NULL_RTX);
+                       operands[3], operands[4], NULL_RTX);
   DONE;
 })
 
          (match_dup 4)))]
 {
   ix86_split_fp_branch (GET_CODE (operands[0]), operands[1], operands[2],
-                       operands[3], operands[4], operands[5]);
+                       operands[3], operands[4], operands[5]);
   DONE;
 })
 \f
                          (const_int 1))
                      (label_ref (match_operand 0 "" ""))
                      (pc)))
-   (set (match_operand:SI 2 "register_operand" "=1,1,*m*r")
+   (set (match_operand:SI 2 "nonimmediate_operand" "=1,1,*m*r")
        (plus:SI (match_dup 1)
                 (const_int -1)))
    (clobber (match_scratch:SI 3 "=X,X,r"))
    (clobber (reg:CC 17))]
-  "!TARGET_64BIT && TARGET_USE_LOOP"
+  "!TARGET_64BIT && TARGET_USE_LOOP
+   && (reload_in_progress || reload_completed
+       || register_operand (operands[2], VOIDmode))"
 {
   if (which_alternative != 0)
     return "#";
   else
     return "dec{l}\t%1\;%+jne\t%l0";
 }
-  [(set_attr "ppro_uops" "many")
-   (set (attr "length")
+  [(set (attr "length")
        (if_then_else (and (eq_attr "alternative" "0")
                           (and (ge (minus (match_dup 0) (pc))
                                    (const_int -126))
    (set_attr "length_immediate" "0")
    (set_attr "modrm" "0")])
 
+;; Used by x86_machine_dependent_reorg to avoid penalty on single byte RET
+;; instruction Athlon and K8 have.
+
+(define_insn "return_internal_long"
+  [(return)
+   (unspec [(const_int 0)] UNSPEC_REP)]
+  "reload_completed"
+  "rep {;} ret"
+  [(set_attr "length" "1")
+   (set_attr "length_immediate" "0")
+   (set_attr "prefix_rep" "1")
+   (set_attr "modrm" "0")])
+
 (define_insn "return_pop_internal"
   [(return)
    (use (match_operand:SI 0 "const_int_operand" ""))]
   "nop"
   [(set_attr "length" "1")
    (set_attr "length_immediate" "0")
-   (set_attr "modrm" "0")
-   (set_attr "ppro_uops" "one")])
+   (set_attr "modrm" "0")])
+
+;; Align to 16-byte boundary, max skip in op0.  Used to avoid
+;; branch prediction penalty for the third jump in a 16-byte
+;; block on K8.
+
+(define_insn "align"
+  [(unspec_volatile [(match_operand 0 "" "")] UNSPECV_ALIGN)]
+  ""
+{
+#ifdef ASM_OUTPUT_MAX_SKIP_ALIGN
+  ASM_OUTPUT_MAX_SKIP_ALIGN (asm_out_file, 4, (int)INTVAL (operands[0]));
+#else
+  /* It is tempting to use ASM_OUTPUT_ALIGN here, but we don't want to do that.
+     The align insn is used to avoid 3 jump instructions in the row to improve
+     branch prediction and the benefits hardly outweight the cost of extra 8
+     nops on the average inserted by full alignment pseudo operation.  */
+#endif
+  return "";
+}
+  [(set_attr "length" "16")])
 
 (define_expand "prologue"
   [(const_int 1)]
 (define_insn_and_split "*ffs_no_cmove"
   [(set (match_operand:SI 0 "nonimmediate_operand" "=r") 
        (ffs:SI (match_operand:SI 1 "nonimmediate_operand" "rm")))
-   (clobber (match_scratch:SI 2 "=&r"))
+   (clobber (match_scratch:SI 2 "=&q"))
    (clobber (reg:CC 17))]
   ""
   "#"
   "reload_completed"
-  [(parallel [(set (match_dup 2) (const_int 0))
-             (clobber (reg:CC 17))])
-   (parallel [(set (reg:CCZ 17) (compare:CCZ (match_dup 1) (const_int 0)))
+  [(parallel [(set (reg:CCZ 17) (compare:CCZ (match_dup 1) (const_int 0)))
              (set (match_dup 0) (ctz:SI (match_dup 1)))])
    (set (strict_low_part (match_dup 3))
        (eq:QI (reg:CCZ 17) (const_int 0)))
              (clobber (reg:CC 17))])]
 {
   operands[3] = gen_lowpart (QImode, operands[2]);
+  ix86_expand_clear (operands[2]);
 })
 
 (define_insn "*ffssi_1"
        (ctz:SI (match_dup 1)))]
   ""
   "bsf{l}\t{%1, %0|%0, %1}"
-  [(set_attr "prefix_0f" "1")
-   (set_attr "ppro_uops" "few")])
+  [(set_attr "prefix_0f" "1")])
 
 (define_insn "ctzsi2"
   [(set (match_operand:SI 0 "register_operand" "=r")
    (clobber (reg:CC 17))]
   ""
   "bsf{l}\t{%1, %0|%0, %1}"
-  [(set_attr "prefix_0f" "1")
-   (set_attr "ppro_uops" "few")])
+  [(set_attr "prefix_0f" "1")])
 
 (define_expand "clzsi2"
   [(parallel
    (clobber (reg:CC 17))]
   ""
   "bsr{l}\t{%1, %0|%0, %1}"
-  [(set_attr "prefix_0f" "1")
-   (set_attr "ppro_uops" "few")])
+  [(set_attr "prefix_0f" "1")])
 \f
 ;; Thread-local storage patterns for ELF.
 ;;
              (clobber (match_dup 5))
              (clobber (reg:CC 17))])]
   "")
+
+;; Load and add the thread base pointer from %gs:0.
+
+(define_insn "*load_tp_si"
+  [(set (match_operand:SI 0 "register_operand" "=r")
+       (unspec:SI [(const_int 0)] UNSPEC_TP))]
+  "!TARGET_64BIT"
+  "mov{l}\t{%%gs:0, %0|%0, DWORD PTR %%gs:0}"
+  [(set_attr "type" "imov")
+   (set_attr "modrm" "0")
+   (set_attr "length" "7")
+   (set_attr "memory" "load")
+   (set_attr "imm_disp" "false")])
+
+(define_insn "*add_tp_si"
+  [(set (match_operand:SI 0 "register_operand" "=r")
+       (plus:SI (unspec:SI [(const_int 0)] UNSPEC_TP)
+                (match_operand:SI 1 "register_operand" "0")))
+   (clobber (reg:CC 17))]
+  "!TARGET_64BIT"
+  "add{l}\t{%%gs:0, %0|%0, DWORD PTR %%gs:0}"
+  [(set_attr "type" "alu")
+   (set_attr "modrm" "0")
+   (set_attr "length" "7")
+   (set_attr "memory" "load")
+   (set_attr "imm_disp" "false")])
+
+(define_insn "*load_tp_di"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (unspec:DI [(const_int 0)] UNSPEC_TP))]
+  "TARGET_64BIT"
+  "mov{q}\t{%%fs:0, %0|%0, QWORD PTR %%fs:0}"
+  [(set_attr "type" "imov")
+   (set_attr "modrm" "0")
+   (set_attr "length" "7")
+   (set_attr "memory" "load")
+   (set_attr "imm_disp" "false")])
+
+(define_insn "*add_tp_di"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (plus:DI (unspec:DI [(const_int 0)] UNSPEC_TP)
+                (match_operand:DI 1 "register_operand" "0")))
+   (clobber (reg:CC 17))]
+  "TARGET_64BIT"
+  "add{q}\t{%%fs:0, %0|%0, QWORD PTR %%fs:0}"
+  [(set_attr "type" "alu")
+   (set_attr "modrm" "0")
+   (set_attr "length" "7")
+   (set_attr "memory" "load")
+   (set_attr "imm_disp" "false")])
 \f
 ;; These patterns match the binary 387 instructions for addM3, subM3,
 ;; mulM3 and divM3.  There are three patterns for each of DFmode and
                        [(match_operand:SF 1 "nonimmediate_operand" "%0")
                         (match_operand:SF 2 "nonimmediate_operand" "fm")]))]
   "TARGET_80387 && !TARGET_SSE_MATH
-   && GET_RTX_CLASS (GET_CODE (operands[3])) == 'c'
+   && COMMUTATIVE_ARITH_P (operands[3])
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
                        [(match_operand:SF 1 "nonimmediate_operand" "%0,0")
                         (match_operand:SF 2 "nonimmediate_operand" "fm#x,xm#f")]))]
   "TARGET_80387 && TARGET_SSE_MATH && TARGET_MIX_SSE_I387
-   && GET_RTX_CLASS (GET_CODE (operands[3])) == 'c'
+   && COMMUTATIVE_ARITH_P (operands[3])
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
        (match_operator:SF 3 "binary_fp_operator"
                        [(match_operand:SF 1 "nonimmediate_operand" "%0")
                         (match_operand:SF 2 "nonimmediate_operand" "xm")]))]
-  "TARGET_SSE_MATH && GET_RTX_CLASS (GET_CODE (operands[3])) == 'c'
+  "TARGET_SSE_MATH && COMMUTATIVE_ARITH_P (operands[3])
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
                        [(match_operand:DF 1 "nonimmediate_operand" "%0")
                         (match_operand:DF 2 "nonimmediate_operand" "fm")]))]
   "TARGET_80387 && (!TARGET_SSE2 || !TARGET_SSE_MATH)
-   && GET_RTX_CLASS (GET_CODE (operands[3])) == 'c'
+   && COMMUTATIVE_ARITH_P (operands[3])
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
                        [(match_operand:DF 1 "nonimmediate_operand" "%0,0")
                         (match_operand:DF 2 "nonimmediate_operand" "fm#Y,Ym#f")]))]
   "TARGET_80387 && TARGET_SSE_MATH && TARGET_SSE2 && TARGET_MIX_SSE_I387
-   && GET_RTX_CLASS (GET_CODE (operands[3])) == 'c'
+   && COMMUTATIVE_ARITH_P (operands[3])
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
                        [(match_operand:DF 1 "nonimmediate_operand" "%0")
                         (match_operand:DF 2 "nonimmediate_operand" "Ym")]))]
   "TARGET_SSE2 && TARGET_SSE_MATH
-   && GET_RTX_CLASS (GET_CODE (operands[3])) == 'c'
+   && COMMUTATIVE_ARITH_P (operands[3])
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
        (match_operator:XF 3 "binary_fp_operator"
                        [(match_operand:XF 1 "register_operand" "%0")
                         (match_operand:XF 2 "register_operand" "f")]))]
-  "!TARGET_64BIT && TARGET_80387
-   && GET_RTX_CLASS (GET_CODE (operands[3])) == 'c'"
+  "TARGET_80387
+   && COMMUTATIVE_ARITH_P (operands[3])"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
         (if_then_else (match_operand:XF 3 "mult_operator" "") 
            (const_string "fop")))
    (set_attr "mode" "XF")])
 
-(define_insn "*fop_tf_comm"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (match_operator:TF 3 "binary_fp_operator"
-                       [(match_operand:TF 1 "register_operand" "%0")
-                        (match_operand:TF 2 "register_operand" "f")]))]
-  "TARGET_80387 && GET_RTX_CLASS (GET_CODE (operands[3])) == 'c'"
-  "* return output_387_binary_op (insn, operands);"
-  [(set (attr "type") 
-        (if_then_else (match_operand:TF 3 "mult_operator" "") 
-           (const_string "fmul")
-           (const_string "fop")))
-   (set_attr "mode" "XF")])
-
 (define_insn "*fop_sf_1_nosse"
   [(set (match_operand:SF 0 "register_operand" "=f,f")
        (match_operator:SF 3 "binary_fp_operator"
                        [(match_operand:SF 1 "nonimmediate_operand" "0,fm")
                         (match_operand:SF 2 "nonimmediate_operand" "fm,0")]))]
   "TARGET_80387 && !TARGET_SSE_MATH
-   && GET_RTX_CLASS (GET_CODE (operands[3])) != 'c'
+   && !COMMUTATIVE_ARITH_P (operands[3])
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
                        [(match_operand:SF 1 "nonimmediate_operand" "0,fm,0")
                         (match_operand:SF 2 "nonimmediate_operand" "fm,0,xm#f")]))]
   "TARGET_80387 && TARGET_SSE_MATH && TARGET_MIX_SSE_I387
-   && GET_RTX_CLASS (GET_CODE (operands[3])) != 'c'
+   && !COMMUTATIVE_ARITH_P (operands[3])
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
                        [(match_operand:SF 1 "register_operand" "0")
                         (match_operand:SF 2 "nonimmediate_operand" "xm")]))]
   "TARGET_SSE_MATH
-   && GET_RTX_CLASS (GET_CODE (operands[3])) != 'c'"
+   && !COMMUTATIVE_ARITH_P (operands[3])"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
         (cond [(match_operand:SF 3 "mult_operator" "")
               ]
               (const_string "fop")))
    (set_attr "fp_int_src" "true")
-   (set_attr "ppro_uops" "many")
    (set_attr "mode" "SI")])
 
 (define_insn "*fop_sf_3"
               ]
               (const_string "fop")))
    (set_attr "fp_int_src" "true")
-   (set_attr "ppro_uops" "many")
    (set_attr "mode" "SI")])
 
 (define_insn "*fop_df_1_nosse"
                        [(match_operand:DF 1 "nonimmediate_operand" "0,fm")
                         (match_operand:DF 2 "nonimmediate_operand" "fm,0")]))]
   "TARGET_80387 && (!TARGET_SSE2 || !TARGET_SSE_MATH)
-   && GET_RTX_CLASS (GET_CODE (operands[3])) != 'c'
+   && !COMMUTATIVE_ARITH_P (operands[3])
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
                        [(match_operand:DF 1 "nonimmediate_operand" "0,fm,0")
                         (match_operand:DF 2 "nonimmediate_operand" "fm,0,Ym#f")]))]
   "TARGET_80387 && TARGET_SSE2 && TARGET_SSE_MATH && TARGET_MIX_SSE_I387
-   && GET_RTX_CLASS (GET_CODE (operands[3])) != 'c'
+   && !COMMUTATIVE_ARITH_P (operands[3])
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
                        [(match_operand:DF 1 "register_operand" "0")
                         (match_operand:DF 2 "nonimmediate_operand" "Ym")]))]
   "TARGET_SSE2 && TARGET_SSE_MATH
-   && GET_RTX_CLASS (GET_CODE (operands[3])) != 'c'"
+   && !COMMUTATIVE_ARITH_P (operands[3])"
   "* return output_387_binary_op (insn, operands);"
   [(set_attr "mode" "DF")
    (set (attr "type") 
               ]
               (const_string "fop")))
    (set_attr "fp_int_src" "true")
-   (set_attr "ppro_uops" "many")
    (set_attr "mode" "SI")])
 
 (define_insn "*fop_df_3"
               ]
               (const_string "fop")))
    (set_attr "fp_int_src" "true")
-   (set_attr "ppro_uops" "many")
    (set_attr "mode" "SI")])
 
 (define_insn "*fop_df_4"
        (match_operator:XF 3 "binary_fp_operator"
                        [(match_operand:XF 1 "register_operand" "0,f")
                         (match_operand:XF 2 "register_operand" "f,0")]))]
-  "!TARGET_64BIT && TARGET_80387
-   && GET_RTX_CLASS (GET_CODE (operands[3])) != 'c'"
+  "TARGET_80387
+   && !COMMUTATIVE_ARITH_P (operands[3])"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
         (cond [(match_operand:XF 3 "mult_operator" "") 
               (const_string "fop")))
    (set_attr "mode" "XF")])
 
-(define_insn "*fop_tf_1"
-  [(set (match_operand:TF 0 "register_operand" "=f,f")
-       (match_operator:TF 3 "binary_fp_operator"
-                       [(match_operand:TF 1 "register_operand" "0,f")
-                        (match_operand:TF 2 "register_operand" "f,0")]))]
-  "TARGET_80387
-   && GET_RTX_CLASS (GET_CODE (operands[3])) != 'c'"
-  "* return output_387_binary_op (insn, operands);"
-  [(set (attr "type") 
-        (cond [(match_operand:TF 3 "mult_operator" "") 
-                 (const_string "fmul")
-               (match_operand:TF 3 "div_operator" "") 
-                 (const_string "fdiv")
-              ]
-              (const_string "fop")))
-   (set_attr "mode" "XF")])
-
 (define_insn "*fop_xf_2"
   [(set (match_operand:XF 0 "register_operand" "=f,f")
        (match_operator:XF 3 "binary_fp_operator"
           [(float:XF (match_operand:SI 1 "nonimmediate_operand" "m,?r"))
            (match_operand:XF 2 "register_operand" "0,0")]))]
-  "!TARGET_64BIT && TARGET_80387 && TARGET_USE_FIOP"
+  "TARGET_80387 && TARGET_USE_FIOP"
   "* return which_alternative ? \"#\" : output_387_binary_op (insn, operands);"
   [(set (attr "type") 
         (cond [(match_operand:XF 3 "mult_operator" "") 
               ]
               (const_string "fop")))
    (set_attr "fp_int_src" "true")
-   (set_attr "mode" "SI")
-   (set_attr "ppro_uops" "many")])
-
-(define_insn "*fop_tf_2"
-  [(set (match_operand:TF 0 "register_operand" "=f,f")
-       (match_operator:TF 3 "binary_fp_operator"
-          [(float:TF (match_operand:SI 1 "nonimmediate_operand" "m,?r"))
-           (match_operand:TF 2 "register_operand" "0,0")]))]
-  "TARGET_80387 && TARGET_USE_FIOP"
-  "* return which_alternative ? \"#\" : output_387_binary_op (insn, operands);"
-  [(set (attr "type") 
-        (cond [(match_operand:TF 3 "mult_operator" "") 
-                 (const_string "fmul")
-               (match_operand:TF 3 "div_operator" "") 
-                 (const_string "fdiv")
-              ]
-              (const_string "fop")))
-   (set_attr "fp_int_src" "true")
-   (set_attr "mode" "SI")
-   (set_attr "ppro_uops" "many")])
+   (set_attr "mode" "SI")])
 
 (define_insn "*fop_xf_3"
   [(set (match_operand:XF 0 "register_operand" "=f,f")
        (match_operator:XF 3 "binary_fp_operator"
          [(match_operand:XF 1 "register_operand" "0,0")
           (float:XF (match_operand:SI 2 "nonimmediate_operand" "m,?r"))]))]
-  "!TARGET_64BIT && TARGET_80387 && TARGET_USE_FIOP"
+  "TARGET_80387 && TARGET_USE_FIOP"
   "* return which_alternative ? \"#\" : output_387_binary_op (insn, operands);"
   [(set (attr "type") 
         (cond [(match_operand:XF 3 "mult_operator" "") 
               ]
               (const_string "fop")))
    (set_attr "fp_int_src" "true")
-   (set_attr "mode" "SI")
-   (set_attr "ppro_uops" "many")])
-
-(define_insn "*fop_tf_3"
-  [(set (match_operand:TF 0 "register_operand" "=f,f")
-       (match_operator:TF 3 "binary_fp_operator"
-         [(match_operand:TF 1 "register_operand" "0,0")
-          (float:TF (match_operand:SI 2 "nonimmediate_operand" "m,?r"))]))]
-  "TARGET_80387 && TARGET_USE_FIOP"
-  "* return which_alternative ? \"#\" : output_387_binary_op (insn, operands);"
-  [(set (attr "type") 
-        (cond [(match_operand:TF 3 "mult_operator" "") 
-                 (const_string "fmul")
-               (match_operand:TF 3 "div_operator" "") 
-                 (const_string "fdiv")
-              ]
-              (const_string "fop")))
-   (set_attr "fp_int_src" "true")
-   (set_attr "mode" "SI")
-   (set_attr "ppro_uops" "many")])
+   (set_attr "mode" "SI")])
 
 (define_insn "*fop_xf_4"
   [(set (match_operand:XF 0 "register_operand" "=f,f")
        (match_operator:XF 3 "binary_fp_operator"
           [(float_extend:XF (match_operand 1 "nonimmediate_operand" "fm,0"))
            (match_operand:XF 2 "register_operand" "0,f")]))]
-  "!TARGET_64BIT && TARGET_80387"
+  "TARGET_80387"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
         (cond [(match_operand:XF 3 "mult_operator" "") 
               (const_string "fop")))
    (set_attr "mode" "SF")])
 
-(define_insn "*fop_tf_4"
-  [(set (match_operand:TF 0 "register_operand" "=f,f")
-       (match_operator:TF 3 "binary_fp_operator"
-          [(float_extend:TF (match_operand 1 "nonimmediate_operand" "fm,0"))
-           (match_operand:TF 2 "register_operand" "0,f")]))]
-  "TARGET_80387"
-  "* return output_387_binary_op (insn, operands);"
-  [(set (attr "type") 
-        (cond [(match_operand:TF 3 "mult_operator" "") 
-                 (const_string "fmul")
-               (match_operand:TF 3 "div_operator" "") 
-                 (const_string "fdiv")
-              ]
-              (const_string "fop")))
-   (set_attr "mode" "SF")])
-
 (define_insn "*fop_xf_5"
   [(set (match_operand:XF 0 "register_operand" "=f,f")
        (match_operator:XF 3 "binary_fp_operator"
          [(match_operand:XF 1 "register_operand" "0,f")
           (float_extend:XF
            (match_operand 2 "nonimmediate_operand" "fm,0"))]))]
-  "!TARGET_64BIT && TARGET_80387"
+  "TARGET_80387"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
         (cond [(match_operand:XF 3 "mult_operator" "") 
               (const_string "fop")))
    (set_attr "mode" "SF")])
 
-(define_insn "*fop_tf_5"
-  [(set (match_operand:TF 0 "register_operand" "=f,f")
-       (match_operator:TF 3 "binary_fp_operator"
-         [(match_operand:TF 1 "register_operand" "0,f")
-          (float_extend:TF
-           (match_operand 2 "nonimmediate_operand" "fm,0"))]))]
-  "TARGET_80387"
-  "* return output_387_binary_op (insn, operands);"
-  [(set (attr "type") 
-        (cond [(match_operand:TF 3 "mult_operator" "") 
-                 (const_string "fmul")
-               (match_operand:TF 3 "div_operator" "") 
-                 (const_string "fdiv")
-              ]
-              (const_string "fop")))
-   (set_attr "mode" "SF")])
-
 (define_insn "*fop_xf_6"
   [(set (match_operand:XF 0 "register_operand" "=f,f")
        (match_operator:XF 3 "binary_fp_operator"
            (match_operand 1 "register_operand" "0,f"))
           (float_extend:XF
            (match_operand 2 "nonimmediate_operand" "fm,0"))]))]
-  "!TARGET_64BIT && TARGET_80387"
+  "TARGET_80387"
   "* return output_387_binary_op (insn, operands);"
   [(set (attr "type") 
         (cond [(match_operand:XF 3 "mult_operator" "") 
               (const_string "fop")))
    (set_attr "mode" "SF")])
 
-(define_insn "*fop_tf_6"
-  [(set (match_operand:TF 0 "register_operand" "=f,f")
-       (match_operator:TF 3 "binary_fp_operator"
-         [(float_extend:TF
-           (match_operand 1 "register_operand" "0,f"))
-          (float_extend:TF
-           (match_operand 2 "nonimmediate_operand" "fm,0"))]))]
-  "TARGET_80387"
-  "* return output_387_binary_op (insn, operands);"
-  [(set (attr "type") 
-        (cond [(match_operand:TF 3 "mult_operator" "") 
-                 (const_string "fmul")
-               (match_operand:TF 3 "div_operator" "") 
-                 (const_string "fdiv")
-              ]
-              (const_string "fop")))
-   (set_attr "mode" "SF")])
-
 (define_split
   [(set (match_operand 0 "register_operand" "")
        (match_operator 3 "binary_fp_operator"
 (define_insn "sqrtxf2"
   [(set (match_operand:XF 0 "register_operand" "=f")
        (sqrt:XF (match_operand:XF 1 "register_operand" "0")))]
-  "!TARGET_64BIT && TARGET_80387 && !TARGET_NO_FANCY_MATH_387 
-   && (TARGET_IEEE_FP || flag_unsafe_math_optimizations) "
-  "fsqrt"
-  [(set_attr "type" "fpspc")
-   (set_attr "mode" "XF")
-   (set_attr "athlon_decode" "direct")])
-
-(define_insn "sqrttf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (sqrt:TF (match_operand:TF 1 "register_operand" "0")))]
-  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
+  "TARGET_80387 && !TARGET_NO_FANCY_MATH_387 
    && (TARGET_IEEE_FP || flag_unsafe_math_optimizations) "
   "fsqrt"
   [(set_attr "type" "fpspc")
   [(set (match_operand:XF 0 "register_operand" "=f")
        (sqrt:XF (float_extend:XF
                  (match_operand:DF 1 "register_operand" "0"))))]
-  "!TARGET_64BIT && TARGET_80387 && !TARGET_NO_FANCY_MATH_387"
-  "fsqrt"
-  [(set_attr "type" "fpspc")
-   (set_attr "mode" "XF")
-   (set_attr "athlon_decode" "direct")])
-
-(define_insn "*sqrtextenddftf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (sqrt:TF (float_extend:TF
-                 (match_operand:DF 1 "register_operand" "0"))))]
-  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387"
+  "TARGET_80387 && !TARGET_NO_FANCY_MATH_387"
   "fsqrt"
   [(set_attr "type" "fpspc")
    (set_attr "mode" "XF")
   [(set (match_operand:XF 0 "register_operand" "=f")
        (sqrt:XF (float_extend:XF
                  (match_operand:SF 1 "register_operand" "0"))))]
-  "!TARGET_64BIT && TARGET_80387 && !TARGET_NO_FANCY_MATH_387"
-  "fsqrt"
-  [(set_attr "type" "fpspc")
-   (set_attr "mode" "XF")
-   (set_attr "athlon_decode" "direct")])
-
-(define_insn "*sqrtextendsftf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (sqrt:TF (float_extend:TF
-                 (match_operand:SF 1 "register_operand" "0"))))]
-  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387"
+  "TARGET_80387 && !TARGET_NO_FANCY_MATH_387"
   "fsqrt"
   [(set_attr "type" "fpspc")
    (set_attr "mode" "XF")
    (set_attr "athlon_decode" "direct")])
 
-(define_insn "sindf2"
+(define_insn "*sindf2"
   [(set (match_operand:DF 0 "register_operand" "=f")
        (unspec:DF [(match_operand:DF 1 "register_operand" "0")] UNSPEC_SIN))]
   "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
   [(set_attr "type" "fpspc")
    (set_attr "mode" "DF")])
 
-(define_insn "sinsf2"
+(define_insn "*sinsf2"
   [(set (match_operand:SF 0 "register_operand" "=f")
        (unspec:SF [(match_operand:SF 1 "register_operand" "0")] UNSPEC_SIN))]
   "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
   [(set_attr "type" "fpspc")
    (set_attr "mode" "DF")])
 
-(define_insn "sinxf2"
+(define_insn "*sinxf2"
   [(set (match_operand:XF 0 "register_operand" "=f")
        (unspec:XF [(match_operand:XF 1 "register_operand" "0")] UNSPEC_SIN))]
-  "!TARGET_64BIT && TARGET_80387 && !TARGET_NO_FANCY_MATH_387
-   && flag_unsafe_math_optimizations"
-  "fsin"
-  [(set_attr "type" "fpspc")
-   (set_attr "mode" "XF")])
-
-(define_insn "sintf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (unspec:TF [(match_operand:TF 1 "register_operand" "0")] UNSPEC_SIN))]
-  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
+  "TARGET_80387 && !TARGET_NO_FANCY_MATH_387
    && flag_unsafe_math_optimizations"
   "fsin"
   [(set_attr "type" "fpspc")
    (set_attr "mode" "XF")])
 
-(define_insn "cosdf2"
+(define_insn "*cosdf2"
   [(set (match_operand:DF 0 "register_operand" "=f")
        (unspec:DF [(match_operand:DF 1 "register_operand" "0")] UNSPEC_COS))]
   "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
   [(set_attr "type" "fpspc")
    (set_attr "mode" "DF")])
 
-(define_insn "cossf2"
+(define_insn "*cossf2"
   [(set (match_operand:SF 0 "register_operand" "=f")
        (unspec:SF [(match_operand:SF 1 "register_operand" "0")] UNSPEC_COS))]
   "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
   [(set_attr "type" "fpspc")
    (set_attr "mode" "DF")])
 
-(define_insn "cosxf2"
+(define_insn "*cosxf2"
   [(set (match_operand:XF 0 "register_operand" "=f")
        (unspec:XF [(match_operand:XF 1 "register_operand" "0")] UNSPEC_COS))]
-  "!TARGET_64BIT && ! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
    && flag_unsafe_math_optimizations"
   "fcos"
   [(set_attr "type" "fpspc")
    (set_attr "mode" "XF")])
 
-(define_insn "costf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (unspec:TF [(match_operand:TF 1 "register_operand" "0")] UNSPEC_COS))]
-  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
-   && flag_unsafe_math_optimizations"
-  "fcos"
-  [(set_attr "type" "fpspc")
-   (set_attr "mode" "XF")])
+;; With sincos pattern defined, sin and cos builtin function will be
+;; expanded to sincos pattern with one of its outputs left unused. 
+;; Cse pass  will detected, if two sincos patterns can be combined,
+;; otherwise sincos pattern will be splitted back to sin or cos pattern,
+;; depending on the unused output.
 
-(define_insn "atan2df3"
-  [(parallel [(set (match_operand:DF 0 "register_operand" "=f")
-                  (unspec:DF [(match_operand:DF 2 "register_operand" "0")
-                              (match_operand:DF 1 "register_operand" "u")]
-                   UNSPEC_FPATAN))
-             (clobber (match_dup 1))])]
-  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+(define_insn "sincosdf3"
+  [(set (match_operand:DF 0 "register_operand" "=f")
+       (unspec:DF [(match_operand:DF 2 "register_operand" "0")]
+                  UNSPEC_SINCOS_COS))
+   (set (match_operand:DF 1 "register_operand" "=u")
+        (unspec:DF [(match_dup 2)] UNSPEC_SINCOS_SIN))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
    && flag_unsafe_math_optimizations"
-  "fpatan"
+  "fsincos"
   [(set_attr "type" "fpspc")
    (set_attr "mode" "DF")])
 
-(define_insn "atan2sf3"
-  [(parallel [(set (match_operand:SF 0 "register_operand" "=f")
-                  (unspec:SF [(match_operand:SF 2 "register_operand" "0")
-                              (match_operand:SF 1 "register_operand" "u")]
-                   UNSPEC_FPATAN))
-             (clobber (match_dup 1))])]
-  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+(define_split
+  [(set (match_operand:DF 0 "register_operand" "")
+       (unspec:DF [(match_operand:DF 2 "register_operand" "")]
+                  UNSPEC_SINCOS_COS))
+   (set (match_operand:DF 1 "register_operand" "")
+       (unspec:DF [(match_dup 2)] UNSPEC_SINCOS_SIN))]
+  "find_regno_note (insn, REG_UNUSED, REGNO (operands[0]))
+   && !reload_completed && !reload_in_progress"
+  [(set (match_dup 1) (unspec:DF [(match_dup 2)] UNSPEC_SIN))]
+  "")
+
+(define_split
+  [(set (match_operand:DF 0 "register_operand" "")
+       (unspec:DF [(match_operand:DF 2 "register_operand" "")]
+                  UNSPEC_SINCOS_COS))
+   (set (match_operand:DF 1 "register_operand" "")
+       (unspec:DF [(match_dup 2)] UNSPEC_SINCOS_SIN))]
+  "find_regno_note (insn, REG_UNUSED, REGNO (operands[1]))
+   && !reload_completed && !reload_in_progress"
+  [(set (match_dup 0) (unspec:DF [(match_dup 2)] UNSPEC_COS))]
+  "")
+
+(define_insn "sincossf3"
+  [(set (match_operand:SF 0 "register_operand" "=f")
+       (unspec:SF [(match_operand:SF 2 "register_operand" "0")]
+                  UNSPEC_SINCOS_COS))
+   (set (match_operand:SF 1 "register_operand" "=u")
+        (unspec:SF [(match_dup 2)] UNSPEC_SINCOS_SIN))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
    && flag_unsafe_math_optimizations"
-  "fpatan"
+  "fsincos"
   [(set_attr "type" "fpspc")
    (set_attr "mode" "SF")])
 
-(define_insn "atan2xf3"
-  [(parallel [(set (match_operand:XF 0 "register_operand" "=f")
-                  (unspec:XF [(match_operand:XF 2 "register_operand" "0")
-                              (match_operand:XF 1 "register_operand" "u")]
-                   UNSPEC_FPATAN))
-             (clobber (match_dup 1))])]
-  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+(define_split
+  [(set (match_operand:SF 0 "register_operand" "")
+       (unspec:SF [(match_operand:SF 2 "register_operand" "")]
+                  UNSPEC_SINCOS_COS))
+   (set (match_operand:SF 1 "register_operand" "")
+       (unspec:SF [(match_dup 2)] UNSPEC_SINCOS_SIN))]
+  "find_regno_note (insn, REG_UNUSED, REGNO (operands[0]))
+   && !reload_completed && !reload_in_progress"
+  [(set (match_dup 1) (unspec:SF [(match_dup 2)] UNSPEC_SIN))]
+  "")
+
+(define_split
+  [(set (match_operand:SF 0 "register_operand" "")
+       (unspec:SF [(match_operand:SF 2 "register_operand" "")]
+                  UNSPEC_SINCOS_COS))
+   (set (match_operand:SF 1 "register_operand" "")
+       (unspec:SF [(match_dup 2)] UNSPEC_SINCOS_SIN))]
+  "find_regno_note (insn, REG_UNUSED, REGNO (operands[1]))
+   && !reload_completed && !reload_in_progress"
+  [(set (match_dup 0) (unspec:SF [(match_dup 2)] UNSPEC_COS))]
+  "")
+
+(define_insn "*sincosextendsfdf3"
+  [(set (match_operand:DF 0 "register_operand" "=f")
+       (unspec:DF [(float_extend:DF
+                    (match_operand:SF 2 "register_operand" "0"))]
+                  UNSPEC_SINCOS_COS))
+   (set (match_operand:DF 1 "register_operand" "=u")
+        (unspec:DF [(float_extend:DF
+                    (match_dup 2))] UNSPEC_SINCOS_SIN))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
    && flag_unsafe_math_optimizations"
-  "fpatan"
+  "fsincos"
   [(set_attr "type" "fpspc")
-   (set_attr "mode" "XF")])
+   (set_attr "mode" "DF")])
 
-(define_insn "atan2tf3"
-  [(parallel [(set (match_operand:TF 0 "register_operand" "=f")
-                  (unspec:TF [(match_operand:TF 2 "register_operand" "0")
-                              (match_operand:TF 1 "register_operand" "u")]
-                   UNSPEC_FPATAN))
-             (clobber (match_dup 1))])]
-  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+(define_split
+  [(set (match_operand:DF 0 "register_operand" "")
+       (unspec:DF [(float_extend:DF
+                    (match_operand:SF 2 "register_operand" ""))]
+                  UNSPEC_SINCOS_COS))
+   (set (match_operand:DF 1 "register_operand" "")
+        (unspec:DF [(float_extend:DF
+                    (match_dup 2))] UNSPEC_SINCOS_SIN))]
+  "find_regno_note (insn, REG_UNUSED, REGNO (operands[0]))
+   && !reload_completed && !reload_in_progress"
+  [(set (match_dup 1) (unspec:DF [(float_extend:DF
+                                  (match_dup 2))] UNSPEC_SIN))]
+  "")
+
+(define_split
+  [(set (match_operand:DF 0 "register_operand" "")
+       (unspec:DF [(float_extend:DF
+                    (match_operand:SF 2 "register_operand" ""))]
+                  UNSPEC_SINCOS_COS))
+   (set (match_operand:DF 1 "register_operand" "")
+        (unspec:DF [(float_extend:DF
+                    (match_dup 2))] UNSPEC_SINCOS_SIN))]
+  "find_regno_note (insn, REG_UNUSED, REGNO (operands[1]))
+   && !reload_completed && !reload_in_progress"
+  [(set (match_dup 0) (unspec:DF [(float_extend:DF
+                                  (match_dup 2))] UNSPEC_COS))]
+  "")
+
+(define_insn "sincosxf3"
+  [(set (match_operand:XF 0 "register_operand" "=f")
+       (unspec:XF [(match_operand:XF 2 "register_operand" "0")]
+                  UNSPEC_SINCOS_COS))
+   (set (match_operand:XF 1 "register_operand" "=u")
+        (unspec:XF [(match_dup 2)] UNSPEC_SINCOS_SIN))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
    && flag_unsafe_math_optimizations"
-  "fpatan"
+  "fsincos"
   [(set_attr "type" "fpspc")
    (set_attr "mode" "XF")])
 
-(define_insn "*fyl2x_sfxf3"
-  [(parallel [(set (match_operand:SF 0 "register_operand" "=f")
-                  (unspec:SF [(match_operand:SF 2 "register_operand" "0")
-                              (match_operand:XF 1 "register_operand" "u")]
-                   UNSPEC_FYL2X))
-             (clobber (match_dup 1))])]
-  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
-   && flag_unsafe_math_optimizations"
-  "fyl2x"
-  [(set_attr "type" "fpspc")
-   (set_attr "mode" "SF")])
+(define_split
+  [(set (match_operand:XF 0 "register_operand" "")
+       (unspec:XF [(match_operand:XF 2 "register_operand" "")]
+                  UNSPEC_SINCOS_COS))
+   (set (match_operand:XF 1 "register_operand" "")
+       (unspec:XF [(match_dup 2)] UNSPEC_SINCOS_SIN))]
+  "find_regno_note (insn, REG_UNUSED, REGNO (operands[0]))
+   && !reload_completed && !reload_in_progress"
+  [(set (match_dup 1) (unspec:XF [(match_dup 2)] UNSPEC_SIN))]
+  "")
 
-(define_insn "*fyl2x_dfxf3"
-  [(parallel [(set (match_operand:DF 0 "register_operand" "=f")
-                  (unspec:DF [(match_operand:DF 2 "register_operand" "0")
-                              (match_operand:XF 1 "register_operand" "u")]
-                   UNSPEC_FYL2X))
-             (clobber (match_dup 1))])]
-  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+(define_split
+  [(set (match_operand:XF 0 "register_operand" "")
+       (unspec:XF [(match_operand:XF 2 "register_operand" "")]
+                  UNSPEC_SINCOS_COS))
+   (set (match_operand:XF 1 "register_operand" "")
+       (unspec:XF [(match_dup 2)] UNSPEC_SINCOS_SIN))]
+  "find_regno_note (insn, REG_UNUSED, REGNO (operands[1]))
+   && !reload_completed && !reload_in_progress"
+  [(set (match_dup 0) (unspec:XF [(match_dup 2)] UNSPEC_COS))]
+  "")
+
+(define_insn "*tandf3_1"
+  [(set (match_operand:DF 0 "register_operand" "=f")
+       (unspec:DF [(match_operand:DF 2 "register_operand" "0")]
+                  UNSPEC_TAN_ONE))
+   (set (match_operand:DF 1 "register_operand" "=u")
+        (unspec:DF [(match_dup 2)] UNSPEC_TAN_TAN))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
    && flag_unsafe_math_optimizations"
-  "fyl2x"
+  "fptan"
   [(set_attr "type" "fpspc")
    (set_attr "mode" "DF")])
 
-(define_insn "*fyl2x_xf3"
-  [(parallel [(set (match_operand:XF 0 "register_operand" "=f")
-                  (unspec:XF [(match_operand:XF 2 "register_operand" "0")
-                              (match_operand:XF 1 "register_operand" "u")]
-                   UNSPEC_FYL2X))
-             (clobber (match_dup 1))])]
-  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
-   && flag_unsafe_math_optimizations"
-  "fyl2x"
-  [(set_attr "type" "fpspc")
-   (set_attr "mode" "XF")])
+;; optimize sequence: fptan
+;;                   fstp    %st(0)
+;;                   fld1
+;; into fptan insn.
 
-(define_insn "*fyl2x_tfxf3"
-  [(parallel [(set (match_operand:TF 0 "register_operand" "=f")
-                  (unspec:TF [(match_operand:TF 2 "register_operand" "0")
-                              (match_operand:XF 1 "register_operand" "u")]
-                   UNSPEC_FYL2X))
-             (clobber (match_dup 1))])]
-  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
-   && flag_unsafe_math_optimizations"
-  "fyl2x"
-  [(set_attr "type" "fpspc")
-   (set_attr "mode" "XF")])
+(define_peephole2
+  [(parallel[(set (match_operand:DF 0 "register_operand" "")
+                 (unspec:DF [(match_operand:DF 2 "register_operand" "")]
+                            UNSPEC_TAN_ONE))
+            (set (match_operand:DF 1 "register_operand" "")
+                 (unspec:DF [(match_dup 2)] UNSPEC_TAN_TAN))])
+   (set (match_dup 0)
+        (match_operand:DF 3 "immediate_operand" ""))]
+  "standard_80387_constant_p (operands[3]) == 2"
+  [(parallel[(set (match_dup 0) (unspec:DF [(match_dup 2)] UNSPEC_TAN_ONE))
+            (set (match_dup 1) (unspec:DF [(match_dup 2)] UNSPEC_TAN_TAN))])]
+  "")
 
-(define_expand "logsf2"
-  [(parallel [(set (match_operand:SF 0 "register_operand" "")
-                  (unspec:SF [(match_operand:SF 1 "register_operand" "")
-                              (match_dup 2)] UNSPEC_FYL2X))
-             (clobber (match_dup 2))])]
-  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+(define_expand "tandf2"
+  [(parallel [(set (match_dup 2)
+                  (unspec:DF [(match_operand:DF 1 "register_operand" "")]
+                             UNSPEC_TAN_ONE))
+             (set (match_operand:DF 0 "register_operand" "")
+                  (unspec:DF [(match_dup 1)] UNSPEC_TAN_TAN))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
    && flag_unsafe_math_optimizations"
 {
-  rtx temp;
-
-  operands[2] = gen_reg_rtx (XFmode);
-  temp = standard_80387_constant_rtx (4); /* fldln2 */
-  emit_move_insn (operands[2], temp);
+  operands[2] = gen_reg_rtx (DFmode);
 })
 
-(define_expand "logdf2"
+(define_insn "*tansf3_1"
+  [(set (match_operand:SF 0 "register_operand" "=f")
+       (unspec:SF [(match_operand:SF 2 "register_operand" "0")]
+                  UNSPEC_TAN_ONE))
+   (set (match_operand:SF 1 "register_operand" "=u")
+        (unspec:SF [(match_dup 2)] UNSPEC_TAN_TAN))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
+   && flag_unsafe_math_optimizations"
+  "fptan"
+  [(set_attr "type" "fpspc")
+   (set_attr "mode" "SF")])
+
+;; optimize sequence: fptan
+;;                   fstp    %st(0)
+;;                   fld1
+;; into fptan insn.
+
+(define_peephole2
+  [(parallel[(set (match_operand:SF 0 "register_operand" "")
+                 (unspec:SF [(match_operand:SF 2 "register_operand" "")]
+                            UNSPEC_TAN_ONE))
+            (set (match_operand:SF 1 "register_operand" "")
+                 (unspec:SF [(match_dup 2)] UNSPEC_TAN_TAN))])
+   (set (match_dup 0)
+        (match_operand:SF 3 "immediate_operand" ""))]
+  "standard_80387_constant_p (operands[3]) == 2"
+  [(parallel[(set (match_dup 0) (unspec:SF [(match_dup 2)] UNSPEC_TAN_ONE))
+            (set (match_dup 1) (unspec:SF [(match_dup 2)] UNSPEC_TAN_TAN))])]
+  "")
+
+(define_expand "tansf2"
+  [(parallel [(set (match_dup 2)
+                  (unspec:SF [(match_operand:SF 1 "register_operand" "")]
+                             UNSPEC_TAN_ONE))
+             (set (match_operand:SF 0 "register_operand" "")
+                  (unspec:SF [(match_dup 1)] UNSPEC_TAN_TAN))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
+   && flag_unsafe_math_optimizations"
+{
+  operands[2] = gen_reg_rtx (SFmode);
+})
+
+(define_insn "*tanxf3_1"
+  [(set (match_operand:XF 0 "register_operand" "=f")
+       (unspec:XF [(match_operand:XF 2 "register_operand" "0")]
+                  UNSPEC_TAN_ONE))
+   (set (match_operand:XF 1 "register_operand" "=u")
+        (unspec:XF [(match_dup 2)] UNSPEC_TAN_TAN))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
+   && flag_unsafe_math_optimizations"
+  "fptan"
+  [(set_attr "type" "fpspc")
+   (set_attr "mode" "XF")])
+
+;; optimize sequence: fptan
+;;                   fstp    %st(0)
+;;                   fld1
+;; into fptan insn.
+
+(define_peephole2
+  [(parallel[(set (match_operand:XF 0 "register_operand" "")
+                 (unspec:XF [(match_operand:XF 2 "register_operand" "")]
+                            UNSPEC_TAN_ONE))
+            (set (match_operand:XF 1 "register_operand" "")
+                 (unspec:XF [(match_dup 2)] UNSPEC_TAN_TAN))])
+   (set (match_dup 0)
+        (match_operand:XF 3 "immediate_operand" ""))]
+  "standard_80387_constant_p (operands[3]) == 2"
+  [(parallel[(set (match_dup 0) (unspec:XF [(match_dup 2)] UNSPEC_TAN_ONE))
+            (set (match_dup 1) (unspec:XF [(match_dup 2)] UNSPEC_TAN_TAN))])]
+  "")
+
+(define_expand "tanxf2"
+  [(parallel [(set (match_dup 2)
+                  (unspec:XF [(match_operand:XF 1 "register_operand" "")]
+                             UNSPEC_TAN_ONE))
+             (set (match_operand:XF 0 "register_operand" "")
+                  (unspec:XF [(match_dup 1)] UNSPEC_TAN_TAN))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
+   && flag_unsafe_math_optimizations"
+{
+  operands[2] = gen_reg_rtx (XFmode);
+})
+
+(define_insn "atan2df3_1"
+  [(set (match_operand:DF 0 "register_operand" "=f")
+       (unspec:DF [(match_operand:DF 2 "register_operand" "0")
+                   (match_operand:DF 1 "register_operand" "u")]
+                  UNSPEC_FPATAN))
+   (clobber (match_scratch:DF 3 "=1"))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+  "fpatan"
+  [(set_attr "type" "fpspc")
+   (set_attr "mode" "DF")])
+
+(define_expand "atan2df3"
+  [(use (match_operand:DF 0 "register_operand" "=f"))
+   (use (match_operand:DF 2 "register_operand" "0"))
+   (use (match_operand:DF 1 "register_operand" "u"))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  rtx copy = gen_reg_rtx (DFmode);
+  emit_move_insn (copy, operands[1]);
+  emit_insn (gen_atan2df3_1 (operands[0], copy, operands[2]));
+  DONE;
+})
+
+(define_insn "atan2sf3_1"
+  [(set (match_operand:SF 0 "register_operand" "=f")
+        (unspec:SF [(match_operand:SF 2 "register_operand" "0")
+                   (match_operand:SF 1 "register_operand" "u")]
+                  UNSPEC_FPATAN))
+   (clobber (match_scratch:SF 3 "=1"))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+  "fpatan"
+  [(set_attr "type" "fpspc")
+   (set_attr "mode" "SF")])
+
+(define_expand "atan2sf3"
+  [(use (match_operand:SF 0 "register_operand" "=f"))
+   (use (match_operand:SF 2 "register_operand" "0"))
+   (use (match_operand:SF 1 "register_operand" "u"))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  rtx copy = gen_reg_rtx (SFmode);
+  emit_move_insn (copy, operands[1]);
+  emit_insn (gen_atan2sf3_1 (operands[0], copy, operands[2]));
+  DONE;
+})
+
+(define_insn "atan2xf3_1"
+  [(set (match_operand:XF 0 "register_operand" "=f")
+        (unspec:XF [(match_operand:XF 2 "register_operand" "0")
+                   (match_operand:XF 1 "register_operand" "u")]
+                  UNSPEC_FPATAN))
+   (clobber (match_scratch:XF 3 "=1"))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+  "fpatan"
+  [(set_attr "type" "fpspc")
+   (set_attr "mode" "XF")])
+
+(define_expand "atan2xf3"
+  [(use (match_operand:XF 0 "register_operand" "=f"))
+   (use (match_operand:XF 2 "register_operand" "0"))
+   (use (match_operand:XF 1 "register_operand" "u"))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  rtx copy = gen_reg_rtx (XFmode);
+  emit_move_insn (copy, operands[1]);
+  emit_insn (gen_atan2xf3_1 (operands[0], copy, operands[2]));
+  DONE;
+})
+
+(define_expand "asindf2"
+  [(set (match_dup 2)
+       (float_extend:XF (match_operand:DF 1 "register_operand" "")))
+   (set (match_dup 3) (mult:XF (match_dup 2) (match_dup 2)))
+   (set (match_dup 5) (minus:XF (match_dup 4) (match_dup 3)))
+   (set (match_dup 6) (sqrt:XF (match_dup 5)))
+   (parallel [(set (match_dup 7)
+                  (unspec:XF [(match_dup 6) (match_dup 2)]
+                             UNSPEC_FPATAN))
+             (clobber (match_scratch:XF 8 ""))])
+   (set (match_operand:DF 0 "register_operand" "")
+       (float_truncate:DF (match_dup 7)))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  int i;
+
+  for (i=2; i<8; i++)
+    operands[i] = gen_reg_rtx (XFmode);
+
+  emit_move_insn (operands[4], CONST1_RTX (XFmode));  /* fld1 */
+})
+
+(define_expand "asinsf2"
+  [(set (match_dup 2)
+       (float_extend:XF (match_operand:SF 1 "register_operand" "")))
+   (set (match_dup 3) (mult:XF (match_dup 2) (match_dup 2)))
+   (set (match_dup 5) (minus:XF (match_dup 4) (match_dup 3)))
+   (set (match_dup 6) (sqrt:XF (match_dup 5)))
+   (parallel [(set (match_dup 7)
+                  (unspec:XF [(match_dup 6) (match_dup 2)]
+                             UNSPEC_FPATAN))
+             (clobber (match_scratch:XF 8 ""))])
+   (set (match_operand:SF 0 "register_operand" "")
+       (float_truncate:SF (match_dup 7)))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  int i;
+
+  for (i=2; i<8; i++)
+    operands[i] = gen_reg_rtx (XFmode);
+
+  emit_move_insn (operands[4], CONST1_RTX (XFmode));  /* fld1 */
+})
+
+(define_expand "asinxf2"
+  [(set (match_dup 2)
+       (mult:XF (match_operand:XF 1 "register_operand" "")
+                (match_dup 1)))
+   (set (match_dup 4) (minus:XF (match_dup 3) (match_dup 2)))
+   (set (match_dup 5) (sqrt:XF (match_dup 4)))
+   (parallel [(set (match_operand:XF 0 "register_operand" "")
+                  (unspec:XF [(match_dup 5) (match_dup 1)]
+                             UNSPEC_FPATAN))
+             (clobber (match_scratch:XF 6 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  int i;
+
+  for (i=2; i<6; i++)
+    operands[i] = gen_reg_rtx (XFmode);
+
+  emit_move_insn (operands[3], CONST1_RTX (XFmode));  /* fld1 */
+})
+
+(define_expand "acosdf2"
+  [(set (match_dup 2)
+       (float_extend:XF (match_operand:DF 1 "register_operand" "")))
+   (set (match_dup 3) (mult:XF (match_dup 2) (match_dup 2)))
+   (set (match_dup 5) (minus:XF (match_dup 4) (match_dup 3)))
+   (set (match_dup 6) (sqrt:XF (match_dup 5)))
+   (parallel [(set (match_dup 7)
+                  (unspec:XF [(match_dup 2) (match_dup 6)]
+                             UNSPEC_FPATAN))
+             (clobber (match_scratch:XF 8 ""))])
+   (set (match_operand:DF 0 "register_operand" "")
+       (float_truncate:DF (match_dup 7)))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  int i;
+
+  for (i=2; i<8; i++)
+    operands[i] = gen_reg_rtx (XFmode);
+
+  emit_move_insn (operands[4], CONST1_RTX (XFmode));  /* fld1 */
+})
+
+(define_expand "acossf2"
+  [(set (match_dup 2)
+       (float_extend:XF (match_operand:SF 1 "register_operand" "")))
+   (set (match_dup 3) (mult:XF (match_dup 2) (match_dup 2)))
+   (set (match_dup 5) (minus:XF (match_dup 4) (match_dup 3)))
+   (set (match_dup 6) (sqrt:XF (match_dup 5)))
+   (parallel [(set (match_dup 7)
+                  (unspec:XF [(match_dup 2) (match_dup 6)]
+                             UNSPEC_FPATAN))
+             (clobber (match_scratch:XF 8 ""))])
+   (set (match_operand:SF 0 "register_operand" "")
+       (float_truncate:SF (match_dup 7)))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  int i;
+
+  for (i=2; i<8; i++)
+    operands[i] = gen_reg_rtx (XFmode);
+
+  emit_move_insn (operands[4], CONST1_RTX (XFmode));  /* fld1 */
+})
+
+(define_expand "acosxf2"
+  [(set (match_dup 2)
+       (mult:XF (match_operand:XF 1 "register_operand" "")
+                (match_dup 1)))
+   (set (match_dup 4) (minus:XF (match_dup 3) (match_dup 2)))
+   (set (match_dup 5) (sqrt:XF (match_dup 4)))
+   (parallel [(set (match_operand:XF 0 "register_operand" "")
+                  (unspec:XF [(match_dup 1) (match_dup 5)]
+                             UNSPEC_FPATAN))
+             (clobber (match_scratch:XF 6 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  int i;
+
+  for (i=2; i<6; i++)
+    operands[i] = gen_reg_rtx (XFmode);
+
+  emit_move_insn (operands[3], CONST1_RTX (XFmode));  /* fld1 */
+})
+
+(define_insn "*fyl2x_sfxf3"
+  [(set (match_operand:SF 0 "register_operand" "=f")
+         (unspec:SF [(match_operand:SF 2 "register_operand" "0")
+                    (match_operand:XF 1 "register_operand" "u")]
+                   UNSPEC_FYL2X))
+   (clobber (match_scratch:SF 3 "=1"))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+  "fyl2x"
+  [(set_attr "type" "fpspc")
+   (set_attr "mode" "SF")])
+
+(define_insn "*fyl2x_dfxf3"
+  [(set (match_operand:DF 0 "register_operand" "=f")
+         (unspec:DF [(match_operand:DF 2 "register_operand" "0")
+                    (match_operand:XF 1 "register_operand" "u")]
+                   UNSPEC_FYL2X))
+   (clobber (match_scratch:DF 3 "=1"))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+  "fyl2x"
+  [(set_attr "type" "fpspc")
+   (set_attr "mode" "DF")])
+
+(define_insn "*fyl2x_xf3"
+  [(set (match_operand:XF 0 "register_operand" "=f")
+        (unspec:XF [(match_operand:XF 2 "register_operand" "0")
+                   (match_operand:XF 1 "register_operand" "u")]
+                  UNSPEC_FYL2X))
+   (clobber (match_scratch:XF 3 "=1"))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+  "fyl2x"
+  [(set_attr "type" "fpspc")
+   (set_attr "mode" "XF")])
+
+(define_expand "logsf2"
+  [(parallel [(set (match_operand:SF 0 "register_operand" "")
+                  (unspec:SF [(match_operand:SF 1 "register_operand" "")
+                              (match_dup 2)] UNSPEC_FYL2X))
+             (clobber (match_scratch:SF 3 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  rtx temp;
+
+  operands[2] = gen_reg_rtx (XFmode);
+  temp = standard_80387_constant_rtx (4); /* fldln2 */
+  emit_move_insn (operands[2], temp);
+})
+
+(define_expand "logdf2"
   [(parallel [(set (match_operand:DF 0 "register_operand" "")
                   (unspec:DF [(match_operand:DF 1 "register_operand" "")
                               (match_dup 2)] UNSPEC_FYL2X))
-             (clobber (match_dup 2))])]
+             (clobber (match_scratch:DF 3 ""))])]
   "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
    && flag_unsafe_math_optimizations"
 {
   [(parallel [(set (match_operand:XF 0 "register_operand" "")
                   (unspec:XF [(match_operand:XF 1 "register_operand" "")
                               (match_dup 2)] UNSPEC_FYL2X))
-             (clobber (match_dup 2))])]
+             (clobber (match_scratch:XF 3 ""))])]
   "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
    && flag_unsafe_math_optimizations"
 {
   emit_move_insn (operands[2], temp);
 })
 
-(define_expand "logtf2"
-  [(parallel [(set (match_operand:TF 0 "register_operand" "")
-                  (unspec:TF [(match_operand:TF 1 "register_operand" "")
+(define_expand "log10sf2"
+  [(parallel [(set (match_operand:SF 0 "register_operand" "")
+                  (unspec:SF [(match_operand:SF 1 "register_operand" "")
                               (match_dup 2)] UNSPEC_FYL2X))
-             (clobber (match_dup 2))])]
+             (clobber (match_scratch:SF 3 ""))])]
   "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
    && flag_unsafe_math_optimizations"
 {
   rtx temp;
 
   operands[2] = gen_reg_rtx (XFmode);
-  temp = standard_80387_constant_rtx (4); /* fldln2 */
+  temp = standard_80387_constant_rtx (3); /* fldlg2 */
   emit_move_insn (operands[2], temp);
 })
-\f
-;; Block operation instructions
 
-(define_insn "cld"
- [(set (reg:SI 19) (const_int 0))]
- ""
- "cld"
-  [(set_attr "type" "cld")])
+(define_expand "log10df2"
+  [(parallel [(set (match_operand:DF 0 "register_operand" "")
+                  (unspec:DF [(match_operand:DF 1 "register_operand" "")
+                              (match_dup 2)] UNSPEC_FYL2X))
+             (clobber (match_scratch:DF 3 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  rtx temp;
 
-(define_expand "movstrsi"
-  [(use (match_operand:BLK 0 "memory_operand" ""))
-   (use (match_operand:BLK 1 "memory_operand" ""))
-   (use (match_operand:SI 2 "nonmemory_operand" ""))
-   (use (match_operand:SI 3 "const_int_operand" ""))]
-  ""
+  operands[2] = gen_reg_rtx (XFmode);
+  temp = standard_80387_constant_rtx (3); /* fldlg2 */
+  emit_move_insn (operands[2], temp);
+})
+
+(define_expand "log10xf2"
+  [(parallel [(set (match_operand:XF 0 "register_operand" "")
+                  (unspec:XF [(match_operand:XF 1 "register_operand" "")
+                              (match_dup 2)] UNSPEC_FYL2X))
+             (clobber (match_scratch:XF 3 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
 {
- if (ix86_expand_movstr (operands[0], operands[1], operands[2], operands[3]))
-   DONE;
- else
-   FAIL;
+  rtx temp;
+
+  operands[2] = gen_reg_rtx (XFmode);
+  temp = standard_80387_constant_rtx (3); /* fldlg2 */
+  emit_move_insn (operands[2], temp);
 })
 
-(define_expand "movstrdi"
-  [(use (match_operand:BLK 0 "memory_operand" ""))
-   (use (match_operand:BLK 1 "memory_operand" ""))
-   (use (match_operand:DI 2 "nonmemory_operand" ""))
-   (use (match_operand:DI 3 "const_int_operand" ""))]
-  "TARGET_64BIT"
+(define_expand "log2sf2"
+  [(parallel [(set (match_operand:SF 0 "register_operand" "")
+                  (unspec:SF [(match_operand:SF 1 "register_operand" "")
+                              (match_dup 2)] UNSPEC_FYL2X))
+             (clobber (match_scratch:SF 3 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
 {
- if (ix86_expand_movstr (operands[0], operands[1], operands[2], operands[3]))
-   DONE;
- else
-   FAIL;
+  operands[2] = gen_reg_rtx (XFmode);
+  emit_move_insn (operands[2], CONST1_RTX (XFmode));  /* fld1 */
+
 })
 
-;; Most CPUs don't like single string operations
-;; Handle this case here to simplify previous expander.
+(define_expand "log2df2"
+  [(parallel [(set (match_operand:DF 0 "register_operand" "")
+                  (unspec:DF [(match_operand:DF 1 "register_operand" "")
+                              (match_dup 2)] UNSPEC_FYL2X))
+             (clobber (match_scratch:DF 3 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  operands[2] = gen_reg_rtx (XFmode);
+  emit_move_insn (operands[2], CONST1_RTX (XFmode));  /* fld1 */
+})
 
-(define_expand "strmovdi_rex64"
-  [(set (match_dup 2)
-       (mem:DI (match_operand:DI 1 "register_operand" "")))
-   (set (mem:DI (match_operand:DI 0 "register_operand" ""))
-        (match_dup 2))
-   (parallel [(set (match_dup 0) (plus:DI (match_dup 0) (const_int 8)))
-             (clobber (reg:CC 17))])
-   (parallel [(set (match_dup 1) (plus:DI (match_dup 1) (const_int 8)))
+(define_expand "log2xf2"
+  [(parallel [(set (match_operand:XF 0 "register_operand" "")
+                  (unspec:XF [(match_operand:XF 1 "register_operand" "")
+                              (match_dup 2)] UNSPEC_FYL2X))
+             (clobber (match_scratch:XF 3 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  operands[2] = gen_reg_rtx (XFmode);
+  emit_move_insn (operands[2], CONST1_RTX (XFmode));  /* fld1 */
+})
+
+(define_insn "*fxtractdf3"
+  [(set (match_operand:DF 0 "register_operand" "=f")
+       (unspec:DF [(match_operand:DF 2 "register_operand" "0")]
+                  UNSPEC_XTRACT_FRACT))
+   (set (match_operand:DF 1 "register_operand" "=u")
+        (unspec:DF [(match_dup 2)] UNSPEC_XTRACT_EXP))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
+   && flag_unsafe_math_optimizations"
+  "fxtract"
+  [(set_attr "type" "fpspc")
+   (set_attr "mode" "DF")])
+
+(define_expand "logbdf2"
+  [(parallel [(set (match_dup 2)
+                  (unspec:DF [(match_operand:DF 1 "register_operand" "")]
+                             UNSPEC_XTRACT_FRACT))
+             (set (match_operand:DF 0 "register_operand" "")
+                  (unspec:DF [(match_dup 1)] UNSPEC_XTRACT_EXP))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
+   && flag_unsafe_math_optimizations"
+{
+  operands[2] = gen_reg_rtx (DFmode);
+})
+
+(define_insn "*fxtractsf3"
+  [(set (match_operand:SF 0 "register_operand" "=f")
+       (unspec:SF [(match_operand:SF 2 "register_operand" "0")]
+                  UNSPEC_XTRACT_FRACT))
+   (set (match_operand:SF 1 "register_operand" "=u")
+        (unspec:SF [(match_dup 2)] UNSPEC_XTRACT_EXP))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
+   && flag_unsafe_math_optimizations"
+  "fxtract"
+  [(set_attr "type" "fpspc")
+   (set_attr "mode" "SF")])
+
+(define_expand "logbsf2"
+  [(parallel [(set (match_dup 2)
+                  (unspec:SF [(match_operand:SF 1 "register_operand" "")]
+                             UNSPEC_XTRACT_FRACT))
+             (set (match_operand:SF 0 "register_operand" "")
+                  (unspec:SF [(match_dup 1)] UNSPEC_XTRACT_EXP))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
+   && flag_unsafe_math_optimizations"
+{
+  operands[2] = gen_reg_rtx (SFmode);
+})
+
+(define_insn "*fxtractxf3"
+  [(set (match_operand:XF 0 "register_operand" "=f")
+       (unspec:XF [(match_operand:XF 2 "register_operand" "0")]
+                  UNSPEC_XTRACT_FRACT))
+   (set (match_operand:XF 1 "register_operand" "=u")
+        (unspec:XF [(match_dup 2)] UNSPEC_XTRACT_EXP))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
+   && flag_unsafe_math_optimizations"
+  "fxtract"
+  [(set_attr "type" "fpspc")
+   (set_attr "mode" "XF")])
+
+(define_expand "logbxf2"
+  [(parallel [(set (match_dup 2)
+                  (unspec:XF [(match_operand:XF 1 "register_operand" "")]
+                             UNSPEC_XTRACT_FRACT))
+             (set (match_operand:XF 0 "register_operand" "")
+                  (unspec:XF [(match_dup 1)] UNSPEC_XTRACT_EXP))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
+   && flag_unsafe_math_optimizations"
+{
+  operands[2] = gen_reg_rtx (XFmode);
+})
+
+(define_expand "ilogbsi2"
+  [(parallel [(set (match_dup 2)
+                  (unspec:XF [(match_operand:XF 1 "register_operand" "")]
+                             UNSPEC_XTRACT_FRACT))
+             (set (match_operand:XF 3 "register_operand" "")
+                  (unspec:XF [(match_dup 1)] UNSPEC_XTRACT_EXP))])
+   (parallel [(set (match_operand:SI 0 "register_operand" "")
+                  (fix:SI (match_dup 3)))
              (clobber (reg:CC 17))])]
-  "TARGET_64BIT"
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387 
+   && flag_unsafe_math_optimizations"
 {
-  if (TARGET_SINGLE_STRINGOP || optimize_size)
-    {
-      emit_insn (gen_strmovdi_rex_1 (operands[0], operands[1], operands[0],
-                                    operands[1]));
-      DONE;
-    }
-  else 
-    operands[2] = gen_reg_rtx (DImode);
+  operands[2] = gen_reg_rtx (XFmode);
+  operands[3] = gen_reg_rtx (XFmode);
 })
 
+(define_insn "*fscale_sfxf3"
+  [(set (match_operand:SF 0 "register_operand" "=f")
+        (unspec:SF [(match_operand:XF 2 "register_operand" "0")
+                    (match_operand:XF 1 "register_operand" "u")]
+                   UNSPEC_FSCALE))
+   (clobber (match_scratch:SF 3 "=1"))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+  "fscale\;fstp\t%y1"
+  [(set_attr "type" "fpspc")
+   (set_attr "mode" "SF")])
 
-(define_expand "strmovsi"
+(define_insn "*fscale_dfxf3"
+  [(set (match_operand:DF 0 "register_operand" "=f")
+        (unspec:DF [(match_operand:XF 2 "register_operand" "0")
+                    (match_operand:XF 1 "register_operand" "u")]
+                   UNSPEC_FSCALE))
+   (clobber (match_scratch:DF 3 "=1"))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+  "fscale\;fstp\t%y1"
+  [(set_attr "type" "fpspc")
+   (set_attr "mode" "DF")])
+
+(define_insn "*fscale_xf3"
+  [(set (match_operand:XF 0 "register_operand" "=f")
+       (unspec:XF [(match_operand:XF 2 "register_operand" "0")
+                   (match_operand:XF 1 "register_operand" "u")]
+                  UNSPEC_FSCALE))
+   (clobber (match_scratch:XF 3 "=1"))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+  "fscale\;fstp\t%y1"
+  [(set_attr "type" "fpspc")
+   (set_attr "mode" "XF")])
+
+(define_insn "*frndintxf2"
+  [(set (match_operand:XF 0 "register_operand" "=f")
+       (unspec:XF [(match_operand:XF 1 "register_operand" "0")]
+        UNSPEC_FRNDINT))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+  "frndint"
+  [(set_attr "type" "fpspc")
+   (set_attr "mode" "XF")])
+
+(define_insn "*f2xm1xf2"
+  [(set (match_operand:XF 0 "register_operand" "=f")
+       (unspec:XF [(match_operand:XF 1 "register_operand" "0")]
+        UNSPEC_F2XM1))]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+  "f2xm1"
+  [(set_attr "type" "fpspc")
+   (set_attr "mode" "XF")])
+
+(define_expand "expsf2"
   [(set (match_dup 2)
-       (mem:SI (match_operand:SI 1 "register_operand" "")))
-   (set (mem:SI (match_operand:SI 0 "register_operand" ""))
-        (match_dup 2))
-   (parallel [(set (match_dup 0) (plus:SI (match_dup 0) (const_int 4)))
-             (clobber (reg:CC 17))])
-   (parallel [(set (match_dup 1) (plus:SI (match_dup 1) (const_int 4)))
-             (clobber (reg:CC 17))])]
-  ""
+       (float_extend:XF (match_operand:SF 1 "register_operand" "")))
+   (set (match_dup 4) (mult:XF (match_dup 2) (match_dup 3)))
+   (set (match_dup 5) (unspec:XF [(match_dup 4)] UNSPEC_FRNDINT))
+   (set (match_dup 6) (minus:XF (match_dup 4) (match_dup 5)))
+   (set (match_dup 7) (unspec:XF [(match_dup 6)] UNSPEC_F2XM1))
+   (set (match_dup 9) (plus:XF (match_dup 7) (match_dup 8)))
+   (parallel [(set (match_operand:SF 0 "register_operand" "")
+                  (unspec:SF [(match_dup 9) (match_dup 5)] UNSPEC_FSCALE))
+             (clobber (match_scratch:SF 5 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
 {
-  if (TARGET_64BIT)
-    {
-      emit_insn (gen_strmovsi_rex64 (operands[0], operands[1]));
-      DONE;
-    }
-  if (TARGET_SINGLE_STRINGOP || optimize_size)
-    {
-      emit_insn (gen_strmovsi_1 (operands[0], operands[1], operands[0],
-                               operands[1]));
-      DONE;
-    }
-  else 
-    operands[2] = gen_reg_rtx (SImode);
+  rtx temp;
+  int i;
+
+  for (i=2; i<10; i++)
+    operands[i] = gen_reg_rtx (XFmode);
+  temp = standard_80387_constant_rtx (5); /* fldl2e */
+  emit_move_insn (operands[3], temp);
+  emit_move_insn (operands[8], CONST1_RTX (XFmode));  /* fld1 */
 })
 
-(define_expand "strmovsi_rex64"
+(define_expand "expdf2"
   [(set (match_dup 2)
-       (mem:SI (match_operand:DI 1 "register_operand" "")))
-   (set (mem:SI (match_operand:DI 0 "register_operand" ""))
-        (match_dup 2))
-   (parallel [(set (match_dup 0) (plus:DI (match_dup 0) (const_int 4)))
-             (clobber (reg:CC 17))])
-   (parallel [(set (match_dup 1) (plus:DI (match_dup 1) (const_int 4)))
-             (clobber (reg:CC 17))])]
-  "TARGET_64BIT"
+       (float_extend:XF (match_operand:DF 1 "register_operand" "")))
+   (set (match_dup 4) (mult:XF (match_dup 2) (match_dup 3)))
+   (set (match_dup 5) (unspec:XF [(match_dup 4)] UNSPEC_FRNDINT))
+   (set (match_dup 6) (minus:XF (match_dup 4) (match_dup 5)))
+
+   (set (match_dup 7) (unspec:XF [(match_dup 6)] UNSPEC_F2XM1))
+   (set (match_dup 9) (plus:XF (match_dup 7) (match_dup 8)))
+   (parallel [(set (match_operand:DF 0 "register_operand" "")
+                  (unspec:DF [(match_dup 9) (match_dup 5)] UNSPEC_FSCALE))
+             (clobber (match_scratch:DF 5 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
 {
-  if (TARGET_SINGLE_STRINGOP || optimize_size)
-    {
-      emit_insn (gen_strmovsi_rex_1 (operands[0], operands[1], operands[0],
-                                    operands[1]));
-      DONE;
-    }
-  else 
-    operands[2] = gen_reg_rtx (SImode);
+  rtx temp;
+  int i;
+
+  for (i=2; i<10; i++)
+    operands[i] = gen_reg_rtx (XFmode);
+  temp = standard_80387_constant_rtx (5); /* fldl2e */
+  emit_move_insn (operands[3], temp);
+  emit_move_insn (operands[8], CONST1_RTX (XFmode));  /* fld1 */
 })
 
-(define_expand "strmovhi"
+(define_expand "expxf2"
+  [(set (match_dup 3) (mult:XF (match_operand:XF 1 "register_operand" "")
+                              (match_dup 2)))
+   (set (match_dup 4) (unspec:XF [(match_dup 3)] UNSPEC_FRNDINT))
+   (set (match_dup 5) (minus:XF (match_dup 3) (match_dup 4)))
+   (set (match_dup 6) (unspec:XF [(match_dup 5)] UNSPEC_F2XM1))
+   (set (match_dup 8) (plus:XF (match_dup 6) (match_dup 7)))
+   (parallel [(set (match_operand:XF 0 "register_operand" "")
+                  (unspec:XF [(match_dup 8) (match_dup 4)] UNSPEC_FSCALE))
+             (clobber (match_scratch:XF 5 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  rtx temp;
+  int i;
+
+  for (i=2; i<9; i++)
+    operands[i] = gen_reg_rtx (XFmode);
+  temp = standard_80387_constant_rtx (5); /* fldl2e */
+  emit_move_insn (operands[2], temp);
+  emit_move_insn (operands[7], CONST1_RTX (XFmode));  /* fld1 */
+})
+
+(define_expand "exp10sf2"
   [(set (match_dup 2)
-       (mem:HI (match_operand:SI 1 "register_operand" "")))
-   (set (mem:HI (match_operand:SI 0 "register_operand" ""))
-        (match_dup 2))
-   (parallel [(set (match_dup 0) (plus:SI (match_dup 0) (const_int 2)))
-             (clobber (reg:CC 17))])
-   (parallel [(set (match_dup 1) (plus:SI (match_dup 1) (const_int 2)))
-             (clobber (reg:CC 17))])]
-  ""
+       (float_extend:XF (match_operand:SF 1 "register_operand" "")))
+   (set (match_dup 4) (mult:XF (match_dup 2) (match_dup 3)))
+   (set (match_dup 5) (unspec:XF [(match_dup 4)] UNSPEC_FRNDINT))
+   (set (match_dup 6) (minus:XF (match_dup 4) (match_dup 5)))
+   (set (match_dup 7) (unspec:XF [(match_dup 6)] UNSPEC_F2XM1))
+   (set (match_dup 9) (plus:XF (match_dup 7) (match_dup 8)))
+   (parallel [(set (match_operand:SF 0 "register_operand" "")
+                  (unspec:SF [(match_dup 9) (match_dup 5)] UNSPEC_FSCALE))
+             (clobber (match_scratch:SF 5 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
 {
-  if (TARGET_64BIT)
-    {
-      emit_insn (gen_strmovhi_rex64 (operands[0], operands[1]));
-      DONE;
-    }
-  if (TARGET_SINGLE_STRINGOP || optimize_size)
-    {
-      emit_insn (gen_strmovhi_1 (operands[0], operands[1], operands[0],
-                               operands[1]));
-      DONE;
-    }
-  else 
-    operands[2] = gen_reg_rtx (HImode);
+  rtx temp;
+  int i;
+
+  for (i=2; i<10; i++)
+    operands[i] = gen_reg_rtx (XFmode);
+  temp = standard_80387_constant_rtx (6); /* fldl2t */
+  emit_move_insn (operands[3], temp);
+  emit_move_insn (operands[8], CONST1_RTX (XFmode));  /* fld1 */
+})
+
+(define_expand "exp10df2"
+  [(set (match_dup 2)
+       (float_extend:XF (match_operand:DF 1 "register_operand" "")))
+   (set (match_dup 4) (mult:XF (match_dup 2) (match_dup 3)))
+   (set (match_dup 5) (unspec:XF [(match_dup 4)] UNSPEC_FRNDINT))
+   (set (match_dup 6) (minus:XF (match_dup 4) (match_dup 5)))
+   (set (match_dup 7) (unspec:XF [(match_dup 6)] UNSPEC_F2XM1))
+   (set (match_dup 9) (plus:XF (match_dup 7) (match_dup 8)))
+   (parallel [(set (match_operand:DF 0 "register_operand" "")
+                  (unspec:DF [(match_dup 9) (match_dup 5)] UNSPEC_FSCALE))
+             (clobber (match_scratch:DF 5 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  rtx temp;
+  int i;
+
+  for (i=2; i<10; i++)
+    operands[i] = gen_reg_rtx (XFmode);
+  temp = standard_80387_constant_rtx (6); /* fldl2t */
+  emit_move_insn (operands[3], temp);
+  emit_move_insn (operands[8], CONST1_RTX (XFmode));  /* fld1 */
+})
+
+(define_expand "exp10xf2"
+  [(set (match_dup 3) (mult:XF (match_operand:XF 1 "register_operand" "")
+                              (match_dup 2)))
+   (set (match_dup 4) (unspec:XF [(match_dup 3)] UNSPEC_FRNDINT))
+   (set (match_dup 5) (minus:XF (match_dup 3) (match_dup 4)))
+   (set (match_dup 6) (unspec:XF [(match_dup 5)] UNSPEC_F2XM1))
+   (set (match_dup 8) (plus:XF (match_dup 6) (match_dup 7)))
+   (parallel [(set (match_operand:XF 0 "register_operand" "")
+                  (unspec:XF [(match_dup 8) (match_dup 4)] UNSPEC_FSCALE))
+             (clobber (match_scratch:XF 5 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  rtx temp;
+  int i;
+
+  for (i=2; i<9; i++)
+    operands[i] = gen_reg_rtx (XFmode);
+  temp = standard_80387_constant_rtx (6); /* fldl2t */
+  emit_move_insn (operands[2], temp);
+  emit_move_insn (operands[7], CONST1_RTX (XFmode));  /* fld1 */
+})
+
+(define_expand "exp2sf2"
+  [(set (match_dup 2)
+       (float_extend:XF (match_operand:SF 1 "register_operand" "")))
+   (set (match_dup 3) (unspec:XF [(match_dup 2)] UNSPEC_FRNDINT))
+   (set (match_dup 4) (minus:XF (match_dup 2) (match_dup 3)))
+   (set (match_dup 5) (unspec:XF [(match_dup 4)] UNSPEC_F2XM1))
+   (set (match_dup 7) (plus:XF (match_dup 5) (match_dup 6)))
+   (parallel [(set (match_operand:SF 0 "register_operand" "")
+                  (unspec:SF [(match_dup 7) (match_dup 3)] UNSPEC_FSCALE))
+             (clobber (match_scratch:SF 3 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  int i;
+
+  for (i=2; i<8; i++)
+    operands[i] = gen_reg_rtx (XFmode);
+  emit_move_insn (operands[6], CONST1_RTX (XFmode));  /* fld1 */
+})
+
+(define_expand "exp2df2"
+  [(set (match_dup 2)
+       (float_extend:XF (match_operand:DF 1 "register_operand" "")))
+   (set (match_dup 3) (unspec:XF [(match_dup 2)] UNSPEC_FRNDINT))
+   (set (match_dup 4) (minus:XF (match_dup 2) (match_dup 3)))
+   (set (match_dup 5) (unspec:XF [(match_dup 4)] UNSPEC_F2XM1))
+   (set (match_dup 7) (plus:XF (match_dup 5) (match_dup 6)))
+   (parallel [(set (match_operand:DF 0 "register_operand" "")
+                  (unspec:DF [(match_dup 7) (match_dup 3)] UNSPEC_FSCALE))
+             (clobber (match_scratch:DF 3 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  int i;
+
+  for (i=2; i<8; i++)
+    operands[i] = gen_reg_rtx (XFmode);
+  emit_move_insn (operands[6], CONST1_RTX (XFmode));  /* fld1 */
+})
+
+(define_expand "exp2xf2"
+  [(set (match_dup 2) (match_operand:XF 1 "register_operand" ""))
+   (set (match_dup 3) (unspec:XF [(match_dup 2)] UNSPEC_FRNDINT))
+   (set (match_dup 4) (minus:XF (match_dup 2) (match_dup 3)))
+   (set (match_dup 5) (unspec:XF [(match_dup 4)] UNSPEC_F2XM1))
+   (set (match_dup 7) (plus:XF (match_dup 5) (match_dup 6)))
+   (parallel [(set (match_operand:XF 0 "register_operand" "")
+                  (unspec:XF [(match_dup 7) (match_dup 3)] UNSPEC_FSCALE))
+             (clobber (match_scratch:XF 3 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  int i;
+
+  for (i=2; i<8; i++)
+    operands[i] = gen_reg_rtx (XFmode);
+  emit_move_insn (operands[6], CONST1_RTX (XFmode));  /* fld1 */
+})
+
+(define_expand "atansf2"
+  [(parallel [(set (match_operand:SF 0 "register_operand" "")
+                  (unspec:SF [(match_dup 2)
+                              (match_operand:SF 1 "register_operand" "")]
+                   UNSPEC_FPATAN))
+             (clobber (match_scratch:SF 3 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  operands[2] = gen_reg_rtx (SFmode);
+  emit_move_insn (operands[2], CONST1_RTX (SFmode));  /* fld1 */
+})
+
+(define_expand "atandf2"
+  [(parallel [(set (match_operand:DF 0 "register_operand" "")
+                  (unspec:DF [(match_dup 2)
+                              (match_operand:DF 1 "register_operand" "")]
+                   UNSPEC_FPATAN))
+             (clobber (match_scratch:DF 3 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  operands[2] = gen_reg_rtx (DFmode);
+  emit_move_insn (operands[2], CONST1_RTX (DFmode));  /* fld1 */
+})
+
+(define_expand "atanxf2"
+  [(parallel [(set (match_operand:XF 0 "register_operand" "")
+                  (unspec:XF [(match_dup 2)
+                              (match_operand:XF 1 "register_operand" "")]
+                   UNSPEC_FPATAN))
+             (clobber (match_scratch:XF 3 ""))])]
+  "! TARGET_NO_FANCY_MATH_387 && TARGET_80387
+   && flag_unsafe_math_optimizations"
+{
+  operands[2] = gen_reg_rtx (XFmode);
+  emit_move_insn (operands[2], CONST1_RTX (XFmode));  /* fld1 */
+})
+\f
+;; Block operation instructions
+
+(define_insn "cld"
+ [(set (reg:SI 19) (const_int 0))]
+ ""
+ "cld"
+  [(set_attr "type" "cld")])
+
+(define_expand "movstrsi"
+  [(use (match_operand:BLK 0 "memory_operand" ""))
+   (use (match_operand:BLK 1 "memory_operand" ""))
+   (use (match_operand:SI 2 "nonmemory_operand" ""))
+   (use (match_operand:SI 3 "const_int_operand" ""))]
+  "! optimize_size"
+{
+ if (ix86_expand_movstr (operands[0], operands[1], operands[2], operands[3]))
+   DONE;
+ else
+   FAIL;
 })
 
-(define_expand "strmovhi_rex64"
-  [(set (match_dup 2)
-       (mem:HI (match_operand:DI 1 "register_operand" "")))
-   (set (mem:HI (match_operand:DI 0 "register_operand" ""))
-        (match_dup 2))
-   (parallel [(set (match_dup 0) (plus:DI (match_dup 0) (const_int 2)))
-             (clobber (reg:CC 17))])
-   (parallel [(set (match_dup 1) (plus:DI (match_dup 1) (const_int 2)))
-             (clobber (reg:CC 17))])]
+(define_expand "movstrdi"
+  [(use (match_operand:BLK 0 "memory_operand" ""))
+   (use (match_operand:BLK 1 "memory_operand" ""))
+   (use (match_operand:DI 2 "nonmemory_operand" ""))
+   (use (match_operand:DI 3 "const_int_operand" ""))]
   "TARGET_64BIT"
 {
-  if (TARGET_SINGLE_STRINGOP || optimize_size)
-    {
-      emit_insn (gen_strmovhi_rex_1 (operands[0], operands[1], operands[0],
-                                    operands[1]));
-      DONE;
-    }
-  else 
-    operands[2] = gen_reg_rtx (HImode);
+ if (ix86_expand_movstr (operands[0], operands[1], operands[2], operands[3]))
+   DONE;
+ else
+   FAIL;
 })
 
-(define_expand "strmovqi"
-  [(set (match_dup 2)
-       (mem:QI (match_operand:SI 1 "register_operand" "")))
-   (set (mem:QI (match_operand:SI 0 "register_operand" ""))
-        (match_dup 2))
-   (parallel [(set (match_dup 0) (plus:SI (match_dup 0) (const_int 1)))
+;; Most CPUs don't like single string operations
+;; Handle this case here to simplify previous expander.
+
+(define_expand "strmov"
+  [(set (match_dup 4) (match_operand 3 "memory_operand" ""))
+   (set (match_operand 1 "memory_operand" "") (match_dup 4))
+   (parallel [(set (match_operand 0 "register_operand" "") (match_dup 5))
              (clobber (reg:CC 17))])
-   (parallel [(set (match_dup 1) (plus:SI (match_dup 1) (const_int 1)))
+   (parallel [(set (match_operand 2 "register_operand" "") (match_dup 6))
              (clobber (reg:CC 17))])]
   ""
 {
-  if (TARGET_64BIT)
-    {
-      emit_insn (gen_strmovqi_rex64 (operands[0], operands[1]));
-      DONE;
-    }
-  if (TARGET_SINGLE_STRINGOP || optimize_size)
-    {
-      emit_insn (gen_strmovqi_1 (operands[0], operands[1], operands[0],
-                               operands[1]));
-      DONE;
-    }
-  else 
-    operands[2] = gen_reg_rtx (QImode);
-})
+  rtx adjust = GEN_INT (GET_MODE_SIZE (GET_MODE (operands[1])));
+
+  /* If .md ever supports :P for Pmode, these can be directly
+     in the pattern above.  */
+  operands[5] = gen_rtx_PLUS (Pmode, operands[0], adjust);
+  operands[6] = gen_rtx_PLUS (Pmode, operands[2], adjust);
 
-(define_expand "strmovqi_rex64"
-  [(set (match_dup 2)
-       (mem:QI (match_operand:DI 1 "register_operand" "")))
-   (set (mem:QI (match_operand:DI 0 "register_operand" ""))
-        (match_dup 2))
-   (parallel [(set (match_dup 0) (plus:DI (match_dup 0) (const_int 1)))
-             (clobber (reg:CC 17))])
-   (parallel [(set (match_dup 1) (plus:DI (match_dup 1) (const_int 1)))
-             (clobber (reg:CC 17))])]
-  "TARGET_64BIT"
-{
   if (TARGET_SINGLE_STRINGOP || optimize_size)
     {
-      emit_insn (gen_strmovqi_rex_1 (operands[0], operands[1], operands[0],
-                                    operands[1]));
+      emit_insn (gen_strmov_singleop (operands[0], operands[1],
+                                     operands[2], operands[3],
+                                     operands[5], operands[6]));
       DONE;
     }
-  else 
-    operands[2] = gen_reg_rtx (QImode);
+
+  operands[4] = gen_reg_rtx (GET_MODE (operands[1]));
 })
 
-(define_insn "strmovdi_rex_1"
+(define_expand "strmov_singleop"
+  [(parallel [(set (match_operand 1 "memory_operand" "")
+                  (match_operand 3 "memory_operand" ""))
+             (set (match_operand 0 "register_operand" "")
+                  (match_operand 4 "" ""))
+             (set (match_operand 2 "register_operand" "")
+                  (match_operand 5 "" ""))
+             (use (reg:SI 19))])]
+  "TARGET_SINGLE_STRINGOP || optimize_size"
+  "")
+
+(define_insn "*strmovdi_rex_1"
   [(set (mem:DI (match_operand:DI 2 "register_operand" "0"))
        (mem:DI (match_operand:DI 3 "register_operand" "1")))
    (set (match_operand:DI 0 "register_operand" "=D")
    (set_attr "mode" "DI")
    (set_attr "memory" "both")])
 
-(define_insn "strmovsi_1"
+(define_insn "*strmovsi_1"
   [(set (mem:SI (match_operand:SI 2 "register_operand" "0"))
        (mem:SI (match_operand:SI 3 "register_operand" "1")))
    (set (match_operand:SI 0 "register_operand" "=D")
    (set_attr "mode" "SI")
    (set_attr "memory" "both")])
 
-(define_insn "strmovsi_rex_1"
+(define_insn "*strmovsi_rex_1"
   [(set (mem:SI (match_operand:DI 2 "register_operand" "0"))
        (mem:SI (match_operand:DI 3 "register_operand" "1")))
    (set (match_operand:DI 0 "register_operand" "=D")
    (set_attr "mode" "SI")
    (set_attr "memory" "both")])
 
-(define_insn "strmovhi_1"
+(define_insn "*strmovhi_1"
   [(set (mem:HI (match_operand:SI 2 "register_operand" "0"))
        (mem:HI (match_operand:SI 3 "register_operand" "1")))
    (set (match_operand:SI 0 "register_operand" "=D")
    (set_attr "memory" "both")
    (set_attr "mode" "HI")])
 
-(define_insn "strmovhi_rex_1"
+(define_insn "*strmovhi_rex_1"
   [(set (mem:HI (match_operand:DI 2 "register_operand" "0"))
        (mem:HI (match_operand:DI 3 "register_operand" "1")))
    (set (match_operand:DI 0 "register_operand" "=D")
    (set_attr "memory" "both")
    (set_attr "mode" "HI")])
 
-(define_insn "strmovqi_1"
+(define_insn "*strmovqi_1"
   [(set (mem:QI (match_operand:SI 2 "register_operand" "0"))
        (mem:QI (match_operand:SI 3 "register_operand" "1")))
    (set (match_operand:SI 0 "register_operand" "=D")
    (set_attr "memory" "both")
    (set_attr "mode" "QI")])
 
-(define_insn "strmovqi_rex_1"
+(define_insn "*strmovqi_rex_1"
   [(set (mem:QI (match_operand:DI 2 "register_operand" "0"))
        (mem:QI (match_operand:DI 3 "register_operand" "1")))
    (set (match_operand:DI 0 "register_operand" "=D")
    (set_attr "memory" "both")
    (set_attr "mode" "QI")])
 
-(define_insn "rep_movdi_rex64"
+(define_expand "rep_mov"
+  [(parallel [(set (match_operand 4 "register_operand" "") (const_int 0))
+             (set (match_operand 0 "register_operand" "")
+                  (match_operand 5 "" ""))
+             (set (match_operand 2 "register_operand" "")
+                  (match_operand 6 "" ""))
+             (set (match_operand 1 "memory_operand" "")
+                  (match_operand 3 "memory_operand" ""))
+             (use (match_dup 4))
+             (use (reg:SI 19))])]
+  ""
+  "")
+
+(define_insn "*rep_movdi_rex64"
   [(set (match_operand:DI 2 "register_operand" "=c") (const_int 0))
    (set (match_operand:DI 0 "register_operand" "=D") 
         (plus:DI (ashift:DI (match_operand:DI 5 "register_operand" "2")
    (set_attr "memory" "both")
    (set_attr "mode" "DI")])
 
-(define_insn "rep_movsi"
+(define_insn "*rep_movsi"
   [(set (match_operand:SI 2 "register_operand" "=c") (const_int 0))
    (set (match_operand:SI 0 "register_operand" "=D") 
         (plus:SI (ashift:SI (match_operand:SI 5 "register_operand" "2")
    (set_attr "memory" "both")
    (set_attr "mode" "SI")])
 
-(define_insn "rep_movsi_rex64"
+(define_insn "*rep_movsi_rex64"
   [(set (match_operand:DI 2 "register_operand" "=c") (const_int 0))
    (set (match_operand:DI 0 "register_operand" "=D") 
         (plus:DI (ashift:DI (match_operand:DI 5 "register_operand" "2")
    (set_attr "memory" "both")
    (set_attr "mode" "SI")])
 
-(define_insn "rep_movqi"
+(define_insn "*rep_movqi"
   [(set (match_operand:SI 2 "register_operand" "=c") (const_int 0))
    (set (match_operand:SI 0 "register_operand" "=D") 
         (plus:SI (match_operand:SI 3 "register_operand" "0")
    (set_attr "memory" "both")
    (set_attr "mode" "SI")])
 
-(define_insn "rep_movqi_rex64"
+(define_insn "*rep_movqi_rex64"
   [(set (match_operand:DI 2 "register_operand" "=c") (const_int 0))
    (set (match_operand:DI 0 "register_operand" "=D") 
         (plus:DI (match_operand:DI 3 "register_operand" "0")
 ;; Most CPUs don't like single string operations
 ;; Handle this case here to simplify previous expander.
 
-(define_expand "strsetdi_rex64"
-  [(set (mem:DI (match_operand:DI 0 "register_operand" ""))
-       (match_operand:DI 1 "register_operand" ""))
-   (parallel [(set (match_dup 0) (plus:DI (match_dup 0) (const_int 8)))
-             (clobber (reg:CC 17))])]
-  "TARGET_64BIT"
-{
-  if (TARGET_SINGLE_STRINGOP || optimize_size)
-    {
-      emit_insn (gen_strsetdi_rex_1 (operands[0], operands[0], operands[1]));
-      DONE;
-    }
-})
-
-(define_expand "strsetsi"
-  [(set (mem:SI (match_operand:SI 0 "register_operand" ""))
-       (match_operand:SI 1 "register_operand" ""))
-   (parallel [(set (match_dup 0) (plus:SI (match_dup 0) (const_int 4)))
-             (clobber (reg:CC 17))])]
-  ""
-{
-  if (TARGET_64BIT)
-    {
-      emit_insn (gen_strsetsi_rex64 (operands[0], operands[1]));
-      DONE;
-    }
-  else if (TARGET_SINGLE_STRINGOP || optimize_size)
-    {
-      emit_insn (gen_strsetsi_1 (operands[0], operands[0], operands[1]));
-      DONE;
-    }
-})
-
-(define_expand "strsetsi_rex64"
-  [(set (mem:SI (match_operand:DI 0 "register_operand" ""))
-       (match_operand:SI 1 "register_operand" ""))
-   (parallel [(set (match_dup 0) (plus:DI (match_dup 0) (const_int 4)))
-             (clobber (reg:CC 17))])]
-  "TARGET_64BIT"
-{
-  if (TARGET_SINGLE_STRINGOP || optimize_size)
-    {
-      emit_insn (gen_strsetsi_rex_1 (operands[0], operands[0], operands[1]));
-      DONE;
-    }
-})
-
-(define_expand "strsethi"
-  [(set (mem:HI (match_operand:SI 0 "register_operand" ""))
-       (match_operand:HI 1 "register_operand" ""))
-   (parallel [(set (match_dup 0) (plus:SI (match_dup 0) (const_int 2)))
+(define_expand "strset"
+  [(set (match_operand 1 "memory_operand" "")
+       (match_operand 2 "register_operand" ""))
+   (parallel [(set (match_operand 0 "register_operand" "")
+                  (match_dup 3))
              (clobber (reg:CC 17))])]
   ""
 {
-  if (TARGET_64BIT)
-    {
-      emit_insn (gen_strsethi_rex64 (operands[0], operands[1]));
-      DONE;
-    }
-  else if (TARGET_SINGLE_STRINGOP || optimize_size)
-    {
-      emit_insn (gen_strsethi_1 (operands[0], operands[0], operands[1]));
-      DONE;
-    }
-})
+  if (GET_MODE (operands[1]) != GET_MODE (operands[2]))
+    operands[1] = adjust_address_nv (operands[1], GET_MODE (operands[2]), 0);
 
-(define_expand "strsethi_rex64"
-  [(set (mem:HI (match_operand:DI 0 "register_operand" ""))
-       (match_operand:HI 1 "register_operand" ""))
-   (parallel [(set (match_dup 0) (plus:DI (match_dup 0) (const_int 2)))
-             (clobber (reg:CC 17))])]
-  "TARGET_64BIT"
-{
+  /* If .md ever supports :P for Pmode, this can be directly
+     in the pattern above.  */
+  operands[3] = gen_rtx_PLUS (Pmode, operands[0],
+                             GEN_INT (GET_MODE_SIZE (GET_MODE
+                                                     (operands[2]))));
   if (TARGET_SINGLE_STRINGOP || optimize_size)
     {
-      emit_insn (gen_strsethi_rex_1 (operands[0], operands[0], operands[1]));
-      DONE;
-    }
-})
-
-(define_expand "strsetqi"
-  [(set (mem:QI (match_operand:SI 0 "register_operand" ""))
-       (match_operand:QI 1 "register_operand" ""))
-   (parallel [(set (match_dup 0) (plus:SI (match_dup 0) (const_int 1)))
-             (clobber (reg:CC 17))])]
-  ""
-{
-  if (TARGET_64BIT)
-    {
-      emit_insn (gen_strsetqi_rex64 (operands[0], operands[1]));
-      DONE;
-    }
-  else if (TARGET_SINGLE_STRINGOP || optimize_size)
-    {
-      emit_insn (gen_strsetqi_1 (operands[0], operands[0], operands[1]));
+      emit_insn (gen_strset_singleop (operands[0], operands[1], operands[2],
+                                     operands[3]));
       DONE;
     }
 })
 
-(define_expand "strsetqi_rex64"
-  [(set (mem:QI (match_operand:DI 0 "register_operand" ""))
-       (match_operand:QI 1 "register_operand" ""))
-   (parallel [(set (match_dup 0) (plus:DI (match_dup 0) (const_int 1)))
-             (clobber (reg:CC 17))])]
-  "TARGET_64BIT"
-{
-  if (TARGET_SINGLE_STRINGOP || optimize_size)
-    {
-      emit_insn (gen_strsetqi_rex_1 (operands[0], operands[0], operands[1]));
-      DONE;
-    }
-})
+(define_expand "strset_singleop"
+  [(parallel [(set (match_operand 1 "memory_operand" "")
+                  (match_operand 2 "register_operand" ""))
+             (set (match_operand 0 "register_operand" "")
+                  (match_operand 3 "" ""))
+             (use (reg:SI 19))])]
+  "TARGET_SINGLE_STRINGOP || optimize_size"
+  "")
 
-(define_insn "strsetdi_rex_1"
+(define_insn "*strsetdi_rex_1"
   [(set (mem:SI (match_operand:DI 1 "register_operand" "0"))
        (match_operand:SI 2 "register_operand" "a"))
    (set (match_operand:DI 0 "register_operand" "=D")
    (set_attr "memory" "store")
    (set_attr "mode" "DI")])
 
-(define_insn "strsetsi_1"
+(define_insn "*strsetsi_1"
   [(set (mem:SI (match_operand:SI 1 "register_operand" "0"))
        (match_operand:SI 2 "register_operand" "a"))
    (set (match_operand:SI 0 "register_operand" "=D")
    (set_attr "memory" "store")
    (set_attr "mode" "SI")])
 
-(define_insn "strsetsi_rex_1"
+(define_insn "*strsetsi_rex_1"
   [(set (mem:SI (match_operand:DI 1 "register_operand" "0"))
        (match_operand:SI 2 "register_operand" "a"))
    (set (match_operand:DI 0 "register_operand" "=D")
    (set_attr "memory" "store")
    (set_attr "mode" "SI")])
 
-(define_insn "strsethi_1"
+(define_insn "*strsethi_1"
   [(set (mem:HI (match_operand:SI 1 "register_operand" "0"))
        (match_operand:HI 2 "register_operand" "a"))
    (set (match_operand:SI 0 "register_operand" "=D")
    (set_attr "memory" "store")
    (set_attr "mode" "HI")])
 
-(define_insn "strsethi_rex_1"
+(define_insn "*strsethi_rex_1"
   [(set (mem:HI (match_operand:DI 1 "register_operand" "0"))
        (match_operand:HI 2 "register_operand" "a"))
    (set (match_operand:DI 0 "register_operand" "=D")
    (set_attr "memory" "store")
    (set_attr "mode" "HI")])
 
-(define_insn "strsetqi_1"
+(define_insn "*strsetqi_1"
   [(set (mem:QI (match_operand:SI 1 "register_operand" "0"))
        (match_operand:QI 2 "register_operand" "a"))
    (set (match_operand:SI 0 "register_operand" "=D")
    (set_attr "memory" "store")
    (set_attr "mode" "QI")])
 
-(define_insn "strsetqi_rex_1"
+(define_insn "*strsetqi_rex_1"
   [(set (mem:QI (match_operand:DI 1 "register_operand" "0"))
        (match_operand:QI 2 "register_operand" "a"))
    (set (match_operand:DI 0 "register_operand" "=D")
    (set_attr "memory" "store")
    (set_attr "mode" "QI")])
 
-(define_insn "rep_stosdi_rex64"
+(define_expand "rep_stos"
+  [(parallel [(set (match_operand 1 "register_operand" "") (const_int 0))
+             (set (match_operand 0 "register_operand" "")
+                  (match_operand 4 "" ""))
+             (set (match_operand 2 "memory_operand" "") (const_int 0))
+             (use (match_operand 3 "register_operand" ""))
+             (use (match_dup 1))
+             (use (reg:SI 19))])]
+  ""
+  "")
+
+(define_insn "*rep_stosdi_rex64"
   [(set (match_operand:DI 1 "register_operand" "=c") (const_int 0))
    (set (match_operand:DI 0 "register_operand" "=D") 
         (plus:DI (ashift:DI (match_operand:DI 4 "register_operand" "1")
    (set_attr "memory" "store")
    (set_attr "mode" "DI")])
 
-(define_insn "rep_stossi"
+(define_insn "*rep_stossi"
   [(set (match_operand:SI 1 "register_operand" "=c") (const_int 0))
    (set (match_operand:SI 0 "register_operand" "=D") 
         (plus:SI (ashift:SI (match_operand:SI 4 "register_operand" "1")
    (set_attr "memory" "store")
    (set_attr "mode" "SI")])
 
-(define_insn "rep_stossi_rex64"
+(define_insn "*rep_stossi_rex64"
   [(set (match_operand:DI 1 "register_operand" "=c") (const_int 0))
    (set (match_operand:DI 0 "register_operand" "=D") 
         (plus:DI (ashift:DI (match_operand:DI 4 "register_operand" "1")
    (set_attr "memory" "store")
    (set_attr "mode" "SI")])
 
-(define_insn "rep_stosqi"
+(define_insn "*rep_stosqi"
   [(set (match_operand:SI 1 "register_operand" "=c") (const_int 0))
    (set (match_operand:SI 0 "register_operand" "=D") 
         (plus:SI (match_operand:SI 3 "register_operand" "0")
    (set_attr "memory" "store")
    (set_attr "mode" "QI")])
 
-(define_insn "rep_stosqi_rex64"
+(define_insn "*rep_stosqi_rex64"
   [(set (match_operand:DI 1 "register_operand" "=c") (const_int 0))
    (set (match_operand:DI 0 "register_operand" "=D") 
         (plus:DI (match_operand:DI 3 "register_operand" "0")
        (const_int 0))
    (use (match_operand:QI 2 "register_operand" "a"))
    (use (match_dup 4))
-   (use (reg:DI 19))]
+   (use (reg:SI 19))]
   "TARGET_64BIT"
   "{rep\;stosb|rep stosb}"
   [(set_attr "type" "str")
                    (match_operand:BLK 2 "general_operand" "")))
    (use (match_operand 3 "general_operand" ""))
    (use (match_operand 4 "immediate_operand" ""))]
-  ""
+  "! optimize_size || TARGET_INLINE_ALL_STRINGOPS"
 {
   rtx addr1, addr2, out, outlow, count, countreg, align;
 
 
   addr1 = copy_to_mode_reg (Pmode, XEXP (operands[1], 0));
   addr2 = copy_to_mode_reg (Pmode, XEXP (operands[2], 0));
-  
+  if (addr1 != XEXP (operands[1], 0))
+    operands[1] = replace_equiv_address_nv (operands[1], addr1);
+  if (addr2 != XEXP (operands[2], 0))
+    operands[2] = replace_equiv_address_nv (operands[2], addr2);
+
   count = operands[3];
   countreg = ix86_zero_extend_to_Pmode (count);
 
          emit_move_insn (operands[0], const0_rtx);
          DONE;
        }
-      if (TARGET_64BIT)
-       emit_insn (gen_cmpstrqi_nz_rex_1 (addr1, addr2, countreg, align,
-                                         addr1, addr2, countreg));
-      else
-       emit_insn (gen_cmpstrqi_nz_1 (addr1, addr2, countreg, align,
-                                     addr1, addr2, countreg));
+      emit_insn (gen_cmpstrqi_nz_1 (addr1, addr2, countreg, align,
+                                   operands[1], operands[2]));
     }
   else
     {
       if (TARGET_64BIT)
-       {
-         emit_insn (gen_cmpdi_1_rex64 (countreg, countreg));
-         emit_insn (gen_cmpstrqi_rex_1 (addr1, addr2, countreg, align,
-                                        addr1, addr2, countreg));
-       }
+       emit_insn (gen_cmpdi_1_rex64 (countreg, countreg));
       else
-       {
-         emit_insn (gen_cmpsi_1 (countreg, countreg));
-         emit_insn (gen_cmpstrqi_1 (addr1, addr2, countreg, align,
-                                    addr1, addr2, countreg));
-       }
+       emit_insn (gen_cmpsi_1 (countreg, countreg));
+      emit_insn (gen_cmpstrqi_1 (addr1, addr2, countreg, align,
+                                operands[1], operands[2]));
     }
 
   outlow = gen_lowpart (QImode, out);
 ;; memcmp recognizers.  The `cmpsb' opcode does nothing if the count is
 ;; zero.  Emit extra code to make sure that a zero-length compare is EQ.
 
-(define_insn "cmpstrqi_nz_1"
+(define_expand "cmpstrqi_nz_1"
+  [(parallel [(set (reg:CC 17)
+                  (compare:CC (match_operand 4 "memory_operand" "")
+                              (match_operand 5 "memory_operand" "")))
+             (use (match_operand 2 "register_operand" ""))
+             (use (match_operand:SI 3 "immediate_operand" ""))
+             (use (reg:SI 19))
+             (clobber (match_operand 0 "register_operand" ""))
+             (clobber (match_operand 1 "register_operand" ""))
+             (clobber (match_dup 2))])]
+  ""
+  "")
+
+(define_insn "*cmpstrqi_nz_1"
   [(set (reg:CC 17)
        (compare:CC (mem:BLK (match_operand:SI 4 "register_operand" "0"))
                    (mem:BLK (match_operand:SI 5 "register_operand" "1"))))
    (set_attr "mode" "QI")
    (set_attr "prefix_rep" "1")])
 
-(define_insn "cmpstrqi_nz_rex_1"
+(define_insn "*cmpstrqi_nz_rex_1"
   [(set (reg:CC 17)
        (compare:CC (mem:BLK (match_operand:DI 4 "register_operand" "0"))
                    (mem:BLK (match_operand:DI 5 "register_operand" "1"))))
 
 ;; The same, but the count is not known to not be zero.
 
-(define_insn "cmpstrqi_1"
+(define_expand "cmpstrqi_1"
+  [(parallel [(set (reg:CC 17)
+               (if_then_else:CC (ne (match_operand 2 "register_operand" "")
+                                    (const_int 0))
+                 (compare:CC (match_operand 4 "memory_operand" "")
+                             (match_operand 5 "memory_operand" ""))
+                 (const_int 0)))
+             (use (match_operand:SI 3 "immediate_operand" ""))
+             (use (reg:CC 17))
+             (use (reg:SI 19))
+             (clobber (match_operand 0 "register_operand" ""))
+             (clobber (match_operand 1 "register_operand" ""))
+             (clobber (match_dup 2))])]
+  ""
+  "")
+
+(define_insn "*cmpstrqi_1"
   [(set (reg:CC 17)
        (if_then_else:CC (ne (match_operand:SI 6 "register_operand" "2")
                             (const_int 0))
    (set_attr "mode" "QI")
    (set_attr "prefix_rep" "1")])
 
-(define_insn "cmpstrqi_rex_1"
+(define_insn "*cmpstrqi_rex_1"
   [(set (reg:CC 17)
        (if_then_else:CC (ne (match_operand:DI 6 "register_operand" "2")
                             (const_int 0))
    FAIL;
 })
 
-(define_insn "strlenqi_1"
+(define_expand "strlenqi_1"
+  [(parallel [(set (match_operand 0 "register_operand" "") (match_operand 2 "" ""))
+             (use (reg:SI 19))
+             (clobber (match_operand 1 "register_operand" ""))
+             (clobber (reg:CC 17))])]
+  ""
+  "")
+
+(define_insn "*strlenqi_1"
   [(set (match_operand:SI 0 "register_operand" "=&c")
        (unspec:SI [(mem:BLK (match_operand:SI 5 "register_operand" "1"))
                    (match_operand:QI 2 "register_operand" "a")
    (set_attr "mode" "QI")
    (set_attr "prefix_rep" "1")])
 
-(define_insn "strlenqi_rex_1"
+(define_insn "*strlenqi_rex_1"
   [(set (match_operand:DI 0 "register_operand" "=&c")
        (unspec:DI [(mem:BLK (match_operand:DI 5 "register_operand" "1"))
                    (match_operand:QI 2 "register_operand" "a")
        (if_then_else:XF (match_operand 1 "comparison_operator" "")
                         (match_operand:XF 2 "register_operand" "")
                         (match_operand:XF 3 "register_operand" "")))]
-  "!TARGET_64BIT && TARGET_CMOVE"
-  "if (! ix86_expand_fp_movcc (operands)) FAIL; DONE;")
-
-(define_expand "movtfcc"
-  [(set (match_operand:TF 0 "register_operand" "")
-       (if_then_else:TF (match_operand 1 "comparison_operator" "")
-                        (match_operand:TF 2 "register_operand" "")
-                        (match_operand:TF 3 "register_operand" "")))]
   "TARGET_CMOVE"
   "if (! ix86_expand_fp_movcc (operands)) FAIL; DONE;")
 
                                [(reg 17) (const_int 0)])
                      (match_operand:XF 2 "register_operand" "f,0")
                      (match_operand:XF 3 "register_operand" "0,f")))]
-  "!TARGET_64BIT && TARGET_CMOVE"
-  "@
-   fcmov%F1\t{%2, %0|%0, %2}
-   fcmov%f1\t{%3, %0|%0, %3}"
-  [(set_attr "type" "fcmov")
-   (set_attr "mode" "XF")])
-
-(define_insn "*movtfcc_1"
-  [(set (match_operand:TF 0 "register_operand" "=f,f")
-       (if_then_else:TF (match_operator 1 "fcmov_comparison_operator" 
-                               [(reg 17) (const_int 0)])
-                     (match_operand:TF 2 "register_operand" "f,0")
-                     (match_operand:TF 3 "register_operand" "0,f")))]
   "TARGET_CMOVE"
   "@
    fcmov%F1\t{%2, %0|%0, %2}
           && operands_match_p (operands[2], operands[3])))"
   [(set (reg:CCFP 17)
        (compare:CCFP (match_dup 2)
-                     (match_dup 2)))
+                     (match_dup 1)))
    (set (match_dup 0)
        (if_then_else:DF (ge (reg:CCFP 17) (const_int 0))
                         (match_dup 1)
 ;; [(set (mem (plus (reg ebp) (const_int -160000))) (const_int 0))]
 ;;
 ;; in proper program order.
-(define_expand "pro_epilogue_adjust_stack"
-  [(parallel [(set (match_operand:SI 0 "register_operand" "=r,r")
-                  (plus:SI (match_operand:SI 1 "register_operand" "0,r")
-                           (match_operand:SI 2 "immediate_operand" "i,i")))
-             (clobber (reg:CC 17))
-             (clobber (mem:BLK (scratch)))])]
- ""
-{
-  if (TARGET_64BIT)
-    {
-      emit_insn (gen_pro_epilogue_adjust_stack_rex64
-                (operands[0], operands[1], operands[2]));
-      DONE;
-    }
-})
-
-(define_insn "*pro_epilogue_adjust_stack_1"
+(define_insn "pro_epilogue_adjust_stack_1"
   [(set (match_operand:SI 0 "register_operand" "=r,r")
        (plus:SI (match_operand:SI 1 "register_operand" "0,r")
                 (match_operand:SI 2 "immediate_operand" "i,i")))
 
     case TYPE_ALU:
       if (GET_CODE (operands[2]) == CONST_INT
+         /* Avoid overflows.  */
+         && ((INTVAL (operands[2]) & ((((unsigned int) 1) << 31) - 1)))
           && (INTVAL (operands[2]) == 128
              || (INTVAL (operands[2]) < 0
                  && INTVAL (operands[2]) != -128)))
              (const_string "lea")))
    (set_attr "mode" "DI")])
 
+(define_insn "pro_epilogue_adjust_stack_rex64_2"
+  [(set (match_operand:DI 0 "register_operand" "=r,r")
+       (plus:DI (match_operand:DI 1 "register_operand" "0,r")
+                (match_operand:DI 3 "immediate_operand" "i,i")))
+   (use (match_operand:DI 2 "register_operand" "r,r"))
+   (clobber (reg:CC 17))
+   (clobber (mem:BLK (scratch)))]
+  "TARGET_64BIT"
+{
+  switch (get_attr_type (insn))
+    {
+    case TYPE_ALU:
+      return "add{q}\t{%2, %0|%0, %2}";
+
+    case TYPE_LEA:
+      operands[2] = gen_rtx_PLUS (DImode, operands[1], operands[2]);
+      return "lea{q}\t{%a2, %0|%0, %a2}";
+
+    default:
+      abort ();
+    }
+}
+  [(set_attr "type" "alu,lea")
+   (set_attr "mode" "DI")])
 
 ;; Placeholder for the conditional moves.  This one is split either to SSE
 ;; based moves emulation or to usual cmove sequence.  Little bit unfortunate
 ;; nand  op0, op3   -  load op3 to op0 if comparison was false
 ;; or   op2, op0   -  get the nonzero one into the result.
 (define_split
-  [(set (match_operand 0 "register_operand" "")
-       (if_then_else (match_operator 1 "sse_comparison_operator"
-                       [(match_operand 4 "register_operand" "")
-                        (match_operand 5 "nonimmediate_operand" "")])
-                     (match_operand 2 "register_operand" "")
-                     (match_operand 3 "register_operand" "")))
+  [(set (match_operand:SF 0 "register_operand" "")
+       (if_then_else (match_operator:SF 1 "sse_comparison_operator"
+                       [(match_operand:SF 4 "register_operand" "")
+                        (match_operand:SF 5 "nonimmediate_operand" "")])
+                     (match_operand:SF 2 "register_operand" "")
+                     (match_operand:SF 3 "register_operand" "")))
+   (clobber (match_operand 6 "" ""))
+   (clobber (reg:CC 17))]
+  "SSE_REG_P (operands[0]) && reload_completed"
+  [(set (match_dup 4) (match_op_dup 1 [(match_dup 4) (match_dup 5)]))
+   (set (match_dup 2) (and:V4SF (match_dup 2)
+                               (match_dup 8)))
+   (set (match_dup 8) (and:V4SF (not:V4SF (match_dup 8))
+                                         (match_dup 3)))
+   (set (match_dup 0) (ior:V4SF (match_dup 6)
+                               (match_dup 7)))]
+{
+  /* If op2 == op3, op3 would be clobbered before it is used.  */
+  if (operands_match_p (operands[2], operands[3]))
+    {
+      emit_move_insn (operands[0], operands[2]);
+      DONE;
+    }
+
+  PUT_MODE (operands[1], GET_MODE (operands[0]));
+  if (operands_match_p (operands[0], operands[4]))
+    operands[6] = operands[4], operands[7] = operands[2];
+  else
+    operands[6] = operands[2], operands[7] = operands[4];
+  operands[0] = simplify_gen_subreg (V4SFmode, operands[0], SFmode, 0);
+  operands[2] = simplify_gen_subreg (V4SFmode, operands[2], SFmode, 0);
+  operands[3] = simplify_gen_subreg (V4SFmode, operands[3], SFmode, 0);
+  operands[8] = simplify_gen_subreg (V4SFmode, operands[4], SFmode, 0);
+  operands[6] = simplify_gen_subreg (V4SFmode, operands[6], SFmode, 0);
+  operands[7] = simplify_gen_subreg (V4SFmode, operands[7], SFmode, 0);
+})
+
+(define_split
+  [(set (match_operand:DF 0 "register_operand" "")
+       (if_then_else (match_operator:DF 1 "sse_comparison_operator"
+                       [(match_operand:DF 4 "register_operand" "")
+                        (match_operand:DF 5 "nonimmediate_operand" "")])
+                     (match_operand:DF 2 "register_operand" "")
+                     (match_operand:DF 3 "register_operand" "")))
    (clobber (match_operand 6 "" ""))
    (clobber (reg:CC 17))]
   "SSE_REG_P (operands[0]) && reload_completed"
   [(set (match_dup 4) (match_op_dup 1 [(match_dup 4) (match_dup 5)]))
-   (set (subreg:TI (match_dup 2) 0) (and:TI (subreg:TI (match_dup 2) 0)
-                                           (subreg:TI (match_dup 4) 0)))
-   (set (subreg:TI (match_dup 4) 0) (and:TI (not:TI (subreg:TI (match_dup 4) 0))
-                                           (subreg:TI (match_dup 3) 0)))
-   (set (subreg:TI (match_dup 0) 0) (ior:TI (subreg:TI (match_dup 6) 0)
-                                           (subreg:TI (match_dup 7) 0)))]
+   (set (match_dup 2) (and:V2DF (match_dup 2)
+                               (match_dup 8)))
+   (set (match_dup 8) (and:V2DF (not:V2DF (match_dup 8))
+                                         (match_dup 3)))
+   (set (match_dup 0) (ior:V2DF (match_dup 6)
+                               (match_dup 7)))]
 {
   if (GET_MODE (operands[2]) == DFmode
       && TARGET_SSE_PARTIAL_REGS && !optimize_size)
     operands[6] = operands[4], operands[7] = operands[2];
   else
     operands[6] = operands[2], operands[7] = operands[4];
+  operands[0] = simplify_gen_subreg (V2DFmode, operands[0], DFmode, 0);
+  operands[2] = simplify_gen_subreg (V2DFmode, operands[2], DFmode, 0);
+  operands[3] = simplify_gen_subreg (V2DFmode, operands[3], DFmode, 0);
+  operands[8] = simplify_gen_subreg (V2DFmode, operands[4], DFmode, 0);
+  operands[6] = simplify_gen_subreg (V2DFmode, operands[6], DFmode, 0);
+  operands[7] = simplify_gen_subreg (V2DFmode, operands[7], DFmode, 0);
 })
 
 ;; Special case of conditional move we can handle effectively.
   "#")
 
 (define_split
-  [(set (match_operand 0 "register_operand" "")
-       (if_then_else (match_operator 1 "comparison_operator"
-                       [(match_operand 4 "nonimmediate_operand" "")
-                        (match_operand 5 "nonimmediate_operand" "")])
-                     (match_operand 2 "nonmemory_operand" "")
-                     (match_operand 3 "nonmemory_operand" "")))]
+  [(set (match_operand:SF 0 "register_operand" "")
+       (if_then_else (match_operator:SF 1 "comparison_operator"
+                       [(match_operand:SF 4 "nonimmediate_operand" "")
+                        (match_operand:SF 5 "nonimmediate_operand" "")])
+                     (match_operand:SF 2 "nonmemory_operand" "")
+                     (match_operand:SF 3 "nonmemory_operand" "")))]
+  "SSE_REG_P (operands[0]) && reload_completed
+   && (const0_operand (operands[2], GET_MODE (operands[0]))
+       || const0_operand (operands[3], GET_MODE (operands[0])))"
+  [(set (match_dup 0) (match_op_dup 1 [(match_dup 0) (match_dup 5)]))
+   (set (match_dup 8) (and:V4SF (match_dup 6) (match_dup 7)))]
+{
+  PUT_MODE (operands[1], GET_MODE (operands[0]));
+  if (!sse_comparison_operator (operands[1], VOIDmode)
+      || !rtx_equal_p (operands[0], operands[4]))
+    {
+      rtx tmp = operands[5];
+      operands[5] = operands[4];
+      operands[4] = tmp;
+      PUT_CODE (operands[1], swap_condition (GET_CODE (operands[1])));
+    }
+  if (!rtx_equal_p (operands[0], operands[4]))
+    abort ();
+  operands[8] = simplify_gen_subreg (V4SFmode, operands[0], SFmode, 0);
+  if (const0_operand (operands[2], GET_MODE (operands[2])))
+    {
+      operands[7] = operands[3];
+      operands[6] = gen_rtx_NOT (V4SFmode, operands[5]);
+    }
+  else
+    {
+      operands[7] = operands[2];
+      operands[6] = operands[0];
+    }
+  operands[7] = simplify_gen_subreg (V4SFmode, operands[7], SFmode, 0);
+})
+
+(define_split
+  [(set (match_operand:DF 0 "register_operand" "")
+       (if_then_else (match_operator:DF 1 "comparison_operator"
+                       [(match_operand:DF 4 "nonimmediate_operand" "")
+                        (match_operand:DF 5 "nonimmediate_operand" "")])
+                     (match_operand:DF 2 "nonmemory_operand" "")
+                     (match_operand:DF 3 "nonmemory_operand" "")))]
   "SSE_REG_P (operands[0]) && reload_completed
    && (const0_operand (operands[2], GET_MODE (operands[0]))
        || const0_operand (operands[3], GET_MODE (operands[0])))"
   [(set (match_dup 0) (match_op_dup 1 [(match_dup 0) (match_dup 5)]))
-   (set (subreg:TI (match_dup 0) 0) (and:TI (match_dup 6)
-                                           (match_dup 7)))]
+   (set (match_dup 8) (and:V2DF (match_dup 6) (match_dup 7)))]
 {
   if (TARGET_SSE_PARTIAL_REGS && !optimize_size
       && GET_MODE (operands[2]) == DFmode)
     }
   if (!rtx_equal_p (operands[0], operands[4]))
     abort ();
-  if (const0_operand (operands[2], GET_MODE (operands[0])))
+  operands[8] = simplify_gen_subreg (V2DFmode, operands[0], DFmode, 0);
+  if (const0_operand (operands[2], GET_MODE (operands[2])))
     {
       operands[7] = operands[3];
-      operands[6] = gen_rtx_NOT (TImode, gen_rtx_SUBREG (TImode, operands[0],
-                                                        0));
+      operands[6] = gen_rtx_NOT (V2DFmode, operands[8]);
     }
   else
     {
       operands[7] = operands[2];
-      operands[6] = gen_rtx_SUBREG (TImode, operands[0], 0);
+      operands[6] = operands[8];
     }
-  operands[7] = simplify_gen_subreg (TImode, operands[7],
-                                    GET_MODE (operands[7]), 0);
+  operands[7] = simplify_gen_subreg (V2DFmode, operands[7], DFmode, 0);
 })
 
 (define_expand "allocate_stack_worker"
   [(match_operand:SI 0 "register_operand" "")]
   "TARGET_STACK_PROBE"
 {
-  if (TARGET_64BIT)
-    emit_insn (gen_allocate_stack_worker_rex64 (operands[0]));
+  if (reload_completed)
+    {
+      if (TARGET_64BIT)
+       emit_insn (gen_allocate_stack_worker_rex64_postreload (operands[0]));
+      else
+       emit_insn (gen_allocate_stack_worker_postreload (operands[0]));
+    }
   else
-    emit_insn (gen_allocate_stack_worker_1 (operands[0]));
+    {
+      if (TARGET_64BIT)
+       emit_insn (gen_allocate_stack_worker_rex64 (operands[0]));
+      else
+       emit_insn (gen_allocate_stack_worker_1 (operands[0]));
+    }
   DONE;
 })
 
 (define_insn "allocate_stack_worker_1"
   [(unspec:SI [(match_operand:SI 0 "register_operand" "a")] UNSPEC_STACK_PROBE)
    (set (reg:SI 7) (minus:SI (reg:SI 7) (match_dup 0)))
-   (clobber (match_dup 0))
+   (clobber (match_scratch:SI 1 "=0"))
    (clobber (reg:CC 17))]
   "!TARGET_64BIT && TARGET_STACK_PROBE"
   "call\t__alloca"
   [(set_attr "type" "multi")
    (set_attr "length" "5")])
 
+(define_expand "allocate_stack_worker_postreload"
+  [(parallel [(unspec:SI [(match_operand:SI 0 "register_operand" "a")]
+                          UNSPEC_STACK_PROBE)
+             (set (reg:SI 7) (minus:SI (reg:SI 7) (match_dup 0)))
+             (clobber (match_dup 0))
+             (clobber (reg:CC 17))])]
+  ""
+  "")
+
 (define_insn "allocate_stack_worker_rex64"
   [(unspec:DI [(match_operand:DI 0 "register_operand" "a")] UNSPEC_STACK_PROBE)
    (set (reg:DI 7) (minus:DI (reg:DI 7) (match_dup 0)))
-   (clobber (match_dup 0))
+   (clobber (match_scratch:DI 1 "=0"))
    (clobber (reg:CC 17))]
   "TARGET_64BIT && TARGET_STACK_PROBE"
   "call\t__alloca"
   [(set_attr "type" "multi")
    (set_attr "length" "5")])
 
+(define_expand "allocate_stack_worker_rex64_postreload"
+  [(parallel [(unspec:DI [(match_operand:DI 0 "register_operand" "a")]
+                          UNSPEC_STACK_PROBE)
+             (set (reg:DI 7) (minus:DI (reg:DI 7) (match_dup 0)))
+             (clobber (match_dup 0))
+             (clobber (reg:CC 17))])]
+  ""
+  "")
+
 (define_expand "allocate_stack"
   [(parallel [(set (match_operand:SI 0 "register_operand" "=r")
                   (minus:SI (reg:SI 7)
 (define_insn "movv2di_internal"
   [(set (match_operand:V2DI 0 "nonimmediate_operand" "=x,x,m")
        (match_operand:V2DI 1 "vector_move_operand" "C,xm,x"))]
-  "TARGET_SSE2"
+  "TARGET_SSE"
 {
   switch (which_alternative)
     {
   DONE;
 })
 
+(define_expand "movtf"
+  [(set (match_operand:TF 0 "nonimmediate_operand" "")
+       (match_operand:TF 1 "nonimmediate_operand" ""))]
+  "TARGET_64BIT"
+{
+  if (TARGET_64BIT)
+    ix86_expand_move (TFmode, operands);
+  else
+    ix86_expand_vector_move (TFmode, operands);
+  DONE;
+})
+
 (define_insn "movv2df_internal"
   [(set (match_operand:V2DF 0 "nonimmediate_operand" "=x,x,m")
        (match_operand:V2DF 1 "vector_move_operand" "C,xm,x"))]
                   (const_string "TI"))]
               (const_string "DI")))])
 
+(define_insn "*movtf_rex64"
+  [(set (match_operand:TF 0 "nonimmediate_operand" "=r,o,x,x,xm")
+       (match_operand:TF 1 "general_operand" "riFo,riF,C,xm,x"))]
+  "TARGET_64BIT
+   && (GET_CODE (operands[0]) != MEM || GET_CODE (operands[1]) != MEM)"
+{
+  switch (which_alternative)
+    {
+    case 0:
+    case 1:
+      return "#";
+    case 2:
+      if (get_attr_mode (insn) == MODE_V4SF)
+       return "xorps\t%0, %0";
+      else
+       return "pxor\t%0, %0";
+    case 3:
+    case 4:
+      if (get_attr_mode (insn) == MODE_V4SF)
+       return "movaps\t{%1, %0|%0, %1}";
+      else
+       return "movdqa\t{%1, %0|%0, %1}";
+    default:
+      abort ();
+    }
+}
+  [(set_attr "type" "*,*,ssemov,ssemov,ssemov")
+   (set (attr "mode")
+        (cond [(eq_attr "alternative" "2,3")
+                (if_then_else
+                  (ne (symbol_ref "optimize_size")
+                      (const_int 0))
+                  (const_string "V4SF")
+                  (const_string "TI"))
+              (eq_attr "alternative" "4")
+                (if_then_else
+                  (ior (ne (symbol_ref "TARGET_SSE_TYPELESS_STORES")
+                           (const_int 0))
+                       (ne (symbol_ref "optimize_size")
+                           (const_int 0)))
+                  (const_string "V4SF")
+                  (const_string "TI"))]
+              (const_string "DI")))])
+
 (define_split
   [(set (match_operand:TI 0 "nonimmediate_operand" "")
         (match_operand:TI 1 "general_operand" ""))]
   [(const_int 0)]
   "ix86_split_long_move (operands); DONE;")
 
+(define_split
+  [(set (match_operand:TF 0 "nonimmediate_operand" "")
+        (match_operand:TF 1 "general_operand" ""))]
+  "reload_completed && !SSE_REG_P (operands[0])
+   && !SSE_REG_P (operands[1])"
+  [(const_int 0)]
+  "ix86_split_long_move (operands); DONE;")
+
 ;; These two patterns are useful for specifying exactly whether to use
 ;; movaps or movups
-(define_insn "sse_movaps"
+(define_expand "sse_movaps"
+  [(set (match_operand:V4SF 0 "nonimmediate_operand" "")
+       (unspec:V4SF [(match_operand:V4SF 1 "nonimmediate_operand" "")]
+                    UNSPEC_MOVA))]
+  "TARGET_SSE"
+{
+  if (GET_CODE (operands[0]) == MEM && GET_CODE (operands[1]) == MEM)
+    {
+      rtx tmp = gen_reg_rtx (V4SFmode);
+      emit_insn (gen_sse_movaps (tmp, operands[1]));
+      emit_move_insn (operands[0], tmp);
+      DONE;
+    }
+})
+
+(define_insn "*sse_movaps_1"
   [(set (match_operand:V4SF 0 "nonimmediate_operand" "=x,m")
        (unspec:V4SF [(match_operand:V4SF 1 "nonimmediate_operand" "xm,x")]
                     UNSPEC_MOVA))]
   [(set_attr "type" "ssemov,ssemov")
    (set_attr "mode" "V4SF")])
 
-(define_insn "sse_movups"
+(define_expand "sse_movups"
+  [(set (match_operand:V4SF 0 "nonimmediate_operand" "")
+       (unspec:V4SF [(match_operand:V4SF 1 "nonimmediate_operand" "")]
+                    UNSPEC_MOVU))]
+  "TARGET_SSE"
+{
+  if (GET_CODE (operands[0]) == MEM && GET_CODE (operands[1]) == MEM)
+    {
+      rtx tmp = gen_reg_rtx (V4SFmode);
+      emit_insn (gen_sse_movups (tmp, operands[1]));
+      emit_move_insn (operands[0], tmp);
+      DONE;
+    }
+})
+
+(define_insn "*sse_movups_1"
   [(set (match_operand:V4SF 0 "nonimmediate_operand" "=x,m")
        (unspec:V4SF [(match_operand:V4SF 1 "nonimmediate_operand" "xm,x")]
                     UNSPEC_MOVU))]
   [(set_attr "type" "ssecvt,ssecvt")
    (set_attr "mode" "V4SF")])
 
-
 ;; SSE Strange Moves.
 
 (define_insn "sse_movmskps"
 ;; of DImode subregs again!
 ;; SSE1 single precision floating point logical operation
 (define_expand "sse_andv4sf3"
-  [(set (subreg:TI (match_operand:V4SF 0 "register_operand" "") 0)
-        (and:TI (subreg:TI (match_operand:V4SF 1 "register_operand" "") 0)
-               (subreg:TI (match_operand:V4SF 2 "nonimmediate_operand" "") 0)))]
+  [(set (match_operand:V4SF 0 "register_operand" "")
+        (and:V4SF (match_operand:V4SF 1 "register_operand" "")
+                 (match_operand:V4SF 2 "nonimmediate_operand" "")))]
   "TARGET_SSE"
   "")
 
 (define_insn "*sse_andv4sf3"
-  [(set (subreg:TI (match_operand:V4SF 0 "register_operand" "=x") 0)
-        (and:TI (match_operand:TI 1 "nonimmediate_operand" "%0")
-               (match_operand:TI 2 "nonimmediate_operand" "xm")))]
-  "TARGET_SSE
-   && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
-  "andps\t{%2, %0|%0, %2}"
-  [(set_attr "type" "sselog")
-   (set_attr "mode" "V4SF")])
-
-(define_insn "*sse_andsf3"
-  [(set (subreg:TI (match_operand:SF 0 "register_operand" "=x") 0)
-        (and:TI (match_operand:TI 1 "nonimmediate_operand" "%0")
-               (match_operand:TI 2 "nonimmediate_operand" "xm")))]
+  [(set (match_operand:V4SF 0 "register_operand" "=x")
+        (and:V4SF (match_operand:V4SF 1 "nonimmediate_operand" "%0")
+                 (match_operand:V4SF 2 "nonimmediate_operand" "xm")))]
   "TARGET_SSE
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "andps\t{%2, %0|%0, %2}"
    (set_attr "mode" "V4SF")])
 
 (define_expand "sse_nandv4sf3"
-  [(set (subreg:TI (match_operand:V4SF 0 "register_operand" "") 0)
-        (and:TI (not:TI (subreg:TI (match_operand:V4SF 1 "register_operand" "") 0))
-               (subreg:TI (match_operand:V4SF 2 "nonimmediate_operand" "") 0)))]
+  [(set (match_operand:V4SF 0 "register_operand" "")
+        (and:V4SF (not:V4SF (match_operand:V4SF 1 "register_operand" ""))
+                 (match_operand:V4SF 2 "nonimmediate_operand" "")))]
   "TARGET_SSE"
   "")
 
 (define_insn "*sse_nandv4sf3"
-  [(set (subreg:TI (match_operand:V4SF 0 "register_operand" "=x") 0)
-        (and:TI (not:TI (match_operand:TI 1 "register_operand" "0"))
-               (match_operand:TI 2 "nonimmediate_operand" "xm")))]
-  "TARGET_SSE"
-  "andnps\t{%2, %0|%0, %2}"
-  [(set_attr "type" "sselog")
-   (set_attr "mode" "V4SF")])
-
-(define_insn "*sse_nandsf3"
-  [(set (subreg:TI (match_operand:SF 0 "register_operand" "=x") 0)
-        (and:TI (not:TI (match_operand:TI 1 "register_operand" "0"))
-               (match_operand:TI 2 "nonimmediate_operand" "xm")))]
+  [(set (match_operand:V4SF 0 "register_operand" "=x")
+        (and:V4SF (not:V4SF (match_operand:V4SF 1 "register_operand" "0"))
+                 (match_operand:V4SF 2 "nonimmediate_operand" "xm")))]
   "TARGET_SSE"
   "andnps\t{%2, %0|%0, %2}"
   [(set_attr "type" "sselog")
    (set_attr "mode" "V4SF")])
 
 (define_expand "sse_iorv4sf3"
-  [(set (subreg:TI (match_operand:V4SF 0 "register_operand" "") 0)
-        (ior:TI (subreg:TI (match_operand:V4SF 1 "register_operand" "") 0)
-               (subreg:TI (match_operand:V4SF 2 "nonimmediate_operand" "") 0)))]
+  [(set (match_operand:V4SF 0 "register_operand" "")
+        (ior:V4SF (match_operand:V4SF 1 "register_operand" "")
+                 (match_operand:V4SF 2 "nonimmediate_operand" "")))]
   "TARGET_SSE"
   "")
 
 (define_insn "*sse_iorv4sf3"
-  [(set (subreg:TI (match_operand:V4SF 0 "register_operand" "=x") 0)
-        (ior:TI (match_operand:TI 1 "nonimmediate_operand" "%0")
-               (match_operand:TI 2 "nonimmediate_operand" "xm")))]
-  "TARGET_SSE
-   && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
-  "orps\t{%2, %0|%0, %2}"
-  [(set_attr "type" "sselog")
-   (set_attr "mode" "V4SF")])
-
-(define_insn "*sse_iorsf3"
-  [(set (subreg:TI (match_operand:SF 0 "register_operand" "=x") 0)
-        (ior:TI (match_operand:TI 1 "nonimmediate_operand" "%0")
-               (match_operand:TI 2 "nonimmediate_operand" "xm")))]
+  [(set (match_operand:V4SF 0 "register_operand" "=x")
+        (ior:V4SF (match_operand:V4SF 1 "nonimmediate_operand" "%0")
+                 (match_operand:V4SF 2 "nonimmediate_operand" "xm")))]
   "TARGET_SSE
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "orps\t{%2, %0|%0, %2}"
    (set_attr "mode" "V4SF")])
 
 (define_expand "sse_xorv4sf3"
-  [(set (subreg:TI (match_operand:V4SF 0 "register_operand" "") 0)
-        (xor:TI (subreg:TI (match_operand:V4SF 1 "register_operand" "") 0)
-               (subreg:TI (match_operand:V4SF 2 "nonimmediate_operand" "") 0)))]
-  "TARGET_SSE
-   && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
+  [(set (match_operand:V4SF 0 "register_operand" "")
+        (xor:V4SF (match_operand:V4SF 1 "register_operand" "")
+                 (match_operand:V4SF 2 "nonimmediate_operand" "")))]
+  "TARGET_SSE"
   "")
 
 (define_insn "*sse_xorv4sf3"
-  [(set (subreg:TI (match_operand:V4SF 0 "register_operand" "=x") 0)
-        (xor:TI (match_operand:TI 1 "nonimmediate_operand" "%0")
-               (match_operand:TI 2 "nonimmediate_operand" "xm")))]
-  "TARGET_SSE
-   && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
-  "xorps\t{%2, %0|%0, %2}"
-  [(set_attr "type" "sselog")
-   (set_attr "mode" "V4SF")])
-
-(define_insn "*sse_xorsf3"
-  [(set (subreg:TI (match_operand:SF 0 "register_operand" "=x") 0)
-        (xor:TI (match_operand:TI 1 "nonimmediate_operand" "%0")
-               (match_operand:TI 2 "nonimmediate_operand" "xm")))]
+  [(set (match_operand:V4SF 0 "register_operand" "=x")
+        (xor:V4SF (match_operand:V4SF 1 "nonimmediate_operand" "%0")
+                 (match_operand:V4SF 2 "nonimmediate_operand" "xm")))]
   "TARGET_SSE
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "xorps\t{%2, %0|%0, %2}"
 ;; SSE2 double precision floating point logical operation
 
 (define_expand "sse2_andv2df3"
-  [(set (subreg:TI (match_operand:V2DF 0 "register_operand" "") 0)
-        (and:TI (subreg:TI (match_operand:V2DF 1 "register_operand" "") 0)
-               (subreg:TI (match_operand:V2DF 2 "nonimmediate_operand" "") 0)))]
+  [(set (match_operand:V2DF 0 "register_operand" "")
+        (and:V2DF (match_operand:V2DF 1 "register_operand" "")
+                 (match_operand:V2DF 2 "nonimmediate_operand" "")))]
   "TARGET_SSE2"
   "")
 
 (define_insn "*sse2_andv2df3"
-  [(set (subreg:TI (match_operand:V2DF 0 "register_operand" "=x") 0)
-        (and:TI (match_operand:TI 1 "nonimmediate_operand" "%0")
-               (match_operand:TI 2 "nonimmediate_operand" "xm")))]
-  "TARGET_SSE2
-   && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
-  "andpd\t{%2, %0|%0, %2}"
-  [(set_attr "type" "sselog")
-   (set_attr "mode" "V2DF")])
-
-(define_insn "*sse2_andv2df3"
-  [(set (subreg:TI (match_operand:DF 0 "register_operand" "=x") 0)
-        (and:TI (match_operand:TI 1 "nonimmediate_operand" "%0")
-               (match_operand:TI 2 "nonimmediate_operand" "xm")))]
+  [(set (match_operand:V2DF 0 "register_operand" "=x")
+        (and:V2DF (match_operand:V2DF 1 "nonimmediate_operand" "%0")
+                 (match_operand:V2DF 2 "nonimmediate_operand" "xm")))]
   "TARGET_SSE2
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "andpd\t{%2, %0|%0, %2}"
    (set_attr "mode" "V2DF")])
 
 (define_expand "sse2_nandv2df3"
-  [(set (subreg:TI (match_operand:V2DF 0 "register_operand" "") 0)
-        (and:TI (not:TI (subreg:TI (match_operand:V2DF 1 "register_operand" "") 0))
-               (subreg:TI (match_operand:V2DF 2 "nonimmediate_operand" "") 0)))]
+  [(set (match_operand:V2DF 0 "register_operand" "")
+        (and:V2DF (not:V2DF (match_operand:V2DF 1 "register_operand" ""))
+                 (match_operand:V2DF 2 "nonimmediate_operand" "")))]
   "TARGET_SSE2"
   "")
 
 (define_insn "*sse2_nandv2df3"
-  [(set (subreg:TI (match_operand:V2DF 0 "register_operand" "=x") 0)
-        (and:TI (not:TI (match_operand:TI 1 "register_operand" "0"))
-               (match_operand:TI 2 "nonimmediate_operand" "xm")))]
-  "TARGET_SSE2"
-  "andnpd\t{%2, %0|%0, %2}"
-  [(set_attr "type" "sselog")
-   (set_attr "mode" "V2DF")])
-
-(define_insn "*sse_nandti3_df"
-  [(set (subreg:TI (match_operand:DF 0 "register_operand" "=Y") 0)
-        (and:TI (not:TI (match_operand:TI 1 "register_operand" "0"))
-               (match_operand:TI 2 "nonimmediate_operand" "Ym")))]
+  [(set (match_operand:V2DF 0 "register_operand" "=x")
+        (and:V2DF (not:V2DF (match_operand:V2DF 1 "register_operand" "0"))
+                 (match_operand:V2DF 2 "nonimmediate_operand" "xm")))]
   "TARGET_SSE2"
   "andnpd\t{%2, %0|%0, %2}"
   [(set_attr "type" "sselog")
    (set_attr "mode" "V2DF")])
 
 (define_expand "sse2_iorv2df3"
-  [(set (subreg:TI (match_operand:V2DF 0 "register_operand" "") 0)
-        (ior:TI (subreg:TI (match_operand:V2DF 1 "register_operand" "") 0)
-               (subreg:TI (match_operand:V2DF 2 "nonimmediate_operand" "") 0)))]
+  [(set (match_operand:V2DF 0 "register_operand" "")
+        (ior:V2DF (match_operand:V2DF 1 "register_operand" "")
+                 (match_operand:V2DF 2 "nonimmediate_operand" "")))]
   "TARGET_SSE2"
   "")
 
 (define_insn "*sse2_iorv2df3"
-  [(set (subreg:TI (match_operand:V2DF 0 "register_operand" "=x") 0)
-        (ior:TI (match_operand:TI 1 "nonimmediate_operand" "%0")
-               (match_operand:TI 2 "nonimmediate_operand" "xm")))]
-  "TARGET_SSE2
-   && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
-  "orpd\t{%2, %0|%0, %2}"
-  [(set_attr "type" "sselog")
-   (set_attr "mode" "V2DF")])
-
-(define_insn "*sse2_iordf3"
-  [(set (subreg:TI (match_operand:DF 0 "register_operand" "=x") 0)
-        (ior:TI (match_operand:TI 1 "nonimmediate_operand" "%0")
-               (match_operand:TI 2 "nonimmediate_operand" "xm")))]
+  [(set (match_operand:V2DF 0 "register_operand" "=x")
+        (ior:V2DF (match_operand:V2DF 1 "nonimmediate_operand" "%0")
+                 (match_operand:V2DF 2 "nonimmediate_operand" "xm")))]
   "TARGET_SSE2
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "orpd\t{%2, %0|%0, %2}"
    (set_attr "mode" "V2DF")])
 
 (define_expand "sse2_xorv2df3"
-  [(set (subreg:TI (match_operand:V2DF 0 "register_operand" "") 0)
-        (xor:TI (subreg:TI (match_operand:V2DF 1 "nonimmediate_operand" "") 0)
-               (subreg:TI (match_operand:V2DF 2 "nonimmediate_operand" "") 0)))]
+  [(set (match_operand:V2DF 0 "register_operand" "")
+        (xor:V2DF (match_operand:V2DF 1 "nonimmediate_operand" "")
+                 (match_operand:V2DF 2 "nonimmediate_operand" "")))]
   "TARGET_SSE2"
   "")
 
 (define_insn "*sse2_xorv2df3"
-  [(set (subreg:TI (match_operand:V2DF 0 "register_operand" "=x") 0)
-        (xor:TI (match_operand:TI 1 "nonimmediate_operand" "%0")
-               (match_operand:TI 2 "nonimmediate_operand" "xm")))]
-  "TARGET_SSE2
-   && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
-  "xorpd\t{%2, %0|%0, %2}"
-  [(set_attr "type" "sselog")
-   (set_attr "mode" "V2DF")])
-
-(define_insn "*sse2_xordf3"
-  [(set (subreg:TI (match_operand:DF 0 "register_operand" "=x") 0)
-        (xor:TI (match_operand:TI 1 "nonimmediate_operand" "%0")
-               (match_operand:TI 2 "nonimmediate_operand" "xm")))]
+  [(set (match_operand:V2DF 0 "register_operand" "=x")
+        (xor:V2DF (match_operand:V2DF 1 "nonimmediate_operand" "%0")
+                 (match_operand:V2DF 2 "nonimmediate_operand" "xm")))]
   "TARGET_SSE2
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
   "xorpd\t{%2, %0|%0, %2}"
 
 (define_insn "sse2_nandv2di3"
   [(set (match_operand:V2DI 0 "register_operand" "=x")
-        (and:V2DI (not:V2DI (match_operand:V2DI 1 "nonimmediate_operand" "0"))
+        (and:V2DI (not:V2DI (match_operand:V2DI 1 "register_operand" "0"))
                  (match_operand:V2DI 2 "nonimmediate_operand" "xm")))]
   "TARGET_SSE2
    && (GET_CODE (operands[1]) != MEM || GET_CODE (operands[2]) != MEM)"
         (vec_merge:V4HI (match_operand:V4HI 1 "register_operand" "0")
                        (vec_duplicate:V4HI
                         (truncate:HI (match_operand:SI 2 "nonimmediate_operand" "rm")))
-                       (match_operand:SI 3 "immediate_operand" "i")))]
+                       (match_operand:SI 3 "const_0_to_15_operand" "N")))]
   "TARGET_SSE || TARGET_3DNOW_A"
   "pinsrw\t{%3, %2, %0|%0, %2, %3}"
   [(set_attr "type" "mmxcvt")
   [(set (match_operand:SI 0 "register_operand" "=r")
         (zero_extend:SI (vec_select:HI (match_operand:V4HI 1 "register_operand" "y")
                                       (parallel
-                                       [(match_operand:SI 2 "immediate_operand" "i")]))))]
+                                       [(match_operand:SI 2 "const_0_to_3_operand" "N")]))))]
   "TARGET_SSE || TARGET_3DNOW_A"
   "pextrw\t{%2, %1, %0|%0, %1, %2}"
   [(set_attr "type" "mmxcvt")
 
 (define_insn "mmx_pshufw"
   [(set (match_operand:V4HI 0 "register_operand" "=y")
-        (unspec:V4HI [(match_operand:V4HI 1 "register_operand" "0")
+        (unspec:V4HI [(match_operand:V4HI 1 "nonimmediate_operand" "ym")
                      (match_operand:SI 2 "immediate_operand" "i")]
                     UNSPEC_SHUFFLE))]
   "TARGET_SSE || TARGET_3DNOW_A"
                        (vec_duplicate:V8HI
                         (truncate:HI
                           (match_operand:SI 2 "nonimmediate_operand" "rm")))
-                       (match_operand:SI 3 "immediate_operand" "i")))]
+                       (match_operand:SI 3 "const_0_to_255_operand" "N")))]
   "TARGET_SSE2"
   "pinsrw\t{%3, %2, %0|%0, %2, %3}"
   [(set_attr "type" "ssecvt")
         (zero_extend:SI
          (vec_select:HI (match_operand:V8HI 1 "register_operand" "x")
                         (parallel
-                         [(match_operand:SI 2 "immediate_operand" "i")]))))]
+                         [(match_operand:SI 2 "const_0_to_7_operand" "N")]))))]
   "TARGET_SSE2"
   "pextrw\t{%2, %1, %0|%0, %1, %2}"
   [(set_attr "type" "ssecvt")
 
 (define_insn "sse2_pshufd"
   [(set (match_operand:V4SI 0 "register_operand" "=x")
-        (unspec:V4SI [(match_operand:V4SI 1 "register_operand" "0")
+        (unspec:V4SI [(match_operand:V4SI 1 "nonimmediate_operand" "xm")
                      (match_operand:SI 2 "immediate_operand" "i")]
                     UNSPEC_SHUFFLE))]
   "TARGET_SSE2"
 
 (define_insn "sse2_pshuflw"
   [(set (match_operand:V8HI 0 "register_operand" "=x")
-        (unspec:V8HI [(match_operand:V8HI 1 "register_operand" "0")
+        (unspec:V8HI [(match_operand:V8HI 1 "nonimmediate_operand" "xm")
                      (match_operand:SI 2 "immediate_operand" "i")]
                     UNSPEC_PSHUFLW))]
   "TARGET_SSE2"
 
 (define_insn "sse2_pshufhw"
   [(set (match_operand:V8HI 0 "register_operand" "=x")
-        (unspec:V8HI [(match_operand:V8HI 1 "register_operand" "0")
+        (unspec:V8HI [(match_operand:V8HI 1 "nonimmediate_operand" "xm")
                      (match_operand:SI 2 "immediate_operand" "i")]
                     UNSPEC_PSHUFHW))]
   "TARGET_SSE2"
 (define_insn "ashrv8hi3"
   [(set (match_operand:V8HI 0 "register_operand" "=x")
         (ashiftrt:V8HI (match_operand:V8HI 1 "register_operand" "0")
-                      (match_operand:TI 2 "nonmemory_operand" "xi")))]
+                      (match_operand:SI 2 "nonmemory_operand" "xi")))]
   "TARGET_SSE2"
   "psraw\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseishft")
 (define_insn "ashrv4si3"
   [(set (match_operand:V4SI 0 "register_operand" "=x")
         (ashiftrt:V4SI (match_operand:V4SI 1 "register_operand" "0")
-                      (match_operand:TI 2 "nonmemory_operand" "xi")))]
+                      (match_operand:SI 2 "nonmemory_operand" "xi")))]
   "TARGET_SSE2"
   "psrad\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseishft")
 (define_insn "lshrv8hi3"
   [(set (match_operand:V8HI 0 "register_operand" "=x")
         (lshiftrt:V8HI (match_operand:V8HI 1 "register_operand" "0")
-                      (match_operand:TI 2 "nonmemory_operand" "xi")))]
+                      (match_operand:SI 2 "nonmemory_operand" "xi")))]
   "TARGET_SSE2"
   "psrlw\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseishft")
 (define_insn "lshrv4si3"
   [(set (match_operand:V4SI 0 "register_operand" "=x")
         (lshiftrt:V4SI (match_operand:V4SI 1 "register_operand" "0")
-                      (match_operand:TI 2 "nonmemory_operand" "xi")))]
+                      (match_operand:SI 2 "nonmemory_operand" "xi")))]
   "TARGET_SSE2"
   "psrld\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseishft")
 (define_insn "lshrv2di3"
   [(set (match_operand:V2DI 0 "register_operand" "=x")
         (lshiftrt:V2DI (match_operand:V2DI 1 "register_operand" "0")
-                      (match_operand:TI 2 "nonmemory_operand" "xi")))]
+                      (match_operand:SI 2 "nonmemory_operand" "xi")))]
   "TARGET_SSE2"
   "psrlq\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseishft")
 (define_insn "ashlv8hi3"
   [(set (match_operand:V8HI 0 "register_operand" "=x")
         (ashift:V8HI (match_operand:V8HI 1 "register_operand" "0")
-                    (match_operand:TI 2 "nonmemory_operand" "xi")))]
+                    (match_operand:SI 2 "nonmemory_operand" "xi")))]
   "TARGET_SSE2"
   "psllw\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseishft")
 (define_insn "ashlv4si3"
   [(set (match_operand:V4SI 0 "register_operand" "=x")
         (ashift:V4SI (match_operand:V4SI 1 "register_operand" "0")
-                    (match_operand:TI 2 "nonmemory_operand" "xi")))]
+                    (match_operand:SI 2 "nonmemory_operand" "xi")))]
   "TARGET_SSE2"
   "pslld\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseishft")
 (define_insn "ashlv2di3"
   [(set (match_operand:V2DI 0 "register_operand" "=x")
         (ashift:V2DI (match_operand:V2DI 1 "register_operand" "0")
-                    (match_operand:TI 2 "nonmemory_operand" "xi")))]
+                    (match_operand:SI 2 "nonmemory_operand" "xi")))]
   "TARGET_SSE2"
   "psllq\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseishft")
 (define_insn "ashrv8hi3_ti"
   [(set (match_operand:V8HI 0 "register_operand" "=x")
         (ashiftrt:V8HI (match_operand:V8HI 1 "register_operand" "0")
-                      (subreg:TI (match_operand:V2DI 2 "nonmemory_operand" "xi") 0)))]
+                      (subreg:SI (match_operand:V2DI 2 "nonmemory_operand" "xi") 0)))]
   "TARGET_SSE2"
   "psraw\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseishft")
 (define_insn "ashrv4si3_ti"
   [(set (match_operand:V4SI 0 "register_operand" "=x")
         (ashiftrt:V4SI (match_operand:V4SI 1 "register_operand" "0")
-                      (subreg:TI (match_operand:V2DI 2 "nonmemory_operand" "xi") 0)))]
+                      (subreg:SI (match_operand:V2DI 2 "nonmemory_operand" "xi") 0)))]
   "TARGET_SSE2"
   "psrad\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseishft")
 (define_insn "lshrv8hi3_ti"
   [(set (match_operand:V8HI 0 "register_operand" "=x")
         (lshiftrt:V8HI (match_operand:V8HI 1 "register_operand" "0")
-                      (subreg:TI (match_operand:V2DI 2 "nonmemory_operand" "xi") 0)))]
+                      (subreg:SI (match_operand:V2DI 2 "nonmemory_operand" "xi") 0)))]
   "TARGET_SSE2"
   "psrlw\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseishft")
 (define_insn "lshrv4si3_ti"
   [(set (match_operand:V4SI 0 "register_operand" "=x")
         (lshiftrt:V4SI (match_operand:V4SI 1 "register_operand" "0")
-                      (subreg:TI (match_operand:V2DI 2 "nonmemory_operand" "xi") 0)))]
+                      (subreg:SI (match_operand:V2DI 2 "nonmemory_operand" "xi") 0)))]
   "TARGET_SSE2"
   "psrld\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseishft")
 (define_insn "lshrv2di3_ti"
   [(set (match_operand:V2DI 0 "register_operand" "=x")
         (lshiftrt:V2DI (match_operand:V2DI 1 "register_operand" "0")
-                      (subreg:TI (match_operand:V2DI 2 "nonmemory_operand" "xi") 0)))]
+                      (subreg:SI (match_operand:V2DI 2 "nonmemory_operand" "xi") 0)))]
   "TARGET_SSE2"
   "psrlq\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseishft")
 (define_insn "ashlv8hi3_ti"
   [(set (match_operand:V8HI 0 "register_operand" "=x")
         (ashift:V8HI (match_operand:V8HI 1 "register_operand" "0")
-                    (subreg:TI (match_operand:V2DI 2 "nonmemory_operand" "xi") 0)))]
+                    (subreg:SI (match_operand:V2DI 2 "nonmemory_operand" "xi") 0)))]
   "TARGET_SSE2"
   "psllw\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseishft")
 (define_insn "ashlv4si3_ti"
   [(set (match_operand:V4SI 0 "register_operand" "=x")
         (ashift:V4SI (match_operand:V4SI 1 "register_operand" "0")
-                    (subreg:TI (match_operand:V2DI 2 "nonmemory_operand" "xi") 0)))]
+                    (subreg:SI (match_operand:V2DI 2 "nonmemory_operand" "xi") 0)))]
   "TARGET_SSE2"
   "pslld\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseishft")
 (define_insn "ashlv2di3_ti"
   [(set (match_operand:V2DI 0 "register_operand" "=x")
         (ashift:V2DI (match_operand:V2DI 1 "register_operand" "0")
-                    (subreg:TI (match_operand:V2DI 2 "nonmemory_operand" "xi") 0)))]
+                    (subreg:SI (match_operand:V2DI 2 "nonmemory_operand" "xi") 0)))]
   "TARGET_SSE2"
   "psllq\t{%2, %0|%0, %2}"
   [(set_attr "type" "sseishft")
         (vec_select:DF (match_operand:V2DF 1 "register_operand" "0")
                        (parallel [(const_int 1)]))
         (vec_select:DF (match_operand:V2DF 2 "register_operand" "x")
-                       (parallel [(const_int 0)]))))]
+                       (parallel [(const_int 1)]))))]
   "TARGET_SSE2"
   "unpckhpd\t{%2, %0|%0, %2}"
   [(set_attr "type" "ssecvt")
-   (set_attr "mode" "TI")])
+   (set_attr "mode" "V2DF")])
 
 (define_insn "sse2_unpcklpd"
   [(set (match_operand:V2DF 0 "register_operand" "=x")
         (vec_select:DF (match_operand:V2DF 1 "register_operand" "0")
                        (parallel [(const_int 0)]))
         (vec_select:DF (match_operand:V2DF 2 "register_operand" "x")
-                       (parallel [(const_int 1)]))))]
+                       (parallel [(const_int 0)]))))]
   "TARGET_SSE2"
   "unpcklpd\t{%2, %0|%0, %2}"
   [(set_attr "type" "ssecvt")
-   (set_attr "mode" "TI")])
+   (set_attr "mode" "V2DF")])
 
 ;; MMX pack/unpack insns.
 
   [(set_attr "type" "ssecvt")
    (set_attr "mode" "V2DF")])
 
-(define_insn "sse2_movlpd"
-  [(set (match_operand:V2DF 0 "nonimmediate_operand" "=x,m")
-       (vec_merge:V2DF
-        (match_operand:V2DF 1 "nonimmediate_operand" "0,0")
-        (match_operand:V2DF 2 "nonimmediate_operand" "m,x")
-        (const_int 1)))]
-  "TARGET_SSE2 && (GET_CODE (operands[1]) == MEM || GET_CODE (operands[2]) == MEM)"
-  "movlpd\t{%2, %0|%0, %2}"
-  [(set_attr "type" "ssecvt")
-   (set_attr "mode" "V2DF")])
-
 (define_expand "sse2_loadsd"
   [(match_operand:V2DF 0 "register_operand" "")
    (match_operand:DF 1 "memory_operand" "")]
    (set_attr "mode" "DF")])
 
 (define_insn "sse2_movsd"
-  [(set (match_operand:V2DF 0 "register_operand" "=x")
+  [(set (match_operand:V2DF 0 "nonimmediate_operand" "=x,x,m")
        (vec_merge:V2DF
-        (match_operand:V2DF 1 "register_operand" "0")
-        (match_operand:V2DF 2 "register_operand" "x")
+        (match_operand:V2DF 1 "nonimmediate_operand" "0,0,0")
+        (match_operand:V2DF 2 "nonimmediate_operand" "x,m,x")
         (const_int 1)))]
-  "TARGET_SSE2"
-  "movsd\t{%2, %0|%0, %2}"
+  "TARGET_SSE2 && ix86_binary_operator_ok (UNKNOWN, V2DFmode, operands)"
+  "@movsd\t{%2, %0|%0, %2}
+    movlpd\t{%2, %0|%0, %2}
+    movlpd\t{%2, %0|%0, %2}"
   [(set_attr "type" "ssecvt")
-   (set_attr "mode" "DF")])
+   (set_attr "mode" "DF,V2DF,V2DF")])
 
 (define_insn "sse2_storesd"
   [(set (match_operand:DF 0 "memory_operand" "=m")
   "lfence"
   [(set_attr "type" "sse")
    (set_attr "memory" "unknown")])
+
+;; SSE3
+
+(define_insn "mwait"
+  [(unspec_volatile [(match_operand:SI 0 "register_operand" "a")
+                    (match_operand:SI 1 "register_operand" "c")]
+                   UNSPECV_MWAIT)]
+  "TARGET_SSE3"
+  "mwait\t%0, %1"
+  [(set_attr "length" "3")])
+
+(define_insn "monitor"
+  [(unspec_volatile [(match_operand:SI 0 "register_operand" "a")
+                    (match_operand:SI 1 "register_operand" "c")
+                    (match_operand:SI 2 "register_operand" "d")]
+                   UNSPECV_MONITOR)]
+  "TARGET_SSE3"
+  "monitor\t%0, %1, %2"
+  [(set_attr "length" "3")])
+
+;; SSE3 arithmetic
+
+(define_insn "addsubv4sf3"
+  [(set (match_operand:V4SF 0 "register_operand" "=x")
+        (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "0")
+                     (match_operand:V4SF 2 "nonimmediate_operand" "xm")]
+                    UNSPEC_ADDSUB))]
+  "TARGET_SSE3"
+  "addsubps\t{%2, %0|%0, %2}"
+  [(set_attr "type" "sseadd")
+   (set_attr "mode" "V4SF")])
+
+(define_insn "addsubv2df3"
+  [(set (match_operand:V2DF 0 "register_operand" "=x")
+        (unspec:V2DF [(match_operand:V2DF 1 "register_operand" "0")
+                     (match_operand:V2DF 2 "nonimmediate_operand" "xm")]
+                    UNSPEC_ADDSUB))]
+  "TARGET_SSE3"
+  "addsubpd\t{%2, %0|%0, %2}"
+  [(set_attr "type" "sseadd")
+   (set_attr "mode" "V2DF")])
+
+(define_insn "haddv4sf3"
+  [(set (match_operand:V4SF 0 "register_operand" "=x")
+        (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "0")
+                     (match_operand:V4SF 2 "nonimmediate_operand" "xm")]
+                    UNSPEC_HADD))]
+  "TARGET_SSE3"
+  "haddps\t{%2, %0|%0, %2}"
+  [(set_attr "type" "sseadd")
+   (set_attr "mode" "V4SF")])
+
+(define_insn "haddv2df3"
+  [(set (match_operand:V2DF 0 "register_operand" "=x")
+        (unspec:V2DF [(match_operand:V2DF 1 "register_operand" "0")
+                     (match_operand:V2DF 2 "nonimmediate_operand" "xm")]
+                    UNSPEC_HADD))]
+  "TARGET_SSE3"
+  "haddpd\t{%2, %0|%0, %2}"
+  [(set_attr "type" "sseadd")
+   (set_attr "mode" "V2DF")])
+
+(define_insn "hsubv4sf3"
+  [(set (match_operand:V4SF 0 "register_operand" "=x")
+        (unspec:V4SF [(match_operand:V4SF 1 "register_operand" "0")
+                     (match_operand:V4SF 2 "nonimmediate_operand" "xm")]
+                    UNSPEC_HSUB))]
+  "TARGET_SSE3"
+  "hsubps\t{%2, %0|%0, %2}"
+  [(set_attr "type" "sseadd")
+   (set_attr "mode" "V4SF")])
+
+(define_insn "hsubv2df3"
+  [(set (match_operand:V2DF 0 "register_operand" "=x")
+        (unspec:V2DF [(match_operand:V2DF 1 "register_operand" "0")
+                     (match_operand:V2DF 2 "nonimmediate_operand" "xm")]
+                    UNSPEC_HSUB))]
+  "TARGET_SSE3"
+  "hsubpd\t{%2, %0|%0, %2}"
+  [(set_attr "type" "sseadd")
+   (set_attr "mode" "V2DF")])
+
+(define_insn "movshdup"
+  [(set (match_operand:V4SF 0 "register_operand" "=x")
+        (unspec:V4SF
+        [(match_operand:V4SF 1 "nonimmediate_operand" "xm")] UNSPEC_MOVSHDUP))]
+  "TARGET_SSE3"
+  "movshdup\t{%1, %0|%0, %1}"
+  [(set_attr "type" "sse")
+   (set_attr "mode" "V4SF")])
+
+(define_insn "movsldup"
+  [(set (match_operand:V4SF 0 "register_operand" "=x")
+        (unspec:V4SF
+        [(match_operand:V4SF 1 "nonimmediate_operand" "xm")] UNSPEC_MOVSLDUP))]
+  "TARGET_SSE3"
+  "movsldup\t{%1, %0|%0, %1}"
+  [(set_attr "type" "sse")
+   (set_attr "mode" "V4SF")])
+
+(define_insn "lddqu"
+  [(set (match_operand:V16QI 0 "register_operand" "=x")
+       (unspec:V16QI [(match_operand:V16QI 1 "memory_operand" "m")]
+                      UNSPEC_LDQQU))]
+  "TARGET_SSE3"
+  "lddqu\t{%1, %0|%0, %1}"
+  [(set_attr "type" "ssecvt")
+   (set_attr "mode" "TI")])
+
+(define_insn "loadddup"
+  [(set (match_operand:V2DF 0 "register_operand" "=x")
+       (vec_duplicate:V2DF (match_operand:DF 1 "memory_operand" "m")))]
+  "TARGET_SSE3"
+  "movddup\t{%1, %0|%0, %1}"
+  [(set_attr "type" "ssecvt")
+   (set_attr "mode" "DF")])
+
+(define_insn "movddup"
+  [(set (match_operand:V2DF 0 "register_operand" "=x")
+       (vec_duplicate:V2DF
+        (vec_select:DF (match_operand:V2DF 1 "register_operand" "x")
+                       (parallel [(const_int 0)]))))]
+  "TARGET_SSE3"
+  "movddup\t{%1, %0|%0, %1}"
+  [(set_attr "type" "ssecvt")
+   (set_attr "mode" "DF")])