OSDN Git Service

Backport from mainline
[pf3gnuchains/gcc-fork.git] / gcc / config / i386 / i386.md
index 7d720f9..f1b53a0 100644 (file)
@@ -1,6 +1,6 @@
 ;; GCC machine description for IA-32 and x86-64.
 ;; Copyright (C) 1988, 1994, 1995, 1996, 1997, 1998, 1999, 2000,
-;; 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011
+;; 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012
 ;; Free Software Foundation, Inc.
 ;; Mostly by William Schelter.
 ;; x86_64 support added by Jan Hubicka
@@ -38,6 +38,7 @@
 ;; Z -- likewise, with special suffixes for x87 instructions.
 ;; * -- print a star (in certain assembler syntax)
 ;; A -- print an absolute memory reference.
+;; E -- print address with DImode register names if TARGET_64BIT.
 ;; w -- print the operand as if it's a "word" (HImode) even if it isn't.
 ;; s -- print a shift double count, followed by the assemblers argument
 ;;     delimiter.
@@ -62,8 +63,6 @@
 ;; ; -- print a semicolon (after prefixes due to bug in older gas).
 ;; @ -- print a segment register of thread base pointer load
 
-;; UNSPEC usage:
-
 (define_c_enum "unspec" [
   ;; Relocation specifiers
   UNSPEC_GOT
@@ -82,8 +81,6 @@
   ;; Prologue support
   UNSPEC_STACK_ALLOC
   UNSPEC_SET_GOT
-  UNSPEC_REG_SAVE
-  UNSPEC_DEF_CFA
   UNSPEC_SET_RIP
   UNSPEC_SET_GOT_OFFSET
   UNSPEC_MEMORY_BLOCKAGE
   UNSPEC_LD_MPIC       ; load_macho_picbase
   UNSPEC_TRUNC_NOOP
   UNSPEC_DIV_ALREADY_SPLIT
+  UNSPEC_MS_TO_SYSV_CALL
   UNSPEC_CALL_NEEDS_VZEROUPPER
   UNSPEC_PAUSE
+  UNSPEC_LEA_ADDR
 
   ;; For SSE/MMX support:
   UNSPEC_FIX_NOTRUNC
   UNSPEC_MASKMOV
   UNSPEC_MOVMSK
-  UNSPEC_MOVNT
-  UNSPEC_MOVU
   UNSPEC_RCP
   UNSPEC_RSQRT
-  UNSPEC_SFENCE
-  UNSPEC_PFRCP
-  UNSPEC_PFRCPIT1
-  UNSPEC_PFRCPIT2
-  UNSPEC_PFRSQRT
-  UNSPEC_PFRSQIT1
-  UNSPEC_MFENCE
-  UNSPEC_LFENCE
   UNSPEC_PSADBW
-  UNSPEC_LDDQU
-  UNSPEC_MS_TO_SYSV_CALL
 
   ;; Generic math support
   UNSPEC_COPYSIGN
   UNSPEC_SP_TLS_SET
   UNSPEC_SP_TLS_TEST
 
-  ;; SSSE3
-  UNSPEC_PSHUFB
-  UNSPEC_PSIGN
-  UNSPEC_PALIGNR
-
-  ;; For SSE4A support
-  UNSPEC_EXTRQI
-  UNSPEC_EXTRQ
-  UNSPEC_INSERTQI
-  UNSPEC_INSERTQ
-
-  ;; For SSE4.1 support
-  UNSPEC_BLENDV
-  UNSPEC_INSERTPS
-  UNSPEC_DP
-  UNSPEC_MOVNTDQA
-  UNSPEC_MPSADBW
-  UNSPEC_PHMINPOSUW
-  UNSPEC_PTEST
+  ;; For ROUND support
   UNSPEC_ROUND
 
-  ;; For SSE4.2 support
+  ;; For CRC32 support
   UNSPEC_CRC32
-  UNSPEC_PCMPESTR
-  UNSPEC_PCMPISTR
-
-  ;; For FMA4 support
-  UNSPEC_FMADDSUB
-  UNSPEC_XOP_UNSIGNED_CMP
-  UNSPEC_XOP_TRUEFALSE
-  UNSPEC_XOP_PERMUTE
-  UNSPEC_FRCZ
-
-  ;; For AES support
-  UNSPEC_AESENC
-  UNSPEC_AESENCLAST
-  UNSPEC_AESDEC
-  UNSPEC_AESDECLAST
-  UNSPEC_AESIMC
-  UNSPEC_AESKEYGENASSIST
-
-  ;; For PCLMUL support
-  UNSPEC_PCLMUL
-
-  ;; For AVX support
-  UNSPEC_PCMP
-  UNSPEC_VPERMIL
-  UNSPEC_VPERMIL2
-  UNSPEC_VPERMIL2F128
-  UNSPEC_CAST
-  UNSPEC_VTESTP
-  UNSPEC_VCVTPH2PS
-  UNSPEC_VCVTPS2PH
-
-  ;; For AVX2 support
-  UNSPEC_VPERMSI
-  UNSPEC_VPERMDF
-  UNSPEC_VPERMSF
-  UNSPEC_VPERMTI
-  UNSPEC_GATHER
-  UNSPEC_VSIBADDR
 
   ;; For BMI support
   UNSPEC_BEXTR
 
-  ;; For RDRAND support
-  UNSPEC_RDRAND
-
   ;; For BMI2 support
   UNSPEC_PDEP
   UNSPEC_PEXT
   UNSPECV_BLOCKAGE
   UNSPECV_STACK_PROBE
   UNSPECV_PROBE_STACK_RANGE
-  UNSPECV_EMMS
-  UNSPECV_LDMXCSR
-  UNSPECV_STMXCSR
-  UNSPECV_FEMMS
-  UNSPECV_CLFLUSH
   UNSPECV_ALIGN
-  UNSPECV_MONITOR
-  UNSPECV_MWAIT
-  UNSPECV_CMPXCHG
-  UNSPECV_XCHG
-  UNSPECV_LOCK
   UNSPECV_PROLOGUE_USE
+  UNSPECV_SPLIT_STACK_RETURN
   UNSPECV_CLD
   UNSPECV_NOPS
-  UNSPECV_VZEROALL
-  UNSPECV_VZEROUPPER
   UNSPECV_RDTSC
   UNSPECV_RDTSCP
   UNSPECV_RDPMC
   UNSPECV_RDGSBASE
   UNSPECV_WRFSBASE
   UNSPECV_WRGSBASE
-  UNSPECV_SPLIT_STACK_RETURN
+
+  ;; For RDRAND support
+  UNSPECV_RDRAND
 ])
 
 ;; Constants to represent rounding modes in the ROUND instruction
 (define_attr "movu" "0,1" (const_string "0"))
 
 ;; Used to control the "enabled" attribute on a per-instruction basis.
-(define_attr "isa" "base,sse2,sse2_noavx,sse3,sse4,sse4_noavx,noavx,avx,bmi2"
+(define_attr "isa" "base,sse2,sse2_noavx,sse3,sse4,sse4_noavx,noavx,avx,
+                   bmi2,fma,fma4"
   (const_string "base"))
 
 (define_attr "enabled" ""
         (eq_attr "isa" "avx") (symbol_ref "TARGET_AVX")
         (eq_attr "isa" "noavx") (symbol_ref "!TARGET_AVX")
         (eq_attr "isa" "bmi2") (symbol_ref "TARGET_BMI2")
+        (eq_attr "isa" "fma") (symbol_ref "TARGET_FMA")
+        ;; Fma instruction selection has to be done based on
+        ;; register pressure. For generating fma4, a cost model
+        ;; based on register pressure is required. Till then,
+        ;; fma4 instruction is disabled for targets that implement
+        ;; both fma and fma4 instruction sets.
+        (eq_attr "isa" "fma4")
+          (symbol_ref "TARGET_FMA4 && !TARGET_FMA")
        ]
        (const_int 1)))
 
       return "#";
 
     case TYPE_LEA:
-      return "lea{q}\t{%a1, %0|%0, %a1}";
+      return "lea{q}\t{%E1, %0|%0, %E1}";
 
     default:
       gcc_assert (!flag_pic || LEGITIMATE_PIC_OPERAND_P (operands[1]));
        return "mov{l}\t{%k1, %k0|%k0, %k1}";
       else if (which_alternative == 2)
        return "movabs{q}\t{%1, %0|%0, %1}";
+      else if (ix86_use_lea_for_mov (insn, operands))
+       return "lea{q}\t{%E1, %0|%0, %E1}";
       else
        return "mov{q}\t{%1, %0|%0, %1}";
     }
       return "movd\t{%1, %0|%0, %1}";
 
     case TYPE_LEA:
-      return "lea{l}\t{%a1, %0|%0, %a1}";
+      return "lea{l}\t{%E1, %0|%0, %E1}";
 
     default:
       gcc_assert (!flag_pic || LEGITIMATE_PIC_OPERAND_P (operands[1]));
-      return "mov{l}\t{%1, %0|%0, %1}";
+      if (ix86_use_lea_for_mov (insn, operands))
+       return "lea{l}\t{%E1, %0|%0, %E1}";
+      else
+       return "mov{l}\t{%1, %0|%0, %1}";
     }
 }
   [(set (attr "type")
 ;; into register when rax is not available
 (define_insn "*movabs<mode>_1"
   [(set (mem:SWI1248x (match_operand:DI 0 "x86_64_movabs_operand" "i,r"))
-       (match_operand:SWI1248x 1 "nonmemory_operand" "a,er"))]
-  "TARGET_64BIT && ix86_check_movabs (insn, 0)"
+       (match_operand:SWI1248x 1 "nonmemory_operand" "a,r<i>"))]
+  "TARGET_LP64 && ix86_check_movabs (insn, 0)"
   "@
    movabs{<imodesuffix>}\t{%1, %P0|%P0, %1}
    mov{<imodesuffix>}\t{%1, %a0|%a0, %1}"
 (define_insn "*movabs<mode>_2"
   [(set (match_operand:SWI1248x 0 "register_operand" "=a,r")
         (mem:SWI1248x (match_operand:DI 1 "x86_64_movabs_operand" "i,r")))]
-  "TARGET_64BIT && ix86_check_movabs (insn, 1)"
+  "TARGET_LP64 && ix86_check_movabs (insn, 1)"
   "@
    movabs{<imodesuffix>}\t{%P1, %0|%0, %P1}
    mov{<imodesuffix>}\t{%a1, %0|%0, %a1}"
                             (const_int 8))
        (match_operand:SWI48x 1 "nonmemory_operand" "Qn"))]
   "TARGET_64BIT"
-  "mov{b}\t{%b1, %h0|%h0, %b1}"
+{
+  if (CONST_INT_P (operands[1]))
+    operands[1] = simplify_gen_subreg (QImode, operands[1], <MODE>mode, 0);
+  return "mov{b}\t{%b1, %h0|%h0, %b1}";
+}
   [(set_attr "type" "imov")
    (set_attr "mode" "QI")])
 
                         (const_int 8))
        (match_operand:SI 1 "general_operand" "Qmn"))]
   "!TARGET_64BIT"
-  "mov{b}\t{%b1, %h0|%h0, %b1}"
+{
+  if (CONST_INT_P (operands[1]))
+    operands[1] = simplify_gen_subreg (QImode, operands[1], SImode, 0);
+  return "mov{b}\t{%b1, %h0|%h0, %b1}";
+}
   [(set_attr "type" "imov")
    (set_attr "mode" "QI")])
 
         (match_operand:SI 1 "nonimmediate_operand" "rm,0,r   ,m  ,r   ,m")))]
   "TARGET_64BIT"
   "@
-   mov\t{%k1, %k0|%k0, %k1}
+   mov{l}\t{%1, %k0|%k0, %1}
    #
    movd\t{%1, %0|%0, %1}
    movd\t{%1, %0|%0, %1}
 (define_insn "*floathi<mode>2_i387_with_temp"
   [(set (match_operand:X87MODEF 0 "register_operand" "=f,f")
        (float:X87MODEF (match_operand:HI 1 "nonimmediate_operand" "m,?r")))
-  (clobber (match_operand:HI 2 "memory_operand" "=m,m"))]
+  (clobber (match_operand:HI 2 "memory_operand" "=X,m"))]
   "TARGET_80387
    && (!(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
        || TARGET_MIX_SSE_I387)"
    && reload_completed
    && (SSE_REG_P (operands[0])
        || (GET_CODE (operands[0]) == SUBREG
-          && SSE_REG_P (operands[0])))"
+          && SSE_REG_P (SUBREG_REG (operands[0]))))"
   [(set (match_dup 0) (float:MODEF (match_dup 1)))])
 
 (define_split
    && reload_completed
    && (SSE_REG_P (operands[0])
        || (GET_CODE (operands[0]) == SUBREG
-          && SSE_REG_P (operands[0])))"
+          && SSE_REG_P (SUBREG_REG (operands[0]))))"
   [(set (match_dup 2) (match_dup 1))
    (set (match_dup 0) (float:MODEF (match_dup 2)))])
 
    && reload_completed
    && (SSE_REG_P (operands[0])
        || (GET_CODE (operands[0]) == SUBREG
-          && SSE_REG_P (operands[0])))"
+          && SSE_REG_P (SUBREG_REG (operands[0]))))"
   [(const_int 0)]
 {
   rtx op1 = operands[1];
       emit_insn (gen_sse2_loadld (operands[4],
                                  CONST0_RTX (V4SImode), operands[2]));
     }
-  emit_insn
-    (gen_sse2_cvtdq2<ssevecmodesuffix> (operands[3], operands[4]));
+  if (<ssevecmode>mode == V4SFmode)
+    emit_insn (gen_floatv4siv4sf2 (operands[3], operands[4]));
+  else
+    emit_insn (gen_sse2_cvtdq2pd (operands[3], operands[4]));
   DONE;
 })
 
    && reload_completed
    && (SSE_REG_P (operands[0])
        || (GET_CODE (operands[0]) == SUBREG
-          && SSE_REG_P (operands[0])))"
+          && SSE_REG_P (SUBREG_REG (operands[0]))))"
   [(const_int 0)]
 {
   operands[3] = simplify_gen_subreg (<ssevecmode>mode, operands[0],
 
   emit_insn (gen_sse2_loadld (operands[4],
                              CONST0_RTX (V4SImode), operands[1]));
-  emit_insn
-    (gen_sse2_cvtdq2<ssevecmodesuffix> (operands[3], operands[4]));
+  if (<ssevecmode>mode == V4SFmode)
+    emit_insn (gen_floatv4siv4sf2 (operands[3], operands[4]));
+  else
+    emit_insn (gen_sse2_cvtdq2pd (operands[3], operands[4]));
   DONE;
 })
 
    && reload_completed
    && (SSE_REG_P (operands[0])
        || (GET_CODE (operands[0]) == SUBREG
-          && SSE_REG_P (operands[0])))"
+          && SSE_REG_P (SUBREG_REG (operands[0]))))"
   [(const_int 0)]
 {
   rtx op1 = operands[1];
     operands[4] = simplify_gen_subreg (V4SImode, operands[1], SImode, 0);
   else
     gcc_unreachable ();
-  emit_insn
-    (gen_sse2_cvtdq2<ssevecmodesuffix> (operands[3], operands[4]));
+  if (<ssevecmode>mode == V4SFmode)
+    emit_insn (gen_floatv4siv4sf2 (operands[3], operands[4]));
+  else
+    emit_insn (gen_sse2_cvtdq2pd (operands[3], operands[4]));
   DONE;
 })
 
    && reload_completed
    && (SSE_REG_P (operands[0])
        || (GET_CODE (operands[0]) == SUBREG
-          && SSE_REG_P (operands[0])))"
+          && SSE_REG_P (SUBREG_REG (operands[0]))))"
   [(const_int 0)]
 {
   operands[3] = simplify_gen_subreg (<ssevecmode>mode, operands[0],
 
   emit_insn (gen_sse2_loadld (operands[4],
                              CONST0_RTX (V4SImode), operands[1]));
-  emit_insn
-    (gen_sse2_cvtdq2<ssevecmodesuffix> (operands[3], operands[4]));
+  if (<ssevecmode>mode == V4SFmode)
+    emit_insn (gen_floatv4siv4sf2 (operands[3], operands[4]));
+  else
+    emit_insn (gen_sse2_cvtdq2pd (operands[3], operands[4]));
   DONE;
 })
 
    && reload_completed
    && (SSE_REG_P (operands[0])
        || (GET_CODE (operands[0]) == SUBREG
-          && SSE_REG_P (operands[0])))"
+          && SSE_REG_P (SUBREG_REG (operands[0]))))"
   [(set (match_dup 0) (float:MODEF (match_dup 1)))])
 
 (define_insn "*float<SWI48x:mode><MODEF:mode>2_sse_nointerunit"
    && reload_completed
    && (SSE_REG_P (operands[0])
        || (GET_CODE (operands[0]) == SUBREG
-          && SSE_REG_P (operands[0])))"
+          && SSE_REG_P (SUBREG_REG (operands[0]))))"
   [(set (match_dup 2) (match_dup 1))
    (set (match_dup 0) (float:MODEF (match_dup 2)))])
 
    && reload_completed
    && (SSE_REG_P (operands[0])
        || (GET_CODE (operands[0]) == SUBREG
-          && SSE_REG_P (operands[0])))"
+          && SSE_REG_P (SUBREG_REG (operands[0]))))"
   [(set (match_dup 0) (float:MODEF (match_dup 1)))])
 
 (define_insn "*float<SWI48x:mode><X87MODEF:mode>2_i387_with_temp"
   DONE;
 })
 \f
+;; Load effective address instructions
+
+(define_insn_and_split "*lea<mode>"
+  [(set (match_operand:SWI48 0 "register_operand" "=r")
+       (match_operand:SWI48 1 "lea_address_operand" "p"))]
+  ""
+{
+  rtx addr = operands[1];
+
+  if (SImode_address_operand (addr, VOIDmode))
+    {
+      gcc_assert (TARGET_64BIT);
+      return "lea{l}\t{%E1, %k0|%k0, %E1}";
+    }
+  else 
+    return "lea{<imodesuffix>}\t{%E1, %0|%0, %E1}";
+}
+  "reload_completed && ix86_avoid_lea_for_addr (insn, operands)"
+  [(const_int 0)]
+{
+  ix86_split_lea_for_addr (operands, <MODE>mode);
+  DONE;
+}
+  [(set_attr "type" "lea")
+   (set (attr "mode")
+     (if_then_else
+       (match_operand 1 "SImode_address_operand")
+       (const_string "SI")
+       (const_string "<MODE>")))])
+\f
 ;; Add instructions
 
 (define_expand "add<mode>3"
   [(set_attr "type" "alu")
    (set_attr "mode" "QI")])
 
-(define_insn_and_split "*lea_1"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-       (subreg:SI (match_operand:DI 1 "lea_address_operand" "p") 0))]
-  "TARGET_64BIT"
-  "lea{l}\t{%a1, %0|%0, %a1}"
-  "&& reload_completed && ix86_avoid_lea_for_addr (insn, operands)"
-  [(const_int 0)]
-{
-  ix86_split_lea_for_addr (operands, SImode);
-  DONE;
-}
-  [(set_attr "type" "lea")
-   (set_attr "mode" "SI")])
-
-(define_insn_and_split "*lea<mode>_2"
-  [(set (match_operand:SWI48 0 "register_operand" "=r")
-       (match_operand:SWI48 1 "lea_address_operand" "p"))]
-  ""
-  "lea{<imodesuffix>}\t{%a1, %0|%0, %a1}"
-  "reload_completed && ix86_avoid_lea_for_addr (insn, operands)"
-  [(const_int 0)]
-{
-  ix86_split_lea_for_addr (operands, <MODE>mode);
-  DONE;
-}
-  [(set_attr "type" "lea")
-   (set_attr "mode" "<MODE>")])
-
-(define_insn "*lea_3_zext"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (zero_extend:DI
-         (subreg:SI (match_operand:DI 1 "lea_address_operand" "p") 0)))]
-  "TARGET_64BIT"
-  "lea{l}\t{%a1, %k0|%k0, %a1}"
-  [(set_attr "type" "lea")
-   (set_attr "mode" "SI")])
-
-(define_insn "*lea_4_zext"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (zero_extend:DI
-         (match_operand:SI 1 "lea_address_operand" "p")))]
-  "TARGET_64BIT"
-  "lea{l}\t{%a1, %k0|%k0, %a1}"
-  [(set_attr "type" "lea")
-   (set_attr "mode" "SI")])
-
-(define_insn "*lea_5_zext"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (and:DI
-         (subreg:DI (match_operand:SI 1 "lea_address_operand" "p") 0)
-         (match_operand:DI 2 "const_32bit_mask" "n")))]
-  "TARGET_64BIT"
-  "lea{l}\t{%a1, %k0|%k0, %a1}"
-  [(set_attr "type" "lea")
-   (set_attr "mode" "SI")])
-
-(define_insn "*lea_6_zext"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (and:DI
-         (match_operand:DI 1 "lea_address_operand" "p")
-         (match_operand:DI 2 "const_32bit_mask" "n")))]
-  "TARGET_64BIT"
-  "lea{l}\t{%a1, %k0|%k0, %a1}"
-  [(set_attr "type" "lea")
-   (set_attr "mode" "SI")])
-
 (define_insn "*add<mode>_1"
   [(set (match_operand:SWI48 0 "nonimmediate_operand" "=r,rm,r,r")
        (plus:SWI48
   [(set (reg FLAGS_REG)
        (compare
          (plus:SWI
-           (match_operand:SWI 1 "nonimmediate_operand" "%0,0")
-           (match_operand:SWI 2 "<general_operand>" "<g>,<r><i>"))
+           (match_operand:SWI 1 "nonimmediate_operand" "%0,0,<r>")
+           (match_operand:SWI 2 "<general_operand>" "<g>,<r><i>,0"))
          (const_int 0)))
-   (set (match_operand:SWI 0 "nonimmediate_operand" "=<r>,<r>m")
+   (set (match_operand:SWI 0 "nonimmediate_operand" "=<r>,<r>m,<r>")
        (plus:SWI (match_dup 1) (match_dup 2)))]
   "ix86_match_ccmode (insn, CCGOCmode)
    && ix86_binary_operator_ok (PLUS, <MODE>mode, operands)"
        }
 
     default:
+      if (which_alternative == 2)
+       {
+         rtx tmp;
+         tmp = operands[1], operands[1] = operands[2], operands[2] = tmp;
+       }
+        
+      gcc_assert (rtx_equal_p (operands[0], operands[1]));
       if (x86_maybe_negate_const_int (&operands[2], <MODE>mode))
         return "sub{<imodesuffix>}\t{%2, %0|%0, %2}";
 
 (define_insn "*addsi_2_zext"
   [(set (reg FLAGS_REG)
        (compare
-         (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
-                  (match_operand:SI 2 "x86_64_general_operand" "rme"))
+         (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,r")
+                  (match_operand:SI 2 "x86_64_general_operand" "rme,0"))
          (const_int 0)))
-   (set (match_operand:DI 0 "register_operand" "=r")
+   (set (match_operand:DI 0 "register_operand" "=r,r")
        (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
   "TARGET_64BIT && ix86_match_ccmode (insn, CCGOCmode)
    && ix86_binary_operator_ok (PLUS, SImode, operands)"
        }
 
     default:
+      if (which_alternative == 1)
+       {
+         rtx tmp;
+         tmp = operands[1], operands[1] = operands[2], operands[2] = tmp;
+       }
+
       if (x86_maybe_negate_const_int (&operands[2], SImode))
         return "sub{l}\t{%2, %k0|%k0, %2}";
 
 (define_insn "*add<mode>_3"
   [(set (reg FLAGS_REG)
        (compare
-         (neg:SWI (match_operand:SWI 2 "<general_operand>" "<g>"))
-         (match_operand:SWI 1 "nonimmediate_operand" "%0")))
-   (clobber (match_scratch:SWI 0 "=<r>"))]
+         (neg:SWI (match_operand:SWI 2 "<general_operand>" "<g>,0"))
+         (match_operand:SWI 1 "nonimmediate_operand" "%0,<r>")))
+   (clobber (match_scratch:SWI 0 "=<r>,<r>"))]
   "ix86_match_ccmode (insn, CCZmode)
    && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
 {
        }
 
     default:
+      if (which_alternative == 1)
+       {
+         rtx tmp;
+         tmp = operands[1], operands[1] = operands[2], operands[2] = tmp;
+       }
+
+      gcc_assert (rtx_equal_p (operands[0], operands[1]));
       if (x86_maybe_negate_const_int (&operands[2], <MODE>mode))
         return "sub{<imodesuffix>}\t{%2, %0|%0, %2}";
 
 (define_insn "*addsi_3_zext"
   [(set (reg FLAGS_REG)
        (compare
-         (neg:SI (match_operand:SI 2 "x86_64_general_operand" "rme"))
-         (match_operand:SI 1 "nonimmediate_operand" "%0")))
-   (set (match_operand:DI 0 "register_operand" "=r")
+         (neg:SI (match_operand:SI 2 "x86_64_general_operand" "rme,0"))
+         (match_operand:SI 1 "nonimmediate_operand" "%0,r")))
+   (set (match_operand:DI 0 "register_operand" "=r,r")
        (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))]
   "TARGET_64BIT && ix86_match_ccmode (insn, CCZmode)
    && ix86_binary_operator_ok (PLUS, SImode, operands)"
        }
 
     default:
+      if (which_alternative == 1)
+       {
+         rtx tmp;
+         tmp = operands[1], operands[1] = operands[2], operands[2] = tmp;
+       }
+
       if (x86_maybe_negate_const_int (&operands[2], SImode))
         return "sub{l}\t{%2, %k0|%k0, %2}";
 
   [(set (reg FLAGS_REG)
        (compare
          (plus:SWI
-           (match_operand:SWI 1 "nonimmediate_operand" "%0")
-           (match_operand:SWI 2 "<general_operand>" "<g>"))
+           (match_operand:SWI 1 "nonimmediate_operand" "%0,<r>")
+           (match_operand:SWI 2 "<general_operand>" "<g>,0"))
          (const_int 0)))
-   (clobber (match_scratch:SWI 0 "=<r>"))]
+   (clobber (match_scratch:SWI 0 "=<r>,<r>"))]
   "ix86_match_ccmode (insn, CCGOCmode)
    && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
 {
        }
 
     default:
+      if (which_alternative == 1)
+       {
+         rtx tmp;
+         tmp = operands[1], operands[1] = operands[2], operands[2] = tmp;
+       }
+
+      gcc_assert (rtx_equal_p (operands[0], operands[1]));
       if (x86_maybe_negate_const_int (&operands[2], <MODE>mode))
         return "sub{<imodesuffix>}\t{%2, %0|%0, %2}";
 
          [(and:SI (zero_extract:SI (match_dup 2) (const_int 8) (const_int 8))
                   (match_dup 3))
           (const_int 0)]))]
-  "operands[2] = gen_lowpart (SImode, operands[2]);
-   operands[3] = gen_int_mode (INTVAL (operands[3]) >> 8, SImode);")
+{
+  operands[2] = gen_lowpart (SImode, operands[2]);
+  operands[3] = gen_int_mode (INTVAL (operands[3]) >> 8, SImode);
+})
 
 (define_split
   [(set (match_operand 0 "flags_reg_operand" "")
   [(set (match_dup 0)
        (match_op_dup 1 [(and:QI (match_dup 2) (match_dup 3))
                         (const_int 0)]))]
-  "operands[2] = gen_lowpart (QImode, operands[2]);
-   operands[3] = gen_lowpart (QImode, operands[3]);")
+{
+  operands[2] = gen_lowpart (QImode, operands[2]);
+  operands[3] = gen_lowpart (QImode, operands[3]);
+})
 
 ;; %%% This used to optimize known byte-wide and operations to memory,
 ;; and sometimes to QImode registers.  If this is considered useful,
        (and:SWIM (match_operand:SWIM 1 "nonimmediate_operand" "")
                  (match_operand:SWIM 2 "<general_szext_operand>" "")))]
   ""
-  "ix86_expand_binary_operator (AND, <MODE>mode, operands); DONE;")
+{
+  if (<MODE>mode == DImode
+      && GET_CODE (operands[2]) == CONST_INT
+      && INTVAL (operands[2]) == (HOST_WIDE_INT) 0xffffffff
+      && REG_P (operands[1]))
+    emit_insn (gen_zero_extendsidi2 (operands[0],
+                                    gen_lowpart (SImode, operands[1])));
+  else
+    ix86_expand_binary_operator (AND, <MODE>mode, operands);
+  DONE;
+})
 
 (define_insn "*anddi_1"
   [(set (match_operand:DI 0 "nonimmediate_operand" "=r,rm,r,r")
        enum machine_mode mode;
 
        gcc_assert (CONST_INT_P (operands[2]));
-        if (INTVAL (operands[2]) == 0xff)
-         mode = QImode;
+       if (INTVAL (operands[2]) == (HOST_WIDE_INT) 0xffffffff)
+         mode = SImode;
+       else if (INTVAL (operands[2]) == 0xffff)
+         mode = HImode;
        else
          {
-           gcc_assert (INTVAL (operands[2]) == 0xffff);
-           mode = HImode;
+           gcc_assert (INTVAL (operands[2]) == 0xff);
+           mode = QImode;
          }
 
        operands[1] = gen_lowpart (mode, operands[1]);
-       if (mode == QImode)
-         return "movz{bl|x}\t{%1, %k0|%k0, %1}";
-       else
+       if (mode == SImode)
+         return "mov{l}\t{%1, %k0|%k0, %1}";
+       else if (mode == HImode)
          return "movz{wl|x}\t{%1, %k0|%k0, %1}";
+       else
+         return "movz{bl|x}\t{%1, %k0|%k0, %1}";
       }
 
     default:
        enum machine_mode mode;
 
        gcc_assert (CONST_INT_P (operands[2]));
-        if (INTVAL (operands[2]) == 0xff)
-         mode = QImode;
+        if (INTVAL (operands[2]) == 0xffff)
+         mode = HImode;
        else
          {
-           gcc_assert (INTVAL (operands[2]) == 0xffff);
-           mode = HImode;
+           gcc_assert (INTVAL (operands[2]) == 0xff);
+           mode = QImode;
          }
 
        operands[1] = gen_lowpart (mode, operands[1]);
-       if (mode == QImode)
-         return "movz{bl|x}\t{%1, %0|%0, %1}";
-       else
+       if (mode == HImode)
          return "movz{wl|x}\t{%1, %0|%0, %1}";
+       else
+         return "movz{bl|x}\t{%1, %0|%0, %1}";
       }
 
     default:
                                            (const_int 8) (const_int 8))
                           (match_dup 2)))
              (clobber (reg:CC FLAGS_REG))])]
-  "operands[0] = gen_lowpart (SImode, operands[0]);
-   operands[1] = gen_lowpart (SImode, operands[1]);
-   operands[2] = gen_int_mode ((INTVAL (operands[2]) >> 8) & 0xff, SImode);")
+{
+  operands[0] = gen_lowpart (SImode, operands[0]);
+  operands[1] = gen_lowpart (SImode, operands[1]);
+  operands[2] = gen_int_mode ((INTVAL (operands[2]) >> 8) & 0xff, SImode);
+})
 
 ;; Since AND can be encoded with sign extended immediate, this is only
 ;; profitable when 7th bit is not set.
                   (and:QI (match_dup 1)
                           (match_dup 2)))
              (clobber (reg:CC FLAGS_REG))])]
-  "operands[0] = gen_lowpart (QImode, operands[0]);
-   operands[1] = gen_lowpart (QImode, operands[1]);
-   operands[2] = gen_lowpart (QImode, operands[2]);")
+{
+  operands[0] = gen_lowpart (QImode, operands[0]);
+  operands[1] = gen_lowpart (QImode, operands[1]);
+  operands[2] = gen_lowpart (QImode, operands[2]);
+})
 \f
 ;; Logical inclusive and exclusive OR instructions
 
                                               (const_int 8) (const_int 8))
                              (match_dup 2)))
              (clobber (reg:CC FLAGS_REG))])]
-  "operands[0] = gen_lowpart (SImode, operands[0]);
-   operands[1] = gen_lowpart (SImode, operands[1]);
-   operands[2] = gen_int_mode ((INTVAL (operands[2]) >> 8) & 0xff, SImode);")
+{
+  operands[0] = gen_lowpart (SImode, operands[0]);
+  operands[1] = gen_lowpart (SImode, operands[1]);
+  operands[2] = gen_int_mode ((INTVAL (operands[2]) >> 8) & 0xff, SImode);
+})
 
 ;; Since OR can be encoded with sign extended immediate, this is only
 ;; profitable when 7th bit is set.
                   (any_or:QI (match_dup 1)
                              (match_dup 2)))
              (clobber (reg:CC FLAGS_REG))])]
-  "operands[0] = gen_lowpart (QImode, operands[0]);
-   operands[1] = gen_lowpart (QImode, operands[1]);
-   operands[2] = gen_lowpart (QImode, operands[2]);")
+{
+  operands[0] = gen_lowpart (QImode, operands[0]);
+  operands[1] = gen_lowpart (QImode, operands[1]);
+  operands[2] = gen_lowpart (QImode, operands[2]);
+})
 
 (define_expand "xorqi_cc_ext_1"
   [(parallel [
                            (match_dup 0)))
    (set (match_dup 1)
        (if_then_else:SWI48 (ne (reg:CCZ FLAGS_REG) (const_int 0))
-                           (match_operand:SWI48 3 "register_operand" "r")
+                           (match_operand:SWI48 3 "register_operand" "")
                            (match_dup 1)))]
   "TARGET_CMOVE"
   "operands[4] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode));")
   [(simple_return)]
   "ix86_can_use_return_insn_p ()"
 {
+  ix86_maybe_emit_epilogue_vzeroupper ();
   if (crtl->args.pops_args)
     {
       rtx popc = GEN_INT (crtl->args.pops_args);
   [(simple_return)]
   "!TARGET_SEH"
 {
+  ix86_maybe_emit_epilogue_vzeroupper ();
   if (crtl->args.pops_args)
     {
       rtx popc = GEN_INT (crtl->args.pops_args);
   "!TARGET_64BIT && TARGET_GNU_TLS"
 {
   output_asm_insn
-    ("lea{l}\t{%a2@tlsgd(,%1,1), %0|%0, %a2@tlsgd[%1*1]}", operands);
+    ("lea{l}\t{%E2@tlsgd(,%1,1), %0|%0, %E2@tlsgd[%1*1]}", operands);
   if (TARGET_SUN_TLS)
 #ifdef HAVE_AS_IX86_TLSGDPLT
     return "call\t%a2@tlsgdplt";
   if (!TARGET_X32)
     fputs (ASM_BYTE "0x66\n", asm_out_file);
   output_asm_insn
-    ("lea{q}\t{%a1@tlsgd(%%rip), %%rdi|rdi, %a1@tlsgd[rip]}", operands);
+    ("lea{q}\t{%E1@tlsgd(%%rip), %%rdi|rdi, %E1@tlsgd[rip]}", operands);
   fputs (ASM_SHORT "0x6666\n", asm_out_file);
   fputs ("\trex64\n", asm_out_file);
   if (TARGET_SUN_TLS)
                  (unspec:SI [(match_operand:SI 2 "tls_symbolic_operand" "")]
                              UNSPEC_TLSDESC))))]
   "!TARGET_64BIT && TARGET_GNU2_TLS"
-  "lea{l}\t{%a2@TLSDESC(%1), %0|%0, %a2@TLSDESC[%1]}"
+  "lea{l}\t{%E2@TLSDESC(%1), %0|%0, %E2@TLSDESC[%1]}"
   [(set_attr "type" "lea")
    (set_attr "mode" "SI")
    (set_attr "length" "6")
        (unspec:DI [(match_operand 1 "tls_symbolic_operand" "")]
                   UNSPEC_TLSDESC))]
   "TARGET_64BIT && TARGET_GNU2_TLS"
-  "lea{q}\t{%a1@TLSDESC(%%rip), %0|%0, %a1@TLSDESC[rip]}"
+  "lea{q}\t{%E1@TLSDESC(%%rip), %0|%0, %E1@TLSDESC[rip]}"
   [(set_attr "type" "lea")
    (set_attr "mode" "DI")
    (set_attr "length" "7")
       else if (optimize_insn_for_size_p ())
         FAIL;
       else
-       ix86_expand_rint (operand0, operand1);
+       ix86_expand_rint (operands[0], operands[1]);
     }
   else
     {
       && <SWI248x:MODE>mode != HImode
       && ((<SWI248x:MODE>mode != DImode) || TARGET_64BIT)
       && !flag_trapping_math && !flag_rounding_math)
-    ix86_expand_lround (operand0, operand1);
+    ix86_expand_lround (operands[0], operands[1]);
   else
     ix86_emit_i387_round (operands[0], operands[1]);
   DONE;
       else if (optimize_insn_for_size_p ())
         FAIL;
       else if (TARGET_64BIT || (<MODE>mode != DFmode))
-       ix86_expand_floorceil (operand0, operand1, true);
+       ix86_expand_floorceil (operands[0], operands[1], true);
       else
-       ix86_expand_floorceildf_32 (operand0, operand1, true);
+       ix86_expand_floorceildf_32 (operands[0], operands[1], true);
     }
   else
     {
 {
   if (TARGET_64BIT && optimize_insn_for_size_p ())
     FAIL;
-  ix86_expand_lfloorceil (operand0, operand1, true);
+  ix86_expand_lfloorceil (operands[0], operands[1], true);
   DONE;
 })
 
       else if (optimize_insn_for_size_p ())
        FAIL;
       else if (TARGET_64BIT || (<MODE>mode != DFmode))
-       ix86_expand_floorceil (operand0, operand1, false);
+       ix86_expand_floorceil (operands[0], operands[1], false);
       else
-       ix86_expand_floorceildf_32 (operand0, operand1, false);
+       ix86_expand_floorceildf_32 (operands[0], operands[1], false);
     }
   else
     {
   "SSE_FLOAT_MODE_P (<MODEF:MODE>mode) && TARGET_SSE_MATH
    && !flag_trapping_math"
 {
-  ix86_expand_lfloorceil (operand0, operand1, false);
+  ix86_expand_lfloorceil (operands[0], operands[1], false);
   DONE;
 })
 
       else if (optimize_insn_for_size_p ())
        FAIL;
       else if (TARGET_64BIT || (<MODE>mode != DFmode))
-       ix86_expand_trunc (operand0, operand1);
+       ix86_expand_trunc (operands[0], operands[1]);
       else
-       ix86_expand_truncdf_32 (operand0, operand1);
+       ix86_expand_truncdf_32 (operands[0], operands[1]);
     }
   else
     {
 (define_insn "*x86_mov<mode>cc_0_m1_neg"
   [(set (match_operand:SWI48 0 "register_operand" "=r")
        (neg:SWI48 (match_operator 1 "ix86_carry_flag_operator"
-                   [(reg FLAGS_REG) (const_int 0)])))]
+                   [(reg FLAGS_REG) (const_int 0)])))
+   (clobber (reg:CC FLAGS_REG))]
   ""
   "sbb{<imodesuffix>}\t%0, %0"
   [(set_attr "type" "alu")
   [(set_attr "type" "icmov")
    (set_attr "mode" "<MODE>")])
 
-(define_insn_and_split "*movqicc_noc"
+(define_insn "*movqicc_noc"
   [(set (match_operand:QI 0 "register_operand" "=r,r")
        (if_then_else:QI (match_operator 1 "ix86_comparison_operator"
-                          [(match_operand 4 "flags_reg_operand" "")
-                           (const_int 0)])
+                          [(reg FLAGS_REG) (const_int 0)])
                      (match_operand:QI 2 "register_operand" "r,0")
                      (match_operand:QI 3 "register_operand" "0,r")))]
   "TARGET_CMOVE && !TARGET_PARTIAL_REG_STALL"
   "#"
-  "&& reload_completed"
-  [(set (match_dup 0)
-       (if_then_else:SI (match_op_dup 1 [(match_dup 4) (const_int 0)])
-                     (match_dup 2)
-                     (match_dup 3)))]
-  "operands[0] = gen_lowpart (SImode, operands[0]);
-   operands[2] = gen_lowpart (SImode, operands[2]);
-   operands[3] = gen_lowpart (SImode, operands[3]);"
   [(set_attr "type" "icmov")
-   (set_attr "mode" "SI")])
+   (set_attr "mode" "QI")])
+
+(define_split
+  [(set (match_operand 0 "register_operand")
+       (if_then_else (match_operator 1 "ix86_comparison_operator"
+                       [(reg FLAGS_REG) (const_int 0)])
+                     (match_operand 2 "register_operand")
+                     (match_operand 3 "register_operand")))]
+  "TARGET_CMOVE && !TARGET_PARTIAL_REG_STALL
+   && (GET_MODE (operands[0]) == QImode
+       || GET_MODE (operands[0]) == HImode)
+   && reload_completed"
+  [(set (match_dup 0)
+       (if_then_else:SI (match_dup 1) (match_dup 2) (match_dup 3)))]
+{
+  operands[0] = gen_lowpart (SImode, operands[0]);
+  operands[2] = gen_lowpart (SImode, operands[2]);
+  operands[3] = gen_lowpart (SImode, operands[3]);
+})
 
 (define_expand "mov<mode>cc"
   [(set (match_operand:X87MODEF 0 "register_operand" "")
    (set_attr "mode" "DF,DF,DI,DI")])
 
 (define_split
-  [(set (match_operand:DF 0 "register_and_not_any_fp_reg_operand" "")
+  [(set (match_operand:DF 0 "register_and_not_any_fp_reg_operand")
        (if_then_else:DF (match_operator 1 "fcmov_comparison_operator"
-                               [(match_operand 4 "flags_reg_operand" "")
-                                (const_int 0)])
-                     (match_operand:DF 2 "nonimmediate_operand" "")
-                     (match_operand:DF 3 "nonimmediate_operand" "")))]
+                               [(reg FLAGS_REG) (const_int 0)])
+                     (match_operand:DF 2 "nonimmediate_operand")
+                     (match_operand:DF 3 "nonimmediate_operand")))]
   "!TARGET_64BIT && reload_completed"
   [(set (match_dup 2)
-       (if_then_else:SI (match_op_dup 1 [(match_dup 4) (const_int 0)])
-                     (match_dup 5)
-                     (match_dup 6)))
+       (if_then_else:SI (match_dup 1) (match_dup 4) (match_dup 5)))
    (set (match_dup 3)
-       (if_then_else:SI (match_op_dup 1 [(match_dup 4) (const_int 0)])
-                     (match_dup 7)
-                     (match_dup 8)))]
+       (if_then_else:SI (match_dup 1) (match_dup 6) (match_dup 7)))]
 {
-  split_double_mode (DImode, &operands[2], 2, &operands[5], &operands[7]);
+  split_double_mode (DImode, &operands[2], 2, &operands[4], &operands[6]);
   split_double_mode (DImode, &operands[0], 1, &operands[2], &operands[3]);
 })
 
 
   ;; The % modifier is not operational anymore in peephole2's, so we have to
   ;; swap the operands manually in the case of addition and multiplication.
-  "if (COMMUTATIVE_ARITH_P (operands[2]))
-     operands[4] = gen_rtx_fmt_ee (GET_CODE (operands[2]),
-                                  GET_MODE (operands[2]),
-                                  operands[0], operands[1]);
-   else
-     operands[4] = gen_rtx_fmt_ee (GET_CODE (operands[2]),
-                                  GET_MODE (operands[2]),
-                                  operands[1], operands[0]);")
+{
+  rtx op0, op1;
+
+  if (COMMUTATIVE_ARITH_P (operands[2]))
+    op0 = operands[0], op1 = operands[1];
+  else
+    op0 = operands[1], op1 = operands[0];
+
+  operands[4] = gen_rtx_fmt_ee (GET_CODE (operands[2]),
+                               GET_MODE (operands[2]),
+                               op0, op1);
+})
 
 ;; Conditional addition patterns
 (define_expand "add<mode>cc"
 
     default:
       operands[2] = SET_SRC (XVECEXP (PATTERN (insn), 0, 0));
-      return "lea{<imodesuffix>}\t{%a2, %0|%0, %a2}";
+      return "lea{<imodesuffix>}\t{%E2, %0|%0, %E2}";
     }
 }
   [(set (attr "type")
   [(parallel [(set (match_dup 0)
                   (match_op_dup 3 [(match_dup 1) (match_dup 2)]))
              (clobber (reg:CC FLAGS_REG))])]
-  "operands[0] = gen_lowpart (SImode, operands[0]);
-   operands[1] = gen_lowpart (SImode, operands[1]);
-   if (GET_CODE (operands[3]) != ASHIFT)
-     operands[2] = gen_lowpart (SImode, operands[2]);
-   PUT_MODE (operands[3], SImode);")
+{
+  operands[0] = gen_lowpart (SImode, operands[0]);
+  operands[1] = gen_lowpart (SImode, operands[1]);
+  if (GET_CODE (operands[3]) != ASHIFT)
+    operands[2] = gen_lowpart (SImode, operands[2]);
+  PUT_MODE (operands[3], SImode);
+})
 
 ; Promote the QImode tests, as i386 has encoding of the AND
 ; instruction with 32-bit sign-extended immediate and thus the
   [(parallel [(set (match_dup 0)
                   (neg:SI (match_dup 1)))
              (clobber (reg:CC FLAGS_REG))])]
-  "operands[0] = gen_lowpart (SImode, operands[0]);
-   operands[1] = gen_lowpart (SImode, operands[1]);")
+{
+  operands[0] = gen_lowpart (SImode, operands[0]);
+  operands[1] = gen_lowpart (SImode, operands[1]);
+})
 
 (define_split
   [(set (match_operand 0 "register_operand" "")
               || optimize_insn_for_size_p ())))"
   [(set (match_dup 0)
        (not:SI (match_dup 1)))]
-  "operands[0] = gen_lowpart (SImode, operands[0]);
-   operands[1] = gen_lowpart (SImode, operands[1]);")
-
-(define_split
-  [(set (match_operand 0 "register_operand" "")
-       (if_then_else (match_operator 1 "ordered_comparison_operator"
-                               [(reg FLAGS_REG) (const_int 0)])
-                     (match_operand 2 "register_operand" "")
-                     (match_operand 3 "register_operand" "")))]
-  "! TARGET_PARTIAL_REG_STALL && TARGET_CMOVE
-   && (GET_MODE (operands[0]) == HImode
-       || (GET_MODE (operands[0]) == QImode
-          && (TARGET_PROMOTE_QImode
-              || optimize_insn_for_size_p ())))"
-  [(set (match_dup 0)
-       (if_then_else:SI (match_dup 1) (match_dup 2) (match_dup 3)))]
-  "operands[0] = gen_lowpart (SImode, operands[0]);
-   operands[2] = gen_lowpart (SImode, operands[2]);
-   operands[3] = gen_lowpart (SImode, operands[3]);")
+{
+  operands[0] = gen_lowpart (SImode, operands[0]);
+  operands[1] = gen_lowpart (SImode, operands[1]);
+})
 \f
 ;; RTL Peephole optimizations, run before sched2.  These primarily look to
 ;; transform a complex memory operation into two memory to register operations.
   "(TARGET_READ_MODIFY_WRITE || optimize_insn_for_size_p ())
    && peep2_reg_dead_p (4, operands[0])
    && !reg_overlap_mentioned_p (operands[0], operands[1])
+   && (<MODE>mode != QImode
+       || immediate_operand (operands[2], QImode)
+       || q_regs_operand (operands[2], QImode))
    && ix86_match_ccmode (peep2_next_insn (3),
                         (GET_CODE (operands[3]) == PLUS
                          || GET_CODE (operands[3]) == MINUS)
   [(parallel [(set (match_dup 4) (match_dup 5))
              (set (match_dup 1) (match_op_dup 3 [(match_dup 1)
                                                  (match_dup 2)]))])]
-  "operands[4] = SET_DEST (PATTERN (peep2_next_insn (3)));
-   operands[5] = gen_rtx_fmt_ee (GET_CODE (operands[3]), <MODE>mode,
-                                copy_rtx (operands[1]),
-                                copy_rtx (operands[2]));
-   operands[5] = gen_rtx_COMPARE (GET_MODE (operands[4]),
-                                 operands[5], const0_rtx);")
+{
+  operands[4] = SET_DEST (PATTERN (peep2_next_insn (3)));
+  operands[5] = gen_rtx_fmt_ee (GET_CODE (operands[3]), <MODE>mode,
+                               copy_rtx (operands[1]),
+                               copy_rtx (operands[2]));
+  operands[5] = gen_rtx_COMPARE (GET_MODE (operands[4]),
+                                operands[5], const0_rtx);
+})
 
 (define_peephole2
   [(parallel [(set (match_operand:SWI 0 "register_operand" "")
   [(parallel [(set (match_dup 3) (match_dup 4))
              (set (match_dup 1) (match_op_dup 2 [(match_dup 1)
                                                  (match_dup 0)]))])]
-  "operands[3] = SET_DEST (PATTERN (peep2_next_insn (2)));
-   operands[4] = gen_rtx_fmt_ee (GET_CODE (operands[2]), <MODE>mode,
-                                copy_rtx (operands[1]),
-                                copy_rtx (operands[0]));
-   operands[4] = gen_rtx_COMPARE (GET_MODE (operands[3]),
-                                 operands[4], const0_rtx);")
+{
+  operands[3] = SET_DEST (PATTERN (peep2_next_insn (2)));
+  operands[4] = gen_rtx_fmt_ee (GET_CODE (operands[2]), <MODE>mode,
+                               copy_rtx (operands[1]),
+                               copy_rtx (operands[0]));
+  operands[4] = gen_rtx_COMPARE (GET_MODE (operands[3]),
+                                operands[4], const0_rtx);
+})
 
 (define_peephole2
   [(set (match_operand:SWI12 0 "register_operand" "")
    && REG_P (operands[0]) && REG_P (operands[4])
    && REGNO (operands[0]) == REGNO (operands[4])
    && peep2_reg_dead_p (4, operands[0])
+   && (<MODE>mode != QImode
+       || immediate_operand (operands[2], SImode)
+       || q_regs_operand (operands[2], SImode))
    && !reg_overlap_mentioned_p (operands[0], operands[1])
    && ix86_match_ccmode (peep2_next_insn (3),
                         (GET_CODE (operands[3]) == PLUS
                         ? CCGOCmode : CCNOmode)"
   [(parallel [(set (match_dup 4) (match_dup 5))
              (set (match_dup 1) (match_dup 6))])]
-  "operands[2] = gen_lowpart (<MODE>mode, operands[2]);
-   operands[4] = SET_DEST (PATTERN (peep2_next_insn (3)));
-   operands[5] = gen_rtx_fmt_ee (GET_CODE (operands[3]), <MODE>mode,
-                                copy_rtx (operands[1]), operands[2]);
-   operands[5] = gen_rtx_COMPARE (GET_MODE (operands[4]),
-                                 operands[5], const0_rtx);
-   operands[6] = gen_rtx_fmt_ee (GET_CODE (operands[3]), <MODE>mode,
-                                copy_rtx (operands[1]),
-                                copy_rtx (operands[2]));")
+{
+  operands[2] = gen_lowpart (<MODE>mode, operands[2]);
+  operands[4] = SET_DEST (PATTERN (peep2_next_insn (3)));
+  operands[5] = gen_rtx_fmt_ee (GET_CODE (operands[3]), <MODE>mode,
+                               copy_rtx (operands[1]), operands[2]);
+  operands[5] = gen_rtx_COMPARE (GET_MODE (operands[4]),
+                                operands[5], const0_rtx);
+  operands[6] = gen_rtx_fmt_ee (GET_CODE (operands[3]), <MODE>mode,
+                               copy_rtx (operands[1]),
+                               copy_rtx (operands[2]));
+})
 
 ;; Attempt to always use XOR for zeroing registers.
 (define_peephole2
   int locality = INTVAL (operands[2]);
 
   gcc_assert (rw == 0 || rw == 1);
-  gcc_assert (locality >= 0 && locality <= 3);
-  gcc_assert (GET_MODE (operands[0]) == Pmode
-             || GET_MODE (operands[0]) == VOIDmode);
+  gcc_assert (IN_RANGE (locality, 0, 3));
 
+  if (TARGET_PREFETCHW && rw)
+    operands[2] = GEN_INT (3);
   /* Use 3dNOW prefetch in case we are asking for write prefetch not
      supported by SSE counterpart or the SSE prefetch is not available
      (K6 machines).  Otherwise use SSE prefetch as it allows specifying
      of locality.  */
-  if (TARGET_3DNOW && (!TARGET_PREFETCH_SSE || rw))
+  else if (TARGET_3DNOW && (!TARGET_PREFETCH_SSE || rw))
     operands[2] = GEN_INT (3);
   else
     operands[1] = const0_rtx;
 })
 
-(define_insn "*prefetch_sse_<mode>"
-  [(prefetch (match_operand:P 0 "address_operand" "p")
+(define_insn "*prefetch_sse"
+  [(prefetch (match_operand 0 "address_operand" "p")
             (const_int 0)
             (match_operand:SI 1 "const_int_operand" ""))]
   "TARGET_PREFETCH_SSE"
   };
 
   int locality = INTVAL (operands[1]);
-  gcc_assert (locality >= 0 && locality <= 3);
+  gcc_assert (IN_RANGE (locality, 0, 3));
 
   return patterns[locality];
 }
   [(set_attr "type" "sse")
    (set_attr "atom_sse_attr" "prefetch")
    (set (attr "length_address")
-       (symbol_ref "memory_address_length (operands[0])"))
+       (symbol_ref "memory_address_length (operands[0], false)"))
    (set_attr "memory" "none")])
 
-(define_insn "*prefetch_3dnow_<mode>"
-  [(prefetch (match_operand:P 0 "address_operand" "p")
+(define_insn "*prefetch_3dnow"
+  [(prefetch (match_operand 0 "address_operand" "p")
             (match_operand:SI 1 "const_int_operand" "n")
             (const_int 3))]
-  "TARGET_3DNOW"
+  "TARGET_3DNOW || TARGET_PREFETCHW"
 {
   if (INTVAL (operands[1]) == 0)
     return "prefetch\t%a0";
 }
   [(set_attr "type" "mmx")
    (set (attr "length_address")
-       (symbol_ref "memory_address_length (operands[0])"))
+       (symbol_ref "memory_address_length (operands[0], false)"))
    (set_attr "memory" "none")])
 
 (define_expand "stack_protect_set"
   [(match_operand 0 "memory_operand" "")
    (match_operand 1 "memory_operand" "")]
-  ""
+  "!TARGET_HAS_BIONIC"
 {
   rtx (*insn)(rtx, rtx);
 
                    UNSPEC_SP_SET))
    (set (match_scratch:PTR 2 "=&r") (const_int 0))
    (clobber (reg:CC FLAGS_REG))]
-  ""
+  "!TARGET_HAS_BIONIC"
   "mov{<imodesuffix>}\t{%1, %2|%2, %1}\;mov{<imodesuffix>}\t{%2, %0|%0, %2}\;xor{l}\t%k2, %k2"
   [(set_attr "type" "multi")])
 
   [(match_operand 0 "memory_operand" "")
    (match_operand 1 "memory_operand" "")
    (match_operand 2 "" "")]
-  ""
+  "!TARGET_HAS_BIONIC"
 {
   rtx flags = gen_rtx_REG (CCZmode, FLAGS_REG);
 
                     (match_operand:PTR 2 "memory_operand" "m")]
                    UNSPEC_SP_TEST))
    (clobber (match_scratch:PTR 3 "=&r"))]
-  ""
+  "!TARGET_HAS_BIONIC"
   "mov{<imodesuffix>}\t{%1, %3|%3, %1}\;xor{<imodesuffix>}\t{%2, %3|%3, %2}"
   [(set_attr "type" "multi")])
 
                     (match_operand:SI 3 "const_int_operand" "i")]
                    UNSPECV_LWPVAL_INTRINSIC)]
   "TARGET_LWP"
-  "/* Avoid unused variable warning.  */
-   (void) operand0;")
+  ;; Avoid unused variable warning.
+  "(void) operands[0];")
 
 (define_insn "*lwp_lwpval<mode>3_1"
   [(unspec_volatile [(match_operand:SWI48 0 "register_operand" "r")
 
 (define_insn "rdrand<mode>_1"
   [(set (match_operand:SWI248 0 "register_operand" "=r")
-       (unspec:SWI248 [(const_int 0)] UNSPEC_RDRAND))
+       (unspec_volatile:SWI248 [(const_int 0)] UNSPECV_RDRAND))
    (set (reg:CCC FLAGS_REG)
-       (unspec:CCC [(const_int 0)] UNSPEC_RDRAND))]
+       (unspec_volatile:CCC [(const_int 0)] UNSPECV_RDRAND))]
   "TARGET_RDRND"
   "rdrand\t%0"
   [(set_attr "type" "other")