OSDN Git Service

* config/i386/i386.c (override_options): Don't accept
[pf3gnuchains/gcc-fork.git] / gcc / config / i386 / i386.md
index c011d9b..da28f1c 100644 (file)
@@ -1,6 +1,6 @@
 ;; GCC machine description for IA-32 and x86-64.
 ;; Copyright (C) 1988, 1994, 1995, 1996, 1997, 1998, 1999, 2000,
-;; 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009
+;; 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
 ;; Free Software Foundation, Inc.
 ;; Mostly by William Schelter.
 ;; x86_64 support added by Jan Hubicka
    (UNSPEC_XOP_TRUEFALSE       152)
    (UNSPEC_XOP_PERMUTE         153)
    (UNSPEC_FRCZ                        154)
-   (UNSPEC_LLWP_INTRINSIC      155)
-   (UNSPEC_SLWP_INTRINSIC      156)
-   (UNSPECV_LWPVAL_INTRINSIC   157)
-   (UNSPECV_LWPINS_INTRINSIC   158)
 
    ; For AES support
    (UNSPEC_AESENC              159)
    ; For AVX support
    (UNSPEC_PCMP                        166)
    (UNSPEC_VPERMIL             167)
-   (UNSPEC_VPERMIL2F128                168)
-   (UNSPEC_MASKLOAD            169)
-   (UNSPEC_MASKSTORE           170)
-   (UNSPEC_CAST                        171)
-   (UNSPEC_VTESTP              172)
+   (UNSPEC_VPERMIL2            168)
+   (UNSPEC_VPERMIL2F128                169)
+   (UNSPEC_MASKLOAD            170)
+   (UNSPEC_MASKSTORE           171)
+   (UNSPEC_CAST                        172)
+   (UNSPEC_VTESTP              173)
   ])
 
 (define_constants
    (UNSPECV_RDTSC              18)
    (UNSPECV_RDTSCP             19)
    (UNSPECV_RDPMC              20)
-   (UNSPECV_VSWAPMOV   21)
+   (UNSPECV_VSWAPMOV           21)
+   (UNSPECV_LLWP_INTRINSIC     22)
+   (UNSPECV_SLWP_INTRINSIC     23)
+   (UNSPECV_LWPVAL_INTRINSIC   24)
+   (UNSPECV_LWPINS_INTRINSIC   25)
   ])
 
 ;; Constants to represent pcomtrue/pcomfalse variants
 ;; if the instruction is complex.
 
 (define_attr "memory" "none,load,store,both,unknown"
-  (cond [(eq_attr "type" "other,multi,str")
+  (cond [(eq_attr "type" "other,multi,str,lwp")
           (const_string "unknown")
         (eq_attr "type" "lea,fcmov,fpspc")
           (const_string "none")
    (set_attr "type" "multi")])
 
 ;; All integer comparison codes.
-(define_code_iterator int_cond [ne eq ge gt le lt geu gtu leu ltu ])
+(define_code_iterator int_cond [ne eq ge gt le lt geu gtu leu ltu])
 
 ;; All floating-point comparison codes.
 (define_code_iterator fp_cond [unordered ordered
-                              uneq unge ungt unle unlt ltgt ])
+                              uneq unge ungt unle unlt ltgt])
 
 (define_code_iterator plusminus [plus minus])
 
 (define_code_attr plusminus_mnemonic
   [(plus "add") (ss_plus "adds") (us_plus "addus")
    (minus "sub") (ss_minus "subs") (us_minus "subus")])
+(define_code_attr plusminus_carry_mnemonic
+  [(plus "adc") (minus "sbb")])
 
 ;; Mark commutative operators as such in constraints.
 (define_code_attr comm [(plus "%") (ss_plus "%") (us_plus "%")
                                 (umax "maxu") (umin "minu")])
 (define_code_attr maxminfprefix [(smax "max") (smin "min")])
 
-;; Mapping of parallel logic operators
-(define_code_iterator plogic [and ior xor])
+;; Mapping of logic operators
+(define_code_iterator any_logic [and ior xor])
+(define_code_iterator any_or [ior xor])
 
 ;; Base name for insn mnemonic.
-(define_code_attr plogicprefix [(and "and") (ior "or") (xor "xor")])
+(define_code_attr logicprefix [(and "and") (ior "or") (xor "xor")])
 
 ;; Mapping of abs neg operators
 (define_code_iterator absneg [abs neg])
 ;; Used in signed and unsigned widening multiplications.
 (define_code_iterator any_extend [sign_extend zero_extend])
 
-;; Used in signed and unsigned divisions.
-(define_code_iterator any_div [div udiv])
-
 ;; Various insn prefixes for signed and unsigned operations.
 (define_code_attr u [(sign_extend "") (zero_extend "u")
                     (div "") (udiv "u")])
 (define_code_attr s [(sign_extend "s") (zero_extend "u")])
 
+;; Used in signed and unsigned divisions.
+(define_code_iterator any_div [div udiv])
+
 ;; Instruction prefix for signed and unsigned operations.
 (define_code_attr sgnprefix [(sign_extend "i") (zero_extend "")
                             (div "i") (udiv "")])
 ;; All single word integer modes.
 (define_mode_iterator SWI [QI HI SI (DI "TARGET_64BIT")])
 
+;; Single word integer modes without DImode.
+(define_mode_iterator SWI124 [QI HI SI])
+
 ;; Single word integer modes without QImode.
 (define_mode_iterator SWI248 [HI SI (DI "TARGET_64BIT")])
 
 (define_mode_attr i [(QI "n") (HI "n") (SI "i") (DI "e")])
 
 ;; General operand constraint for word modes.
-(define_mode_attr g [(SI "g") (DI "rme")])
+(define_mode_attr g [(QI "qmn") (HI "rmn") (SI "g") (DI "rme")])
 
 ;; Immediate operand constraint for double integer modes.
 (define_mode_attr di [(SI "iF") (DI "e")])
         (DI "x86_64_general_operand")
         (TI "x86_64_general_operand")])
 
+;; General sign/zero extend operand predicate for integer modes.
+(define_mode_attr general_szext_operand
+       [(QI "general_operand")
+        (HI "general_operand")
+        (SI "general_operand")
+        (DI "x86_64_szext_general_operand")])
+
 ;; SSE and x87 SFmode and DFmode floating point modes
 (define_mode_iterator MODEF [SF DF])
 
 \f
 ;; Compare and branch/compare and store instructions.
 
-(define_expand "cbranchti4"
-  [(set (reg:CC FLAGS_REG)
-       (compare:CC (match_operand:TI 1 "nonimmediate_operand" "")
-                   (match_operand:TI 2 "x86_64_general_operand" "")))
-   (set (pc) (if_then_else
-              (match_operator 0 "comparison_operator"
-               [(reg:CC FLAGS_REG)
-                (const_int 0)])
-              (label_ref (match_operand 3 "" ""))
-              (pc)))]
-  "TARGET_64BIT"
-{
-  if (MEM_P (operands[1]) && MEM_P (operands[2]))
-    operands[1] = force_reg (TImode, operands[1]);
-  ix86_compare_op0 = operands[1];
-  ix86_compare_op1 = operands[2];
-  ix86_expand_branch (GET_CODE (operands[0]), operands[3]);
-  DONE;
-})
-
-(define_expand "cbranchdi4"
-  [(set (reg:CC FLAGS_REG)
-       (compare:CC (match_operand:DI 1 "nonimmediate_operand" "")
-                   (match_operand:DI 2 "x86_64_general_operand" "")))
-   (set (pc) (if_then_else
-              (match_operator 0 "comparison_operator"
-               [(reg:CC FLAGS_REG)
-                (const_int 0)])
-              (label_ref (match_operand 3 "" ""))
-              (pc)))]
-  ""
-{
-  if (MEM_P (operands[1]) && MEM_P (operands[2]))
-    operands[1] = force_reg (DImode, operands[1]);
-  ix86_compare_op0 = operands[1];
-  ix86_compare_op1 = operands[2];
-  ix86_expand_branch (GET_CODE (operands[0]), operands[3]);
-  DONE;
-})
-
-(define_expand "cstoredi4"
-  [(set (reg:CC FLAGS_REG)
-       (compare:CC (match_operand:DI 2 "nonimmediate_operand" "")
-                   (match_operand:DI 3 "x86_64_general_operand" "")))
-   (set (match_operand:QI 0 "register_operand" "")
-              (match_operator 1 "comparison_operator"
-               [(reg:CC FLAGS_REG)
-                (const_int 0)]))]
-  "TARGET_64BIT"
-{
-  if (MEM_P (operands[2]) && MEM_P (operands[3]))
-    operands[2] = force_reg (DImode, operands[2]);
-  ix86_compare_op0 = operands[2];
-  ix86_compare_op1 = operands[3];
-  ix86_expand_setcc (GET_CODE (operands[1]), operands[0]);
-  DONE;
-})
-
-(define_expand "cbranchsi4"
-  [(set (reg:CC FLAGS_REG)
-       (compare:CC (match_operand:SI 1 "cmpsi_operand" "")
-                   (match_operand:SI 2 "general_operand" "")))
-   (set (pc) (if_then_else
-              (match_operator 0 "comparison_operator"
-               [(reg:CC FLAGS_REG)
-                (const_int 0)])
-              (label_ref (match_operand 3 "" ""))
-              (pc)))]
-  ""
-{
-  if (MEM_P (operands[1]) && MEM_P (operands[2]))
-    operands[1] = force_reg (SImode, operands[1]);
-  ix86_compare_op0 = operands[1];
-  ix86_compare_op1 = operands[2];
-  ix86_expand_branch (GET_CODE (operands[0]), operands[3]);
-  DONE;
-})
-
-(define_expand "cstoresi4"
-  [(set (reg:CC FLAGS_REG)
-       (compare:CC (match_operand:SI 2 "cmpsi_operand" "")
-                   (match_operand:SI 3 "general_operand" "")))
-   (set (match_operand:QI 0 "register_operand" "")
-              (match_operator 1 "comparison_operator"
-               [(reg:CC FLAGS_REG)
-                (const_int 0)]))]
-  ""
-{
-  if (MEM_P (operands[2]) && MEM_P (operands[3]))
-    operands[2] = force_reg (SImode, operands[2]);
-  ix86_compare_op0 = operands[2];
-  ix86_compare_op1 = operands[3];
-  ix86_expand_setcc (GET_CODE (operands[1]), operands[0]);
-  DONE;
-})
-
-(define_expand "cbranchhi4"
-  [(set (reg:CC FLAGS_REG)
-       (compare:CC (match_operand:HI 1 "nonimmediate_operand" "")
-                   (match_operand:HI 2 "general_operand" "")))
-   (set (pc) (if_then_else
-              (match_operator 0 "comparison_operator"
-               [(reg:CC FLAGS_REG)
-                (const_int 0)])
-              (label_ref (match_operand 3 "" ""))
-              (pc)))]
-  ""
-{
-  if (MEM_P (operands[1]) && MEM_P (operands[2]))
-    operands[1] = force_reg (HImode, operands[1]);
-  ix86_compare_op0 = operands[1];
-  ix86_compare_op1 = operands[2];
-  ix86_expand_branch (GET_CODE (operands[0]), operands[3]);
-  DONE;
-})
-
-(define_expand "cstorehi4"
-  [(set (reg:CC FLAGS_REG)
-       (compare:CC (match_operand:HI 2 "nonimmediate_operand" "")
-                   (match_operand:HI 3 "general_operand" "")))
-   (set (match_operand:QI 0 "register_operand" "")
-              (match_operator 1 "comparison_operator"
-               [(reg:CC FLAGS_REG)
-                (const_int 0)]))]
-  ""
-{
-  if (MEM_P (operands[2]) && MEM_P (operands[3]))
-    operands[2] = force_reg (HImode, operands[2]);
-  ix86_compare_op0 = operands[2];
-  ix86_compare_op1 = operands[3];
-  ix86_expand_setcc (GET_CODE (operands[1]), operands[0]);
-  DONE;
-})
-
-
-(define_expand "cbranchqi4"
+(define_expand "cbranch<mode>4"
   [(set (reg:CC FLAGS_REG)
-       (compare:CC (match_operand:QI 1 "nonimmediate_operand" "")
-                   (match_operand:QI 2 "general_operand" "")))
+       (compare:CC (match_operand:SDWIM 1 "nonimmediate_operand" "")
+                   (match_operand:SDWIM 2 "<general_operand>" "")))
    (set (pc) (if_then_else
-              (match_operator 0 "comparison_operator"
-               [(reg:CC FLAGS_REG)
-                (const_int 0)])
-              (label_ref (match_operand 3 "" ""))
-              (pc)))]
+              (match_operator 0 "comparison_operator"
+               [(reg:CC FLAGS_REG) (const_int 0)])
+              (label_ref (match_operand 3 "" ""))
+              (pc)))]
   ""
 {
   if (MEM_P (operands[1]) && MEM_P (operands[2]))
-    operands[1] = force_reg (QImode, operands[1]);
+    operands[1] = force_reg (<MODE>mode, operands[1]);
   ix86_compare_op0 = operands[1];
   ix86_compare_op1 = operands[2];
   ix86_expand_branch (GET_CODE (operands[0]), operands[3]);
   DONE;
 })
 
-
-(define_expand "cstoreqi4"
+(define_expand "cstore<mode>4"
   [(set (reg:CC FLAGS_REG)
-       (compare:CC (match_operand:QI 2 "nonimmediate_operand" "")
-                   (match_operand:QI 3 "general_operand" "")))
+       (compare:CC (match_operand:SWIM 2 "nonimmediate_operand" "")
+                   (match_operand:SWIM 3 "<general_operand>" "")))
    (set (match_operand:QI 0 "register_operand" "")
-              (match_operator 1 "comparison_operator"
-               [(reg:CC FLAGS_REG)
-                (const_int 0)]))]
+       (match_operator 1 "comparison_operator"
+         [(reg:CC FLAGS_REG) (const_int 0)]))]
   ""
 {
   if (MEM_P (operands[2]) && MEM_P (operands[3]))
-    operands[2] = force_reg (QImode, operands[2]);
+    operands[2] = force_reg (<MODE>mode, operands[2]);
   ix86_compare_op0 = operands[2];
   ix86_compare_op1 = operands[3];
   ix86_expand_setcc (GET_CODE (operands[1]), operands[0]);
   DONE;
 })
 
-
-(define_insn "cmpdi_ccno_1_rex64"
-  [(set (reg FLAGS_REG)
-       (compare (match_operand:DI 0 "nonimmediate_operand" "r,?mr")
-                (match_operand:DI 1 "const0_operand" "")))]
-  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)"
-  "@
-   test{q}\t%0, %0
-   cmp{q}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "test,icmp")
-   (set_attr "length_immediate" "0,1")
-   (set_attr "mode" "DI")])
-
-(define_insn "*cmpdi_minus_1_rex64"
-  [(set (reg FLAGS_REG)
-       (compare (minus:DI (match_operand:DI 0 "nonimmediate_operand" "rm,r")
-                          (match_operand:DI 1 "x86_64_general_operand" "re,mr"))
-                (const_int 0)))]
-  "TARGET_64BIT && ix86_match_ccmode (insn, CCGOCmode)"
-  "cmp{q}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "icmp")
-   (set_attr "mode" "DI")])
-
-(define_expand "cmpdi_1_rex64"
-  [(set (reg:CC FLAGS_REG)
-       (compare:CC (match_operand:DI 0 "nonimmediate_operand" "")
-                   (match_operand:DI 1 "general_operand" "")))]
-  "TARGET_64BIT"
-  "")
-
-(define_insn "cmpdi_1_insn_rex64"
-  [(set (reg FLAGS_REG)
-       (compare (match_operand:DI 0 "nonimmediate_operand" "mr,r")
-                (match_operand:DI 1 "x86_64_general_operand" "re,mr")))]
-  "TARGET_64BIT && ix86_match_ccmode (insn, CCmode)"
-  "cmp{q}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "icmp")
-   (set_attr "mode" "DI")])
-
-
-(define_insn "*cmpsi_ccno_1"
-  [(set (reg FLAGS_REG)
-       (compare (match_operand:SI 0 "nonimmediate_operand" "r,?mr")
-                (match_operand:SI 1 "const0_operand" "")))]
-  "ix86_match_ccmode (insn, CCNOmode)"
-  "@
-   test{l}\t%0, %0
-   cmp{l}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "test,icmp")
-   (set_attr "length_immediate" "0,1")
-   (set_attr "mode" "SI")])
-
-(define_insn "*cmpsi_minus_1"
-  [(set (reg FLAGS_REG)
-       (compare (minus:SI (match_operand:SI 0 "nonimmediate_operand" "rm,r")
-                          (match_operand:SI 1 "general_operand" "ri,mr"))
-                (const_int 0)))]
-  "ix86_match_ccmode (insn, CCGOCmode)"
-  "cmp{l}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "icmp")
-   (set_attr "mode" "SI")])
-
-(define_expand "cmpsi_1"
+(define_expand "cmp<mode>_1"
   [(set (reg:CC FLAGS_REG)
-       (compare:CC (match_operand:SI 0 "nonimmediate_operand" "")
-                   (match_operand:SI 1 "general_operand" "")))]
+       (compare:CC (match_operand:SWI48 0 "nonimmediate_operand" "")
+                   (match_operand:SWI48 1 "<general_operand>" "")))]
   ""
   "")
 
-(define_insn "*cmpsi_1_insn"
-  [(set (reg FLAGS_REG)
-       (compare (match_operand:SI 0 "nonimmediate_operand" "rm,r")
-                (match_operand:SI 1 "general_operand" "ri,mr")))]
-  "!(MEM_P (operands[0]) && MEM_P (operands[1]))
-    && ix86_match_ccmode (insn, CCmode)"
-  "cmp{l}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "icmp")
-   (set_attr "mode" "SI")])
-
-(define_insn "*cmphi_ccno_1"
-  [(set (reg FLAGS_REG)
-       (compare (match_operand:HI 0 "nonimmediate_operand" "r,?mr")
-                (match_operand:HI 1 "const0_operand" "")))]
-  "ix86_match_ccmode (insn, CCNOmode)"
-  "@
-   test{w}\t%0, %0
-   cmp{w}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "test,icmp")
-   (set_attr "length_immediate" "0,1")
-   (set_attr "mode" "HI")])
-
-(define_insn "*cmphi_minus_1"
-  [(set (reg FLAGS_REG)
-       (compare (minus:HI (match_operand:HI 0 "nonimmediate_operand" "rm,r")
-                          (match_operand:HI 1 "general_operand" "rn,mr"))
-                (const_int 0)))]
-  "ix86_match_ccmode (insn, CCGOCmode)"
-  "cmp{w}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "icmp")
-   (set_attr "mode" "HI")])
-
-(define_insn "*cmphi_1"
-  [(set (reg FLAGS_REG)
-       (compare (match_operand:HI 0 "nonimmediate_operand" "rm,r")
-                (match_operand:HI 1 "general_operand" "rn,mr")))]
-  "!(MEM_P (operands[0]) && MEM_P (operands[1]))
-   && ix86_match_ccmode (insn, CCmode)"
-  "cmp{w}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "icmp")
-   (set_attr "mode" "HI")])
-
-(define_insn "*cmpqi_ccno_1"
+(define_insn "*cmp<mode>_ccno_1"
   [(set (reg FLAGS_REG)
-       (compare (match_operand:QI 0 "nonimmediate_operand" "q,?mq")
-                (match_operand:QI 1 "const0_operand" "")))]
+       (compare (match_operand:SWI 0 "nonimmediate_operand" "<r>,?m<r>")
+                (match_operand:SWI 1 "const0_operand" "")))]
   "ix86_match_ccmode (insn, CCNOmode)"
   "@
-   test{b}\t%0, %0
-   cmp{b}\t{$0, %0|%0, 0}"
+   test{<imodesuffix>}\t%0, %0
+   cmp{<imodesuffix>}\t{%1, %0|%0, %1}"
   [(set_attr "type" "test,icmp")
    (set_attr "length_immediate" "0,1")
-   (set_attr "mode" "QI")])
+   (set_attr "mode" "<MODE>")])
 
-(define_insn "*cmpqi_1"
+(define_insn "*cmp<mode>_1"
   [(set (reg FLAGS_REG)
-       (compare (match_operand:QI 0 "nonimmediate_operand" "qm,q")
-                (match_operand:QI 1 "general_operand" "qn,mq")))]
-  "!(MEM_P (operands[0]) && MEM_P (operands[1]))
-    && ix86_match_ccmode (insn, CCmode)"
-  "cmp{b}\t{%1, %0|%0, %1}"
+       (compare (match_operand:SWI 0 "nonimmediate_operand" "<r>m,<r>")
+                (match_operand:SWI 1 "<general_operand>" "<r><i>,<r>m")))]
+  "ix86_match_ccmode (insn, CCmode)"
+  "cmp{<imodesuffix>}\t{%1, %0|%0, %1}"
   [(set_attr "type" "icmp")
-   (set_attr "mode" "QI")])
+   (set_attr "mode" "<MODE>")])
 
-(define_insn "*cmpqi_minus_1"
+(define_insn "*cmp<mode>_minus_1"
   [(set (reg FLAGS_REG)
-       (compare (minus:QI (match_operand:QI 0 "nonimmediate_operand" "qm,q")
-                          (match_operand:QI 1 "general_operand" "qn,mq"))
-                (const_int 0)))]
+       (compare
+         (minus:SWI (match_operand:SWI 0 "nonimmediate_operand" "<r>m,<r>")
+                    (match_operand:SWI 1 "<general_operand>" "<r><i>,<r>m"))
+         (const_int 0)))]
   "ix86_match_ccmode (insn, CCGOCmode)"
-  "cmp{b}\t{%1, %0|%0, %1}"
+  "cmp{<imodesuffix>}\t{%1, %0|%0, %1}"
   [(set_attr "type" "icmp")
-   (set_attr "mode" "QI")])
+   (set_attr "mode" "<MODE>")])
 
 (define_insn "*cmpqi_ext_1"
   [(set (reg FLAGS_REG)
              (match_operand 0 "ext_register_operand" "")
              (const_int 8)
              (const_int 8)) 0)
-         (match_operand:QI 1 "general_operand" "")))]
+         (match_operand:QI 1 "immediate_operand" "")))]
   ""
   "")
 
-(define_insn "cmpqi_ext_3_insn"
+(define_insn "*cmpqi_ext_3_insn"
   [(set (reg FLAGS_REG)
        (compare
          (subreg:QI
    (set_attr "modrm" "1")
    (set_attr "mode" "QI")])
 
-(define_insn "cmpqi_ext_3_insn_rex64"
+(define_insn "*cmpqi_ext_3_insn_rex64"
   [(set (reg FLAGS_REG)
        (compare
          (subreg:QI
        }
       else
        operands[3] = simplify_gen_subreg (V4SFmode, operands[1], SFmode, 0);
-      emit_insn (gen_sse_unpcklps (operands[3], operands[3], operands[3]));
+      emit_insn (gen_vec_interleave_lowv4sf (operands[3], operands[3],
+                                            operands[3]));
     }
   else
     emit_insn (gen_vec_setv4sf_0 (operands[3],
   "TARGET_80387
    || ((<SSEMODEI24:MODE>mode != DImode || TARGET_64BIT)
        && SSE_FLOAT_MODE_P (<X87MODEF:MODE>mode) && TARGET_SSE_MATH)"
-  "
 {
   if (!((<SSEMODEI24:MODE>mode != DImode || TARGET_64BIT)
        && SSE_FLOAT_MODE_P (<X87MODEF:MODE>mode) && TARGET_SSE_MATH)
       emit_insn (insn);
       DONE;
     }
-}")
+})
 
 ;; Pre-reload splitter to add memory clobber to the pattern.
 (define_insn_and_split "*float<SSEMODEI24:mode><X87MODEF:mode>2_1"
                              gen_rtx_SUBREG (SImode, operands[1], 0)));
   emit_insn (gen_sse2_loadld (operands[4], CONST0_RTX (V4SImode),
                              gen_rtx_SUBREG (SImode, operands[1], 4)));
-  emit_insn (gen_sse2_punpckldq (operands[3], operands[3], operands[4]));
+  emit_insn (gen_vec_interleave_lowv4si (operands[3], operands[3],
+                                        operands[4]));
 
   operands[3] = gen_rtx_REG (DImode, REGNO (operands[3]));
 })
                   (plus:DWIH (match_dup 1) (match_dup 2)))])
    (parallel [(set (match_dup 3)
                   (plus:DWIH
+                    (match_dup 4)
                     (plus:DWIH
                       (ltu:DWIH (reg:CC FLAGS_REG) (const_int 0))
-                      (match_dup 4))
-                    (match_dup 5)))
+                      (match_dup 5))))
              (clobber (reg:CC FLAGS_REG))])]
   "split_<dwi> (&operands[0], 3, &operands[0], &operands[3]);")
 
-(define_insn "add<mode>3_carry"
-  [(set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m,<r>")
-       (plus:SWI
-         (plus:SWI (match_operand:SWI 3 "ix86_carry_flag_operator" "")
-                   (match_operand:SWI 1 "nonimmediate_operand" "%0,0"))
-         (match_operand:SWI 2 "<general_operand>" "<r><i>,<r>m")))
-   (clobber (reg:CC FLAGS_REG))]
-  "ix86_binary_operator_ok (PLUS, <MODE>mode, operands)"
-  "adc{<imodesuffix>}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "use_carry" "1")
-   (set_attr "pent_pair" "pu")
-   (set_attr "mode" "<MODE>")])
-
-(define_insn "*addsi3_carry_zext"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (zero_extend:DI
-         (plus:SI
-           (plus:SI (match_operand:SI 3 "ix86_carry_flag_operator" "")
-                    (match_operand:SI 1 "nonimmediate_operand" "%0"))
-         (match_operand:SI 2 "general_operand" "g"))))
-   (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT && ix86_binary_operator_ok (PLUS, SImode, operands)"
-  "adc{l}\t{%2, %k0|%k0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "use_carry" "1")
-   (set_attr "pent_pair" "pu")
-   (set_attr "mode" "SI")])
-
 (define_insn "*add<mode>3_cc"
   [(set (reg:CC FLAGS_REG)
        (unspec:CC
   [(set_attr "type" "alu")
    (set_attr "mode" "QI")])
 
-(define_insn "*add<mode>3_cconly_overflow"
-  [(set (reg:CCC FLAGS_REG)
-       (compare:CCC
-         (plus:SWI
-           (match_operand:SWI 1 "nonimmediate_operand" "%0")
-           (match_operand:SWI 2 "<general_operand>" "<r><i>m"))
-         (match_dup 1)))
-   (clobber (match_scratch:SWI 0 "=<r>"))]
-  "ix86_binary_operator_ok (PLUS, <MODE>mode, operands)"
-  "add{<imodesuffix>}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "<MODE>")])
-
 (define_insn "*lea_1"
   [(set (match_operand:DWIH 0 "register_operand" "=r")
        (match_operand:DWIH 1 "no_seg_address_operand" "p"))]
              (clobber (reg:CC FLAGS_REG))])]
   "split_<dwi> (&operands[0], 3, &operands[0], &operands[3]);")
 
-(define_insn "sub<mode>3_carry"
+(define_insn "*sub<mode>_1"
   [(set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m,<r>")
-         (minus:SWI
-           (match_operand:SWI 1 "nonimmediate_operand" "0,0")
-           (plus:SWI
-             (match_operand:SWI 3 "ix86_carry_flag_operator" "")
-             (match_operand:SWI 2 "<general_operand>" "<r><i>,<r>m"))))
+       (minus:SWI
+         (match_operand:SWI 1 "nonimmediate_operand" "0,0")
+         (match_operand:SWI 2 "<general_operand>" "<r><i>,<r>m")))
    (clobber (reg:CC FLAGS_REG))]
   "ix86_binary_operator_ok (MINUS, <MODE>mode, operands)"
-  "sbb{<imodesuffix>}\t{%2, %0|%0, %2}"
+  "sub{<imodesuffix>}\t{%2, %0|%0, %2}"
   [(set_attr "type" "alu")
-   (set_attr "use_carry" "1")
-   (set_attr "pent_pair" "pu")
    (set_attr "mode" "<MODE>")])
 
-(define_insn "*subsi3_carry_zext"
+(define_insn "*subsi_1_zext"
   [(set (match_operand:DI 0 "register_operand" "=r")
-         (zero_extend:DI
-           (minus:SI (match_operand:SI 1 "register_operand" "0")
-             (plus:SI (match_operand:SI 3 "ix86_carry_flag_operator" "")
-                (match_operand:SI 2 "general_operand" "g")))))
+       (zero_extend:DI
+         (minus:SI (match_operand:SI 1 "register_operand" "0")
+                   (match_operand:SI 2 "general_operand" "g"))))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_64BIT && ix86_binary_operator_ok (MINUS, SImode, operands)"
-  "sbb{l}\t{%2, %k0|%k0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "pent_pair" "pu")
-   (set_attr "mode" "SI")])
-
-(define_insn "*sub<mode>3_cconly_overflow"
-  [(set (reg:CCC FLAGS_REG)
-       (compare:CCC
-         (minus:SWI
-           (match_operand:SWI 0 "nonimmediate_operand" "<r>m,<r>")
-           (match_operand:SWI 1 "<general_operand>" "<r><i>,<r>m"))
-         (match_dup 0)))]
-  ""
-  "cmp{<imodesuffix>}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "icmp")
-   (set_attr "mode" "<MODE>")])
-
-(define_insn "*sub<mode>_1"
-  [(set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m,<r>")
-       (minus:SWI
-         (match_operand:SWI 1 "nonimmediate_operand" "0,0")
-         (match_operand:SWI 2 "<general_operand>" "<r><i>,<r>m")))
-   (clobber (reg:CC FLAGS_REG))]
-  "ix86_binary_operator_ok (MINUS, <MODE>mode, operands)"
-  "sub{<imodesuffix>}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "<MODE>")])
-
-(define_insn "*subsi_1_zext"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (zero_extend:DI
-         (minus:SI (match_operand:SI 1 "register_operand" "0")
-                   (match_operand:SI 2 "general_operand" "g"))))
-   (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT && ix86_binary_operator_ok (MINUS, SImode, operands)"
-  "sub{l}\t{%2, %k0|%k0, %2}"
+  "sub{l}\t{%2, %k0|%k0, %2}"
   [(set_attr "type" "alu")
    (set_attr "mode" "SI")])
 
   "sub{l}\t{%2, %1|%1, %2}"
   [(set_attr "type" "alu")
    (set_attr "mode" "SI")])
+\f
+;; Add with carry and subtract with borrow
+
+(define_expand "<plusminus_insn><mode>3_carry"
+  [(parallel
+    [(set (match_operand:SWI 0 "nonimmediate_operand" "")
+         (plusminus:SWI
+           (match_operand:SWI 1 "nonimmediate_operand" "")
+           (plus:SWI (match_operator:SWI 4 "ix86_carry_flag_operator"
+                      [(match_operand 3 "flags_reg_operand" "")
+                       (const_int 0)])
+                     (match_operand:SWI 2 "<general_operand>" ""))))
+     (clobber (reg:CC FLAGS_REG))])]
+  "ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
+  "")
+
+(define_insn "*<plusminus_insn><mode>3_carry"
+  [(set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m,<r>")
+       (plusminus:SWI
+         (match_operand:SWI 1 "nonimmediate_operand" "<comm>0,0")
+         (plus:SWI
+           (match_operator 3 "ix86_carry_flag_operator"
+            [(reg FLAGS_REG) (const_int 0)])
+           (match_operand:SWI 2 "<general_operand>" "<r><i>,<r>m"))))
+   (clobber (reg:CC FLAGS_REG))]
+  "ix86_binary_operator_ok (PLUS, <MODE>mode, operands)"
+  "<plusminus_carry_mnemonic>{<imodesuffix>}\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "use_carry" "1")
+   (set_attr "pent_pair" "pu")
+   (set_attr "mode" "<MODE>")])
+
+(define_insn "*addsi3_carry_zext"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (zero_extend:DI
+         (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
+                  (plus:SI (match_operator 3 "ix86_carry_flag_operator"
+                            [(reg FLAGS_REG) (const_int 0)])
+                           (match_operand:SI 2 "general_operand" "g")))))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT && ix86_binary_operator_ok (PLUS, SImode, operands)"
+  "adc{l}\t{%2, %k0|%k0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "use_carry" "1")
+   (set_attr "pent_pair" "pu")
+   (set_attr "mode" "SI")])
+
+(define_insn "*subsi3_carry_zext"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (zero_extend:DI
+         (minus:SI (match_operand:SI 1 "register_operand" "0")
+                   (plus:SI (match_operator 3 "ix86_carry_flag_operator"
+                             [(reg FLAGS_REG) (const_int 0)])
+                            (match_operand:SI 2 "general_operand" "g")))))
+   (clobber (reg:CC FLAGS_REG))]
+  "TARGET_64BIT && ix86_binary_operator_ok (MINUS, SImode, operands)"
+  "sbb{l}\t{%2, %k0|%k0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "pent_pair" "pu")
+   (set_attr "mode" "SI")])
+\f
+;; Overflow setting add and subtract instructions
+
+(define_insn "*add<mode>3_cconly_overflow"
+  [(set (reg:CCC FLAGS_REG)
+       (compare:CCC
+         (plus:SWI
+           (match_operand:SWI 1 "nonimmediate_operand" "%0")
+           (match_operand:SWI 2 "<general_operand>" "<r><i>m"))
+         (match_dup 1)))
+   (clobber (match_scratch:SWI 0 "=<r>"))]
+  "ix86_binary_operator_ok (PLUS, <MODE>mode, operands)"
+  "add{<imodesuffix>}\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "<MODE>")])
 
+(define_insn "*sub<mode>3_cconly_overflow"
+  [(set (reg:CCC FLAGS_REG)
+       (compare:CCC
+         (minus:SWI
+           (match_operand:SWI 0 "nonimmediate_operand" "<r>m,<r>")
+           (match_operand:SWI 1 "<general_operand>" "<r><i>,<r>m"))
+         (match_dup 0)))]
+  ""
+  "cmp{<imodesuffix>}\t{%1, %0|%0, %1}"
+  [(set_attr "type" "icmp")
+   (set_attr "mode" "<MODE>")])
 
 (define_insn "*<plusminus_insn><mode>3_cc_overflow"
   [(set (reg:CCC FLAGS_REG)
 ;; On Pentium, "test imm, reg" is pairable only with eax, ax, and al.
 ;; Note that this excludes ah.
 
-(define_insn "*testdi_1_rex64"
-  [(set (reg FLAGS_REG)
-       (compare
-         (and:DI (match_operand:DI 0 "nonimmediate_operand" "%!*a,r,!*a,r,rm")
-                 (match_operand:DI 1 "x86_64_szext_general_operand" "Z,Z,e,e,re"))
-         (const_int 0)))]
-  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
-   && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
-  "@
-   test{l}\t{%k1, %k0|%k0, %k1}
-   test{l}\t{%k1, %k0|%k0, %k1}
-   test{q}\t{%1, %0|%0, %1}
-   test{q}\t{%1, %0|%0, %1}
-   test{q}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "test")
-   (set_attr "modrm" "0,1,0,1,1")
-   (set_attr "mode" "SI,SI,DI,DI,DI")
-   (set_attr "pent_pair" "uv,np,uv,np,uv")])
-
-(define_insn "testsi_1"
-  [(set (reg FLAGS_REG)
-       (compare
-         (and:SI (match_operand:SI 0 "nonimmediate_operand" "%!*a,r,rm")
-                 (match_operand:SI 1 "general_operand" "i,i,ri"))
-         (const_int 0)))]
-  "ix86_match_ccmode (insn, CCNOmode)
-   && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
-  "test{l}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "test")
-   (set_attr "modrm" "0,1,1")
-   (set_attr "mode" "SI")
-   (set_attr "pent_pair" "uv,np,uv")])
-
 (define_expand "testsi_ccno_1"
   [(set (reg:CCNO FLAGS_REG)
        (compare:CCNO
   ""
   "")
 
-(define_insn "*testhi_1"
-  [(set (reg FLAGS_REG)
-        (compare (and:HI (match_operand:HI 0 "nonimmediate_operand" "%!*a,r,rm")
-                        (match_operand:HI 1 "general_operand" "n,n,rn"))
-                (const_int 0)))]
-  "ix86_match_ccmode (insn, CCNOmode)
-   && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
-  "test{w}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "test")
-   (set_attr "modrm" "0,1,1")
-   (set_attr "mode" "HI")
-   (set_attr "pent_pair" "uv,np,uv")])
-
 (define_expand "testqi_ccz_1"
   [(set (reg:CCZ FLAGS_REG)
         (compare:CCZ (and:QI (match_operand:QI 0 "nonimmediate_operand" "")
   ""
   "")
 
+(define_insn "*testdi_1"
+  [(set (reg FLAGS_REG)
+       (compare
+        (and:DI
+         (match_operand:DI 0 "nonimmediate_operand" "%!*a,r,!*a,r,rm")
+         (match_operand:DI 1 "x86_64_szext_general_operand" "Z,Z,e,e,re"))
+        (const_int 0)))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
+   && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
+  "@
+   test{l}\t{%k1, %k0|%k0, %k1}
+   test{l}\t{%k1, %k0|%k0, %k1}
+   test{q}\t{%1, %0|%0, %1}
+   test{q}\t{%1, %0|%0, %1}
+   test{q}\t{%1, %0|%0, %1}"
+  [(set_attr "type" "test")
+   (set_attr "modrm" "0,1,0,1,1")
+   (set_attr "mode" "SI,SI,DI,DI,DI")])
+
 (define_insn "*testqi_1_maybe_si"
   [(set (reg FLAGS_REG)
         (compare
    (set_attr "mode" "QI,QI,QI,SI")
    (set_attr "pent_pair" "uv,np,uv,np")])
 
-(define_insn "*testqi_1"
+(define_insn "*test<mode>_1"
   [(set (reg FLAGS_REG)
-        (compare
-         (and:QI
-           (match_operand:QI 0 "nonimmediate_operand" "%!*a,q,qm")
-           (match_operand:QI 1 "general_operand" "n,n,qn"))
-         (const_int 0)))]
-  "!(MEM_P (operands[0]) && MEM_P (operands[1]))
-   && ix86_match_ccmode (insn, CCNOmode)"
-  "test{b}\t{%1, %0|%0, %1}"
+       (compare
+        (and:SWI124
+         (match_operand:SWI124 0 "nonimmediate_operand" "%!*a,<r>,<r>m")
+         (match_operand:SWI124 1 "general_operand" "<i>,<i>,<r><i>"))
+        (const_int 0)))]
+  "ix86_match_ccmode (insn, CCNOmode)
+   && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
+  "test{<imodesuffix>}\t{%1, %0|%0, %1}"
   [(set_attr "type" "test")
    (set_attr "modrm" "0,1,1")
-   (set_attr "mode" "QI")
+   (set_attr "mode" "<MODE>")
    (set_attr "pent_pair" "uv,np,uv")])
 
 (define_expand "testqi_ext_ccno_0"
    (set_attr "modrm" "1")
    (set_attr "pent_pair" "np")])
 
-(define_insn "*testqi_ext_1"
+(define_insn "*testqi_ext_1_rex64"
   [(set (reg FLAGS_REG)
        (compare
          (and:SI
              (const_int 8)
              (const_int 8))
            (zero_extend:SI
-             (match_operand:QI 1 "general_operand" "Qm")))
+             (match_operand:QI 1 "register_operand" "Q")))
          (const_int 0)))]
-  "!TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
-   && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)"
   "test{b}\t{%1, %h0|%h0, %1}"
   [(set_attr "type" "test")
    (set_attr "mode" "QI")])
 
-(define_insn "*testqi_ext_1_rex64"
+(define_insn "*testqi_ext_1"
   [(set (reg FLAGS_REG)
        (compare
          (and:SI
              (const_int 8)
              (const_int 8))
            (zero_extend:SI
-             (match_operand:QI 1 "register_operand" "Q")))
+             (match_operand:QI 1 "general_operand" "Qm")))
          (const_int 0)))]
-  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)"
+  "!TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)"
   "test{b}\t{%1, %h0|%h0, %1}"
   [(set_attr "type" "test")
    (set_attr "mode" "QI")])
   [(set_attr "type" "test")
    (set_attr "mode" "QI")])
 
-;; Combine likes to form bit extractions for some tests.  Humor it.
-(define_insn "*testqi_ext_3"
-  [(set (reg FLAGS_REG)
-        (compare (zero_extract:SI
-                  (match_operand 0 "nonimmediate_operand" "rm")
-                  (match_operand:SI 1 "const_int_operand" "")
-                  (match_operand:SI 2 "const_int_operand" ""))
-                (const_int 0)))]
-  "ix86_match_ccmode (insn, CCNOmode)
-   && INTVAL (operands[1]) > 0
-   && INTVAL (operands[2]) >= 0
-   && INTVAL (operands[1]) + INTVAL (operands[2]) <= 32
-   && (GET_MODE (operands[0]) == SImode
-       || (TARGET_64BIT && GET_MODE (operands[0]) == DImode)
-       || GET_MODE (operands[0]) == HImode
-       || GET_MODE (operands[0]) == QImode)"
-  "#")
-
 (define_insn "*testqi_ext_3_rex64"
   [(set (reg FLAGS_REG)
         (compare (zero_extract:DI
        || GET_MODE (operands[0]) == QImode)"
   "#")
 
+;; Combine likes to form bit extractions for some tests.  Humor it.
+(define_insn "*testqi_ext_3"
+  [(set (reg FLAGS_REG)
+        (compare (zero_extract:SI
+                  (match_operand 0 "nonimmediate_operand" "rm")
+                  (match_operand:SI 1 "const_int_operand" "")
+                  (match_operand:SI 2 "const_int_operand" ""))
+                (const_int 0)))]
+  "ix86_match_ccmode (insn, CCNOmode)
+   && INTVAL (operands[1]) > 0
+   && INTVAL (operands[2]) >= 0
+   && INTVAL (operands[1]) + INTVAL (operands[2]) <= 32
+   && (GET_MODE (operands[0]) == SImode
+       || (TARGET_64BIT && GET_MODE (operands[0]) == DImode)
+       || GET_MODE (operands[0]) == HImode
+       || GET_MODE (operands[0]) == QImode)"
+  "#")
+
 (define_split
   [(set (match_operand 0 "flags_reg_operand" "")
         (match_operator 1 "compare_operator"
   "operands[2] = gen_lowpart (QImode, operands[2]);
    operands[3] = gen_lowpart (QImode, operands[3]);")
 
-
 ;; %%% This used to optimize known byte-wide and operations to memory,
 ;; and sometimes to QImode registers.  If this is considered useful,
 ;; it should be done with splitters.
 
-(define_expand "anddi3"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "")
-       (and:DI (match_operand:DI 1 "nonimmediate_operand" "")
-               (match_operand:DI 2 "x86_64_szext_general_operand" "")))]
-  "TARGET_64BIT"
-  "ix86_expand_binary_operator (AND, DImode, operands); DONE;")
+(define_expand "and<mode>3"
+  [(set (match_operand:SWIM 0 "nonimmediate_operand" "")
+       (and:SWIM (match_operand:SWIM 1 "nonimmediate_operand" "")
+                 (match_operand:SWIM 2 "<general_szext_operand>" "")))]
+  ""
+  "ix86_expand_binary_operator (AND, <MODE>mode, operands); DONE;")
 
-(define_insn "*anddi_1_rex64"
+(define_insn "*anddi_1"
   [(set (match_operand:DI 0 "nonimmediate_operand" "=r,rm,r,r")
-       (and:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0,0,qm")
-               (match_operand:DI 2 "x86_64_szext_general_operand" "Z,re,rm,L")))
+       (and:DI
+        (match_operand:DI 1 "nonimmediate_operand" "%0,0,0,qm")
+        (match_operand:DI 2 "x86_64_szext_general_operand" "Z,re,rm,L")))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_64BIT && ix86_binary_operator_ok (AND, DImode, operands)"
 {
        (const_string "*")))
    (set_attr "mode" "SI,DI,DI,SI")])
 
-(define_insn "*anddi_2"
-  [(set (reg FLAGS_REG)
-       (compare (and:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0,0")
-                        (match_operand:DI 2 "x86_64_szext_general_operand" "Z,rem,re"))
-                (const_int 0)))
-   (set (match_operand:DI 0 "nonimmediate_operand" "=r,r,rm")
-       (and:DI (match_dup 1) (match_dup 2)))]
-  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (AND, DImode, operands)"
-  "@
-   and{l}\t{%k2, %k0|%k0, %k2}
-   and{q}\t{%2, %0|%0, %2}
-   and{q}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "SI,DI,DI")])
-
-(define_expand "andsi3"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "")
-       (and:SI (match_operand:SI 1 "nonimmediate_operand" "")
-               (match_operand:SI 2 "general_operand" "")))]
-  ""
-  "ix86_expand_binary_operator (AND, SImode, operands); DONE;")
-
 (define_insn "*andsi_1"
   [(set (match_operand:SI 0 "nonimmediate_operand" "=rm,r,r")
        (and:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,qm")
    (set_attr "length_immediate" "*,*,0")
    (set_attr "mode" "SI")])
 
-(define_split
-  [(set (match_operand 0 "register_operand" "")
-       (and (match_dup 0)
-            (const_int -65536)))
-   (clobber (reg:CC FLAGS_REG))]
-  "optimize_function_for_size_p (cfun) || (TARGET_FAST_PREFIX && !TARGET_PARTIAL_REG_STALL)"
-  [(set (strict_low_part (match_dup 1)) (const_int 0))]
-  "operands[1] = gen_lowpart (HImode, operands[0]);")
-
-(define_split
-  [(set (match_operand 0 "ext_register_operand" "")
-       (and (match_dup 0)
-            (const_int -256)))
-   (clobber (reg:CC FLAGS_REG))]
-  "(optimize_function_for_size_p (cfun) || !TARGET_PARTIAL_REG_STALL) && reload_completed"
-  [(set (strict_low_part (match_dup 1)) (const_int 0))]
-  "operands[1] = gen_lowpart (QImode, operands[0]);")
-
-(define_split
-  [(set (match_operand 0 "ext_register_operand" "")
-       (and (match_dup 0)
-            (const_int -65281)))
-   (clobber (reg:CC FLAGS_REG))]
-  "(optimize_function_for_size_p (cfun) || !TARGET_PARTIAL_REG_STALL) && reload_completed"
-  [(parallel [(set (zero_extract:SI (match_dup 0)
-                                   (const_int 8)
-                                   (const_int 8))
-                  (xor:SI
-                    (zero_extract:SI (match_dup 0)
-                                     (const_int 8)
-                                     (const_int 8))
-                    (zero_extract:SI (match_dup 0)
-                                     (const_int 8)
-                                     (const_int 8))))
-             (clobber (reg:CC FLAGS_REG))])]
-  "operands[0] = gen_lowpart (SImode, operands[0]);")
-
 ;; See comment for addsi_1_zext why we do use nonimmediate_operand
 (define_insn "*andsi_1_zext"
   [(set (match_operand:DI 0 "register_operand" "=r")
   [(set_attr "type" "alu")
    (set_attr "mode" "SI")])
 
-(define_insn "*andsi_2"
-  [(set (reg FLAGS_REG)
-       (compare (and:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                        (match_operand:SI 2 "general_operand" "g,ri"))
-                (const_int 0)))
-   (set (match_operand:SI 0 "nonimmediate_operand" "=r,rm")
-       (and:SI (match_dup 1) (match_dup 2)))]
-  "ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (AND, SImode, operands)"
-  "and{l}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "SI")])
-
-;; See comment for addsi_1_zext why we do use nonimmediate_operand
-(define_insn "*andsi_2_zext"
-  [(set (reg FLAGS_REG)
-       (compare (and:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
-                        (match_operand:SI 2 "general_operand" "g"))
-                (const_int 0)))
-   (set (match_operand:DI 0 "register_operand" "=r")
-       (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
-  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (AND, SImode, operands)"
-  "and{l}\t{%2, %k0|%k0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "SI")])
-
-(define_expand "andhi3"
-  [(set (match_operand:HI 0 "nonimmediate_operand" "")
-       (and:HI (match_operand:HI 1 "nonimmediate_operand" "")
-               (match_operand:HI 2 "general_operand" "")))]
-  "TARGET_HIMODE_MATH"
-  "ix86_expand_binary_operator (AND, HImode, operands); DONE;")
-
 (define_insn "*andhi_1"
   [(set (match_operand:HI 0 "nonimmediate_operand" "=rm,r,r")
        (and:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0,qm")
        (const_string "*")))
    (set_attr "mode" "HI,HI,SI")])
 
-(define_insn "*andhi_2"
-  [(set (reg FLAGS_REG)
-       (compare (and:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0")
-                        (match_operand:HI 2 "general_operand" "rmn,rn"))
-                (const_int 0)))
-   (set (match_operand:HI 0 "nonimmediate_operand" "=r,rm")
-       (and:HI (match_dup 1) (match_dup 2)))]
-  "ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (AND, HImode, operands)"
-  "and{w}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "HI")])
-
-(define_expand "andqi3"
-  [(set (match_operand:QI 0 "nonimmediate_operand" "")
-       (and:QI (match_operand:QI 1 "nonimmediate_operand" "")
-               (match_operand:QI 2 "general_operand" "")))]
-  "TARGET_QIMODE_MATH"
-  "ix86_expand_binary_operator (AND, QImode, operands); DONE;")
-
 ;; %%% Potential partial reg stall on alternative 2.  What to do?
 (define_insn "*andqi_1"
   [(set (match_operand:QI 0 "nonimmediate_operand" "=qm,q,r")
        (and:QI (match_dup 0)
                (match_operand:QI 1 "general_operand" "qn,qmn")))
    (clobber (reg:CC FLAGS_REG))]
-  "(! TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
+  "(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
    && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
   "and{b}\t{%1, %0|%0, %1}"
   [(set_attr "type" "alu1")
    (set_attr "mode" "QI")])
 
+(define_split
+  [(set (match_operand 0 "register_operand" "")
+       (and (match_dup 0)
+            (const_int -65536)))
+   (clobber (reg:CC FLAGS_REG))]
+  "(TARGET_FAST_PREFIX && !TARGET_PARTIAL_REG_STALL)
+    || optimize_function_for_size_p (cfun)"
+  [(set (strict_low_part (match_dup 1)) (const_int 0))]
+  "operands[1] = gen_lowpart (HImode, operands[0]);")
+
+(define_split
+  [(set (match_operand 0 "ext_register_operand" "")
+       (and (match_dup 0)
+            (const_int -256)))
+   (clobber (reg:CC FLAGS_REG))]
+  "(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
+   && reload_completed"
+  [(set (strict_low_part (match_dup 1)) (const_int 0))]
+  "operands[1] = gen_lowpart (QImode, operands[0]);")
+
+(define_split
+  [(set (match_operand 0 "ext_register_operand" "")
+       (and (match_dup 0)
+            (const_int -65281)))
+   (clobber (reg:CC FLAGS_REG))]
+  "(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
+   && reload_completed"
+  [(parallel [(set (zero_extract:SI (match_dup 0)
+                                   (const_int 8)
+                                   (const_int 8))
+                  (xor:SI
+                    (zero_extract:SI (match_dup 0)
+                                     (const_int 8)
+                                     (const_int 8))
+                    (zero_extract:SI (match_dup 0)
+                                     (const_int 8)
+                                     (const_int 8))))
+             (clobber (reg:CC FLAGS_REG))])]
+  "operands[0] = gen_lowpart (SImode, operands[0]);")
+
+(define_insn "*anddi_2"
+  [(set (reg FLAGS_REG)
+       (compare
+        (and:DI
+         (match_operand:DI 1 "nonimmediate_operand" "%0,0,0")
+         (match_operand:DI 2 "x86_64_szext_general_operand" "Z,rem,re"))
+        (const_int 0)))
+   (set (match_operand:DI 0 "nonimmediate_operand" "=r,r,rm")
+       (and:DI (match_dup 1) (match_dup 2)))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
+   && ix86_binary_operator_ok (AND, DImode, operands)"
+  "@
+   and{l}\t{%k2, %k0|%k0, %k2}
+   and{q}\t{%2, %0|%0, %2}
+   and{q}\t{%2, %0|%0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "SI,DI,DI")])
+
 (define_insn "*andqi_2_maybe_si"
   [(set (reg FLAGS_REG)
        (compare (and:QI
-                     (match_operand:QI 1 "nonimmediate_operand" "%0,0,0")
-                     (match_operand:QI 2 "general_operand" "qmn,qn,n"))
+                 (match_operand:QI 1 "nonimmediate_operand" "%0,0,0")
+                 (match_operand:QI 2 "general_operand" "qmn,qn,n"))
                 (const_int 0)))
    (set (match_operand:QI 0 "nonimmediate_operand" "=q,qm,*r")
        (and:QI (match_dup 1) (match_dup 2)))]
   [(set_attr "type" "alu")
    (set_attr "mode" "QI,QI,SI")])
 
-(define_insn "*andqi_2"
+(define_insn "*and<mode>_2"
   [(set (reg FLAGS_REG)
-       (compare (and:QI
-                  (match_operand:QI 1 "nonimmediate_operand" "%0,0")
-                  (match_operand:QI 2 "general_operand" "qmn,qn"))
+       (compare (and:SWI124
+                 (match_operand:SWI124 1 "nonimmediate_operand" "%0,0")
+                 (match_operand:SWI124 2 "general_operand" "<g>,<r><i>"))
                 (const_int 0)))
-   (set (match_operand:QI 0 "nonimmediate_operand" "=q,qm")
-       (and:QI (match_dup 1) (match_dup 2)))]
+   (set (match_operand:SWI124 0 "nonimmediate_operand" "=<r>,<r>m")
+       (and:SWI124 (match_dup 1) (match_dup 2)))]
   "ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (AND, QImode, operands)"
-  "and{b}\t{%2, %0|%0, %2}"
+   && ix86_binary_operator_ok (AND, <MODE>mode, operands)"
+  "and{<imodesuffix>}\t{%2, %0|%0, %2}"
   [(set_attr "type" "alu")
-   (set_attr "mode" "QI")])
+   (set_attr "mode" "<MODE>")])
+
+;; See comment for addsi_1_zext why we do use nonimmediate_operand
+(define_insn "*andsi_2_zext"
+  [(set (reg FLAGS_REG)
+       (compare (and:SI
+                 (match_operand:SI 1 "nonimmediate_operand" "%0")
+                 (match_operand:SI 2 "general_operand" "g"))
+                (const_int 0)))
+   (set (match_operand:DI 0 "register_operand" "=r")
+       (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))]
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
+   && ix86_binary_operator_ok (AND, SImode, operands)"
+  "and{l}\t{%2, %k0|%k0, %2}"
+  [(set_attr "type" "alu")
+   (set_attr "mode" "SI")])
 
 (define_insn "*andqi_2_slp"
   [(set (reg FLAGS_REG)
                 (const_int 0)))
    (set (strict_low_part (match_dup 0))
        (and:QI (match_dup 0) (match_dup 1)))]
-  "(! TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
+  "(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
    && ix86_match_ccmode (insn, CCNOmode)
    && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
   "and{b}\t{%1, %0|%0, %1}"
 ;; ??? A bug in recog prevents it from recognizing a const_int as an
 ;; operand to zero_extend in andqi_ext_1.  It was checking explicitly
 ;; for a QImode operand, which of course failed.
-
 (define_insn "andqi_ext_0"
   [(set (zero_extract:SI (match_operand 0 "ext_register_operand" "=Q")
                         (const_int 8)
 
 ;; Generated by peephole translating test to and.  This shows up
 ;; often in fp comparisons.
-
 (define_insn "*andqi_ext_0_cc"
   [(set (reg FLAGS_REG)
        (compare
    (set_attr "modrm" "1")
    (set_attr "mode" "QI")])
 
-(define_insn "*andqi_ext_1"
+(define_insn "*andqi_ext_1_rex64"
   [(set (zero_extract:SI (match_operand 0 "ext_register_operand" "=Q")
                         (const_int 8)
                         (const_int 8))
            (const_int 8)
            (const_int 8))
          (zero_extend:SI
-           (match_operand:QI 2 "general_operand" "Qm"))))
+           (match_operand 2 "ext_register_operand" "Q"))))
    (clobber (reg:CC FLAGS_REG))]
-  "!TARGET_64BIT"
+  "TARGET_64BIT"
   "and{b}\t{%2, %h0|%h0, %2}"
   [(set_attr "type" "alu")
    (set_attr "length_immediate" "0")
    (set_attr "mode" "QI")])
 
-(define_insn "*andqi_ext_1_rex64"
+(define_insn "*andqi_ext_1"
   [(set (zero_extract:SI (match_operand 0 "ext_register_operand" "=Q")
                         (const_int 8)
                         (const_int 8))
            (const_int 8)
            (const_int 8))
          (zero_extend:SI
-           (match_operand 2 "ext_register_operand" "Q"))))
+           (match_operand:QI 2 "general_operand" "Qm"))))
    (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT"
+  "!TARGET_64BIT"
   "and{b}\t{%2, %h0|%h0, %2}"
   [(set_attr "type" "alu")
    (set_attr "length_immediate" "0")
    operands[1] = gen_lowpart (QImode, operands[1]);
    operands[2] = gen_lowpart (QImode, operands[2]);")
 \f
-;; Logical inclusive OR instructions
+;; Logical inclusive and exclusive OR instructions
 
 ;; %%% This used to optimize known byte-wide and operations to memory.
 ;; If this is considered useful, it should be done with splitters.
 
-(define_expand "iordi3"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "")
-       (ior:DI (match_operand:DI 1 "nonimmediate_operand" "")
-               (match_operand:DI 2 "x86_64_general_operand" "")))]
-  "TARGET_64BIT"
-  "ix86_expand_binary_operator (IOR, DImode, operands); DONE;")
+(define_expand "<code><mode>3"
+  [(set (match_operand:SWIM 0 "nonimmediate_operand" "")
+       (any_or:SWIM (match_operand:SWIM 1 "nonimmediate_operand" "")
+                    (match_operand:SWIM 2 "<general_operand>" "")))]
+  ""
+  "ix86_expand_binary_operator (<CODE>, <MODE>mode, operands); DONE;")
 
-(define_insn "*iordi_1_rex64"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "=rm,r")
-       (ior:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
-               (match_operand:DI 2 "x86_64_general_operand" "re,rme")))
+(define_insn "*<code><mode>_1"
+  [(set (match_operand:SWI248 0 "nonimmediate_operand" "=r,rm")
+       (any_or:SWI248
+        (match_operand:SWI248 1 "nonimmediate_operand" "%0,0")
+        (match_operand:SWI248 2 "<general_operand>" "<g>,r<i>")))
    (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT
-   && ix86_binary_operator_ok (IOR, DImode, operands)"
-  "or{q}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "DI")])
-
-(define_insn "*iordi_2_rex64"
-  [(set (reg FLAGS_REG)
-       (compare (ior:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
-                        (match_operand:DI 2 "x86_64_general_operand" "rem,re"))
-                (const_int 0)))
-   (set (match_operand:DI 0 "nonimmediate_operand" "=r,rm")
-       (ior:DI (match_dup 1) (match_dup 2)))]
-  "TARGET_64BIT
-   && ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (IOR, DImode, operands)"
-  "or{q}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "DI")])
-
-(define_insn "*iordi_3_rex64"
-  [(set (reg FLAGS_REG)
-       (compare (ior:DI (match_operand:DI 1 "nonimmediate_operand" "%0")
-                        (match_operand:DI 2 "x86_64_general_operand" "rem"))
-                (const_int 0)))
-   (clobber (match_scratch:DI 0 "=r"))]
-  "TARGET_64BIT
-   && ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (IOR, DImode, operands)"
-  "or{q}\t{%2, %0|%0, %2}"
+  "ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
+  "<logicprefix>{<imodesuffix>}\t{%2, %0|%0, %2}"
   [(set_attr "type" "alu")
-   (set_attr "mode" "DI")])
-
-
-(define_expand "iorsi3"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "")
-       (ior:SI (match_operand:SI 1 "nonimmediate_operand" "")
-               (match_operand:SI 2 "general_operand" "")))]
-  ""
-  "ix86_expand_binary_operator (IOR, SImode, operands); DONE;")
+   (set_attr "mode" "<MODE>")])
 
-(define_insn "*iorsi_1"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "=rm,r")
-       (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-               (match_operand:SI 2 "general_operand" "ri,g")))
+;; %%% Potential partial reg stall on alternative 2.  What to do?
+(define_insn "*<code>qi_1"
+  [(set (match_operand:QI 0 "nonimmediate_operand" "=q,m,r")
+       (any_or:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0,0")
+                  (match_operand:QI 2 "general_operand" "qmn,qn,rn")))
    (clobber (reg:CC FLAGS_REG))]
-  "ix86_binary_operator_ok (IOR, SImode, operands)"
-  "or{l}\t{%2, %0|%0, %2}"
+  "ix86_binary_operator_ok (<CODE>, QImode, operands)"
+  "@
+   <logicprefix>{b}\t{%2, %0|%0, %2}
+   <logicprefix>{b}\t{%2, %0|%0, %2}
+   <logicprefix>{l}\t{%k2, %k0|%k0, %k2}"
   [(set_attr "type" "alu")
-   (set_attr "mode" "SI")])
+   (set_attr "mode" "QI,QI,SI")])
 
 ;; See comment for addsi_1_zext why we do use nonimmediate_operand
-(define_insn "*iorsi_1_zext"
+(define_insn "*<code>si_1_zext"
   [(set (match_operand:DI 0 "register_operand" "=r")
        (zero_extend:DI
-         (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
-                 (match_operand:SI 2 "general_operand" "g"))))
+        (any_or:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
+                   (match_operand:SI 2 "general_operand" "g"))))
    (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT && ix86_binary_operator_ok (IOR, SImode, operands)"
-  "or{l}\t{%2, %k0|%k0, %2}"
+  "TARGET_64BIT && ix86_binary_operator_ok (<CODE>, SImode, operands)"
+  "<logicprefix>{l}\t{%2, %k0|%k0, %2}"
   [(set_attr "type" "alu")
    (set_attr "mode" "SI")])
 
-(define_insn "*iorsi_1_zext_imm"
+(define_insn "*<code>si_1_zext_imm"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (ior:DI (zero_extend:DI (match_operand:SI 1 "register_operand" "%0"))
-               (match_operand:DI 2 "x86_64_zext_immediate_operand" "Z")))
+       (any_or:DI
+        (zero_extend:DI (match_operand:SI 1 "register_operand" "%0"))
+        (match_operand:DI 2 "x86_64_zext_immediate_operand" "Z")))
    (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT"
-  "or{l}\t{%2, %k0|%k0, %2}"
+  "TARGET_64BIT && ix86_binary_operator_ok (<CODE>, SImode, operands)"
+  "<logicprefix>{l}\t{%2, %k0|%k0, %2}"
   [(set_attr "type" "alu")
    (set_attr "mode" "SI")])
 
-(define_insn "*iorsi_2"
+(define_insn "*<code>qi_1_slp"
+  [(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+q,m"))
+       (any_or:QI (match_dup 0)
+                  (match_operand:QI 1 "general_operand" "qmn,qn")))
+   (clobber (reg:CC FLAGS_REG))]
+  "(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
+   && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
+  "<logicprefix>{b}\t{%1, %0|%0, %1}"
+  [(set_attr "type" "alu1")
+   (set_attr "mode" "QI")])
+
+(define_insn "*<code><mode>_2"
   [(set (reg FLAGS_REG)
-       (compare (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                        (match_operand:SI 2 "general_operand" "g,ri"))
+       (compare (any_or:SWI
+                 (match_operand:SWI 1 "nonimmediate_operand" "%0,0")
+                 (match_operand:SWI 2 "<general_operand>" "<g>,<r><i>"))
                 (const_int 0)))
-   (set (match_operand:SI 0 "nonimmediate_operand" "=r,rm")
-       (ior:SI (match_dup 1) (match_dup 2)))]
+   (set (match_operand:SWI 0 "nonimmediate_operand" "=<r>,<r>m")
+       (any_or:SWI (match_dup 1) (match_dup 2)))]
   "ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (IOR, SImode, operands)"
-  "or{l}\t{%2, %0|%0, %2}"
+   && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
+  "<logicprefix>{<imodesuffix>}\t{%2, %0|%0, %2}"
   [(set_attr "type" "alu")
-   (set_attr "mode" "SI")])
+   (set_attr "mode" "<MODE>")])
 
 ;; See comment for addsi_1_zext why we do use nonimmediate_operand
 ;; ??? Special case for immediate operand is missing - it is tricky.
-(define_insn "*iorsi_2_zext"
+(define_insn "*<code>si_2_zext"
   [(set (reg FLAGS_REG)
-       (compare (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
-                        (match_operand:SI 2 "general_operand" "g"))
+       (compare (any_or:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
+                           (match_operand:SI 2 "general_operand" "g"))
                 (const_int 0)))
    (set (match_operand:DI 0 "register_operand" "=r")
-       (zero_extend:DI (ior:SI (match_dup 1) (match_dup 2))))]
+       (zero_extend:DI (any_or:SI (match_dup 1) (match_dup 2))))]
   "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (IOR, SImode, operands)"
-  "or{l}\t{%2, %k0|%k0, %2}"
+   && ix86_binary_operator_ok (<CODE>, SImode, operands)"
+  "<logicprefix>{l}\t{%2, %k0|%k0, %2}"
   [(set_attr "type" "alu")
    (set_attr "mode" "SI")])
 
-(define_insn "*iorsi_2_zext_imm"
+(define_insn "*<code>si_2_zext_imm"
   [(set (reg FLAGS_REG)
-       (compare (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
-                        (match_operand 2 "x86_64_zext_immediate_operand" "Z"))
+       (compare (any_or:SI
+                 (match_operand:SI 1 "nonimmediate_operand" "%0")
+                 (match_operand:SI 2 "x86_64_zext_immediate_operand" "Z"))
                 (const_int 0)))
    (set (match_operand:DI 0 "register_operand" "=r")
-       (ior:DI (zero_extend:DI (match_dup 1)) (match_dup 2)))]
+       (any_or:DI (zero_extend:DI (match_dup 1)) (match_dup 2)))]
   "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (IOR, SImode, operands)"
-  "or{l}\t{%2, %k0|%k0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "SI")])
-
-(define_insn "*iorsi_3"
-  [(set (reg FLAGS_REG)
-       (compare (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
-                        (match_operand:SI 2 "general_operand" "g"))
-                (const_int 0)))
-   (clobber (match_scratch:SI 0 "=r"))]
-  "ix86_match_ccmode (insn, CCNOmode)
-   && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
-  "or{l}\t{%2, %0|%0, %2}"
+   && ix86_binary_operator_ok (<CODE>, SImode, operands)"
+  "<logicprefix>{l}\t{%2, %k0|%k0, %2}"
   [(set_attr "type" "alu")
    (set_attr "mode" "SI")])
 
-(define_expand "iorhi3"
-  [(set (match_operand:HI 0 "nonimmediate_operand" "")
-       (ior:HI (match_operand:HI 1 "nonimmediate_operand" "")
-               (match_operand:HI 2 "general_operand" "")))]
-  "TARGET_HIMODE_MATH"
-  "ix86_expand_binary_operator (IOR, HImode, operands); DONE;")
-
-(define_insn "*iorhi_1"
-  [(set (match_operand:HI 0 "nonimmediate_operand" "=r,m")
-       (ior:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0")
-               (match_operand:HI 2 "general_operand" "rmn,rn")))
-   (clobber (reg:CC FLAGS_REG))]
-  "ix86_binary_operator_ok (IOR, HImode, operands)"
-  "or{w}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "HI")])
-
-(define_insn "*iorhi_2"
-  [(set (reg FLAGS_REG)
-       (compare (ior:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0")
-                        (match_operand:HI 2 "general_operand" "rmn,rn"))
-                (const_int 0)))
-   (set (match_operand:HI 0 "nonimmediate_operand" "=r,rm")
-       (ior:HI (match_dup 1) (match_dup 2)))]
-  "ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (IOR, HImode, operands)"
-  "or{w}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "HI")])
-
-(define_insn "*iorhi_3"
-  [(set (reg FLAGS_REG)
-       (compare (ior:HI (match_operand:HI 1 "nonimmediate_operand" "%0")
-                        (match_operand:HI 2 "general_operand" "rmn"))
-                (const_int 0)))
-   (clobber (match_scratch:HI 0 "=r"))]
-  "ix86_match_ccmode (insn, CCNOmode)
-   && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
-  "or{w}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "HI")])
-
-(define_expand "iorqi3"
-  [(set (match_operand:QI 0 "nonimmediate_operand" "")
-       (ior:QI (match_operand:QI 1 "nonimmediate_operand" "")
-               (match_operand:QI 2 "general_operand" "")))]
-  "TARGET_QIMODE_MATH"
-  "ix86_expand_binary_operator (IOR, QImode, operands); DONE;")
-
-;; %%% Potential partial reg stall on alternative 2.  What to do?
-(define_insn "*iorqi_1"
-  [(set (match_operand:QI 0 "nonimmediate_operand" "=q,m,r")
-       (ior:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0,0")
-               (match_operand:QI 2 "general_operand" "qmn,qn,rn")))
-   (clobber (reg:CC FLAGS_REG))]
-  "ix86_binary_operator_ok (IOR, QImode, operands)"
-  "@
-   or{b}\t{%2, %0|%0, %2}
-   or{b}\t{%2, %0|%0, %2}
-   or{l}\t{%k2, %k0|%k0, %k2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "QI,QI,SI")])
-
-(define_insn "*iorqi_1_slp"
-  [(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+q,m"))
-       (ior:QI (match_dup 0)
-               (match_operand:QI 1 "general_operand" "qmn,qn")))
-   (clobber (reg:CC FLAGS_REG))]
-  "(! TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
-   && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
-  "or{b}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "alu1")
-   (set_attr "mode" "QI")])
-
-(define_insn "*iorqi_2"
+(define_insn "*<code>qi_2_slp"
   [(set (reg FLAGS_REG)
-       (compare (ior:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0")
-                        (match_operand:QI 2 "general_operand" "qmn,qn"))
-                (const_int 0)))
-   (set (match_operand:QI 0 "nonimmediate_operand" "=q,qm")
-       (ior:QI (match_dup 1) (match_dup 2)))]
-  "ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (IOR, QImode, operands)"
-  "or{b}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "QI")])
-
-(define_insn "*iorqi_2_slp"
-  [(set (reg FLAGS_REG)
-       (compare (ior:QI (match_operand:QI 0 "nonimmediate_operand" "+q,qm")
-                        (match_operand:QI 1 "general_operand" "qmn,qn"))
+       (compare (any_or:QI (match_operand:QI 0 "nonimmediate_operand" "+q,qm")
+                           (match_operand:QI 1 "general_operand" "qmn,qn"))
                 (const_int 0)))
    (set (strict_low_part (match_dup 0))
-       (ior:QI (match_dup 0) (match_dup 1)))]
-  "(! TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
+       (any_or:QI (match_dup 0) (match_dup 1)))]
+  "(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
    && ix86_match_ccmode (insn, CCNOmode)
    && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
-  "or{b}\t{%1, %0|%0, %1}"
+  "<logicprefix>{b}\t{%1, %0|%0, %1}"
   [(set_attr "type" "alu1")
    (set_attr "mode" "QI")])
 
-(define_insn "*iorqi_3"
+(define_insn "*<code><mode>_3"
   [(set (reg FLAGS_REG)
-       (compare (ior:QI (match_operand:QI 1 "nonimmediate_operand" "%0")
-                        (match_operand:QI 2 "general_operand" "qmn"))
+       (compare (any_or:SWI
+                 (match_operand:SWI 1 "nonimmediate_operand" "%0")
+                 (match_operand:SWI 2 "<general_operand>" "<g>"))
                 (const_int 0)))
-   (clobber (match_scratch:QI 0 "=q"))]
+   (clobber (match_scratch:SWI 0 "=<r>"))]
   "ix86_match_ccmode (insn, CCNOmode)
-   && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
-  "or{b}\t{%2, %0|%0, %2}"
+   && ix86_binary_operator_ok (<CODE>, <MODE>mode, operands)"
+  "<logicprefix>{<imodesuffix>}\t{%2, %0|%0, %2}"
   [(set_attr "type" "alu")
-   (set_attr "mode" "QI")])
+   (set_attr "mode" "<MODE>")])
 
-(define_insn "*iorqi_ext_0"
+(define_insn "*<code>qi_ext_0"
   [(set (zero_extract:SI (match_operand 0 "ext_register_operand" "=Q")
                         (const_int 8)
                         (const_int 8))
-       (ior:SI
+       (any_or:SI
          (zero_extract:SI
            (match_operand 1 "ext_register_operand" "0")
            (const_int 8)
            (const_int 8))
          (match_operand 2 "const_int_operand" "n")))
    (clobber (reg:CC FLAGS_REG))]
-  "(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))"
-  "or{b}\t{%2, %h0|%h0, %2}"
+  "!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun)"
+  "<logicprefix>{b}\t{%2, %h0|%h0, %2}"
   [(set_attr "type" "alu")
    (set_attr "length_immediate" "1")
    (set_attr "modrm" "1")
    (set_attr "mode" "QI")])
 
-(define_insn "*iorqi_ext_1"
+(define_insn "*<code>qi_ext_1_rex64"
   [(set (zero_extract:SI (match_operand 0 "ext_register_operand" "=Q")
                         (const_int 8)
                         (const_int 8))
-       (ior:SI
+       (any_or:SI
          (zero_extract:SI
            (match_operand 1 "ext_register_operand" "0")
            (const_int 8)
            (const_int 8))
          (zero_extend:SI
-           (match_operand:QI 2 "general_operand" "Qm"))))
+           (match_operand 2 "ext_register_operand" "Q"))))
    (clobber (reg:CC FLAGS_REG))]
-  "!TARGET_64BIT
+  "TARGET_64BIT
    && (!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))"
-  "or{b}\t{%2, %h0|%h0, %2}"
+  "<logicprefix>{b}\t{%2, %h0|%h0, %2}"
   [(set_attr "type" "alu")
    (set_attr "length_immediate" "0")
    (set_attr "mode" "QI")])
 
-(define_insn "*iorqi_ext_1_rex64"
+(define_insn "*<code>qi_ext_1"
   [(set (zero_extract:SI (match_operand 0 "ext_register_operand" "=Q")
                         (const_int 8)
                         (const_int 8))
-       (ior:SI
+       (any_or:SI
          (zero_extract:SI
            (match_operand 1 "ext_register_operand" "0")
            (const_int 8)
            (const_int 8))
          (zero_extend:SI
-           (match_operand 2 "ext_register_operand" "Q"))))
+           (match_operand:QI 2 "general_operand" "Qm"))))
    (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT
+  "!TARGET_64BIT
    && (!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))"
-  "or{b}\t{%2, %h0|%h0, %2}"
+  "<logicprefix>{b}\t{%2, %h0|%h0, %2}"
   [(set_attr "type" "alu")
    (set_attr "length_immediate" "0")
    (set_attr "mode" "QI")])
 
-(define_insn "*iorqi_ext_2"
+(define_insn "*<code>qi_ext_2"
   [(set (zero_extract:SI (match_operand 0 "ext_register_operand" "=Q")
                         (const_int 8)
                         (const_int 8))
-       (ior:SI
+       (any_or:SI
          (zero_extract:SI (match_operand 1 "ext_register_operand" "0")
                           (const_int 8)
                           (const_int 8))
                           (const_int 8)
                           (const_int 8))))
    (clobber (reg:CC FLAGS_REG))]
-  "(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))"
-  "ior{b}\t{%h2, %h0|%h0, %h2}"
+  "!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun)"
+  "<logicprefix>{b}\t{%h2, %h0|%h0, %h2}"
   [(set_attr "type" "alu")
    (set_attr "length_immediate" "0")
    (set_attr "mode" "QI")])
 
 (define_split
   [(set (match_operand 0 "register_operand" "")
-       (ior (match_operand 1 "register_operand" "")
-            (match_operand 2 "const_int_operand" "")))
+       (any_or (match_operand 1 "register_operand" "")
+               (match_operand 2 "const_int_operand" "")))
    (clobber (reg:CC FLAGS_REG))]
    "reload_completed
     && QI_REG_P (operands[0])
     && !(INTVAL (operands[2]) & ~(255 << 8))
     && GET_MODE (operands[0]) != QImode"
   [(parallel [(set (zero_extract:SI (match_dup 0) (const_int 8) (const_int 8))
-                  (ior:SI (zero_extract:SI (match_dup 1)
-                                           (const_int 8) (const_int 8))
-                          (match_dup 2)))
+                  (any_or:SI (zero_extract:SI (match_dup 1)
+                                              (const_int 8) (const_int 8))
+                             (match_dup 2)))
              (clobber (reg:CC FLAGS_REG))])]
   "operands[0] = gen_lowpart (SImode, operands[0]);
    operands[1] = gen_lowpart (SImode, operands[1]);
 ;; profitable when 7th bit is set.
 (define_split
   [(set (match_operand 0 "register_operand" "")
-       (ior (match_operand 1 "general_operand" "")
-            (match_operand 2 "const_int_operand" "")))
+       (any_or (match_operand 1 "general_operand" "")
+               (match_operand 2 "const_int_operand" "")))
    (clobber (reg:CC FLAGS_REG))]
    "reload_completed
     && ANY_QI_REG_P (operands[0])
     && (INTVAL (operands[2]) & 128)
     && GET_MODE (operands[0]) != QImode"
   [(parallel [(set (strict_low_part (match_dup 0))
-                  (ior:QI (match_dup 1)
-                          (match_dup 2)))
+                  (any_or:QI (match_dup 1)
+                             (match_dup 2)))
              (clobber (reg:CC FLAGS_REG))])]
   "operands[0] = gen_lowpart (QImode, operands[0]);
    operands[1] = gen_lowpart (QImode, operands[1]);
    operands[2] = gen_lowpart (QImode, operands[2]);")
-\f
-;; Logical XOR instructions
-
-;; %%% This used to optimize known byte-wide and operations to memory.
-;; If this is considered useful, it should be done with splitters.
-
-(define_expand "xordi3"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "")
-       (xor:DI (match_operand:DI 1 "nonimmediate_operand" "")
-               (match_operand:DI 2 "x86_64_general_operand" "")))]
-  "TARGET_64BIT"
-  "ix86_expand_binary_operator (XOR, DImode, operands); DONE;")
-
-(define_insn "*xordi_1_rex64"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "=rm,r")
-       (xor:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
-               (match_operand:DI 2 "x86_64_general_operand" "re,rm")))
-   (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT
-   && ix86_binary_operator_ok (XOR, DImode, operands)"
-  "xor{q}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "DI")])
-
-(define_insn "*xordi_2_rex64"
-  [(set (reg FLAGS_REG)
-       (compare (xor:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
-                        (match_operand:DI 2 "x86_64_general_operand" "rem,re"))
-                (const_int 0)))
-   (set (match_operand:DI 0 "nonimmediate_operand" "=r,rm")
-       (xor:DI (match_dup 1) (match_dup 2)))]
-  "TARGET_64BIT
-   && ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (XOR, DImode, operands)"
-  "xor{q}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "DI")])
 
-(define_insn "*xordi_3_rex64"
-  [(set (reg FLAGS_REG)
-       (compare (xor:DI (match_operand:DI 1 "nonimmediate_operand" "%0")
-                        (match_operand:DI 2 "x86_64_general_operand" "rem"))
-                (const_int 0)))
-   (clobber (match_scratch:DI 0 "=r"))]
-  "TARGET_64BIT
-   && ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (XOR, DImode, operands)"
-  "xor{q}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "DI")])
-
-(define_expand "xorsi3"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "")
-       (xor:SI (match_operand:SI 1 "nonimmediate_operand" "")
-               (match_operand:SI 2 "general_operand" "")))]
-  ""
-  "ix86_expand_binary_operator (XOR, SImode, operands); DONE;")
-
-(define_insn "*xorsi_1"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "=rm,r")
-       (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-               (match_operand:SI 2 "general_operand" "ri,rm")))
-   (clobber (reg:CC FLAGS_REG))]
-  "ix86_binary_operator_ok (XOR, SImode, operands)"
-  "xor{l}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "SI")])
-
-;; See comment for addsi_1_zext why we do use nonimmediate_operand
-;; Add speccase for immediates
-(define_insn "*xorsi_1_zext"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (zero_extend:DI
-         (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
-                 (match_operand:SI 2 "general_operand" "g"))))
-   (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT && ix86_binary_operator_ok (XOR, SImode, operands)"
-  "xor{l}\t{%2, %k0|%k0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "SI")])
-
-(define_insn "*xorsi_1_zext_imm"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (xor:DI (zero_extend:DI (match_operand:SI 1 "register_operand" "%0"))
-               (match_operand:DI 2 "x86_64_zext_immediate_operand" "Z")))
-   (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT && ix86_binary_operator_ok (XOR, SImode, operands)"
-  "xor{l}\t{%2, %k0|%k0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "SI")])
-
-(define_insn "*xorsi_2"
-  [(set (reg FLAGS_REG)
-       (compare (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                        (match_operand:SI 2 "general_operand" "g,ri"))
-                (const_int 0)))
-   (set (match_operand:SI 0 "nonimmediate_operand" "=r,rm")
-       (xor:SI (match_dup 1) (match_dup 2)))]
-  "ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (XOR, SImode, operands)"
-  "xor{l}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "SI")])
-
-;; See comment for addsi_1_zext why we do use nonimmediate_operand
-;; ??? Special case for immediate operand is missing - it is tricky.
-(define_insn "*xorsi_2_zext"
-  [(set (reg FLAGS_REG)
-       (compare (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
-                        (match_operand:SI 2 "general_operand" "g"))
-                (const_int 0)))
-   (set (match_operand:DI 0 "register_operand" "=r")
-       (zero_extend:DI (xor:SI (match_dup 1) (match_dup 2))))]
-  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (XOR, SImode, operands)"
-  "xor{l}\t{%2, %k0|%k0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "SI")])
-
-(define_insn "*xorsi_2_zext_imm"
-  [(set (reg FLAGS_REG)
-       (compare (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
-                        (match_operand 2 "x86_64_zext_immediate_operand" "Z"))
-                (const_int 0)))
-   (set (match_operand:DI 0 "register_operand" "=r")
-       (xor:DI (zero_extend:DI (match_dup 1)) (match_dup 2)))]
-  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (XOR, SImode, operands)"
-  "xor{l}\t{%2, %k0|%k0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "SI")])
-
-(define_insn "*xorsi_3"
-  [(set (reg FLAGS_REG)
-       (compare (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
-                        (match_operand:SI 2 "general_operand" "g"))
-                (const_int 0)))
-   (clobber (match_scratch:SI 0 "=r"))]
-  "ix86_match_ccmode (insn, CCNOmode)
-   && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
-  "xor{l}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "SI")])
-
-(define_expand "xorhi3"
-  [(set (match_operand:HI 0 "nonimmediate_operand" "")
-       (xor:HI (match_operand:HI 1 "nonimmediate_operand" "")
-               (match_operand:HI 2 "general_operand" "")))]
-  "TARGET_HIMODE_MATH"
-  "ix86_expand_binary_operator (XOR, HImode, operands); DONE;")
-
-(define_insn "*xorhi_1"
-  [(set (match_operand:HI 0 "nonimmediate_operand" "=r,m")
-       (xor:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0")
-               (match_operand:HI 2 "general_operand" "rmn,rn")))
-   (clobber (reg:CC FLAGS_REG))]
-  "ix86_binary_operator_ok (XOR, HImode, operands)"
-  "xor{w}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "HI")])
-
-(define_insn "*xorhi_2"
-  [(set (reg FLAGS_REG)
-       (compare (xor:HI (match_operand:HI 1 "nonimmediate_operand" "%0,0")
-                        (match_operand:HI 2 "general_operand" "rmn,rn"))
-                (const_int 0)))
-   (set (match_operand:HI 0 "nonimmediate_operand" "=r,rm")
-       (xor:HI (match_dup 1) (match_dup 2)))]
-  "ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (XOR, HImode, operands)"
-  "xor{w}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "HI")])
-
-(define_insn "*xorhi_3"
-  [(set (reg FLAGS_REG)
-       (compare (xor:HI (match_operand:HI 1 "nonimmediate_operand" "%0")
-                        (match_operand:HI 2 "general_operand" "rmn"))
-                (const_int 0)))
-   (clobber (match_scratch:HI 0 "=r"))]
-  "ix86_match_ccmode (insn, CCNOmode)
-   && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
-  "xor{w}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "HI")])
-
-(define_expand "xorqi3"
-  [(set (match_operand:QI 0 "nonimmediate_operand" "")
-       (xor:QI (match_operand:QI 1 "nonimmediate_operand" "")
-               (match_operand:QI 2 "general_operand" "")))]
-  "TARGET_QIMODE_MATH"
-  "ix86_expand_binary_operator (XOR, QImode, operands); DONE;")
-
-;; %%% Potential partial reg stall on alternative 2.  What to do?
-(define_insn "*xorqi_1"
-  [(set (match_operand:QI 0 "nonimmediate_operand" "=q,m,r")
-       (xor:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0,0")
-               (match_operand:QI 2 "general_operand" "qmn,qn,rn")))
-   (clobber (reg:CC FLAGS_REG))]
-  "ix86_binary_operator_ok (XOR, QImode, operands)"
-  "@
-   xor{b}\t{%2, %0|%0, %2}
-   xor{b}\t{%2, %0|%0, %2}
-   xor{l}\t{%k2, %k0|%k0, %k2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "QI,QI,SI")])
-
-(define_insn "*xorqi_1_slp"
-  [(set (strict_low_part (match_operand:QI 0 "nonimmediate_operand" "+qm,q"))
-       (xor:QI (match_dup 0)
-               (match_operand:QI 1 "general_operand" "qn,qmn")))
-   (clobber (reg:CC FLAGS_REG))]
-  "(! TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
-   && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
-  "xor{b}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "alu1")
-   (set_attr "mode" "QI")])
-
-(define_insn "*xorqi_ext_0"
-  [(set (zero_extract:SI (match_operand 0 "ext_register_operand" "=Q")
-                        (const_int 8)
-                        (const_int 8))
-       (xor:SI
-         (zero_extract:SI
-           (match_operand 1 "ext_register_operand" "0")
-           (const_int 8)
-           (const_int 8))
-         (match_operand 2 "const_int_operand" "n")))
-   (clobber (reg:CC FLAGS_REG))]
-  "(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))"
-  "xor{b}\t{%2, %h0|%h0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "length_immediate" "1")
-   (set_attr "modrm" "1")
-   (set_attr "mode" "QI")])
-
-(define_insn "*xorqi_ext_1"
-  [(set (zero_extract:SI (match_operand 0 "ext_register_operand" "=Q")
-                        (const_int 8)
-                        (const_int 8))
-       (xor:SI
-         (zero_extract:SI
-           (match_operand 1 "ext_register_operand" "0")
-           (const_int 8)
-           (const_int 8))
-         (zero_extend:SI
-           (match_operand:QI 2 "general_operand" "Qm"))))
-   (clobber (reg:CC FLAGS_REG))]
-  "!TARGET_64BIT
-   && (!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))"
-  "xor{b}\t{%2, %h0|%h0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "length_immediate" "0")
-   (set_attr "mode" "QI")])
-
-(define_insn "*xorqi_ext_1_rex64"
-  [(set (zero_extract:SI (match_operand 0 "ext_register_operand" "=Q")
-                        (const_int 8)
-                        (const_int 8))
-       (xor:SI
-         (zero_extract:SI
-           (match_operand 1 "ext_register_operand" "0")
-           (const_int 8)
-           (const_int 8))
-         (zero_extend:SI
-           (match_operand 2 "ext_register_operand" "Q"))))
-   (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT
-   && (!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))"
-  "xor{b}\t{%2, %h0|%h0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "length_immediate" "0")
-   (set_attr "mode" "QI")])
-
-(define_insn "*xorqi_ext_2"
-  [(set (zero_extract:SI (match_operand 0 "ext_register_operand" "=Q")
-                        (const_int 8)
-                        (const_int 8))
-       (xor:SI
-         (zero_extract:SI (match_operand 1 "ext_register_operand" "0")
-                          (const_int 8)
+(define_expand "xorqi_cc_ext_1"
+  [(parallel [
+     (set (reg:CCNO FLAGS_REG)
+         (compare:CCNO
+           (xor:SI
+             (zero_extract:SI
+               (match_operand 1 "ext_register_operand" "")
+               (const_int 8)
+               (const_int 8))
+             (match_operand:QI 2 "general_operand" ""))
+           (const_int 0)))
+     (set (zero_extract:SI (match_operand 0 "ext_register_operand" "")
+                          (const_int 8)
                           (const_int 8))
-         (zero_extract:SI (match_operand 2 "ext_register_operand" "Q")
-                          (const_int 8)
-                          (const_int 8))))
-   (clobber (reg:CC FLAGS_REG))]
-  "(!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))"
-  "xor{b}\t{%h2, %h0|%h0, %h2}"
-  [(set_attr "type" "alu")
-   (set_attr "length_immediate" "0")
-   (set_attr "mode" "QI")])
-
-(define_insn "*xorqi_cc_1"
-  [(set (reg FLAGS_REG)
-       (compare
-         (xor:QI (match_operand:QI 1 "nonimmediate_operand" "%0,0")
-                 (match_operand:QI 2 "general_operand" "qmn,qn"))
-         (const_int 0)))
-   (set (match_operand:QI 0 "nonimmediate_operand" "=q,qm")
-       (xor:QI (match_dup 1) (match_dup 2)))]
-  "ix86_match_ccmode (insn, CCNOmode)
-   && ix86_binary_operator_ok (XOR, QImode, operands)"
-  "xor{b}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "QI")])
-
-(define_insn "*xorqi_2_slp"
-  [(set (reg FLAGS_REG)
-       (compare (xor:QI (match_operand:QI 0 "nonimmediate_operand" "+q,qm")
-                        (match_operand:QI 1 "general_operand" "qmn,qn"))
-                (const_int 0)))
-   (set (strict_low_part (match_dup 0))
-       (xor:QI (match_dup 0) (match_dup 1)))]
-  "(! TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
-   && ix86_match_ccmode (insn, CCNOmode)
-   && !(MEM_P (operands[0]) && MEM_P (operands[1]))"
-  "xor{b}\t{%1, %0|%0, %1}"
-  [(set_attr "type" "alu1")
-   (set_attr "mode" "QI")])
-
-(define_insn "*xorqi_cc_2"
-  [(set (reg FLAGS_REG)
-       (compare
-         (xor:QI (match_operand:QI 1 "nonimmediate_operand" "%0")
-                 (match_operand:QI 2 "general_operand" "qmn"))
-         (const_int 0)))
-   (clobber (match_scratch:QI 0 "=q"))]
-  "ix86_match_ccmode (insn, CCNOmode)
-   && !(MEM_P (operands[1]) && MEM_P (operands[2]))"
-  "xor{b}\t{%2, %0|%0, %2}"
-  [(set_attr "type" "alu")
-   (set_attr "mode" "QI")])
+         (xor:SI
+           (zero_extract:SI
+            (match_dup 1)
+            (const_int 8)
+            (const_int 8))
+           (match_dup 2)))])]
+  ""
+  "")
 
-(define_insn "*xorqi_cc_ext_1"
+(define_insn "*xorqi_cc_ext_1_rex64"
   [(set (reg FLAGS_REG)
        (compare
          (xor:SI
              (match_operand 1 "ext_register_operand" "0")
              (const_int 8)
              (const_int 8))
-           (match_operand:QI 2 "general_operand" "qmn"))
+           (match_operand:QI 2 "nonmemory_operand" "Qn"))
          (const_int 0)))
-   (set (zero_extract:SI (match_operand 0 "ext_register_operand" "=q")
+   (set (zero_extract:SI (match_operand 0 "ext_register_operand" "=Q")
                         (const_int 8)
                         (const_int 8))
        (xor:SI
-         (zero_extract:SI (match_dup 1) (const_int 8) (const_int 8))
+         (zero_extract:SI
+          (match_dup 1)
+          (const_int 8)
+          (const_int 8))
          (match_dup 2)))]
-  "!TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)"
+  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)"
   "xor{b}\t{%2, %h0|%h0, %2}"
   [(set_attr "type" "alu")
    (set_attr "modrm" "1")
    (set_attr "mode" "QI")])
 
-(define_insn "*xorqi_cc_ext_1_rex64"
+(define_insn "*xorqi_cc_ext_1"
   [(set (reg FLAGS_REG)
        (compare
          (xor:SI
              (match_operand 1 "ext_register_operand" "0")
              (const_int 8)
              (const_int 8))
-           (match_operand:QI 2 "nonmemory_operand" "Qn"))
+           (match_operand:QI 2 "general_operand" "qmn"))
          (const_int 0)))
-   (set (zero_extract:SI (match_operand 0 "ext_register_operand" "=Q")
+   (set (zero_extract:SI (match_operand 0 "ext_register_operand" "=q")
                         (const_int 8)
                         (const_int 8))
        (xor:SI
-         (zero_extract:SI (match_dup 1) (const_int 8) (const_int 8))
+         (zero_extract:SI
+          (match_dup 1)
+          (const_int 8)
+          (const_int 8))
          (match_dup 2)))]
-  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)"
+  "!TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)"
   "xor{b}\t{%2, %h0|%h0, %2}"
   [(set_attr "type" "alu")
    (set_attr "modrm" "1")
    (set_attr "mode" "QI")])
-
-(define_expand "xorqi_cc_ext_1"
-  [(parallel [
-     (set (reg:CCNO FLAGS_REG)
-         (compare:CCNO
-           (xor:SI
-             (zero_extract:SI
-               (match_operand 1 "ext_register_operand" "")
-               (const_int 8)
-               (const_int 8))
-             (match_operand:QI 2 "general_operand" ""))
-           (const_int 0)))
-     (set (zero_extract:SI (match_operand 0 "ext_register_operand" "")
-                          (const_int 8)
-                          (const_int 8))
-         (xor:SI
-           (zero_extract:SI (match_dup 1) (const_int 8) (const_int 8))
-           (match_dup 2)))])]
-  ""
-  "")
-
-(define_split
-  [(set (match_operand 0 "register_operand" "")
-       (xor (match_operand 1 "register_operand" "")
-            (match_operand 2 "const_int_operand" "")))
-   (clobber (reg:CC FLAGS_REG))]
-   "reload_completed
-    && QI_REG_P (operands[0])
-    && (!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
-    && !(INTVAL (operands[2]) & ~(255 << 8))
-    && GET_MODE (operands[0]) != QImode"
-  [(parallel [(set (zero_extract:SI (match_dup 0) (const_int 8) (const_int 8))
-                  (xor:SI (zero_extract:SI (match_dup 1)
-                                           (const_int 8) (const_int 8))
-                          (match_dup 2)))
-             (clobber (reg:CC FLAGS_REG))])]
-  "operands[0] = gen_lowpart (SImode, operands[0]);
-   operands[1] = gen_lowpart (SImode, operands[1]);
-   operands[2] = gen_int_mode ((INTVAL (operands[2]) >> 8) & 0xff, SImode);")
-
-;; Since XOR can be encoded with sign extended immediate, this is only
-;; profitable when 7th bit is set.
-(define_split
-  [(set (match_operand 0 "register_operand" "")
-       (xor (match_operand 1 "general_operand" "")
-            (match_operand 2 "const_int_operand" "")))
-   (clobber (reg:CC FLAGS_REG))]
-   "reload_completed
-    && ANY_QI_REG_P (operands[0])
-    && (!TARGET_PARTIAL_REG_STALL || optimize_function_for_size_p (cfun))
-    && !(INTVAL (operands[2]) & ~255)
-    && (INTVAL (operands[2]) & 128)
-    && GET_MODE (operands[0]) != QImode"
-  [(parallel [(set (strict_low_part (match_dup 0))
-                  (xor:QI (match_dup 1)
-                          (match_dup 2)))
-             (clobber (reg:CC FLAGS_REG))])]
-  "operands[0] = gen_lowpart (QImode, operands[0]);
-   operands[1] = gen_lowpart (QImode, operands[1]);
-   operands[2] = gen_lowpart (QImode, operands[2]);")
 \f
 ;; Negation instructions
 
-(define_expand "negti2"
-  [(set (match_operand:TI 0 "nonimmediate_operand" "")
-       (neg:TI (match_operand:TI 1 "nonimmediate_operand" "")))]
-  "TARGET_64BIT"
-  "ix86_expand_unary_operator (NEG, TImode, operands); DONE;")
-
-(define_insn "*negti2_1"
-  [(set (match_operand:TI 0 "nonimmediate_operand" "=ro")
-       (neg:TI (match_operand:TI 1 "nonimmediate_operand" "0")))
-   (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT
-   && ix86_unary_operator_ok (NEG, TImode, operands)"
-  "#")
-
-(define_split
-  [(set (match_operand:TI 0 "nonimmediate_operand" "")
-       (neg:TI (match_operand:TI 1 "nonimmediate_operand" "")))
-   (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT && reload_completed"
-  [(parallel
-    [(set (reg:CCZ FLAGS_REG)
-         (compare:CCZ (neg:DI (match_dup 1)) (const_int 0)))
-     (set (match_dup 0) (neg:DI (match_dup 1)))])
-   (parallel
-    [(set (match_dup 2)
-         (plus:DI (plus:DI (ltu:DI (reg:CC FLAGS_REG) (const_int 0))
-                           (match_dup 3))
-                  (const_int 0)))
-     (clobber (reg:CC FLAGS_REG))])
-   (parallel
-    [(set (match_dup 2)
-         (neg:DI (match_dup 2)))
-     (clobber (reg:CC FLAGS_REG))])]
-  "split_ti (&operands[0], 2, &operands[0], &operands[2]);")
-
-(define_expand "negdi2"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "")
-       (neg:DI (match_operand:DI 1 "nonimmediate_operand" "")))]
+(define_expand "neg<mode>2"
+  [(set (match_operand:SDWIM 0 "nonimmediate_operand" "")
+       (neg:SDWIM (match_operand:SDWIM 1 "nonimmediate_operand" "")))]
   ""
-  "ix86_expand_unary_operator (NEG, DImode, operands); DONE;")
+  "ix86_expand_unary_operator (NEG, <MODE>mode, operands); DONE;")
 
-(define_insn "*negdi2_1"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "=ro")
-       (neg:DI (match_operand:DI 1 "general_operand" "0")))
-   (clobber (reg:CC FLAGS_REG))]
-  "!TARGET_64BIT
-   && ix86_unary_operator_ok (NEG, DImode, operands)"
-  "#")
-
-(define_split
-  [(set (match_operand:DI 0 "nonimmediate_operand" "")
-       (neg:DI (match_operand:DI 1 "general_operand" "")))
+(define_insn_and_split "*neg<dwi>2_doubleword"
+  [(set (match_operand:<DWI> 0 "nonimmediate_operand" "=ro")
+       (neg:<DWI> (match_operand:<DWI> 1 "nonimmediate_operand" "0")))
    (clobber (reg:CC FLAGS_REG))]
-  "!TARGET_64BIT && reload_completed"
+  "ix86_unary_operator_ok (NEG, <DWI>mode, operands)"
+  "#"
+  "reload_completed"
   [(parallel
     [(set (reg:CCZ FLAGS_REG)
-         (compare:CCZ (neg:SI (match_dup 1)) (const_int 0)))
-     (set (match_dup 0) (neg:SI (match_dup 1)))])
+         (compare:CCZ (neg:DWIH (match_dup 1)) (const_int 0)))
+     (set (match_dup 0) (neg:DWIH (match_dup 1)))])
    (parallel
     [(set (match_dup 2)
-         (plus:SI (plus:SI (ltu:SI (reg:CC FLAGS_REG) (const_int 0))
-                           (match_dup 3))
-                  (const_int 0)))
+         (plus:DWIH (match_dup 3)
+                    (plus:DWIH (ltu:DWIH (reg:CC FLAGS_REG) (const_int 0))
+                               (const_int 0))))
      (clobber (reg:CC FLAGS_REG))])
    (parallel
     [(set (match_dup 2)
-         (neg:SI (match_dup 2)))
+         (neg:DWIH (match_dup 2)))
      (clobber (reg:CC FLAGS_REG))])]
-  "split_di (&operands[0], 2, &operands[0], &operands[2]);");
+  "split_<dwi> (&operands[0], 2, &operands[0], &operands[2]);")
 
-(define_insn "*negdi2_1_rex64"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "=rm")
-       (neg:DI (match_operand:DI 1 "nonimmediate_operand" "0")))
+(define_insn "*neg<mode>2_1"
+  [(set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m")
+       (neg:SWI (match_operand:SWI 1 "nonimmediate_operand" "0")))
    (clobber (reg:CC FLAGS_REG))]
-  "TARGET_64BIT && ix86_unary_operator_ok (NEG, DImode, operands)"
-  "neg{q}\t%0"
+  "ix86_unary_operator_ok (NEG, <MODE>mode, operands)"
+  "neg{<imodesuffix>}\t%0"
   [(set_attr "type" "negnot")
-   (set_attr "mode" "DI")])
-
-;; The problem with neg is that it does not perform (compare x 0),
-;; it really performs (compare 0 x), which leaves us with the zero
-;; flag being the only useful item.
-
-(define_insn "*negdi2_cmpz_rex64"
-  [(set (reg:CCZ FLAGS_REG)
-       (compare:CCZ (neg:DI (match_operand:DI 1 "nonimmediate_operand" "0"))
-                    (const_int 0)))
-   (set (match_operand:DI 0 "nonimmediate_operand" "=rm")
-       (neg:DI (match_dup 1)))]
-  "TARGET_64BIT && ix86_unary_operator_ok (NEG, DImode, operands)"
-  "neg{q}\t%0"
-  [(set_attr "type" "negnot")
-   (set_attr "mode" "DI")])
-
-
-(define_expand "negsi2"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "")
-       (neg:SI (match_operand:SI 1 "nonimmediate_operand" "")))]
-  ""
-  "ix86_expand_unary_operator (NEG, SImode, operands); DONE;")
-
-(define_insn "*negsi2_1"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "=rm")
-       (neg:SI (match_operand:SI 1 "nonimmediate_operand" "0")))
-   (clobber (reg:CC FLAGS_REG))]
-  "ix86_unary_operator_ok (NEG, SImode, operands)"
-  "neg{l}\t%0"
-  [(set_attr "type" "negnot")
-   (set_attr "mode" "SI")])
+   (set_attr "mode" "<MODE>")])
 
 ;; Combine is quite creative about this pattern.
 (define_insn "*negsi2_1_zext"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (lshiftrt:DI (neg:DI (ashift:DI (match_operand:DI 1 "register_operand" "0")
-                                       (const_int 32)))
-                    (const_int 32)))
+       (lshiftrt:DI
+         (neg:DI (ashift:DI (match_operand:DI 1 "register_operand" "0")
+                            (const_int 32)))
+       (const_int 32)))
    (clobber (reg:CC FLAGS_REG))]
   "TARGET_64BIT && ix86_unary_operator_ok (NEG, SImode, operands)"
   "neg{l}\t%k0"
 ;; it really performs (compare 0 x), which leaves us with the zero
 ;; flag being the only useful item.
 
-(define_insn "*negsi2_cmpz"
+(define_insn "*neg<mode>2_cmpz"
   [(set (reg:CCZ FLAGS_REG)
-       (compare:CCZ (neg:SI (match_operand:SI 1 "nonimmediate_operand" "0"))
-                    (const_int 0)))
-   (set (match_operand:SI 0 "nonimmediate_operand" "=rm")
-       (neg:SI (match_dup 1)))]
-  "ix86_unary_operator_ok (NEG, SImode, operands)"
-  "neg{l}\t%0"
+       (compare:CCZ
+         (neg:SWI (match_operand:SWI 1 "nonimmediate_operand" "0"))
+                  (const_int 0)))
+   (set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m")
+       (neg:SWI (match_dup 1)))]
+  "ix86_unary_operator_ok (NEG, <MODE>mode, operands)"
+  "neg{<imodesuffix>}\t%0"
   [(set_attr "type" "negnot")
-   (set_attr "mode" "SI")])
+   (set_attr "mode" "<MODE>")])
 
 (define_insn "*negsi2_cmpz_zext"
   [(set (reg:CCZ FLAGS_REG)
-       (compare:CCZ (lshiftrt:DI
-                      (neg:DI (ashift:DI
-                                (match_operand:DI 1 "register_operand" "0")
-                                (const_int 32)))
-                      (const_int 32))
-                    (const_int 0)))
+       (compare:CCZ
+         (lshiftrt:DI
+           (neg:DI (ashift:DI
+                     (match_operand:DI 1 "register_operand" "0")
+                     (const_int 32)))
+           (const_int 32))
+         (const_int 0)))
    (set (match_operand:DI 0 "register_operand" "=r")
        (lshiftrt:DI (neg:DI (ashift:DI (match_dup 1)
                                        (const_int 32)))
   [(set_attr "type" "negnot")
    (set_attr "mode" "SI")])
 
-(define_expand "neghi2"
-  [(set (match_operand:HI 0 "nonimmediate_operand" "")
-       (neg:HI (match_operand:HI 1 "nonimmediate_operand" "")))]
-  "TARGET_HIMODE_MATH"
-  "ix86_expand_unary_operator (NEG, HImode, operands); DONE;")
-
-(define_insn "*neghi2_1"
-  [(set (match_operand:HI 0 "nonimmediate_operand" "=rm")
-       (neg:HI (match_operand:HI 1 "nonimmediate_operand" "0")))
-   (clobber (reg:CC FLAGS_REG))]
-  "ix86_unary_operator_ok (NEG, HImode, operands)"
-  "neg{w}\t%0"
-  [(set_attr "type" "negnot")
-   (set_attr "mode" "HI")])
-
-(define_insn "*neghi2_cmpz"
-  [(set (reg:CCZ FLAGS_REG)
-       (compare:CCZ (neg:HI (match_operand:HI 1 "nonimmediate_operand" "0"))
-                    (const_int 0)))
-   (set (match_operand:HI 0 "nonimmediate_operand" "=rm")
-       (neg:HI (match_dup 1)))]
-  "ix86_unary_operator_ok (NEG, HImode, operands)"
-  "neg{w}\t%0"
-  [(set_attr "type" "negnot")
-   (set_attr "mode" "HI")])
-
-(define_expand "negqi2"
-  [(set (match_operand:QI 0 "nonimmediate_operand" "")
-       (neg:QI (match_operand:QI 1 "nonimmediate_operand" "")))]
-  "TARGET_QIMODE_MATH"
-  "ix86_expand_unary_operator (NEG, QImode, operands); DONE;")
-
-(define_insn "*negqi2_1"
-  [(set (match_operand:QI 0 "nonimmediate_operand" "=qm")
-       (neg:QI (match_operand:QI 1 "nonimmediate_operand" "0")))
-   (clobber (reg:CC FLAGS_REG))]
-  "ix86_unary_operator_ok (NEG, QImode, operands)"
-  "neg{b}\t%0"
-  [(set_attr "type" "negnot")
-   (set_attr "mode" "QI")])
-
-(define_insn "*negqi2_cmpz"
-  [(set (reg:CCZ FLAGS_REG)
-       (compare:CCZ (neg:QI (match_operand:QI 1 "nonimmediate_operand" "0"))
-                    (const_int 0)))
-   (set (match_operand:QI 0 "nonimmediate_operand" "=qm")
-       (neg:QI (match_dup 1)))]
-  "ix86_unary_operator_ok (NEG, QImode, operands)"
-  "neg{b}\t%0"
-  [(set_attr "type" "negnot")
-   (set_attr "mode" "QI")])
-
 ;; Changing of sign for FP values is doable using integer unit too.
 
 (define_expand "<code><mode>2"
          UNSPEC_COPYSIGN))
    (clobber (match_scratch:<CSGNVMODE> 1 "=x,x,x,x,x"))]
   "(SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
-   || (TARGET_SSE2 && (<MODE>mode == TFmode))"
-  "#")
-
-(define_split
-  [(set (match_operand:CSGNMODE 0 "register_operand" "")
-       (unspec:CSGNMODE
-         [(match_operand:CSGNMODE 2 "register_operand" "")
-          (match_operand:CSGNMODE 3 "register_operand" "")
-          (match_operand:<CSGNVMODE> 4 "" "")
-          (match_operand:<CSGNVMODE> 5 "" "")]
-         UNSPEC_COPYSIGN))
-   (clobber (match_scratch:<CSGNVMODE> 1 ""))]
-  "((SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
-    || (TARGET_SSE2 && (<MODE>mode == TFmode)))
-   && reload_completed"
-  [(const_int 0)]
-{
-  ix86_split_copysign_var (operands);
-  DONE;
-})
-\f
-;; One complement instructions
-
-(define_expand "one_cmpldi2"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "")
-       (not:DI (match_operand:DI 1 "nonimmediate_operand" "")))]
-  "TARGET_64BIT"
-  "ix86_expand_unary_operator (NOT, DImode, operands); DONE;")
-
-(define_insn "*one_cmpldi2_1_rex64"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "=rm")
-       (not:DI (match_operand:DI 1 "nonimmediate_operand" "0")))]
-  "TARGET_64BIT && ix86_unary_operator_ok (NOT, DImode, operands)"
-  "not{q}\t%0"
-  [(set_attr "type" "negnot")
-   (set_attr "mode" "DI")])
-
-(define_insn "*one_cmpldi2_2_rex64"
-  [(set (reg FLAGS_REG)
-       (compare (not:DI (match_operand:DI 1 "nonimmediate_operand" "0"))
-                (const_int 0)))
-   (set (match_operand:DI 0 "nonimmediate_operand" "=rm")
-       (not:DI (match_dup 1)))]
-  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)
-   && ix86_unary_operator_ok (NOT, DImode, operands)"
-  "#"
-  [(set_attr "type" "alu1")
-   (set_attr "mode" "DI")])
-
-(define_split
-  [(set (match_operand 0 "flags_reg_operand" "")
-       (match_operator 2 "compare_operator"
-         [(not:DI (match_operand:DI 3 "nonimmediate_operand" ""))
-          (const_int 0)]))
-   (set (match_operand:DI 1 "nonimmediate_operand" "")
-       (not:DI (match_dup 3)))]
-  "TARGET_64BIT && ix86_match_ccmode (insn, CCNOmode)"
-  [(parallel [(set (match_dup 0)
-                  (match_op_dup 2
-                    [(xor:DI (match_dup 3) (const_int -1))
-                     (const_int 0)]))
-             (set (match_dup 1)
-                  (xor:DI (match_dup 3) (const_int -1)))])]
-  "")
+   || (TARGET_SSE2 && (<MODE>mode == TFmode))"
+  "#")
 
-(define_expand "one_cmplsi2"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "")
-       (not:SI (match_operand:SI 1 "nonimmediate_operand" "")))]
+(define_split
+  [(set (match_operand:CSGNMODE 0 "register_operand" "")
+       (unspec:CSGNMODE
+         [(match_operand:CSGNMODE 2 "register_operand" "")
+          (match_operand:CSGNMODE 3 "register_operand" "")
+          (match_operand:<CSGNVMODE> 4 "" "")
+          (match_operand:<CSGNVMODE> 5 "" "")]
+         UNSPEC_COPYSIGN))
+   (clobber (match_scratch:<CSGNVMODE> 1 ""))]
+  "((SSE_FLOAT_MODE_P (<MODE>mode) && TARGET_SSE_MATH)
+    || (TARGET_SSE2 && (<MODE>mode == TFmode)))
+   && reload_completed"
+  [(const_int 0)]
+{
+  ix86_split_copysign_var (operands);
+  DONE;
+})
+\f
+;; One complement instructions
+
+(define_expand "one_cmpl<mode>2"
+  [(set (match_operand:SWIM 0 "nonimmediate_operand" "")
+       (not:SWIM (match_operand:SWIM 1 "nonimmediate_operand" "")))]
   ""
-  "ix86_expand_unary_operator (NOT, SImode, operands); DONE;")
+  "ix86_expand_unary_operator (NOT, <MODE>mode, operands); DONE;")
 
-(define_insn "*one_cmplsi2_1"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "=rm")
-       (not:SI (match_operand:SI 1 "nonimmediate_operand" "0")))]
-  "ix86_unary_operator_ok (NOT, SImode, operands)"
-  "not{l}\t%0"
+(define_insn "*one_cmpl<mode>2_1"
+  [(set (match_operand:SWI248 0 "nonimmediate_operand" "=rm")
+       (not:SWI248 (match_operand:SWI248 1 "nonimmediate_operand" "0")))]
+  "ix86_unary_operator_ok (NOT, <MODE>mode, operands)"
+  "not{<imodesuffix>}\t%0"
   [(set_attr "type" "negnot")
-   (set_attr "mode" "SI")])
+   (set_attr "mode" "<MODE>")])
+
+;; %%% Potential partial reg stall on alternative 1.  What to do?
+(define_insn "*one_cmplqi2_1"
+  [(set (match_operand:QI 0 "nonimmediate_operand" "=qm,r")
+       (not:QI (match_operand:QI 1 "nonimmediate_operand" "0,0")))]
+  "ix86_unary_operator_ok (NOT, QImode, operands)"
+  "@
+   not{b}\t%0
+   not{l}\t%k0"
+  [(set_attr "type" "negnot")
+   (set_attr "mode" "QI,SI")])
 
 ;; ??? Currently never generated - xor is used instead.
 (define_insn "*one_cmplsi2_1_zext"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (zero_extend:DI (not:SI (match_operand:SI 1 "register_operand" "0"))))]
+       (zero_extend:DI
+         (not:SI (match_operand:SI 1 "register_operand" "0"))))]
   "TARGET_64BIT && ix86_unary_operator_ok (NOT, SImode, operands)"
   "not{l}\t%k0"
   [(set_attr "type" "negnot")
    (set_attr "mode" "SI")])
 
-(define_insn "*one_cmplsi2_2"
+(define_insn "*one_cmpl<mode>2_2"
   [(set (reg FLAGS_REG)
-       (compare (not:SI (match_operand:SI 1 "nonimmediate_operand" "0"))
+       (compare (not:SWI (match_operand:SWI 1 "nonimmediate_operand" "0"))
                 (const_int 0)))
-   (set (match_operand:SI 0 "nonimmediate_operand" "=rm")
-       (not:SI (match_dup 1)))]
+   (set (match_operand:SWI 0 "nonimmediate_operand" "=<r>m")
+       (not:SWI (match_dup 1)))]
   "ix86_match_ccmode (insn, CCNOmode)
-   && ix86_unary_operator_ok (NOT, SImode, operands)"
+   && ix86_unary_operator_ok (NOT, <MODE>mode, operands)"
   "#"
   [(set_attr "type" "alu1")
-   (set_attr "mode" "SI")])
+   (set_attr "mode" "<MODE>")])
 
 (define_split
   [(set (match_operand 0 "flags_reg_operand" "")
        (match_operator 2 "compare_operator"
-         [(not:SI (match_operand:SI 3 "nonimmediate_operand" ""))
+         [(not:SWI (match_operand:SWI 3 "nonimmediate_operand" ""))
           (const_int 0)]))
-   (set (match_operand:SI 1 "nonimmediate_operand" "")
-       (not:SI (match_dup 3)))]
+   (set (match_operand:SWI 1 "nonimmediate_operand" "")
+       (not:SWI (match_dup 3)))]
   "ix86_match_ccmode (insn, CCNOmode)"
   [(parallel [(set (match_dup 0)
-                  (match_op_dup 2 [(xor:SI (match_dup 3) (const_int -1))
+                  (match_op_dup 2 [(xor:SWI (match_dup 3) (const_int -1))
                                    (const_int 0)]))
              (set (match_dup 1)
-                  (xor:SI (match_dup 3) (const_int -1)))])]
+                  (xor:SWI (match_dup 3) (const_int -1)))])]
   "")
 
 ;; ??? Currently never generated - xor is used instead.
              (set (match_dup 1)
                   (zero_extend:DI (xor:SI (match_dup 3) (const_int -1))))])]
   "")
-
-(define_expand "one_cmplhi2"
-  [(set (match_operand:HI 0 "nonimmediate_operand" "")
-       (not:HI (match_operand:HI 1 "nonimmediate_operand" "")))]
-  "TARGET_HIMODE_MATH"
-  "ix86_expand_unary_operator (NOT, HImode, operands); DONE;")
-
-(define_insn "*one_cmplhi2_1"
-  [(set (match_operand:HI 0 "nonimmediate_operand" "=rm")
-       (not:HI (match_operand:HI 1 "nonimmediate_operand" "0")))]
-  "ix86_unary_operator_ok (NOT, HImode, operands)"
-  "not{w}\t%0"
-  [(set_attr "type" "negnot")
-   (set_attr "mode" "HI")])
-
-(define_insn "*one_cmplhi2_2"
-  [(set (reg FLAGS_REG)
-       (compare (not:HI (match_operand:HI 1 "nonimmediate_operand" "0"))
-                (const_int 0)))
-   (set (match_operand:HI 0 "nonimmediate_operand" "=rm")
-       (not:HI (match_dup 1)))]
-  "ix86_match_ccmode (insn, CCNOmode)
-   && ix86_unary_operator_ok (NEG, HImode, operands)"
-  "#"
-  [(set_attr "type" "alu1")
-   (set_attr "mode" "HI")])
-
-(define_split
-  [(set (match_operand 0 "flags_reg_operand" "")
-       (match_operator 2 "compare_operator"
-         [(not:HI (match_operand:HI 3 "nonimmediate_operand" ""))
-          (const_int 0)]))
-   (set (match_operand:HI 1 "nonimmediate_operand" "")
-       (not:HI (match_dup 3)))]
-  "ix86_match_ccmode (insn, CCNOmode)"
-  [(parallel [(set (match_dup 0)
-                  (match_op_dup 2 [(xor:HI (match_dup 3) (const_int -1))
-                                   (const_int 0)]))
-             (set (match_dup 1)
-                  (xor:HI (match_dup 3) (const_int -1)))])]
-  "")
-
-;; %%% Potential partial reg stall on alternative 1.  What to do?
-(define_expand "one_cmplqi2"
-  [(set (match_operand:QI 0 "nonimmediate_operand" "")
-       (not:QI (match_operand:QI 1 "nonimmediate_operand" "")))]
-  "TARGET_QIMODE_MATH"
-  "ix86_expand_unary_operator (NOT, QImode, operands); DONE;")
-
-(define_insn "*one_cmplqi2_1"
-  [(set (match_operand:QI 0 "nonimmediate_operand" "=qm,r")
-       (not:QI (match_operand:QI 1 "nonimmediate_operand" "0,0")))]
-  "ix86_unary_operator_ok (NOT, QImode, operands)"
-  "@
-   not{b}\t%0
-   not{l}\t%k0"
-  [(set_attr "type" "negnot")
-   (set_attr "mode" "QI,SI")])
-
-(define_insn "*one_cmplqi2_2"
-  [(set (reg FLAGS_REG)
-       (compare (not:QI (match_operand:QI 1 "nonimmediate_operand" "0"))
-                (const_int 0)))
-   (set (match_operand:QI 0 "nonimmediate_operand" "=qm")
-       (not:QI (match_dup 1)))]
-  "ix86_match_ccmode (insn, CCNOmode)
-   && ix86_unary_operator_ok (NOT, QImode, operands)"
-  "#"
-  [(set_attr "type" "alu1")
-   (set_attr "mode" "QI")])
-
-(define_split
-  [(set (match_operand 0 "flags_reg_operand" "")
-       (match_operator 2 "compare_operator"
-         [(not:QI (match_operand:QI 3 "nonimmediate_operand" ""))
-          (const_int 0)]))
-   (set (match_operand:QI 1 "nonimmediate_operand" "")
-       (not:QI (match_dup 3)))]
-  "ix86_match_ccmode (insn, CCNOmode)"
-  [(parallel [(set (match_dup 0)
-                  (match_op_dup 2 [(xor:QI (match_dup 3) (const_int -1))
-                                   (const_int 0)]))
-             (set (match_dup 1)
-                  (xor:QI (match_dup 3) (const_int -1)))])]
-  "")
 \f
 ;; Arithmetic shift instructions
 
   "TARGET_64BIT"
   "ix86_expand_binary_operator (ASHIFT, TImode, operands); DONE;")
 
-;; This pattern must be defined before *ashlti3_1 to prevent
-;; combine pass from converting sse2_ashlti3 to *ashlti3_1.
-
-(define_insn "*avx_ashlti3"
-  [(set (match_operand:TI 0 "register_operand" "=x")
-       (ashift:TI (match_operand:TI 1 "register_operand" "x")
-                  (match_operand:SI 2 "const_0_to_255_mul_8_operand" "n")))]
-  "TARGET_AVX"
-{
-  operands[2] = GEN_INT (INTVAL (operands[2]) / 8);
-  return "vpslldq\t{%2, %1, %0|%0, %1, %2}";
-}
-  [(set_attr "type" "sseishft")
-   (set_attr "prefix" "vex")
-   (set_attr "length_immediate" "1")
-   (set_attr "mode" "TI")])
-
-(define_insn "sse2_ashlti3"
-  [(set (match_operand:TI 0 "register_operand" "=x")
-       (ashift:TI (match_operand:TI 1 "register_operand" "0")
-                  (match_operand:SI 2 "const_0_to_255_mul_8_operand" "n")))]
-  "TARGET_SSE2"
-{
-  operands[2] = GEN_INT (INTVAL (operands[2]) / 8);
-  return "pslldq\t{%2, %0|%0, %2}";
-}
-  [(set_attr "type" "sseishft")
-   (set_attr "prefix_data16" "1")
-   (set_attr "length_immediate" "1")
-   (set_attr "mode" "TI")])
-
 (define_insn "*ashlti3_1"
   [(set (match_operand:TI 0 "register_operand" "=&r,r")
        (ashift:TI (match_operand:TI 1 "reg_or_pm1_operand" "n,0")
   "TARGET_64BIT"
   "ix86_expand_binary_operator (LSHIFTRT, TImode, operands); DONE;")
 
-;; This pattern must be defined before *lshrti3_1 to prevent
-;; combine pass from converting sse2_lshrti3 to *lshrti3_1.
-
-(define_insn "*avx_lshrti3"
-  [(set (match_operand:TI 0 "register_operand" "=x")
-       (lshiftrt:TI (match_operand:TI 1 "register_operand" "x")
-                    (match_operand:SI 2 "const_0_to_255_mul_8_operand" "n")))]
-  "TARGET_AVX"
-{
-  operands[2] = GEN_INT (INTVAL (operands[2]) / 8);
-  return "vpsrldq\t{%2, %1, %0|%0, %1, %2}";
-}
-  [(set_attr "type" "sseishft")
-   (set_attr "prefix" "vex")
-   (set_attr "length_immediate" "1")
-   (set_attr "mode" "TI")])
-
-(define_insn "sse2_lshrti3"
-  [(set (match_operand:TI 0 "register_operand" "=x")
-       (lshiftrt:TI (match_operand:TI 1 "register_operand" "0")
-                    (match_operand:SI 2 "const_0_to_255_mul_8_operand" "n")))]
-  "TARGET_SSE2"
-{
-  operands[2] = GEN_INT (INTVAL (operands[2]) / 8);
-  return "psrldq\t{%2, %0|%0, %2}";
-}
-  [(set_attr "type" "sseishft")
-   (set_attr "prefix_data16" "1")
-   (set_attr "length_immediate" "1")
-   (set_attr "mode" "TI")])
-
 (define_insn "*lshrti3_1"
   [(set (match_operand:TI 0 "register_operand" "=r")
        (lshiftrt:TI (match_operand:TI 1 "register_operand" "0")
 ;; checked for calls.  This is a bug in the generic code, but it isn't that
 ;; easy to fix.  Ignore it for now and be prepared to fix things up.
 
+;; P6 processors will jump to the address after the decrement when %esp
+;; is used as a call operand, so they will execute return address as a code.
+;; See Pentium Pro errata 70, Pentium 2 errata A33 and Pentium 3 errata E17.
 ;; Call subroutine returning no value.
 
 (define_expand "call_pop"
                            (match_operand:SI 3 "" "")))])]
   "!TARGET_64BIT"
 {
-  ix86_expand_call (NULL, operands[0], operands[1], operands[2], operands[3], 0);
+  ix86_expand_call (NULL, operands[0], operands[1],
+                   operands[2], operands[3], 0);
   DONE;
 })
 
 (define_insn "*call_pop_0"
   [(call (mem:QI (match_operand:SI 0 "constant_call_address_operand" ""))
         (match_operand:SI 1 "" ""))
-   (set (reg:SI SP_REG) (plus:SI (reg:SI SP_REG)
-                           (match_operand:SI 2 "immediate_operand" "")))]
+   (set (reg:SI SP_REG)
+       (plus:SI (reg:SI SP_REG)
+                (match_operand:SI 2 "immediate_operand" "")))]
   "!TARGET_64BIT"
 {
   if (SIBLING_CALL_P (insn))
 }
   [(set_attr "type" "call")])
 
-(define_insn "*call_pop_1_esp"
-  [(call (mem:QI (match_operand:SI 0 "call_insn_operand" "rsm"))
-        (match_operand:SI 1 "" ""))
-   (set (reg:SI SP_REG) (plus:SI (reg:SI SP_REG)
-                           (match_operand:SI 2 "immediate_operand" "i")))]
-  "!TARGET_64BIT && TARGET_CALL_ESP && !SIBLING_CALL_P (insn)"
-{
-  if (constant_call_address_operand (operands[0], Pmode))
-    return "call\t%P0";
-  return "call\t%A0";
-}
-  [(set_attr "type" "call")])
-
 (define_insn "*call_pop_1"
   [(call (mem:QI (match_operand:SI 0 "call_insn_operand" "lsm"))
         (match_operand:SI 1 "" ""))
-   (set (reg:SI SP_REG) (plus:SI (reg:SI SP_REG)
-                           (match_operand:SI 2 "immediate_operand" "i")))]
-  "!TARGET_64BIT && !TARGET_CALL_ESP && !SIBLING_CALL_P (insn)"
+   (set (reg:SI SP_REG)
+       (plus:SI (reg:SI SP_REG)
+                (match_operand:SI 2 "immediate_operand" "i")))]
+  "!TARGET_64BIT && !SIBLING_CALL_P (insn)"
 {
   if (constant_call_address_operand (operands[0], Pmode))
     return "call\t%P0";
 (define_insn "*sibcall_pop_1"
   [(call (mem:QI (match_operand:SI 0 "sibcall_insn_operand" "s,U"))
         (match_operand:SI 1 "" ""))
-   (set (reg:SI SP_REG) (plus:SI (reg:SI SP_REG)
-                           (match_operand:SI 2 "immediate_operand" "i,i")))]
+   (set (reg:SI SP_REG)
+       (plus:SI (reg:SI SP_REG)
+                (match_operand:SI 2 "immediate_operand" "i,i")))]
   "!TARGET_64BIT && SIBLING_CALL_P (insn)"
   "@
    jmp\t%P0
 }
   [(set_attr "type" "call")])
 
-(define_insn "*call_1_esp"
-  [(call (mem:QI (match_operand:SI 0 "call_insn_operand" "rsm"))
-        (match_operand 1 "" ""))]
-  "!TARGET_64BIT && TARGET_CALL_ESP && !SIBLING_CALL_P (insn)"
-{
-  if (constant_call_address_operand (operands[0], Pmode))
-    return "call\t%P0";
-  return "call\t%A0";
-}
-  [(set_attr "type" "call")])
-
 (define_insn "*call_1"
   [(call (mem:QI (match_operand:SI 0 "call_insn_operand" "lsm"))
         (match_operand 1 "" ""))]
-  "!TARGET_64BIT && !TARGET_CALL_ESP && !SIBLING_CALL_P (insn)"
+  "!TARGET_64BIT && !SIBLING_CALL_P (insn)"
 {
   if (constant_call_address_operand (operands[0], Pmode))
     return "call\t%P0";
        (call (match_operand:QI 1 "" "")
              (match_operand:SI 2 "" "")))
    (use (match_operand:SI 3 "" ""))]
-  ;; Operand 2 not used on the i386.
+  ;; Operand 3 is not used on the i386.
   ""
 {
-  ix86_expand_call (operands[0], operands[1], operands[2], operands[3], NULL, 0);
+  ix86_expand_call (operands[0], operands[1], operands[2],
+                   operands[3], NULL, 0);
   DONE;
 })
 
        (call (match_operand:QI 1 "" "")
              (match_operand:SI 2 "" "")))
    (use (match_operand:SI 3 "" ""))]
-  ;; Operand 2 not used on the i386.
+  ;; Operand 3 is not used on the i386.
   ""
 {
-  ix86_expand_call (operands[0], operands[1], operands[2], operands[3], NULL, 1);
+  ix86_expand_call (operands[0], operands[1], operands[2],
+                   operands[3], NULL, 1);
   DONE;
 })
 
   [(set_attr "type" "multi")
    (set_attr "length" "12")])
 
-(define_insn "*tls_global_dynamic_32_sun"
-  [(set (match_operand:SI 0 "register_operand" "=a")
-       (unspec:SI [(match_operand:SI 1 "register_operand" "b")
-                   (match_operand:SI 2 "tls_symbolic_operand" "")
-                   (match_operand:SI 3 "call_insn_operand" "")]
-                   UNSPEC_TLS_GD))
-   (clobber (match_scratch:SI 4 "=d"))
-   (clobber (match_scratch:SI 5 "=c"))
-   (clobber (reg:CC FLAGS_REG))]
-  "!TARGET_64BIT && TARGET_SUN_TLS"
-  "lea{l}\t{%a2@DTLNDX(%1), %4|%4, %a2@DTLNDX[%1]}
-       push{l}\t%4\;call\t%a2@TLSPLT\;pop{l}\t%4\;nop"
-  [(set_attr "type" "multi")
-   (set_attr "length" "14")])
-
 (define_expand "tls_global_dynamic_32"
   [(parallel [(set (match_operand:SI 0 "register_operand" "")
                   (unspec:SI
   [(set_attr "type" "multi")
    (set_attr "length" "11")])
 
-(define_insn "*tls_local_dynamic_base_32_sun"
-  [(set (match_operand:SI 0 "register_operand" "=a")
-       (unspec:SI [(match_operand:SI 1 "register_operand" "b")
-                    (match_operand:SI 2 "call_insn_operand" "")]
-                  UNSPEC_TLS_LD_BASE))
-   (clobber (match_scratch:SI 3 "=d"))
-   (clobber (match_scratch:SI 4 "=c"))
-   (clobber (reg:CC FLAGS_REG))]
-  "!TARGET_64BIT && TARGET_SUN_TLS"
-  "lea{l}\t{%&@TMDNX(%1), %3|%3, %&@TMDNX[%1]}
-       push{l}\t%3\;call\t%&@TLSPLT\;pop{l}\t%3"
-  [(set_attr "type" "multi")
-   (set_attr "length" "13")])
-
 (define_expand "tls_local_dynamic_base_32"
   [(parallel [(set (match_operand:SI 0 "register_operand" "")
                   (unspec:SI [(match_dup 1) (match_dup 2)]
     }
   else
     {
+      rtx (*cmp_insn)(rtx, rtx);
+
       if (TARGET_64BIT)
-       emit_insn (gen_cmpdi_1_rex64 (countreg, countreg));
+       cmp_insn = gen_cmpdi_1;
       else
-       emit_insn (gen_cmpsi_1 (countreg, countreg));
+       cmp_insn = gen_cmpsi_1;
+      emit_insn (cmp_insn (countreg, countreg));
       emit_insn (gen_cmpstrnqi_1 (addr1, addr2, countreg, align,
                                  operands[1], operands[2]));
     }
 ;; the register first winds up with `sbbl $0,reg', which is also weird.
 ;; So just document what we're doing explicitly.
 
-(define_insn "x86_mov<mode>cc_0_m1"
+(define_expand "x86_mov<mode>cc_0_m1"
+  [(parallel
+    [(set (match_operand:SWI48 0 "register_operand" "")
+         (if_then_else:SWI48
+           (match_operator:SWI48 2 "ix86_carry_flag_operator"
+            [(match_operand 1 "flags_reg_operand" "")
+             (const_int 0)])
+           (const_int -1)
+           (const_int 0)))
+     (clobber (reg:CC FLAGS_REG))])]
+  ""
+  "")
+
+(define_insn "*x86_mov<mode>cc_0_m1"
   [(set (match_operand:SWI48 0 "register_operand" "=r")
-       (if_then_else:SWI48 (match_operand 1 "ix86_carry_flag_operator" "")
+       (if_then_else:SWI48 (match_operator 1 "ix86_carry_flag_operator"
+                            [(reg FLAGS_REG) (const_int 0)])
          (const_int -1)
          (const_int 0)))
    (clobber (reg:CC FLAGS_REG))]
 
 (define_insn "*x86_mov<mode>cc_0_m1_se"
   [(set (match_operand:SWI48 0 "register_operand" "=r")
-       (sign_extract:SWI48 (match_operand 1 "ix86_carry_flag_operator" "")
+       (sign_extract:SWI48 (match_operator 1 "ix86_carry_flag_operator"
+                            [(reg FLAGS_REG) (const_int 0)])
                            (const_int 1)
                            (const_int 0)))
    (clobber (reg:CC FLAGS_REG))]
 
 (define_insn "*x86_mov<mode>cc_0_m1_neg"
   [(set (match_operand:SWI48 0 "register_operand" "=r")
-       (neg:SWI48 (match_operand 1 "ix86_carry_flag_operator" "")))]
+       (neg:SWI48 (match_operator 1 "ix86_carry_flag_operator"
+                   [(reg FLAGS_REG) (const_int 0)])))]
   ""
   "sbb{<imodesuffix>}\t%0, %0"
   [(set_attr "type" "alu")
          (match_operand:MODEF 1 "register_operand" "x")
          (match_operand:MODEF 2 "register_operand" "x")
          (match_operand:MODEF 3 "register_operand" "x")))]
-  "TARGET_XOP && ix86_fma4_valid_op_p (operands, insn, 4, true, 1, false)"
+  "TARGET_XOP"
   "vpcmov\t{%1, %3, %2, %0|%0, %2, %3, %1}"
   [(set_attr "type" "sse4arg")])
 
   DONE;
 })
 
+;; Use IOR for stack probes, this is shorter.
+(define_expand "probe_stack"
+  [(match_operand 0 "memory_operand" "")]
+  ""
+{
+  if (GET_MODE (operands[0]) == DImode)
+    emit_insn (gen_iordi3 (operands[0], operands[0], const0_rtx));
+  else
+    emit_insn (gen_iorsi3 (operands[0], operands[0], const0_rtx));
+  DONE;
+})
+
 (define_expand "builtin_setjmp_receiver"
   [(label_ref (match_operand 0 "" ""))]
   "!TARGET_64BIT && flag_pic"
                      [(match_dup 0)
                       (match_operand:SI 1 "nonmemory_operand" "")]))
               (clobber (reg:CC FLAGS_REG))])]
-  "optimize_insn_for_speed_p () && ! TARGET_READ_MODIFY_WRITE"
+  "optimize_insn_for_speed_p () && ! TARGET_READ_MODIFY_WRITE
+   /* Do not split stack checking probes.  */
+   && GET_CODE (operands[3]) != IOR && operands[1] != const0_rtx"
   [(set (match_dup 2) (match_dup 0))
    (parallel [(set (match_dup 2)
                    (match_op_dup 3 [(match_dup 2) (match_dup 1)]))
                      [(match_operand:SI 1 "nonmemory_operand" "")
                       (match_dup 0)]))
               (clobber (reg:CC FLAGS_REG))])]
-  "optimize_insn_for_speed_p () && ! TARGET_READ_MODIFY_WRITE"
+  "optimize_insn_for_speed_p () && ! TARGET_READ_MODIFY_WRITE
+   /* Do not split stack checking probes.  */
+   && GET_CODE (operands[3]) != IOR && operands[1] != const0_rtx"
   [(set (match_dup 2) (match_dup 0))
    (parallel [(set (match_dup 2)
                    (match_op_dup 3 [(match_dup 1) (match_dup 2)]))
   [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand:SI 1 "constant_call_address_operand" ""))
              (match_operand:SI 2 "" "")))
-   (set (reg:SI SP_REG) (plus:SI (reg:SI SP_REG)
-                           (match_operand:SI 3 "immediate_operand" "")))]
+   (set (reg:SI SP_REG)
+       (plus:SI (reg:SI SP_REG)
+                (match_operand:SI 3 "immediate_operand" "")))]
   "!TARGET_64BIT"
 {
   if (SIBLING_CALL_P (insn))
 }
   [(set_attr "type" "callv")])
 
-(define_insn "*call_value_pop_1_esp"
-  [(set (match_operand 0 "" "")
-       (call (mem:QI (match_operand:SI 1 "call_insn_operand" "rsm"))
-             (match_operand:SI 2 "" "")))
-   (set (reg:SI SP_REG) (plus:SI (reg:SI SP_REG)
-                           (match_operand:SI 3 "immediate_operand" "i")))]
-  "!TARGET_64BIT && TARGET_CALL_ESP && !SIBLING_CALL_P (insn)"
-{
-  if (constant_call_address_operand (operands[1], Pmode))
-    return "call\t%P1";
-  return "call\t%A1";
-}
-  [(set_attr "type" "callv")])
-
 (define_insn "*call_value_pop_1"
   [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand:SI 1 "call_insn_operand" "lsm"))
              (match_operand:SI 2 "" "")))
-   (set (reg:SI SP_REG) (plus:SI (reg:SI SP_REG)
-                           (match_operand:SI 3 "immediate_operand" "i")))]
-  "!TARGET_64BIT && !TARGET_CALL_ESP && !SIBLING_CALL_P (insn)"
+   (set (reg:SI SP_REG)
+       (plus:SI (reg:SI SP_REG)
+                (match_operand:SI 3 "immediate_operand" "i")))]
+  "!TARGET_64BIT && !SIBLING_CALL_P (insn)"
 {
   if (constant_call_address_operand (operands[1], Pmode))
     return "call\t%P1";
   [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand:SI 1 "sibcall_insn_operand" "s,U"))
              (match_operand:SI 2 "" "")))
-   (set (reg:SI SP_REG) (plus:SI (reg:SI SP_REG)
-                           (match_operand:SI 3 "immediate_operand" "i,i")))]
+   (set (reg:SI SP_REG)
+       (plus:SI (reg:SI SP_REG)
+                (match_operand:SI 3 "immediate_operand" "i,i")))]
   "!TARGET_64BIT && SIBLING_CALL_P (insn)"
   "@
    jmp\t%P1
 }
   [(set_attr "type" "callv")])
 
-(define_insn "*call_value_1_esp"
-  [(set (match_operand 0 "" "")
-       (call (mem:QI (match_operand:SI 1 "call_insn_operand" "rsm"))
-             (match_operand:SI 2 "" "")))]
-  "!TARGET_64BIT && TARGET_CALL_ESP && !SIBLING_CALL_P (insn)"
-{
-  if (constant_call_address_operand (operands[1], Pmode))
-    return "call\t%P1";
-  return "call\t%A1";
-}
-  [(set_attr "type" "callv")])
-
 (define_insn "*call_value_1"
   [(set (match_operand 0 "" "")
        (call (mem:QI (match_operand:SI 1 "call_insn_operand" "lsm"))
              (match_operand:SI 2 "" "")))]
-  "!TARGET_64BIT && !TARGET_CALL_ESP && !SIBLING_CALL_P (insn)"
+  "!TARGET_64BIT && !SIBLING_CALL_P (insn)"
 {
   if (constant_call_address_operand (operands[1], Pmode))
     return "call\t%P1";
   }
   [(set_attr "type" "multi")])
 
-(define_mode_iterator CRC32MODE [QI HI SI])
-(define_mode_attr crc32modesuffix [(QI "b") (HI "w") (SI "l")])
-(define_mode_attr crc32modeconstraint [(QI "qm") (HI "rm") (SI "rm")])
-
 (define_insn "sse4_2_crc32<mode>"
   [(set (match_operand:SI 0 "register_operand" "=r")
        (unspec:SI
          [(match_operand:SI 1 "register_operand" "0")
-          (match_operand:CRC32MODE 2 "nonimmediate_operand" "<crc32modeconstraint>")]
+          (match_operand:SWI124 2 "nonimmediate_operand" "<r>m")]
          UNSPEC_CRC32))]
   "TARGET_SSE4_2 || TARGET_CRC32"
-  "crc32<crc32modesuffix>\t{%2, %0|%0, %2}"
+  "crc32{<imodesuffix>}\t{%2, %0|%0, %2}"
   [(set_attr "type" "sselog1")
    (set_attr "prefix_rep" "1")
    (set_attr "prefix_extra" "1")
           (match_operand:DI 2 "nonimmediate_operand" "rm")]
          UNSPEC_CRC32))]
   "TARGET_64BIT && (TARGET_SSE4_2 || TARGET_CRC32)"
-  "crc32q\t{%2, %0|%0, %2}"
+  "crc32{q}\t{%2, %0|%0, %2}"
   [(set_attr "type" "sselog1")
    (set_attr "prefix_rep" "1")
    (set_attr "prefix_extra" "1")
 ;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(define_insn "lwp_llwpcbhi1"
-  [(unspec [(match_operand:HI 0 "register_operand" "r")]
-          UNSPEC_LLWP_INTRINSIC)]
-  "TARGET_LWP"
-  "llwpcb\t%0"
-  [(set_attr "type" "lwp")
-   (set_attr "mode" "HI")])
-
-(define_insn "lwp_llwpcbsi1"
-  [(unspec [(match_operand:SI 0 "register_operand" "r")]
-          UNSPEC_LLWP_INTRINSIC)]
+(define_expand "lwp_llwpcb"
+  [(unspec_volatile [(match_operand 0 "register_operand" "r")]
+                   UNSPECV_LLWP_INTRINSIC)]
   "TARGET_LWP"
-  "llwpcb\t%0"
-  [(set_attr "type" "lwp")
-   (set_attr "mode" "SI")])
+  "")
 
-(define_insn "lwp_llwpcbdi1"
-  [(unspec [(match_operand:DI 0 "register_operand" "r")]
-          UNSPEC_LLWP_INTRINSIC)]
+(define_insn "*lwp_llwpcb<mode>1"
+  [(unspec_volatile [(match_operand:P 0 "register_operand" "r")]
+                   UNSPECV_LLWP_INTRINSIC)]
   "TARGET_LWP"
   "llwpcb\t%0"
   [(set_attr "type" "lwp")
-   (set_attr "mode" "DI")])
-
-(define_insn "lwp_slwpcbhi1"
-  [(unspec [(match_operand:HI 0 "register_operand" "r")]
-          UNSPEC_SLWP_INTRINSIC)]
-  "TARGET_LWP"
-  "slwpcb\t%0"
-  [(set_attr "type" "lwp")
-   (set_attr "mode" "HI")])
+   (set_attr "mode" "<MODE>")
+   (set_attr "length" "5")])
 
-(define_insn "lwp_slwpcbsi1"
-  [(unspec [(match_operand:SI 0 "register_operand" "r")]
-          UNSPEC_SLWP_INTRINSIC)]
+(define_expand "lwp_slwpcb"
+  [(set (match_operand 0 "register_operand" "=r")
+       (unspec_volatile [(const_int 0)] UNSPECV_SLWP_INTRINSIC))]
   "TARGET_LWP"
-  "slwpcb\t%0"
-  [(set_attr "type" "lwp")
-   (set_attr "mode" "SI")])
-
-(define_insn "lwp_slwpcbdi1"
-  [(unspec [(match_operand:DI 0 "register_operand" "r")]
-          UNSPEC_SLWP_INTRINSIC)]
+  {
+    if (TARGET_64BIT)
+      emit_insn (gen_lwp_slwpcbdi (operands[0]));
+    else
+      emit_insn (gen_lwp_slwpcbsi (operands[0]));
+    DONE;
+  })
+
+(define_insn "lwp_slwpcb<mode>"
+  [(set (match_operand:P 0 "register_operand" "=r")
+       (unspec_volatile:P [(const_int 0)] UNSPECV_SLWP_INTRINSIC))]
   "TARGET_LWP"
   "slwpcb\t%0"
   [(set_attr "type" "lwp")
-   (set_attr "mode" "DI")])
-
-(define_insn "lwp_lwpvalhi3"
-  [(unspec_volatile [(match_operand:HI 0 "register_operand" "r")
-                    (match_operand:SI 1 "nonimmediate_operand" "rm")
-                    (match_operand:HI 2 "const_int_operand" "")]
-                   UNSPECV_LWPVAL_INTRINSIC)]
-  "TARGET_LWP"
-  "lwpval\t{%2, %1, %0|%0, %1, %2}"
-  [(set_attr "type" "lwp")
-   (set_attr "mode" "HI")])
+   (set_attr "mode" "<MODE>")
+   (set_attr "length" "5")])
 
-(define_insn "lwp_lwpvalsi3"
-  [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")
-                    (match_operand:SI 1 "nonimmediate_operand" "rm")
-                    (match_operand:SI 2 "const_int_operand" "")]
+(define_expand "lwp_lwpval<mode>3"
+  [(unspec_volatile [(match_operand:SWI48 1 "register_operand" "r")
+                    (match_operand:SI 2 "nonimmediate_operand" "rm")
+                    (match_operand:SI 3 "const_int_operand" "i")]
                    UNSPECV_LWPVAL_INTRINSIC)]
   "TARGET_LWP"
-  "lwpval\t{%2, %1, %0|%0, %1, %2}"
-  [(set_attr "type" "lwp")
-   (set_attr "mode" "SI")])
+  "/* Avoid unused variable warning.  */
+   (void) operand0;")
 
-(define_insn "lwp_lwpvaldi3"
-  [(unspec_volatile [(match_operand:DI 0 "register_operand" "r")
-                    (match_operand:SI 1 "nonimmediate_operand" "rm")
-                    (match_operand:SI 2 "const_int_operand" "")]
+(define_insn "*lwp_lwpval<mode>3_1"
+  [(unspec_volatile [(match_operand:SWI48 0 "register_operand" "r")
+                    (match_operand:SI 1 "nonimmediate_operand" "rm")
+                    (match_operand:SI 2 "const_int_operand" "i")]
                    UNSPECV_LWPVAL_INTRINSIC)]
   "TARGET_LWP"
   "lwpval\t{%2, %1, %0|%0, %1, %2}"
   [(set_attr "type" "lwp")
-   (set_attr "mode" "DI")])
-
-(define_insn "lwp_lwpinshi3"
-  [(unspec_volatile [(match_operand:HI 0 "register_operand" "r")
-                    (match_operand:SI 1 "nonimmediate_operand" "rm")
-                    (match_operand:HI 2 "const_int_operand" "")]
-                   UNSPECV_LWPINS_INTRINSIC)]
-  "TARGET_LWP"
-  "lwpins\t{%2, %1, %0|%0, %1, %2}"
-  [(set_attr "type" "lwp")
-   (set_attr "mode" "HI")])
+   (set_attr "mode" "<MODE>")
+   (set (attr "length")
+        (symbol_ref "ix86_attr_length_address_default (insn) + 9"))])
 
-(define_insn "lwp_lwpinssi3"
-  [(unspec_volatile [(match_operand:SI 0 "register_operand" "r")
-                    (match_operand:SI 1 "nonimmediate_operand" "rm")
-                    (match_operand:SI 2 "const_int_operand" "")]
-                   UNSPECV_LWPINS_INTRINSIC)]
+(define_expand "lwp_lwpins<mode>3"
+  [(set (reg:CCC FLAGS_REG)
+       (unspec_volatile:CCC [(match_operand:SWI48 1 "register_operand" "r")
+                             (match_operand:SI 2 "nonimmediate_operand" "rm")
+                             (match_operand:SI 3 "const_int_operand" "i")]
+                            UNSPECV_LWPINS_INTRINSIC))
+   (set (match_operand:QI 0 "nonimmediate_operand" "=qm")
+       (eq:QI (reg:CCC FLAGS_REG) (const_int 0)))]
   "TARGET_LWP"
-  "lwpins\t{%2, %1, %0|%0, %1, %2}"
-  [(set_attr "type" "lwp")
-   (set_attr "mode" "SI")])
+  "")
 
-(define_insn "lwp_lwpinsdi3"
-  [(unspec_volatile [(match_operand:DI 0 "register_operand" "r")
-                    (match_operand:SI 1 "nonimmediate_operand" "rm")
-                    (match_operand:SI 2 "const_int_operand" "")]
-                   UNSPECV_LWPINS_INTRINSIC)]
+(define_insn "*lwp_lwpins<mode>3_1"
+  [(set (reg:CCC FLAGS_REG)
+       (unspec_volatile:CCC [(match_operand:SWI48 0 "register_operand" "r")
+                             (match_operand:SI 1 "nonimmediate_operand" "rm")
+                             (match_operand:SI 2 "const_int_operand" "i")]
+                            UNSPECV_LWPINS_INTRINSIC))]
   "TARGET_LWP"
   "lwpins\t{%2, %1, %0|%0, %1, %2}"
   [(set_attr "type" "lwp")
-   (set_attr "mode" "DI")])
+   (set_attr "mode" "<MODE>")
+   (set (attr "length")
+        (symbol_ref "ix86_attr_length_address_default (insn) + 9"))])
 
 (include "mmx.md")
 (include "sse.md")