X-Git-Url: http://git.sourceforge.jp/view?a=blobdiff_plain;f=gcc%2Fconfig%2Fs390%2Fs390.md;h=d7f0e0763b405cba5e9856b660381296c0f4e74d;hb=76dbb8df4c3713a6ee6c3c6f97617e44f5df1efc;hp=650d82d200f648a3c38fa74493fd0067a22d87a3;hpb=a40b2054a6a1d0dcd3b9ec1c66e572598e20fa46;p=pf3gnuchains%2Fgcc-fork.git

diff --git a/gcc/config/s390/s390.md b/gcc/config/s390/s390.md
index 650d82d200f..d7f0e0763b4 100644
--- a/gcc/config/s390/s390.md
+++ b/gcc/config/s390/s390.md
@@ -1,23 +1,25 @@
 ;;- Machine description for GNU compiler -- S/390 / zSeries version.
-;;  Copyright (C) 1999, 2000, 2001, 2002 Free Software Foundation, Inc.
+;;  Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004
+;;  Free Software Foundation, Inc.
 ;;  Contributed by Hartmut Penner (hpenner@de.ibm.com) and
 ;;                 Ulrich Weigand (uweigand@de.ibm.com).
-;; This file is part of GNU CC.
 
-;; GNU CC is free software; you can redistribute it and/or modify
-;; it under the terms of the GNU General Public License as published by
-;; the Free Software Foundation; either version 2, or (at your option)
-;; any later version.
+;; This file is part of GCC.
 
-;; GNU CC is distributed in the hope that it will be useful,
-;; but WITHOUT ANY WARRANTY; without even the implied warranty of
-;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-;; GNU General Public License for more details.
+;; GCC is free software; you can redistribute it and/or modify it under
+;; the terms of the GNU General Public License as published by the Free
+;; Software Foundation; either version 2, or (at your option) any later
+;; version.
+
+;; GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+;; WARRANTY; without even the implied warranty of MERCHANTABILITY or
+;; FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+;; for more details.
 
 ;; You should have received a copy of the GNU General Public License
-;; along with GNU CC; see the file COPYING.  If not, write to
-;; the Free Software Foundation, 59 Temple Place - Suite 330,
-;; Boston, MA 02111-1307, USA.
+;; along with GCC; see the file COPYING.  If not, write to the Free
+;; Software Foundation, 59 Temple Place - Suite 330, Boston, MA
+;; 02111-1307, USA.
 
 ;;
 ;; Special constraints for s/390 machine description:
@@ -27,99 +29,169 @@
 ;;    I -- An 8-bit constant (0..255).
 ;;    J -- A 12-bit constant (0..4095).
 ;;    K -- A 16-bit constant (-32768..32767).
-;;    Q -- A memory reference without index-register.
-;;    S -- Valid operand for the LARL instruction.
+;;    L -- Value appropriate as displacement.
+;;         (0..4095) for short displacement
+;;         (-524288..524287) for long displacement
+;;    M -- Constant integer with a value of 0x7fffffff.
+;;    N -- Multiple letter constraint followed by 4 parameter letters.
+;;         0..9:  number of the part counting from most to least significant
+;;         H,Q:   mode of the part
+;;         D,S,H: mode of the containing operand
+;;         0,F:   value of the other parts (F - all bits set)
+;;
+;;         The constraint matches if the specified part of a constant
+;;         has a value different from its other parts.
+;;    Q -- Memory reference without index register and with short displacement.
+;;    R -- Memory reference with index register and short displacement.
+;;    S -- Memory reference without index register but with long displacement.
+;;    T -- Memory reference with index register and long displacement.
+;;    U -- Pointer with short displacement.
+;;    W -- Pointer with long displacement.
+;;    Y -- Shift count operand.
 ;;
 ;; Special formats used for outputting 390 instructions.
 ;;
-;;   %b -- Print a constant byte integer.               xy
-;;   %h -- Print a signed 16-bit.                       wxyz
-;;   %N -- Print next register (second word of a DImode reg) or next word.
-;;   %M -- Print next register (second word of a TImode reg) or next word.
-;;   %O -- Print the offset of a memory reference (PLUS (REG) (CONST_INT)).
-;;   %R -- Print the register of a memory reference (PLUS (REG) (CONST_INT)).
+;;     %C: print opcode suffix for branch condition.
+;;     %D: print opcode suffix for inverse branch condition.
+;;     %J: print tls_load/tls_gdcall/tls_ldcall suffix
+;;     %O: print only the displacement of a memory reference.
+;;     %R: print only the base register of a memory reference.
+;;     %N: print the second word of a DImode operand.
+;;     %M: print the second word of a TImode operand.
+
+;;     %b: print integer X as if it's an unsigned byte.
+;;     %x: print integer X as if it's an unsigned word.
+;;     %h: print integer X as if it's a signed word.
+;;     %i: print the first nonzero HImode part of X
+;;     %j: print the first HImode part unequal to 0xffff of X
+
 ;;
 ;; We have a special constraint for pattern matching.
 ;;
 ;;   s_operand -- Matches a valid S operand in a RS, SI or SS type instruction.
 ;;
 
+;;
+;; UNSPEC usage
+;;
 
-;; Define an insn type attribute.  This is used in function unit delay
-;; computations.
-
-(define_attr "type" "none,integer,load,lr,la,lm,stm,cs,vs,store,imul,lmul,fmul,idiv,ldiv,fdiv,branch,jsr,other,o2,o3"
-  (const_string "integer"))
-
-;; Insn are devide in two classes:
-;;   mem: Insn accessing memory
-;;   reg: Insn operands all in registers
-
-(define_attr "atype" "reg,mem"
-  (const_string "reg"))
-
-;; Generic pipeline function unit.  
-
-(define_function_unit "integer" 1 0
-  (eq_attr "type" "none") 0 0)
-
-(define_function_unit "integer" 1 0
-  (eq_attr "type" "integer") 1 1)
-
-(define_function_unit "integer" 1 0
-  (eq_attr "type" "load") 1 1)
-
-(define_function_unit "integer" 1 0
-  (eq_attr "type" "la") 1 1)
-
-(define_function_unit "integer" 1 0
-  (eq_attr "type" "lr") 1 1)
-
-(define_function_unit "integer" 1 0
-  (eq_attr "type" "store") 1 1)
-
-(define_function_unit "integer" 1 0
-  (eq_attr "type" "lm") 2 2)
-
-(define_function_unit "integer" 1 0
-  (eq_attr "type" "stm") 2 2)
+(define_constants
+  [; Miscellaneous
+   (UNSPEC_ROUND		1)
+   (UNSPEC_SETHIGH		10)
+
+   ; GOT/PLT and lt-relative accesses
+   (UNSPEC_LTREL_OFFSET		100)
+   (UNSPEC_LTREL_BASE		101)
+   (UNSPEC_GOTENT		110)
+   (UNSPEC_GOT			111)
+   (UNSPEC_GOTOFF		112)
+   (UNSPEC_PLT			113)
+   (UNSPEC_PLTOFF		114)
+
+   ; Literal pool
+   (UNSPEC_RELOAD_BASE		210)
+   (UNSPEC_MAIN_BASE		211)
+   (UNSPEC_LTREF		212)
+
+   ; TLS relocation specifiers
+   (UNSPEC_TLSGD		500)
+   (UNSPEC_TLSLDM		501)
+   (UNSPEC_NTPOFF               502)
+   (UNSPEC_DTPOFF               503)
+   (UNSPEC_GOTNTPOFF            504)
+   (UNSPEC_INDNTPOFF            505)
+
+   ; TLS support
+   (UNSPEC_TP			510)
+   (UNSPEC_TLSLDM_NTPOFF	511)
+   (UNSPEC_TLS_LOAD		512)
+
+   ; String Functions
+   (UNSPEC_SRST		600)
+ ])
 
-(define_function_unit "integer" 1 0
-  (eq_attr "type" "cs") 5 5)
+;;
+;; UNSPEC_VOLATILE usage
+;;
 
-(define_function_unit "integer" 1 0
-  (eq_attr "type" "vs") 30 30)
+(define_constants
+  [; Blockage
+   (UNSPECV_BLOCKAGE		0)
 
-(define_function_unit "integer" 1 0
-  (eq_attr "type" "jsr") 5 5)
+   ; TPF Support
+   (UNSPECV_TPF_PROLOGUE        20)
+   (UNSPECV_TPF_EPILOGUE        21)
 
-(define_function_unit "integer" 1 0
-  (eq_attr "type" "imul") 7 7)
+   ; Literal pool
+   (UNSPECV_POOL		200)
+   (UNSPECV_POOL_START		201)
+   (UNSPECV_POOL_END		202)
+   (UNSPECV_POOL_ENTRY		203)
+   (UNSPECV_MAIN_POOL		300)
 
-(define_function_unit "integer" 1 0
-  (eq_attr "type" "fmul") 6 6)
+   ; TLS support
+   (UNSPECV_SET_TP		500)
+  ])
 
-(define_function_unit "integer" 1 0
-  (eq_attr "type" "idiv") 33 33)
 
-(define_function_unit "integer" 1 0
-  (eq_attr "type" "fdiv") 33 33)
+;; Processor type.  This attribute must exactly match the processor_type
+;; enumeration in s390.h.  The current machine description does not
+;; distinguish between g5 and g6, but there are differences between the two
+;; CPUs could in theory be modeled.
 
-(define_function_unit "integer" 1 0
-  (eq_attr "type" "o2") 2 2)
+(define_attr "cpu" "g5,g6,z900,z990"
+  (const (symbol_ref "s390_tune")))
 
-(define_function_unit "integer" 1 0
-  (eq_attr "type" "o3") 3 3)
+;; Define an insn type attribute.  This is used in function unit delay
+;; computations.
 
-(define_function_unit "integer" 1 0
-  (eq_attr "type" "other") 5 5)
+(define_attr "type" "none,integer,load,lr,la,larl,lm,stm,
+	             cs,vs,store,imul,idiv,
+		     branch,jsr,fsimpd,fsimps,
+		     floadd,floads,fstored, fstores,
+		     fmuld,fmuls,fdivd,fdivs,
+		     ftoi,itof,fsqrtd,fsqrts,
+                     other,o2,o3"
+  (const_string "integer"))
 
 ;; Operand type. Used to default length attribute values
 
 (define_attr "op_type"
-  "NN,E,RR,RRE,RX,RS,RSI,RI,SI,S,SS,SSE,RXE,RSE,RIL,RIE"
+  "NN,E,RR,RRE,RX,RS,RSI,RI,SI,S,SS,SSE,RXE,RSE,RIL,RIE,RXY,RSY,SIY"
   (const_string "RX"))
 
+;; Insn are devide in two classes:
+;;   agen: Insn using agen
+;;   reg: Insn not using agen
+
+(define_attr "atype" "agen,reg"
+(cond [ (eq_attr "op_type" "E")    (const_string "reg")
+         (eq_attr "op_type" "RR")  (const_string "reg")
+         (eq_attr "op_type" "RX")  (const_string "agen")
+         (eq_attr "op_type" "RI")  (const_string "reg")
+         (eq_attr "op_type" "RRE") (const_string "reg")
+         (eq_attr "op_type" "RS")  (const_string "agen")
+         (eq_attr "op_type" "RSI") (const_string "agen")
+         (eq_attr "op_type" "S")   (const_string "agen")
+         (eq_attr "op_type" "SI")  (const_string "agen")
+         (eq_attr "op_type" "SS")  (const_string "agen")
+         (eq_attr "op_type" "SSE") (const_string "agen")
+         (eq_attr "op_type" "RXE") (const_string "agen")
+         (eq_attr "op_type" "RSE") (const_string "agen")
+         (eq_attr "op_type" "RIL") (const_string "agen")
+         (eq_attr "op_type" "RXY") (const_string "agen")
+         (eq_attr "op_type" "RSY") (const_string "agen")
+         (eq_attr "op_type" "SIY") (const_string "agen")]
+  (const_string "reg")))
+
+;; Pipeline description for z900.  For lack of anything better,
+;; this description is also used for the g5 and g6.
+(include "2064.md")
+
+;; Pipeline description for z990. 
+(include "2084.md")
+
 ;; Length in bytes.
 
 (define_attr "length" ""
@@ -130,14 +202,16 @@
          (eq_attr "op_type" "RRE") (const_int 4)
          (eq_attr "op_type" "RS")  (const_int 4)
          (eq_attr "op_type" "RSI") (const_int 4)
-         (eq_attr "op_type" "RX")  (const_int 4)
          (eq_attr "op_type" "S")   (const_int 4)
          (eq_attr "op_type" "SI")  (const_int 4)
          (eq_attr "op_type" "SS")  (const_int 6)
          (eq_attr "op_type" "SSE") (const_int 6)
          (eq_attr "op_type" "RXE") (const_int 6)
          (eq_attr "op_type" "RSE") (const_int 6)
-         (eq_attr "op_type" "RIL") (const_int 6)]
+         (eq_attr "op_type" "RIL") (const_int 6)
+         (eq_attr "op_type" "RXY") (const_int 6)
+         (eq_attr "op_type" "RSY") (const_int 6)
+         (eq_attr "op_type" "SIY") (const_int 6)]
          (const_int 4)))
 
 ;; Define attributes for `asm' insns.
@@ -154,11 +228,11 @@
 ;   CCU:  Equal    ULess     UGreater  --           (CL, CLR, CLI, CLM)
 ;   CCS:  Equal    SLess     SGreater  --           (C, CR, CH, CHI, ICM)
 ;   CCT:  Zero     Mixed     Mixed     Ones         (TM, TMH, TML)
- 
+
 ;   CCZ  -> CCL / CCZ1
 ;   CCZ1 -> CCA/CCU/CCS/CCT
 ;   CCS  -> CCA
- 
+
 ;   String:    CLC, CLCL, CLCLE, CLST, CUSE, MVCL, MVCLE, MVPG, MVST, SRST
 ;   Clobber:   CKSM, CFC, CS, CDS, CUUTF, CUTFU, PLO, SPM, STCK, STCKE, TS, TRT, TRE, UPT
 
@@ -172,259 +246,101 @@
         (compare:CC (match_operand:DI 0 "register_operand" "")
                     (match_operand:DI 1 "general_operand" "")))]
   "TARGET_64BIT"
-  "
 {
   s390_compare_op0 = operands[0];
   s390_compare_op1 = operands[1];
   DONE;
-}")
+})
 
 (define_expand "cmpsi"
   [(set (reg:CC 33)
         (compare:CC (match_operand:SI 0 "register_operand" "")
                     (match_operand:SI 1 "general_operand" "")))]
   ""
-  "
 {
   s390_compare_op0 = operands[0];
   s390_compare_op1 = operands[1];
   DONE;
-}")
+})
 
 (define_expand "cmpdf"
   [(set (reg:CC 33)
         (compare:CC (match_operand:DF 0 "register_operand" "")
                     (match_operand:DF 1 "general_operand" "")))]
   "TARGET_HARD_FLOAT"
-  "
 {
   s390_compare_op0 = operands[0];
   s390_compare_op1 = operands[1];
   DONE;
-}")
+})
 
 (define_expand "cmpsf"
   [(set (reg:CC 33)
         (compare:CC (match_operand:SF 0 "register_operand" "")
                     (match_operand:SF 1 "general_operand" "")))]
   "TARGET_HARD_FLOAT"
-  "
 {
   s390_compare_op0 = operands[0];
   s390_compare_op1 = operands[1];
   DONE;
-}")
-
-
-; Test-under-Mask (zero_extract) instructions
-
-(define_insn "*tmdi_ext"
-  [(set (reg 33)
-        (compare (zero_extract:DI (match_operand:DI 0 "register_operand" "d")
-	                          (match_operand:DI 1 "const_int_operand" "n")
-                                  (match_operand:DI 2 "const_int_operand" "n"))
-                 (const_int 0)))]
-  "s390_match_ccmode(insn, CCTmode) && TARGET_64BIT
-   && INTVAL (operands[1]) >= 1 && INTVAL (operands[2]) >= 0 
-   && INTVAL (operands[1]) + INTVAL (operands[2]) <= 64
-   && (INTVAL (operands[1]) + INTVAL (operands[2]) - 1) >> 4
-      == INTVAL (operands[2]) >> 4"
-  "*
-{
-  int part = INTVAL (operands[2]) >> 4;
-  int block = (1 << INTVAL (operands[1])) - 1;
-  int shift = 16 - INTVAL (operands[1]) - (INTVAL (operands[2]) & 15);
-
-  operands[2] = GEN_INT (block << shift);
-
-  switch (part)
-    {
-      case 0: return \"tmhh\\t%0,%x2\";
-      case 1: return \"tmhl\\t%0,%x2\";
-      case 2: return \"tmlh\\t%0,%x2\";
-      case 3: return \"tmll\\t%0,%x2\";
-      default: abort ();
-    }
-}"
-  [(set_attr "op_type" "RI")])
-
-(define_insn "*tmsi_ext"
-  [(set (reg 33)
-        (compare (zero_extract:SI (match_operand:SI 0 "register_operand" "d")
-	                          (match_operand:SI 1 "const_int_operand" "n")
-                                  (match_operand:SI 2 "const_int_operand" "n"))
-                 (const_int 0)))]
-  "s390_match_ccmode(insn, CCTmode)
-   && INTVAL (operands[1]) >= 1 && INTVAL (operands[2]) >= 0 
-   && INTVAL (operands[1]) + INTVAL (operands[2]) <= 32
-   && (INTVAL (operands[1]) + INTVAL (operands[2]) - 1) >> 4
-      == INTVAL (operands[2]) >> 4"
-  "*
-{
-  int part = INTVAL (operands[2]) >> 4;
-  int block = (1 << INTVAL (operands[1])) - 1;
-  int shift = 16 - INTVAL (operands[1]) - (INTVAL (operands[2]) & 15);
-
-  operands[2] = GEN_INT (block << shift);
-
-  switch (part)
-    {
-      case 0: return \"tmh\\t%0,%x2\";
-      case 1: return \"tml\\t%0,%x2\";
-      default: abort ();
-    }
-}"
-  [(set_attr "op_type" "RI")])
-
-(define_insn "*tmqi_ext"
-  [(set (reg 33)
-        (compare (zero_extract:SI (match_operand:QI 0 "memory_operand" "Q")
-	                          (match_operand:SI 1 "const_int_operand" "n")
-                                  (match_operand:SI 2 "const_int_operand" "n"))
-                 (const_int 0)))]
-  "s390_match_ccmode(insn, CCTmode)
-   && INTVAL (operands[1]) >= 1 && INTVAL (operands[2]) >= 0 
-   && INTVAL (operands[1]) + INTVAL (operands[2]) <= 8"
-  "*
-{
-  int block = (1 << INTVAL (operands[1])) - 1;
-  int shift = 8 - INTVAL (operands[1]) - INTVAL (operands[2]);
+})
 
-  operands[2] = GEN_INT (block << shift);
-  return \"tm\\t%0,%b2\";
-}"
-  [(set_attr "op_type" "SI")
-   (set_attr "atype"   "mem")])
 
 ; Test-under-Mask instructions
 
-(define_insn "*tmdi_mem"
-  [(set (reg 33)
-        (compare (and:DI (match_operand:DI 0 "memory_operand" "Q")
-                         (match_operand:DI 1 "immediate_operand" "n"))
-                 (match_operand:DI 2 "immediate_operand" "n")))]
-  "TARGET_64BIT
-   && s390_match_ccmode (insn, s390_tm_ccmode (operands[1], operands[2], 0))
-   && s390_single_qi (operands[1], DImode, 0) >= 0"
-  "*
-{
-  int part = s390_single_qi (operands[1], DImode, 0);
-  operands[1] = GEN_INT (s390_extract_qi (operands[1], DImode, part));
-
-  operands[0] = gen_rtx_MEM (QImode, 
-			     plus_constant (XEXP (operands[0], 0), part));
-  return \"tm\\t%0,%b1\";
-}"
-  [(set_attr "op_type" "SI")
-   (set_attr "atype"   "mem")])
-
-(define_insn "*tmsi_mem"
-  [(set (reg 33)
-        (compare (and:SI (match_operand:SI 0 "memory_operand" "Q")
-                         (match_operand:SI 1 "immediate_operand" "n"))
-                 (match_operand:SI 2 "immediate_operand" "n")))]
-  "s390_match_ccmode (insn, s390_tm_ccmode (operands[1], operands[2], 0))
-   && s390_single_qi (operands[1], SImode, 0) >= 0"
-  "*
-{
-  int part = s390_single_qi (operands[1], SImode, 0);
-  operands[1] = GEN_INT (s390_extract_qi (operands[1], SImode, part));
-
-  operands[0] = gen_rtx_MEM (QImode, 
-			     plus_constant (XEXP (operands[0], 0), part));
-  return \"tm\\t%0,%b1\";
-}"
-  [(set_attr "op_type" "SI")
-   (set_attr "atype"   "mem")])
-
-(define_insn "*tmhi_mem"
-  [(set (reg 33)
-        (compare (and:SI (subreg:SI (match_operand:HI 0 "memory_operand" "Q") 0)
-                         (match_operand:SI 1 "immediate_operand" "n"))
-                 (match_operand:SI 2 "immediate_operand" "n")))]
-  "s390_match_ccmode (insn, s390_tm_ccmode (operands[1], operands[2], 0))
-   && s390_single_qi (operands[1], HImode, 0) >= 0"
-  "*
-{
-  int part = s390_single_qi (operands[1], HImode, 0);
-  operands[1] = GEN_INT (s390_extract_qi (operands[1], HImode, part));
-
-  operands[0] = gen_rtx_MEM (QImode, 
-			     plus_constant (XEXP (operands[0], 0), part));
-  return \"tm\\t%0,%b1\";
-}"
-  [(set_attr "op_type" "SI")
-   (set_attr "atype"   "mem")])
-
 (define_insn "*tmqi_mem"
   [(set (reg 33)
-        (compare (and:SI (subreg:SI (match_operand:QI 0 "memory_operand" "Q") 0)
-                         (match_operand:SI 1 "immediate_operand" "n"))
-                 (match_operand:SI 2 "immediate_operand" "n")))]
+        (compare (and:QI (match_operand:QI 0 "memory_operand" "Q,S")
+                         (match_operand:QI 1 "immediate_operand" "n,n"))
+                 (match_operand:QI 2 "immediate_operand" "n,n")))]
   "s390_match_ccmode (insn, s390_tm_ccmode (operands[1], operands[2], 0))"
-  "tm\\t%0,%b1"
-  [(set_attr "op_type" "SI")
-   (set_attr "atype"   "mem")])
+  "@
+   tm\t%0,%b1
+   tmy\t%0,%b1"
+  [(set_attr "op_type" "SI,SIY")])
 
 (define_insn "*tmdi_reg"
   [(set (reg 33)
-        (compare (and:DI (match_operand:DI 0 "nonimmediate_operand" "d")
-                         (match_operand:DI 1 "immediate_operand" "n"))
-                 (match_operand:DI 2 "immediate_operand" "n")))]
+        (compare (and:DI (match_operand:DI 0 "nonimmediate_operand" "d,d,d,d")
+                         (match_operand:DI 1 "immediate_operand"
+					     "N0HD0,N1HD0,N2HD0,N3HD0"))
+                 (match_operand:DI 2 "immediate_operand" "n,n,n,n")))]
   "TARGET_64BIT
    && s390_match_ccmode (insn, s390_tm_ccmode (operands[1], operands[2], 1))
-   && s390_single_hi (operands[1], DImode, 0) >= 0"
-  "*
-{
-  int part = s390_single_hi (operands[1], DImode, 0);
-  operands[1] = GEN_INT (s390_extract_hi (operands[1], DImode, part));
-
-  switch (part)
-    {
-      case 0: return \"tmhh\\t%0,%x1\";
-      case 1: return \"tmhl\\t%0,%x1\";
-      case 2: return \"tmlh\\t%0,%x1\";
-      case 3: return \"tmll\\t%0,%x1\";
-      default: abort ();
-    }
-}"
+   && s390_single_part (operands[1], DImode, HImode, 0) >= 0"
+  "@
+   tmhh\t%0,%i1
+   tmhl\t%0,%i1
+   tmlh\t%0,%i1
+   tmll\t%0,%i1"
   [(set_attr "op_type" "RI")])
 
 (define_insn "*tmsi_reg"
   [(set (reg 33)
-        (compare (and:SI (match_operand:SI 0 "nonimmediate_operand" "d")
-                         (match_operand:SI 1 "immediate_operand" "n"))
-                 (match_operand:SI 2 "immediate_operand" "n")))]
+        (compare (and:SI (match_operand:SI 0 "nonimmediate_operand" "d,d")
+                         (match_operand:SI 1 "immediate_operand" "N0HS0,N1HS0"))
+                 (match_operand:SI 2 "immediate_operand" "n,n")))]
   "s390_match_ccmode (insn, s390_tm_ccmode (operands[1], operands[2], 1))
-   && s390_single_hi (operands[1], SImode, 0) >= 0"
-  "*
-{
-  int part = s390_single_hi (operands[1], SImode, 0);
-  operands[1] = GEN_INT (s390_extract_hi (operands[1], SImode, part));
-
-  switch (part)
-    {
-      case 0: return \"tmh\\t%0,%x1\";
-      case 1: return \"tml\\t%0,%x1\";
-      default: abort ();
-    }
-}"
+   && s390_single_part (operands[1], SImode, HImode, 0) >= 0"
+  "@
+   tmh\t%0,%i1
+   tml\t%0,%i1"
   [(set_attr "op_type" "RI")])
 
 (define_insn "*tmhi_full"
   [(set (reg 33)
         (compare (match_operand:HI 0 "register_operand" "d")
                  (match_operand:HI 1 "immediate_operand" "n")))]
-  "s390_match_ccmode (insn, s390_tm_ccmode (GEN_INT (-1), operands[1], 1))"
-  "tml\\t%0,65535"
+  "s390_match_ccmode (insn, s390_tm_ccmode (constm1_rtx, operands[1], 1))"
+  "tml\t%0,65535"
   [(set_attr "op_type" "RX")])
 
 (define_insn "*tmqi_full"
   [(set (reg 33)
         (compare (match_operand:QI 0 "register_operand" "d")
                  (match_operand:QI 1 "immediate_operand" "n")))]
-  "s390_match_ccmode (insn, s390_tm_ccmode (GEN_INT (-1), operands[1], 1))"
-  "tml\\t%0,255"
+  "s390_match_ccmode (insn, s390_tm_ccmode (constm1_rtx, operands[1], 1))"
+  "tml\t%0,255"
   [(set_attr "op_type" "RI")])
 
 
@@ -438,7 +354,7 @@
    (set (match_operand:DI 2 "register_operand" "=d")
         (sign_extend:DI (match_dup 0)))]
   "s390_match_ccmode(insn, CCSmode) && TARGET_64BIT"
-  "ltgfr\\t%2,%0"
+  "ltgfr\t%2,%0"
   [(set_attr "op_type" "RRE")])
 
 (define_insn "*tstdi"
@@ -448,7 +364,7 @@
    (set (match_operand:DI 2 "register_operand" "=d")
         (match_dup 0))]
   "s390_match_ccmode(insn, CCSmode) && TARGET_64BIT"
-  "ltgr\\t%2,%0"
+  "ltgr\t%2,%0"
   [(set_attr "op_type" "RRE")])
 
 (define_insn "*tstdi_cconly"
@@ -456,7 +372,7 @@
         (compare (match_operand:DI 0 "register_operand" "d")
                  (match_operand:DI 1 "const0_operand" "")))]
   "s390_match_ccmode(insn, CCSmode) && TARGET_64BIT"
-  "ltgr\\t%0,%0"
+  "ltgr\t%0,%0"
   [(set_attr "op_type" "RRE")])
 
 (define_insn "*tstdi_cconly_31"
@@ -464,83 +380,138 @@
         (compare (match_operand:DI 0 "register_operand" "d")
                  (match_operand:DI 1 "const0_operand" "")))]
   "s390_match_ccmode(insn, CCSmode) && !TARGET_64BIT"
-  "srda\\t%0,0"
-  [(set_attr "op_type" "RS")])
+  "srda\t%0,0"
+  [(set_attr "op_type" "RS")
+   (set_attr "atype"   "reg")])
+
 
 (define_insn "*tstsi"
   [(set (reg 33)
-        (compare (match_operand:SI 0 "nonimmediate_operand" "d,Q")
+        (compare (match_operand:SI 0 "nonimmediate_operand" "d,Q,S")
                  (match_operand:SI 1 "const0_operand" "")))
-   (set (match_operand:SI 2 "register_operand" "=d,d")
+   (set (match_operand:SI 2 "register_operand" "=d,d,d")
         (match_dup 0))]
   "s390_match_ccmode(insn, CCSmode)"
   "@
-   ltr\\t%2,%0
-   icm\\t%2,15,%0"
-  [(set_attr "op_type" "RR,RS")
-   (set_attr "atype"   "reg,mem")])
+   ltr\t%2,%0
+   icm\t%2,15,%0
+   icmy\t%2,15,%0"
+  [(set_attr "op_type" "RR,RS,RSY")])
 
 (define_insn "*tstsi_cconly"
   [(set (reg 33)
-        (compare (match_operand:SI 0 "nonimmediate_operand" "d,Q")
+        (compare (match_operand:SI 0 "nonimmediate_operand" "d,Q,S")
                  (match_operand:SI 1 "const0_operand" "")))
-   (clobber (match_scratch:SI 2 "=X,d"))]
+   (clobber (match_scratch:SI 2 "=X,d,d"))]
   "s390_match_ccmode(insn, CCSmode)"
   "@
-   ltr\\t%0,%0
-   icm\\t%2,15,%0"
-  [(set_attr "op_type" "RR,RS")
-   (set_attr "atype"   "reg,mem")])
+   ltr\t%0,%0
+   icm\t%2,15,%0
+   icmy\t%2,15,%0"
+  [(set_attr "op_type" "RR,RS,RSY")])
 
 (define_insn "*tstsi_cconly2"
   [(set (reg 33)
         (compare (match_operand:SI 0 "register_operand" "d")
                  (match_operand:SI 1 "const0_operand" "")))]
   "s390_match_ccmode(insn, CCSmode)"
-  "ltr\\t%0,%0"
+  "ltr\t%0,%0"
   [(set_attr "op_type" "RR")])
 
+(define_insn "*tsthiCCT"
+  [(set (reg 33)
+        (compare (match_operand:HI 0 "nonimmediate_operand" "?Q,?S,d")
+                 (match_operand:HI 1 "const0_operand" "")))
+   (set (match_operand:HI 2 "register_operand" "=d,d,0")
+        (match_dup 0))]
+  "s390_match_ccmode(insn, CCTmode)"
+  "@
+   icm\t%2,3,%0
+   icmy\t%2,3,%0
+   tml\t%0,65535"
+  [(set_attr "op_type" "RS,RSY,RI")])
+
+(define_insn "*tsthiCCT_cconly"
+  [(set (reg 33)
+        (compare (match_operand:HI 0 "nonimmediate_operand" "Q,S,d")
+                 (match_operand:HI 1 "const0_operand" "")))
+   (clobber (match_scratch:HI 2 "=d,d,X"))]
+  "s390_match_ccmode(insn, CCTmode)"
+  "@
+   icm\t%2,3,%0
+   icmy\t%2,3,%0
+   tml\t%0,65535"
+  [(set_attr "op_type" "RS,RSY,RI")])
+
 (define_insn "*tsthi"
   [(set (reg 33)
-        (compare (match_operand:HI 0 "s_operand" "Q")
+        (compare (match_operand:HI 0 "s_operand" "Q,S")
                  (match_operand:HI 1 "const0_operand" "")))
-   (set (match_operand:HI 2 "register_operand" "=d")
+   (set (match_operand:HI 2 "register_operand" "=d,d")
         (match_dup 0))]
   "s390_match_ccmode(insn, CCSmode)"
-  "icm\\t%2,3,%0"
-  [(set_attr "op_type" "RS")
-   (set_attr "atype"   "mem")])
+  "@
+   icm\t%2,3,%0
+   icmy\t%2,3,%0"
+  [(set_attr "op_type" "RS,RSY")])
 
 (define_insn "*tsthi_cconly"
   [(set (reg 33)
-        (compare (match_operand:HI 0 "s_operand" "Q")
+        (compare (match_operand:HI 0 "s_operand" "Q,S")
                  (match_operand:HI 1 "const0_operand" "")))
-   (clobber (match_scratch:HI 2 "=d"))]
+   (clobber (match_scratch:HI 2 "=d,d"))]
   "s390_match_ccmode(insn, CCSmode)"
-  "icm\\t%2,3,%0"
-  [(set_attr "op_type" "RS")
-   (set_attr "atype"   "mem")])
+  "@
+   icm\t%2,3,%0
+   icmy\t%2,3,%0"
+  [(set_attr "op_type" "RS,RSY")])
+
+(define_insn "*tstqiCCT"
+  [(set (reg 33)
+        (compare (match_operand:QI 0 "nonimmediate_operand" "?Q,?S,d")
+                 (match_operand:QI 1 "const0_operand" "")))
+   (set (match_operand:QI 2 "register_operand" "=d,d,0")
+        (match_dup 0))]
+  "s390_match_ccmode(insn, CCTmode)"
+  "@
+   icm\t%2,1,%0
+   icmy\t%2,1,%0
+   tml\t%0,255"
+  [(set_attr "op_type" "RS,RSY,RI")])
+
+(define_insn "*tstqiCCT_cconly"
+  [(set (reg 33)
+        (compare (match_operand:QI 0 "nonimmediate_operand" "?Q,?S,d")
+                 (match_operand:QI 1 "const0_operand" "")))]
+  "s390_match_ccmode(insn, CCTmode)"
+  "@
+   cli\t%0,0
+   cliy\t%0,0
+   tml\t%0,255"
+  [(set_attr "op_type" "SI,SIY,RI")])
 
 (define_insn "*tstqi"
   [(set (reg 33)
-        (compare (match_operand:QI 0 "s_operand" "Q")
+        (compare (match_operand:QI 0 "s_operand" "Q,S")
                  (match_operand:QI 1 "const0_operand" "")))
-   (set (match_operand:QI 2 "register_operand" "=d")
+   (set (match_operand:QI 2 "register_operand" "=d,d")
         (match_dup 0))]
   "s390_match_ccmode(insn, CCSmode)"
-  "icm\\t%2,1,%0"
-  [(set_attr "op_type" "RS")
-   (set_attr "atype"   "mem")])
+  "@
+   icm\t%2,1,%0
+   icmy\t%2,1,%0"
+  [(set_attr "op_type" "RS,RSY")])
 
 (define_insn "*tstqi_cconly"
   [(set (reg 33)
-        (compare (match_operand:QI 0 "s_operand" "Q")
+        (compare (match_operand:QI 0 "s_operand" "Q,S")
                  (match_operand:QI 1 "const0_operand" "")))
-   (clobber (match_scratch:QI 2 "=d"))]
+   (clobber (match_scratch:QI 2 "=d,d"))]
   "s390_match_ccmode(insn, CCSmode)"
-  "icm\\t%2,1,%0"
-  [(set_attr "op_type" "RS")
-   (set_attr "atype"   "mem")])
+  "@
+   icm\t%2,1,%0
+   icmy\t%2,1,%0"
+  [(set_attr "op_type" "RS,RSY")])
 
 
 ; Compare (signed) instructions
@@ -551,10 +522,9 @@
                  (match_operand:DI 0 "register_operand" "d,d")))]
   "s390_match_ccmode(insn, CCSRmode) && TARGET_64BIT"
   "@
-   cgfr\\t%0,%1
-   cgf\\t%0,%1"
-  [(set_attr "op_type" "RRE,RXE")
-   (set_attr "atype"   "reg,mem")])
+   cgfr\t%0,%1
+   cgf\t%0,%1"
+  [(set_attr "op_type" "RRE,RXY")])
 
 (define_insn "*cmpdi_ccs"
   [(set (reg 33)
@@ -562,33 +532,33 @@
                  (match_operand:DI 1 "general_operand" "d,K,m")))]
   "s390_match_ccmode(insn, CCSmode) && TARGET_64BIT"
   "@
-   cgr\\t%0,%1
-   cghi\\t%0,%c1
-   cg\\t%0,%1"
-  [(set_attr "op_type" "RRE,RI,RXE")
-   (set_attr "atype"    "reg,reg,mem")])
-   
+   cgr\t%0,%1
+   cghi\t%0,%c1
+   cg\t%0,%1"
+  [(set_attr "op_type" "RRE,RI,RXY")])
+
 (define_insn "*cmpsi_ccs_sign"
   [(set (reg 33)
-        (compare (sign_extend:SI (match_operand:HI 1 "memory_operand" "m"))
-                 (match_operand:SI 0 "register_operand" "d")))]
+        (compare (sign_extend:SI (match_operand:HI 1 "memory_operand" "R,T"))
+                 (match_operand:SI 0 "register_operand" "d,d")))]
   "s390_match_ccmode(insn, CCSRmode)"
-  "ch\\t%0,%1"
-  [(set_attr "op_type" "RX")
-   (set_attr "atype"   "mem")])
+  "@
+   ch\t%0,%1
+   chy\t%0,%1"
+  [(set_attr "op_type" "RX,RXY")])
 
 (define_insn "*cmpsi_ccs"
   [(set (reg 33)
-        (compare (match_operand:SI 0 "register_operand" "d,d,d")
-                 (match_operand:SI 1 "general_operand" "d,K,m")))]
+        (compare (match_operand:SI 0 "register_operand" "d,d,d,d")
+                 (match_operand:SI 1 "general_operand" "d,K,R,T")))]
   "s390_match_ccmode(insn, CCSmode)"
   "@
-   cr\\t%0,%1
-   chi\\t%0,%c1
-   c\\t%0,%1"
-  [(set_attr "op_type" "RR,RI,RX")
-   (set_attr "atype"   "reg,reg,mem")])
-   
+   cr\t%0,%1
+   chi\t%0,%c1
+   c\t%0,%1
+   cy\t%0,%1"
+  [(set_attr "op_type" "RR,RI,RX,RXY")])
+
 
 ; Compare (unsigned) instructions
 
@@ -598,10 +568,9 @@
                  (match_operand:DI 0 "register_operand" "d,d")))]
   "s390_match_ccmode(insn, CCURmode) && TARGET_64BIT"
   "@
-   clgfr\\t%0,%1
-   clgf\\t%0,%1"
-  [(set_attr "op_type" "RRE,RXE")
-   (set_attr "atype"   "reg,mem")])
+   clgfr\t%0,%1
+   clgf\t%0,%1"
+  [(set_attr "op_type" "RRE,RXY")])
 
 (define_insn "*cmpdi_ccu"
   [(set (reg 33)
@@ -609,84 +578,82 @@
                  (match_operand:DI 1 "general_operand" "d,m")))]
   "s390_match_ccmode(insn, CCUmode) && TARGET_64BIT"
   "@
-   clgr\\t%0,%1
-   clg\\t%0,%1"
-  [(set_attr "op_type" "RRE,RXE")
-   (set_attr "atype"   "reg,mem")])
+   clgr\t%0,%1
+   clg\t%0,%1"
+  [(set_attr "op_type" "RRE,RXY")])
 
 (define_insn "*cmpsi_ccu"
   [(set (reg 33)
-        (compare (match_operand:SI 0 "register_operand" "d,d")
-                 (match_operand:SI 1 "general_operand" "d,m")))]
+        (compare (match_operand:SI 0 "register_operand" "d,d,d")
+                 (match_operand:SI 1 "general_operand" "d,R,T")))]
   "s390_match_ccmode(insn, CCUmode)"
   "@
-   clr\\t%0,%1
-   cl\\t%0,%1"
-  [(set_attr "op_type" "RR,RX")
-   (set_attr "atype"   "reg,mem")])
+   clr\t%0,%1
+   cl\t%0,%1
+   cly\t%0,%1"
+  [(set_attr "op_type" "RR,RX,RXY")])
 
 (define_insn "*cmphi_ccu"
   [(set (reg 33)
-        (compare (match_operand:HI 0 "register_operand" "d")
-                 (match_operand:HI 1 "s_imm_operand" "Q")))]
+        (compare (match_operand:HI 0 "register_operand" "d,d")
+                 (match_operand:HI 1 "s_imm_operand" "Q,S")))]
   "s390_match_ccmode(insn, CCUmode)"
-  "clm\\t%0,3,%1"
-  [(set_attr "op_type" "RS")
-   (set_attr "atype"   "mem")])
+  "@
+   clm\t%0,3,%1
+   clmy\t%0,3,%1"
+  [(set_attr "op_type" "RS,RSY")])
 
 (define_insn "*cmpqi_ccu"
   [(set (reg 33)
-        (compare (match_operand:QI 0 "register_operand" "d")
-                 (match_operand:QI 1 "s_imm_operand" "Q")))]
+        (compare (match_operand:QI 0 "register_operand" "d,d")
+                 (match_operand:QI 1 "s_imm_operand" "Q,S")))]
   "s390_match_ccmode(insn, CCUmode)"
-  "clm\\t%0,1,%1"
-  [(set_attr "op_type" "RS")
-   (set_attr "atype"   "mem")])
+  "@
+   clm\t%0,1,%1
+   clmy\t%0,1,%1"
+  [(set_attr "op_type" "RS,RSY")])
 
 (define_insn "*cli"
   [(set (reg 33)
-        (compare (match_operand:QI 0 "memory_operand" "Q")
-                 (match_operand:QI 1 "immediate_operand" "n")))]
+        (compare (match_operand:QI 0 "memory_operand" "Q,S")
+                 (match_operand:QI 1 "immediate_operand" "n,n")))]
   "s390_match_ccmode (insn, CCUmode)"
-  "cli\\t%0,%b1"
-  [(set_attr "op_type" "SI")
-   (set_attr "atype"   "mem")])
+  "@
+   cli\t%0,%b1
+   cliy\t%0,%b1"
+  [(set_attr "op_type" "SI,SIY")])
 
 (define_insn "*cmpdi_ccu_mem"
   [(set (reg 33)
         (compare (match_operand:DI 0 "s_operand" "Q")
                  (match_operand:DI 1 "s_imm_operand" "Q")))]
   "s390_match_ccmode(insn, CCUmode)"
-  "clc\\t%O0(8,%R0),%1"
-  [(set_attr "op_type" "SS")
-   (set_attr "atype"   "mem")])
+  "clc\t%O0(8,%R0),%1"
+  [(set_attr "op_type" "SS")])
 
 (define_insn "*cmpsi_ccu_mem"
   [(set (reg 33)
         (compare (match_operand:SI 0 "s_operand" "Q")
                  (match_operand:SI 1 "s_imm_operand" "Q")))]
   "s390_match_ccmode(insn, CCUmode)"
-  "clc\\t%O0(4,%R0),%1"
-   [(set_attr "op_type" "SS")
-    (set_attr "atype"   "mem")])
+  "clc\t%O0(4,%R0),%1"
+   [(set_attr "op_type" "SS")])
 
 (define_insn "*cmphi_ccu_mem"
   [(set (reg 33)
         (compare (match_operand:HI 0 "s_operand" "Q")
                  (match_operand:HI 1 "s_imm_operand" "Q")))]
   "s390_match_ccmode(insn, CCUmode)"
-  "clc\\t%O0(2,%R0),%1"
-  [(set_attr "op_type" "SS")
-   (set_attr "atype"   "mem")])
+  "clc\t%O0(2,%R0),%1"
+  [(set_attr "op_type" "SS")])
 
 (define_insn "*cmpqi_ccu_mem"
   [(set (reg 33)
         (compare (match_operand:QI 0 "s_operand" "Q")
                  (match_operand:QI 1 "s_imm_operand" "Q")))]
   "s390_match_ccmode(insn, CCUmode)"
-  "clc\\t%O0(1,%R0),%1"
-  [(set_attr "op_type" "SS")
-   (set_attr "atype"   "mem")])
+  "clc\t%O0(1,%R0),%1"
+  [(set_attr "op_type" "SS")])
 
 
 ; DF instructions
@@ -696,38 +663,40 @@
         (compare (match_operand:DF 0 "register_operand" "f")
                  (match_operand:DF 1 "const0_operand" "")))]
   "s390_match_ccmode(insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "ltdbr\\t%0,%0"
-   [(set_attr "op_type" "RRE")])
+  "ltdbr\t%0,%0"
+   [(set_attr "op_type" "RRE")
+    (set_attr "type"  "fsimpd")])
 
 (define_insn "*cmpdf_ccs_0_ibm"
   [(set (reg 33)
         (compare (match_operand:DF 0 "register_operand" "f")
                  (match_operand:DF 1 "const0_operand" "")))]
   "s390_match_ccmode(insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
-  "ltdr\\t%0,%0"
-   [(set_attr "op_type" "RR")])
+  "ltdr\t%0,%0"
+   [(set_attr "op_type" "RR")
+    (set_attr "type"  "fsimpd")])
 
 (define_insn "*cmpdf_ccs"
   [(set (reg 33)
         (compare (match_operand:DF 0 "register_operand" "f,f")
-                 (match_operand:DF 1 "general_operand" "f,m")))]
+                 (match_operand:DF 1 "general_operand" "f,R")))]
   "s390_match_ccmode(insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   cdbr\\t%0,%1
-   cdb\\t%0,%1"
+   cdbr\t%0,%1
+   cdb\t%0,%1"
    [(set_attr "op_type" "RRE,RXE")
-    (set_attr "atype"   "reg,mem")])  
+    (set_attr "type"  "fsimpd")])
 
 (define_insn "*cmpdf_ccs_ibm"
   [(set (reg 33)
         (compare (match_operand:DF 0 "register_operand" "f,f")
-                 (match_operand:DF 1 "general_operand" "f,m")))]
+                 (match_operand:DF 1 "general_operand" "f,R")))]
   "s390_match_ccmode(insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
   "@
-   cdr\\t%0,%1
-   cd\\t%0,%1"
+   cdr\t%0,%1
+   cd\t%0,%1"
    [(set_attr "op_type" "RR,RX")
-    (set_attr "atype"   "reg,mem")])  
+    (set_attr "type"  "fsimpd")])
 
 
 ; SF instructions
@@ -737,38 +706,40 @@
         (compare (match_operand:SF 0 "register_operand" "f")
                  (match_operand:SF 1 "const0_operand" "")))]
   "s390_match_ccmode(insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "ltebr\\t%0,%0"
-   [(set_attr "op_type" "RRE")])
+  "ltebr\t%0,%0"
+   [(set_attr "op_type" "RRE")
+    (set_attr "type"  "fsimps")])
 
 (define_insn "*cmpsf_ccs_0_ibm"
   [(set (reg 33)
         (compare (match_operand:SF 0 "register_operand" "f")
                  (match_operand:SF 1 "const0_operand" "")))]
   "s390_match_ccmode(insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
-  "lter\\t%0,%0"
-   [(set_attr "op_type" "RR")])
+  "lter\t%0,%0"
+   [(set_attr "op_type" "RR")
+    (set_attr "type"  "fsimps")])
 
 (define_insn "*cmpsf_ccs"
   [(set (reg 33)
         (compare (match_operand:SF 0 "register_operand" "f,f")
-                 (match_operand:SF 1 "general_operand" "f,m")))]
+                 (match_operand:SF 1 "general_operand" "f,R")))]
   "s390_match_ccmode(insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   cebr\\t%0,%1
-   ceb\\t%0,%1"
-  [(set_attr "op_type" "RRE,RXE")
-   (set_attr "atype"   "reg,mem")])
+   cebr\t%0,%1
+   ceb\t%0,%1"
+   [(set_attr "op_type" "RRE,RXE")
+    (set_attr "type"  "fsimps")])
 
 (define_insn "*cmpsf_ccs"
   [(set (reg 33)
         (compare (match_operand:SF 0 "register_operand" "f,f")
-                 (match_operand:SF 1 "general_operand" "f,m")))]
+                 (match_operand:SF 1 "general_operand" "f,R")))]
   "s390_match_ccmode(insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
   "@
-   cer\\t%0,%1
-   ce\\t%0,%1"
-  [(set_attr "op_type" "RR,RX")
-   (set_attr "atype"   "reg,mem")])
+   cer\t%0,%1
+   ce\t%0,%1"
+   [(set_attr "op_type" "RR,RX")
+    (set_attr "type"  "fsimps")])
 
 
 ;;
@@ -780,52 +751,45 @@
 ;
 
 (define_insn "movti"
-  [(set (match_operand:TI 0 "nonimmediate_operand" "=d,Q,d,m,Q")
-        (match_operand:TI 1 "general_operand" "Q,d,dKm,d,Q"))]
+  [(set (match_operand:TI 0 "nonimmediate_operand" "=d,QS,d,o,Q")
+        (match_operand:TI 1 "general_operand" "QS,d,dKm,d,Q"))]
   "TARGET_64BIT"
   "@
-   lmg\\t%0,%N0,%1
-   stmg\\t%1,%N1,%0
+   lmg\t%0,%N0,%1
+   stmg\t%1,%N1,%0
    #
    #
-   mvc\\t%O0(16,%R0),%1"
-  [(set_attr "op_type" "RSE,RSE,NN,NN,SS")
-   (set_attr "atype"   "mem")])
+   mvc\t%O0(16,%R0),%1"
+  [(set_attr "op_type" "RSY,RSY,NN,NN,SS")
+   (set_attr "type" "lm,stm,*,*,cs")])
 
 (define_split
   [(set (match_operand:TI 0 "nonimmediate_operand" "")
         (match_operand:TI 1 "general_operand" ""))]
   "TARGET_64BIT && reload_completed
-   && !s_operand (operands[0], VOIDmode)
-   && !s_operand (operands[1], VOIDmode)
-   && (register_operand (operands[0], VOIDmode)
-       || register_operand (operands[1], VOIDmode))
-   && (!register_operand (operands[0], VOIDmode)
-       || !reg_overlap_mentioned_p (operand_subword (operands[0], 0, 0, TImode),
-                                    operands[1])
-       || !reg_overlap_mentioned_p (operand_subword (operands[0], 1, 0, TImode),
-                                    operands[1]))"
+   && s390_split_ok_p (operands[0], operands[1], TImode, 0)"
   [(set (match_dup 2) (match_dup 4))
    (set (match_dup 3) (match_dup 5))]
-  "
 {
-  if (!register_operand (operands[0], VOIDmode)
-      || !reg_overlap_mentioned_p (operand_subword (operands[0], 0, 0, TImode),
-                                   operands[1]))
-    {
-      operands[2] = operand_subword (operands[0], 0, 0, TImode);
-      operands[3] = operand_subword (operands[0], 1, 0, TImode);
-      operands[4] = operand_subword (operands[1], 0, 0, TImode);
-      operands[5] = operand_subword (operands[1], 1, 0, TImode);
-    }
-  else
-    {
-      operands[2] = operand_subword (operands[0], 1, 0, TImode);
-      operands[3] = operand_subword (operands[0], 0, 0, TImode);
-      operands[4] = operand_subword (operands[1], 1, 0, TImode);
-      operands[5] = operand_subword (operands[1], 0, 0, TImode);
-    }
-}")
+  operands[2] = operand_subword (operands[0], 0, 0, TImode);
+  operands[3] = operand_subword (operands[0], 1, 0, TImode);
+  operands[4] = operand_subword (operands[1], 0, 0, TImode);
+  operands[5] = operand_subword (operands[1], 1, 0, TImode);
+})
+
+(define_split
+  [(set (match_operand:TI 0 "nonimmediate_operand" "")
+        (match_operand:TI 1 "general_operand" ""))]
+  "TARGET_64BIT && reload_completed
+   && s390_split_ok_p (operands[0], operands[1], TImode, 1)"
+  [(set (match_dup 2) (match_dup 4))
+   (set (match_dup 3) (match_dup 5))]
+{
+  operands[2] = operand_subword (operands[0], 1, 0, TImode);
+  operands[3] = operand_subword (operands[0], 0, 0, TImode);
+  operands[4] = operand_subword (operands[1], 1, 0, TImode);
+  operands[5] = operand_subword (operands[1], 0, 0, TImode);
+})
 
 (define_split
   [(set (match_operand:TI 0 "register_operand" "")
@@ -833,184 +797,223 @@
   "TARGET_64BIT && reload_completed
    && !s_operand (operands[1], VOIDmode)"
   [(set (match_dup 0) (match_dup 1))]
-  "
 {
   rtx addr = operand_subword (operands[0], 1, 0, TImode);
   s390_load_address (addr, XEXP (operands[1], 0));
   operands[1] = replace_equiv_address (operands[1], addr);
-}")
+})
+
+(define_expand "reload_outti"
+  [(parallel [(match_operand:TI 0 "memory_operand" "")
+              (match_operand:TI 1 "register_operand" "d")
+              (match_operand:DI 2 "register_operand" "=&a")])]
+  "TARGET_64BIT"
+{
+  s390_load_address (operands[2], XEXP (operands[0], 0));
+  operands[0] = replace_equiv_address (operands[0], operands[2]);
+  emit_move_insn (operands[0], operands[1]);
+  DONE;
+})
 
 ;
 ; movdi instruction pattern(s).
 ;
 
-;; If generating PIC code and operands[1] is a symbolic CONST, emit a
-;; move to get the address of the symbolic object from the GOT.
-
 (define_expand "movdi"
   [(set (match_operand:DI 0 "general_operand" "")
         (match_operand:DI 1 "general_operand" ""))]
   ""
-  "
 {
-  /* Handle PIC symbolic constants.  */
-  if (TARGET_64BIT && flag_pic && SYMBOLIC_CONST (operands[1]))
-    emit_pic_move (operands, DImode);
+  /* Handle symbolic constants.  */
+  if (TARGET_64BIT && SYMBOLIC_CONST (operands[1]))
+    emit_symbolic_move (operands);
 
   /* During and after reload, we need to force constants
      to the literal pool ourselves, if necessary.  */
   if ((reload_in_progress || reload_completed)
-      && CONSTANT_P (operands[1]) 
+      && CONSTANT_P (operands[1])
       && (!legitimate_reload_constant_p (operands[1])
-          || fp_operand (operands[0], VOIDmode)))
+          || FP_REG_P (operands[0])))
     operands[1] = force_const_mem (DImode, operands[1]);
-}")
-
-(define_insn "*movdi_lhi"
-  [(set (match_operand:DI 0 "register_operand" "=d")
-        (match_operand:DI 1 "immediate_operand" "K"))]
-  "TARGET_64BIT
-   && GET_CODE (operands[1]) == CONST_INT
-   && CONST_OK_FOR_LETTER_P (INTVAL (operands[1]), 'K')
-   && !fp_operand (operands[0], VOIDmode)"
-  "lghi\\t%0,%h1"
-  [(set_attr "op_type" "RI")
-   (set_attr "atype"   "reg")])
-
-(define_insn "*movdi_lli"
-  [(set (match_operand:DI 0 "register_operand" "=d")
-        (match_operand:DI 1 "immediate_operand" "n"))]
-  "TARGET_64BIT && s390_single_hi (operands[1], DImode, 0) >= 0
-   && !fp_operand (operands[0], VOIDmode)"
-  "*
-{
-  int part = s390_single_hi (operands[1], DImode, 0);
-  operands[1] = GEN_INT (s390_extract_hi (operands[1], DImode, part));
-
-  switch (part)
-    {
-      case 0: return \"llihh\\t%0,%x1\";
-      case 1: return \"llihl\\t%0,%x1\";
-      case 2: return \"llilh\\t%0,%x1\";
-      case 3: return \"llill\\t%0,%x1\";
-      default: abort ();
-    }
-}"
-  [(set_attr "op_type" "RI")
-   (set_attr "atype"   "reg")])
+})
 
 (define_insn "*movdi_larl"
   [(set (match_operand:DI 0 "register_operand" "=d")
         (match_operand:DI 1 "larl_operand" "X"))]
   "TARGET_64BIT
-   && !fp_operand (operands[0], VOIDmode)"
-  "larl\\t%0,%1"
+   && !FP_REG_P (operands[0])"
+  "larl\t%0,%1"
    [(set_attr "op_type" "RIL")
-    (set_attr "atype"   "reg")
-    (set_attr "type"    "la")])
+    (set_attr "type"    "larl")])
 
 (define_insn "*movdi_64"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "=d,d,m,!*f,!*f,!m,Q")
-        (match_operand:DI 1 "general_operand" "d,m,d,*f,m,*f,Q"))]
+  [(set (match_operand:DI 0 "nonimmediate_operand"
+                            "=d,d,d,d,d,d,d,d,m,!*f,!*f,!*f,!R,!T,?Q")
+        (match_operand:DI 1 "general_operand"
+                            "K,N0HD0,N1HD0,N2HD0,N3HD0,L,d,m,d,*f,R,T,*f,*f,?Q"))]
   "TARGET_64BIT"
   "@
-   lgr\\t%0,%1
-   lg\\t%0,%1
-   stg\\t%1,%0
-   ldr\\t%0,%1
-   ld\\t%0,%1
-   std\\t%1,%0
-   mvc\\t%O0(8,%R0),%1"
-  [(set_attr "op_type" "RR,RXE,RXE,RR,RX,RX,SS")
-   (set_attr "atype"   "reg,mem,mem,reg,mem,mem,mem")])
+   lghi\t%0,%h1
+   llihh\t%0,%i1
+   llihl\t%0,%i1
+   llilh\t%0,%i1
+   llill\t%0,%i1
+   lay\t%0,%a1
+   lgr\t%0,%1
+   lg\t%0,%1
+   stg\t%1,%0
+   ldr\t%0,%1
+   ld\t%0,%1
+   ldy\t%0,%1
+   std\t%1,%0
+   stdy\t%1,%0
+   mvc\t%O0(8,%R0),%1"
+  [(set_attr "op_type" "RI,RI,RI,RI,RI,RXY,RRE,RXY,RXY,RR,RX,RXY,RX,RXY,SS")
+   (set_attr "type" "*,*,*,*,*,la,lr,load,store,floadd,floadd,floadd,
+                     fstored,fstored,cs")])
 
 (define_insn "*movdi_31"
-  [(set (match_operand:DI 0 "nonimmediate_operand" "=d,Q,d,m,!*f,!*f,!m,Q")
-        (match_operand:DI 1 "general_operand" "Q,d,dKm,d,*f,m,*f,Q"))]
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=d,Q,d,o,!*f,!*f,!*f,!R,!T,Q")
+        (match_operand:DI 1 "general_operand" "Q,d,dKm,d,*f,R,T,*f,*f,Q"))]
   "!TARGET_64BIT"
   "@
-   lm\\t%0,%N0,%1
-   stm\\t%1,%N1,%0
+   lm\t%0,%N0,%1
+   stm\t%1,%N1,%0
    #
    #
-   ldr\\t%0,%1
-   ld\\t%0,%1
-   std\\t%1,%0
-   mvc\\t%O0(8,%R0),%1"
-  [(set_attr "op_type" "RS,RS,NN,NN,RR,RX,RX,SS")
-   (set_attr "atype"   "mem,mem,*,*,reg,mem,mem,mem")])
+   ldr\t%0,%1
+   ld\t%0,%1
+   ldy\t%0,%1
+   std\t%1,%0
+   stdy\t%1,%0
+   mvc\t%O0(8,%R0),%1"
+  [(set_attr "op_type" "RS,RS,NN,NN,RR,RX,RXY,RX,RXY,SS")
+   (set_attr "type" "lm,stm,*,*,floadd,floadd,floadd,fstored,fstored,cs")])
 
 (define_split
   [(set (match_operand:DI 0 "nonimmediate_operand" "")
         (match_operand:DI 1 "general_operand" ""))]
   "!TARGET_64BIT && reload_completed
-   && !fp_operand (operands[0], VOIDmode)
-   && !fp_operand (operands[1], VOIDmode)
-   && !s_operand (operands[0], VOIDmode)
-   && !s_operand (operands[1], VOIDmode)
-   && (register_operand (operands[0], VOIDmode)
-       || register_operand (operands[1], VOIDmode))
-   && (!register_operand (operands[0], VOIDmode)
-       || !reg_overlap_mentioned_p (operand_subword (operands[0], 0, 0, DImode),
-                                    operands[1])
-       || !reg_overlap_mentioned_p (operand_subword (operands[0], 1, 0, DImode),
-                                    operands[1]))"
+   && s390_split_ok_p (operands[0], operands[1], DImode, 0)"
   [(set (match_dup 2) (match_dup 4))
    (set (match_dup 3) (match_dup 5))]
-  "
 {
-  if (!register_operand (operands[0], VOIDmode)
-      || !reg_overlap_mentioned_p (operand_subword (operands[0], 0, 0, DImode),
-                                   operands[1]))
-    {
-      operands[2] = operand_subword (operands[0], 0, 0, DImode);
-      operands[3] = operand_subword (operands[0], 1, 0, DImode);
-      operands[4] = operand_subword (operands[1], 0, 0, DImode);
-      operands[5] = operand_subword (operands[1], 1, 0, DImode);
-    }
-  else
-    {
-      operands[2] = operand_subword (operands[0], 1, 0, DImode);
-      operands[3] = operand_subword (operands[0], 0, 0, DImode);
-      operands[4] = operand_subword (operands[1], 1, 0, DImode);
-      operands[5] = operand_subword (operands[1], 0, 0, DImode);
-    }
-}")
+  operands[2] = operand_subword (operands[0], 0, 0, DImode);
+  operands[3] = operand_subword (operands[0], 1, 0, DImode);
+  operands[4] = operand_subword (operands[1], 0, 0, DImode);
+  operands[5] = operand_subword (operands[1], 1, 0, DImode);
+})
+
+(define_split
+  [(set (match_operand:DI 0 "nonimmediate_operand" "")
+        (match_operand:DI 1 "general_operand" ""))]
+  "!TARGET_64BIT && reload_completed
+   && s390_split_ok_p (operands[0], operands[1], DImode, 1)"
+  [(set (match_dup 2) (match_dup 4))
+   (set (match_dup 3) (match_dup 5))]
+{
+  operands[2] = operand_subword (operands[0], 1, 0, DImode);
+  operands[3] = operand_subword (operands[0], 0, 0, DImode);
+  operands[4] = operand_subword (operands[1], 1, 0, DImode);
+  operands[5] = operand_subword (operands[1], 0, 0, DImode);
+})
 
 (define_split
   [(set (match_operand:DI 0 "register_operand" "")
         (match_operand:DI 1 "memory_operand" ""))]
   "!TARGET_64BIT && reload_completed
-   && !fp_operand (operands[0], VOIDmode)
-   && !fp_operand (operands[1], VOIDmode)
+   && !FP_REG_P (operands[0])
    && !s_operand (operands[1], VOIDmode)"
   [(set (match_dup 0) (match_dup 1))]
-  "
 {
   rtx addr = operand_subword (operands[0], 1, 0, DImode);
   s390_load_address (addr, XEXP (operands[1], 0));
   operands[1] = replace_equiv_address (operands[1], addr);
-}")
+})
+
+(define_expand "reload_outdi"
+  [(parallel [(match_operand:DI 0 "memory_operand" "")
+              (match_operand:DI 1 "register_operand" "d")
+              (match_operand:SI 2 "register_operand" "=&a")])]
+  "!TARGET_64BIT"
+{
+  s390_load_address (operands[2], XEXP (operands[0], 0));
+  operands[0] = replace_equiv_address (operands[0], operands[2]);
+  emit_move_insn (operands[0], operands[1]);
+  DONE;
+})
+
+(define_peephole2
+  [(set (match_operand:DI 0 "register_operand" "")
+        (mem:DI (match_operand 1 "address_operand" "")))]
+  "TARGET_64BIT
+   && !FP_REG_P (operands[0])
+   && GET_CODE (operands[1]) == SYMBOL_REF
+   && CONSTANT_POOL_ADDRESS_P (operands[1])
+   && get_pool_mode (operands[1]) == DImode
+   && legitimate_reload_constant_p (get_pool_constant (operands[1]))"
+  [(set (match_dup 0) (match_dup 2))]
+  "operands[2] = get_pool_constant (operands[1]);")
+
+(define_insn "*la_64"
+  [(set (match_operand:DI 0 "register_operand" "=d,d")
+        (match_operand:QI 1 "address_operand" "U,W"))]
+  "TARGET_64BIT"
+  "@
+   la\t%0,%a1
+   lay\t%0,%a1"
+  [(set_attr "op_type" "RX,RXY")
+   (set_attr "type"    "la")])
+
+(define_peephole2
+  [(parallel
+    [(set (match_operand:DI 0 "register_operand" "")
+          (match_operand:QI 1 "address_operand" ""))
+     (clobber (reg:CC 33))])]
+  "TARGET_64BIT
+   && preferred_la_operand_p (operands[1], const0_rtx)"
+  [(set (match_dup 0) (match_dup 1))]
+  "")
+
+(define_peephole2
+  [(set (match_operand:DI 0 "register_operand" "")
+        (match_operand:DI 1 "register_operand" ""))
+   (parallel
+    [(set (match_dup 0)
+          (plus:DI (match_dup 0)
+                   (match_operand:DI 2 "nonmemory_operand" "")))
+     (clobber (reg:CC 33))])]
+  "TARGET_64BIT
+   && !reg_overlap_mentioned_p (operands[0], operands[2])
+   && preferred_la_operand_p (operands[1], operands[2])"
+  [(set (match_dup 0) (plus:DI (match_dup 1) (match_dup 2)))]
+  "")
+
+(define_expand "reload_indi"
+  [(parallel [(match_operand:DI 0 "register_operand" "=a")
+              (match_operand:DI 1 "s390_plus_operand" "")
+              (match_operand:DI 2 "register_operand" "=&a")])]
+  "TARGET_64BIT"
+{
+  s390_expand_plus_operand (operands[0], operands[1], operands[2]);
+  DONE;
+})
 
 ;
 ; movsi instruction pattern(s).
 ;
 
-;; If generating PIC code and operands[1] is a symbolic CONST, emit a
-;; move to get the address of the symbolic object from the GOT.
-
 (define_expand "movsi"
   [(set (match_operand:SI 0 "general_operand" "")
         (match_operand:SI 1 "general_operand" ""))]
   ""
-  "
 {
-  /* Handle PIC symbolic constants.  */
-  if (!TARGET_64BIT && flag_pic && SYMBOLIC_CONST (operands[1]))
-    emit_pic_move (operands, SImode);
+  /* Handle symbolic constants.  */
+  if (!TARGET_64BIT && SYMBOLIC_CONST (operands[1]))
+    emit_symbolic_move (operands);
 
-  /* expr.c tries to load an effective address using 
-     force_reg.  This fails because we don't have a 
+  /* expr.c tries to load an effective address using
+     force_reg.  This fails because we don't have a
      generic load_address pattern.  Convert the move
      to a proper arithmetic operation instead, unless
      it is guaranteed to be OK.  */
@@ -1025,148 +1028,290 @@
   /* During and after reload, we need to force constants
      to the literal pool ourselves, if necessary.  */
   if ((reload_in_progress || reload_completed)
-      && CONSTANT_P (operands[1]) 
+      && CONSTANT_P (operands[1])
       && (!legitimate_reload_constant_p (operands[1])
-          || fp_operand (operands[0], VOIDmode)))
+          || FP_REG_P (operands[0])))
     operands[1] = force_const_mem (SImode, operands[1]);
-}")
-
-(define_insn "*movsi_lhi"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (match_operand:SI 1 "immediate_operand" "K"))]
-  "GET_CODE (operands[1]) == CONST_INT
-   && CONST_OK_FOR_LETTER_P (INTVAL (operands[1]), 'K')
-   && !fp_operand (operands[0], VOIDmode)"
-  "lhi\\t%0,%h1"
-  [(set_attr "op_type" "RI")])
+})
 
-(define_insn "*movsi_lli"
+(define_insn "*movsi_larl"
   [(set (match_operand:SI 0 "register_operand" "=d")
-        (match_operand:SI 1 "immediate_operand" "n"))]
-  "TARGET_64BIT && s390_single_hi (operands[1], SImode, 0) >= 0
-   && !fp_operand (operands[0], VOIDmode)"
-  "*
-{
-  int part = s390_single_hi (operands[1], SImode, 0);
-  operands[1] = GEN_INT (s390_extract_hi (operands[1], SImode, part));
-
-  switch (part)
-    {
-      case 0: return \"llilh\\t%0,%x1\";
-      case 1: return \"llill\\t%0,%x1\";
-      default: abort ();
-    }
-}"
-  [(set_attr "op_type" "RI")])
-
-(define_insn "*movsi"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "=d,d,m,!*f,!*f,!m,Q")
-        (match_operand:SI 1 "general_operand" "d,m,d,*f,m,*f,Q"))]
-  ""
+        (match_operand:SI 1 "larl_operand" "X"))]
+  "!TARGET_64BIT && TARGET_CPU_ZARCH
+   && !FP_REG_P (operands[0])"
+  "larl\t%0,%1"
+   [(set_attr "op_type" "RIL")
+    (set_attr "type"    "larl")])
+
+(define_insn "*movsi_zarch"
+  [(set (match_operand:SI 0 "nonimmediate_operand"
+                            "=d,d,d,d,d,d,d,R,T,!*f,!*f,!*f,!R,!T,?Q")
+        (match_operand:SI 1 "general_operand"
+                            "K,N0HS0,N1HS0,L,d,R,T,d,d,*f,R,T,*f,*f,?Q"))]
+  "TARGET_ZARCH"
   "@
-   lr\\t%0,%1
-   l\\t%0,%1
-   st\\t%1,%0
-   ler\\t%0,%1
-   le\\t%0,%1
-   ste\\t%1,%0
-   mvc\\t%O0(4,%R0),%1"
-  [(set_attr "op_type" "RR,RX,RX,RR,RX,RX,SS")
-   (set_attr "atype"   "reg,mem,mem,reg,mem,mem,mem")])
-
-
-;
-; movhi instruction pattern(s).
-;
-
-(define_insn "movhi"
-  [(set (match_operand:HI 0 "nonimmediate_operand" "=d,d,d,m,Q")
-        (match_operand:HI 1 "general_operand" "d,n,m,d,Q"))]
-  ""
+   lhi\t%0,%h1
+   llilh\t%0,%i1
+   llill\t%0,%i1
+   lay\t%0,%a1
+   lr\t%0,%1
+   l\t%0,%1
+   ly\t%0,%1
+   st\t%1,%0
+   sty\t%1,%0
+   ler\t%0,%1
+   le\t%0,%1
+   ley\t%0,%1
+   ste\t%1,%0
+   stey\t%1,%0
+   mvc\t%O0(4,%R0),%1"
+  [(set_attr "op_type" "RI,RI,RI,RXY,RR,RX,RXY,RX,RXY,RR,RX,RXY,RX,RXY,SS")
+   (set_attr "type" "*,*,*,la,lr,load,load,store,store,floads,floads,floads,fstores,fstores,cs")])
+
+(define_insn "*movsi_esa"
+  [(set (match_operand:SI 0 "nonimmediate_operand" "=d,d,d,R,!*f,!*f,!R,?Q")
+        (match_operand:SI 1 "general_operand" "K,d,R,d,*f,R,*f,?Q"))]
+  "!TARGET_ZARCH"
   "@
-   lr\\t%0,%1
-   lhi\\t%0,%h1
-   lh\\t%0,%1
-   sth\\t%1,%0
-   mvc\\t%O0(2,%R0),%1"
-  [(set_attr "op_type" "RR,RI,RX,RX,SS")
-   (set_attr "atype"   "reg,reg,mem,mem,mem")])
-
-
-;
-; movqi instruction pattern(s).
-;
+   lhi\t%0,%h1
+   lr\t%0,%1
+   l\t%0,%1
+   st\t%1,%0
+   ler\t%0,%1
+   le\t%0,%1
+   ste\t%1,%0
+   mvc\t%O0(4,%R0),%1"
+  [(set_attr "op_type" "RI,RR,RX,RX,RR,RX,RX,SS")
+   (set_attr "type" "*,lr,load,store,floads,floads,fstores,cs")])
+
+(define_peephole2
+  [(set (match_operand:SI 0 "register_operand" "")
+        (mem:SI (match_operand 1 "address_operand" "")))]
+  "!FP_REG_P (operands[0])
+   && GET_CODE (operands[1]) == SYMBOL_REF
+   && CONSTANT_POOL_ADDRESS_P (operands[1])
+   && get_pool_mode (operands[1]) == SImode
+   && legitimate_reload_constant_p (get_pool_constant (operands[1]))"
+  [(set (match_dup 0) (match_dup 2))]
+  "operands[2] = get_pool_constant (operands[1]);")
 
-(define_insn "movqi_64"
-  [(set (match_operand:QI 0 "nonimmediate_operand" "=d,d,d,m,Q,Q")
-        (match_operand:QI 1 "general_operand" "d,n,m,d,n,Q"))]
-  "TARGET_64BIT"
+(define_insn "*la_31"
+  [(set (match_operand:SI 0 "register_operand" "=d,d")
+        (match_operand:QI 1 "address_operand" "U,W"))]
+  "!TARGET_64BIT && legitimate_la_operand_p (operands[1])"
   "@
-   lr\\t%0,%1
-   lhi\\t%0,%b1
-   llgc\\t%0,%1
-   stc\\t%1,%0
-   mvi\\t%0,%b1
-   mvc\\t%O0(1,%R0),%1"
-  [(set_attr "op_type" "RR,RI,RXE,RX,SI,SS")
-   (set_attr "atype"   "reg,reg,mem,mem,mem,mem")])
+   la\t%0,%a1
+   lay\t%0,%a1"
+  [(set_attr "op_type"  "RX,RXY")
+   (set_attr "type"     "la")])
 
+(define_peephole2
+  [(parallel
+    [(set (match_operand:SI 0 "register_operand" "")
+          (match_operand:QI 1 "address_operand" ""))
+     (clobber (reg:CC 33))])]
+  "!TARGET_64BIT
+   && preferred_la_operand_p (operands[1], const0_rtx)"
+  [(set (match_dup 0) (match_dup 1))]
+  "")
+
+(define_peephole2
+  [(set (match_operand:SI 0 "register_operand" "")
+        (match_operand:SI 1 "register_operand" ""))
+   (parallel
+    [(set (match_dup 0)
+          (plus:SI (match_dup 0)
+                   (match_operand:SI 2 "nonmemory_operand" "")))
+     (clobber (reg:CC 33))])]
+  "!TARGET_64BIT
+   && !reg_overlap_mentioned_p (operands[0], operands[2])
+   && preferred_la_operand_p (operands[1], operands[2])"
+  [(set (match_dup 0) (plus:SI (match_dup 1) (match_dup 2)))]
+  "")
+
+(define_insn "*la_31_and"
+  [(set (match_operand:SI 0 "register_operand" "=d,d")
+        (and:SI (match_operand:QI 1 "address_operand" "U,W")
+                (const_int 2147483647)))]
+  "!TARGET_64BIT"
+  "@
+   la\t%0,%a1
+   lay\t%0,%a1"
+  [(set_attr "op_type"  "RX,RXY")
+   (set_attr "type"     "la")])
+
+(define_insn_and_split "*la_31_and_cc"
+  [(set (match_operand:SI 0 "register_operand" "=d")
+        (and:SI (match_operand:QI 1 "address_operand" "p")
+                (const_int 2147483647)))
+   (clobber (reg:CC 33))]
+  "!TARGET_64BIT"
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 0)
+        (and:SI (match_dup 1) (const_int 2147483647)))]
+  ""
+  [(set_attr "op_type"  "RX")
+   (set_attr "type"     "la")])
+
+(define_insn "force_la_31"
+  [(set (match_operand:SI 0 "register_operand" "=d,d")
+        (match_operand:QI 1 "address_operand" "U,W"))
+   (use (const_int 0))]
+  "!TARGET_64BIT"
+  "@
+   la\t%0,%a1
+   lay\t%0,%a1"
+  [(set_attr "op_type"  "RX")
+   (set_attr "type"     "la")])
+
+(define_expand "reload_insi"
+  [(parallel [(match_operand:SI 0 "register_operand" "=a")
+              (match_operand:SI 1 "s390_plus_operand" "")
+              (match_operand:SI 2 "register_operand" "=&a")])]
+  "!TARGET_64BIT"
+{
+  s390_expand_plus_operand (operands[0], operands[1], operands[2]);
+  DONE;
+})
+
+;
+; movhi instruction pattern(s).
+;
 
-(define_insn "movqi"
-  [(set (match_operand:QI 0 "nonimmediate_operand" "=d,d,d,m,Q,Q")
-        (match_operand:QI 1 "general_operand" "d,n,m,d,n,Q"))]
+(define_expand "movhi"
+  [(set (match_operand:HI 0 "nonimmediate_operand" "")
+        (match_operand:HI 1 "general_operand" ""))]
+  ""
+{
+  /* Make it explicit that loading a register from memory
+     always sign-extends (at least) to SImode.  */
+  if (optimize && !no_new_pseudos
+      && register_operand (operands[0], VOIDmode)
+      && GET_CODE (operands[1]) == MEM)
+    {
+      rtx tmp = gen_reg_rtx (SImode);
+      rtx ext = gen_rtx_SIGN_EXTEND (SImode, operands[1]);
+      emit_insn (gen_rtx_SET (VOIDmode, tmp, ext));
+      operands[1] = gen_lowpart (HImode, tmp);
+    }
+})
+
+(define_insn "*movhi"
+  [(set (match_operand:HI 0 "nonimmediate_operand" "=d,d,d,d,R,T,?Q")
+        (match_operand:HI 1 "general_operand" "d,n,R,T,d,d,?Q"))]
   ""
   "@
-   lr\\t%0,%1
-   lhi\\t%0,%b1
-   ic\\t%0,%1
-   stc\\t%1,%0
-   mvi\\t%0,%b1
-   mvc\\t%O0(1,%R0),%1"
-  [(set_attr "op_type" "RR,RI,RX,RX,SI,SS")
-   (set_attr "atype"   "reg,reg,mem,mem,mem,mem")])
+   lr\t%0,%1
+   lhi\t%0,%h1
+   lh\t%0,%1
+   lhy\t%0,%1
+   sth\t%1,%0
+   sthy\t%1,%0
+   mvc\t%O0(2,%R0),%1"
+  [(set_attr "op_type" "RR,RI,RX,RXY,RX,RXY,SS")
+   (set_attr "type" "lr,*,*,*,store,store,cs")])
+
+(define_peephole2
+  [(set (match_operand:HI 0 "register_operand" "")
+        (mem:HI (match_operand 1 "address_operand" "")))]
+  "GET_CODE (operands[1]) == SYMBOL_REF
+   && CONSTANT_POOL_ADDRESS_P (operands[1])
+   && get_pool_mode (operands[1]) == HImode
+   && GET_CODE (get_pool_constant (operands[1])) == CONST_INT"
+  [(set (match_dup 0) (match_dup 2))]
+  "operands[2] = get_pool_constant (operands[1]);")
+
+;
+; movqi instruction pattern(s).
+;
+
+(define_expand "movqi"
+  [(set (match_operand:QI 0 "nonimmediate_operand" "")
+        (match_operand:QI 1 "general_operand" ""))]
+  ""
+{
+  /* On z/Architecture, zero-extending from memory to register
+     is just as fast as a QImode load.  */
+  if (TARGET_ZARCH && optimize && !no_new_pseudos
+      && register_operand (operands[0], VOIDmode)
+      && GET_CODE (operands[1]) == MEM)
+    {
+      rtx tmp = gen_reg_rtx (word_mode);
+      rtx ext = gen_rtx_ZERO_EXTEND (word_mode, operands[1]);
+      emit_insn (gen_rtx_SET (VOIDmode, tmp, ext));
+      operands[1] = gen_lowpart (QImode, tmp);
+    }
+})
 
+(define_insn "*movqi"
+  [(set (match_operand:QI 0 "nonimmediate_operand" "=d,d,d,d,R,T,Q,S,?Q")
+        (match_operand:QI 1 "general_operand" "d,n,R,T,d,d,n,n,?Q"))]
+  ""
+  "@
+   lr\t%0,%1
+   lhi\t%0,%b1
+   ic\t%0,%1
+   icy\t%0,%1
+   stc\t%1,%0
+   stcy\t%1,%0
+   mvi\t%0,%b1
+   mviy\t%0,%b1
+   mvc\t%O0(1,%R0),%1"
+  [(set_attr "op_type" "RR,RI,RX,RXY,RX,RXY,SI,SIY,SS")
+   (set_attr "type" "lr,*,*,*,store,store,store,store,cs")])
+
+(define_peephole2
+  [(set (match_operand:QI 0 "nonimmediate_operand" "")
+        (mem:QI (match_operand 1 "address_operand" "")))]
+  "GET_CODE (operands[1]) == SYMBOL_REF
+   && CONSTANT_POOL_ADDRESS_P (operands[1])
+   && get_pool_mode (operands[1]) == QImode
+   && GET_CODE (get_pool_constant (operands[1])) == CONST_INT"
+  [(set (match_dup 0) (match_dup 2))]
+  "operands[2] = get_pool_constant (operands[1]);")
 
 ;
 ; movstrictqi instruction pattern(s).
 ;
 
 (define_insn "*movstrictqi"
-  [(set (strict_low_part (match_operand:QI 0 "register_operand" "+d"))
-                         (match_operand:QI 1 "memory_operand" "m"))]
+  [(set (strict_low_part (match_operand:QI 0 "register_operand" "+d,d"))
+                         (match_operand:QI 1 "memory_operand" "R,T"))]
   ""
-  "ic\\t%0,%1"
-  [(set_attr "op_type"  "RX")
-   (set_attr "atype"    "mem")])
+  "@
+   ic\t%0,%1
+   icy\t%0,%1"
+  [(set_attr "op_type"  "RX,RXY")])
 
 ;
 ; movstricthi instruction pattern(s).
 ;
 
 (define_insn "*movstricthi"
-  [(set (strict_low_part (match_operand:HI 0 "register_operand" "+d"))
-                         (match_operand:HI 1 "s_imm_operand" "Q"))
+  [(set (strict_low_part (match_operand:HI 0 "register_operand" "+d,d"))
+                         (match_operand:HI 1 "s_imm_operand" "Q,S"))
    (clobber (reg:CC 33))]
   ""
-  "icm\\t%0,3,%1"
-  [(set_attr "op_type" "RS")
-   (set_attr "atype"   "mem")])
-
+  "@
+   icm\t%0,3,%1
+   icmy\t%0,3,%1"
+  [(set_attr "op_type" "RS,RSY")])
 
 ;
 ; movstrictsi instruction pattern(s).
 ;
 
 (define_insn "movstrictsi"
-  [(set (strict_low_part (match_operand:SI 0 "register_operand" "+d,d"))
-                         (match_operand:SI 1 "general_operand" "d,m"))]
+  [(set (strict_low_part (match_operand:SI 0 "register_operand" "+d,d,d"))
+                         (match_operand:SI 1 "general_operand" "d,R,T"))]
   "TARGET_64BIT"
   "@
-   lr\\t%0,%1
-   l\\t%0,%1"
-  [(set_attr "op_type" "RR,RS")
-   (set_attr "atype"   "reg,mem")])
-
+   lr\t%0,%1
+   l\t%0,%1
+   ly\t%0,%1"
+  [(set_attr "op_type" "RR,RX,RXY")
+   (set_attr "type" "lr,load,load")])
 
 ;
 ; movdf instruction pattern(s).
@@ -1176,97 +1321,101 @@
   [(set (match_operand:DF 0 "nonimmediate_operand" "")
         (match_operand:DF 1 "general_operand"  ""))]
   ""
-  "
 {
   /* During and after reload, we need to force constants
      to the literal pool ourselves, if necessary.  */
   if ((reload_in_progress || reload_completed)
       && CONSTANT_P (operands[1]))
     operands[1] = force_const_mem (DFmode, operands[1]);
-}")
+})
 
 (define_insn "*movdf_64"
-  [(set (match_operand:DF 0 "nonimmediate_operand" "=f,f,m,d,d,m,Q")
-        (match_operand:DF 1 "general_operand" "f,m,f,d,m,d,Q"))]
+  [(set (match_operand:DF 0 "nonimmediate_operand" "=f,f,f,R,T,d,d,m,?Q")
+        (match_operand:DF 1 "general_operand" "f,R,T,f,f,d,m,d,?Q"))]
   "TARGET_64BIT"
   "@
-   ldr\\t%0,%1
-   ld\\t%0,%1
-   std\\t%1,%0
-   lgr\\t%0,%1
-   lg\\t%0,%1
-   stg\\t%1,%0
-   mvc\\t%O0(8,%R0),%1"
-  [(set_attr "op_type" "RR,RX,RX,RR,RXE,RXE,SS")
-   (set_attr "atype"   "reg,mem,mem,reg,mem,mem,mem")])
+   ldr\t%0,%1
+   ld\t%0,%1
+   ldy\t%0,%1
+   std\t%1,%0
+   stdy\t%1,%0
+   lgr\t%0,%1
+   lg\t%0,%1
+   stg\t%1,%0
+   mvc\t%O0(8,%R0),%1"
+  [(set_attr "op_type" "RR,RX,RXY,RX,RXY,RRE,RXY,RXY,SS")
+   (set_attr "type" "floadd,floadd,floadd,fstored,fstored,lr,load,store,cs")])
 
 (define_insn "*movdf_31"
-  [(set (match_operand:DF 0 "nonimmediate_operand" "=f,f,m,d,Q,d,m,Q")
-        (match_operand:DF 1 "general_operand" "f,m,f,Q,d,dKm,d,Q"))]
+  [(set (match_operand:DF 0 "nonimmediate_operand" "=f,f,f,R,T,d,Q,d,o,Q")
+        (match_operand:DF 1 "general_operand" "f,R,T,f,f,Q,d,dKm,d,Q"))]
   "!TARGET_64BIT"
   "@
-   ldr\\t%0,%1
-   ld\\t%0,%1
-   std\\t%1,%0
-   lm\\t%0,%N0,%1
-   stm\\t%1,%N1,%0
+   ldr\t%0,%1
+   ld\t%0,%1
+   ldy\t%0,%1
+   std\t%1,%0
+   stdy\t%1,%0
+   lm\t%0,%N0,%1
+   stm\t%1,%N1,%0
    #
    #
-   mvc\\t%O0(8,%R0),%1"
-  [(set_attr "op_type" "RR,RX,RX,RS,RS,NN,NN,SS")
-   (set_attr "atype"   "reg,mem,mem,mem,mem,*,*,mem")])
+   mvc\t%O0(8,%R0),%1"
+  [(set_attr "op_type" "RR,RX,RXY,RX,RXY,RS,RS,NN,NN,SS")
+   (set_attr "type" "floadd,floadd,floadd,fstored,fstored,lm,stm,*,*,cs")])
 
 (define_split
   [(set (match_operand:DF 0 "nonimmediate_operand" "")
         (match_operand:DF 1 "general_operand" ""))]
   "!TARGET_64BIT && reload_completed
-   && !fp_operand (operands[0], VOIDmode)
-   && !fp_operand (operands[1], VOIDmode)
-   && !s_operand (operands[0], VOIDmode)
-   && !s_operand (operands[1], VOIDmode)
-   && (register_operand (operands[0], VOIDmode)
-       || register_operand (operands[1], VOIDmode))
-   && (!register_operand (operands[0], VOIDmode)
-       || !reg_overlap_mentioned_p (operand_subword (operands[0], 0, 0, DFmode),
-                                    operands[1])
-       || !reg_overlap_mentioned_p (operand_subword (operands[0], 1, 0, DFmode),
-                                    operands[1]))"
+   && s390_split_ok_p (operands[0], operands[1], DFmode, 0)"
   [(set (match_dup 2) (match_dup 4))
    (set (match_dup 3) (match_dup 5))]
-  "
 {
-  if (!register_operand (operands[0], VOIDmode)
-      || !reg_overlap_mentioned_p (operand_subword (operands[0], 0, 0, DFmode),
-                                   operands[1]))
-    {
-      operands[2] = operand_subword (operands[0], 0, 0, DFmode);
-      operands[3] = operand_subword (operands[0], 1, 0, DFmode);
-      operands[4] = operand_subword (operands[1], 0, 0, DFmode);
-      operands[5] = operand_subword (operands[1], 1, 0, DFmode);
-    }
-  else
-    {
-      operands[2] = operand_subword (operands[0], 1, 0, DFmode);
-      operands[3] = operand_subword (operands[0], 0, 0, DFmode);
-      operands[4] = operand_subword (operands[1], 1, 0, DFmode);
-      operands[5] = operand_subword (operands[1], 0, 0, DFmode);
-    }
-}")
+  operands[2] = operand_subword (operands[0], 0, 0, DFmode);
+  operands[3] = operand_subword (operands[0], 1, 0, DFmode);
+  operands[4] = operand_subword (operands[1], 0, 0, DFmode);
+  operands[5] = operand_subword (operands[1], 1, 0, DFmode);
+})
+
+(define_split
+  [(set (match_operand:DF 0 "nonimmediate_operand" "")
+        (match_operand:DF 1 "general_operand" ""))]
+  "!TARGET_64BIT && reload_completed
+   && s390_split_ok_p (operands[0], operands[1], DFmode, 1)"
+  [(set (match_dup 2) (match_dup 4))
+   (set (match_dup 3) (match_dup 5))]
+{
+  operands[2] = operand_subword (operands[0], 1, 0, DFmode);
+  operands[3] = operand_subword (operands[0], 0, 0, DFmode);
+  operands[4] = operand_subword (operands[1], 1, 0, DFmode);
+  operands[5] = operand_subword (operands[1], 0, 0, DFmode);
+})
 
 (define_split
   [(set (match_operand:DF 0 "register_operand" "")
         (match_operand:DF 1 "memory_operand" ""))]
   "!TARGET_64BIT && reload_completed
-   && !fp_operand (operands[0], VOIDmode)
-   && !fp_operand (operands[1], VOIDmode)
+   && !FP_REG_P (operands[0])
    && !s_operand (operands[1], VOIDmode)"
   [(set (match_dup 0) (match_dup 1))]
-  "
 {
   rtx addr = operand_subword (operands[0], 1, 0, DFmode);
   s390_load_address (addr, XEXP (operands[1], 0));
   operands[1] = replace_equiv_address (operands[1], addr);
-}")
+})
+
+(define_expand "reload_outdf"
+  [(parallel [(match_operand:DF 0 "memory_operand" "")
+              (match_operand:DF 1 "register_operand" "d")
+              (match_operand:SI 2 "register_operand" "=&a")])]
+  "!TARGET_64BIT"
+{
+  s390_load_address (operands[2], XEXP (operands[0], 0));
+  operands[0] = replace_equiv_address (operands[0], operands[2]);
+  emit_move_insn (operands[0], operands[1]);
+  DONE;
+})
 
 ;
 ; movsf instruction pattern(s).
@@ -1276,41 +1425,47 @@
   [(set (match_operand:SF 0 "nonimmediate_operand" "")
         (match_operand:SF 1 "general_operand"  ""))]
   ""
-  "
 {
   /* During and after reload, we need to force constants
      to the literal pool ourselves, if necessary.  */
   if ((reload_in_progress || reload_completed)
       && CONSTANT_P (operands[1]))
     operands[1] = force_const_mem (SFmode, operands[1]);
-}")
+})
 
 (define_insn "*movsf"
-  [(set (match_operand:SF 0 "nonimmediate_operand" "=f,f,m,d,d,m,Q")
-        (match_operand:SF 1 "general_operand" "f,m,f,d,m,d,Q"))]
+  [(set (match_operand:SF 0 "nonimmediate_operand" "=f,f,f,R,T,d,d,d,R,T,?Q")
+        (match_operand:SF 1 "general_operand" "f,R,T,f,f,d,R,T,d,d,?Q"))]
   ""
   "@
-   ler\\t%0,%1
-   le\\t%0,%1
-   ste\\t%1,%0
-   lr\\t%0,%1
-   l\\t%0,%1
-   st\\t%1,%0
-   mvc\\t%O0(4,%R0),%1"
-  [(set_attr "op_type" "RR,RX,RX,RR,RX,RX,SS")
-   (set_attr "atype"   "reg,mem,mem,reg,mem,mem,mem")])
+   ler\t%0,%1
+   le\t%0,%1
+   ley\t%0,%1
+   ste\t%1,%0
+   stey\t%1,%0
+   lr\t%0,%1
+   l\t%0,%1
+   ly\t%0,%1
+   st\t%1,%0
+   sty\t%1,%0
+   mvc\t%O0(4,%R0),%1"
+  [(set_attr "op_type" "RR,RX,RXY,RX,RXY,RR,RX,RXY,RX,RXY,SS")
+   (set_attr "type" "floads,floads,floads,fstores,fstores,lr,load,load,store,store,cs")])
 
 ;
 ; load_multiple pattern(s).
 ;
+; ??? Due to reload problems with replacing registers inside match_parallel
+; we currently support load_multiple/store_multiple only after reload.
+;
 
 (define_expand "load_multiple"
   [(match_par_dup 3 [(set (match_operand 0 "" "")
 			  (match_operand 1 "" ""))
 		     (use (match_operand 2 "" ""))])]
-  ""
-  "
+  "reload_completed"
 {
+  enum machine_mode mode;
   int regno;
   int count;
   rtx from;
@@ -1328,6 +1483,9 @@
 
   count = INTVAL (operands[2]);
   regno = REGNO (operands[0]);
+  mode = GET_MODE (operands[0]);
+  if (mode != SImode && mode != word_mode)
+    FAIL;
 
   operands[3] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (count));
   if (no_new_pseudos)
@@ -1346,9 +1504,6 @@
 	}
       else
 	FAIL;
-
-      if (from == frame_pointer_rtx || from == arg_pointer_rtx)
-	FAIL;
     }
   else
     {
@@ -1358,61 +1513,48 @@
 
   for (i = 0; i < count; i++)
     XVECEXP (operands[3], 0, i)
-      = gen_rtx_SET (VOIDmode, gen_rtx_REG (Pmode, regno + i),
-		     change_address (operands[1], Pmode,
-				     plus_constant (from,
-						    off + i * UNITS_PER_WORD)));
-}")
+      = gen_rtx_SET (VOIDmode, gen_rtx_REG (mode, regno + i),
+		     change_address (operands[1], mode,
+		       plus_constant (from, off + i * GET_MODE_SIZE (mode))));
+})
 
 (define_insn "*load_multiple_di"
   [(match_parallel 0 "load_multiple_operation"
 		   [(set (match_operand:DI 1 "register_operand" "=r")
-			 (match_operand:DI 2 "s_operand" "Q"))])]
-  ""
-  "*
+			 (match_operand:DI 2 "s_operand" "QS"))])]
+  "reload_completed && word_mode == DImode"
 {
   int words = XVECLEN (operands[0], 0);
-
-  if (XVECLEN (operands[0], 0) == 1)
-    return \"lg\\t%1,0(%2)\";
-
   operands[0] = gen_rtx_REG (DImode, REGNO (operands[1]) + words - 1);
-    return \"lmg\\t%1,%0,%2\";
-}"
-   [(set_attr "op_type" "RXE")
-    (set_attr "atype"   "mem")
+  return "lmg\t%1,%0,%2";
+}
+   [(set_attr "op_type" "RSY")
     (set_attr "type"    "lm")])
 
 (define_insn "*load_multiple_si"
   [(match_parallel 0 "load_multiple_operation"
-		   [(set (match_operand:SI 1 "register_operand" "=r")
-			 (match_operand:SI 2 "s_operand" "Q"))])]
-  ""
-  "*
+		   [(set (match_operand:SI 1 "register_operand" "=r,r")
+			 (match_operand:SI 2 "s_operand" "Q,S"))])]
+  "reload_completed"
 {
   int words = XVECLEN (operands[0], 0);
-
-  if (XVECLEN (operands[0], 0) == 1)
-    return \"l\\t%1,0(%2)\";
-
   operands[0] = gen_rtx_REG (SImode, REGNO (operands[1]) + words - 1);
-    return \"lm\\t%1,%0,%2\";
-}"
-   [(set_attr "op_type" "RXE")
-    (set_attr "atype"   "mem")
+  return which_alternative == 0 ? "lm\t%1,%0,%2" : "lmy\t%1,%0,%2";
+}
+   [(set_attr "op_type" "RS,RSY")
     (set_attr "type"    "lm")])
 
 ;
-; store multiple pattern(s). 
+; store multiple pattern(s).
 ;
 
 (define_expand "store_multiple"
   [(match_par_dup 3 [(set (match_operand 0 "" "")
 			  (match_operand 1 "" ""))
 		     (use (match_operand 2 "" ""))])]
-  ""
-  "
+  "reload_completed"
 {
+  enum machine_mode mode;
   int regno;
   int count;
   rtx to;
@@ -1430,6 +1572,9 @@
 
   count = INTVAL (operands[2]);
   regno = REGNO (operands[1]);
+  mode = GET_MODE (operands[1]);
+  if (mode != SImode && mode != word_mode)
+    FAIL;
 
   operands[3] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (count));
 
@@ -1449,11 +1594,8 @@
 	}
       else
 	FAIL;
-
-      if (to == frame_pointer_rtx || to == arg_pointer_rtx)
-	FAIL;
     }
-  else	
+  else
     {
       to = force_reg (Pmode, XEXP (operands[0], 0));
       off = 0;
@@ -1462,49 +1604,36 @@
   for (i = 0; i < count; i++)
     XVECEXP (operands[3], 0, i)
       = gen_rtx_SET (VOIDmode,
-		     change_address (operands[0], Pmode,
-				     plus_constant (to,
-						    off + i * UNITS_PER_WORD)),
-		     gen_rtx_REG (Pmode, regno + i));
-}")
+		     change_address (operands[0], mode,
+		       plus_constant (to, off + i * GET_MODE_SIZE (mode))),
+		     gen_rtx_REG (mode, regno + i));
+})
 
 (define_insn "*store_multiple_di"
   [(match_parallel 0 "store_multiple_operation"
-		   [(set (match_operand:DI 1 "s_operand" "=Q")
+		   [(set (match_operand:DI 1 "s_operand" "=QS")
 			 (match_operand:DI 2 "register_operand" "r"))])]
-  ""
-  "*
+  "reload_completed && word_mode == DImode"
 {
   int words = XVECLEN (operands[0], 0);
-
-  if (XVECLEN (operands[0], 0) == 1)
-    return \"stg\\t%1,0(%2)\";
-
   operands[0] = gen_rtx_REG (DImode, REGNO (operands[2]) + words - 1);
-    return \"stmg\\t%2,%0,%1\";
-}"
-   [(set_attr "op_type" "RXE")
-    (set_attr "atype"   "mem")
+  return "stmg\t%2,%0,%1";
+}
+   [(set_attr "op_type" "RSY")
     (set_attr "type"    "stm")])
 
 
 (define_insn "*store_multiple_si"
   [(match_parallel 0 "store_multiple_operation"
-		   [(set (match_operand:SI 1 "s_operand" "=Q")
-			 (match_operand:SI 2 "register_operand" "r"))])]
-  ""
-  "*
+		   [(set (match_operand:SI 1 "s_operand" "=Q,S")
+			 (match_operand:SI 2 "register_operand" "r,r"))])]
+  "reload_completed"
 {
   int words = XVECLEN (operands[0], 0);
-
-  if (XVECLEN (operands[0], 0) == 1)
-    return \"st\\t%1,0(%2)\";
-
   operands[0] = gen_rtx_REG (SImode, REGNO (operands[2]) + words - 1);
-    return \"stm\\t%2,%0,%1\";
-}"
-   [(set_attr "op_type" "RXE")
-    (set_attr "atype"   "mem")
+  return which_alternative == 0 ? "stm\t%2,%0,%1" : "stmy\t%2,%0,%1";
+}
+   [(set_attr "op_type" "RS,RSY")
     (set_attr "type"    "stm")])
 
 ;;
@@ -1512,321 +1641,420 @@
 ;;
 
 ;
-; movstrM instruction pattern(s).
+; strlenM instruction pattern(s).
+;
+
+(define_expand "strlendi"
+  [(set (reg:QI 0) (match_operand:QI 2 "immediate_operand" ""))
+   (parallel
+    [(set (match_dup 4)
+	  (unspec:DI [(const_int 0)
+		      (match_operand:BLK 1 "memory_operand" "")
+		      (reg:QI 0)
+		      (match_operand 3 "immediate_operand" "")] UNSPEC_SRST))
+     (clobber (scratch:DI))
+     (clobber (reg:CC 33))])
+   (parallel
+    [(set (match_operand:DI 0 "register_operand" "")
+          (minus:DI (match_dup 4) (match_dup 5)))
+     (clobber (reg:CC 33))])]
+  "TARGET_64BIT"
+{
+  operands[4] = gen_reg_rtx (DImode);
+  operands[5] = gen_reg_rtx (DImode);
+  emit_move_insn (operands[5], force_operand (XEXP (operands[1], 0), NULL_RTX));
+  operands[1] = replace_equiv_address (operands[1], operands[5]);
+})
+
+(define_insn "*strlendi"
+  [(set (match_operand:DI 0 "register_operand" "=a")
+	(unspec:DI [(match_operand:DI 2 "general_operand" "0")
+		    (mem:BLK (match_operand:DI 3 "register_operand" "1"))
+		    (reg:QI 0)
+		    (match_operand 4 "immediate_operand" "")] UNSPEC_SRST))
+   (clobber (match_scratch:DI 1 "=a"))
+   (clobber (reg:CC 33))]
+  "TARGET_64BIT"
+  "srst\t%0,%1\;jo\t.-4"
+  [(set_attr "op_type" "NN")
+   (set_attr "type"    "vs")
+   (set_attr "length"  "8")])
+
+(define_expand "strlensi"
+  [(set (reg:QI 0) (match_operand:QI 2 "immediate_operand" ""))
+   (parallel
+    [(set (match_dup 4)
+	  (unspec:SI [(const_int 0)
+		      (match_operand:BLK 1 "memory_operand" "")
+		      (reg:QI 0)
+		      (match_operand 3 "immediate_operand" "")] UNSPEC_SRST))
+     (clobber (scratch:SI))
+     (clobber (reg:CC 33))])
+   (parallel
+    [(set (match_operand:SI 0 "register_operand" "")
+          (minus:SI (match_dup 4) (match_dup 5)))
+     (clobber (reg:CC 33))])]
+  "!TARGET_64BIT"
+{
+  operands[4] = gen_reg_rtx (SImode);
+  operands[5] = gen_reg_rtx (SImode);
+  emit_move_insn (operands[5], force_operand (XEXP (operands[1], 0), NULL_RTX));
+  operands[1] = replace_equiv_address (operands[1], operands[5]);
+})
+
+(define_insn "*strlensi"
+  [(set (match_operand:SI 0 "register_operand" "=a")
+	(unspec:SI [(match_operand:SI 2 "general_operand" "0")
+		    (mem:BLK (match_operand:SI 3 "register_operand" "1"))
+		    (reg:QI 0)
+		    (match_operand 4 "immediate_operand" "")] UNSPEC_SRST))
+   (clobber (match_scratch:SI 1 "=a"))
+   (clobber (reg:CC 33))]
+  "!TARGET_64BIT"
+  "srst\t%0,%1\;jo\t.-4"
+  [(set_attr "op_type" "NN")
+   (set_attr "type"    "vs")
+   (set_attr "length"  "8")])
+
+;
+; movmemM instruction pattern(s).
 ;
 
-(define_expand "movstrdi"
+(define_expand "movmemdi"
   [(set (match_operand:BLK 0 "memory_operand" "")
         (match_operand:BLK 1 "memory_operand" ""))
    (use (match_operand:DI 2 "general_operand" ""))
    (match_operand 3 "" "")]
   "TARGET_64BIT"
-  "s390_expand_movstr (operands[0], operands[1], operands[2]); DONE;")
+  "s390_expand_movmem (operands[0], operands[1], operands[2]); DONE;")
 
-(define_expand "movstrsi"
+(define_expand "movmemsi"
   [(set (match_operand:BLK 0 "memory_operand" "")
         (match_operand:BLK 1 "memory_operand" ""))
    (use (match_operand:SI 2 "general_operand" ""))
    (match_operand 3 "" "")]
   ""
-  "s390_expand_movstr (operands[0], operands[1], operands[2]); DONE;")
+  "s390_expand_movmem (operands[0], operands[1], operands[2]); DONE;")
 
 ; Move a block that is up to 256 bytes in length.
 ; The block length is taken as (operands[2] % 256) + 1.
 
-(define_insn "movstr_short_64"
-  [(set (match_operand:BLK 0 "memory_operand" "=Q,Q")
-        (match_operand:BLK 1 "memory_operand" "Q,Q"))
-   (use (match_operand:DI 2 "nonmemory_operand" "n,a"))
-   (clobber (match_scratch:DI 3 "=X,&a"))]
-  "TARGET_64BIT"
-  "*
-{
-  switch (which_alternative)
-    {
-      case 0:
-	return \"mvc\\t%O0(%b2+1,%R0),%1\";
-
-      case 1:
-	output_asm_insn (\"bras\\t%3,.+10\", operands);
-	output_asm_insn (\"mvc\\t%O0(1,%R0),%1\", operands);
-	return \"ex\\t%2,0(%3)\";
-
-      default:
-        abort ();
-    }
-}"
-  [(set_attr "op_type" "SS,NN")
-   (set_attr "type"    "cs,cs")
-   (set_attr "atype"   "mem,mem")
-   (set_attr "length"  "*,14")])
+(define_expand "movmem_short"
+  [(parallel
+    [(set (match_operand:BLK 0 "memory_operand" "")
+          (match_operand:BLK 1 "memory_operand" ""))
+     (use (match_operand 2 "nonmemory_operand" ""))
+     (clobber (match_dup 3))])]
+  ""
+  "operands[3] = gen_rtx_SCRATCH (Pmode);")
 
-(define_insn "movstr_short_31"
+(define_insn "*movmem_short"
   [(set (match_operand:BLK 0 "memory_operand" "=Q,Q")
         (match_operand:BLK 1 "memory_operand" "Q,Q"))
-   (use (match_operand:SI 2 "nonmemory_operand" "n,a"))
-   (clobber (match_scratch:SI 3 "=X,&a"))]
-  "!TARGET_64BIT"
-  "*
+   (use (match_operand 2 "nonmemory_operand" "n,a"))
+   (clobber (match_scratch 3 "=X,&a"))]
+  "(GET_MODE (operands[2]) == Pmode || GET_MODE (operands[2]) == VOIDmode)
+   && GET_MODE (operands[3]) == Pmode"
 {
   switch (which_alternative)
     {
       case 0:
-	return \"mvc\\t%O0(%b2+1,%R0),%1\";
+	return "mvc\t%O0(%b2+1,%R0),%1";
 
       case 1:
-	output_asm_insn (\"bras\\t%3,.+10\", operands);
-	output_asm_insn (\"mvc\\t%O0(1,%R0),%1\", operands);
-	return \"ex\\t%2,0(%3)\";
+	output_asm_insn ("bras\t%3,.+10", operands);
+	output_asm_insn ("mvc\t%O0(1,%R0),%1", operands);
+	return "ex\t%2,0(%3)";
 
       default:
         abort ();
     }
-}"
+}
   [(set_attr "op_type" "SS,NN")
    (set_attr "type"    "cs,cs")
-   (set_attr "atype"   "mem,mem")
+   (set_attr "atype"   "*,agen")
    (set_attr "length"  "*,14")])
 
 ; Move a block of arbitrary length.
 
-(define_insn "movstr_long_64"
-  [(set (match_operand:TI 0 "register_operand" "=d")
-        (ashift:TI (plus:TI (match_operand:TI 2 "register_operand" "0")
-                            (lshiftrt:TI (match_dup 2) (const_int 64)))
-                   (const_int 64)))
-   (set (match_operand:TI 1 "register_operand" "=d")
-        (ashift:TI (plus:TI (match_operand:TI 3 "register_operand" "1")
-                            (lshiftrt:TI (match_dup 3) (const_int 64)))
-                   (const_int 64)))
-   (set (mem:BLK (subreg:DI (match_dup 2) 0))
-        (mem:BLK (subreg:DI (match_dup 3) 0)))
+(define_expand "movmem_long"
+  [(parallel
+    [(clobber (match_dup 2))
+     (clobber (match_dup 3))
+     (set (match_operand:BLK 0 "memory_operand" "")
+          (match_operand:BLK 1 "memory_operand" ""))
+     (use (match_operand 2 "general_operand" ""))
+     (use (match_dup 3))
+     (clobber (reg:CC 33))])]
+  ""
+{
+  enum machine_mode dword_mode = word_mode == DImode ? TImode : DImode;
+  rtx reg0 = gen_reg_rtx (dword_mode);
+  rtx reg1 = gen_reg_rtx (dword_mode);
+  rtx addr0 = gen_lowpart (Pmode, gen_highpart (word_mode, reg0));
+  rtx addr1 = gen_lowpart (Pmode, gen_highpart (word_mode, reg1));
+  rtx len0 = gen_lowpart (Pmode, reg0);
+  rtx len1 = gen_lowpart (Pmode, reg1);
+
+  emit_insn (gen_rtx_CLOBBER (VOIDmode, reg0));
+  emit_move_insn (addr0, force_operand (XEXP (operands[0], 0), NULL_RTX));
+  emit_move_insn (len0, operands[2]);
+
+  emit_insn (gen_rtx_CLOBBER (VOIDmode, reg1));
+  emit_move_insn (addr1, force_operand (XEXP (operands[1], 0), NULL_RTX));
+  emit_move_insn (len1, operands[2]);
+
+  operands[0] = replace_equiv_address_nv (operands[0], addr0);
+  operands[1] = replace_equiv_address_nv (operands[1], addr1);
+  operands[2] = reg0;
+  operands[3] = reg1;
+})
+
+(define_insn "*movmem_long_64"
+  [(clobber (match_operand:TI 0 "register_operand" "=d"))
+   (clobber (match_operand:TI 1 "register_operand" "=d"))
+   (set (mem:BLK (subreg:DI (match_operand:TI 2 "register_operand" "0") 0))
+        (mem:BLK (subreg:DI (match_operand:TI 3 "register_operand" "1") 0)))
+   (use (match_dup 2))
+   (use (match_dup 3))
    (clobber (reg:CC 33))]
   "TARGET_64BIT"
-  "mvcle\\t%0,%1,0\;jo\\t.-4"
+  "mvcle\t%0,%1,0\;jo\t.-4"
   [(set_attr "op_type" "NN")
    (set_attr "type"    "vs")
-   (set_attr "atype"   "mem")
    (set_attr "length"  "8")])
 
-(define_insn "movstr_long_31"
-  [(set (match_operand:DI 0 "register_operand" "=d")
-        (ashift:DI (plus:DI (match_operand:DI 2 "register_operand" "0")
-                            (lshiftrt:DI (match_dup 2) (const_int 32)))
-                   (const_int 32)))
-   (set (match_operand:DI 1 "register_operand" "=d")
-        (ashift:DI (plus:DI (match_operand:DI 3 "register_operand" "1")
-                            (lshiftrt:DI (match_dup 3) (const_int 32)))
-                   (const_int 32)))
-   (set (mem:BLK (subreg:SI (match_dup 2) 0))
-        (mem:BLK (subreg:SI (match_dup 3) 0)))
+(define_insn "*movmem_long_31"
+  [(clobber (match_operand:DI 0 "register_operand" "=d"))
+   (clobber (match_operand:DI 1 "register_operand" "=d"))
+   (set (mem:BLK (subreg:SI (match_operand:DI 2 "register_operand" "0") 0))
+        (mem:BLK (subreg:SI (match_operand:DI 3 "register_operand" "1") 0)))
+   (use (match_dup 2))
+   (use (match_dup 3))
    (clobber (reg:CC 33))]
   "!TARGET_64BIT"
-  "mvcle\\t%0,%1,0\;jo\\t.-4"
+  "mvcle\t%0,%1,0\;jo\t.-4"
   [(set_attr "op_type" "NN")
    (set_attr "type"    "vs")
-   (set_attr "atype"   "mem")
    (set_attr "length"  "8")])
 
 ;
-; clrstrM instruction pattern(s).
+; clrmemM instruction pattern(s).
 ;
 
-(define_expand "clrstrdi"
+(define_expand "clrmemdi"
   [(set (match_operand:BLK 0 "memory_operand" "")
         (const_int 0))
    (use (match_operand:DI 1 "general_operand" ""))
    (match_operand 2 "" "")]
   "TARGET_64BIT"
-  "s390_expand_clrstr (operands[0], operands[1]); DONE;")
+  "s390_expand_clrmem (operands[0], operands[1]); DONE;")
 
-(define_expand "clrstrsi"
+(define_expand "clrmemsi"
   [(set (match_operand:BLK 0 "memory_operand" "")
         (const_int 0))
    (use (match_operand:SI 1 "general_operand" ""))
    (match_operand 2 "" "")]
   ""
-  "s390_expand_clrstr (operands[0], operands[1]); DONE;")
+  "s390_expand_clrmem (operands[0], operands[1]); DONE;")
 
 ; Clear a block that is up to 256 bytes in length.
-; The block length is taken as (operands[2] % 256) + 1.
+; The block length is taken as (operands[1] % 256) + 1.
+
+(define_expand "clrmem_short"
+  [(parallel
+    [(set (match_operand:BLK 0 "memory_operand" "")
+          (const_int 0))
+     (use (match_operand 1 "nonmemory_operand" ""))
+     (clobber (match_dup 2))
+     (clobber (reg:CC 33))])]
+  ""
+  "operands[2] = gen_rtx_SCRATCH (Pmode);")
 
-(define_insn "clrstr_short_64"
+(define_insn "*clrmem_short"
   [(set (match_operand:BLK 0 "memory_operand" "=Q,Q")
         (const_int 0))
-   (use (match_operand:DI 1 "nonmemory_operand" "n,a"))
-   (clobber (match_scratch:DI 2 "=X,&a"))
+   (use (match_operand 1 "nonmemory_operand" "n,a"))
+   (clobber (match_scratch 2 "=X,&a"))
    (clobber (reg:CC 33))]
-  "TARGET_64BIT"
-  "*
+  "(GET_MODE (operands[1]) == Pmode || GET_MODE (operands[1]) == VOIDmode)
+   && GET_MODE (operands[2]) == Pmode"
 {
   switch (which_alternative)
     {
       case 0:
-	return \"xc\\t%O0(%b1+1,%R0),%0\";
+	return "xc\t%O0(%b1+1,%R0),%0";
 
       case 1:
-	output_asm_insn (\"bras\\t%2,.+10\", operands);
-	output_asm_insn (\"xc\\t%O0(1,%R0),%0\", operands);
-	return \"ex\\t%1,0(%2)\";
+	output_asm_insn ("bras\t%2,.+10", operands);
+	output_asm_insn ("xc\t%O0(1,%R0),%0", operands);
+	return "ex\t%1,0(%2)";
 
       default:
         abort ();
     }
-}"
+}
   [(set_attr "op_type" "SS,NN")
    (set_attr "type"    "cs,cs")
-   (set_attr "atype"   "mem,mem")
+   (set_attr "atype"   "*,agen")
    (set_attr "length"  "*,14")])
 
-(define_insn "clrstr_short_31"
-  [(set (match_operand:BLK 0 "memory_operand" "=Q,Q")
-        (const_int 0))
-   (use (match_operand:SI 1 "nonmemory_operand" "n,a"))
-   (clobber (match_scratch:SI 2 "=X,&a"))
-   (clobber (reg:CC 33))]
-  "!TARGET_64BIT"
-  "*
+; Clear a block of arbitrary length.
+
+(define_expand "clrmem_long"
+  [(parallel
+    [(clobber (match_dup 1))
+     (set (match_operand:BLK 0 "memory_operand" "")
+          (const_int 0))
+     (use (match_operand 1 "general_operand" ""))
+     (use (match_dup 2))
+     (clobber (reg:CC 33))])]
+  ""
 {
-  switch (which_alternative)
-    {
-      case 0:
-	return \"xc\\t%O0(%b1+1,%R0),%0\";
+  enum machine_mode dword_mode = word_mode == DImode ? TImode : DImode;
+  rtx reg0 = gen_reg_rtx (dword_mode);
+  rtx reg1 = gen_reg_rtx (dword_mode);
+  rtx addr0 = gen_lowpart (Pmode, gen_highpart (word_mode, reg0));
+  rtx len0 = gen_lowpart (Pmode, reg0);
 
-      case 1:
-	output_asm_insn (\"bras\\t%2,.+10\", operands);
-	output_asm_insn (\"xc\\t%O0(1,%R0),%0\", operands);
-	return \"ex\\t%1,0(%2)\";
+  emit_insn (gen_rtx_CLOBBER (VOIDmode, reg0));
+  emit_move_insn (addr0, force_operand (XEXP (operands[0], 0), NULL_RTX));
+  emit_move_insn (len0, operands[1]);
 
-      default:
-        abort ();
-    }
-}"
-  [(set_attr "op_type" "SS,NN")
-   (set_attr "type"    "cs,cs")
-   (set_attr "atype"   "mem,mem")
-   (set_attr "length"  "*,14")])
+  emit_move_insn (reg1, const0_rtx);
 
-; Clear a block of arbitrary length.
+  operands[0] = replace_equiv_address_nv (operands[0], addr0);
+  operands[1] = reg0;
+  operands[2] = reg1;
+})
 
-(define_insn "clrstr_long_64"
-  [(set (match_operand:TI 0 "register_operand" "=d")
-        (ashift:TI (plus:TI (match_operand:TI 2 "register_operand" "0")
-                            (lshiftrt:TI (match_dup 2) (const_int 64)))
-                   (const_int 64)))
-   (set (mem:BLK (subreg:DI (match_dup 2) 0))
+(define_insn "*clrmem_long_64"
+  [(clobber (match_operand:TI 0 "register_operand" "=d"))
+   (set (mem:BLK (subreg:DI (match_operand:TI 2 "register_operand" "0") 0))
         (const_int 0))
+   (use (match_dup 2))
    (use (match_operand:TI 1 "register_operand" "d"))
    (clobber (reg:CC 33))]
   "TARGET_64BIT"
-  "mvcle\\t%0,%1,0\;jo\\t.-4"
+  "mvcle\t%0,%1,0\;jo\t.-4"
   [(set_attr "op_type" "NN")
-   (set_attr "atype"   "mem")
    (set_attr "type"    "vs")
    (set_attr "length"  "8")])
 
-(define_insn "clrstr_long_31"
-  [(set (match_operand:DI 0 "register_operand" "=d")
-        (ashift:DI (plus:DI (match_operand:DI 2 "register_operand" "0")
-                            (lshiftrt:DI (match_dup 2) (const_int 32)))
-                   (const_int 32)))
-   (set (mem:BLK (subreg:SI (match_dup 2) 0))
+(define_insn "*clrmem_long_31"
+  [(clobber (match_operand:DI 0 "register_operand" "=d"))
+   (set (mem:BLK (subreg:SI (match_operand:DI 2 "register_operand" "0") 0))
         (const_int 0))
+   (use (match_dup 2))
    (use (match_operand:DI 1 "register_operand" "d"))
    (clobber (reg:CC 33))]
   "!TARGET_64BIT"
-  "mvcle\\t%0,%1,0\;jo\\t.-4"
+  "mvcle\t%0,%1,0\;jo\t.-4"
   [(set_attr "op_type" "NN")
-   (set_attr "atype"   "mem")
    (set_attr "type"    "vs")
    (set_attr "length"  "8")])
 
 ;
-; cmpstrM instruction pattern(s).
+; cmpmemM instruction pattern(s).
 ;
 
-(define_expand "cmpstrdi"
+(define_expand "cmpmemdi"
   [(set (match_operand:DI 0 "register_operand" "")
         (compare:DI (match_operand:BLK 1 "memory_operand" "")
                     (match_operand:BLK 2 "memory_operand" "") ) )
    (use (match_operand:DI 3 "general_operand" ""))
    (use (match_operand:DI 4 "" ""))]
   "TARGET_64BIT"
-  "s390_expand_cmpstr (operands[0], operands[1], 
+  "s390_expand_cmpmem (operands[0], operands[1],
                        operands[2], operands[3]); DONE;")
 
-(define_expand "cmpstrsi"
+(define_expand "cmpmemsi"
   [(set (match_operand:SI 0 "register_operand" "")
         (compare:SI (match_operand:BLK 1 "memory_operand" "")
                     (match_operand:BLK 2 "memory_operand" "") ) )
    (use (match_operand:SI 3 "general_operand" ""))
    (use (match_operand:SI 4 "" ""))]
   ""
-  "s390_expand_cmpstr (operands[0], operands[1], 
+  "s390_expand_cmpmem (operands[0], operands[1],
                        operands[2], operands[3]); DONE;")
 
 ; Compare a block that is up to 256 bytes in length.
 ; The block length is taken as (operands[2] % 256) + 1.
 
-(define_insn "cmpstr_short_64"
-  [(set (reg:CCS 33)
-        (compare:CCS (match_operand:BLK 0 "memory_operand" "=Q,Q")
-                     (match_operand:BLK 1 "memory_operand" "Q,Q")))
-   (use (match_operand:DI 2 "nonmemory_operand" "n,a"))
-   (clobber (match_scratch:DI 3 "=X,&a"))]
-  "TARGET_64BIT"
-  "*
-{
-  switch (which_alternative)
-    {
-      case 0:
-	return \"clc\\t%O0(%b2+1,%R0),%1\";
-
-      case 1:
-	output_asm_insn (\"bras\\t%3,.+10\", operands);
-	output_asm_insn (\"clc\\t%O0(1,%R0),%1\", operands);
-	return \"ex\\t%2,0(%3)\";
-
-      default:
-        abort ();
-    }
-}"
-  [(set_attr "op_type" "SS,NN")
-   (set_attr "type"    "cs,cs")
-   (set_attr "atype"   "mem,mem")
-   (set_attr "length"  "*,14")])
+(define_expand "cmpmem_short"
+  [(parallel
+    [(set (reg:CCS 33)
+          (compare:CCS (match_operand:BLK 0 "memory_operand" "")
+                       (match_operand:BLK 1 "memory_operand" "")))
+     (use (match_operand 2 "nonmemory_operand" ""))
+     (clobber (match_dup 3))])]
+  ""
+  "operands[3] = gen_rtx_SCRATCH (Pmode);")
 
-(define_insn "cmpstr_short_31"
+(define_insn "*cmpmem_short"
   [(set (reg:CCS 33)
         (compare:CCS (match_operand:BLK 0 "memory_operand" "=Q,Q")
                      (match_operand:BLK 1 "memory_operand" "Q,Q")))
-   (use (match_operand:SI 2 "nonmemory_operand" "n,a"))
-   (clobber (match_scratch:SI 3 "=X,&a"))]
-  "!TARGET_64BIT"
-  "*
+   (use (match_operand 2 "nonmemory_operand" "n,a"))
+   (clobber (match_scratch 3 "=X,&a"))]
+  "(GET_MODE (operands[2]) == Pmode || GET_MODE (operands[2]) == VOIDmode)
+   && GET_MODE (operands[3]) == Pmode"
 {
   switch (which_alternative)
     {
       case 0:
-	return \"clc\\t%O0(%b2+1,%R0),%1\";
+	return "clc\t%O0(%b2+1,%R0),%1";
 
       case 1:
-	output_asm_insn (\"bras\\t%3,.+10\", operands);
-	output_asm_insn (\"clc\\t%O0(1,%R0),%1\", operands);
-	return \"ex\\t%2,0(%3)\";
+	output_asm_insn ("bras\t%3,.+10", operands);
+	output_asm_insn ("clc\t%O0(1,%R0),%1", operands);
+	return "ex\t%2,0(%3)";
 
       default:
         abort ();
     }
-}"
+}
   [(set_attr "op_type" "SS,NN")
    (set_attr "type"    "cs,cs")
-   (set_attr "atype"   "mem,mem")
+   (set_attr "atype"   "*,agen")
    (set_attr "length"  "*,14")])
 
 ; Compare a block of arbitrary length.
 
-(define_insn "cmpstr_long_64"
+(define_expand "cmpmem_long"
+  [(parallel
+    [(clobber (match_dup 2))
+     (clobber (match_dup 3))
+     (set (reg:CCS 33)
+          (compare:CCS (match_operand:BLK 0 "memory_operand" "")
+                       (match_operand:BLK 1 "memory_operand" "")))
+     (use (match_operand 2 "general_operand" ""))
+     (use (match_dup 3))])]
+  ""
+{
+  enum machine_mode dword_mode = word_mode == DImode ? TImode : DImode;
+  rtx reg0 = gen_reg_rtx (dword_mode);
+  rtx reg1 = gen_reg_rtx (dword_mode);
+  rtx addr0 = gen_lowpart (Pmode, gen_highpart (word_mode, reg0));
+  rtx addr1 = gen_lowpart (Pmode, gen_highpart (word_mode, reg1));
+  rtx len0 = gen_lowpart (Pmode, reg0);
+  rtx len1 = gen_lowpart (Pmode, reg1);
+
+  emit_insn (gen_rtx_CLOBBER (VOIDmode, reg0));
+  emit_move_insn (addr0, force_operand (XEXP (operands[0], 0), NULL_RTX));
+  emit_move_insn (len0, operands[2]);
+
+  emit_insn (gen_rtx_CLOBBER (VOIDmode, reg1));
+  emit_move_insn (addr1, force_operand (XEXP (operands[1], 0), NULL_RTX));
+  emit_move_insn (len1, operands[2]);
+
+  operands[0] = replace_equiv_address_nv (operands[0], addr0);
+  operands[1] = replace_equiv_address_nv (operands[1], addr1);
+  operands[2] = reg0;
+  operands[3] = reg1;
+})
+
+(define_insn "*cmpmem_long_64"
   [(clobber (match_operand:TI 0 "register_operand" "=d"))
    (clobber (match_operand:TI 1 "register_operand" "=d"))
    (set (reg:CCS 33)
@@ -1835,12 +2063,12 @@
    (use (match_dup 2))
    (use (match_dup 3))]
   "TARGET_64BIT"
-  "clcl\\t%0,%1"
-  [(set_attr "op_type" "RR")
-   (set_attr "atype"   "mem")
-   (set_attr "type"    "vs")])
+  "clcle\t%0,%1,0\;jo\t.-4"
+  [(set_attr "op_type" "NN")
+   (set_attr "type"    "vs")
+   (set_attr "length"  "8")])
 
-(define_insn "cmpstr_long_31"
+(define_insn "*cmpmem_long_31"
   [(clobber (match_operand:DI 0 "register_operand" "=d"))
    (clobber (match_operand:DI 1 "register_operand" "=d"))
    (set (reg:CCS 33)
@@ -1849,10 +2077,10 @@
    (use (match_dup 2))
    (use (match_dup 3))]
   "!TARGET_64BIT"
-  "clcl\\t%0,%1"
-  [(set_attr "op_type" "RR")
-   (set_attr "atype"   "mem")
-   (set_attr "type"    "vs")])
+  "clcle\t%0,%1,0\;jo\t.-4"
+  [(set_attr "op_type" "NN")
+   (set_attr "type"    "vs")
+   (set_attr "length"  "8")])
 
 ; Convert condition code to integer in range (-1, 0, 1)
 
@@ -1860,34 +2088,30 @@
   [(set (match_operand:SI 0 "register_operand" "=d")
         (compare:SI (reg:CCS 33) (const_int 0)))]
   ""
-  "*
 {
-   output_asm_insn (\"lhi\\t%0,1\", operands);
-   output_asm_insn (\"jh\\t.+12\", operands);
-   output_asm_insn (\"jl\\t.+6\", operands);
-   output_asm_insn (\"sr\\t%0,%0\", operands);
-   return \"lcr\\t%0,%0\";
-}"
+   output_asm_insn ("lhi\t%0,1", operands);
+   output_asm_insn ("jh\t.+12", operands);
+   output_asm_insn ("jl\t.+6", operands);
+   output_asm_insn ("sr\t%0,%0", operands);
+   return "lcr\t%0,%0";
+}
   [(set_attr "op_type" "NN")
    (set_attr "length"  "16")
-   (set_attr "atype"   "reg")
    (set_attr "type"    "other")])
 
 (define_insn "cmpint_di"
   [(set (match_operand:DI 0 "register_operand" "=d")
         (compare:DI (reg:CCS 33) (const_int 0)))]
   "TARGET_64BIT"
-  "*
 {
-   output_asm_insn (\"lghi\\t%0,1\", operands);
-   output_asm_insn (\"jh\\t.+12\", operands);
-   output_asm_insn (\"jl\\t.+6\", operands);
-   output_asm_insn (\"sgr\\t%0,%0\", operands);
-   return \"lcgr\\t%0,%0\";
-}"
+   output_asm_insn ("lghi\t%0,1", operands);
+   output_asm_insn ("jh\t.+16", operands);
+   output_asm_insn ("jl\t.+8", operands);
+   output_asm_insn ("sgr\t%0,%0", operands);
+   return "lcgr\t%0,%0";
+}
   [(set_attr "op_type" "NN")
-   (set_attr "length"  "22")
-   (set_attr "atype"   "reg")
+   (set_attr "length"  "20")
    (set_attr "type"    "other")])
 
 
@@ -1896,40 +2120,42 @@
 ;;
 
 (define_insn "*sethighqisi"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (unspec:SI [(match_operand:QI 1 "s_operand" "Q")] 10))
+  [(set (match_operand:SI 0 "register_operand" "=d,d")
+        (unspec:SI [(match_operand:QI 1 "s_operand" "Q,S")] UNSPEC_SETHIGH))
    (clobber (reg:CC 33))]
   ""
-  "icm\\t%0,8,%1"
-  [(set_attr "op_type" "RS")
-   (set_attr "atype"   "mem")])
+  "@
+   icm\t%0,8,%1
+   icmy\t%0,8,%1"
+  [(set_attr "op_type" "RS,RSY")])
 
 (define_insn "*sethighhisi"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (unspec:SI [(match_operand:HI 1 "s_operand" "Q")] 10))
+  [(set (match_operand:SI 0 "register_operand" "=d,d")
+        (unspec:SI [(match_operand:HI 1 "s_operand" "Q,S")] UNSPEC_SETHIGH))
    (clobber (reg:CC 33))]
   ""
-  "icm\\t%0,12,%1"
-  [(set_attr "op_type" "RS")
-   (set_attr "atype"   "mem")])
+  "@
+   icm\t%0,12,%1
+   icmy\t%0,12,%1"
+  [(set_attr "op_type" "RS,RSY")])
 
 (define_insn "*sethighqidi_64"
   [(set (match_operand:DI 0 "register_operand" "=d")
-        (unspec:DI [(match_operand:QI 1 "s_operand" "Q")] 10))
+        (unspec:DI [(match_operand:QI 1 "s_operand" "QS")] UNSPEC_SETHIGH))
    (clobber (reg:CC 33))]
   "TARGET_64BIT"
-  "icmh\\t%0,8,%1"
-  [(set_attr "op_type" "RSE")
-   (set_attr "atype"   "mem")])
+  "icmh\t%0,8,%1"
+  [(set_attr "op_type" "RSY")])
 
 (define_insn "*sethighqidi_31"
-  [(set (match_operand:DI 0 "register_operand" "=d")
-        (unspec:DI [(match_operand:QI 1 "s_operand" "Q")] 10))
+  [(set (match_operand:DI 0 "register_operand" "=d,d")
+        (unspec:DI [(match_operand:QI 1 "s_operand" "Q,S")] UNSPEC_SETHIGH))
    (clobber (reg:CC 33))]
   "!TARGET_64BIT"
-  "icm\\t%0,8,%1"
-  [(set_attr "op_type" "RS")
-   (set_attr "atype"   "mem")])
+  "@
+   icm\t%0,8,%1
+   icmy\t%0,8,%1"
+  [(set_attr "op_type" "RS,RSY")])
 
 (define_insn_and_split "*extractqi"
   [(set (match_operand:SI 0 "register_operand" "=d")
@@ -1942,16 +2168,14 @@
   "#"
   "&& reload_completed"
   [(parallel
-    [(set (match_dup 0) (unspec:SI [(match_dup 1)] 10))
+    [(set (match_dup 0) (unspec:SI [(match_dup 1)] UNSPEC_SETHIGH))
      (clobber (reg:CC 33))])
     (set (match_dup 0) (lshiftrt:SI (match_dup 0) (match_dup 2)))]
-  "
 {
   operands[2] = GEN_INT (32 - INTVAL (operands[2]));
   operands[1] = change_address (operands[1], QImode, 0);
-}"
-  [(set_attr "type"    "o2")
-   (set_attr "atype"   "mem")])
+}
+  [(set_attr "atype"   "agen")])
 
 (define_insn_and_split "*extracthi"
   [(set (match_operand:SI 0 "register_operand" "=d")
@@ -1964,16 +2188,14 @@
   "#"
   "&& reload_completed"
   [(parallel
-    [(set (match_dup 0) (unspec:SI [(match_dup 1)] 10))
+    [(set (match_dup 0) (unspec:SI [(match_dup 1)] UNSPEC_SETHIGH))
      (clobber (reg:CC 33))])
     (set (match_dup 0) (lshiftrt:SI (match_dup 0) (match_dup 2)))]
-  "
 {
   operands[2] = GEN_INT (32 - INTVAL (operands[2]));
   operands[1] = change_address (operands[1], HImode, 0);
-}"
-  [(set_attr "type"    "o2")
-   (set_attr "atype"   "mem")])
+}
+  [(set_attr "atype"   "agen")])
 
 ;
 ; extendsidi2 instruction pattern(s).
@@ -2001,10 +2223,9 @@
         (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "d,m")))]
   "TARGET_64BIT"
   "@
-   lgfr\\t%0,%1
-   lgf\\t%0,%1"
-  [(set_attr "op_type" "RRE,RXE")
-   (set_attr "atype"   "reg,mem")])
+   lgfr\t%0,%1
+   lgf\t%0,%1"
+  [(set_attr "op_type" "RRE,RXY")])
 
 ;
 ; extendhidi2 instruction pattern(s).
@@ -2027,7 +2248,7 @@
     {
       operands[1] = gen_lowpart (DImode, operands[1]);
       emit_insn (gen_ashldi3 (operands[0], operands[1], GEN_INT (48)));
-      emit_insn (gen_ashrdi3 (operands[0], operands[0], GEN_INT (48))); 
+      emit_insn (gen_ashrdi3 (operands[0], operands[0], GEN_INT (48)));
       DONE;
     }
 }
@@ -2037,9 +2258,8 @@
   [(set (match_operand:DI 0 "register_operand" "=d")
         (sign_extend:DI (match_operand:HI 1 "memory_operand" "m")))]
   "TARGET_64BIT"
-  "lgh\\t%0,%1"
-  [(set_attr "op_type" "RXE")
-   (set_attr "atype"   "mem")])
+  "lgh\t%0,%1"
+  [(set_attr "op_type" "RXY")])
 
 ;
 ; extendqidi2 instruction pattern(s).
@@ -2062,18 +2282,28 @@
     {
       operands[1] = gen_lowpart (DImode, operands[1]);
       emit_insn (gen_ashldi3 (operands[0], operands[1], GEN_INT (56)));
-      emit_insn (gen_ashrdi3 (operands[0], operands[0], GEN_INT (56))); 
+      emit_insn (gen_ashrdi3 (operands[0], operands[0], GEN_INT (56)));
       DONE;
     }
 }
 ")
 
-(define_split
-  [(set (match_operand:DI 0 "register_operand" "")
-        (sign_extend:DI (match_operand:QI 1 "s_operand" "")))]
-  "TARGET_64BIT && !reload_completed"
+(define_insn "*extendqidi2"
+  [(set (match_operand:DI 0 "register_operand" "=d")
+        (sign_extend:DI (match_operand:QI 1 "memory_operand" "m")))]
+  "TARGET_64BIT && TARGET_LONG_DISPLACEMENT"
+  "lgb\t%0,%1"
+  [(set_attr "op_type" "RXY")])
+
+(define_insn_and_split "*extendqidi2_short_displ"
+  [(set (match_operand:DI 0 "register_operand" "=d")
+        (sign_extend:DI (match_operand:QI 1 "s_operand" "Q")))
+   (clobber (reg:CC 33))]
+  "TARGET_64BIT && !TARGET_LONG_DISPLACEMENT"
+  "#"
+  "&& reload_completed"
   [(parallel
-    [(set (match_dup 0) (unspec:DI [(match_dup 1)] 10))
+    [(set (match_dup 0) (unspec:DI [(match_dup 1)] UNSPEC_SETHIGH))
      (clobber (reg:CC 33))])
    (parallel
     [(set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 56)))
@@ -2092,18 +2322,19 @@
 {
   operands[1] = gen_lowpart (SImode, operands[1]);
   emit_insn (gen_ashlsi3 (operands[0], operands[1], GEN_INT (16)));
-  emit_insn (gen_ashrsi3 (operands[0], operands[0], GEN_INT (16))); 
+  emit_insn (gen_ashrsi3 (operands[0], operands[0], GEN_INT (16)));
   DONE;
 }
 ")
 
 (define_insn "*extendhisi2"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (sign_extend:SI (match_operand:HI 1 "memory_operand" "m")))]
+  [(set (match_operand:SI 0 "register_operand" "=d,d")
+        (sign_extend:SI (match_operand:HI 1 "memory_operand" "R,T")))]
   ""
-  "lh\\t%0,%1"
-  [(set_attr "op_type" "RX")
-   (set_attr "atype"   "mem")])
+  "@
+   lh\t%0,%1
+   lhy\t%0,%1"
+  [(set_attr "op_type" "RX,RXY")])
 
 ;
 ; extendqisi2 instruction pattern(s).
@@ -2117,17 +2348,27 @@
 {
   operands[1] = gen_lowpart (SImode, operands[1]);
   emit_insn (gen_ashlsi3 (operands[0], operands[1], GEN_INT (24)));
-  emit_insn (gen_ashrsi3 (operands[0], operands[0], GEN_INT (24))); 
+  emit_insn (gen_ashrsi3 (operands[0], operands[0], GEN_INT (24)));
   DONE;
 }
 ")
 
-(define_split
-  [(set (match_operand:SI 0 "register_operand" "")
-        (sign_extend:SI (match_operand:QI 1 "s_operand" "")))]
-  "!reload_completed"
+(define_insn "*extendqisi2"
+  [(set (match_operand:SI 0 "register_operand" "=d")
+        (sign_extend:SI (match_operand:QI 1 "memory_operand" "m")))]
+  "TARGET_LONG_DISPLACEMENT"
+  "lb\t%0,%1"
+  [(set_attr "op_type" "RXY")])
+
+(define_insn_and_split "*extendqisi2_short_displ"
+  [(set (match_operand:SI 0 "register_operand" "=d")
+        (sign_extend:SI (match_operand:QI 1 "s_operand" "Q")))
+   (clobber (reg:CC 33))]
+  "!TARGET_LONG_DISPLACEMENT"
+  "#"
+  "&& reload_completed"
   [(parallel
-    [(set (match_dup 0) (unspec:SI [(match_dup 1)] 10))
+    [(set (match_dup 0) (unspec:SI [(match_dup 1)] UNSPEC_SETHIGH))
      (clobber (reg:CC 33))])
    (parallel
     [(set (match_dup 0) (ashiftrt:SI (match_dup 0) (const_int 24)))
@@ -2164,10 +2405,9 @@
         (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "d,m")))]
   "TARGET_64BIT"
   "@
-   llgfr\\t%0,%1
-   llgf\\t%0,%1"
-  [(set_attr "op_type" "RRE,RXE")
-   (set_attr "atype"   "reg,mem")])
+   llgfr\t%0,%1
+   llgf\t%0,%1"
+  [(set_attr "op_type" "RRE,RXY")])
 
 ;
 ; zero_extendhidi2 instruction pattern(s).
@@ -2190,7 +2430,7 @@
     {
       operands[1] = gen_lowpart (DImode, operands[1]);
       emit_insn (gen_ashldi3 (operands[0], operands[1], GEN_INT (48)));
-      emit_insn (gen_lshrdi3 (operands[0], operands[0], GEN_INT (48))); 
+      emit_insn (gen_lshrdi3 (operands[0], operands[0], GEN_INT (48)));
       DONE;
     }
 }
@@ -2200,9 +2440,75 @@
   [(set (match_operand:DI 0 "register_operand" "=d")
         (zero_extend:DI (match_operand:HI 1 "memory_operand" "m")))]
   "TARGET_64BIT"
-  "llgh\\t%0,%1"
-  [(set_attr "op_type" "RXE")
-   (set_attr "atype"   "mem")])
+  "llgh\t%0,%1"
+  [(set_attr "op_type" "RXY")])
+
+;
+; LLGT-type instructions (zero-extend from 31 bit to 64 bit).
+;
+
+(define_insn "*llgt_sisi"
+  [(set (match_operand:SI 0 "register_operand" "=d,d")
+        (and:SI (match_operand:SI 1 "nonimmediate_operand" "d,m")
+		(const_int 2147483647)))]
+  "TARGET_64BIT"
+  "@
+   llgtr\t%0,%1
+   llgt\t%0,%1"
+  [(set_attr "op_type"  "RRE,RXE")])
+
+(define_split
+  [(set (match_operand:SI 0 "register_operand" "")
+        (and:SI (match_operand:SI 1 "nonimmediate_operand" "")
+		(const_int 2147483647)))
+   (clobber (reg:CC 33))]
+  "TARGET_64BIT && reload_completed"
+  [(set (match_dup 0)
+        (and:SI (match_dup 1)
+		(const_int 2147483647)))]
+  "")
+
+(define_insn "*llgt_didi"
+  [(set (match_operand:DI 0 "register_operand" "=d,d")
+        (and:DI (match_operand:DI 1 "nonimmediate_operand" "d,o")
+                (const_int 2147483647)))]
+  "TARGET_64BIT"
+  "@
+   llgtr\t%0,%1
+   llgt\t%0,%N1"
+  [(set_attr "op_type"  "RRE,RXE")])
+
+(define_split
+  [(set (match_operand:DI 0 "register_operand" "")
+        (and:DI (match_operand:DI 1 "nonimmediate_operand" "")
+                (const_int 2147483647)))
+   (clobber (reg:CC 33))]
+  "TARGET_64BIT && reload_completed"
+  [(set (match_dup 0)
+        (and:DI (match_dup 1)
+                (const_int 2147483647)))]
+  "")
+
+(define_insn "*llgt_sidi"
+  [(set (match_operand:DI 0 "register_operand" "=d")
+        (and:DI (subreg:DI (match_operand:SI 1 "memory_operand" "m") 0)
+		(const_int 2147483647)))]
+  "TARGET_64BIT"
+  "llgt\t%0,%1"
+  [(set_attr "op_type"  "RXE")])
+
+(define_insn_and_split "*llgt_sidi_split"
+  [(set (match_operand:DI 0 "register_operand" "=d")
+        (and:DI (subreg:DI (match_operand:SI 1 "memory_operand" "m") 0)
+		(const_int 2147483647)))
+   (clobber (reg:CC 33))]
+  "TARGET_64BIT"
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 0)
+        (and:DI (subreg:DI (match_dup 1) 0)
+		(const_int 2147483647)))]
+  "")
 
 ;
 ; zero_extendqidi2 instruction pattern(s)
@@ -2225,7 +2531,7 @@
     {
       operands[1] = gen_lowpart (DImode, operands[1]);
       emit_insn (gen_ashldi3 (operands[0], operands[1], GEN_INT (56)));
-      emit_insn (gen_lshrdi3 (operands[0], operands[0], GEN_INT (56))); 
+      emit_insn (gen_lshrdi3 (operands[0], operands[0], GEN_INT (56)));
       DONE;
     }
 }
@@ -2235,9 +2541,8 @@
   [(set (match_operand:DI 0 "register_operand" "=d")
         (zero_extend:DI (match_operand:QI 1 "memory_operand" "m")))]
   "TARGET_64BIT"
-  "llgc\\t%0,%1"
-  [(set_attr "op_type" "RXE")
-   (set_attr "atype"   "mem")])
+  "llgc\t%0,%1"
+  [(set_attr "op_type" "RXY")])
 
 ;
 ; zero_extendhisi2 instruction pattern(s).
@@ -2259,13 +2564,12 @@
   [(set (match_operand:SI 0 "register_operand" "=d")
         (zero_extend:SI (match_operand:HI 1 "memory_operand" "m")))]
   "TARGET_64BIT"
-  "llgh\\t%0,%1"
-  [(set_attr "op_type" "RXE")
-   (set_attr "atype"   "mem")])
+  "llgh\t%0,%1"
+  [(set_attr "op_type" "RXY")])
 
 (define_insn_and_split "*zero_extendhisi2_31"
   [(set (match_operand:SI 0 "register_operand" "=&d")
-        (zero_extend:SI (match_operand:HI 1 "memory_operand" "Q")))
+        (zero_extend:SI (match_operand:HI 1 "s_operand" "QS")))
    (clobber (reg:CC 33))]
   "!TARGET_64BIT"
   "#"
@@ -2275,9 +2579,8 @@
     [(set (strict_low_part (match_dup 2)) (match_dup 1))
      (clobber (reg:CC 33))])]
   "operands[2] = gen_lowpart (HImode, operands[0]);"
-  [(set_attr "type" "o2")
-   (set_attr "atype" "mem")])
- 
+  [(set_attr "atype" "agen")])
+
 ;
 ; zero_extendqisi2 instruction pattern(s).
 ;
@@ -2297,23 +2600,21 @@
 (define_insn "*zero_extendqisi2_64"
   [(set (match_operand:SI 0 "register_operand" "=d")
         (zero_extend:SI (match_operand:QI 1 "memory_operand" "m")))]
-  "TARGET_64BIT"
-  "llgc\\t%0,%1"
-  [(set_attr "op_type" "RXE")
-   (set_attr "atype"   "mem")])
+  "TARGET_ZARCH"
+  "llgc\t%0,%1"
+  [(set_attr "op_type" "RXY")])
 
 (define_insn_and_split "*zero_extendqisi2_31"
   [(set (match_operand:SI 0 "register_operand" "=&d")
         (zero_extend:SI (match_operand:QI 1 "memory_operand" "m")))]
-  "!TARGET_64BIT"
+  "!TARGET_ZARCH"
   "#"
   "&& reload_completed"
   [(set (match_dup 0) (const_int 0))
    (set (strict_low_part (match_dup 2)) (match_dup 1))]
   "operands[2] = gen_lowpart (QImode, operands[0]);"
-  [(set_attr "type" "o2")
-   (set_attr "atype" "mem")])
- 
+  [(set_attr "atype" "agen")])
+
 ;
 ; zero_extendqihi2 instruction pattern(s).
 ;
@@ -2321,7 +2622,7 @@
 (define_expand "zero_extendqihi2"
   [(set (match_operand:HI 0 "register_operand" "")
         (zero_extend:HI (match_operand:QI 1 "register_operand" "")))]
-  "TARGET_64BIT"
+  "TARGET_ZARCH"
   "
 {
   operands[1] = gen_lowpart (HImode, operands[1]);
@@ -2333,22 +2634,20 @@
 (define_insn "*zero_extendqihi2_64"
   [(set (match_operand:HI 0 "register_operand" "=d")
         (zero_extend:HI (match_operand:QI 1 "memory_operand" "m")))]
-  "TARGET_64BIT"
-  "llgc\\t%0,%1"
-  [(set_attr "op_type" "RXE")
-   (set_attr "atype"   "mem")])
+  "TARGET_ZARCH"
+  "llgc\t%0,%1"
+  [(set_attr "op_type" "RXY")])
 
 (define_insn_and_split "*zero_extendqihi2_31"
   [(set (match_operand:HI 0 "register_operand" "=&d")
         (zero_extend:HI (match_operand:QI 1 "memory_operand" "m")))]
-  "!TARGET_64BIT"
+  "!TARGET_ZARCH"
   "#"
   "&& reload_completed"
   [(set (match_dup 0) (const_int 0))
    (set (strict_low_part (match_dup 2)) (match_dup 1))]
   "operands[2] = gen_lowpart (QImode, operands[0]);"
-  [(set_attr "type" "o2")
-   (set_attr "atype" "mem")])
+  [(set_attr "atype" "agen")])
 
 
 ;
@@ -2359,20 +2658,19 @@
   [(set (match_operand:DI 0 "register_operand" "")
         (unsigned_fix:DI (match_operand:DF 1 "register_operand" "")))]
   "TARGET_64BIT && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "
 {
   rtx label1 = gen_label_rtx ();
   rtx label2 = gen_label_rtx ();
   rtx temp = gen_reg_rtx (DFmode);
   operands[1] = force_reg (DFmode, operands[1]);
 
-  emit_insn (gen_cmpdf (operands[1], 
+  emit_insn (gen_cmpdf (operands[1],
 	CONST_DOUBLE_FROM_REAL_VALUE (
-          REAL_VALUE_ATOF (\"9223372036854775808.0\", DFmode), DFmode)));
+          REAL_VALUE_ATOF ("9223372036854775808.0", DFmode), DFmode)));
   emit_jump_insn (gen_blt (label1));
   emit_insn (gen_subdf3 (temp, operands[1],
 	CONST_DOUBLE_FROM_REAL_VALUE (
-          REAL_VALUE_ATOF (\"18446744073709551616.0\", DFmode), DFmode)));
+          REAL_VALUE_ATOF ("18446744073709551616.0", DFmode), DFmode)));
   emit_insn (gen_fix_truncdfdi2_ieee (operands[0], temp, GEN_INT(7)));
   emit_jump (label2);
 
@@ -2380,28 +2678,27 @@
   emit_insn (gen_fix_truncdfdi2_ieee (operands[0], operands[1], GEN_INT(5)));
   emit_label (label2);
   DONE;
-}")
+})
 
 (define_expand "fix_truncdfdi2"
   [(set (match_operand:DI 0 "register_operand" "")
         (fix:DI (match_operand:DF 1 "nonimmediate_operand" "")))]
   "TARGET_64BIT && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "
 {
   operands[1] = force_reg (DFmode, operands[1]);
   emit_insn (gen_fix_truncdfdi2_ieee (operands[0], operands[1], GEN_INT(5)));
   DONE;
-}")
+})
 
 (define_insn "fix_truncdfdi2_ieee"
   [(set (match_operand:DI 0 "register_operand" "=d")
         (fix:DI (match_operand:DF 1 "register_operand" "f")))
-   (unspec:DI [(match_operand:DI 2 "immediate_operand" "K")] 1)
+   (unspec:DI [(match_operand:DI 2 "immediate_operand" "K")] UNSPEC_ROUND)
    (clobber (reg:CC 33))]
   "TARGET_64BIT && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "cgdbr\\t%0,%h2,%1"
+  "cgdbr\t%0,%h2,%1"
   [(set_attr "op_type" "RRE")
-   (set_attr "type"    "other")])
+   (set_attr "type"    "ftoi")])
 
 ;
 ; fixuns_truncdfsi2 and fix_truncdfsi2 instruction pattern(s).
@@ -2411,20 +2708,19 @@
   [(set (match_operand:SI 0 "register_operand" "")
         (unsigned_fix:SI (match_operand:DF 1 "register_operand" "")))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "
 {
   rtx label1 = gen_label_rtx ();
   rtx label2 = gen_label_rtx ();
   rtx temp = gen_reg_rtx (DFmode);
 
   operands[1] = force_reg (DFmode,operands[1]);
-  emit_insn (gen_cmpdf (operands[1], 
+  emit_insn (gen_cmpdf (operands[1],
 	CONST_DOUBLE_FROM_REAL_VALUE (
-          REAL_VALUE_ATOF (\"2147483648.0\", DFmode), DFmode)));
+          REAL_VALUE_ATOF ("2147483648.0", DFmode), DFmode)));
   emit_jump_insn (gen_blt (label1));
   emit_insn (gen_subdf3 (temp, operands[1],
 	CONST_DOUBLE_FROM_REAL_VALUE (
-          REAL_VALUE_ATOF (\"4294967296.0\", DFmode), DFmode)));
+          REAL_VALUE_ATOF ("4294967296.0", DFmode), DFmode)));
   emit_insn (gen_fix_truncdfsi2_ieee (operands[0], temp, GEN_INT (7)));
   emit_jump (label2);
 
@@ -2432,42 +2728,41 @@
   emit_insn (gen_fix_truncdfsi2_ieee (operands[0], operands[1], GEN_INT (5)));
   emit_label (label2);
   DONE;
-}")
+})
 
 (define_expand "fix_truncdfsi2"
   [(set (match_operand:SI 0 "register_operand" "")
         (fix:SI (match_operand:DF 1 "nonimmediate_operand" "")))]
   "TARGET_HARD_FLOAT"
-  "
 {
-  if (TARGET_IBM_FLOAT) 
+  if (TARGET_IBM_FLOAT)
     {
       /* This is the algorithm from POP chapter A.5.7.2.  */
 
-      rtx temp   = assign_stack_local (BLKmode, 2 * UNITS_PER_WORD, BITS_PER_WORD);
+      rtx temp   = assign_stack_local (BLKmode, 8, BITS_PER_WORD);
       rtx two31r = s390_gen_rtx_const_DI (0x4f000000, 0x08000000);
       rtx two32  = s390_gen_rtx_const_DI (0x4e000001, 0x00000000);
 
       operands[1] = force_reg (DFmode, operands[1]);
-      emit_insn (gen_fix_truncdfsi2_ibm (operands[0], operands[1], 
+      emit_insn (gen_fix_truncdfsi2_ibm (operands[0], operands[1],
 					 two31r, two32, temp));
-    } 
-  else 
+    }
+  else
     {
       operands[1] = force_reg (DFmode, operands[1]);
       emit_insn (gen_fix_truncdfsi2_ieee (operands[0], operands[1], GEN_INT (5)));
     }
 
   DONE;
-}")
+})
 
 (define_insn "fix_truncdfsi2_ieee"
   [(set (match_operand:SI 0 "register_operand" "=d")
         (fix:SI (match_operand:DF 1 "register_operand" "f")))
-    (unspec:SI [(match_operand:SI 2 "immediate_operand" "K")] 1)
+    (unspec:SI [(match_operand:SI 2 "immediate_operand" "K")] UNSPEC_ROUND)
     (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "cfdbr\\t%0,%h2,%1"
+  "cfdbr\t%0,%h2,%1"
    [(set_attr "op_type" "RRE")
     (set_attr "type"    "other" )])
 
@@ -2479,16 +2774,16 @@
    (use (match_operand:BLK 4 "memory_operand" "m"))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
-  "*
 {
-   output_asm_insn (\"sd\\t%1,%2\", operands);
-   output_asm_insn (\"aw\\t%1,%3\", operands);
-   output_asm_insn (\"std\\t%1,%4\", operands);
-   output_asm_insn (\"xi\\t%N4,128\", operands);
-   return \"l\\t%0,%N4\";
-}"
+   output_asm_insn ("sd\t%1,%2", operands);
+   output_asm_insn ("aw\t%1,%3", operands);
+   output_asm_insn ("std\t%1,%4", operands);
+   output_asm_insn ("xi\t%N4,128", operands);
+   return "l\t%0,%N4";
+}
   [(set_attr "op_type" "NN")
-   (set_attr "type"    "other")
+   (set_attr "type"    "ftoi")
+   (set_attr "atype"   "agen")
    (set_attr "length"  "20")])
 
 ;
@@ -2499,21 +2794,20 @@
   [(set (match_operand:DI 0 "register_operand" "")
         (unsigned_fix:DI (match_operand:SF 1 "register_operand" "")))]
   "TARGET_64BIT && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "
 {
   rtx label1 = gen_label_rtx ();
   rtx label2 = gen_label_rtx ();
   rtx temp = gen_reg_rtx (SFmode);
 
   operands[1] = force_reg (SFmode, operands[1]);
-  emit_insn (gen_cmpsf (operands[1], 
+  emit_insn (gen_cmpsf (operands[1],
 	CONST_DOUBLE_FROM_REAL_VALUE (
-          REAL_VALUE_ATOF (\"9223372036854775808.0\", SFmode), SFmode)));
+          REAL_VALUE_ATOF ("9223372036854775808.0", SFmode), SFmode)));
   emit_jump_insn (gen_blt (label1));
 
   emit_insn (gen_subsf3 (temp, operands[1],
 	CONST_DOUBLE_FROM_REAL_VALUE (
-          REAL_VALUE_ATOF (\"18446744073709551616.0\", SFmode), SFmode)));
+          REAL_VALUE_ATOF ("18446744073709551616.0", SFmode), SFmode)));
   emit_insn (gen_fix_truncsfdi2_ieee (operands[0], temp, GEN_INT(7)));
   emit_jump (label2);
 
@@ -2521,28 +2815,27 @@
   emit_insn (gen_fix_truncsfdi2_ieee (operands[0], operands[1], GEN_INT(5)));
   emit_label (label2);
   DONE;
-}")
+})
 
 (define_expand "fix_truncsfdi2"
   [(set (match_operand:DI 0 "register_operand" "")
         (fix:DI (match_operand:SF 1 "nonimmediate_operand" "")))]
   "TARGET_64BIT && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "
 {
   operands[1] = force_reg (SFmode, operands[1]);
   emit_insn (gen_fix_truncsfdi2_ieee (operands[0], operands[1], GEN_INT(5)));
   DONE;
-}")
+})
 
 (define_insn "fix_truncsfdi2_ieee"
   [(set (match_operand:DI 0 "register_operand" "=d")
         (fix:DI (match_operand:SF 1 "register_operand"  "f")))
-   (unspec:DI [(match_operand:DI 2 "immediate_operand" "K")] 1)
+   (unspec:DI [(match_operand:DI 2 "immediate_operand" "K")] UNSPEC_ROUND)
    (clobber (reg:CC 33))]
   "TARGET_64BIT && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "cgebr\\t%0,%h2,%1"
+  "cgebr\t%0,%h2,%1"
   [(set_attr "op_type" "RRE")
-   (set_attr "type"    "other")])
+   (set_attr "type"    "ftoi")])
 
 ;
 ; fixuns_truncsfsi2 and fix_truncsfsi2 instruction pattern(s).
@@ -2552,7 +2845,6 @@
   [(set (match_operand:SI 0 "register_operand" "")
         (unsigned_fix:SI (match_operand:SF 1 "register_operand" "")))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "
 {
   rtx label1 = gen_label_rtx ();
   rtx label2 = gen_label_rtx ();
@@ -2561,11 +2853,11 @@
   operands[1] = force_reg (SFmode, operands[1]);
   emit_insn (gen_cmpsf (operands[1],
 	CONST_DOUBLE_FROM_REAL_VALUE (
-          REAL_VALUE_ATOF (\"2147483648.0\", SFmode), SFmode)));
+          REAL_VALUE_ATOF ("2147483648.0", SFmode), SFmode)));
   emit_jump_insn (gen_blt (label1));
   emit_insn (gen_subsf3 (temp, operands[1],
 	CONST_DOUBLE_FROM_REAL_VALUE (
-          REAL_VALUE_ATOF (\"4294967296.0\", SFmode), SFmode)));
+          REAL_VALUE_ATOF ("4294967296.0", SFmode), SFmode)));
   emit_insn (gen_fix_truncsfsi2_ieee (operands[0], temp, GEN_INT (7)));
   emit_jump (label2);
 
@@ -2573,13 +2865,12 @@
   emit_insn (gen_fix_truncsfsi2_ieee (operands[0], operands[1], GEN_INT (5)));
   emit_label (label2);
   DONE;
-}")
+})
 
 (define_expand "fix_truncsfsi2"
   [(set (match_operand:SI 0 "register_operand" "")
         (fix:SI (match_operand:SF 1 "nonimmediate_operand" "")))]
   "TARGET_HARD_FLOAT"
-  "
 {
   if (TARGET_IBM_FLOAT)
     {
@@ -2595,17 +2886,17 @@
     }
 
   DONE;
-}")
+})
 
 (define_insn "fix_truncsfsi2_ieee"
   [(set (match_operand:SI 0 "register_operand" "=d")
         (fix:SI (match_operand:SF 1 "register_operand" "f")))
-    (unspec:SI [(match_operand:SI 2 "immediate_operand" "K")] 1)
+    (unspec:SI [(match_operand:SI 2 "immediate_operand" "K")] UNSPEC_ROUND)
     (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "cfebr\\t%0,%h2,%1"
+  "cfebr\t%0,%h2,%1"
   [(set_attr "op_type" "RRE")
-   (set_attr "type"    "other")])
+   (set_attr "type"    "ftoi")])
 
 ;
 ; floatdidf2 instruction pattern(s).
@@ -2616,9 +2907,9 @@
         (float:DF (match_operand:DI 1 "register_operand" "d")))
    (clobber (reg:CC 33))]
   "TARGET_64BIT && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "cdgbr\\t%0,%1"
+  "cdgbr\t%0,%1"
   [(set_attr "op_type" "RRE")
-   (set_attr "type"    "other" )])
+   (set_attr "type"    "itof" )])
 
 ;
 ; floatdisf2 instruction pattern(s).
@@ -2629,9 +2920,9 @@
         (float:SF (match_operand:DI 1 "register_operand" "d")))
    (clobber (reg:CC 33))]
   "TARGET_64BIT && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "cegbr\\t%0,%1"
+  "cegbr\t%0,%1"
   [(set_attr "op_type" "RRE")
-   (set_attr "type"    "other" )])
+   (set_attr "type"    "itof" )])
 
 ;
 ; floatsidf2 instruction pattern(s).
@@ -2643,28 +2934,27 @@
           (float:DF (match_operand:SI 1 "register_operand" "")))
      (clobber (reg:CC 33))])]
   "TARGET_HARD_FLOAT"
-  "
 {
-  if (TARGET_IBM_FLOAT) 
+  if (TARGET_IBM_FLOAT)
     {
       /* This is the algorithm from POP chapter A.5.7.1.  */
 
-      rtx temp  = assign_stack_local (BLKmode, 2 * UNITS_PER_WORD, BITS_PER_WORD);
-      rtx two31 = s390_gen_rtx_const_DI (0x4e000000, 0x80000000);	   
+      rtx temp  = assign_stack_local (BLKmode, 8, BITS_PER_WORD);
+      rtx two31 = s390_gen_rtx_const_DI (0x4e000000, 0x80000000);
 
       emit_insn (gen_floatsidf2_ibm (operands[0], operands[1], two31, temp));
       DONE;
     }
-}")
+})
 
 (define_insn "floatsidf2_ieee"
   [(set (match_operand:DF 0 "register_operand" "=f")
         (float:DF (match_operand:SI 1 "register_operand"  "d")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "cdfbr\\t%0,%1"
+  "cdfbr\t%0,%1"
   [(set_attr "op_type" "RRE")
-   (set_attr "type"   "other" )])
+   (set_attr "type"   "itof" )])
 
 (define_insn "floatsidf2_ibm"
   [(set (match_operand:DF 0 "register_operand" "=f")
@@ -2673,16 +2963,16 @@
    (use (match_operand:BLK 3 "memory_operand" "m"))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
-  "*
 {
-   output_asm_insn (\"st\\t%1,%N3\", operands);
-   output_asm_insn (\"xi\\t%N3,128\", operands);
-   output_asm_insn (\"mvc\\t%O3(4,%R3),%2\", operands);
-   output_asm_insn (\"ld\\t%0,%3\", operands);
-   return \"sd\\t%0,%2\";
-}"
+   output_asm_insn ("st\t%1,%N3", operands);
+   output_asm_insn ("xi\t%N3,128", operands);
+   output_asm_insn ("mvc\t%O3(4,%R3),%2", operands);
+   output_asm_insn ("ld\t%0,%3", operands);
+   return "sd\t%0,%2";
+}
   [(set_attr "op_type" "NN")
    (set_attr "type"    "other" )
+   (set_attr "atype"   "agen")
    (set_attr "length"  "20")])
 
 ;
@@ -2695,7 +2985,6 @@
           (float:SF (match_operand:SI 1 "register_operand" "")))
      (clobber (reg:CC 33))])]
   "TARGET_HARD_FLOAT"
-  "
 {
   if (TARGET_IBM_FLOAT)
     {
@@ -2705,16 +2994,16 @@
       emit_insn (gen_truncdfsf2 (operands[0], temp));
       DONE;
     }
-}")
+})
 
 (define_insn "floatsisf2_ieee"
   [(set (match_operand:SF 0 "register_operand" "=f")
         (float:SF (match_operand:SI 1 "register_operand" "d")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "cefbr\\t%0,%1"
+  "cefbr\t%0,%1"
   [(set_attr "op_type" "RRE")
-   (set_attr "type"    "other" )])
+   (set_attr "type"    "itof" )])
 
 ;
 ; truncdfsf2 instruction pattern(s).
@@ -2730,18 +3019,18 @@
   [(set (match_operand:SF 0 "register_operand" "=f")
         (float_truncate:SF (match_operand:DF 1 "general_operand" "f")))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "ledbr\\t%0,%1"
+  "ledbr\t%0,%1"
   [(set_attr "op_type"  "RRE")])
 
 (define_insn "truncdfsf2_ibm"
   [(set (match_operand:SF 0 "register_operand" "=f,f")
-        (float_truncate:SF (match_operand:DF 1 "general_operand" "f,m")))]
+        (float_truncate:SF (match_operand:DF 1 "general_operand" "f,R")))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
   "@
-   lrer\\t%0,%1
-   le\\t%0,%1"
+   lrer\t%0,%1
+   le\t%0,%1"
   [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
+   (set_attr "type"   "floads,floads")])
 
 ;
 ; extendsfdf2 instruction pattern(s).
@@ -2751,41 +3040,42 @@
   [(set (match_operand:DF 0 "register_operand" "")
         (float_extend:DF (match_operand:SF 1 "nonimmediate_operand" "")))]
   "TARGET_HARD_FLOAT"
-  "
 {
   if (TARGET_IBM_FLOAT)
     {
       emit_insn (gen_extendsfdf2_ibm (operands[0], operands[1]));
       DONE;
     }
-}")
+})
 
 (define_insn "extendsfdf2_ieee"
   [(set (match_operand:DF 0 "register_operand" "=f,f")
-        (float_extend:DF (match_operand:SF 1 "nonimmediate_operand"  "f,m")))]
+        (float_extend:DF (match_operand:SF 1 "nonimmediate_operand"  "f,R")))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   ldebr\\t%0,%1
-   ldeb\\t%0,%1"
-  [(set_attr "op_type"  "RRE,RXE")])
+   ldebr\t%0,%1
+   ldeb\t%0,%1"
+  [(set_attr "op_type"  "RRE,RXE")
+   (set_attr "type"   "floads,floads")])
 
 (define_insn "extendsfdf2_ibm"
   [(set (match_operand:DF 0 "register_operand" "=f,f")
-        (float_extend:DF (match_operand:SF 1 "nonimmediate_operand" "f,m")))
+        (float_extend:DF (match_operand:SF 1 "nonimmediate_operand" "f,R")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
   "@
-   sdr\\t%0,%0\;ler\\t%0,%1
-   sdr\\t%0,%0\;le\\t%0,%1"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")
-   (set_attr "type"     "o2,o2")]) 
+   sdr\t%0,%0\;ler\t%0,%1
+   sdr\t%0,%0\;le\t%0,%1"
+  [(set_attr "op_type"  "NN,NN")
+   (set_attr "atype"    "reg,agen")
+   (set_attr "length"   "4,6")
+   (set_attr "type"     "o2,o2")])
 
 
 ;;
-;; ARITHMETRIC OPERATIONS
+;; ARITHMETIC OPERATIONS
 ;;
-;  arithmetric operations set the ConditionCode,
+;  arithmetic operations set the ConditionCode,
 ;  because of unpredictable Bits in Register for Halfword and Byte
 ;  the ConditionCode can be set wrong in operations for Halfword and Byte
 
@@ -2794,27 +3084,37 @@
 ;;
 
 ;
-; adddi3 instruction pattern(s).
+; addti3 instruction pattern(s).
 ;
 
-(define_insn "*la_64_cc"
-  [(set (match_operand:DI 0 "register_operand" "=d")
-        (match_operand:QI 1 "address_operand"  "p"))
+(define_insn_and_split "addti3"
+  [(set (match_operand:TI 0 "register_operand" "=&d")
+        (plus:TI (match_operand:TI 1 "nonimmediate_operand" "%0")
+                 (match_operand:TI 2 "general_operand" "do") ) )
    (clobber (reg:CC 33))]
-  "TARGET_64BIT
-   && preferred_la_operand_p (operands[1], 1)"
+  "TARGET_64BIT"
   "#"
-  [(set_attr "op_type"  "RX")
-   (set_attr "atype"    "mem")
-   (set_attr "type"     "la")])
+  "&& reload_completed"
+  [(parallel
+    [(set (reg:CCL1 33)
+          (compare:CCL1 (plus:DI (match_dup 7) (match_dup 8))
+                        (match_dup 7)))
+     (set (match_dup 6) (plus:DI (match_dup 7) (match_dup 8)))])
+   (parallel
+    [(set (match_dup 3) (plus:DI (plus:DI (match_dup 4) (match_dup 5))
+                                 (ltu:DI (reg:CCL1 33) (const_int 0))))
+     (clobber (reg:CC 33))])]
+  "operands[3] = operand_subword (operands[0], 0, 0, TImode);
+   operands[4] = operand_subword (operands[1], 0, 0, TImode);
+   operands[5] = operand_subword (operands[2], 0, 0, TImode);
+   operands[6] = operand_subword (operands[0], 1, 0, TImode);
+   operands[7] = operand_subword (operands[1], 1, 0, TImode);
+   operands[8] = operand_subword (operands[2], 1, 0, TImode);"
+  [(set_attr "op_type"  "NN")])
 
-(define_split
-  [(set (match_operand:DI 0 "register_operand" "")
-        (match_operand:QI 1 "address_operand" ""))
-   (clobber (reg:CC 33))]
-  "TARGET_64BIT && reload_completed
-   && preferred_la_operand_p (operands[1], 0)"
-  [(set (match_dup 0) (match_dup 1))])
+;
+; adddi3 instruction pattern(s).
+;
 
 (define_insn "*adddi3_sign"
   [(set (match_operand:DI 0 "register_operand" "=d,d")
@@ -2823,13 +3123,12 @@
    (clobber (reg:CC 33))]
   "TARGET_64BIT"
   "@
-   agfr\\t%0,%2
-   agf\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
+   agfr\t%0,%2
+   agf\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "*adddi3_zero_cc"
-  [(set (reg 33) 
+  [(set (reg 33)
         (compare (plus:DI (zero_extend:DI (match_operand:SI 2 "general_operand" "d,m"))
                           (match_operand:DI 1 "register_operand" "0,0"))
                  (const_int 0)))
@@ -2837,23 +3136,21 @@
         (plus:DI (zero_extend:DI (match_dup 2)) (match_dup 1)))]
   "s390_match_ccmode (insn, CCLmode) && TARGET_64BIT"
   "@
-   algfr\\t%0,%2
-   algf\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])  
+   algfr\t%0,%2
+   algf\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "*adddi3_zero_cconly"
-  [(set (reg 33) 
+  [(set (reg 33)
         (compare (plus:DI (zero_extend:DI (match_operand:SI 2 "general_operand" "d,m"))
                           (match_operand:DI 1 "register_operand" "0,0"))
                  (const_int 0)))
    (clobber (match_scratch:DI 0 "=d,d"))]
   "s390_match_ccmode (insn, CCLmode) && TARGET_64BIT"
   "@
-   algfr\\t%0,%2
-   algf\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])  
+   algfr\t%0,%2
+   algf\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "*adddi3_zero"
   [(set (match_operand:DI 0 "register_operand" "=d,d")
@@ -2862,27 +3159,75 @@
    (clobber (reg:CC 33))]
   "TARGET_64BIT"
   "@
-   algfr\\t%0,%2
-   algf\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
+   algfr\t%0,%2
+   algf\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "*adddi3_imm_cc"
-  [(set (reg 33) 
+  [(set (reg 33)
         (compare (plus:DI (match_operand:DI 1 "nonimmediate_operand" "0")
                           (match_operand:DI 2 "const_int_operand" "K"))
                  (const_int 0)))
    (set (match_operand:DI 0 "register_operand" "=d")
         (plus:DI (match_dup 1) (match_dup 2)))]
-  "TARGET_64BIT	 
-   && s390_match_ccmode (insn, CCAmode) 
-   && CONST_OK_FOR_LETTER_P (INTVAL (operands[2]), 'K')"
-  "aghi\\t%0,%h2"
-  [(set_attr "op_type"  "RI")
-   (set_attr "atype"    "reg")])  
+  "TARGET_64BIT
+   && s390_match_ccmode (insn, CCAmode)
+   && CONST_OK_FOR_CONSTRAINT_P (INTVAL (operands[2]), 'K', \"K\")"
+  "aghi\t%0,%h2"
+  [(set_attr "op_type"  "RI")])
+
+(define_insn "*adddi3_carry1_cc"
+  [(set (reg 33)
+        (compare (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
+                          (match_operand:DI 2 "general_operand" "d,m"))
+                 (match_dup 1)))
+   (set (match_operand:DI 0 "register_operand" "=d,d")
+        (plus:DI (match_dup 1) (match_dup 2)))]
+  "s390_match_ccmode (insn, CCL1mode) && TARGET_64BIT"
+  "@
+   algr\t%0,%2
+   alg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
+
+(define_insn "*adddi3_carry1_cconly"
+  [(set (reg 33)
+        (compare (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
+                          (match_operand:DI 2 "general_operand" "d,m"))
+                 (match_dup 1)))
+   (clobber (match_scratch:DI 0 "=d,d"))]
+  "s390_match_ccmode (insn, CCL1mode) && TARGET_64BIT"
+  "@
+   algr\t%0,%2
+   alg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
+
+(define_insn "*adddi3_carry2_cc"
+  [(set (reg 33)
+        (compare (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
+                          (match_operand:DI 2 "general_operand" "d,m"))
+                 (match_dup 2)))
+   (set (match_operand:DI 0 "register_operand" "=d,d")
+        (plus:DI (match_dup 1) (match_dup 2)))]
+  "s390_match_ccmode (insn, CCL1mode) && TARGET_64BIT"
+  "@
+   algr\t%0,%2
+   alg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
+
+(define_insn "*adddi3_carry2_cconly"
+  [(set (reg 33)
+        (compare (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
+                          (match_operand:DI 2 "general_operand" "d,m"))
+                 (match_dup 2)))
+   (clobber (match_scratch:DI 0 "=d,d"))]
+  "s390_match_ccmode (insn, CCL1mode) && TARGET_64BIT"
+  "@
+   algr\t%0,%2
+   alg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "*adddi3_cc"
-  [(set (reg 33) 
+  [(set (reg 33)
         (compare (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
                           (match_operand:DI 2 "general_operand" "d,m"))
                  (const_int 0)))
@@ -2890,35 +3235,32 @@
         (plus:DI (match_dup 1) (match_dup 2)))]
   "s390_match_ccmode (insn, CCLmode) && TARGET_64BIT"
   "@
-   algr\\t%0,%2
-   alg\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])  
+   algr\t%0,%2
+   alg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "*adddi3_cconly"
-  [(set (reg 33) 
+  [(set (reg 33)
         (compare (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
                           (match_operand:DI 2 "general_operand" "d,m"))
                  (const_int 0)))
    (clobber (match_scratch:DI 0 "=d,d"))]
   "s390_match_ccmode (insn, CCLmode) && TARGET_64BIT"
   "@
-   algr\\t%0,%2
-   alg\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])  
+   algr\t%0,%2
+   alg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "*adddi3_cconly2"
-  [(set (reg 33) 
+  [(set (reg 33)
         (compare (match_operand:DI 1 "nonimmediate_operand" "%0,0")
                  (neg:SI (match_operand:DI 2 "general_operand" "d,m"))))
    (clobber (match_scratch:DI 0 "=d,d"))]
   "s390_match_ccmode(insn, CCLmode) && TARGET_64BIT"
   "@
-   algr\\t%0,%2
-   alg\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])  
+   algr\t%0,%2
+   alg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "*adddi3_64"
   [(set (match_operand:DI 0 "register_operand" "=d,d,d")
@@ -2927,18 +3269,42 @@
    (clobber (reg:CC 33))]
   "TARGET_64BIT"
   "@
-   agr\\t%0,%2
-   aghi\\t%0,%h2
-   ag\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RI,RXE")
-   (set_attr "atype"    "reg,reg,mem")])
+   agr\t%0,%2
+   aghi\t%0,%h2
+   ag\t%0,%2"
+  [(set_attr "op_type"  "RRE,RI,RXY")])
+
+(define_insn_and_split "*adddi3_31z"
+  [(set (match_operand:DI 0 "register_operand" "=&d")
+        (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0")
+                 (match_operand:DI 2 "general_operand" "do") ) )
+   (clobber (reg:CC 33))]
+  "!TARGET_64BIT && TARGET_CPU_ZARCH"
+  "#"
+  "&& reload_completed"
+  [(parallel
+    [(set (reg:CCL1 33)
+          (compare:CCL1 (plus:SI (match_dup 7) (match_dup 8))
+                        (match_dup 7)))
+     (set (match_dup 6) (plus:SI (match_dup 7) (match_dup 8)))])
+   (parallel
+    [(set (match_dup 3) (plus:SI (plus:SI (match_dup 4) (match_dup 5))
+                                 (ltu:SI (reg:CCL1 33) (const_int 0))))
+     (clobber (reg:CC 33))])]
+  "operands[3] = operand_subword (operands[0], 0, 0, DImode);
+   operands[4] = operand_subword (operands[1], 0, 0, DImode);
+   operands[5] = operand_subword (operands[2], 0, 0, DImode);
+   operands[6] = operand_subword (operands[0], 1, 0, DImode);
+   operands[7] = operand_subword (operands[1], 1, 0, DImode);
+   operands[8] = operand_subword (operands[2], 1, 0, DImode);"
+  [(set_attr "op_type"  "NN")])
 
 (define_insn_and_split "*adddi3_31"
   [(set (match_operand:DI 0 "register_operand" "=&d")
         (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0")
-                 (match_operand:DI 2 "general_operand" "dm") ) )
+                 (match_operand:DI 2 "general_operand" "do") ) )
    (clobber (reg:CC 33))]
-  "!TARGET_64BIT"
+  "!TARGET_CPU_ZARCH"
   "#"
   "&& reload_completed"
   [(parallel
@@ -2957,15 +3323,14 @@
     [(set (match_dup 3) (plus:SI (match_dup 3) (const_int 1)))
      (clobber (reg:CC 33))])
    (match_dup 9)]
-  "operands[3] = operand_subword (operands[0], 0, 1, DImode);
-   operands[4] = operand_subword (operands[1], 0, 1, DImode);
-   operands[5] = operand_subword (operands[2], 0, 1, DImode);
-   operands[6] = operand_subword (operands[0], 1, 1, DImode);
-   operands[7] = operand_subword (operands[1], 1, 1, DImode);
-   operands[8] = operand_subword (operands[2], 1, 1, DImode);
+  "operands[3] = operand_subword (operands[0], 0, 0, DImode);
+   operands[4] = operand_subword (operands[1], 0, 0, DImode);
+   operands[5] = operand_subword (operands[2], 0, 0, DImode);
+   operands[6] = operand_subword (operands[0], 1, 0, DImode);
+   operands[7] = operand_subword (operands[1], 1, 0, DImode);
+   operands[8] = operand_subword (operands[2], 1, 0, DImode);
    operands[9] = gen_label_rtx ();"
-  [(set_attr "op_type"  "NN")
-   (set_attr "type"     "o3")])
+  [(set_attr "op_type"  "NN")])
 
 (define_expand "adddi3"
   [(parallel
@@ -2976,244 +3341,138 @@
   ""
   "")
 
-(define_insn "*la_64"
-  [(set (match_operand:DI 0 "register_operand" "=d")
-        (match_operand:QI 1 "address_operand" "p"))]
-  "TARGET_64BIT"
-  "la\\t%0,%a1"      
-  [(set_attr "op_type" "RX")
-   (set_attr "atype"   "mem")
-   (set_attr "type"    "la")])
-
-(define_expand "reload_indi"
-  [(parallel [(match_operand:DI 0 "register_operand" "=a")
-              (match_operand:DI 1 "s390_plus_operand" "")
-              (match_operand:DI 2 "register_operand" "=&a")])]
-  "TARGET_64BIT"
-  "
-{
-  s390_expand_plus_operand (operands[0], operands[1], operands[2]);
-  DONE;
-}")
-
- 
 ;
 ; addsi3 instruction pattern(s).
 ;
 
-(define_insn "*la_31_cc"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (match_operand:QI 1 "address_operand"  "p"))
-   (clobber (reg:CC 33))]
-  "!TARGET_64BIT
-   && preferred_la_operand_p (operands[1], 1)"
-  "#"
-  [(set_attr "op_type"  "RX")
-   (set_attr "atype"    "mem")
-   (set_attr "type"     "la")])
-
-(define_split
-  [(set (match_operand:SI 0 "register_operand" "")
-        (match_operand:QI 1 "address_operand" ""))
-   (clobber (reg:CC 33))]
-  "!TARGET_64BIT && reload_completed
-   && preferred_la_operand_p (operands[1], 0)"
-  [(set (match_dup 0) (match_dup 1))])
-
 (define_insn "*addsi3_imm_cc"
-  [(set (reg 33) 
+  [(set (reg 33)
         (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "0")
                           (match_operand:SI 2 "const_int_operand" "K"))
                  (const_int 0)))
    (set (match_operand:SI 0 "register_operand" "=d")
         (plus:SI (match_dup 1) (match_dup 2)))]
   "s390_match_ccmode (insn, CCAmode)
-   && CONST_OK_FOR_LETTER_P (INTVAL (operands[2]), 'K')"
-  "ahi\\t%0,%h2"
-  [(set_attr "op_type"  "RI")
-   (set_attr "atype"    "reg")])  
+   && CONST_OK_FOR_CONSTRAINT_P (INTVAL (operands[2]), 'K', \"K\")"
+  "ahi\t%0,%h2"
+  [(set_attr "op_type"  "RI")])
 
 (define_insn "*addsi3_carry1_cc"
-  [(set (reg 33) 
-        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                          (match_operand:SI 2 "general_operand" "d,m"))
+  [(set (reg 33)
+        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
+                          (match_operand:SI 2 "general_operand" "d,R,T"))
                  (match_dup 1)))
-   (set (match_operand:SI 0 "register_operand" "=d,d")
+   (set (match_operand:SI 0 "register_operand" "=d,d,d")
         (plus:SI (match_dup 1) (match_dup 2)))]
-  "s390_match_ccmode (insn, CCL1mode)" 
+  "s390_match_ccmode (insn, CCL1mode)"
   "@
-   alr\\t%0,%2
-   al\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])  
+   alr\t%0,%2
+   al\t%0,%2
+   aly\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
 (define_insn "*addsi3_carry1_cconly"
-  [(set (reg 33) 
-        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                          (match_operand:SI 2 "general_operand" "d,m"))
+  [(set (reg 33)
+        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
+                          (match_operand:SI 2 "general_operand" "d,R,T"))
                  (match_dup 1)))
-   (clobber (match_scratch:SI 0 "=d,d"))]
-  "s390_match_ccmode (insn, CCL1mode)" 
+   (clobber (match_scratch:SI 0 "=d,d,d"))]
+  "s390_match_ccmode (insn, CCL1mode)"
   "@
-   alr\\t%0,%2
-   al\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])  
+   alr\t%0,%2
+   al\t%0,%2
+   aly\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
 (define_insn "*addsi3_carry2_cc"
-  [(set (reg 33) 
-        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                          (match_operand:SI 2 "general_operand" "d,m"))
+  [(set (reg 33)
+        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
+                          (match_operand:SI 2 "general_operand" "d,R,T"))
                  (match_dup 2)))
-   (set (match_operand:SI 0 "register_operand" "=d,d")
+   (set (match_operand:SI 0 "register_operand" "=d,d,d")
         (plus:SI (match_dup 1) (match_dup 2)))]
-  "s390_match_ccmode (insn, CCL1mode)" 
+  "s390_match_ccmode (insn, CCL1mode)"
   "@
-   alr\\t%0,%2
-   al\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])  
+   alr\t%0,%2
+   al\t%0,%2
+   aly\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
 (define_insn "*addsi3_carry2_cconly"
-  [(set (reg 33) 
-        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                          (match_operand:SI 2 "general_operand" "d,m"))
+  [(set (reg 33)
+        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
+                          (match_operand:SI 2 "general_operand" "d,R,T"))
                  (match_dup 2)))
-   (clobber (match_scratch:SI 0 "=d,d"))]
-  "s390_match_ccmode (insn, CCL1mode)" 
+   (clobber (match_scratch:SI 0 "=d,d,d"))]
+  "s390_match_ccmode (insn, CCL1mode)"
   "@
-   alr\\t%0,%2
-   al\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])  
+   alr\t%0,%2
+   al\t%0,%2
+   aly\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
 (define_insn "*addsi3_cc"
-  [(set (reg 33) 
-        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                          (match_operand:SI 2 "general_operand" "d,m"))
+  [(set (reg 33)
+        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
+                          (match_operand:SI 2 "general_operand" "d,R,T"))
                  (const_int 0)))
-   (set (match_operand:SI 0 "register_operand" "=d,d")
+   (set (match_operand:SI 0 "register_operand" "=d,d,d")
         (plus:SI (match_dup 1) (match_dup 2)))]
-  "s390_match_ccmode (insn, CCLmode)" 
+  "s390_match_ccmode (insn, CCLmode)"
   "@
-   alr\\t%0,%2
-   al\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])  
+   alr\t%0,%2
+   al\t%0,%2
+   aly\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
 (define_insn "*addsi3_cconly"
-  [(set (reg 33) 
-        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                          (match_operand:SI 2 "general_operand" "d,m"))
+  [(set (reg 33)
+        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
+                          (match_operand:SI 2 "general_operand" "d,R,T"))
                  (const_int 0)))
-   (clobber (match_scratch:SI 0 "=d,d"))]
-  "s390_match_ccmode (insn, CCLmode)" 
+   (clobber (match_scratch:SI 0 "=d,d,d"))]
+  "s390_match_ccmode (insn, CCLmode)"
   "@
-   alr\\t%0,%2
-   al\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])  
+   alr\t%0,%2
+   al\t%0,%2
+   aly\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
 (define_insn "*addsi3_cconly2"
-  [(set (reg 33) 
-        (compare (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                 (neg:SI (match_operand:SI 2 "general_operand" "d,m"))))
-   (clobber (match_scratch:SI 0 "=d,d"))]
-  "s390_match_ccmode(insn, CCLmode)"
-  "@
-   alr\\t%0,%2
-   al\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])  
+  [(set (reg 33)
+        (compare (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
+                 (neg:SI (match_operand:SI 2 "general_operand" "d,R,T"))))
+   (clobber (match_scratch:SI 0 "=d,d,d"))]
+  "s390_match_ccmode (insn, CCLmode)"
+  "@
+   alr\t%0,%2
+   al\t%0,%2
+   aly\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
 (define_insn "*addsi3_sign"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (plus:SI (match_operand:SI 1 "register_operand" "0")
-                 (sign_extend:SI (match_operand:HI 2 "memory_operand" "m"))))
-   (clobber (reg:CC 33))]
-  ""
-  "ah\\t%0,%2"
-  [(set_attr "op_type"  "RX")
-   (set_attr "atype"    "mem")])
-
-(define_insn "*addsi3_sub"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (plus:SI (match_operand:SI 1 "register_operand" "0")
-                 (subreg:SI (match_operand:HI 2 "memory_operand" "m") 0)))
+  [(set (match_operand:SI 0 "register_operand" "=d,d")
+        (plus:SI (match_operand:SI 1 "register_operand" "0,0")
+                 (sign_extend:SI (match_operand:HI 2 "memory_operand" "R,T"))))
    (clobber (reg:CC 33))]
   ""
-  "ah\\t%0,%2"
-  [(set_attr "op_type"  "RX")
-   (set_attr "atype"    "mem")])
+  "@
+   ah\t%0,%2
+   ahy\t%0,%2"
+  [(set_attr "op_type"  "RX,RXY")])
 
 (define_insn "addsi3"
-  [(set (match_operand:SI 0 "register_operand" "=d,d,d")
-        (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
-                 (match_operand:SI 2 "general_operand" "d,K,m")))
+  [(set (match_operand:SI 0 "register_operand" "=d,d,d,d")
+        (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0,0")
+                 (match_operand:SI 2 "general_operand" "d,K,R,T")))
    (clobber (reg:CC 33))]
   ""
   "@
-   ar\\t%0,%2
-   ahi\\t%0,%h2
-   a\\t%0,%2"
-  [(set_attr "op_type"  "RR,RI,RX")
-   (set_attr "atype"    "reg,reg,mem")])
-
-(define_insn "*la_31"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (match_operand:QI 1 "address_operand" "p"))]
-  "!TARGET_64BIT && legitimate_la_operand_p (operands[1])"
-  "la\\t%0,%a1"
-  [(set_attr "op_type"  "RX")
-   (set_attr "atype"    "mem")
-   (set_attr "type"     "la")])
-
-(define_insn "*la_31_and"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (and:SI (match_operand:QI 1 "address_operand" "p")
-                (const_int 2147483647)))]
-  "!TARGET_64BIT"
-  "la\\t%0,%a1"
-  [(set_attr "op_type"  "RX")
-   (set_attr "atype"    "mem")
-   (set_attr "type"     "la")])
-
-(define_insn_and_split "*la_31_and_cc"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (and:SI (match_operand:QI 1 "address_operand" "p")
-                (const_int 2147483647)))
-   (clobber (reg:CC 33))]
-  "!TARGET_64BIT"
-  "#"
-  "&& reload_completed"
-  [(set (match_dup 0) 
-        (and:SI (match_dup 1) (const_int 2147483647)))]
-  ""
-  [(set_attr "op_type"  "RX")
-   (set_attr "atype"    "mem")
-   (set_attr "type"     "la")])
-
-(define_insn "force_la_31"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (match_operand:QI 1 "address_operand" "p"))
-   (use (const_int 0))]
-  "!TARGET_64BIT"
-  "la\\t%0,%a1"
-  [(set_attr "op_type"  "RX")
-   (set_attr "atype"    "mem")
-   (set_attr "type"     "la")])
-
-(define_expand "reload_insi"
-  [(parallel [(match_operand:SI 0 "register_operand" "=a")
-              (match_operand:SI 1 "s390_plus_operand" "")
-              (match_operand:SI 2 "register_operand" "=&a")])]
-  "!TARGET_64BIT"
-  "
-{
-  s390_expand_plus_operand (operands[0], operands[1], operands[2]);
-  DONE;
-}")
-
+   ar\t%0,%2
+   ahi\t%0,%h2
+   a\t%0,%2
+   ay\t%0,%2"
+  [(set_attr "op_type"  "RR,RI,RX,RXY")])
 
 ;
 ; adddf3 instruction pattern(s).
@@ -3223,7 +3482,7 @@
   [(parallel
     [(set (match_operand:DF 0 "register_operand" "=f,f")
           (plus:DF (match_operand:DF 1 "nonimmediate_operand" "%0,0")
-                   (match_operand:DF 2 "general_operand" "f,m")))
+                   (match_operand:DF 2 "general_operand" "f,R")))
      (clobber (reg:CC 33))])]
   "TARGET_HARD_FLOAT"
   "")
@@ -3231,26 +3490,53 @@
 (define_insn "*adddf3"
   [(set (match_operand:DF 0 "register_operand" "=f,f")
         (plus:DF (match_operand:DF 1 "nonimmediate_operand" "%0,0")
-                 (match_operand:DF 2 "general_operand" "f,m")))
+                 (match_operand:DF 2 "general_operand" "f,R")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   adbr\\t%0,%2
-   adb\\t%0,%2"
+   adbr\t%0,%2
+   adb\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXE")
+   (set_attr "type"     "fsimpd,fsimpd")])
+
+(define_insn "*adddf3_cc"
+  [(set (reg 33)
+	(compare (plus:DF (match_operand:DF 1 "nonimmediate_operand" "%0,0")
+			  (match_operand:DF 2 "general_operand" "f,R"))
+		 (match_operand:DF 3 "const0_operand" "")))
+   (set (match_operand:DF 0 "register_operand" "=f,f")
+	(plus:DF (match_dup 1) (match_dup 2)))]
+  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "@
+   adbr\t%0,%2
+   adb\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXE")
+   (set_attr "type"     "fsimpd,fsimpd")])
+
+(define_insn "*adddf3_cconly"
+  [(set (reg 33)
+	(compare (plus:DF (match_operand:DF 1 "nonimmediate_operand" "%0,0")
+			  (match_operand:DF 2 "general_operand" "f,R"))
+		 (match_operand:DF 3 "const0_operand" "")))
+   (clobber (match_scratch:DF 0 "=f,f"))]
+  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "@
+  adbr\t%0,%2
+  adb\t%0,%2"
   [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
+   (set_attr "type"     "fsimpd,fsimpd")])
 
 (define_insn "*adddf3_ibm"
   [(set (match_operand:DF 0 "register_operand" "=f,f")
         (plus:DF (match_operand:DF 1 "nonimmediate_operand" "%0,0")
-                 (match_operand:DF 2 "general_operand" "f,m")))
+                 (match_operand:DF 2 "general_operand" "f,R")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
   "@
-   adr\\t%0,%2
-   ad\\t%0,%2"
+   adr\t%0,%2
+   ad\t%0,%2"
   [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
+   (set_attr "type"     "fsimpd,fsimpd")])
 
 ;
 ; addsf3 instruction pattern(s).
@@ -3260,7 +3546,7 @@
   [(parallel
     [(set (match_operand:SF 0 "register_operand" "=f,f")
           (plus:SF (match_operand:SF 1 "nonimmediate_operand" "%0,0")
-                   (match_operand:SF 2 "general_operand" "f,m")))
+                   (match_operand:SF 2 "general_operand" "f,R")))
      (clobber (reg:CC 33))])]
   "TARGET_HARD_FLOAT"
   "")
@@ -3268,26 +3554,53 @@
 (define_insn "*addsf3"
   [(set (match_operand:SF 0 "register_operand" "=f,f")
         (plus:SF (match_operand:SF 1 "nonimmediate_operand" "%0,0")
-                 (match_operand:SF 2 "general_operand" "f,m")))
+                 (match_operand:SF 2 "general_operand" "f,R")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   aebr\\t%0,%2
-   aeb\\t%0,%2"
+   aebr\t%0,%2
+   aeb\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXE")
+   (set_attr "type"     "fsimps,fsimps")])
+
+(define_insn "*addsf3_cc"
+  [(set (reg 33)
+	(compare (plus:SF (match_operand:SF 1 "nonimmediate_operand" "%0,0")
+			  (match_operand:SF 2 "general_operand" "f,R"))
+		 (match_operand:SF 3 "const0_operand" "")))
+   (set (match_operand:SF 0 "register_operand" "=f,f")
+	(plus:SF (match_dup 1) (match_dup 2)))]
+  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "@
+   aebr\t%0,%2
+   aeb\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXE")
+   (set_attr "type"     "fsimps,fsimps")])
+
+(define_insn "*addsf3_cconly"
+  [(set (reg 33)
+	(compare (plus:SF (match_operand:SF 1 "nonimmediate_operand" "%0,0")
+			  (match_operand:SF 2 "general_operand" "f,R"))
+		 (match_operand:SF 3 "const0_operand" "")))
+   (clobber (match_scratch:SF 0 "=f,f"))]
+  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "@
+   aebr\t%0,%2
+   aeb\t%0,%2"
   [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
+   (set_attr "type"     "fsimps,fsimps")])
 
 (define_insn "*addsf3"
   [(set (match_operand:SF 0 "register_operand" "=f,f")
         (plus:SF (match_operand:SF 1 "nonimmediate_operand" "%0,0")
-                 (match_operand:SF 2 "general_operand" "f,m")))
+                 (match_operand:SF 2 "general_operand" "f,R")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
   "@
-   aer\\t%0,%2
-   ae\\t%0,%2"
+   aer\t%0,%2
+   ae\t%0,%2"
   [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
+   (set_attr "type"     "fsimps,fsimps")])
 
 
 ;;
@@ -3295,6 +3608,35 @@
 ;;
 
 ;
+; subti3 instruction pattern(s).
+;
+
+(define_insn_and_split "subti3"
+  [(set (match_operand:TI 0 "register_operand" "=&d")
+        (minus:TI (match_operand:TI 1 "register_operand" "0")
+                  (match_operand:TI 2 "general_operand" "do") ) )
+   (clobber (reg:CC 33))]
+  "TARGET_64BIT"
+  "#"
+  "&& reload_completed"
+  [(parallel
+    [(set (reg:CCL2 33)
+          (compare:CCL2 (minus:DI (match_dup 7) (match_dup 8))
+                        (match_dup 7)))
+     (set (match_dup 6) (minus:DI (match_dup 7) (match_dup 8)))])
+   (parallel
+    [(set (match_dup 3) (minus:DI (minus:DI (match_dup 4) (match_dup 5))
+                                  (gtu:DI (reg:CCL2 33) (const_int 0))))
+     (clobber (reg:CC 33))])]
+  "operands[3] = operand_subword (operands[0], 0, 0, TImode);
+   operands[4] = operand_subword (operands[1], 0, 0, TImode);
+   operands[5] = operand_subword (operands[2], 0, 0, TImode);
+   operands[6] = operand_subword (operands[0], 1, 0, TImode);
+   operands[7] = operand_subword (operands[1], 1, 0, TImode);
+   operands[8] = operand_subword (operands[2], 1, 0, TImode);"
+  [(set_attr "op_type"  "NN")])
+
+;
 ; subdi3 instruction pattern(s).
 ;
 
@@ -3305,13 +3647,12 @@
    (clobber (reg:CC 33))]
   "TARGET_64BIT"
   "@
-   sgfr\\t%0,%2
-   sgf\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
+   sgfr\t%0,%2
+   sgf\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "*subdi3_zero_cc"
-  [(set (reg 33) 
+  [(set (reg 33)
         (compare (minus:DI (match_operand:DI 1 "register_operand" "0,0")
                            (zero_extend:DI (match_operand:SI 2 "general_operand" "d,m")))
                  (const_int 0)))
@@ -3319,23 +3660,21 @@
         (minus:DI (match_dup 1) (zero_extend:DI (match_dup 2))))]
   "s390_match_ccmode (insn, CCLmode) && TARGET_64BIT"
   "@
-   slgfr\\t%0,%2
-   slgf\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])  
+   slgfr\t%0,%2
+   slgf\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "*subdi3_zero_cconly"
-  [(set (reg 33) 
+  [(set (reg 33)
         (compare (minus:DI (match_operand:DI 1 "register_operand" "0,0")
                            (zero_extend:DI (match_operand:SI 2 "general_operand" "d,m")))
                  (const_int 0)))
    (clobber (match_scratch:DI 0 "=d,d"))]
   "s390_match_ccmode (insn, CCLmode) && TARGET_64BIT"
   "@
-   slgfr\\t%0,%2
-   slgf\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])  
+   slgfr\t%0,%2
+   slgf\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "*subdi3_zero"
   [(set (match_operand:DI 0 "register_operand" "=d,d")
@@ -3344,10 +3683,34 @@
    (clobber (reg:CC 33))]
   "TARGET_64BIT"
   "@
-   slgfr\\t%0,%2
-   slgf\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
+   slgfr\t%0,%2
+   slgf\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
+
+(define_insn "*subdi3_borrow_cc"
+  [(set (reg 33)
+        (compare (minus:DI (match_operand:DI 1 "register_operand" "0,0")
+                           (match_operand:DI 2 "general_operand" "d,m"))
+                 (match_dup 1)))
+   (set (match_operand:DI 0 "register_operand" "=d,d")
+        (minus:DI (match_dup 1) (match_dup 2)))]
+  "s390_match_ccmode (insn, CCL2mode) && TARGET_64BIT"
+  "@
+   slgr\t%0,%2
+   slg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
+
+(define_insn "*subdi3_borrow_cconly"
+  [(set (reg 33)
+        (compare (minus:DI (match_operand:DI 1 "register_operand" "0,0")
+                           (match_operand:DI 2 "general_operand" "d,m"))
+                 (match_dup 1)))
+   (clobber (match_scratch:DI 0 "=d,d"))]
+  "s390_match_ccmode (insn, CCL2mode) && TARGET_64BIT"
+  "@
+   slgr\t%0,%2
+   slg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "*subdi3_cc"
   [(set (reg 33)
@@ -3356,12 +3719,23 @@
                  (const_int 0)))
    (set (match_operand:DI 0 "register_operand" "=d,d")
         (minus:DI (match_dup 1) (match_dup 2)))]
-  "s390_match_ccmode (insn, CCLmode)"
+  "s390_match_ccmode (insn, CCLmode) && TARGET_64BIT"
   "@
-   slgr\\t%0,%2
-   slg\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
+   slgr\t%0,%2
+   slg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
+
+(define_insn "*subdi3_cc2"
+  [(set (reg 33)
+        (compare (match_operand:DI 1 "register_operand" "0,0")
+                 (match_operand:DI 2 "general_operand" "d,m")))
+   (set (match_operand:DI 0 "register_operand" "=d,d")
+        (minus:DI (match_dup 1) (match_dup 2)))]
+  "s390_match_ccmode (insn, CCL3mode) && TARGET_64BIT"
+  "@
+   slgr\t%0,%2
+   slg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "*subdi3_cconly"
   [(set (reg 33)
@@ -3369,12 +3743,22 @@
                            (match_operand:DI 2 "general_operand" "d,m"))
                  (const_int 0)))
    (clobber (match_scratch:DI 0 "=d,d"))]
-  "s390_match_ccmode (insn, CCLmode)"
+  "s390_match_ccmode (insn, CCLmode) && TARGET_64BIT"
   "@
-   slgr\\t%0,%2
-   slg\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
+   slgr\t%0,%2
+   slg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
+
+(define_insn "*subdi3_cconly2"
+  [(set (reg 33)
+        (compare (match_operand:DI 1 "register_operand" "0,0")
+                 (match_operand:DI 2 "general_operand" "d,m")))
+   (clobber (match_scratch:DI 0 "=d,d"))]
+  "s390_match_ccmode (insn, CCL3mode) && TARGET_64BIT"
+  "@
+   slgr\t%0,%2
+   slg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "*subdi3_64"
   [(set (match_operand:DI 0 "register_operand" "=d,d")
@@ -3383,17 +3767,41 @@
    (clobber (reg:CC 33))]
   "TARGET_64BIT"
   "@
-   sgr\\t%0,%2
-   sg\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RRE")
-   (set_attr "atype"    "reg,mem")])
+   sgr\t%0,%2
+   sg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RRE")])
+
+(define_insn_and_split "*subdi3_31z"
+  [(set (match_operand:DI 0 "register_operand" "=&d")
+        (minus:DI (match_operand:DI 1 "register_operand" "0")
+                  (match_operand:DI 2 "general_operand" "do") ) )
+   (clobber (reg:CC 33))]
+  "!TARGET_64BIT && TARGET_CPU_ZARCH"
+  "#"
+  "&& reload_completed"
+  [(parallel
+    [(set (reg:CCL2 33)
+          (compare:CCL2 (minus:SI (match_dup 7) (match_dup 8))
+                        (match_dup 7)))
+     (set (match_dup 6) (minus:SI (match_dup 7) (match_dup 8)))])
+   (parallel
+    [(set (match_dup 3) (minus:SI (minus:SI (match_dup 4) (match_dup 5))
+                                  (gtu:SI (reg:CCL2 33) (const_int 0))))
+     (clobber (reg:CC 33))])]
+  "operands[3] = operand_subword (operands[0], 0, 0, DImode);
+   operands[4] = operand_subword (operands[1], 0, 0, DImode);
+   operands[5] = operand_subword (operands[2], 0, 0, DImode);
+   operands[6] = operand_subword (operands[0], 1, 0, DImode);
+   operands[7] = operand_subword (operands[1], 1, 0, DImode);
+   operands[8] = operand_subword (operands[2], 1, 0, DImode);"
+  [(set_attr "op_type"  "NN")])
 
 (define_insn_and_split "*subdi3_31"
   [(set (match_operand:DI 0 "register_operand" "=&d")
         (minus:DI (match_operand:DI 1 "register_operand" "0")
-                  (match_operand:DI 2 "general_operand" "dm") ) )
+                  (match_operand:DI 2 "general_operand" "do") ) )
    (clobber (reg:CC 33))]
-  "!TARGET_64BIT"
+  "!TARGET_CPU_ZARCH"
   "#"
   "&& reload_completed"
   [(parallel
@@ -3412,15 +3820,14 @@
     [(set (match_dup 3) (plus:SI (match_dup 3) (const_int -1)))
      (clobber (reg:CC 33))])
    (match_dup 9)]
-  "operands[3] = operand_subword (operands[0], 0, 1, DImode);
-   operands[4] = operand_subword (operands[1], 0, 1, DImode);
-   operands[5] = operand_subword (operands[2], 0, 1, DImode);
-   operands[6] = operand_subword (operands[0], 1, 1, DImode);
-   operands[7] = operand_subword (operands[1], 1, 1, DImode);
-   operands[8] = operand_subword (operands[2], 1, 1, DImode);
+  "operands[3] = operand_subword (operands[0], 0, 0, DImode);
+   operands[4] = operand_subword (operands[1], 0, 0, DImode);
+   operands[5] = operand_subword (operands[2], 0, 0, DImode);
+   operands[6] = operand_subword (operands[0], 1, 0, DImode);
+   operands[7] = operand_subword (operands[1], 1, 0, DImode);
+   operands[8] = operand_subword (operands[2], 1, 0, DImode);
    operands[9] = gen_label_rtx ();"
-  [(set_attr "op_type"  "NN")
-   (set_attr "type"     "o3")])
+  [(set_attr "op_type"  "NN")])
 
 (define_expand "subdi3"
   [(parallel
@@ -3437,89 +3844,105 @@
 
 (define_insn "*subsi3_borrow_cc"
   [(set (reg 33)
-        (compare (minus:SI (match_operand:SI 1 "register_operand" "0,0")
-                           (match_operand:SI 2 "general_operand" "d,m"))
+        (compare (minus:SI (match_operand:SI 1 "register_operand" "0,0,0")
+                           (match_operand:SI 2 "general_operand" "d,R,T"))
                  (match_dup 1)))
-   (set (match_operand:SI 0 "register_operand" "=d,d")
+   (set (match_operand:SI 0 "register_operand" "=d,d,d")
         (minus:SI (match_dup 1) (match_dup 2)))]
-  "s390_match_ccmode(insn, CCL2mode)"
+  "s390_match_ccmode (insn, CCL2mode)"
   "@
-   slr\\t%0,%2
-   sl\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
+   slr\t%0,%2
+   sl\t%0,%2
+   sly\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
 (define_insn "*subsi3_borrow_cconly"
   [(set (reg 33)
-        (compare (minus:SI (match_operand:SI 1 "register_operand" "0,0")
-                           (match_operand:SI 2 "general_operand" "d,m"))
+        (compare (minus:SI (match_operand:SI 1 "register_operand" "0,0,0")
+                           (match_operand:SI 2 "general_operand" "d,R,T"))
                  (match_dup 1)))
-   (clobber (match_scratch:SI 0 "=d,d"))]
-  "s390_match_ccmode(insn, CCL2mode)"
+   (clobber (match_scratch:SI 0 "=d,d,d"))]
+  "s390_match_ccmode (insn, CCL2mode)"
   "@
-   slr\\t%0,%2
-   sl\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
+   slr\t%0,%2
+   sl\t%0,%2
+   sly\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
 (define_insn "*subsi3_cc"
   [(set (reg 33)
-        (compare (minus:SI (match_operand:SI 1 "register_operand" "0,0")
-                           (match_operand:SI 2 "general_operand" "d,m"))
+        (compare (minus:SI (match_operand:SI 1 "register_operand" "0,0,0")
+                           (match_operand:SI 2 "general_operand" "d,R,T"))
                  (const_int 0)))
-   (set (match_operand:SI 0 "register_operand" "=d,d")
+   (set (match_operand:SI 0 "register_operand" "=d,d,d")
         (minus:SI (match_dup 1) (match_dup 2)))]
-  "s390_match_ccmode(insn, CCLmode)"
+  "s390_match_ccmode (insn, CCLmode)"
   "@
-   slr\\t%0,%2
-   sl\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
+   slr\t%0,%2
+   sl\t%0,%2
+   sly\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
+
+(define_insn "*subsi3_cc2"
+  [(set (reg 33)
+        (compare (match_operand:SI 1 "register_operand" "0,0,0")
+                 (match_operand:SI 2 "general_operand" "d,R,T")))
+   (set (match_operand:SI 0 "register_operand" "=d,d,d")
+        (minus:SI (match_dup 1) (match_dup 2)))]
+  "s390_match_ccmode (insn, CCL3mode)"
+  "@
+   slr\t%0,%2
+   sl\t%0,%2
+   sly\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
 (define_insn "*subsi3_cconly"
   [(set (reg 33)
-        (compare (minus:SI (match_operand:SI 1 "register_operand" "0,0")
-                           (match_operand:SI 2 "general_operand" "d,m"))
+        (compare (minus:SI (match_operand:SI 1 "register_operand" "0,0,0")
+                           (match_operand:SI 2 "general_operand" "d,R,T"))
                  (const_int 0)))
-   (clobber (match_scratch:SI 0 "=d,d"))]
-  "s390_match_ccmode(insn, CCLmode)"
+   (clobber (match_scratch:SI 0 "=d,d,d"))]
+  "s390_match_ccmode (insn, CCLmode)"
   "@
-   slr\\t%0,%2
-   sl\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
+   slr\t%0,%2
+   sl\t%0,%2
+   sly\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
-(define_insn "*subsi3_sign"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (minus:SI (match_operand:SI 1 "register_operand" "0")
-                  (sign_extend:SI (match_operand:HI 2 "memory_operand" "m"))))
-   (clobber (reg:CC 33))]
-  ""
-  "sh\\t%0,%2"
-  [(set_attr "op_type"  "RX")
-   (set_attr "atype"    "mem")])
+(define_insn "*subsi3_cconly2"
+  [(set (reg 33)
+        (compare (match_operand:SI 1 "register_operand" "0,0,0")
+                 (match_operand:SI 2 "general_operand" "d,R,T")))
+   (clobber (match_scratch:SI 0 "=d,d,d"))]
+  "s390_match_ccmode (insn, CCL3mode)"
+  "@
+   slr\t%0,%2
+   sl\t%0,%2
+   sly\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
-(define_insn "*subsi3_sub"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (minus:SI (match_operand:SI 1 "register_operand" "0")
-                  (subreg:SI (match_operand:HI 2 "memory_operand" "m") 0)))
+(define_insn "*subsi3_sign"
+  [(set (match_operand:SI 0 "register_operand" "=d,d")
+        (minus:SI (match_operand:SI 1 "register_operand" "0,0")
+                  (sign_extend:SI (match_operand:HI 2 "memory_operand" "R,T"))))
    (clobber (reg:CC 33))]
   ""
-  "sh\\t%0,%2"
-  [(set_attr "op_type"  "RX")
-   (set_attr "atype"    "mem")])
+  "@
+   sh\t%0,%2
+   shy\t%0,%2"
+  [(set_attr "op_type"  "RX,RXY")])
 
 (define_insn "subsi3"
-  [(set (match_operand:SI 0 "register_operand" "=d,d")
-        (minus:SI (match_operand:SI 1 "register_operand" "0,0")
-                  (match_operand:SI 2 "general_operand" "d,m")))
+  [(set (match_operand:SI 0 "register_operand" "=d,d,d")
+        (minus:SI (match_operand:SI 1 "register_operand" "0,0,0")
+                  (match_operand:SI 2 "general_operand" "d,R,T")))
    (clobber (reg:CC 33))]
   ""
   "@
-   sr\\t%0,%2
-   s\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
+   sr\t%0,%2
+   s\t%0,%2
+   sy\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
 
 ;
@@ -3530,7 +3953,7 @@
   [(parallel
     [(set (match_operand:DF 0 "register_operand" "=f,f")
           (minus:DF (match_operand:DF 1 "register_operand" "0,0")
-                    (match_operand:DF 2 "general_operand" "f,m")))
+                    (match_operand:DF 2 "general_operand" "f,R")))
      (clobber (reg:CC 33))])]
   "TARGET_HARD_FLOAT"
   "")
@@ -3538,26 +3961,53 @@
 (define_insn "*subdf3"
   [(set (match_operand:DF 0 "register_operand" "=f,f")
         (minus:DF (match_operand:DF 1 "register_operand" "0,0")
-                  (match_operand:DF 2 "general_operand" "f,m")))
+                  (match_operand:DF 2 "general_operand" "f,R")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   sdbr\\t%0,%2
-   sdb\\t%0,%2"
+   sdbr\t%0,%2
+   sdb\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXE")
+   (set_attr "type"     "fsimpd,fsimpd")])
+
+(define_insn "*subdf3_cc"
+  [(set (reg 33)
+	(compare (minus:DF (match_operand:DF 1 "nonimmediate_operand" "0,0")
+			   (match_operand:DF 2 "general_operand" "f,R"))
+		 (match_operand:DF 3 "const0_operand" "")))
+   (set (match_operand:DF 0 "register_operand" "=f,f")
+	(plus:DF (match_dup 1) (match_dup 2)))]
+  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "@
+   sdbr\t%0,%2
+   sdb\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXE")
+   (set_attr "type"     "fsimpd,fsimpd")])
+
+(define_insn "*subdf3_cconly"
+  [(set (reg 33)
+	(compare (minus:DF (match_operand:DF 1 "nonimmediate_operand" "0,0")
+			   (match_operand:DF 2 "general_operand" "f,R"))
+		 (match_operand:DF 3 "const0_operand" "")))
+   (clobber (match_scratch:DF 0 "=f,f"))]
+  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "@
+   sdbr\t%0,%2
+   sdb\t%0,%2"
   [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
+   (set_attr "type"     "fsimpd,fsimpd")])
 
 (define_insn "*subdf3_ibm"
   [(set (match_operand:DF 0 "register_operand" "=f,f")
         (minus:DF (match_operand:DF 1 "register_operand" "0,0")
-                  (match_operand:DF 2 "general_operand" "f,m")))
+                  (match_operand:DF 2 "general_operand" "f,R")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
   "@
-   sdr\\t%0,%2
-   sd\\t%0,%2"
+   sdr\t%0,%2
+   sd\t%0,%2"
   [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
+   (set_attr "type"     "fsimpd,fsimpd")])
 
 ;
 ; subsf3 instruction pattern(s).
@@ -3567,7 +4017,7 @@
   [(parallel
     [(set (match_operand:SF 0 "register_operand" "=f,f")
           (minus:SF (match_operand:SF 1 "register_operand" "0,0")
-                    (match_operand:SF 2 "general_operand" "f,m")))
+                    (match_operand:SF 2 "general_operand" "f,R")))
      (clobber (reg:CC 33))])]
   "TARGET_HARD_FLOAT"
   "")
@@ -3575,196 +4025,496 @@
 (define_insn "*subsf3"
   [(set (match_operand:SF 0 "register_operand" "=f,f")
         (minus:SF (match_operand:SF 1 "register_operand" "0,0")
-                  (match_operand:SF 2 "general_operand" "f,m")))
+                  (match_operand:SF 2 "general_operand" "f,R")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   sebr\\t%0,%2
-   seb\\t%0,%2"
+   sebr\t%0,%2
+   seb\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXE")
+   (set_attr "type"     "fsimps,fsimps")])
+
+(define_insn "*subsf3_cc"
+  [(set (reg 33)
+	(compare (minus:SF (match_operand:SF 1 "nonimmediate_operand" "0,0")
+			   (match_operand:SF 2 "general_operand" "f,R"))
+		 (match_operand:SF 3 "const0_operand" "")))
+   (set (match_operand:SF 0 "register_operand" "=f,f")
+	(minus:SF (match_dup 1) (match_dup 2)))]
+  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "@
+   sebr\t%0,%2
+   seb\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXE")
+   (set_attr "type"     "fsimps,fsimps")])
+
+(define_insn "*subsf3_cconly"
+  [(set (reg 33)
+	(compare (minus:SF (match_operand:SF 1 "nonimmediate_operand" "0,0")
+			   (match_operand:SF 2 "general_operand" "f,R"))
+		 (match_operand:SF 3 "const0_operand" "")))
+   (clobber (match_scratch:SF 0 "=f,f"))]
+  "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "@
+   sebr\t%0,%2
+   seb\t%0,%2"
   [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
+   (set_attr "type"     "fsimps,fsimps")])
 
 (define_insn "*subsf3_ibm"
   [(set (match_operand:SF 0 "register_operand" "=f,f")
         (minus:SF (match_operand:SF 1 "register_operand" "0,0")
-                  (match_operand:SF 2 "general_operand" "f,m")))
+                  (match_operand:SF 2 "general_operand" "f,R")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
   "@
-   ser\\t%0,%2
-   se\\t%0,%2"
+   ser\t%0,%2
+   se\t%0,%2"
   [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
+   (set_attr "type"     "fsimps,fsimps")])
 
 
 ;;
-;;- Multiply instructions.
+;;- Conditional add/subtract instructions.
 ;;
 
 ;
-; muldi3 instruction pattern(s).
+; adddicc instruction pattern(s).
 ;
 
-(define_insn "*muldi3_sign"
+(define_insn "*adddi3_alc_cc"
+  [(set (reg 33)
+        (compare
+          (plus:DI (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
+                            (match_operand:DI 2 "general_operand" "d,m"))
+                   (match_operand:DI 3 "s390_alc_comparison" ""))
+          (const_int 0)))
+   (set (match_operand:DI 0 "register_operand" "=d,d")
+        (plus:DI (plus:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
+  "s390_match_ccmode (insn, CCLmode) && TARGET_64BIT"
+  "@
+   alcgr\\t%0,%2
+   alcg\\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
+
+(define_insn "*adddi3_alc"
   [(set (match_operand:DI 0 "register_operand" "=d,d")
-        (mult:DI (sign_extend:DI (match_operand:SI 2 "nonimmediate_operand" "d,m"))
-                 (match_operand:DI 1 "register_operand" "0,0")))]
+        (plus:DI (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
+                          (match_operand:DI 2 "general_operand" "d,m"))
+                 (match_operand:DI 3 "s390_alc_comparison" "")))
+   (clobber (reg:CC 33))]
   "TARGET_64BIT"
   "@
-   msgfr\\t%0,%2
-   msgf\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")
-   (set_attr "type"     "imul")])
+   alcgr\\t%0,%2
+   alcg\\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
+(define_insn "*subdi3_slb_cc"
+  [(set (reg 33)
+        (compare
+          (minus:DI (minus:DI (match_operand:DI 1 "nonimmediate_operand" "0,0")
+                              (match_operand:DI 2 "general_operand" "d,m"))
+                    (match_operand:DI 3 "s390_slb_comparison" ""))
+          (const_int 0)))
+   (set (match_operand:DI 0 "register_operand" "=d,d")
+        (minus:DI (minus:DI (match_dup 1) (match_dup 2)) (match_dup 3)))]
+  "s390_match_ccmode (insn, CCLmode) && TARGET_64BIT"
+  "@
+   slbgr\\t%0,%2
+   slbg\\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
-(define_insn "muldi3"
-  [(set (match_operand:DI 0 "register_operand" "=d,d,d")
-        (mult:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0,0")
-                 (match_operand:DI 2 "general_operand" "d,K,m")))]
+(define_insn "*subdi3_slb"
+  [(set (match_operand:DI 0 "register_operand" "=d,d")
+        (minus:DI (minus:DI (match_operand:DI 1 "nonimmediate_operand" "0,0")
+                            (match_operand:DI 2 "general_operand" "d,m"))
+                  (match_operand:DI 3 "s390_slb_comparison" "")))
+   (clobber (reg:CC 33))]
   "TARGET_64BIT"
   "@
-   msgr\\t%0,%2
-   mghi\\t%0,%h2
-   msg\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RI,RXE")
-   (set_attr "atype"    "reg,reg,mem")
-   (set_attr "type"     "imul")])
+   slbgr\\t%0,%2
+   slbg\\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
+
+(define_expand "adddicc"
+  [(match_operand:DI 0 "register_operand" "")
+   (match_operand 1 "comparison_operator" "")
+   (match_operand:DI 2 "register_operand" "")
+   (match_operand:DI 3 "const_int_operand" "")]
+  "TARGET_64BIT"
+  "if (!s390_expand_addcc (GET_CODE (operands[1]), 
+			   s390_compare_op0, s390_compare_op1, 
+			   operands[0], operands[2], 
+			   operands[3])) FAIL; DONE;")
 
 ;
-; mulsi3 instruction pattern(s).
+; addsicc instruction pattern(s).
 ;
 
-(define_insn "mulsi3"
-  [(set (match_operand:SI 0 "register_operand" "=d,d,d")
-        (mult:SI  (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
-                  (match_operand:SI 2 "general_operand" "d,K,m")))]
-  ""
+(define_insn "*addsi3_alc_cc"
+  [(set (reg 33)
+        (compare
+          (plus:SI (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
+                            (match_operand:SI 2 "general_operand" "d,m"))
+                   (match_operand:SI 3 "s390_alc_comparison" ""))
+          (const_int 0)))
+   (set (match_operand:SI 0 "register_operand" "=d,d")
+        (plus:SI (plus:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
+  "s390_match_ccmode (insn, CCLmode) && TARGET_CPU_ZARCH"
   "@
-   msr\\t%0,%2
-   mhi\\t%0,%h2
-   ms\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RI,RX")
-   (set_attr "atype"    "reg,reg,mem")
-   (set_attr "type"     "imul")])
-
-;
-; mulsidi3 instruction pattern(s).
-;
-
-(define_expand "mulsidi3"
-  [(set (match_operand:DI 0 "register_operand" "")
-	(mult:DI (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" ""))
-		 (sign_extend:DI (match_operand:SI 2 "nonimmediate_operand" ""))))]
-  "!TARGET_64BIT"
-  "
-{
-  rtx insn;
-
-  emit_insn (gen_zero_extendsidi2 (operands[0], operands[1]));
-  insn = emit_insn (gen_mulsi_6432 (operands[0], operands[0], operands[2]));
-
-  REG_NOTES (insn) =
-	gen_rtx_EXPR_LIST (REG_EQUAL, 
-                           gen_rtx_MULT (DImode, 
-				gen_rtx_SIGN_EXTEND (DImode, operands[1]),
-				gen_rtx_SIGN_EXTEND (DImode, operands[2])),
-			   REG_NOTES (insn));
-  DONE;
-}")
- 
-(define_insn "mulsi_6432"
-   [(set (match_operand:DI 0 "register_operand" "=d,d")
-         (mult:DI (sign_extend:DI 
-	            (truncate:SI (match_operand:DI 1 "register_operand" "0,0")))
-                  (sign_extend:DI
-	            (match_operand:SI 2 "nonimmediate_operand" "d,m"))))]
-   "!TARGET_64BIT"
-   "@
-    mr\\t%0,%2
-    m\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")
-   (set_attr "type"     "imul")])
- 
-;
-; muldf3 instruction pattern(s).
-;
-
-(define_expand "muldf3"
-  [(parallel
-    [(set (match_operand:DF 0 "register_operand" "=f,f")
-          (mult:DF (match_operand:DF 1 "nonimmediate_operand" "%0,0")
-                   (match_operand:DF 2 "general_operand" "f,m")))
-     (clobber (reg:CC 33))])]
-  "TARGET_HARD_FLOAT"
-  "")
+   alcr\\t%0,%2
+   alc\\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
-(define_insn "*muldf3"
-  [(set (match_operand:DF 0 "register_operand" "=f,f")
-        (mult:DF (match_operand:DF 1 "nonimmediate_operand" "%0,0")
-                 (match_operand:DF 2 "general_operand" "f,m")))
+(define_insn "*addsi3_alc"
+  [(set (match_operand:SI 0 "register_operand" "=d,d")
+        (plus:SI (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
+                          (match_operand:SI 2 "general_operand" "d,m"))
+                 (match_operand:SI 3 "s390_alc_comparison" "")))
    (clobber (reg:CC 33))]
-  "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "TARGET_CPU_ZARCH"
   "@
-   mdbr\\t%0,%2
-   mdb\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "type"     "fmul")
-   (set_attr "atype"    "reg,mem")])
+   alcr\\t%0,%2
+   alc\\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
-(define_insn "*muldf3_ibm"
-  [(set (match_operand:DF 0 "register_operand" "=f,f")
-        (mult:DF (match_operand:DF 1 "nonimmediate_operand" "%0,0")
-                 (match_operand:DF 2 "general_operand" "f,m")))
+(define_insn "*subsi3_slb_cc"
+  [(set (reg 33)
+        (compare
+          (minus:SI (minus:SI (match_operand:SI 1 "nonimmediate_operand" "0,0")
+                              (match_operand:SI 2 "general_operand" "d,m"))
+                    (match_operand:SI 3 "s390_slb_comparison" ""))
+          (const_int 0)))
+   (set (match_operand:SI 0 "register_operand" "=d,d")
+        (minus:SI (minus:SI (match_dup 1) (match_dup 2)) (match_dup 3)))]
+  "s390_match_ccmode (insn, CCLmode) && TARGET_CPU_ZARCH"
+  "@
+   slbr\\t%0,%2
+   slb\\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
+
+(define_insn "*subsi3_slb"
+  [(set (match_operand:SI 0 "register_operand" "=d,d")
+        (minus:SI (minus:SI (match_operand:SI 1 "nonimmediate_operand" "0,0")
+                            (match_operand:SI 2 "general_operand" "d,m"))
+                  (match_operand:SI 3 "s390_slb_comparison" "")))
    (clobber (reg:CC 33))]
-  "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
+  "TARGET_CPU_ZARCH"
   "@
-   mdr\\t%0,%2
-   md\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "type"    "fmul")
-   (set_attr "atype"    "reg,mem")])
+   slbr\\t%0,%2
+   slb\\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
+
+(define_expand "addsicc"
+  [(match_operand:SI 0 "register_operand" "")
+   (match_operand 1 "comparison_operator" "")
+   (match_operand:SI 2 "register_operand" "")
+   (match_operand:SI 3 "const_int_operand" "")]
+  "TARGET_CPU_ZARCH"
+  "if (!s390_expand_addcc (GET_CODE (operands[1]), 
+			   s390_compare_op0, s390_compare_op1, 
+			   operands[0], operands[2], 
+			   operands[3])) FAIL; DONE;")
 
 ;
-; mulsf3 instruction pattern(s).
+; scond instruction pattern(s).
 ;
 
-(define_expand "mulsf3"
-  [(parallel
-    [(set (match_operand:SF 0 "register_operand" "=f,f")
-          (mult:SF (match_operand:SF 1 "nonimmediate_operand" "%0,0")
-                   (match_operand:SF 2 "general_operand" "f,m")))
+(define_insn_and_split "*sconddi"
+  [(set (match_operand:DI 0 "register_operand" "=&d")
+        (match_operand:DI 1 "s390_alc_comparison" ""))
+   (clobber (reg:CC 33))]
+  "TARGET_64BIT"
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 0) (const_int 0))
+   (parallel
+    [(set (match_dup 0) (plus:DI (plus:DI (match_dup 0) (match_dup 0))
+                                 (match_dup 1)))
      (clobber (reg:CC 33))])]
-  "TARGET_HARD_FLOAT"
-  "")
+  ""
+  [(set_attr "op_type"  "NN")])
 
-(define_insn "*mulsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f,f")
-        (mult:SF (match_operand:SF 1 "nonimmediate_operand" "%0,0")
-                 (match_operand:SF 2 "general_operand" "f,m")))
+(define_insn_and_split "*scondsi"
+  [(set (match_operand:SI 0 "register_operand" "=&d")
+        (match_operand:SI 1 "s390_alc_comparison" ""))
+   (clobber (reg:CC 33))]
+  "TARGET_CPU_ZARCH"
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 0) (const_int 0))
+   (parallel
+    [(set (match_dup 0) (plus:SI (plus:SI (match_dup 0) (match_dup 0))
+                                 (match_dup 1)))
+     (clobber (reg:CC 33))])]
+  ""
+  [(set_attr "op_type"  "NN")])
+
+(define_insn_and_split "*sconddi_neg"
+  [(set (match_operand:DI 0 "register_operand" "=&d")
+        (match_operand:DI 1 "s390_slb_comparison" ""))
+   (clobber (reg:CC 33))]
+  "TARGET_64BIT"
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 0) (const_int 0))
+   (parallel
+    [(set (match_dup 0) (minus:DI (minus:DI (match_dup 0) (match_dup 0))
+                                  (match_dup 1)))
+     (clobber (reg:CC 33))])
+   (parallel
+    [(set (match_dup 0) (neg:DI (match_dup 0)))
+     (clobber (reg:CC 33))])]
+  ""
+  [(set_attr "op_type"  "NN")])
+
+(define_insn_and_split "*scondsi_neg"
+  [(set (match_operand:SI 0 "register_operand" "=&d")
+        (match_operand:SI 1 "s390_slb_comparison" ""))
    (clobber (reg:CC 33))]
+  "TARGET_CPU_ZARCH"
+  "#"
+  "&& reload_completed"
+  [(set (match_dup 0) (const_int 0))
+   (parallel
+    [(set (match_dup 0) (minus:SI (minus:SI (match_dup 0) (match_dup 0))
+                                  (match_dup 1)))
+     (clobber (reg:CC 33))])
+   (parallel
+    [(set (match_dup 0) (neg:SI (match_dup 0)))
+     (clobber (reg:CC 33))])]
+  ""
+  [(set_attr "op_type"  "NN")])
+
+(define_expand "sltu"
+  [(match_operand:SI 0 "register_operand" "")]
+  "TARGET_CPU_ZARCH"
+  "if (!s390_expand_addcc (LTU, s390_compare_op0, s390_compare_op1,
+			   operands[0], const0_rtx, const1_rtx)) FAIL; DONE;")
+
+(define_expand "sgtu"
+  [(match_operand:SI 0 "register_operand" "")]
+  "TARGET_CPU_ZARCH"
+  "if (!s390_expand_addcc (GTU, s390_compare_op0, s390_compare_op1,
+			   operands[0], const0_rtx, const1_rtx)) FAIL; DONE;")
+
+(define_expand "sleu"
+  [(match_operand:SI 0 "register_operand" "")]
+  "TARGET_CPU_ZARCH"
+  "if (!s390_expand_addcc (LEU, s390_compare_op0, s390_compare_op1,
+			   operands[0], const0_rtx, const1_rtx)) FAIL; DONE;")
+
+(define_expand "sgeu"
+  [(match_operand:SI 0 "register_operand" "")]
+  "TARGET_CPU_ZARCH"
+  "if (!s390_expand_addcc (GEU, s390_compare_op0, s390_compare_op1,
+			   operands[0], const0_rtx, const1_rtx)) FAIL; DONE;")
+
+
+;;
+;;- Multiply instructions.
+;;
+
+;
+; muldi3 instruction pattern(s).
+;
+
+(define_insn "*muldi3_sign"
+  [(set (match_operand:DI 0 "register_operand" "=d,d")
+        (mult:DI (sign_extend:DI (match_operand:SI 2 "nonimmediate_operand" "d,m"))
+                 (match_operand:DI 1 "register_operand" "0,0")))]
+  "TARGET_64BIT"
+  "@
+   msgfr\t%0,%2
+   msgf\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")
+   (set_attr "type"     "imul")])
+
+(define_insn "muldi3"
+  [(set (match_operand:DI 0 "register_operand" "=d,d,d")
+        (mult:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0,0")
+                 (match_operand:DI 2 "general_operand" "d,K,m")))]
+  "TARGET_64BIT"
+  "@
+   msgr\t%0,%2
+   mghi\t%0,%h2
+   msg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RI,RXY")
+   (set_attr "type"     "imul")])
+
+;
+; mulsi3 instruction pattern(s).
+;
+
+(define_insn "*mulsi3_sign"
+  [(set (match_operand:SI 0 "register_operand" "=d")
+        (mult:SI (sign_extend:SI (match_operand:HI 2 "memory_operand" "R"))
+                 (match_operand:SI 1 "register_operand" "0")))]
+  ""
+  "mh\t%0,%2"
+  [(set_attr "op_type"  "RX")
+   (set_attr "type"     "imul")])
+
+(define_insn "mulsi3"
+  [(set (match_operand:SI 0 "register_operand" "=d,d,d,d")
+        (mult:SI  (match_operand:SI 1 "nonimmediate_operand" "%0,0,0,0")
+                  (match_operand:SI 2 "general_operand" "d,K,R,T")))]
+  ""
+  "@
+   msr\t%0,%2
+   mhi\t%0,%h2
+   ms\t%0,%2
+   msy\t%0,%2"
+  [(set_attr "op_type"  "RRE,RI,RX,RXY")
+   (set_attr "type"     "imul")])
+
+;
+; mulsidi3 instruction pattern(s).
+;
+
+(define_insn "mulsidi3"
+  [(set (match_operand:DI 0 "register_operand" "=d,d")
+        (mult:DI (sign_extend:DI
+	           (match_operand:SI 1 "register_operand" "%0,0"))
+                 (sign_extend:DI
+	           (match_operand:SI 2 "nonimmediate_operand" "d,R"))))]
+  "!TARGET_64BIT"
+  "@
+   mr\t%0,%2
+   m\t%0,%2"
+  [(set_attr "op_type"  "RR,RX")
+   (set_attr "type"     "imul")])
+
+;
+; umulsidi3 instruction pattern(s).
+;
+
+(define_insn "umulsidi3"
+  [(set (match_operand:DI 0 "register_operand" "=d,d")
+        (mult:DI (zero_extend:DI
+	           (match_operand:SI 1 "register_operand" "%0,0"))
+                 (zero_extend:DI
+	           (match_operand:SI 2 "nonimmediate_operand" "d,m"))))]
+  "!TARGET_64BIT && TARGET_CPU_ZARCH"
+  "@
+   mlr\t%0,%2
+   ml\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")
+   (set_attr "type"     "imul")])
+
+;
+; muldf3 instruction pattern(s).
+;
+
+(define_expand "muldf3"
+  [(set (match_operand:DF 0 "register_operand" "=f,f")
+        (mult:DF (match_operand:DF 1 "nonimmediate_operand" "%0,0")
+                 (match_operand:DF 2 "general_operand" "f,R")))]
+  "TARGET_HARD_FLOAT"
+  "")
+
+(define_insn "*muldf3"
+  [(set (match_operand:DF 0 "register_operand" "=f,f")
+        (mult:DF (match_operand:DF 1 "nonimmediate_operand" "%0,0")
+                 (match_operand:DF 2 "general_operand" "f,R")))]
+  "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "@
+   mdbr\t%0,%2
+   mdb\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXE")
+   (set_attr "type"     "fmuld")])
+
+(define_insn "*muldf3_ibm"
+  [(set (match_operand:DF 0 "register_operand" "=f,f")
+        (mult:DF (match_operand:DF 1 "nonimmediate_operand" "%0,0")
+                 (match_operand:DF 2 "general_operand" "f,R")))]
+  "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
+  "@
+   mdr\t%0,%2
+   md\t%0,%2"
+  [(set_attr "op_type"  "RR,RX")
+   (set_attr "type"    "fmuld")])
+
+(define_insn "*fmadddf"
+  [(set (match_operand:DF 0 "register_operand" "=f,f")
+	(plus:DF (mult:DF (match_operand:DF 1 "register_operand" "%f,f")
+			  (match_operand:DF 2 "nonimmediate_operand"  "f,R"))
+		 (match_operand:DF 3 "register_operand" "0,0")))]
+  "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT && TARGET_FUSED_MADD"
+  "@
+   madbr\t%0,%1,%2
+   madb\t%0,%1,%2"
+  [(set_attr "op_type"  "RRE,RXE")
+   (set_attr "type" "fmuld")])
+
+(define_insn "*fmsubdf"
+  [(set (match_operand:DF 0 "register_operand" "=f,f")
+	(minus:DF (mult:DF (match_operand:DF 1 "register_operand" "f,f")
+			   (match_operand:DF 2 "nonimmediate_operand"  "f,R"))
+		 (match_operand:DF 3 "register_operand" "0,0")))]
+  "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT && TARGET_FUSED_MADD"
+  "@
+   msdbr\t%0,%1,%2
+   msdb\t%0,%1,%2"
+  [(set_attr "op_type"  "RRE,RXE")
+   (set_attr "type" "fmuld")])
+
+;
+; mulsf3 instruction pattern(s).
+;
+
+(define_expand "mulsf3"
+  [(set (match_operand:SF 0 "register_operand" "=f,f")
+        (mult:SF (match_operand:SF 1 "nonimmediate_operand" "%0,0")
+                 (match_operand:SF 2 "general_operand" "f,R")))]
+  "TARGET_HARD_FLOAT"
+  "")
+
+(define_insn "*mulsf3"
+  [(set (match_operand:SF 0 "register_operand" "=f,f")
+        (mult:SF (match_operand:SF 1 "nonimmediate_operand" "%0,0")
+                 (match_operand:SF 2 "general_operand" "f,R")))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   meebr\\t%0,%2
-   meeb\\t%0,%2"
+   meebr\t%0,%2
+   meeb\t%0,%2"
   [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "type"     "fmul")
-   (set_attr "atype"    "reg,mem")])
+   (set_attr "type"     "fmuls")])
 
 (define_insn "*mulsf3_ibm"
   [(set (match_operand:SF 0 "register_operand" "=f,f")
         (mult:SF (match_operand:SF 1 "nonimmediate_operand" "%0,0")
-                 (match_operand:SF 2 "general_operand" "f,m")))
-   (clobber (reg:CC 33))]
+                 (match_operand:SF 2 "general_operand" "f,R")))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
   "@
-   mer\\t%0,%2
-   me\\t%0,%2"
+   mer\t%0,%2
+   me\t%0,%2"
   [(set_attr "op_type"  "RR,RX")
-   (set_attr "type"     "fmul")
-   (set_attr "atype"    "reg,mem")])
+   (set_attr "type"     "fmuls")])
+
+(define_insn "*fmaddsf"
+  [(set (match_operand:SF 0 "register_operand" "=f,f")
+	(plus:SF (mult:SF (match_operand:SF 1 "register_operand" "%f,f")
+			  (match_operand:SF 2 "nonimmediate_operand"  "f,R"))
+		 (match_operand:SF 3 "register_operand" "0,0")))]
+  "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT && TARGET_FUSED_MADD"
+  "@
+   maebr\t%0,%1,%2
+   maeb\t%0,%1,%2"
+  [(set_attr "op_type"  "RRE,RXE")
+   (set_attr "type" "fmuls")])
 
+(define_insn "*fmsubsf"
+  [(set (match_operand:SF 0 "register_operand" "=f,f")
+	(minus:SF (mult:SF (match_operand:SF 1 "register_operand" "f,f")
+			   (match_operand:SF 2 "nonimmediate_operand"  "f,R"))
+		  (match_operand:SF 3 "register_operand" "0,0")))]
+  "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT && TARGET_FUSED_MADD"
+  "@
+   msebr\t%0,%1,%2
+   mseb\t%0,%1,%2"
+  [(set_attr "op_type"  "RRE,RXE")
+   (set_attr "type" "fmuls")])
 
 ;;
 ;;- Divide and modulo instructions.
@@ -3776,31 +4526,20 @@
 
 (define_expand "divmoddi4"
   [(parallel [(set (match_operand:DI 0 "general_operand" "")
-		   (div:DI (match_operand:DI 1 "general_operand" "")
+		   (div:DI (match_operand:DI 1 "register_operand" "")
 			   (match_operand:DI 2 "general_operand" "")))
 	      (set (match_operand:DI 3 "general_operand" "")
 		   (mod:DI (match_dup 1) (match_dup 2)))])
    (clobber (match_dup 4))]
   "TARGET_64BIT"
-  "
 {
-  rtx insn, div_equal, mod_equal, equal;
+  rtx insn, div_equal, mod_equal;
 
   div_equal = gen_rtx_DIV (DImode, operands[1], operands[2]);
   mod_equal = gen_rtx_MOD (DImode, operands[1], operands[2]);
-  equal = gen_rtx_IOR (TImode,
-		       gen_rtx_ZERO_EXTEND (TImode, div_equal),
-		       gen_rtx_ASHIFT (TImode,
-				       gen_rtx_ZERO_EXTEND (TImode, mod_equal),
-				       GEN_INT (64)));
 
   operands[4] = gen_reg_rtx(TImode);
-  emit_insn (gen_rtx_CLOBBER (VOIDmode, operands[4]));
-  emit_move_insn (gen_lowpart (DImode, operands[4]), operands[1]);
-  emit_move_insn (gen_highpart (DImode, operands[4]), const0_rtx);
-  insn = emit_insn (gen_divmodtidi3 (operands[4], operands[4], operands[2]));
-  REG_NOTES (insn) =
-	gen_rtx_EXPR_LIST (REG_EQUAL, equal, REG_NOTES (insn));
+  emit_insn (gen_divmodtidi3 (operands[4], operands[1], operands[2]));
 
   insn = emit_move_insn (operands[0], gen_lowpart (DImode, operands[4]));
   REG_NOTES (insn) =
@@ -3811,45 +4550,41 @@
         gen_rtx_EXPR_LIST (REG_EQUAL, mod_equal, REG_NOTES (insn));
 
   DONE;
-}")
+})
 
 (define_insn "divmodtidi3"
   [(set (match_operand:TI 0 "register_operand" "=d,d")
         (ior:TI
-          (zero_extend:TI
-            (div:DI (truncate:DI (match_operand:TI 1 "register_operand" "0,0"))
-                    (match_operand:DI 2 "general_operand" "d,m")))
           (ashift:TI
             (zero_extend:TI
-              (mod:DI (truncate:DI (match_dup 1)) 
-                      (match_dup 2)))
-            (const_int 64))))]
+              (mod:DI (match_operand:DI 1 "register_operand" "0,0")
+                      (match_operand:DI 2 "general_operand" "d,m")))
+            (const_int 64))
+          (zero_extend:TI (div:DI (match_dup 1) (match_dup 2)))))]
   "TARGET_64BIT"
   "@
-   dsgr\\t%0,%2
-   dsg\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "type"     "idiv")
-   (set_attr "atype"    "reg,mem")])
+   dsgr\t%0,%2
+   dsg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")
+   (set_attr "type"     "idiv")])
 
 (define_insn "divmodtisi3"
   [(set (match_operand:TI 0 "register_operand" "=d,d")
         (ior:TI
-          (zero_extend:TI
-            (div:DI (truncate:DI (match_operand:TI 1 "register_operand" "0,0"))
-                    (sign_extend:DI (match_operand:SI 2 "nonimmediate_operand" "d,m"))))
           (ashift:TI
             (zero_extend:TI
-              (mod:DI (truncate:DI (match_dup 1)) 
-                      (sign_extend:DI (match_dup 2))))
-            (const_int 64))))]
+              (mod:DI (match_operand:DI 1 "register_operand" "0,0")
+                      (sign_extend:DI
+                        (match_operand:SI 2 "nonimmediate_operand" "d,m"))))
+            (const_int 64))
+          (zero_extend:TI
+            (div:DI (match_dup 1) (sign_extend:DI (match_dup 2))))))]
   "TARGET_64BIT"
   "@
-   dsgfr\\t%0,%2
-   dsgf\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "type"     "idiv")
-   (set_attr "atype"    "reg,mem")])
+   dsgfr\t%0,%2
+   dsgf\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")
+   (set_attr "type"     "idiv")])
 
 ;
 ; udivmoddi4 instruction pattern(s).
@@ -3863,17 +4598,16 @@
 		   (umod:DI (match_dup 1) (match_dup 2)))])
    (clobber (match_dup 4))]
   "TARGET_64BIT"
-  "
 {
   rtx insn, div_equal, mod_equal, equal;
 
   div_equal = gen_rtx_UDIV (DImode, operands[1], operands[2]);
   mod_equal = gen_rtx_UMOD (DImode, operands[1], operands[2]);
   equal = gen_rtx_IOR (TImode,
-		       gen_rtx_ZERO_EXTEND (TImode, div_equal),
 		       gen_rtx_ASHIFT (TImode,
 				       gen_rtx_ZERO_EXTEND (TImode, mod_equal),
-				       GEN_INT (64)));
+				       GEN_INT (64)),
+		       gen_rtx_ZERO_EXTEND (TImode, div_equal));
 
   operands[4] = gen_reg_rtx(TImode);
   emit_insn (gen_rtx_CLOBBER (VOIDmode, operands[4]));
@@ -3892,27 +4626,27 @@
         gen_rtx_EXPR_LIST (REG_EQUAL, mod_equal, REG_NOTES (insn));
 
   DONE;
-}")
+})
 
 (define_insn "udivmodtidi3"
   [(set (match_operand:TI 0 "register_operand" "=d,d")
-        (ior:TI (zero_extend:TI 
-                  (truncate:DI
-                    (udiv:TI (match_operand:TI 1 "register_operand" "0,0")
-                             (zero_extend:TI
-                               (match_operand:DI 2 "nonimmediate_operand" "d,m")))))
-                (ashift:TI
-                  (zero_extend:TI
-                    (truncate:DI
-                      (umod:TI (match_dup 1) (zero_extend:TI (match_dup 2)))))
-                  (const_int 64))))]
+        (ior:TI
+          (ashift:TI
+            (zero_extend:TI
+              (truncate:DI
+                (umod:TI (match_operand:TI 1 "register_operand" "0,0")
+                         (zero_extend:TI
+                           (match_operand:DI 2 "nonimmediate_operand" "d,m")))))
+            (const_int 64))
+          (zero_extend:TI
+            (truncate:DI
+              (udiv:TI (match_dup 1) (zero_extend:TI (match_dup 2)))))))]
   "TARGET_64BIT"
   "@
-   dlgr\\t%0,%2
-   dlg\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "type"     "idiv")
-   (set_attr "atype"    "reg,mem")])
+   dlgr\t%0,%2
+   dlg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")
+   (set_attr "type"     "idiv")])
 
 ;
 ; divmodsi4 instruction pattern(s).
@@ -3926,17 +4660,16 @@
 		   (mod:SI (match_dup 1) (match_dup 2)))])
    (clobber (match_dup 4))]
   "!TARGET_64BIT"
-  "
 {
   rtx insn, div_equal, mod_equal, equal;
 
   div_equal = gen_rtx_DIV (SImode, operands[1], operands[2]);
   mod_equal = gen_rtx_MOD (SImode, operands[1], operands[2]);
   equal = gen_rtx_IOR (DImode,
-		       gen_rtx_ZERO_EXTEND (DImode, div_equal),
 		       gen_rtx_ASHIFT (DImode,
 				       gen_rtx_ZERO_EXTEND (DImode, mod_equal),
-				       GEN_INT (32)));
+				       GEN_INT (32)),
+		       gen_rtx_ZERO_EXTEND (DImode, div_equal));
 
   operands[4] = gen_reg_rtx(DImode);
   emit_insn (gen_extendsidi2 (operands[4], operands[1]));
@@ -3953,50 +4686,106 @@
         gen_rtx_EXPR_LIST (REG_EQUAL, mod_equal, REG_NOTES (insn));
 
   DONE;
-}")
+})
 
 (define_insn "divmoddisi3"
   [(set (match_operand:DI 0 "register_operand" "=d,d")
-        (ior:DI (zero_extend:DI
-                  (truncate:SI
-                    (div:DI (match_operand:DI 1 "register_operand" "0,0")
-                            (sign_extend:DI 
-                              (match_operand:SI 2 "nonimmediate_operand" "d,m")))))
-                (ashift:DI
-                  (zero_extend:DI
-                    (truncate:SI
-                      (mod:DI (match_dup 1) (sign_extend:SI (match_dup 2)))))
-                  (const_int 32))))]
+        (ior:DI
+          (ashift:DI
+            (zero_extend:DI
+              (truncate:SI
+                (mod:DI (match_operand:DI 1 "register_operand" "0,0")
+                        (sign_extend:DI
+                          (match_operand:SI 2 "nonimmediate_operand" "d,R")))))
+            (const_int 32))
+          (zero_extend:DI
+            (truncate:SI
+              (div:DI (match_dup 1) (sign_extend:DI (match_dup 2)))))))]
   "!TARGET_64BIT"
   "@
-   dr\\t%0,%2
-   d\\t%0,%2"
+   dr\t%0,%2
+   d\t%0,%2"
   [(set_attr "op_type"  "RR,RX")
-   (set_attr "type"     "idiv")
-   (set_attr "atype"    "reg,mem")])
+   (set_attr "type"     "idiv")])
 
 ;
 ; udivsi3 and umodsi3 instruction pattern(s).
 ;
 
+(define_expand "udivmodsi4"
+  [(parallel [(set (match_operand:SI 0 "general_operand" "")
+		   (udiv:SI (match_operand:SI 1 "general_operand" "")
+			    (match_operand:SI 2 "nonimmediate_operand" "")))
+	      (set (match_operand:SI 3 "general_operand" "")
+		   (umod:SI (match_dup 1) (match_dup 2)))])
+   (clobber (match_dup 4))]
+  "!TARGET_64BIT && TARGET_CPU_ZARCH"
+{
+  rtx insn, div_equal, mod_equal, equal;
+
+  div_equal = gen_rtx_UDIV (SImode, operands[1], operands[2]);
+  mod_equal = gen_rtx_UMOD (SImode, operands[1], operands[2]);
+  equal = gen_rtx_IOR (DImode,
+		       gen_rtx_ASHIFT (DImode,
+				       gen_rtx_ZERO_EXTEND (DImode, mod_equal),
+				       GEN_INT (32)),
+		       gen_rtx_ZERO_EXTEND (DImode, div_equal));
+
+  operands[4] = gen_reg_rtx(DImode);
+  emit_insn (gen_rtx_CLOBBER (VOIDmode, operands[4]));
+  emit_move_insn (gen_lowpart (SImode, operands[4]), operands[1]);
+  emit_move_insn (gen_highpart (SImode, operands[4]), const0_rtx);
+  insn = emit_insn (gen_udivmoddisi3 (operands[4], operands[4], operands[2]));
+  REG_NOTES (insn) =
+	gen_rtx_EXPR_LIST (REG_EQUAL, equal, REG_NOTES (insn));
+
+  insn = emit_move_insn (operands[0], gen_lowpart (SImode, operands[4]));
+  REG_NOTES (insn) =
+        gen_rtx_EXPR_LIST (REG_EQUAL, div_equal, REG_NOTES (insn));
+
+  insn = emit_move_insn (operands[3], gen_highpart (SImode, operands[4]));
+  REG_NOTES (insn) =
+        gen_rtx_EXPR_LIST (REG_EQUAL, mod_equal, REG_NOTES (insn));
+
+  DONE;
+})
+
+(define_insn "udivmoddisi3"
+  [(set (match_operand:DI 0 "register_operand" "=d,d")
+        (ior:DI
+          (ashift:DI
+            (zero_extend:DI
+              (truncate:SI
+                (umod:DI (match_operand:DI 1 "register_operand" "0,0")
+                         (zero_extend:DI
+                           (match_operand:SI 2 "nonimmediate_operand" "d,m")))))
+            (const_int 32))
+          (zero_extend:DI
+            (truncate:SI
+              (udiv:DI (match_dup 1) (zero_extend:DI (match_dup 2)))))))]
+  "!TARGET_64BIT && TARGET_CPU_ZARCH"
+  "@
+   dlr\t%0,%2
+   dl\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")
+   (set_attr "type"     "idiv")])
 
 (define_expand "udivsi3"
   [(set (match_operand:SI 0 "register_operand" "=d")
         (udiv:SI (match_operand:SI 1 "general_operand" "")
                  (match_operand:SI 2 "general_operand" "")))
    (clobber (match_dup 3))]
-  "!TARGET_64BIT"
-  "
+  "!TARGET_64BIT && !TARGET_CPU_ZARCH"
 {
   rtx insn, udiv_equal, umod_equal, equal;
 
   udiv_equal = gen_rtx_UDIV (SImode, operands[1], operands[2]);
   umod_equal = gen_rtx_UMOD (SImode, operands[1], operands[2]);
   equal = gen_rtx_IOR (DImode,
-		       gen_rtx_ZERO_EXTEND (DImode, udiv_equal),
 		       gen_rtx_ASHIFT (DImode,
 				       gen_rtx_ZERO_EXTEND (DImode, umod_equal),
-				       GEN_INT (32)));
+				       GEN_INT (32)),
+		       gen_rtx_ZERO_EXTEND (DImode, udiv_equal));
 
   operands[3] = gen_reg_rtx (DImode);
 
@@ -4015,37 +4804,37 @@
         }
       else
         {
-          operands[2] = force_reg (SImode, operands[2]);	
-          operands[2] = make_safe_from (operands[2], operands[0]);	
+          operands[2] = force_reg (SImode, operands[2]);
+          operands[2] = make_safe_from (operands[2], operands[0]);
 
 	  emit_insn (gen_zero_extendsidi2 (operands[3], operands[1]));
 	  insn = emit_insn (gen_divmoddisi3 (operands[3], operands[3],
 					     operands[2]));
 	  REG_NOTES (insn) =
 	    gen_rtx_EXPR_LIST (REG_EQUAL, equal, REG_NOTES (insn));
-	  
-	  insn = emit_move_insn (operands[0], 
+
+	  insn = emit_move_insn (operands[0],
 				 gen_lowpart (SImode, operands[3]));
 	  REG_NOTES (insn) =
-	    gen_rtx_EXPR_LIST (REG_EQUAL, 
+	    gen_rtx_EXPR_LIST (REG_EQUAL,
 			       udiv_equal, REG_NOTES (insn));
         }
     }
   else
-    {  
+    {
       rtx label1 = gen_label_rtx ();
       rtx label2 = gen_label_rtx ();
       rtx label3 = gen_label_rtx ();
 
-      operands[1] = force_reg (SImode, operands[1]);	
-      operands[1] = make_safe_from (operands[1], operands[0]);	
-      operands[2] = force_reg (SImode, operands[2]);	
-      operands[2] = make_safe_from (operands[2], operands[0]);	
+      operands[1] = force_reg (SImode, operands[1]);
+      operands[1] = make_safe_from (operands[1], operands[0]);
+      operands[2] = force_reg (SImode, operands[2]);
+      operands[2] = make_safe_from (operands[2], operands[0]);
 
       emit_move_insn (operands[0], const0_rtx);
       emit_insn (gen_cmpsi (operands[2], operands[1]));
       emit_jump_insn (gen_bgtu (label3));
-      emit_insn (gen_cmpsi (operands[2], const1_rtx));
+      emit_insn (gen_cmpsi (operands[2], const0_rtx));
       emit_jump_insn (gen_blt (label2));
       emit_insn (gen_cmpsi (operands[2], const1_rtx));
       emit_jump_insn (gen_beq (label1));
@@ -4054,11 +4843,11 @@
 					 operands[2]));
       REG_NOTES (insn) =
       gen_rtx_EXPR_LIST (REG_EQUAL, equal, REG_NOTES (insn));
-      
-      insn = emit_move_insn (operands[0], 
+
+      insn = emit_move_insn (operands[0],
 			     gen_lowpart (SImode, operands[3]));
       REG_NOTES (insn) =
-      gen_rtx_EXPR_LIST (REG_EQUAL, 
+      gen_rtx_EXPR_LIST (REG_EQUAL,
 			       udiv_equal, REG_NOTES (insn));
       emit_jump (label3);
       emit_label (label1);
@@ -4068,27 +4857,26 @@
       emit_move_insn (operands[0], const1_rtx);
       emit_label (label3);
     }
-  emit_move_insn (operands[0], operands[0]);	
+  emit_move_insn (operands[0], operands[0]);
   DONE;
-}")
+})
 
 (define_expand "umodsi3"
   [(set (match_operand:SI 0 "register_operand" "=d")
         (umod:SI (match_operand:SI 1 "nonimmediate_operand" "")
                  (match_operand:SI 2 "nonimmediate_operand" "")))
    (clobber (match_dup 3))]
-  "!TARGET_64BIT"
-  "
+  "!TARGET_64BIT && !TARGET_CPU_ZARCH"
 {
   rtx insn, udiv_equal, umod_equal, equal;
 
   udiv_equal = gen_rtx_UDIV (SImode, operands[1], operands[2]);
   umod_equal = gen_rtx_UMOD (SImode, operands[1], operands[2]);
   equal = gen_rtx_IOR (DImode,
-		       gen_rtx_ZERO_EXTEND (DImode, udiv_equal),
 		       gen_rtx_ASHIFT (DImode,
 				       gen_rtx_ZERO_EXTEND (DImode, umod_equal),
-				       GEN_INT (32)));
+				       GEN_INT (32)),
+		       gen_rtx_ZERO_EXTEND (DImode, udiv_equal));
 
   operands[3] = gen_reg_rtx (DImode);
 
@@ -4108,19 +4896,19 @@
         }
       else
         {
-          operands[2] = force_reg (SImode, operands[2]);	
-          operands[2] = make_safe_from (operands[2], operands[0]);	
+          operands[2] = force_reg (SImode, operands[2]);
+          operands[2] = make_safe_from (operands[2], operands[0]);
 
 	  emit_insn (gen_zero_extendsidi2 (operands[3], operands[1]));
 	  insn = emit_insn (gen_divmoddisi3 (operands[3], operands[3],
 					     operands[2]));
 	  REG_NOTES (insn) =
 	    gen_rtx_EXPR_LIST (REG_EQUAL, equal, REG_NOTES (insn));
-	  
-	  insn = emit_move_insn (operands[0], 
+
+	  insn = emit_move_insn (operands[0],
 				 gen_highpart (SImode, operands[3]));
 	  REG_NOTES (insn) =
-	    gen_rtx_EXPR_LIST (REG_EQUAL, 
+	    gen_rtx_EXPR_LIST (REG_EQUAL,
 			       umod_equal, REG_NOTES (insn));
         }
     }
@@ -4130,15 +4918,15 @@
       rtx label2 = gen_label_rtx ();
       rtx label3 = gen_label_rtx ();
 
-      operands[1] = force_reg (SImode, operands[1]);	
-      operands[1] = make_safe_from (operands[1], operands[0]);	
-      operands[2] = force_reg (SImode, operands[2]);	
-      operands[2] = make_safe_from (operands[2], operands[0]);	
+      operands[1] = force_reg (SImode, operands[1]);
+      operands[1] = make_safe_from (operands[1], operands[0]);
+      operands[2] = force_reg (SImode, operands[2]);
+      operands[2] = make_safe_from (operands[2], operands[0]);
 
-      emit_move_insn(operands[0], operands[1]);	
+      emit_move_insn(operands[0], operands[1]);
       emit_insn (gen_cmpsi (operands[2], operands[1]));
       emit_jump_insn (gen_bgtu (label3));
-      emit_insn (gen_cmpsi (operands[2], const1_rtx));
+      emit_insn (gen_cmpsi (operands[2], const0_rtx));
       emit_jump_insn (gen_blt (label2));
       emit_insn (gen_cmpsi (operands[2], const1_rtx));
       emit_jump_insn (gen_beq (label1));
@@ -4147,11 +4935,11 @@
 					 operands[2]));
       REG_NOTES (insn) =
       gen_rtx_EXPR_LIST (REG_EQUAL, equal, REG_NOTES (insn));
-      
-      insn = emit_move_insn (operands[0], 
+
+      insn = emit_move_insn (operands[0],
 			     gen_highpart (SImode, operands[3]));
       REG_NOTES (insn) =
-      gen_rtx_EXPR_LIST (REG_EQUAL, 
+      gen_rtx_EXPR_LIST (REG_EQUAL,
 			 umod_equal, REG_NOTES (insn));
       emit_jump (label3);
       emit_label (label1);
@@ -4162,85 +4950,73 @@
       emit_label (label3);
     }
   DONE;
-}")
+})
 
 ;
 ; divdf3 instruction pattern(s).
 ;
 
 (define_expand "divdf3"
-  [(parallel
-    [(set (match_operand:DF 0 "register_operand" "=f,f")
-          (div:DF (match_operand:DF 1 "register_operand" "0,0")
-                  (match_operand:DF 2 "general_operand" "f,m")))
-     (clobber (reg:CC 33))])]
+  [(set (match_operand:DF 0 "register_operand" "=f,f")
+        (div:DF (match_operand:DF 1 "register_operand" "0,0")
+                (match_operand:DF 2 "general_operand" "f,R")))]
   "TARGET_HARD_FLOAT"
   "")
 
 (define_insn "*divdf3"
   [(set (match_operand:DF 0 "register_operand" "=f,f")
         (div:DF (match_operand:DF 1 "register_operand" "0,0")
-                (match_operand:DF 2 "general_operand" "f,m")))
-   (clobber (reg:CC 33))]
+                (match_operand:DF 2 "general_operand" "f,R")))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   ddbr\\t%0,%2
-   ddb\\t%0,%2"
+   ddbr\t%0,%2
+   ddb\t%0,%2"
   [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "type"     "fdiv")
-   (set_attr "atype"    "reg,mem")])
+   (set_attr "type"     "fdivd")])
 
 (define_insn "*divdf3_ibm"
   [(set (match_operand:DF 0 "register_operand" "=f,f")
         (div:DF (match_operand:DF 1 "register_operand" "0,0")
-                (match_operand:DF 2 "general_operand" "f,m")))
-   (clobber (reg:CC 33))]
+                (match_operand:DF 2 "general_operand" "f,R")))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
   "@
-   ddr\\t%0,%2
-   dd\\t%0,%2"
+   ddr\t%0,%2
+   dd\t%0,%2"
   [(set_attr "op_type"  "RR,RX")
-   (set_attr "type"     "fdiv")
-   (set_attr "atype"    "reg,mem")])
+   (set_attr "type"     "fdivd")])
 
 ;
 ; divsf3 instruction pattern(s).
 ;
 
 (define_expand "divsf3"
-  [(parallel
-    [(set (match_operand:SF 0 "register_operand" "=f,f")
-          (div:SF (match_operand:SF 1 "register_operand" "0,0")
-                  (match_operand:SF 2 "general_operand" "f,m")))
-     (clobber (reg:CC 33))])]
+  [(set (match_operand:SF 0 "register_operand" "=f,f")
+        (div:SF (match_operand:SF 1 "register_operand" "0,0")
+                (match_operand:SF 2 "general_operand" "f,R")))]
   "TARGET_HARD_FLOAT"
   "")
 
 (define_insn "*divsf3"
   [(set (match_operand:SF 0 "register_operand" "=f,f")
         (div:SF (match_operand:SF 1 "register_operand" "0,0")
-                (match_operand:SF 2 "general_operand" "f,m")))
-   (clobber (reg:CC 33))]
+                (match_operand:SF 2 "general_operand" "f,R")))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   debr\\t%0,%2
-   deb\\t%0,%2"
+   debr\t%0,%2
+   deb\t%0,%2"
   [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "type"     "fdiv")
-   (set_attr "atype"    "reg,mem")])
+   (set_attr "type"     "fdivs")])
 
 (define_insn "*divsf3"
   [(set (match_operand:SF 0 "register_operand" "=f,f")
         (div:SF (match_operand:SF 1 "register_operand" "0,0")
-                (match_operand:SF 2 "general_operand" "f,m")))
-   (clobber (reg:CC 33))]
+                (match_operand:SF 2 "general_operand" "f,R")))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
   "@
-   der\\t%0,%2
-   de\\t%0,%2"
+   der\t%0,%2
+   de\t%0,%2"
   [(set_attr "op_type"  "RR,RX")
-   (set_attr "type"     "fdiv")
-   (set_attr "atype"    "reg,mem")])
+   (set_attr "type"     "fdivs")])
 
 
 ;;
@@ -4260,10 +5036,9 @@
         (and:DI (match_dup 1) (match_dup 2)))]
   "s390_match_ccmode(insn, CCTmode) && TARGET_64BIT"
   "@
-   ngr\\t%0,%2
-   ng\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
+   ngr\t%0,%2
+   ng\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "*anddi3_cconly"
   [(set (reg 33)
@@ -4271,47 +5046,31 @@
                          (match_operand:DI 2 "general_operand" "d,m"))
                  (const_int 0)))
    (clobber (match_scratch:DI 0 "=d,d"))]
-  "s390_match_ccmode(insn, CCTmode) && TARGET_64BIT"
+  "s390_match_ccmode(insn, CCTmode) && TARGET_64BIT
+   /* Do not steal TM patterns.  */
+   && s390_single_part (operands[2], DImode, HImode, 0) < 0"
   "@
-   ngr\\t%0,%2
-   ng\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
-
-(define_insn "*anddi3_ni"
-  [(set (match_operand:DI 0 "register_operand" "=d")
-        (and:DI (match_operand:DI 1 "nonimmediate_operand" "0")
-                (match_operand:DI 2 "immediate_operand" "n")))
-   (clobber (reg:CC 33))]
-  "TARGET_64BIT && s390_single_hi (operands[2], DImode, -1) >= 0"
-  "*
-{
-  int part = s390_single_hi (operands[2], DImode, -1);
-  operands[2] = GEN_INT (s390_extract_hi (operands[2], DImode, part));
-
-  switch (part)
-    {
-      case 0: return \"nihh\\t%0,%x2\";
-      case 1: return \"nihl\\t%0,%x2\";
-      case 2: return \"nilh\\t%0,%x2\";
-      case 3: return \"nill\\t%0,%x2\";
-      default: abort ();
-    }
-}"
-  [(set_attr "op_type"  "RI")
-   (set_attr "atype"    "reg")])
+   ngr\t%0,%2
+   ng\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "anddi3"
-  [(set (match_operand:DI 0 "register_operand" "=d,d")
-        (and:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
-                (match_operand:DI 2 "general_operand" "d,m")))
-   (clobber (reg:CC 33))]
-  "TARGET_64BIT"
-  "@
-   ngr\\t%0,%2
-   ng\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
+  [(set (match_operand:DI 0 "register_operand" "=d,d,d,d,d,d,d,d")
+	(and:DI (match_operand:DI 1 "nonimmediate_operand" "d,o,0,0,0,0,0,0")
+ 		(match_operand:DI 2 "general_operand"
+				    "M,M,N0HDF,N1HDF,N2HDF,N3HDF,d,m")))
+     (clobber (reg:CC 33))]
+   "TARGET_64BIT"
+   "@
+    #
+    #
+    nihh\t%0,%j2
+    nihl\t%0,%j2
+    nilh\t%0,%j2
+    nill\t%0,%j2
+    ngr\t%0,%2
+    ng\t%0,%2"
+ [(set_attr "op_type" "RRE,RXE,RI,RI,RI,RI,RRE,RXY")])
 
 (define_insn "*anddi3_ss"
   [(set (match_operand:DI 0 "s_operand" "=Q")
@@ -4319,9 +5078,8 @@
                 (match_operand:DI 1 "s_imm_operand" "Q")))
    (clobber (reg:CC 33))]
   ""
-  "nc\\t%O0(8,%R0),%1"
-  [(set_attr "op_type"  "SS")
-   (set_attr "atype"    "mem")])
+  "nc\t%O0(8,%R0),%1"
+  [(set_attr "op_type"  "SS")])
 
 (define_insn "*anddi3_ss_inv"
   [(set (match_operand:DI 0 "s_operand" "=Q")
@@ -4329,9 +5087,8 @@
                 (match_dup 0)))
    (clobber (reg:CC 33))]
   ""
-  "nc\\t%O0(8,%R0),%1"
-  [(set_attr "op_type"  "SS")
-   (set_attr "atype"    "mem")])
+  "nc\t%O0(8,%R0),%1"
+  [(set_attr "op_type"  "SS")])
 
 ;
 ; andsi3 instruction pattern(s).
@@ -4339,63 +5096,68 @@
 
 (define_insn "*andsi3_cc"
   [(set (reg 33)
-        (compare (and:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                         (match_operand:SI 2 "general_operand" "d,m"))
+        (compare (and:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
+                         (match_operand:SI 2 "general_operand" "d,R,T"))
                  (const_int 0)))
-   (set (match_operand:SI 0 "register_operand" "=d,d")
+   (set (match_operand:SI 0 "register_operand" "=d,d,d")
         (and:SI (match_dup 1) (match_dup 2)))]
   "s390_match_ccmode(insn, CCTmode)"
   "@
-   nr\\t%0,%2
-   n\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
+   nr\t%0,%2
+   n\t%0,%2
+   ny\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
 (define_insn "*andsi3_cconly"
   [(set (reg 33)
-        (compare (and:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                         (match_operand:SI 2 "general_operand" "d,m"))
+        (compare (and:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
+                         (match_operand:SI 2 "general_operand" "d,R,T"))
                  (const_int 0)))
-   (clobber (match_scratch:SI 0 "=d,d"))]
-  "s390_match_ccmode(insn, CCTmode)"
+   (clobber (match_scratch:SI 0 "=d,d,d"))]
+  "s390_match_ccmode(insn, CCTmode)
+   /* Do not steal TM patterns.  */
+   && s390_single_part (operands[2], SImode, HImode, 0) < 0"
   "@
-   nr\\t%0,%2
-   n\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
+   nr\t%0,%2
+   n\t%0,%2
+   ny\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
-(define_insn "*andsi3_ni"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (and:SI (match_operand:SI 1 "nonimmediate_operand" "0")
-                (match_operand:SI 2 "immediate_operand" "n")))
-   (clobber (reg:CC 33))]
-  "TARGET_64BIT && s390_single_hi (operands[2], SImode, -1) >= 0"
-  "*
-{
-  int part = s390_single_hi (operands[2], SImode, -1);
-  operands[2] = GEN_INT (s390_extract_hi (operands[2], SImode, part));
+(define_expand "andsi3"
+  [(parallel
+    [(set (match_operand:SI 0 "register_operand" "")
+	  (and:SI (match_operand:SI 1 "nonimmediate_operand" "")
+		  (match_operand:SI 2 "general_operand" "")))
+     (clobber (reg:CC 33))])]
+  ""
+  "")
 
-  switch (part)
-    {
-      case 0: return \"nilh\\t%0,%x2\";
-      case 1: return \"nill\\t%0,%x2\";
-      default: abort ();
-    }
-}"
-  [(set_attr "op_type"  "RI")
-   (set_attr "atype"    "reg")])
-
-(define_insn "andsi3"
+(define_insn "*andsi3_zarch"
+  [(set (match_operand:SI 0 "register_operand" "=d,d,d,d,d,d,d")
+        (and:SI (match_operand:SI 1 "nonimmediate_operand" "d,o,0,0,0,0,0")
+                (match_operand:SI 2 "general_operand" "M,M,N0HSF,N1HSF,d,R,T")))
+   (clobber (reg:CC 33))]
+  "TARGET_ZARCH"
+  "@
+   #
+   #
+   nilh\t%0,%j2
+   nill\t%0,%j2
+   nr\t%0,%2
+   n\t%0,%2
+   ny\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXE,RI,RI,RR,RX,RXY")])
+
+(define_insn "*andsi3_esa"
   [(set (match_operand:SI 0 "register_operand" "=d,d")
         (and:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                (match_operand:SI 2 "general_operand" "d,m")))
+                (match_operand:SI 2 "general_operand" "d,R")))
    (clobber (reg:CC 33))]
-  ""
+  "!TARGET_ZARCH"
   "@
-   nr\\t%0,%2
-   n\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
+   nr\t%0,%2
+   n\t%0,%2"
+  [(set_attr "op_type"  "RR,RX")])
 
 (define_insn "*andsi3_ss"
   [(set (match_operand:SI 0 "s_operand" "=Q")
@@ -4403,9 +5165,8 @@
                 (match_operand:SI 1 "s_imm_operand" "Q")))
    (clobber (reg:CC 33))]
   ""
-  "nc\\t%O0(4,%R0),%1"
-  [(set_attr "op_type"  "SS")
-   (set_attr "atype"    "mem")])
+  "nc\t%O0(4,%R0),%1"
+  [(set_attr "op_type"  "SS")])
 
 (define_insn "*andsi3_ss_inv"
   [(set (match_operand:SI 0 "s_operand" "=Q")
@@ -4413,9 +5174,8 @@
                 (match_dup 0)))
    (clobber (reg:CC 33))]
   ""
-  "nc\\t%O0(4,%R0),%1"
-  [(set_attr "op_type"  "SS")
-   (set_attr "atype"    "mem")])
+  "nc\t%O0(4,%R0),%1"
+  [(set_attr "op_type"  "SS")])
 
 ;
 ; andhi3 instruction pattern(s).
@@ -4426,12 +5186,11 @@
         (and:HI (match_operand:HI 1 "register_operand" "%0,0")
                 (match_operand:HI 2 "nonmemory_operand" "d,n")))
    (clobber (reg:CC 33))]
-  "TARGET_64BIT"
+  "TARGET_ZARCH"
   "@
-   nr\\t%0,%2
-   nill\\t%0,%x2"
-  [(set_attr "op_type"  "RR,RI")
-   (set_attr "atype"    "reg")])
+   nr\t%0,%2
+   nill\t%0,%x2"
+  [(set_attr "op_type"  "RR,RI")])
 
 (define_insn "andhi3"
   [(set (match_operand:HI 0 "register_operand" "=d")
@@ -4439,9 +5198,8 @@
                 (match_operand:HI 2 "nonmemory_operand" "d")))
    (clobber (reg:CC 33))]
   ""
-  "nr\\t%0,%2"
-  [(set_attr "op_type"  "RR")
-   (set_attr "atype"    "reg")])
+  "nr\t%0,%2"
+  [(set_attr "op_type"  "RR")])
 
 (define_insn "*andhi3_ss"
   [(set (match_operand:HI 0 "s_operand" "=Q")
@@ -4449,9 +5207,8 @@
                 (match_operand:HI 1 "s_imm_operand" "Q")))
    (clobber (reg:CC 33))]
   ""
-  "nc\\t%O0(2,%R0),%1"
-  [(set_attr "op_type"  "SS")
-   (set_attr "atype"    "mem")])
+  "nc\t%O0(2,%R0),%1"
+  [(set_attr "op_type"  "SS")])
 
 (define_insn "*andhi3_ss_inv"
   [(set (match_operand:HI 0 "s_operand" "=Q")
@@ -4459,9 +5216,8 @@
                 (match_dup 0)))
    (clobber (reg:CC 33))]
   ""
-  "nc\\t%O0(2,%R0),%1"
-  [(set_attr "op_type"  "SS")
-   (set_attr "atype"    "mem")])
+  "nc\t%O0(2,%R0),%1"
+  [(set_attr "op_type"  "SS")])
 
 ;
 ; andqi3 instruction pattern(s).
@@ -4472,12 +5228,11 @@
         (and:QI (match_operand:QI 1 "register_operand" "%0,0")
                 (match_operand:QI 2 "nonmemory_operand" "d,n")))
    (clobber (reg:CC 33))]
-  "TARGET_64BIT"
+  "TARGET_ZARCH"
   "@
-   nr\\t%0,%2
-   nill\\t%0,%b2"
-  [(set_attr "op_type"  "RR,RI")
-   (set_attr "atype"    "reg")])
+   nr\t%0,%2
+   nill\t%0,%b2"
+  [(set_attr "op_type"  "RR,RI")])
 
 (define_insn "andqi3"
   [(set (match_operand:QI 0 "register_operand" "=d")
@@ -4485,33 +5240,32 @@
                 (match_operand:QI 2 "nonmemory_operand" "d")))
    (clobber (reg:CC 33))]
   ""
-  "nr\\t%0,%2"
-  [(set_attr "op_type"  "RR")
-   (set_attr "atype"    "reg")])
+  "nr\t%0,%2"
+  [(set_attr "op_type"  "RR")])
 
 (define_insn "*andqi3_ss"
-  [(set (match_operand:QI 0 "s_operand" "=Q,Q")
+  [(set (match_operand:QI 0 "s_operand" "=Q,S,Q")
         (and:QI (match_dup 0)
-                (match_operand:QI 1 "s_imm_operand" "n,Q")))
+                (match_operand:QI 1 "s_imm_operand" "n,n,Q")))
    (clobber (reg:CC 33))]
   ""
   "@
-   ni\\t%0,%b1
-   nc\\t%O0(1,%R0),%1"
-  [(set_attr "op_type"  "SI,SS")
-   (set_attr "atype"    "mem")])
+   ni\t%0,%b1
+   niy\t%0,%b1
+   nc\t%O0(1,%R0),%1"
+  [(set_attr "op_type"  "SI,SIY,SS")])
 
 (define_insn "*andqi3_ss_inv"
-  [(set (match_operand:QI 0 "s_operand" "=Q,Q")
-        (and:QI (match_operand:QI 1 "s_imm_operand" "n,Q")
+  [(set (match_operand:QI 0 "s_operand" "=Q,S,Q")
+        (and:QI (match_operand:QI 1 "s_imm_operand" "n,n,Q")
                 (match_dup 0)))
    (clobber (reg:CC 33))]
   ""
   "@
-   ni\\t%0,%b1
-   nc\\t%O0(1,%R0),%1"
-  [(set_attr "op_type"  "SI,SS")
-   (set_attr "atype"    "mem")])
+   ni\t%0,%b1
+   niy\t%0,%b1
+   nc\t%O0(1,%R0),%1"
+  [(set_attr "op_type"  "SI,SIY,SS")])
 
 
 ;;
@@ -4531,10 +5285,9 @@
         (ior:DI (match_dup 1) (match_dup 2)))]
   "s390_match_ccmode(insn, CCTmode) && TARGET_64BIT"
   "@
-   ogr\\t%0,%2
-   og\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
+   ogr\t%0,%2
+   og\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "*iordi3_cconly"
   [(set (reg 33)
@@ -4544,45 +5297,24 @@
    (clobber (match_scratch:DI 0 "=d,d"))]
   "s390_match_ccmode(insn, CCTmode) && TARGET_64BIT"
   "@
-   ogr\\t%0,%2
-   og\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
-
-(define_insn "*iordi3_oi"
-  [(set (match_operand:DI 0 "register_operand" "=d")
-        (ior:DI (match_operand:DI 1 "nonimmediate_operand" "%0")
-                (match_operand:DI 2 "immediate_operand" "n")))
-   (clobber (reg:CC 33))]
-  "TARGET_64BIT && s390_single_hi (operands[2], DImode, 0) >= 0"
-  "*
-{
-  int part = s390_single_hi (operands[2], DImode, 0);
-  operands[2] = GEN_INT (s390_extract_hi (operands[2], DImode, part));
-
-  switch (part)
-    {
-      case 0: return \"oihh\\t%0,%x2\";
-      case 1: return \"oihl\\t%0,%x2\";
-      case 2: return \"oilh\\t%0,%x2\";
-      case 3: return \"oill\\t%0,%x2\";
-      default: abort ();
-    }
-}"
-  [(set_attr "op_type"  "RI")
-   (set_attr "atype"    "reg")])
+   ogr\t%0,%2
+   og\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "iordi3"
-  [(set (match_operand:DI 0 "register_operand" "=d,d")
-        (ior:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
-                (match_operand:DI 2 "general_operand" "d,m")))
+  [(set (match_operand:DI 0 "register_operand" "=d,d,d,d,d,d")
+        (ior:DI (match_operand:DI 1 "nonimmediate_operand" "0,0,0,0,0,0")
+                (match_operand:DI 2 "general_operand" "N0HD0,N1HD0,N2HD0,N3HD0,d,m")))
    (clobber (reg:CC 33))]
   "TARGET_64BIT"
   "@
-   ogr\\t%0,%2
-   og\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
+   oihh\t%0,%i2
+   oihl\t%0,%i2
+   oilh\t%0,%i2
+   oill\t%0,%i2
+   ogr\t%0,%2
+   og\t%0,%2"
+  [(set_attr "op_type"  "RI,RI,RI,RI,RRE,RXY")])
 
 (define_insn "*iordi3_ss"
   [(set (match_operand:DI 0 "s_operand" "=Q")
@@ -4590,9 +5322,8 @@
                 (match_operand:DI 1 "s_imm_operand" "Q")))
    (clobber (reg:CC 33))]
   ""
-  "oc\\t%O0(8,%R0),%1"
-  [(set_attr "op_type"  "SS")
-   (set_attr "atype"    "mem")])
+  "oc\t%O0(8,%R0),%1"
+  [(set_attr "op_type"  "SS")])
 
 (define_insn "*iordi3_ss_inv"
   [(set (match_operand:DI 0 "s_operand" "=Q")
@@ -4600,9 +5331,8 @@
                 (match_dup 0)))
    (clobber (reg:CC 33))]
   ""
-  "oc\\t%O0(8,%R0),%1"
-  [(set_attr "op_type"  "SS")
-   (set_attr "atype"    "mem")])
+  "oc\t%O0(8,%R0),%1"
+  [(set_attr "op_type"  "SS")])
 
 ;
 ; iorsi3 instruction pattern(s).
@@ -4610,63 +5340,64 @@
 
 (define_insn "*iorsi3_cc"
   [(set (reg 33)
-        (compare (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                         (match_operand:SI 2 "general_operand" "d,m"))
+        (compare (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
+                         (match_operand:SI 2 "general_operand" "d,R,T"))
                  (const_int 0)))
-   (set (match_operand:SI 0 "register_operand" "=d,d")
+   (set (match_operand:SI 0 "register_operand" "=d,d,d")
         (ior:SI (match_dup 1) (match_dup 2)))]
   "s390_match_ccmode(insn, CCTmode)"
   "@
-   or\\t%0,%2
-   o\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
+   or\t%0,%2
+   o\t%0,%2
+   oy\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
 (define_insn "*iorsi3_cconly"
   [(set (reg 33)
-        (compare (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                         (match_operand:SI 2 "general_operand" "d,m"))
+        (compare (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
+                         (match_operand:SI 2 "general_operand" "d,R,T"))
                  (const_int 0)))
-   (clobber (match_scratch:SI 0 "=d,d"))]
+   (clobber (match_scratch:SI 0 "=d,d,d"))]
   "s390_match_ccmode(insn, CCTmode)"
   "@
-   or\\t%0,%2
-   o\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
-
-(define_insn "*iorsi3_oi"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0")
-                (match_operand:SI 2 "immediate_operand" "n")))
-   (clobber (reg:CC 33))]
-  "TARGET_64BIT && s390_single_hi (operands[2], SImode, 0) >= 0"
-  "*
-{
-  int part = s390_single_hi (operands[2], SImode, 0);
-  operands[2] = GEN_INT (s390_extract_hi (operands[2], SImode, part));
+   or\t%0,%2
+   o\t%0,%2
+   oy\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
-  switch (part)
-    {
-      case 0: return \"oilh\\t%0,%x2\";
-      case 1: return \"oill\\t%0,%x2\";
-      default: abort ();
-    }
-}"
-  [(set_attr "op_type"  "RI")
-   (set_attr "atype"    "reg")])
+(define_expand "iorsi3"
+  [(parallel
+    [(set (match_operand:SI 0 "register_operand" "")
+	  (ior:SI (match_operand:SI 1 "nonimmediate_operand" "")
+		  (match_operand:SI 2 "general_operand" "")))
+     (clobber (reg:CC 33))])]
+  ""
+  "")
 
-(define_insn "iorsi3"
+(define_insn "iorsi3_zarch"
+  [(set (match_operand:SI 0 "register_operand" "=d,d,d,d,d")
+        (ior:SI (match_operand:SI 1 "nonimmediate_operand" "0,0,0,0,0")
+                (match_operand:SI 2 "general_operand" "N0HS0,N1HS0,d,R,T")))
+   (clobber (reg:CC 33))]
+  "TARGET_ZARCH"
+  "@
+   oilh\t%0,%i2
+   oill\t%0,%i2
+   or\t%0,%2
+   o\t%0,%2
+   oy\t%0,%2"
+  [(set_attr "op_type"  "RI,RI,RR,RX,RXY")])
+
+(define_insn "iorsi3_esa"
   [(set (match_operand:SI 0 "register_operand" "=d,d")
-        (ior:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                (match_operand:SI 2 "general_operand" "d,m")))
+        (ior:SI (match_operand:SI 1 "nonimmediate_operand" "0,0")
+                (match_operand:SI 2 "general_operand" "d,R")))
    (clobber (reg:CC 33))]
-  ""
+  "!TARGET_ZARCH"
   "@
-   or\\t%0,%2
-   o\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
+   or\t%0,%2
+   o\t%0,%2"
+  [(set_attr "op_type"  "RR,RX")])
 
 (define_insn "*iorsi3_ss"
   [(set (match_operand:SI 0 "s_operand" "=Q")
@@ -4674,9 +5405,8 @@
                 (match_operand:SI 1 "s_imm_operand" "Q")))
    (clobber (reg:CC 33))]
   ""
-  "oc\\t%O0(4,%R0),%1"
-  [(set_attr "op_type"  "SS")
-   (set_attr "atype"    "mem")])
+  "oc\t%O0(4,%R0),%1"
+  [(set_attr "op_type"  "SS")])
 
 (define_insn "*iorsi3_ss_inv"
   [(set (match_operand:SI 0 "s_operand" "=Q")
@@ -4684,9 +5414,8 @@
                 (match_dup 0)))
    (clobber (reg:CC 33))]
   ""
-  "oc\\t%O0(4,%R0),%1"
-  [(set_attr "op_type"  "SS")
-   (set_attr "atype"    "mem")])
+  "oc\t%O0(4,%R0),%1"
+  [(set_attr "op_type"  "SS")])
 
 ;
 ; iorhi3 instruction pattern(s).
@@ -4697,12 +5426,11 @@
         (ior:HI (match_operand:HI 1 "register_operand" "%0,0")
                 (match_operand:HI 2 "nonmemory_operand" "d,n")))
    (clobber (reg:CC 33))]
-  "TARGET_64BIT"
+  "TARGET_ZARCH"
   "@
-   or\\t%0,%2
-   oill\\t%0,%x2"
-  [(set_attr "op_type"  "RR,RI")
-   (set_attr "atype"    "reg")])
+   or\t%0,%2
+   oill\t%0,%x2"
+  [(set_attr "op_type"  "RR,RI")])
 
 (define_insn "iorhi3"
   [(set (match_operand:HI 0 "register_operand" "=d")
@@ -4710,9 +5438,8 @@
                 (match_operand:HI 2 "nonmemory_operand" "d")))
    (clobber (reg:CC 33))]
   ""
-  "or\\t%0,%2"
-  [(set_attr "op_type"  "RR")
-   (set_attr "atype"    "reg")])
+  "or\t%0,%2"
+  [(set_attr "op_type"  "RR")])
 
 (define_insn "*iorhi3_ss"
   [(set (match_operand:HI 0 "s_operand" "=Q")
@@ -4720,9 +5447,8 @@
                 (match_operand:HI 1 "s_imm_operand" "Q")))
    (clobber (reg:CC 33))]
   ""
-  "oc\\t%O0(2,%R0),%1"
-  [(set_attr "op_type"  "SS")
-   (set_attr "atype"    "mem")])
+  "oc\t%O0(2,%R0),%1"
+  [(set_attr "op_type"  "SS")])
 
 (define_insn "*iorhi3_ss_inv"
   [(set (match_operand:HI 0 "s_operand" "=Q")
@@ -4730,9 +5456,8 @@
                 (match_dup 0)))
    (clobber (reg:CC 33))]
   ""
-  "oc\\t%O0(2,%R0),%1"
-  [(set_attr "op_type"  "SS")
-   (set_attr "atype"    "mem")])
+  "oc\t%O0(2,%R0),%1"
+  [(set_attr "op_type"  "SS")])
 
 ;
 ; iorqi3 instruction pattern(s).
@@ -4743,12 +5468,11 @@
         (ior:QI (match_operand:QI 1 "register_operand" "%0,0")
                 (match_operand:QI 2 "nonmemory_operand" "d,n")))
    (clobber (reg:CC 33))]
-  "TARGET_64BIT"
+  "TARGET_ZARCH"
   "@
-   or\\t%0,%2
-   oill\\t%0,%b2"
-  [(set_attr "op_type"  "RR,RI")
-   (set_attr "atype"    "reg")])
+   or\t%0,%2
+   oill\t%0,%b2"
+  [(set_attr "op_type"  "RR,RI")])
 
 (define_insn "iorqi3"
   [(set (match_operand:QI 0 "register_operand" "=d")
@@ -4756,33 +5480,32 @@
                 (match_operand:QI 2 "nonmemory_operand" "d")))
    (clobber (reg:CC 33))]
   ""
-  "or\\t%0,%2"
-  [(set_attr "op_type"  "RR")
-   (set_attr "atype"    "reg")])
+  "or\t%0,%2"
+  [(set_attr "op_type"  "RR")])
 
 (define_insn "*iorqi3_ss"
-  [(set (match_operand:QI 0 "s_operand" "=Q,Q")
+  [(set (match_operand:QI 0 "s_operand" "=Q,S,Q")
         (ior:QI (match_dup 0)
-                (match_operand:QI 1 "s_imm_operand" "n,Q")))
+                (match_operand:QI 1 "s_imm_operand" "n,n,Q")))
    (clobber (reg:CC 33))]
   ""
   "@
-   oi\\t%0,%b1
-   oc\\t%O0(1,%R0),%1"
-  [(set_attr "op_type"  "SI,SS")
-   (set_attr "atype"    "reg,mem")])
+   oi\t%0,%b1
+   oiy\t%0,%b1
+   oc\t%O0(1,%R0),%1"
+  [(set_attr "op_type"  "SI,SIY,SS")])
 
 (define_insn "*iorqi3_ss_inv"
-  [(set (match_operand:QI 0 "s_operand" "=Q,Q")
-        (ior:QI (match_operand:QI 1 "s_imm_operand" "n,Q")
+  [(set (match_operand:QI 0 "s_operand" "=Q,S,Q")
+        (ior:QI (match_operand:QI 1 "s_imm_operand" "n,n,Q")
                 (match_dup 0)))
    (clobber (reg:CC 33))]
   ""
   "@
-   oi\\t%0,%b1
-   oc\\t%O0(1,%R0),%1"
-  [(set_attr "op_type"  "SI,SS")
-   (set_attr "atype"    "reg,mem")])
+   oi\t%0,%b1
+   oiy\t%0,%b1
+   oc\t%O0(1,%R0),%1"
+  [(set_attr "op_type"  "SI,SIY,SS")])
 
 
 ;;
@@ -4802,10 +5525,9 @@
         (xor:DI (match_dup 1) (match_dup 2)))]
   "s390_match_ccmode(insn, CCTmode) && TARGET_64BIT"
   "@
-   xgr\\t%0,%2
-   xg\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
+   xgr\t%0,%2
+   xg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "*xordi3_cconly"
   [(set (reg 33)
@@ -4815,10 +5537,9 @@
    (clobber (match_scratch:DI 0 "=d,d"))]
   "s390_match_ccmode(insn, CCTmode) && TARGET_64BIT"
   "@
-   xgr\\t%0,%2
-   xr\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
+   xgr\t%0,%2
+   xr\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "xordi3"
   [(set (match_operand:DI 0 "register_operand" "=d,d")
@@ -4827,10 +5548,9 @@
    (clobber (reg:CC 33))]
   "TARGET_64BIT"
   "@
-   xgr\\t%0,%2
-   xg\\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "atype"    "reg,mem")])
+   xgr\t%0,%2
+   xg\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn "*xordi3_ss"
   [(set (match_operand:DI 0 "s_operand" "=Q")
@@ -4838,9 +5558,8 @@
                 (match_operand:DI 1 "s_imm_operand" "Q")))
    (clobber (reg:CC 33))]
   ""
-  "xc\\t%O0(8,%R0),%1"
-  [(set_attr "op_type"  "SS")
-   (set_attr "atype"    "mem")])
+  "xc\t%O0(8,%R0),%1"
+  [(set_attr "op_type"  "SS")])
 
 (define_insn "*xordi3_ss_inv"
   [(set (match_operand:DI 0 "s_operand" "=Q")
@@ -4848,9 +5567,8 @@
                 (match_dup 0)))
    (clobber (reg:CC 33))]
   ""
-  "xc\\t%O0(8,%R0),%1"
-  [(set_attr "op_type"  "SS")
-   (set_attr "atype"    "mem")])
+  "xc\t%O0(8,%R0),%1"
+  [(set_attr "op_type"  "SS")])
 
 ;
 ; xorsi3 instruction pattern(s).
@@ -4858,42 +5576,42 @@
 
 (define_insn "*xorsi3_cc"
   [(set (reg 33)
-        (compare (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                         (match_operand:SI 2 "general_operand" "d,m"))
+        (compare (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
+                         (match_operand:SI 2 "general_operand" "d,R,T"))
                  (const_int 0)))
-   (set (match_operand:SI 0 "register_operand" "=d,d")
+   (set (match_operand:SI 0 "register_operand" "=d,d,d")
         (xor:SI (match_dup 1) (match_dup 2)))]
   "s390_match_ccmode(insn, CCTmode)"
   "@
-   xr\\t%0,%2
-   x\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
+   xr\t%0,%2
+   x\t%0,%2
+   xy\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
 (define_insn "*xorsi3_cconly"
   [(set (reg 33)
-        (compare (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                         (match_operand:SI 2 "general_operand" "d,m"))
+        (compare (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
+                         (match_operand:SI 2 "general_operand" "d,R,T"))
                  (const_int 0)))
-   (clobber (match_scratch:SI 0 "=d,d"))]
+   (clobber (match_scratch:SI 0 "=d,d,d"))]
   "s390_match_ccmode(insn, CCTmode)"
   "@
-   xr\\t%0,%2
-   x\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
+   xr\t%0,%2
+   x\t%0,%2
+   xy\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
 (define_insn "xorsi3"
-  [(set (match_operand:SI 0 "register_operand" "=d,d")
-        (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0")
-                (match_operand:SI 2 "general_operand" "d,m")))
+  [(set (match_operand:SI 0 "register_operand" "=d,d,d")
+        (xor:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
+                (match_operand:SI 2 "general_operand" "d,R,T")))
    (clobber (reg:CC 33))]
   ""
   "@
-   xr\\t%0,%2
-   x\\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
-   (set_attr "atype"    "reg,mem")])
+   xr\t%0,%2
+   x\t%0,%2
+   xy\t%0,%2"
+  [(set_attr "op_type"  "RR,RX,RXY")])
 
 (define_insn "*xorsi3_ss"
   [(set (match_operand:SI 0 "s_operand" "=Q")
@@ -4901,9 +5619,8 @@
                 (match_operand:SI 1 "s_imm_operand" "Q")))
    (clobber (reg:CC 33))]
   ""
-  "xc\\t%O0(4,%R0),%1"
-  [(set_attr "op_type"  "SS")
-   (set_attr "atype"    "mem")])
+  "xc\t%O0(4,%R0),%1"
+  [(set_attr "op_type"  "SS")])
 
 (define_insn "*xorsi3_ss_inv"
   [(set (match_operand:SI 0 "s_operand" "=Q")
@@ -4911,9 +5628,8 @@
                 (match_dup 0)))
    (clobber (reg:CC 33))]
   ""
-  "xc\\t%O0(4,%R0),%1"
-  [(set_attr "op_type"  "SS")
-   (set_attr "atype"    "mem")])
+  "xc\t%O0(4,%R0),%1"
+  [(set_attr "op_type"  "SS")])
 
 ;
 ; xorhi3 instruction pattern(s).
@@ -4925,9 +5641,8 @@
                 (match_operand:HI 2 "nonmemory_operand" "d")))
    (clobber (reg:CC 33))]
   ""
-  "xr\\t%0,%2"
-  [(set_attr "op_type"  "RR")
-   (set_attr "atype"    "reg")])
+  "xr\t%0,%2"
+  [(set_attr "op_type"  "RR")])
 
 (define_insn "*xorhi3_ss"
   [(set (match_operand:HI 0 "s_operand" "=Q")
@@ -4935,9 +5650,8 @@
                 (match_operand:HI 1 "s_imm_operand" "Q")))
    (clobber (reg:CC 33))]
   ""
-  "xc\\t%O0(2,%R0),%1"
-  [(set_attr "op_type"  "SS")
-   (set_attr "atype"    "mem")])
+  "xc\t%O0(2,%R0),%1"
+  [(set_attr "op_type"  "SS")])
 
 (define_insn "*xorhi3_ss_inv"
   [(set (match_operand:HI 0 "s_operand" "=Q")
@@ -4945,9 +5659,8 @@
                 (match_dup 0)))
    (clobber (reg:CC 33))]
   ""
-  "xc\\t%O0(2,%R0),%1"
-  [(set_attr "op_type"  "SS")
-   (set_attr "atype"    "mem")])
+  "xc\t%O0(2,%R0),%1"
+  [(set_attr "op_type"  "SS")])
 
 ;
 ; xorqi3 instruction pattern(s).
@@ -4959,33 +5672,32 @@
                 (match_operand:QI 2 "nonmemory_operand" "d")))
    (clobber (reg:CC 33))]
   ""
-  "xr\\t%0,%2"
-  [(set_attr "op_type"  "RR")
-   (set_attr "atype"    "reg")])
+  "xr\t%0,%2"
+  [(set_attr "op_type"  "RR")])
 
 (define_insn "*xorqi3_ss"
-  [(set (match_operand:QI 0 "s_operand" "=Q,Q")
+  [(set (match_operand:QI 0 "s_operand" "=Q,S,Q")
         (xor:QI (match_dup 0)
-                (match_operand:QI 1 "s_imm_operand" "n,Q")))
+                (match_operand:QI 1 "s_imm_operand" "n,n,Q")))
    (clobber (reg:CC 33))]
   ""
   "@
-   xi\\t%0,%b1
-   xc\\t%O0(1,%R0),%1"
-  [(set_attr "op_type"  "SI,SS")
-   (set_attr "atype"    "mem")])
+   xi\t%0,%b1
+   xiy\t%0,%b1
+   xc\t%O0(1,%R0),%1"
+  [(set_attr "op_type"  "SI,SIY,SS")])
 
 (define_insn "*xorqi3_ss_inv"
-  [(set (match_operand:QI 0 "s_operand" "=Q,Q")
-        (xor:QI (match_operand:QI 1 "s_imm_operand" "n,Q")
+  [(set (match_operand:QI 0 "s_operand" "=Q,S,Q")
+        (xor:QI (match_operand:QI 1 "s_imm_operand" "n,n,Q")
                 (match_dup 0)))
    (clobber (reg:CC 33))]
   ""
   "@
-   xi\\t%0,%b1
-   xc\\t%O0(1,%R0),%1"
-  [(set_attr "op_type"  "SI,SS")
-   (set_attr "atype"    "mem")])
+   xi\t%0,%b1
+   xiy\t%0,%b1
+   xc\t%O0(1,%R0),%1"
+  [(set_attr "op_type"  "SI,SIY,SS")])
 
 
 ;;
@@ -5009,7 +5721,7 @@
         (neg:DI (match_operand:DI 1 "register_operand" "d")))
    (clobber (reg:CC 33))]
   "TARGET_64BIT"
-  "lcgr\\t%0,%1"
+  "lcgr\t%0,%1"
   [(set_attr "op_type"  "RR")])
 
 (define_insn "*negdi2_31"
@@ -5017,18 +5729,17 @@
         (neg:DI (match_operand:DI 1 "register_operand" "d")))
    (clobber (reg:CC 33))]
   "!TARGET_64BIT"
-  "*
 {
   rtx xop[1];
   xop[0] = gen_label_rtx ();
-  output_asm_insn (\"lcr\\t%0,%1\", operands);
-  output_asm_insn (\"lcr\\t%N0,%N1\", operands);
-  output_asm_insn (\"je\\t%l0\", xop);
-  output_asm_insn (\"bctr\\t%0,0\", operands);
-  ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, \"L\",
+  output_asm_insn ("lcr\t%0,%1", operands);
+  output_asm_insn ("lcr\t%N0,%N1", operands);
+  output_asm_insn ("je\t%l0", xop);
+  output_asm_insn ("bctr\t%0,0", operands);
+  targetm.asm_out.internal_label (asm_out_file, "L",
 			     CODE_LABEL_NUMBER (xop[0]));
-  return \"\";
-}"
+  return "";
+}
   [(set_attr "op_type"  "NN")
    (set_attr "type"     "other")
    (set_attr "length"   "10")])
@@ -5042,7 +5753,7 @@
         (neg:SI (match_operand:SI 1 "register_operand" "d")))
    (clobber (reg:CC 33))]
   ""
-  "lcr\\t%0,%1"
+  "lcr\t%0,%1"
   [(set_attr "op_type"  "RR")])
 
 ;
@@ -5062,16 +5773,18 @@
         (neg:DF (match_operand:DF 1 "register_operand" "f")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "lcdbr\\t%0,%1"
-  [(set_attr "op_type"  "RRE")])
+  "lcdbr\t%0,%1"
+  [(set_attr "op_type"  "RRE")
+   (set_attr "type"     "fsimpd")])
 
 (define_insn "*negdf2_ibm"
   [(set (match_operand:DF 0 "register_operand" "=f")
         (neg:DF (match_operand:DF 1 "register_operand" "f")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
-  "lcdr\\t%0,%1"
-  [(set_attr "op_type"  "RR")])
+  "lcdr\t%0,%1"
+  [(set_attr "op_type"  "RR")
+   (set_attr "type"     "fsimpd")])
 
 ;
 ; negsf2 instruction pattern(s).
@@ -5090,16 +5803,18 @@
         (neg:SF (match_operand:SF 1 "register_operand" "f")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "lcebr\\t%0,%1"
-  [(set_attr "op_type"  "RRE")])
+  "lcebr\t%0,%1"
+  [(set_attr "op_type"  "RRE")
+   (set_attr "type"     "fsimps")])
 
 (define_insn "*negsf2"
   [(set (match_operand:SF 0 "register_operand" "=f")
         (neg:SF (match_operand:SF 1 "register_operand" "f")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
-  "lcer\\t%0,%1"
-  [(set_attr "op_type"  "RR")])
+  "lcer\t%0,%1"
+  [(set_attr "op_type"  "RR")
+   (set_attr "type"     "fsimps")])
 
 
 ;;
@@ -5115,7 +5830,7 @@
         (abs:DI (match_operand:DI 1 "register_operand" "d")))
    (clobber (reg:CC 33))]
   "TARGET_64BIT"
-  "lpgr\\t%0,%1"
+  "lpgr\t%0,%1"
   [(set_attr "op_type"  "RRE")])
 
 ;
@@ -5127,7 +5842,7 @@
         (abs:SI (match_operand:SI 1 "register_operand" "d")))
    (clobber (reg:CC 33))]
   ""
-  "lpr\\t%0,%1"
+  "lpr\t%0,%1"
   [(set_attr "op_type"  "RR")])
 
 ;
@@ -5147,16 +5862,18 @@
         (abs:DF (match_operand:DF 1 "register_operand" "f")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "lpdbr\\t%0,%1"
-  [(set_attr "op_type"  "RRE")])
+  "lpdbr\t%0,%1"
+  [(set_attr "op_type"  "RRE")
+   (set_attr "type"     "fsimpd")])
 
 (define_insn "*absdf2_ibm"
   [(set (match_operand:DF 0 "register_operand" "=f")
         (abs:DF (match_operand:DF 1 "register_operand" "f")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
-  "lpdr\\t%0,%1"
-  [(set_attr "op_type"  "RR")])
+  "lpdr\t%0,%1"
+  [(set_attr "op_type"  "RR")
+   (set_attr "type"     "fsimpd")])
 
 ;
 ; abssf2 instruction pattern(s).
@@ -5175,16 +5892,64 @@
         (abs:SF (match_operand:SF 1 "register_operand" "f")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "lpebr\\t%0,%1"
-  [(set_attr "op_type"  "RRE")])
+  "lpebr\t%0,%1"
+  [(set_attr "op_type"  "RRE")
+   (set_attr "type"     "fsimps")])
 
 (define_insn "*abssf2_ibm"
   [(set (match_operand:SF 0 "register_operand" "=f")
         (abs:SF (match_operand:SF 1 "register_operand" "f")))
    (clobber (reg:CC 33))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
-  "lper\\t%0,%1"
-  [(set_attr "op_type"  "RR")])
+  "lper\t%0,%1"
+  [(set_attr "op_type"  "RR")
+   (set_attr "type"     "fsimps")])
+
+;;
+;;- Negated absolute value instructions
+;;
+
+;
+; Integer
+;
+
+(define_insn "*negabssi2"
+  [(set (match_operand:SI 0 "register_operand" "=d")
+	(neg:SI (abs:SI (match_operand:SI 1 "register_operand" "d"))))
+   (clobber (reg:CC 33))]
+  ""
+  "lnr\t%0,%1"
+  [(set_attr "op_type" "RR")])
+
+(define_insn "*negabsdi2"
+  [(set (match_operand:DI 0 "register_operand" "=d")
+	(neg:DI (abs:DI (match_operand:DI 1 "register_operand" "d"))))
+   (clobber (reg:CC 33))]
+  "TARGET_64BIT"
+  "lngr\t%0,%1"
+  [(set_attr "op_type" "RRE")])
+
+;
+; Floating point
+;
+
+(define_insn "*negabssf2"
+  [(set (match_operand:SF 0 "register_operand" "=f")
+        (neg:SF (abs:SF (match_operand:SF 1 "register_operand" "f"))))
+   (clobber (reg:CC 33))]
+  "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "lnebr\t%0,%1"
+  [(set_attr "op_type"  "RRE")
+   (set_attr "type"     "fsimps")])
+
+(define_insn "*negabsdf2"
+  [(set (match_operand:DF 0 "register_operand" "=f")
+        (neg:DF (abs:DF (match_operand:DF 1 "register_operand" "f"))))
+   (clobber (reg:CC 33))]
+  "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "lndbr\t%0,%1"
+  [(set_attr "op_type"  "RRE")
+   (set_attr "type"     "fsimpd")])
 
 ;;
 ;;- Square root instructions.
@@ -5196,12 +5961,12 @@
 
 (define_insn "sqrtdf2"
   [(set (match_operand:DF 0 "register_operand" "=f,f")
-	(sqrt:DF (match_operand:DF 1 "general_operand" "f,m")))]
+	(sqrt:DF (match_operand:DF 1 "general_operand" "f,R")))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   sqdbr\\t%0,%1
-   sqdb\\t%0,%1"
-  [(set_attr "op_type"  "RRE,RSE")])
+   sqdbr\t%0,%1
+   sqdb\t%0,%1"
+  [(set_attr "op_type"  "RRE,RXE")])
 
 ;
 ; sqrtsf2 instruction pattern(s).
@@ -5209,12 +5974,12 @@
 
 (define_insn "sqrtsf2"
   [(set (match_operand:SF 0 "register_operand" "=f,f")
-	(sqrt:SF (match_operand:SF 1 "general_operand" "f,m")))]
+	(sqrt:SF (match_operand:SF 1 "general_operand" "f,R")))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   sqebr\\t%0,%1
-   sqeb\\t%0,%1"
-  [(set_attr "op_type"  "RRE,RSE")])
+   sqebr\t%0,%1
+   sqeb\t%0,%1"
+  [(set_attr "op_type"  "RRE,RXE")])
 
 ;;
 ;;- One complement instructions.
@@ -5223,7 +5988,7 @@
 ;
 ; one_cmpldi2 instruction pattern(s).
 ;
- 
+
 (define_expand "one_cmpldi2"
   [(parallel
     [(set (match_operand:DI 0 "register_operand" "")
@@ -5232,11 +5997,11 @@
      (clobber (reg:CC 33))])]
   "TARGET_64BIT"
   "")
- 
+
 ;
 ; one_cmplsi2 instruction pattern(s).
 ;
- 
+
 (define_expand "one_cmplsi2"
   [(parallel
     [(set (match_operand:SI 0 "register_operand" "")
@@ -5245,11 +6010,11 @@
      (clobber (reg:CC 33))])]
   ""
   "")
- 
+
 ;
 ; one_cmplhi2 instruction pattern(s).
 ;
- 
+
 (define_expand "one_cmplhi2"
   [(parallel
     [(set (match_operand:HI 0 "register_operand" "")
@@ -5258,11 +6023,11 @@
      (clobber (reg:CC 33))])]
   ""
   "")
- 
+
 ;
 ; one_cmplqi2 instruction pattern(s).
 ;
- 
+
 (define_expand "one_cmplqi2"
   [(parallel
     [(set (match_operand:QI 0 "register_operand" "")
@@ -5282,28 +6047,26 @@
 ;
 
 (define_insn "rotldi3"
-  [(set (match_operand:DI 0 "register_operand" "=d,d")
-	(rotate:DI (match_operand:DI 1 "register_operand" "d,d")
-		   (match_operand:SI 2 "nonmemory_operand" "J,a")))]
+  [(set (match_operand:DI 0 "register_operand" "=d")
+	(rotate:DI (match_operand:DI 1 "register_operand" "d")
+		   (match_operand:SI 2 "shift_count_operand" "Y")))]
   "TARGET_64BIT"
-  "@
-   rllg\\t%0,%1,%c2
-   rllg\\t%0,%1,0(%2)"
-  [(set_attr "op_type"  "RSE")])
+  "rllg\t%0,%1,%Y2"
+  [(set_attr "op_type"  "RSE")
+   (set_attr "atype"    "reg")])
 
 ;
 ; rotlsi3 instruction pattern(s).
 ;
 
 (define_insn "rotlsi3"
-  [(set (match_operand:SI 0 "register_operand" "=d,d")
-	(rotate:SI (match_operand:SI 1 "register_operand" "d,d")
-		   (match_operand:SI 2 "nonmemory_operand" "J,a")))]
-  "TARGET_64BIT"
-  "@
-   rll\\t%0,%1,%c2
-   rll\\t%0,%1,0(%2)"
-  [(set_attr "op_type"  "RSE")])
+  [(set (match_operand:SI 0 "register_operand" "=d")
+	(rotate:SI (match_operand:SI 1 "register_operand" "d")
+		   (match_operand:SI 2 "shift_count_operand" "Y")))]
+  "TARGET_CPU_ZARCH"
+  "rll\t%0,%1,%Y2"
+  [(set_attr "op_type"  "RSE")
+   (set_attr "atype"    "reg")])
 
 
 ;;
@@ -5317,29 +6080,27 @@
 (define_expand "ashldi3"
   [(set (match_operand:DI 0 "register_operand" "")
         (ashift:DI (match_operand:DI 1 "register_operand" "")
-                   (match_operand:SI 2 "nonmemory_operand" "")))]
+                   (match_operand:SI 2 "shift_count_operand" "")))]
   ""
   "")
 
 (define_insn "*ashldi3_31"
-  [(set (match_operand:DI 0 "register_operand" "=d,d")
-        (ashift:DI (match_operand:DI 1 "register_operand" "0,0")
-                   (match_operand:SI 2 "nonmemory_operand" "J,a")))]
+  [(set (match_operand:DI 0 "register_operand" "=d")
+        (ashift:DI (match_operand:DI 1 "register_operand" "0")
+                   (match_operand:SI 2 "shift_count_operand" "Y")))]
   "!TARGET_64BIT"
-  "@
-   sldl\\t%0,%c2
-   sldl\\t%0,0(%2)"     
-  [(set_attr "op_type"  "RS")])
+  "sldl\t%0,%Y2"
+  [(set_attr "op_type"  "RS")
+   (set_attr "atype"    "reg")])
 
 (define_insn "*ashldi3_64"
-  [(set (match_operand:DI 0 "register_operand" "=d,d")
-        (ashift:DI (match_operand:DI 1 "register_operand" "d,d")
-                   (match_operand:SI 2 "nonmemory_operand" "J,a")))]
+  [(set (match_operand:DI 0 "register_operand" "=d")
+        (ashift:DI (match_operand:DI 1 "register_operand" "d")
+                   (match_operand:SI 2 "shift_count_operand" "Y")))]
   "TARGET_64BIT"
-  "@
-   sllg\\t%0,%1,%2
-   sllg\\t%0,%1,0(%2)"
-  [(set_attr "op_type"  "RSE")])
+  "sllg\t%0,%1,%Y2"
+  [(set_attr "op_type"  "RSE")
+   (set_attr "atype"    "reg")])
 
 ;
 ; ashrdi3 instruction pattern(s).
@@ -5349,96 +6110,90 @@
   [(parallel
     [(set (match_operand:DI 0 "register_operand" "")
           (ashiftrt:DI (match_operand:DI 1 "register_operand" "")
-                       (match_operand:SI 2 "nonmemory_operand" "")))
+                       (match_operand:SI 2 "shift_count_operand" "")))
      (clobber (reg:CC 33))])]
   ""
   "")
 
 (define_insn "*ashrdi3_cc_31"
   [(set (reg 33)
-        (compare (ashiftrt:DI (match_operand:DI 1 "register_operand" "0,0")
-                              (match_operand:SI 2 "nonmemory_operand" "J,a"))
+        (compare (ashiftrt:DI (match_operand:DI 1 "register_operand" "0")
+                              (match_operand:SI 2 "shift_count_operand" "Y"))
                  (const_int 0)))
-   (set (match_operand:DI 0 "register_operand" "=d,d")
+   (set (match_operand:DI 0 "register_operand" "=d")
         (ashiftrt:DI (match_dup 1) (match_dup 2)))]
   "!TARGET_64BIT && s390_match_ccmode(insn, CCSmode)"
-  "@
-   srda\\t%0,%c2
-   srda\\t%0,0(%2)"     
-  [(set_attr "op_type"  "RS")])
+  "srda\t%0,%Y2"
+  [(set_attr "op_type"  "RS")
+   (set_attr "atype"    "reg")])
 
 (define_insn "*ashrdi3_cconly_31"
   [(set (reg 33)
-        (compare (ashiftrt:DI (match_operand:DI 1 "register_operand" "0,0")
-                              (match_operand:SI 2 "nonmemory_operand" "J,a"))
+        (compare (ashiftrt:DI (match_operand:DI 1 "register_operand" "0")
+                              (match_operand:SI 2 "shift_count_operand" "Y"))
                  (const_int 0)))
-   (clobber (match_scratch:DI 0 "=d,d"))]
+   (clobber (match_scratch:DI 0 "=d"))]
   "!TARGET_64BIT && s390_match_ccmode(insn, CCSmode)"
-  "@
-   srda\\t%0,%c2
-   srda\\t%0,0(%2)"     
-  [(set_attr "op_type"  "RS")])
+  "srda\t%0,%Y2"
+  [(set_attr "op_type"  "RS")
+   (set_attr "atype"    "reg")])
 
 (define_insn "*ashrdi3_31"
-  [(set (match_operand:DI 0 "register_operand" "=d,d")
-        (ashiftrt:DI (match_operand:DI 1 "register_operand" "0,0")
-                     (match_operand:SI 2 "nonmemory_operand" "J,a")))
+  [(set (match_operand:DI 0 "register_operand" "=d")
+        (ashiftrt:DI (match_operand:DI 1 "register_operand" "0")
+                     (match_operand:SI 2 "shift_count_operand" "Y")))
    (clobber (reg:CC 33))]
   "!TARGET_64BIT"
-  "@
-   srda\\t%0,%c2
-   srda\\t%0,0(%2)"     
-  [(set_attr "op_type"  "RS")])
+  "srda\t%0,%Y2"
+  [(set_attr "op_type"  "RS")
+   (set_attr "atype"    "reg")])
 
 (define_insn "*ashrdi3_cc_64"
   [(set (reg 33)
-        (compare (ashiftrt:DI (match_operand:DI 1 "register_operand" "d,d")
-                              (match_operand:SI 2 "nonmemory_operand" "J,a"))
+        (compare (ashiftrt:DI (match_operand:DI 1 "register_operand" "d")
+                              (match_operand:SI 2 "shift_count_operand" "Y"))
                  (const_int 0)))
-   (set (match_operand:DI 0 "register_operand" "=d,d")
+   (set (match_operand:DI 0 "register_operand" "=d")
         (ashiftrt:DI (match_dup 1) (match_dup 2)))]
   "s390_match_ccmode(insn, CCSmode) && TARGET_64BIT"
-  "@
-   srag\\t%0,%1,%c2
-   srag\\t%0,%1,0(%2)"
-  [(set_attr "op_type"  "RSE")])
+  "srag\t%0,%1,%Y2"
+  [(set_attr "op_type"  "RSE")
+   (set_attr "atype"    "reg")])
 
 (define_insn "*ashrdi3_cconly_64"
   [(set (reg 33)
-        (compare (ashiftrt:DI (match_operand:DI 1 "register_operand" "d,d")
-                              (match_operand:SI 2 "nonmemory_operand" "J,a"))
+        (compare (ashiftrt:DI (match_operand:DI 1 "register_operand" "d")
+                              (match_operand:SI 2 "shift_count_operand" "Y"))
                  (const_int 0)))
-   (clobber (match_scratch:DI 0 "=d,d"))]
+   (clobber (match_scratch:DI 0 "=d"))]
   "s390_match_ccmode(insn, CCSmode) && TARGET_64BIT"
-  "@
-   srag\\t%0,%1,%c2
-   srag\\t%0,%1,0(%2)"
-  [(set_attr "op_type"  "RSE")])
+  "srag\t%0,%1,%Y2"
+  [(set_attr "op_type"  "RSE")
+   (set_attr "atype"    "reg")])
 
 (define_insn "*ashrdi3_64"
-  [(set (match_operand:DI 0 "register_operand" "=d,d")
-        (ashiftrt:DI (match_operand:DI 1 "register_operand" "d,d")
-                     (match_operand:SI 2 "nonmemory_operand" "J,a")))
+  [(set (match_operand:DI 0 "register_operand" "=d")
+        (ashiftrt:DI (match_operand:DI 1 "register_operand" "d")
+                     (match_operand:SI 2 "shift_count_operand" "Y")))
    (clobber (reg:CC 33))]
   "TARGET_64BIT"
-  "@
-   srag\\t%0,%1,%c2
-   srag\\t%0,%1,0(%2)"
-  [(set_attr "op_type"  "RSE")])
+  "srag\t%0,%1,%Y2"
+  [(set_attr "op_type"  "RSE")
+   (set_attr "atype"    "reg")])
+
 
 ;
 ; ashlsi3 instruction pattern(s).
 ;
 
 (define_insn "ashlsi3"
-  [(set (match_operand:SI 0 "register_operand" "=d,d")
-        (ashift:SI (match_operand:SI 1 "register_operand" "0,0")
-                   (match_operand:SI 2 "nonmemory_operand" "J,a")))]
+  [(set (match_operand:SI 0 "register_operand" "=d")
+        (ashift:SI (match_operand:SI 1 "register_operand" "0")
+                   (match_operand:SI 2 "shift_count_operand" "Y")))]
   ""
-  "@
-   sll\\t%0,%c2
-   sll\\t%0,0(%2)"
-  [(set_attr "op_type"  "RS")])
+  "sll\t%0,%Y2"
+  [(set_attr "op_type"  "RS")
+   (set_attr "atype"    "reg")])
 
 ;
 ; ashrsi3 instruction pattern(s).
@@ -5446,39 +6201,37 @@
 
 (define_insn "*ashrsi3_cc"
   [(set (reg 33)
-        (compare (ashiftrt:SI (match_operand:SI 1 "register_operand" "0,0")
-                              (match_operand:SI 2 "nonmemory_operand" "J,a"))
+        (compare (ashiftrt:SI (match_operand:SI 1 "register_operand" "0")
+                              (match_operand:SI 2 "shift_count_operand" "Y"))
                  (const_int 0)))
-   (set (match_operand:SI 0 "register_operand" "=d,d")
+   (set (match_operand:SI 0 "register_operand" "=d")
         (ashiftrt:SI (match_dup 1) (match_dup 2)))]
   "s390_match_ccmode(insn, CCSmode)"
-  "@
-   sra\\t%0,%c2
-   sra\\t%0,0(%2)"
-  [(set_attr "op_type"  "RS")])
+  "sra\t%0,%Y2"
+  [(set_attr "op_type"  "RS")
+   (set_attr "atype"    "reg")])
+
 
 (define_insn "*ashrsi3_cconly"
   [(set (reg 33)
-        (compare (ashiftrt:SI (match_operand:SI 1 "register_operand" "0,0")
-                              (match_operand:SI 2 "nonmemory_operand" "J,a"))
+        (compare (ashiftrt:SI (match_operand:SI 1 "register_operand" "0")
+                              (match_operand:SI 2 "shift_count_operand" "Y"))
                  (const_int 0)))
-   (clobber (match_scratch:SI 0 "=d,d"))]
+   (clobber (match_scratch:SI 0 "=d"))]
   "s390_match_ccmode(insn, CCSmode)"
-  "@
-   sra\\t%0,%c2
-   sra\\t%0,0(%2)"
-  [(set_attr "op_type"  "RS")])
+  "sra\t%0,%Y2"
+  [(set_attr "op_type"  "RS")
+   (set_attr "atype"    "reg")])
 
 (define_insn "ashrsi3"
-  [(set (match_operand:SI 0 "register_operand" "=d,d")
-        (ashiftrt:SI (match_operand:SI 1 "register_operand" "0,0")
-                     (match_operand:SI 2 "nonmemory_operand" "J,a")))
+  [(set (match_operand:SI 0 "register_operand" "=d")
+        (ashiftrt:SI (match_operand:SI 1 "register_operand" "0")
+                     (match_operand:SI 2 "shift_count_operand" "Y")))
    (clobber (reg:CC 33))]
   ""
-  "@
-   sra\\t%0,%c2
-   sra\\t%0,0(%2)"
-  [(set_attr "op_type"  "RS")])
+  "sra\t%0,%Y2"
+  [(set_attr "op_type"  "RS")
+   (set_attr "atype"    "reg")])
 
 
 ;;
@@ -5492,43 +6245,40 @@
 (define_expand "lshrdi3"
   [(set (match_operand:DI 0 "register_operand" "")
         (lshiftrt:DI (match_operand:DI 1 "register_operand" "")
-                     (match_operand:SI 2 "nonmemory_operand" "")))]
+                     (match_operand:SI 2 "shift_count_operand" "")))]
   ""
   "")
 
 (define_insn "*lshrdi3_31"
-  [(set (match_operand:DI 0 "register_operand" "=d,d")
-        (lshiftrt:DI (match_operand:DI 1 "register_operand" "0,0")
-                     (match_operand:SI 2 "nonmemory_operand" "J,a")))]
+  [(set (match_operand:DI 0 "register_operand" "=d")
+        (lshiftrt:DI (match_operand:DI 1 "register_operand" "0")
+                     (match_operand:SI 2 "shift_count_operand" "Y")))]
   "!TARGET_64BIT"
-  "@
-   srdl\\t%0,%c2
-   srdl\\t%0,0(%2)"     
-   [(set_attr "op_type"  "RS,RS")])
+  "srdl\t%0,%Y2"
+   [(set_attr "op_type"  "RS")
+    (set_attr "atype"    "reg")])
 
 (define_insn "*lshrdi3_64"
-  [(set (match_operand:DI 0 "register_operand" "=d,d")
-        (lshiftrt:DI (match_operand:DI 1 "register_operand" "d,d")
-                     (match_operand:SI 2 "nonmemory_operand" "J,a")))]
+  [(set (match_operand:DI 0 "register_operand" "=d")
+        (lshiftrt:DI (match_operand:DI 1 "register_operand" "d")
+                     (match_operand:SI 2 "shift_count_operand" "Y")))]
   "TARGET_64BIT"
-  "@
-   srlg\\t%0,%1,%c2
-   srlg\\t%0,%1,0(%2)"
-  [(set_attr "op_type"  "RS,RS")])
+  "srlg\t%0,%1,%Y2"
+  [(set_attr "op_type"  "RSE")
+   (set_attr "atype"    "reg")])
 
 ;
 ; lshrsi3 instruction pattern(s).
 ;
 
 (define_insn "lshrsi3"
-  [(set (match_operand:SI 0 "register_operand" "=d,d")
-        (lshiftrt:SI (match_operand:SI 1 "register_operand" "0,0")
-                     (match_operand:SI 2 "nonmemory_operand" "J,a")))]
+  [(set (match_operand:SI 0 "register_operand" "=d")
+        (lshiftrt:SI (match_operand:SI 1 "register_operand" "0")
+                     (match_operand:SI 2 "shift_count_operand" "Y")))]
   ""
-  "@
-   srl\\t%0,%c2
-   srl\\t%0,0(%2)"
-  [(set_attr "op_type"  "RS")])
+  "srl\t%0,%Y2"
+  [(set_attr "op_type"  "RS")
+   (set_attr "atype"    "reg")])
 
 
 ;;
@@ -5536,270 +6286,242 @@
 ;;
 
 (define_expand "beq"
-  [(set (reg:CCZ 33) (compare:CCZ (match_dup 1) (match_dup 2)))
-   (set (pc)
-        (if_then_else (eq (reg:CCZ 33) (const_int 0))
-                      (label_ref (match_operand 0 "" ""))
-                      (pc)))]
+  [(match_operand 0 "" "")]
   ""
-  "{ operands[1] = s390_compare_op0; operands[2] = s390_compare_op1; }")
+  "s390_emit_jump (operands[0],
+    s390_emit_compare (EQ, s390_compare_op0, s390_compare_op1)); DONE;")
 
 (define_expand "bne"
-  [(set (reg:CCZ 33) (compare:CCZ (match_dup 1) (match_dup 2)))
-   (set (pc)
-        (if_then_else (ne (reg:CCZ 33) (const_int 0))
-                      (label_ref (match_operand 0 "" ""))
-                      (pc)))]
+  [(match_operand 0 "" "")]
   ""
-  "{ operands[1] = s390_compare_op0; operands[2] = s390_compare_op1; }")
+  "s390_emit_jump (operands[0],
+    s390_emit_compare (NE, s390_compare_op0, s390_compare_op1)); DONE;")
 
 (define_expand "bgt"
-  [(set (reg:CCS 33) (compare:CCS (match_dup 1) (match_dup 2)))
-   (set (pc)
-        (if_then_else (gt (reg:CCS 33) (const_int 0))
-                      (label_ref (match_operand 0 "" ""))
-                      (pc)))]
+  [(match_operand 0 "" "")]
   ""
-  "{ operands[1] = s390_compare_op0; operands[2] = s390_compare_op1; }")
+  "s390_emit_jump (operands[0],
+    s390_emit_compare (GT, s390_compare_op0, s390_compare_op1)); DONE;")
 
 (define_expand "bgtu"
-  [(set (reg:CCU 33) (compare:CCU (match_dup 1) (match_dup 2)))
-   (set (pc)
-        (if_then_else (gtu (reg:CCU 33) (const_int 0))
-                      (label_ref (match_operand 0 "" ""))
-                      (pc)))]
+  [(match_operand 0 "" "")]
   ""
-  "{ operands[1] = s390_compare_op0; operands[2] = s390_compare_op1; }")
+  "s390_emit_jump (operands[0],
+    s390_emit_compare (GTU, s390_compare_op0, s390_compare_op1)); DONE;")
 
 (define_expand "blt"
-  [(set (reg:CCS 33) (compare:CCS (match_dup 1) (match_dup 2)))
-   (set (pc)
-        (if_then_else (lt (reg:CCS 33) (const_int 0))
-                      (label_ref (match_operand 0 "" ""))
-                      (pc)))]
+  [(match_operand 0 "" "")]
   ""
-  "{ operands[1] = s390_compare_op0; operands[2] = s390_compare_op1; }")
+  "s390_emit_jump (operands[0],
+    s390_emit_compare (LT, s390_compare_op0, s390_compare_op1)); DONE;")
 
 (define_expand "bltu"
-  [(set (reg:CCU 33) (compare:CCU (match_dup 1) (match_dup 2)))
-   (set (pc)
-        (if_then_else (ltu (reg:CCU 33) (const_int 0))
-                      (label_ref (match_operand 0 "" ""))
-                      (pc)))]
+  [(match_operand 0 "" "")]
   ""
-  "{ operands[1] = s390_compare_op0; operands[2] = s390_compare_op1; }")
+  "s390_emit_jump (operands[0],
+    s390_emit_compare (LTU, s390_compare_op0, s390_compare_op1)); DONE;")
 
 (define_expand "bge"
-  [(set (reg:CCS 33) (compare:CCS (match_dup 1) (match_dup 2)))
-   (set (pc)
-        (if_then_else (ge (reg:CCS 33) (const_int 0))
-                      (label_ref (match_operand 0 "" ""))
-                      (pc)))]
+  [(match_operand 0 "" "")]
   ""
-  "{ operands[1] = s390_compare_op0; operands[2] = s390_compare_op1; }")
+  "s390_emit_jump (operands[0],
+    s390_emit_compare (GE, s390_compare_op0, s390_compare_op1)); DONE;")
 
 (define_expand "bgeu"
-  [(set (reg:CCU 33) (compare:CCU (match_dup 1) (match_dup 2)))
-   (set (pc)
-        (if_then_else (geu (reg:CCU 33) (const_int 0))
-                      (label_ref (match_operand 0 "" ""))
-                      (pc)))]
+  [(match_operand 0 "" "")]
   ""
-  "{ operands[1] = s390_compare_op0; operands[2] = s390_compare_op1; }")
+  "s390_emit_jump (operands[0],
+    s390_emit_compare (GEU, s390_compare_op0, s390_compare_op1)); DONE;")
 
 (define_expand "ble"
-  [(set (reg:CCS 33) (compare:CCS (match_dup 1) (match_dup 2)))
-   (set (pc)
-        (if_then_else (le (reg:CCS 33) (const_int 0))
-                      (label_ref (match_operand 0 "" ""))
-                      (pc)))]
+  [(match_operand 0 "" "")]
   ""
-  "{ operands[1] = s390_compare_op0; operands[2] = s390_compare_op1; }")
+  "s390_emit_jump (operands[0],
+    s390_emit_compare (LE, s390_compare_op0, s390_compare_op1)); DONE;")
 
 (define_expand "bleu"
-  [(set (reg:CCU 33) (compare:CCU (match_dup 1) (match_dup 2)))
-   (set (pc)
-        (if_then_else (leu (reg:CCU 33) (const_int 0))
-                      (label_ref (match_operand 0 "" ""))
-                      (pc)))]
+  [(match_operand 0 "" "")]
   ""
-  "{ operands[1] = s390_compare_op0; operands[2] = s390_compare_op1; }")
+  "s390_emit_jump (operands[0],
+    s390_emit_compare (LEU, s390_compare_op0, s390_compare_op1)); DONE;")
 
 (define_expand "bunordered"
-  [(set (reg:CCS 33) (compare:CCS (match_dup 1) (match_dup 2)))
-   (set (pc)
-        (if_then_else (unordered (reg:CCS 33) (const_int 0))
-                      (label_ref (match_operand 0 "" ""))
-                      (pc)))]
+  [(match_operand 0 "" "")]
   ""
-  "{ operands[1] = s390_compare_op0; operands[2] = s390_compare_op1; }")
+  "s390_emit_jump (operands[0],
+    s390_emit_compare (UNORDERED, s390_compare_op0, s390_compare_op1)); DONE;")
 
 (define_expand "bordered"
-  [(set (reg:CCS 33) (compare:CCS (match_dup 1) (match_dup 2)))
-   (set (pc)
-        (if_then_else (ordered (reg:CCS 33) (const_int 0))
-                      (label_ref (match_operand 0 "" ""))
-                      (pc)))]
+  [(match_operand 0 "" "")]
   ""
-  "{ operands[1] = s390_compare_op0; operands[2] = s390_compare_op1; }")
+  "s390_emit_jump (operands[0],
+    s390_emit_compare (ORDERED, s390_compare_op0, s390_compare_op1)); DONE;")
 
 (define_expand "buneq"
-  [(set (reg:CCS 33) (compare:CCS (match_dup 1) (match_dup 2)))
-   (set (pc)
-        (if_then_else (uneq (reg:CCS 33) (const_int 0))
-                      (label_ref (match_operand 0 "" ""))
-                      (pc)))]
-  ""
-  "{ operands[1] = s390_compare_op0; operands[2] = s390_compare_op1; }")
-
-(define_expand "bungt"
-  [(set (reg:CCS 33) (compare:CCS (match_dup 1) (match_dup 2)))
-   (set (pc)
-        (if_then_else (ungt (reg:CCS 33) (const_int 0))
-                      (label_ref (match_operand 0 "" ""))
-                      (pc)))]
+  [(match_operand 0 "" "")]
   ""
-  "{ operands[1] = s390_compare_op0; operands[2] = s390_compare_op1; }")
+  "s390_emit_jump (operands[0],
+    s390_emit_compare (UNEQ, s390_compare_op0, s390_compare_op1)); DONE;")
 
 (define_expand "bunlt"
-  [(set (reg:CCS 33) (compare:CCS (match_dup 1) (match_dup 2)))
-   (set (pc)
-        (if_then_else (unlt (reg:CCS 33) (const_int 0))
-                      (label_ref (match_operand 0 "" ""))
-                      (pc)))]
+  [(match_operand 0 "" "")]
   ""
-  "{ operands[1] = s390_compare_op0; operands[2] = s390_compare_op1; }")
+  "s390_emit_jump (operands[0],
+    s390_emit_compare (UNLT, s390_compare_op0, s390_compare_op1)); DONE;")
 
-(define_expand "bunge"
-  [(set (reg:CCS 33) (compare:CCS (match_dup 1) (match_dup 2)))
-   (set (pc)
-        (if_then_else (unge (reg:CCS 33) (const_int 0))
-                      (label_ref (match_operand 0 "" ""))
-                      (pc)))]
+(define_expand "bungt"
+  [(match_operand 0 "" "")]
   ""
-  "{ operands[1] = s390_compare_op0; operands[2] = s390_compare_op1; }")
+  "s390_emit_jump (operands[0],
+    s390_emit_compare (UNGT, s390_compare_op0, s390_compare_op1)); DONE;")
 
 (define_expand "bunle"
-  [(set (reg:CCS 33) (compare:CCS (match_dup 1) (match_dup 2)))
-   (set (pc)
-        (if_then_else (unle (reg:CCS 33) (const_int 0))
-                      (label_ref (match_operand 0 "" ""))
-                      (pc)))]
+  [(match_operand 0 "" "")]
+  ""
+  "s390_emit_jump (operands[0],
+    s390_emit_compare (UNLE, s390_compare_op0, s390_compare_op1)); DONE;")
+
+(define_expand "bunge"
+  [(match_operand 0 "" "")]
   ""
-  "{ operands[1] = s390_compare_op0; operands[2] = s390_compare_op1; }")
+  "s390_emit_jump (operands[0],
+    s390_emit_compare (UNGE, s390_compare_op0, s390_compare_op1)); DONE;")
 
 (define_expand "bltgt"
-  [(set (reg:CCS 33) (compare:CCS (match_dup 1) (match_dup 2)))
-   (set (pc)
-        (if_then_else (ltgt (reg:CCS 33) (const_int 0))
-                      (label_ref (match_operand 0 "" ""))
-                      (pc)))]
+  [(match_operand 0 "" "")]
   ""
-  "{ operands[1] = s390_compare_op0; operands[2] = s390_compare_op1; }")
+  "s390_emit_jump (operands[0],
+    s390_emit_compare (LTGT, s390_compare_op0, s390_compare_op1)); DONE;")
 
 
 ;;
 ;;- Conditional jump instructions.
 ;;
 
-(define_insn "cjump"
- [(set (pc)
-       (if_then_else 
-         (match_operator 1 "comparison_operator" [(reg 33) (const_int 0)])
-	 (label_ref (match_operand 0 "" ""))
-	 (pc)))]
-  ""
-  "*
+(define_insn "*cjump_64"
+  [(set (pc)
+        (if_then_else
+          (match_operator 1 "comparison_operator" [(reg 33) (const_int 0)])
+          (label_ref (match_operand 0 "" ""))
+          (pc)))]
+  "TARGET_CPU_ZARCH"
+{
+  if (get_attr_length (insn) == 4)
+    return "j%C1\t%l0";
+  else
+    return "jg%C1\t%l0";
+}
+  [(set_attr "op_type" "RI")
+   (set_attr "type"    "branch")
+   (set (attr "length")
+        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
+                      (const_int 4) (const_int 6)))])
+
+(define_insn "*cjump_31"
+  [(set (pc)
+        (if_then_else
+          (match_operator 1 "comparison_operator" [(reg 33) (const_int 0)])
+          (label_ref (match_operand 0 "" ""))
+          (pc)))]
+  "!TARGET_CPU_ZARCH"
 {
   if (get_attr_length (insn) == 4)
-    return \"j%C1\\t%l0\";
-  else if (TARGET_64BIT)
-    return \"jg%C1\\t%l0\";
+    return "j%C1\t%l0";
   else
     abort ();
-}"
+}
   [(set_attr "op_type" "RI")
+   (set_attr "type"    "branch")
    (set (attr "length")
-        (cond [(lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
-                (const_int 4)
-               (ne (symbol_ref "TARGET_64BIT") (const_int 0))
-                 (const_int 6)
-	       (ne (symbol_ref "s390_pool_overflow") (const_int 0))
-                 (if_then_else (eq (symbol_ref "flag_pic") (const_int 0))
-                               (const_int 12) (const_int 14))
-               (eq (symbol_ref "flag_pic") (const_int 0))
-                 (const_int 6)] (const_int 8)))])
+        (if_then_else (eq (symbol_ref "flag_pic") (const_int 0))
+          (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
+                        (const_int 4) (const_int 6))
+          (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
+                        (const_int 4) (const_int 8))))])
 
 (define_insn "*cjump_long"
- [(set (pc)
-       (if_then_else
-         (match_operator 1 "comparison_operator" [(reg 33) (const_int 0)])
-	 (match_operand 0 "address_operand" "p")
-	 (pc)))]
+  [(set (pc)
+        (if_then_else
+          (match_operator 1 "comparison_operator" [(reg 33) (const_int 0)])
+          (match_operand 0 "address_operand" "U")
+          (pc)))]
   ""
-  "*
 {
   if (get_attr_op_type (insn) == OP_TYPE_RR)
-    return \"b%C1r\\t%0\";
+    return "b%C1r\t%0";
   else
-    return \"b%C1\\t%a0\";
-}"
-  [(set (attr "op_type") 
+    return "b%C1\t%a0";
+}
+  [(set (attr "op_type")
         (if_then_else (match_operand 0 "register_operand" "")
                       (const_string "RR") (const_string "RX")))
-   (set_attr "atype" "mem")])
+   (set_attr "type"  "branch")
+   (set_attr "atype" "agen")])
 
 
 ;;
 ;;- Negated conditional jump instructions.
 ;;
 
-(define_insn "icjump"
- [(set (pc)
-       (if_then_else
-         (match_operator 1 "comparison_operator" [(reg 33) (const_int 0)])
-         (pc) 
-	 (label_ref (match_operand 0 "" ""))))]
-  ""
-  "*
-{  
+(define_insn "*icjump_64"
+  [(set (pc)
+        (if_then_else
+          (match_operator 1 "comparison_operator" [(reg 33) (const_int 0)])
+          (pc)
+          (label_ref (match_operand 0 "" ""))))]
+  "TARGET_CPU_ZARCH"
+{
   if (get_attr_length (insn) == 4)
-    return \"j%D1\\t%l0\";
-  else if (TARGET_64BIT)
-    return \"jg%D1\\t%l0\";
+    return "j%D1\t%l0";
+  else
+    return "jg%D1\t%l0";
+}
+  [(set_attr "op_type" "RI")
+   (set_attr "type"    "branch")
+   (set (attr "length")
+        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
+                      (const_int 4) (const_int 6)))])
+
+(define_insn "*icjump_31"
+  [(set (pc)
+        (if_then_else
+          (match_operator 1 "comparison_operator" [(reg 33) (const_int 0)])
+          (pc)
+          (label_ref (match_operand 0 "" ""))))]
+  "!TARGET_CPU_ZARCH"
+{
+  if (get_attr_length (insn) == 4)
+    return "j%D1\t%l0";
   else
     abort ();
-}"
+}
   [(set_attr "op_type" "RI")
+   (set_attr "type"    "branch")
    (set (attr "length")
-        (cond [(lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
-                (const_int 4)
-               (ne (symbol_ref "TARGET_64BIT") (const_int 0))
-                 (const_int 6)
-	       (ne (symbol_ref "s390_pool_overflow") (const_int 0))
-                 (if_then_else (eq (symbol_ref "flag_pic") (const_int 0))
-                               (const_int 12) (const_int 14))
-               (eq (symbol_ref "flag_pic") (const_int 0))
-                 (const_int 6)] (const_int 8)))])
+        (if_then_else (eq (symbol_ref "flag_pic") (const_int 0))
+          (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
+                        (const_int 4) (const_int 6))
+          (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
+                        (const_int 4) (const_int 8))))])
 
 (define_insn "*icjump_long"
- [(set (pc)
-       (if_then_else
-         (match_operator 1 "comparison_operator" [(reg 33) (const_int 0)])
-         (pc)
-	 (match_operand 0 "address_operand" "p")))]
+  [(set (pc)
+        (if_then_else
+          (match_operator 1 "comparison_operator" [(reg 33) (const_int 0)])
+          (pc)
+          (match_operand 0 "address_operand" "U")))]
   ""
-  "*
 {
   if (get_attr_op_type (insn) == OP_TYPE_RR)
-    return \"b%D1r\\t%0\";
+    return "b%D1r\t%0";
   else
-    return \"b%D1\\t%a0\";
-}"
-  [(set (attr "op_type") 
+    return "b%D1\t%a0";
+}
+  [(set (attr "op_type")
         (if_then_else (match_operand 0 "register_operand" "")
                       (const_string "RR") (const_string "RX")))
-   (set_attr "atype" "mem")])
+   (set_attr "type"  "branch")
+   (set_attr "atype" "agen")])
 
 ;;
 ;;- Trap instructions.
@@ -5808,40 +6530,34 @@
 (define_insn "trap"
   [(trap_if (const_int 1) (const_int 0))]
   ""
-  "j\\t.+2"
-  [(set_attr "op_type" "RX")])
+  "j\t.+2"
+  [(set_attr "op_type" "RI")
+   (set_attr "type"  "branch")])
 
 (define_expand "conditional_trap"
-  [(set (match_dup 2) (match_dup 3))
-   (trap_if (match_operator 0 "comparison_operator"
-			    [(match_dup 2) (const_int 0)])
-	    (match_operand:SI 1 "general_operand" ""))]
+  [(trap_if (match_operand 0 "comparison_operator" "")
+	    (match_operand 1 "general_operand" ""))]
   ""
-  "
 {
-   enum machine_mode ccmode;
-
-   if (operands[1] != const0_rtx) FAIL;	
-
-   ccmode = s390_select_ccmode (GET_CODE (operands[0]), 
-			        s390_compare_op0, s390_compare_op1);    
-   operands[2] = gen_rtx_REG (ccmode, 33);
-   operands[3] = gen_rtx_COMPARE (ccmode, s390_compare_op0, s390_compare_op1);
-}")
+  if (operands[1] != const0_rtx) FAIL;
+  operands[0] = s390_emit_compare (GET_CODE (operands[0]), 
+                                   s390_compare_op0, s390_compare_op1);
+})
 
 (define_insn "*trap"
   [(trap_if (match_operator 0 "comparison_operator" [(reg 33) (const_int 0)])
 	    (const_int 0))]
   ""
-  "j%C0\\t.+2";
-  [(set_attr "op_type" "RX")])
+  "j%C0\t.+2";
+  [(set_attr "op_type" "RI")
+   (set_attr "type"  "branch")])
 
 ;;
 ;;- Loop instructions.
 ;;
 ;;  This is all complicated by the fact that since this is a jump insn
 ;;  we must handle our own output reloads.
- 
+
 (define_expand "doloop_end"
   [(use (match_operand 0 "" ""))        ; loop pseudo
    (use (match_operand 1 "" ""))        ; iterations; zero if unknown
@@ -5849,86 +6565,79 @@
    (use (match_operand 3 "" ""))        ; loop level
    (use (match_operand 4 "" ""))]       ; label
   ""
-  "
 {
-  if (GET_MODE (operands[0]) == SImode)
-    emit_jump_insn (gen_doloop_si (operands[4], operands[0], operands[0]));
+  if (GET_MODE (operands[0]) == SImode && !TARGET_CPU_ZARCH)
+    emit_jump_insn (gen_doloop_si31 (operands[4], operands[0], operands[0]));
+  else if (GET_MODE (operands[0]) == SImode && TARGET_CPU_ZARCH)
+    emit_jump_insn (gen_doloop_si64 (operands[4], operands[0], operands[0]));
   else if (GET_MODE (operands[0]) == DImode && TARGET_64BIT)
     emit_jump_insn (gen_doloop_di (operands[4], operands[0], operands[0]));
   else
     FAIL;
 
   DONE;
-}")
+})
 
-(define_insn "doloop_si"
+(define_insn_and_split "doloop_si64"
   [(set (pc)
         (if_then_else
           (ne (match_operand:SI 1 "register_operand" "d,d")
               (const_int 1))
           (label_ref (match_operand 0 "" ""))
           (pc)))
-   (set (match_operand:SI 2 "register_operand" "=1,?*m*d")
+   (set (match_operand:SI 2 "nonimmediate_operand" "=1,?*m*d")
         (plus:SI (match_dup 1) (const_int -1)))
    (clobber (match_scratch:SI 3 "=X,&d"))
    (clobber (reg:CC 33))]
-  ""
-  "*
+  "TARGET_CPU_ZARCH"
 {
   if (which_alternative != 0)
-    return \"#\";
+    return "#";
   else if (get_attr_length (insn) == 4)
-    return \"brct\\t%1,%l0\";
+    return "brct\t%1,%l0";
   else
-    abort ();
-}"
+    return "ahi\t%1,-1\;jgne\t%l0";
+}
+  "&& reload_completed
+   && (! REG_P (operands[2])
+       || ! rtx_equal_p (operands[1], operands[2]))"
+  [(set (match_dup 3) (match_dup 1))
+   (parallel [(set (reg:CCAN 33)
+                   (compare:CCAN (plus:SI (match_dup 3) (const_int -1))
+                                 (const_int 0)))
+              (set (match_dup 3) (plus:SI (match_dup 3) (const_int -1)))])
+   (set (match_dup 2) (match_dup 3))
+   (set (pc) (if_then_else (ne (reg:CCAN 33) (const_int 0))
+                           (label_ref (match_dup 0))
+                           (pc)))]
+  ""
   [(set_attr "op_type"  "RI")
+   (set_attr "type"  "branch")
    (set (attr "length")
-        (cond [(lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
-                (const_int 4)
-               (ne (symbol_ref "TARGET_64BIT") (const_int 0))
-                 (const_int 10)
-	       (ne (symbol_ref "s390_pool_overflow") (const_int 0))
-                 (if_then_else (eq (symbol_ref "flag_pic") (const_int 0))
-                               (const_int 12) (const_int 14))
-               (eq (symbol_ref "flag_pic") (const_int 0))
-                 (const_int 6)] (const_int 8)))])
+        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
+                      (const_int 4) (const_int 10)))])
 
-(define_insn "*doloop_si_long"
+(define_insn_and_split "doloop_si31"
   [(set (pc)
         (if_then_else
           (ne (match_operand:SI 1 "register_operand" "d,d")
               (const_int 1))
-          (match_operand 0 "address_operand" "p,p")
+          (label_ref (match_operand 0 "" ""))
           (pc)))
-   (set (match_operand:SI 2 "register_operand" "=1,?*m*d")
+   (set (match_operand:SI 2 "nonimmediate_operand" "=1,?*m*d")
         (plus:SI (match_dup 1) (const_int -1)))
    (clobber (match_scratch:SI 3 "=X,&d"))
    (clobber (reg:CC 33))]
-  ""
-  "*
+  "!TARGET_CPU_ZARCH"
 {
-  if (get_attr_op_type (insn) == OP_TYPE_RR)
-    return \"bctr\\t%0\";
+  if (which_alternative != 0)
+    return "#";
+  else if (get_attr_length (insn) == 4)
+    return "brct\t%1,%l0";
   else
-    return \"bct\\t%a0\";
-}"
-  [(set (attr "op_type") 
-        (if_then_else (match_operand 0 "register_operand" "")
-                      (const_string "RR") (const_string "RX")))
-   (set_attr "atype" "mem")])
-
-(define_split
-  [(set (pc)
-        (if_then_else (ne (match_operand:SI 1 "register_operand" "")
-                          (const_int 1))
-                      (match_operand 0 "" "")
-                      (pc)))
-   (set (match_operand:SI 2 "nonimmediate_operand" "")
-        (plus:SI (match_dup 1) (const_int -1)))
-   (clobber (match_scratch:SI 3 ""))
-   (clobber (reg:CC 33))]
-  "reload_completed
+    abort ();
+}
+  "&& reload_completed
    && (! REG_P (operands[2])
        || ! rtx_equal_p (operands[1], operands[2]))"
   [(set (match_dup 3) (match_dup 1))
@@ -5938,71 +6647,63 @@
               (set (match_dup 3) (plus:SI (match_dup 3) (const_int -1)))])
    (set (match_dup 2) (match_dup 3))
    (set (pc) (if_then_else (ne (reg:CCAN 33) (const_int 0))
-                           (match_dup 0)
+                           (label_ref (match_dup 0))
                            (pc)))]
-  "")
+  ""
+  [(set_attr "op_type"  "RI")
+   (set_attr "type"  "branch")
+   (set (attr "length")
+        (if_then_else (eq (symbol_ref "flag_pic") (const_int 0))
+          (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
+                        (const_int 4) (const_int 6))
+          (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
+                        (const_int 4) (const_int 8))))])
 
-(define_insn "doloop_di"
+(define_insn "*doloop_si_long"
   [(set (pc)
         (if_then_else
-          (ne (match_operand:DI 1 "register_operand" "d,d")
+          (ne (match_operand:SI 1 "register_operand" "d,d")
               (const_int 1))
-          (label_ref (match_operand 0 "" ""))
+          (match_operand 0 "address_operand" "U,U")
           (pc)))
-   (set (match_operand:DI 2 "register_operand" "=1,?*m*r")
-        (plus:DI (match_dup 1) (const_int -1)))
-   (clobber (match_scratch:DI 3 "=X,&d"))
+   (set (match_operand:SI 2 "register_operand" "=1,?*m*d")
+        (plus:SI (match_dup 1) (const_int -1)))
+   (clobber (match_scratch:SI 3 "=X,&d"))
    (clobber (reg:CC 33))]
-  "TARGET_64BIT"
-  "*
+  "!TARGET_CPU_ZARCH"
 {
-  if (which_alternative != 0)
-    return \"#\";
-  else if (get_attr_length (insn) == 4)
-    return \"brctg\\t%1,%l0\";
+  if (get_attr_op_type (insn) == OP_TYPE_RR)
+    return "bctr\t%1,%0";
   else
-    abort ();
-}"
-  [(set_attr "op_type"  "RI")
-   (set (attr "length")
-        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
-                      (const_int 4) (const_int 12)))])
+    return "bct\t%1,%a0";
+}
+  [(set (attr "op_type")
+        (if_then_else (match_operand 0 "register_operand" "")
+                      (const_string "RR") (const_string "RX")))
+   (set_attr "type"  "branch")
+   (set_attr "atype" "agen")])
 
-(define_insn "*doloop_di_long"
+(define_insn_and_split "doloop_di"
   [(set (pc)
         (if_then_else
           (ne (match_operand:DI 1 "register_operand" "d,d")
               (const_int 1))
-          (match_operand 0 "address_operand" "p,p")
+          (label_ref (match_operand 0 "" ""))
           (pc)))
-   (set (match_operand:DI 2 "register_operand" "=1,?*m*d")
+   (set (match_operand:DI 2 "nonimmediate_operand" "=1,?*m*r")
         (plus:DI (match_dup 1) (const_int -1)))
    (clobber (match_scratch:DI 3 "=X,&d"))
    (clobber (reg:CC 33))]
-  ""
-  "*
+  "TARGET_64BIT"
 {
-  if (get_attr_op_type (insn) == OP_TYPE_RRE)
-    return \"bctgr\\t%0\";
+  if (which_alternative != 0)
+    return "#";
+  else if (get_attr_length (insn) == 4)
+    return "brctg\t%1,%l0";
   else
-    return \"bctg\\t%a0\";
-}"
-  [(set (attr "op_type") 
-        (if_then_else (match_operand 0 "register_operand" "")
-                      (const_string "RRE") (const_string "RXE")))
-   (set_attr "atype" "mem")])
-
-(define_split
-  [(set (pc)
-        (if_then_else (ne (match_operand:DI 1 "register_operand" "")
-                          (const_int 1))
-                      (match_operand 0 "" "")
-                      (pc)))
-   (set (match_operand:DI 2 "nonimmediate_operand" "")
-        (plus:DI (match_dup 1) (const_int -1)))
-   (clobber (match_scratch:DI 3 ""))
-   (clobber (reg:CC 33))]
-  "reload_completed
+    return "aghi\t%1,-1\;jgne\t%l0";
+}
+  "&& reload_completed
    && (! REG_P (operands[2])
        || ! rtx_equal_p (operands[1], operands[2]))"
   [(set (match_dup 3) (match_dup 1))
@@ -6012,9 +6713,14 @@
               (set (match_dup 3) (plus:DI (match_dup 3) (const_int -1)))])
    (set (match_dup 2) (match_dup 3))
    (set (pc) (if_then_else (ne (reg:CCAN 33) (const_int 0))
-                           (match_dup 0)
+                           (label_ref (match_dup 0))
                            (pc)))]
-  "")
+  ""
+  [(set_attr "op_type"  "RI")
+   (set_attr "type"  "branch")
+   (set (attr "length")
+        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
+                      (const_int 4) (const_int 10)))])
 
 ;;
 ;;- Unconditional jump instructions.
@@ -6024,65 +6730,82 @@
 ; jump instruction pattern(s).
 ;
 
-(define_insn "jump"
-  [(set (pc) (label_ref (match_operand 0 "" "")))]
+(define_expand "jump"
+  [(match_operand 0 "" "")]
   ""
-  "*
+  "s390_emit_jump (operands[0], NULL_RTX); DONE;")
+
+(define_insn "*jump64"
+  [(set (pc) (label_ref (match_operand 0 "" "")))]
+  "TARGET_CPU_ZARCH"
+{
+  if (get_attr_length (insn) == 4)
+    return "j\t%l0";
+  else
+    return "jg\t%l0";
+}
+  [(set_attr "op_type" "RI")
+   (set_attr "type"  "branch")
+   (set (attr "length")
+        (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
+                      (const_int 4) (const_int 6)))])
+
+(define_insn "*jump31"
+  [(set (pc) (label_ref (match_operand 0 "" "")))]
+  "!TARGET_CPU_ZARCH"
 {
   if (get_attr_length (insn) == 4)
-    return \"j\\t%l0\";
-  else if (TARGET_64BIT)
-    return \"jg\\t%l0\";
+    return "j\t%l0";
   else
     abort ();
-}"
+}
   [(set_attr "op_type" "RI")
+   (set_attr "type"  "branch")
    (set (attr "length")
-        (cond [(lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
-                (const_int 4)
-               (ne (symbol_ref "TARGET_64BIT") (const_int 0))
-                 (const_int 6)
-               (eq (symbol_ref "flag_pic") (const_int 0))
-                 (const_int 6)] (const_int 8)))])
+        (if_then_else (eq (symbol_ref "flag_pic") (const_int 0))
+          (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
+                        (const_int 4) (const_int 6))
+          (if_then_else (lt (abs (minus (pc) (match_dup 0))) (const_int 60000))
+                        (const_int 4) (const_int 8))))])
 
 ;
 ; indirect-jump instruction pattern(s).
 ;
 
 (define_insn "indirect_jump"
- [(set (pc) (match_operand 0 "address_operand" "p"))]
+ [(set (pc) (match_operand 0 "address_operand" "U"))]
   ""
-  "*
 {
   if (get_attr_op_type (insn) == OP_TYPE_RR)
-    return \"br\\t%0\";
+    return "br\t%0";
   else
-    return \"b\\t%a0\";
-}"
-  [(set (attr "op_type") 
+    return "b\t%a0";
+}
+  [(set (attr "op_type")
         (if_then_else (match_operand 0 "register_operand" "")
                       (const_string "RR") (const_string "RX")))
-   (set_attr "atype" "mem")])
+   (set_attr "type"  "branch")
+   (set_attr "atype" "agen")])
 
 ;
 ; casesi instruction pattern(s).
 ;
 
 (define_insn "casesi_jump"
- [(set (pc) (match_operand 0 "address_operand" "p"))
+ [(set (pc) (match_operand 0 "address_operand" "U"))
    (use (label_ref (match_operand 1 "" "")))]
   ""
-  "*
 {
   if (get_attr_op_type (insn) == OP_TYPE_RR)
-    return \"br\\t%0\";
+    return "br\t%0";
   else
-    return \"b\\t%a0\";
-}"
-  [(set (attr "op_type") 
+    return "b\t%a0";
+}
+  [(set (attr "op_type")
         (if_then_else (match_operand 0 "register_operand" "")
                       (const_string "RR") (const_string "RX")))
-   (set_attr "atype" "mem")])
+   (set_attr "type"  "branch")
+   (set_attr "atype" "agen")])
 
 (define_expand "casesi"
   [(match_operand:SI 0 "general_operand" "")
@@ -6091,7 +6814,6 @@
    (label_ref (match_operand 3 "" ""))
    (label_ref (match_operand 4 "" ""))]
   ""
-  "
 {
    rtx index  = gen_reg_rtx (SImode);
    rtx base   = gen_reg_rtx (Pmode);
@@ -6110,11 +6832,11 @@
    if (TARGET_64BIT)
        emit_insn (gen_ashldi3 (index, index, GEN_INT (3)));
    else
-       emit_insn (gen_ashlsi3 (index, index, GEN_INT (2)));
+       emit_insn (gen_ashlsi3 (index, index, const2_rtx));
 
    emit_move_insn (base, gen_rtx_LABEL_REF (Pmode, operands[3]));
 
-   index = gen_rtx_MEM (Pmode, gen_rtx_PLUS (Pmode, base, index));
+   index = gen_const_mem (Pmode, gen_rtx_PLUS (Pmode, base, index));
    emit_move_insn (target, index);
 
    if (flag_pic)
@@ -6122,7 +6844,7 @@
    emit_jump_insn (gen_casesi_jump (target, operands[3]));
 
    DONE;
-}")
+})
 
 
 ;;
@@ -6141,7 +6863,6 @@
               (match_operand 1 "" "")
               (match_operand 2 "" "")])]
   ""
-  "
 {
   int i;
 
@@ -6160,138 +6881,154 @@
   emit_insn (gen_blockage ());
 
   DONE;
-}")
+})
 
 ;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and
 ;; all of memory.  This blocks insns from being moved across this point.
 
 (define_insn "blockage"
-  [(unspec_volatile [(const_int 0)] 0)]
+  [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
   ""
   ""
-  [(set_attr "type"    "none")])
-
-
+  [(set_attr "type"    "none")
+   (set_attr "length"  "0")])
 
 ;
-; call instruction pattern(s).
+; sibcall patterns
 ;
 
-(define_expand "call"
+(define_expand "sibcall"
   [(call (match_operand 0 "" "")
-         (match_operand 1 "" ""))
-   (use (match_operand 2 "" ""))]
+	 (match_operand 1 "" ""))]
   ""
-  "
 {
-  int plt_call = 0;
-  rtx insn;
+  s390_emit_call (XEXP (operands[0], 0), NULL_RTX, NULL_RTX, NULL_RTX);
+  DONE;
+})
+
+(define_insn "*sibcall_br"
+  [(call (mem:QI (reg 1))
+         (match_operand 0 "const_int_operand" "n"))]
+  "SIBLING_CALL_P (insn)
+   && GET_MODE (XEXP (XEXP (PATTERN (insn), 0), 0)) == Pmode"
+  "br\t%%r1"
+  [(set_attr "op_type" "RR")
+   (set_attr "type"  "branch")
+   (set_attr "atype" "agen")])
+
+(define_insn "*sibcall_brc"
+  [(call (mem:QI (match_operand 0 "bras_sym_operand" "X"))
+         (match_operand 1 "const_int_operand" "n"))]
+  "SIBLING_CALL_P (insn) && TARGET_SMALL_EXEC"
+  "j\t%0"
+  [(set_attr "op_type" "RI")
+   (set_attr "type"    "branch")])
 
-  /* Direct function calls need special treatment.  */
-  if (GET_CODE (XEXP (operands[0], 0)) == SYMBOL_REF)
-    {
-      rtx sym = XEXP (operands[0], 0);
+(define_insn "*sibcall_brcl"
+  [(call (mem:QI (match_operand 0 "bras_sym_operand" "X"))
+         (match_operand 1 "const_int_operand" "n"))]
+  "SIBLING_CALL_P (insn) && TARGET_CPU_ZARCH"
+  "jg\t%0"
+  [(set_attr "op_type" "RIL")
+   (set_attr "type"    "branch")])
 
-      /* When calling a global routine in PIC mode, we must
-         replace the symbol itself with the PLT stub.  */
-      if (flag_pic && !SYMBOL_REF_FLAG (sym))
-        {
-          sym = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, sym), 113);
-          sym = gen_rtx_CONST (Pmode, sym);
+;
+; sibcall_value patterns
+;
 
-	  plt_call = 1;
-        }
+(define_expand "sibcall_value"
+  [(set (match_operand 0 "" "")
+	(call (match_operand 1 "" "")
+	      (match_operand 2 "" "")))]
+  ""
+{
+  s390_emit_call (XEXP (operands[1], 0), NULL_RTX, operands[0], NULL_RTX);
+  DONE;
+})
 
-      /* Unless we can use the bras(l) insn, force the 
-         routine address into a register.  */
-      if (!TARGET_SMALL_EXEC && !TARGET_64BIT)
-        {
-          rtx target = gen_reg_rtx (Pmode);
-          emit_move_insn (target, sym);
-          sym = target;
-        }
+(define_insn "*sibcall_value_br"
+  [(set (match_operand 0 "" "")
+	(call (mem:QI (reg 1))
+	      (match_operand 1 "const_int_operand" "n")))]
+  "SIBLING_CALL_P (insn)
+   && GET_MODE (XEXP (XEXP (XEXP (PATTERN (insn), 1), 0), 0)) == Pmode"
+  "br\t%%r1"
+  [(set_attr "op_type" "RR")
+   (set_attr "type"  "branch")
+   (set_attr "atype" "agen")])
 
-      operands[0] = gen_rtx_MEM (QImode, sym);
-    }
+(define_insn "*sibcall_value_brc"
+  [(set (match_operand 0 "" "")
+	(call (mem:QI (match_operand 1 "bras_sym_operand" "X"))
+	      (match_operand 2 "const_int_operand" "n")))]
+  "SIBLING_CALL_P (insn) && TARGET_SMALL_EXEC"
+  "j\t%1"
+  [(set_attr "op_type" "RI")
+   (set_attr "type"    "branch")])
 
-  /* Emit insn.  */
-  insn = emit_call_insn (gen_call_exp (operands[0], operands[1],
-  				       gen_rtx_REG (Pmode, RETURN_REGNUM)));
+(define_insn "*sibcall_value_brcl"
+  [(set (match_operand 0 "" "")
+	(call (mem:QI (match_operand 1 "bras_sym_operand" "X"))
+	      (match_operand 2 "const_int_operand" "n")))]
+  "SIBLING_CALL_P (insn) && TARGET_CPU_ZARCH"
+  "jg\t%1"
+  [(set_attr "op_type" "RIL")
+   (set_attr "type"    "branch")])
 
-  /* In 31-bit, we must load the GOT register even if the 
-     compiler doesn't know about it, because the PLT glue 
-     code uses it.  In 64-bit, this is not necessary.  */
-  if (plt_call && !TARGET_64BIT)
-    use_reg (&CALL_INSN_FUNCTION_USAGE (insn), pic_offset_table_rtx);
 
-  DONE;
-}")
+;
+; call instruction pattern(s).
+;
 
-(define_expand "call_exp"
-  [(parallel [(call (match_operand 0 "" "")
-                    (match_operand 1 "" ""))
-              (clobber (match_operand 2 "" ""))])]
+(define_expand "call"
+  [(call (match_operand 0 "" "")
+         (match_operand 1 "" ""))
+   (use (match_operand 2 "" ""))]
   ""
-  "")
-
-(define_insn "brasl"
-  [(call (mem:QI (match_operand:DI 0 "bras_sym_operand" "X"))
-         (match_operand:SI 1 "const_int_operand" "n"))
-   (clobber (match_operand:DI 2 "register_operand" "=r"))]
-  "TARGET_64BIT"
-  "brasl\\t%2,%0"
-  [(set_attr "op_type" "RIL")
-   (set_attr "type"    "jsr")])
-
-(define_insn "bras"
-  [(call (mem:QI (match_operand:SI 0 "bras_sym_operand" "X"))
-         (match_operand:SI 1 "const_int_operand" "n"))
-   (clobber (match_operand:SI 2 "register_operand" "=r"))]
-  "TARGET_SMALL_EXEC"
-  "bras\\t%2,%0"
+{
+  s390_emit_call (XEXP (operands[0], 0), NULL_RTX, NULL_RTX,
+		  gen_rtx_REG (Pmode, RETURN_REGNUM));
+  DONE;
+})
+
+(define_insn "*bras"
+  [(call (mem:QI (match_operand 0 "bras_sym_operand" "X"))
+         (match_operand 1 "const_int_operand" "n"))
+   (clobber (match_operand 2 "register_operand" "=r"))]
+  "!SIBLING_CALL_P (insn)
+   && TARGET_SMALL_EXEC
+   && GET_MODE (operands[2]) == Pmode"
+  "bras\t%2,%0"
   [(set_attr "op_type" "RI")
    (set_attr "type"    "jsr")])
 
-(define_insn "basr_64"
-  [(call (mem:QI (match_operand:DI 0 "register_operand" "a"))
-         (match_operand:SI 1 "const_int_operand" "n"))
-   (clobber (match_operand:DI 2 "register_operand" "=r"))]
-  "TARGET_64BIT"
-  "basr\\t%2,%0"
-  [(set_attr "op_type" "RR")
-   (set_attr "type"    "jsr")
-   (set_attr "atype"   "mem")])
-
-(define_insn "basr_31"
-  [(call (mem:QI (match_operand:SI 0 "register_operand" "a"))
-         (match_operand:SI 1 "const_int_operand" "n"))
-   (clobber (match_operand:SI 2 "register_operand" "=r"))]
-  "!TARGET_64BIT"
-  "basr\\t%2,%0"
-  [(set_attr "op_type" "RR")
-   (set_attr "type"    "jsr")
-   (set_attr "atype"    "mem")])
-
-(define_insn "bas_64"
-  [(call (mem:QI (match_operand:QI 0 "address_operand" "p"))
-         (match_operand:SI 1 "const_int_operand" "n"))
-   (clobber (match_operand:DI 2 "register_operand" "=r"))]
-  "TARGET_64BIT"
-  "bas\\t%2,%a0"
-  [(set_attr "op_type" "RX")
-   (set_attr "type"    "jsr")
-   (set_attr "atype"   "mem")])
-
-(define_insn "bas_31"
-  [(call (mem:QI (match_operand:QI 0 "address_operand" "p"))
-         (match_operand:SI 1 "const_int_operand" "n"))
-   (clobber (match_operand:SI 2 "register_operand" "=r"))]
-  "!TARGET_64BIT"
-  "bas\\t%2,%a0"
-  [(set_attr "op_type" "RX")
-   (set_attr "type"    "jsr")
-   (set_attr "atype"   "mem")])
+(define_insn "*brasl"
+  [(call (mem:QI (match_operand 0 "bras_sym_operand" "X"))
+         (match_operand 1 "const_int_operand" "n"))
+   (clobber (match_operand 2 "register_operand" "=r"))]
+  "!SIBLING_CALL_P (insn)
+   && TARGET_CPU_ZARCH
+   && GET_MODE (operands[2]) == Pmode"
+  "brasl\t%2,%0"
+  [(set_attr "op_type" "RIL")
+   (set_attr "type"    "jsr")])
 
+(define_insn "*basr"
+  [(call (mem:QI (match_operand 0 "address_operand" "U"))
+         (match_operand 1 "const_int_operand" "n"))
+   (clobber (match_operand 2 "register_operand" "=r"))]
+  "!SIBLING_CALL_P (insn) && GET_MODE (operands[2]) == Pmode"
+{
+  if (get_attr_op_type (insn) == OP_TYPE_RR)
+    return "basr\t%2,%0";
+  else
+    return "bas\t%2,%a0";
+}
+  [(set (attr "op_type")
+        (if_then_else (match_operand 0 "register_operand" "")
+                      (const_string "RR") (const_string "RX")))
+   (set_attr "type"  "jsr")
+   (set_attr "atype" "agen")])
 
 ;
 ; call_value instruction pattern(s).
@@ -6303,123 +7040,163 @@
               (match_operand 2 "" "")))
    (use (match_operand 3 "" ""))]
   ""
-  "
 {
-  int plt_call = 0;
-  rtx insn;
-
-  /* Direct function calls need special treatment.  */
-  if (GET_CODE (XEXP (operands[1], 0)) == SYMBOL_REF)
-    {
-      rtx sym = XEXP (operands[1], 0);
-
-      /* When calling a global routine in PIC mode, we must
-         replace the symbol itself with the PLT stub.  */
-      if (flag_pic && !SYMBOL_REF_FLAG (sym))
-        {
-          sym = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, sym), 113);
-          sym = gen_rtx_CONST (Pmode, sym);
-
-	  plt_call = 1;
-        }
-
-      /* Unless we can use the bras(l) insn, force the 
-         routine address into a register.  */
-      if (!TARGET_SMALL_EXEC && !TARGET_64BIT)
-        {
-          rtx target = gen_reg_rtx (Pmode);
-          emit_move_insn (target, sym);
-          sym = target;
-        }
-
-      operands[1] = gen_rtx_MEM (QImode, sym);
-    }
-
-  /* Emit insn.  */
-  insn = emit_call_insn (
-	    gen_call_value_exp (operands[0], operands[1], operands[2],
-  				gen_rtx_REG (Pmode, RETURN_REGNUM)));
-
-  /* In 31-bit, we must load the GOT register even if the 
-     compiler doesn't know about it, because the PLT glue 
-     code uses it.  In 64-bit, this is not necessary.  */
-  if (plt_call && !TARGET_64BIT)
-    use_reg (&CALL_INSN_FUNCTION_USAGE (insn), pic_offset_table_rtx);
-
+  s390_emit_call (XEXP (operands[1], 0), NULL_RTX, operands[0],
+		  gen_rtx_REG (Pmode, RETURN_REGNUM));
   DONE;
-}")
+})
 
-(define_expand "call_value_exp"
-  [(parallel [(set (match_operand 0 "" "")
-                   (call (match_operand 1 "" "")
-                         (match_operand 2 "" "")))
-              (clobber (match_operand 3 "" ""))])]
-  ""
-  "")
-
-(define_insn "brasl_r"
-  [(set (match_operand 0 "register_operand" "=df")
-        (call (mem:QI (match_operand:DI 1 "bras_sym_operand" "X"))
+(define_insn "*bras_r"
+  [(set (match_operand 0 "" "")
+        (call (mem:QI (match_operand 1 "bras_sym_operand" "X"))
               (match_operand:SI 2 "const_int_operand" "n")))
-   (clobber (match_operand:DI 3 "register_operand" "=r"))]
-  "TARGET_64BIT"
-  "brasl\\t%3,%1"
-  [(set_attr "op_type" "RIL")
+   (clobber (match_operand 3 "register_operand" "=r"))]
+  "!SIBLING_CALL_P (insn)
+   && TARGET_SMALL_EXEC
+   && GET_MODE (operands[3]) == Pmode"
+  "bras\t%3,%1"
+  [(set_attr "op_type" "RI")
    (set_attr "type"    "jsr")])
 
-(define_insn "bras_r"
-  [(set (match_operand 0 "register_operand" "=df")
-        (call (mem:QI (match_operand:SI 1 "bras_sym_operand" "X"))
-              (match_operand:SI 2 "const_int_operand" "n")))
-   (clobber (match_operand:SI 3 "register_operand" "=r"))]
-  "TARGET_SMALL_EXEC"
-  "bras\\t%3,%1"
-  [(set_attr "op_type" "RI")
+(define_insn "*brasl_r"
+  [(set (match_operand 0 "" "")
+        (call (mem:QI (match_operand 1 "bras_sym_operand" "X"))
+              (match_operand 2 "const_int_operand" "n")))
+   (clobber (match_operand 3 "register_operand" "=r"))]
+  "!SIBLING_CALL_P (insn)
+   && TARGET_CPU_ZARCH
+   && GET_MODE (operands[3]) == Pmode"
+  "brasl\t%3,%1"
+  [(set_attr "op_type" "RIL")
    (set_attr "type"    "jsr")])
 
-(define_insn "basr_r_64"
-  [(set (match_operand 0 "register_operand" "=df")
-        (call (mem:QI (match_operand:DI 1 "register_operand" "a"))
-              (match_operand:SI 2 "const_int_operand" "n")))
-   (clobber (match_operand:DI 3 "register_operand" "=r"))]
+(define_insn "*basr_r"
+  [(set (match_operand 0 "" "")
+        (call (mem:QI (match_operand 1 "address_operand" "U"))
+              (match_operand 2 "const_int_operand" "n")))
+   (clobber (match_operand 3 "register_operand" "=r"))]
+  "!SIBLING_CALL_P (insn) && GET_MODE (operands[3]) == Pmode"
+{
+  if (get_attr_op_type (insn) == OP_TYPE_RR)
+    return "basr\t%3,%1";
+  else
+    return "bas\t%3,%a1";
+}
+  [(set (attr "op_type")
+        (if_then_else (match_operand 1 "register_operand" "")
+                      (const_string "RR") (const_string "RX")))
+   (set_attr "type"  "jsr")
+   (set_attr "atype" "agen")])
+
+;;
+;;- Thread-local storage support.
+;;
+
+(define_insn "get_tp_64"
+  [(set (match_operand:DI 0 "nonimmediate_operand" "=??d,Q")
+        (unspec:DI [(const_int 0)] UNSPEC_TP))]
   "TARGET_64BIT"
-  "basr\\t%3,%1"
-  [(set_attr "op_type" "RR")
-   (set_attr "type"    "jsr")])
+  "@
+   ear\t%0,%%a0\;sllg\t%0,%0,32\;ear\t%0,%%a1
+   stam\t%%a0,%%a1,%0"
+  [(set_attr "op_type" "NN,RS")
+   (set_attr "atype"   "reg,*")
+   (set_attr "type"    "o3,*")
+   (set_attr "length"  "14,*")])
+
+(define_insn "get_tp_31"
+  [(set (match_operand:SI 0 "nonimmediate_operand" "=d,Q")
+        (unspec:SI [(const_int 0)] UNSPEC_TP))]
+  "!TARGET_64BIT"
+  "@
+   ear\t%0,%%a0
+   stam\t%%a0,%%a0,%0"
+  [(set_attr "op_type" "RRE,RS")])
 
-(define_insn "basr_r_31"
-  [(set (match_operand 0 "register_operand" "=df")
-        (call (mem:QI (match_operand:SI 1 "register_operand" "a"))
-              (match_operand:SI 2 "const_int_operand" "n")))
-   (clobber (match_operand:SI 3 "register_operand" "=r"))]
+(define_insn "set_tp_64"
+  [(unspec_volatile [(match_operand:DI 0 "general_operand" "??d,Q")] UNSPECV_SET_TP)
+   (clobber (match_scratch:SI 1 "=d,X"))]
+  "TARGET_64BIT"
+  "@
+   sar\t%%a1,%0\;srlg\t%1,%0,32\;sar\t%%a0,%1
+   lam\t%%a0,%%a1,%0"
+  [(set_attr "op_type" "NN,RS")
+   (set_attr "atype"   "reg,*")
+   (set_attr "type"    "o3,*")
+   (set_attr "length"  "14,*")])
+
+(define_insn "set_tp_31"
+  [(unspec_volatile [(match_operand:SI 0 "general_operand" "d,Q")] UNSPECV_SET_TP)]
   "!TARGET_64BIT"
-  "basr\\t%3,%1"
-  [(set_attr "op_type" "RR")
-   (set_attr "type"    "jsr")
-   (set_attr "atype"   "mem")])
+  "@
+   sar\t%%a0,%0
+   lam\t%%a0,%%a0,%0"
+  [(set_attr "op_type" "RRE,RS")])
 
-(define_insn "bas_r_64"
-  [(set (match_operand 0 "register_operand" "=df")
-        (call (mem:QI (match_operand:QI 1 "address_operand" "p"))
-              (match_operand:SI 2 "const_int_operand" "n")))
-   (clobber (match_operand:DI 3 "register_operand" "=r"))]
+(define_insn "*tls_load_64"
+  [(set (match_operand:DI 0 "register_operand" "=d")
+        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
+                    (match_operand:DI 2 "" "")]
+		   UNSPEC_TLS_LOAD))]
   "TARGET_64BIT"
-  "bas\\t%3,%a1"
-  [(set_attr "op_type" "RX")
-   (set_attr "type"    "jsr")
-   (set_attr "atype"   "mem")])
+  "lg\t%0,%1%J2"
+  [(set_attr "op_type" "RXE")])
 
-(define_insn "bas_r_31"
-  [(set (match_operand 0 "register_operand" "=df")
-        (call (mem:QI (match_operand:QI 1 "address_operand" "p"))
-              (match_operand:SI 2 "const_int_operand" "n")))
-   (clobber (match_operand:SI 3 "register_operand" "=r"))]
+(define_insn "*tls_load_31"
+  [(set (match_operand:SI 0 "register_operand" "=d,d")
+        (unspec:SI [(match_operand:SI 1 "memory_operand" "R,T")
+                    (match_operand:SI 2 "" "")]
+		   UNSPEC_TLS_LOAD))]
   "!TARGET_64BIT"
-  "bas\\t%3,%a1"
-   [(set_attr "op_type" "RX")
-    (set_attr "type"    "jsr")
-    (set_attr "atype"   "mem")])
+  "@
+   l\t%0,%1%J2
+   ly\t%0,%1%J2"
+  [(set_attr "op_type" "RX,RXY")])
 
+(define_insn "*bras_tls"
+  [(set (match_operand 0 "" "")
+        (call (mem:QI (match_operand 1 "bras_sym_operand" "X"))
+              (match_operand 2 "const_int_operand" "n")))
+   (clobber (match_operand 3 "register_operand" "=r"))
+   (use (match_operand 4 "" ""))]
+  "!SIBLING_CALL_P (insn)
+   && TARGET_SMALL_EXEC
+   && GET_MODE (operands[3]) == Pmode"
+  "bras\t%3,%1%J4"
+  [(set_attr "op_type" "RI")
+   (set_attr "type"    "jsr")])
+
+(define_insn "*brasl_tls"
+  [(set (match_operand 0 "" "")
+        (call (mem:QI (match_operand 1 "bras_sym_operand" "X"))
+              (match_operand 2 "const_int_operand" "n")))
+   (clobber (match_operand 3 "register_operand" "=r"))
+   (use (match_operand 4 "" ""))]
+  "!SIBLING_CALL_P (insn)
+   && TARGET_CPU_ZARCH
+   && GET_MODE (operands[3]) == Pmode"
+  "brasl\t%3,%1%J4"
+  [(set_attr "op_type" "RIL")
+   (set_attr "type"    "jsr")])
+
+(define_insn "*basr_tls"
+  [(set (match_operand 0 "" "")
+        (call (mem:QI (match_operand 1 "address_operand" "U"))
+              (match_operand 2 "const_int_operand" "n")))
+   (clobber (match_operand 3 "register_operand" "=r"))
+   (use (match_operand 4 "" ""))]
+  "!SIBLING_CALL_P (insn) && GET_MODE (operands[3]) == Pmode"
+{
+  if (get_attr_op_type (insn) == OP_TYPE_RR)
+    return "basr\t%3,%1%J4";
+  else
+    return "bas\t%3,%a1%J4";
+}
+  [(set (attr "op_type")
+        (if_then_else (match_operand 1 "register_operand" "")
+                      (const_string "RR") (const_string "RX")))
+   (set_attr "type"  "jsr")
+   (set_attr "atype" "agen")])
 
 ;;
 ;;- Miscellaneous instructions.
@@ -6430,86 +7207,33 @@
 ;
 
 (define_expand "allocate_stack"
-  [(set (reg 15)
-        (plus (reg 15) (match_operand 1 "general_operand" "")))
-   (set (match_operand 0 "general_operand" "")
-        (reg 15))]
- ""
- "
+  [(match_operand 0 "general_operand" "")
+   (match_operand 1 "general_operand" "")]
+ "TARGET_BACKCHAIN || TARGET_KERNEL_BACKCHAIN"
 {
-    rtx stack = gen_rtx (REG, Pmode, STACK_POINTER_REGNUM);
-    rtx chain = gen_rtx (MEM, Pmode, stack);
-    rtx temp = gen_reg_rtx (Pmode);
-	
-    emit_move_insn (temp, chain);
-
-    if (TARGET_64BIT)
-      emit_insn (gen_adddi3 (stack, stack, negate_rtx (Pmode, operands[1])));
-    else
-      emit_insn (gen_addsi3 (stack, stack, negate_rtx (Pmode, operands[1])));
+  rtx temp = gen_reg_rtx (Pmode);
 
-    emit_move_insn (chain, temp);
+  emit_move_insn (temp, s390_back_chain_rtx ());
+  anti_adjust_stack (operands[1]);
+  emit_move_insn (s390_back_chain_rtx (), temp);
 
-    emit_move_insn (operands[0], virtual_stack_dynamic_rtx);	
-    DONE;
-}")
+  emit_move_insn (operands[0], virtual_stack_dynamic_rtx);
+  DONE;
+})
 
 
 ;
-; setjmp/longjmp instruction pattern(s).
+; setjmp instruction pattern.
 ;
 
-(define_expand "builtin_setjmp_setup"
-  [(unspec [(match_operand 0 "register_operand" "a")] 1)]
-  ""
-  "
-{
-  rtx base = gen_rtx_MEM (Pmode, plus_constant (operands[0], 4 * GET_MODE_SIZE (Pmode)));
-  rtx basereg = gen_rtx_REG (Pmode, BASE_REGISTER);
-
-  emit_move_insn (base, basereg);
-  DONE;
-}")
-
 (define_expand "builtin_setjmp_receiver"
-  [(unspec_volatile [(label_ref (match_operand 0 "" ""))] 2)]
+  [(match_operand 0 "" "")]
   "flag_pic"
-  "
 {
-  rtx gotreg = gen_rtx_REG (Pmode, PIC_OFFSET_TABLE_REGNUM);
-  rtx got = gen_rtx_SYMBOL_REF (Pmode, \"_GLOBAL_OFFSET_TABLE_\");
-  SYMBOL_REF_FLAG (got) = 1;
-
-  emit_move_insn (gotreg, got);
-  emit_insn (gen_rtx_USE (VOIDmode, gotreg));
+  emit_insn (s390_load_got ());
+  emit_insn (gen_rtx_USE (VOIDmode, pic_offset_table_rtx));
   DONE;
-}")
-
-(define_expand "builtin_longjmp"
-  [(unspec_volatile [(match_operand 0 "register_operand" "r")] 3)]
-  ""
-  "
-{
-  /* The elements of the buffer are, in order:  */
-  rtx fp = gen_rtx_MEM (Pmode, operands[0]);
-  rtx lab = gen_rtx_MEM (Pmode, plus_constant (operands[0], GET_MODE_SIZE (Pmode)));
-  rtx stack = gen_rtx_MEM (Pmode, plus_constant (operands[0], 2 * GET_MODE_SIZE (Pmode)));
-  rtx base = gen_rtx_MEM (Pmode, plus_constant (operands[0], 4 * GET_MODE_SIZE (Pmode)));
-  rtx basereg = gen_rtx_REG (Pmode, BASE_REGISTER);
-  rtx jmp = gen_rtx_REG (Pmode, 14);
-
-  emit_move_insn (jmp, lab);
-  emit_move_insn (basereg, base);
-  emit_stack_restore (SAVE_NONLOCAL, stack, NULL_RTX);
-  emit_move_insn (hard_frame_pointer_rtx, fp);
-
-  emit_insn (gen_rtx_USE (VOIDmode, hard_frame_pointer_rtx));
-  emit_insn (gen_rtx_USE (VOIDmode, stack_pointer_rtx));
-  emit_insn (gen_rtx_USE (VOIDmode, basereg));
-  emit_indirect_jump (jmp);
-  DONE;
-}")
-
+})
 
 ;; These patterns say how to save and restore the stack pointer.  We need not
 ;; save the stack pointer at function level since we are careful to
@@ -6533,54 +7257,66 @@
   "DONE;")
 
 (define_expand "restore_stack_block"
-  [(use (match_operand 0 "register_operand" ""))
-   (set (match_dup 2) (match_dup 3))
-   (set (match_dup 0) (match_operand 1 "register_operand" ""))
-   (set (match_dup 3) (match_dup 2))]
-  ""
-  "
+  [(match_operand 0 "register_operand" "")
+   (match_operand 1 "register_operand" "")]
+  "TARGET_BACKCHAIN || TARGET_KERNEL_BACKCHAIN"
 {
-  operands[2] = gen_reg_rtx (Pmode);
-  operands[3] = gen_rtx_MEM (Pmode, operands[0]);
-}")
+  rtx temp = gen_reg_rtx (Pmode);
+
+  emit_move_insn (temp, s390_back_chain_rtx ());
+  emit_move_insn (operands[0], operands[1]);
+  emit_move_insn (s390_back_chain_rtx (), temp);
+
+  DONE;
+})
 
 (define_expand "save_stack_nonlocal"
   [(match_operand 0 "memory_operand" "")
    (match_operand 1 "register_operand" "")]
   ""
-  "
 {
-  rtx temp = gen_reg_rtx (Pmode);
+  enum machine_mode mode = TARGET_64BIT ? OImode : TImode;
+  rtx base = gen_rtx_REG (Pmode, BASE_REGNUM);
+
+  /* Copy the backchain to the first word, sp to the second and the
+     literal pool base to the third.  */
+
+  if (TARGET_BACKCHAIN || TARGET_KERNEL_BACKCHAIN)
+    {
+      rtx temp = force_reg (Pmode, s390_back_chain_rtx ());
+      emit_move_insn (operand_subword (operands[0], 0, 0, mode), temp);
+    }
+
+  emit_move_insn (operand_subword (operands[0], 1, 0, mode), operands[1]);
+  emit_move_insn (operand_subword (operands[0], 2, 0, mode), base);
 
-  /* Copy the backchain to the first word, sp to the second.  */
-  emit_move_insn (temp, gen_rtx_MEM (Pmode, operands[1]));
-  emit_move_insn (operand_subword (operands[0], 0, 0,
-                 TARGET_64BIT ? TImode : DImode),
-                 temp);
-  emit_move_insn (operand_subword (operands[0], 1, 0,
-                 TARGET_64BIT ? TImode : DImode),
-                 operands[1]);
   DONE;
-}")
+})
 
 (define_expand "restore_stack_nonlocal"
   [(match_operand 0 "register_operand" "")
    (match_operand 1 "memory_operand" "")]
   ""
-  "
 {
-  rtx temp = gen_reg_rtx (Pmode);
+  enum machine_mode mode = TARGET_64BIT ? OImode : TImode;
+  rtx base = gen_rtx_REG (Pmode, BASE_REGNUM);
+  rtx temp = NULL_RTX;
+
+  /* Restore the backchain from the first word, sp from the second and the
+     literal pool base from the third.  */
 
-  /* Restore the backchain from the first word, sp from the second.  */
-  emit_move_insn (temp,
-                 operand_subword (operands[1], 0, 0,
-                 TARGET_64BIT ? TImode : DImode));
-  emit_move_insn (operands[0],
-                 operand_subword (operands[1], 1, 0,
-                 TARGET_64BIT ? TImode : DImode));
-  emit_move_insn (gen_rtx_MEM (Pmode, operands[0]), temp);
+  if (TARGET_BACKCHAIN || TARGET_KERNEL_BACKCHAIN)
+    temp = force_reg (Pmode, operand_subword (operands[1], 0, 0, mode));
+    
+  emit_move_insn (base, operand_subword (operands[1], 2, 0, mode));
+  emit_move_insn (operands[0], operand_subword (operands[1], 1, 0, mode));
+
+  if (temp)
+    emit_move_insn (s390_back_chain_rtx (), temp);
+
+  emit_insn (gen_rtx_USE (VOIDmode, base));
   DONE;
-}")
+})
 
 
 ;
@@ -6590,7 +7326,7 @@
 (define_insn "nop"
   [(const_int 0)]
   ""
-  "lr\\t0,0"
+  "lr\t0,0"
   [(set_attr "op_type" "RR")])
 
 
@@ -6598,133 +7334,105 @@
 ; Special literal pool access instruction pattern(s).
 ;
 
-(define_insn "consttable_qi"
-  [(unspec_volatile [(match_operand:QI 0 "consttable_operand" "X")] 200)]
+(define_insn "*pool_entry"
+  [(unspec_volatile [(match_operand 0 "consttable_operand" "X")]
+                    UNSPECV_POOL_ENTRY)]
   ""
-  "*
 {
-  assemble_integer (operands[0], 1, BITS_PER_UNIT, 1);
-  return \"\";
-}"
-  [(set_attr "op_type"  "NN")
-   (set_attr "length"   "1")])
-
-(define_insn "consttable_hi"
-  [(unspec_volatile [(match_operand:HI 0 "consttable_operand" "X")] 201)]
-  ""
-  "*
-{
-  assemble_integer (operands[0], 2, 2*BITS_PER_UNIT, 1);
-  return \"\";
-}"
-  [(set_attr "op_type"  "NN")
-   (set_attr "length"   "2")])
-
-(define_insn "consttable_si"
-  [(unspec_volatile [(match_operand:SI 0 "consttable_operand" "X")] 202)]
-  ""
-  "*
-{
-  if (!TARGET_64BIT && flag_pic && SYMBOLIC_CONST (operands[0]))
-    return \".long\\t%0\";
-
-  assemble_integer (operands[0], 4, 4*BITS_PER_UNIT, 1);
-  return \"\";
-}"
-  [(set_attr "op_type"  "NN")
-   (set_attr "length"   "4")])
-
-(define_insn "consttable_di"
-  [(unspec_volatile [(match_operand:DI 0 "consttable_operand" "X")] 203)]
-  ""
-  "*
-{
-  assemble_integer (operands[0], 8, 8*BITS_PER_UNIT, 1);
-  return \"\";
-}"
-  [(set_attr "op_type"  "NN")
-   (set_attr "length"   "8")])
-
-(define_insn "consttable_sf"
-  [(unspec_volatile [(match_operand:SF 0 "consttable_operand" "X")] 204)]
-  ""
-  "*
-{
-  REAL_VALUE_TYPE r;
-
-  if (GET_CODE (operands[0]) != CONST_DOUBLE)
-    abort ();
-
-  REAL_VALUE_FROM_CONST_DOUBLE (r, operands[0]);
-  assemble_real (r, SFmode, 4*BITS_PER_UNIT);
-  return \"\";
-}"
-  [(set_attr "op_type"  "NN")
-   (set_attr "length"   "4")])
-
-(define_insn "consttable_df"
-  [(unspec_volatile [(match_operand:DF 0 "consttable_operand" "X")] 205)]
-  ""
-  "*
-{
-  REAL_VALUE_TYPE r;
-
-  if (GET_CODE (operands[0]) != CONST_DOUBLE)
-    abort ();
-
-  REAL_VALUE_FROM_CONST_DOUBLE (r, operands[0]);
-  assemble_real (r, DFmode, 8*BITS_PER_UNIT);
-  return \"\";
-}"
-  [(set_attr "op_type"  "NN")
-   (set_attr "length"   "8")])
+  enum machine_mode mode = GET_MODE (PATTERN (insn));
+  unsigned int align = GET_MODE_BITSIZE (mode);
+  s390_output_pool_entry (operands[0], mode, align);
+  return "";
+}
+  [(set_attr "op_type" "NN")
+   (set (attr "length")
+        (symbol_ref "GET_MODE_SIZE (GET_MODE (PATTERN (insn)))"))])
 
 (define_insn "pool_start_31"
-  [(unspec_volatile [(const_int 0)] 206)]
-  "!TARGET_64BIT"
-  ".align\\t4"
+  [(unspec_volatile [(const_int 0)] UNSPECV_POOL_START)]
+  "!TARGET_CPU_ZARCH"
+  ".align\t4"
   [(set_attr "op_type"  "NN")
    (set_attr "length"   "2")])
 
 (define_insn "pool_end_31"
-  [(unspec_volatile [(const_int 0)] 207)]
-  "!TARGET_64BIT"
-  ".align\\t2"
+  [(unspec_volatile [(const_int 0)] UNSPECV_POOL_END)]
+  "!TARGET_CPU_ZARCH"
+  ".align\t2"
   [(set_attr "op_type"  "NN")
    (set_attr "length"   "2")])
 
 (define_insn "pool_start_64"
-  [(unspec_volatile [(const_int 0)] 206)]
-  "TARGET_64BIT"
-  ".section\\t.rodata\;.align\\t8"
+  [(unspec_volatile [(const_int 0)] UNSPECV_POOL_START)]
+  "TARGET_CPU_ZARCH"
+  ".section\t.rodata\;.align\t8"
   [(set_attr "op_type"  "NN")
    (set_attr "length"   "0")])
 
 (define_insn "pool_end_64"
-  [(unspec_volatile [(const_int 0)] 207)]
-  "TARGET_64BIT"
+  [(unspec_volatile [(const_int 0)] UNSPECV_POOL_END)]
+  "TARGET_CPU_ZARCH"
   ".previous"
   [(set_attr "op_type"  "NN")
    (set_attr "length"   "0")])
 
-(define_insn "reload_base"
-  [(set (match_operand:SI 0 "register_operand" "=a")
-        (unspec:SI [(label_ref (match_operand 1 "" ""))] 210))]
-  "!TARGET_64BIT"
-  "basr\\t%0,0\;la\\t%0,%1-.(%0)"
+(define_insn "main_base_31_small"
+  [(set (match_operand 0 "register_operand" "=a")
+        (unspec [(label_ref (match_operand 1 "" ""))] UNSPEC_MAIN_BASE))]
+  "!TARGET_CPU_ZARCH && GET_MODE (operands[0]) == Pmode"
+  "basr\t%0,0"
+  [(set_attr "op_type" "RR")
+   (set_attr "type"    "la")])
+
+(define_insn "main_base_31_large"
+  [(set (match_operand 0 "register_operand" "=a")
+        (unspec [(label_ref (match_operand 1 "" ""))] UNSPEC_MAIN_BASE))
+   (set (pc) (label_ref (match_operand 2 "" "")))]
+  "!TARGET_CPU_ZARCH && GET_MODE (operands[0]) == Pmode"
+  "bras\t%0,%2"
+  [(set_attr "op_type" "RI")])
+
+(define_insn "main_base_64"
+  [(set (match_operand 0 "register_operand" "=a")
+        (unspec [(label_ref (match_operand 1 "" ""))] UNSPEC_MAIN_BASE))]
+  "TARGET_CPU_ZARCH && GET_MODE (operands[0]) == Pmode"
+  "larl\t%0,%1"
+  [(set_attr "op_type" "RIL")
+   (set_attr "type"    "larl")])
+
+(define_insn "main_pool"
+  [(set (match_operand 0 "register_operand" "=a")
+        (unspec_volatile [(const_int 0)] UNSPECV_MAIN_POOL))]
+  "GET_MODE (operands[0]) == Pmode"
+  "* abort ();"
+  [(set_attr "op_type" "NN")
+   (set (attr "type") 
+        (if_then_else (ne (symbol_ref "TARGET_CPU_ZARCH") (const_int 0))
+                      (const_string "larl") (const_string "la")))])
+
+(define_insn "reload_base_31"
+  [(set (match_operand 0 "register_operand" "=a")
+        (unspec [(label_ref (match_operand 1 "" ""))] UNSPEC_RELOAD_BASE))]
+  "!TARGET_CPU_ZARCH && GET_MODE (operands[0]) == Pmode"
+  "basr\t%0,0\;la\t%0,%1-.(%0)"
   [(set_attr "op_type" "NN")
    (set_attr "type"    "la")
    (set_attr "length"  "6")])
 
-(define_insn "reload_base2"
-  [(set (match_operand:SI 0 "register_operand" "=a")
-        (unspec:SI [(label_ref (match_operand 1 "" ""))] 211))]
-  "!TARGET_64BIT"
-  "la\\t%0,%1-.(%0)"
-  [(set_attr "op_type" "NN")
-   (set_attr "type"    "la")
-   (set_attr "length"  "4")])
+(define_insn "reload_base_64"
+  [(set (match_operand 0 "register_operand" "=a")
+        (unspec [(label_ref (match_operand 1 "" ""))] UNSPEC_RELOAD_BASE))]
+  "TARGET_CPU_ZARCH && GET_MODE (operands[0]) == Pmode"
+  "larl\t%0,%1"
+  [(set_attr "op_type" "RIL")
+   (set_attr "type"    "larl")])
 
+(define_insn "pool"
+  [(unspec_volatile [(match_operand 0 "const_int_operand" "n")] UNSPECV_POOL)]
+  ""
+  "* abort ();"
+  [(set_attr "op_type" "NN")
+   (set (attr "length") (symbol_ref "INTVAL (operands[0])"))])
 
 ;;
 ;; Insns related to generating the function prologue and epilogue.
@@ -6734,196 +7442,67 @@
 (define_expand "prologue"
   [(use (const_int 0))]
   ""
-  "
-{
-      s390_emit_prologue ();
-      DONE;
-}")
+  "s390_emit_prologue (); DONE;")
+
+(define_insn "prologue_tpf"
+  [(unspec_volatile [(const_int 0)] UNSPECV_TPF_PROLOGUE)
+   (clobber (reg:DI 1))]
+  "TARGET_TPF_PROFILING"
+  "bas\t%%r1,4064"
+  [(set_attr "type" "jsr")
+   (set_attr "op_type" "RX")])
 
 (define_expand "epilogue"
   [(use (const_int 1))]
   ""
-  "
-{
-      s390_emit_epilogue ();
-      DONE;
-}")
+  "s390_emit_epilogue (false); DONE;")
 
+(define_insn "epilogue_tpf"
+  [(unspec_volatile [(const_int 0)] UNSPECV_TPF_EPILOGUE)
+   (clobber (reg:DI 1))]
+  "TARGET_TPF_PROFILING"
+  "bas\t%%r1,4070"
+  [(set_attr "type" "jsr")
+   (set_attr "op_type" "RX")])
 
-(define_insn "*return_si"
-  [(return)
-   (use (match_operand:SI 0 "register_operand" "a"))]
-  "!TARGET_64BIT"
-  "br\\t%0"
-  [(set_attr "op_type" "RR")
-   (set_attr "type"    "jsr")		
-   (set_attr "atype"   "mem")])
-
-(define_insn "*return_di"
-  [(return)
-   (use (match_operand:DI 0 "register_operand" "a"))]
-  "TARGET_64BIT"
-  "br\\t%0"
-  [(set_attr "op_type" "RR")
-   (set_attr "type"    "jsr")		
-   (set_attr "atype"   "mem")])
 
-
-(define_insn "lit"
-  [(set (reg 13) (pc))
-   (unspec_volatile [(const_int 0)] 200)]
+(define_expand "sibcall_epilogue"
+  [(use (const_int 0))]
   ""
-  "*
-{
-   s390_output_constant_pool (asm_out_file);
-   return \"\";
-}"
-  [(set_attr "op_type" "NN")
-   (set_attr "type"    "integer")])
+  "s390_emit_epilogue (true); DONE;")
 
+(define_insn "*return"
+  [(return)
+   (use (match_operand 0 "register_operand" "a"))]
+  "GET_MODE (operands[0]) == Pmode"
+  "br\t%0"
+  [(set_attr "op_type" "RR")
+   (set_attr "type"    "jsr")
+   (set_attr "atype"   "agen")])
 
-;;
-;; Peephole optimization patterns.
-;;
 
-(define_peephole
-  [(set (match_operand:SI 0 "memory_operand" "m")
-        (match_operand:SI 1 "register_operand" "d"))
-   (set (match_dup 1)
-        (match_dup 0))]
- ""
- "st\\t%1,%0")
-
-(define_peephole
-  [(set (match_operand:SI 0 "memory_operand" "m")
-        (match_operand:SI 1 "register_operand" "d"))
-   (set (match_dup 0)
-        (match_dup 1))]
- ""
- "st\\t%1,%0")
-
-(define_peephole
-  [(set (match_operand:SI 0 "register_operand" "")
-        (match_operand:SI 1 "register_operand" ""))
-   (parallel
-    [(set (match_dup 0)
-          (plus:SI (match_dup 0)
-                   (match_operand:SI 2 "immediate_operand" "")))
-     (clobber (reg:CC 33))])]
- "(REGNO (operands[0]) == STACK_POINTER_REGNUM ||
-   REGNO (operands[1]) == STACK_POINTER_REGNUM ||
-   REGNO (operands[0]) == BASE_REGISTER ||
-   REGNO (operands[1]) == BASE_REGISTER) &&
-  INTVAL (operands[2]) > 0 && INTVAL (operands[2]) < 4096"
- "la\\t%0,%c2(%1)")
-
-;
-; peepholes for fast char instructions
-;
-
-;(define_peephole
-;  [(set (match_operand:QI 0 "register_operand" "d")
-;        (match_operand:QI 1 "s_operand" "Q"))
-;  (set  (match_operand:SI 2 "register_operand" "0")
-;        (zero_extend:SI (match_dup 0)))]
-; "REGNO(operands[0]) == REGNO(operands[2])"
-; "icm\\t%0,8,%1\;srl\\t%0,24")
-
-;(define_peephole
-;  [(set (match_operand:QI 0 "register_operand" "d")
-;        (match_operand:QI 1 "s_operand" "Q"))
-;   (set (match_operand:SI 2 "register_operand" "0")
-;        (sign_extend:SI (match_dup 0)))]
-; "REGNO(operands[0]) == REGNO(operands[2])"
-; "icm\\t%0,8,%1\;sra\\t%0,24")
-
-(define_peephole
-  [(set (match_operand:QI 0 "register_operand" "d")
-        (match_operand:QI 1 "immediate_operand" "J"))
-   (set (match_operand:SI 2 "register_operand" "0" )
-        (sign_extend:SI (match_dup 0) ) )]
- "REGNO(operands[0]) == REGNO(operands[2])"
- "lhi\\t%0,%h1")
-
-;
-; peepholes for fast short instructions
-;
-
-;(define_peephole
-;  [(set (match_operand:HI 0 "register_operand" "d")
-;        (match_operand:HI 1 "s_operand" "Q"))
-;   (set (match_operand:SI 2 "register_operand" "0" )
-;        (zero_extend:SI (match_dup 0)))]
-; "REGNO(operands[0]) == REGNO(operands[2])"
-; "icm\\t%0,12,%1\;srl\\t%0,16")
-
-(define_peephole
-  [(set (match_operand:HI 0 "register_operand" "d")
-        (match_operand:HI 1 "memory_operand" "m"))
-   (set (match_operand:SI 2 "register_operand" "0" )
-        (sign_extend:SI (match_dup 0)))]
- "REGNO(operands[0]) == REGNO(operands[2])"
- "lh\\t%0,%1")
-
-(define_peephole
-  [(set (match_operand:HI 0 "register_operand" "d")
-        (match_operand:HI 1 "immediate_operand" "K"))
-   (set (match_operand:SI 2 "register_operand" "0" )
-        (sign_extend:SI (match_dup 0) ) )]
- "REGNO(operands[0]) == REGNO(operands[2])"
- "lhi\\t%0,%h1")
-
-;
-; peepholes for divide instructions
-;
-
-(define_peephole
-  [(set (match_operand:DI 0 "register_operand" "d")
-        (match_operand:DI 1 "memory_operand" "m"))
-   (set (match_dup 0)
-        (lshiftrt:DI (match_dup 0)
-        (match_operand:SI 2 "immediate_operand" "J")))
-   (set (match_dup 0)
-        (div:SI (match_dup 0)
-        (match_operand:SI 3 "nonimmediate_operand" "g")))
-   (set (match_dup 1)
-        (match_dup 0))]
-  ""
-  "*
-{
-   output_asm_insn (\"l\\t%0,%1\", operands);
-   output_asm_insn (\"srdl\\t%0,%b2\", operands);
+;; Instruction definition to extend a 31-bit pointer into a 64-bit
+;; pointer. This is used for compatibility.
 
-   if (REG_P (operands[3]))
-     output_asm_insn (\"dr\\t%0,%3\", operands);
-   else
-     output_asm_insn (\"d\\t%0,%3\", operands);
-
-   return \"st\\t%N0,%N1\";
-}")
-
-(define_peephole
-  [(set (match_operand:DI 0 "register_operand" "d")
-        (match_operand:DI 1 "memory_operand" "m"))
-   (set (match_dup 0)
-        (lshiftrt:DI (match_dup 0)
-        (match_operand:SI 2 "immediate_operand" "J")))
-   (set (match_dup 0)
-        (mod:SI (match_dup 0)
-        (match_operand:SI 3 "nonimmediate_operand" "g")))
-   (set (match_dup 1)
-        (match_dup 0))]
-  ""
-  "*
+(define_expand "ptr_extend"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+        (match_operand:SI 1 "register_operand" "r"))]
+  "TARGET_64BIT"
 {
-   output_asm_insn (\"l\\t%0,%1\", operands);
-   output_asm_insn (\"srdl\\t%0,%b2\", operands);
+  emit_insn (gen_anddi3 (operands[0],
+			 gen_lowpart (DImode, operands[1]),
+			 GEN_INT (0x7fffffff)));
+  DONE;
+})
 
-   if (REG_P (operands[3]))
-     output_asm_insn (\"dr\\t%0,%3\", operands);
-   else
-     output_asm_insn (\"d\\t%0,%3\", operands);
+;; Instruction definition to expand eh_return macro to support
+;; swapping in special linkage return addresses.
 
-   return \"st\\t%0,%1\";
-}")
+(define_expand "eh_return"
+  [(use (match_operand 0 "register_operand" ""))]
+  "TARGET_TPF"
+{
+  s390_emit_tpf_eh_return (operands[0]);
+  DONE;
+})