OSDN Git Service

* builtins.c, c-pragma.h, c-typeck.c, cgraph.c, cgraphunit.c,
[pf3gnuchains/gcc-fork.git] / gcc / config / s390 / s390.md
index bc9e4c3..c03f880 100644 (file)
@@ -1,5 +1,5 @@
 ;;- Machine description for GNU compiler -- S/390 / zSeries version.
-;;  Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005
+;;  Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006
 ;;  Free Software Foundation, Inc.
 ;;  Contributed by Hartmut Penner (hpenner@de.ibm.com) and
 ;;                 Ulrich Weigand (uweigand@de.ibm.com).
 (define_attr "type" "none,integer,load,lr,la,larl,lm,stm,
                     cs,vs,store,sem,idiv,
                      imulhi,imulsi,imuldi,
-                    branch,jsr,fsimpdf,fsimpsf,
-                    floaddf,floadsf,fstoredf,fstoresf,
-                    fmuldf,fmulsf,fdivdf,fdivsf,
-                    ftoi,itof,fsqrtdf,fsqrtsf,
-                     other"
+                    branch,jsr,fsimptf,fsimpdf,fsimpsf,
+                    floadtf,floaddf,floadsf,fstoredf,fstoresf,
+                    fmultf,fmuldf,fmulsf,fdivtf,fdivdf,fdivsf,
+                    ftoi,itof,fsqrttf,fsqrtdf,fsqrtsf,
+                     ftrunctf,ftruncdf,other"
   (cond [(eq_attr "op_type" "NN")  (const_string "other")
          (eq_attr "op_type" "SS")  (const_string "cs")]
     (const_string "integer")))
 ;;   reg: Instruction does not use the agen unit
 
 (define_attr "atype" "agen,reg"
-  (cond [(eq_attr "op_type" "E")   (const_string "reg")
-         (eq_attr "op_type" "RR")  (const_string "reg")
-         (eq_attr "op_type" "RX")  (const_string "agen")
-         (eq_attr "op_type" "RI")  (const_string "reg")
-         (eq_attr "op_type" "RRE") (const_string "reg")
-         (eq_attr "op_type" "RS")  (const_string "agen")
-         (eq_attr "op_type" "RSI") (const_string "agen")
-         (eq_attr "op_type" "S")   (const_string "agen")
-         (eq_attr "op_type" "SI")  (const_string "agen")
-         (eq_attr "op_type" "SS")  (const_string "agen")
-         (eq_attr "op_type" "SSE") (const_string "agen")
-         (eq_attr "op_type" "RXE") (const_string "agen")
-         (eq_attr "op_type" "RSE") (const_string "agen")
-         (eq_attr "op_type" "RIL") (const_string "agen")
-         (eq_attr "op_type" "RXY") (const_string "agen")
-         (eq_attr "op_type" "RSY") (const_string "agen")
-         (eq_attr "op_type" "SIY") (const_string "agen")]
-    (const_string "agen")))
+  (if_then_else (eq_attr "op_type" "E,RR,RI,RRE")  
+               (const_string "reg")
+               (const_string "agen")))
 
 ;; Length in bytes.
 
 (define_attr "length" ""
-  (cond [(eq_attr "op_type" "E")   (const_int 2)
-         (eq_attr "op_type" "RR")  (const_int 2)
-         (eq_attr "op_type" "RX")  (const_int 4)
-         (eq_attr "op_type" "RI")  (const_int 4)
-         (eq_attr "op_type" "RRE") (const_int 4)
-         (eq_attr "op_type" "RS")  (const_int 4)
-         (eq_attr "op_type" "RSI") (const_int 4)
-         (eq_attr "op_type" "S")   (const_int 4)
-         (eq_attr "op_type" "SI")  (const_int 4)
-         (eq_attr "op_type" "SS")  (const_int 6)
-         (eq_attr "op_type" "SSE") (const_int 6)
-         (eq_attr "op_type" "RXE") (const_int 6)
-         (eq_attr "op_type" "RSE") (const_int 6)
-         (eq_attr "op_type" "RIL") (const_int 6)
-         (eq_attr "op_type" "RXY") (const_int 6)
-         (eq_attr "op_type" "RSY") (const_int 6)
-         (eq_attr "op_type" "SIY") (const_int 6)]
+  (cond [(eq_attr "op_type" "E,RR")                  (const_int 2)
+         (eq_attr "op_type" "RX,RI,RRE,RS,RSI,S,SI")  (const_int 4)]
     (const_int 6)))
 
 
 
 ;; Macros
 
-;; This mode macro allows DF and SF patterns to be generated from the
+;; This mode macro allows floating point patterns to be generated from the
 ;; same template.
-(define_mode_macro FPR     [DF SF])
+(define_mode_macro FPR [TF DF SF])
+(define_mode_macro DSF [DF SF])
 
 ;; These mode macros allow 31-bit and 64-bit TDSI patterns to be generated
 ;; from the same template.
 ;; the same template.
 (define_code_macro SHIFT [ashift lshiftrt])
 
+;; These macros allow to combine most atomic operations.
+(define_code_macro ATOMIC [and ior xor plus minus mult])
+(define_code_attr atomic [(and "and") (ior "ior") (xor "xor") 
+                         (plus "add") (minus "sub") (mult "nand")])
 
-;; In FPR templates, a string like "lt<de>br" will expand to "ltdbr" in DFmode
-;; and "ltebr" in SFmode.
-(define_mode_attr de [(DF "d") (SF "e")])
 
-;; In FPR templates, a string like "m<dee>br" will expand to "mdbr" in DFmode
-;; and "meebr" in SFmode.  This is needed for the 'mul<mode>3' pattern. 
-(define_mode_attr dee [(DF "d") (SF "ee")])
+;; In FPR templates, a string like "lt<de>br" will expand to "ltxbr" in TFmode,
+;; "ltdbr" in DFmode, and "ltebr" in SFmode.
+(define_mode_attr xde [(TF "x") (DF "d") (SF "e")])
 
+;; In FPR templates, a string like "m<dee>br" will expand to "mxbr" in TFmode,
+;; "mdbr" in DFmode, and "meebr" in SFmode.
+(define_mode_attr xdee [(TF "x") (DF "d") (SF "ee")])
+
+;; In FPR templates, "<RRe>" will expand to "RRE" in TFmode and "RR" otherwise.
+;; Likewise for "<RXe>".
+(define_mode_attr RRe [(TF "RRE") (DF "RR") (SF "RR")])
+(define_mode_attr RXe [(TF "RXE") (DF "RX") (SF "RX")])
+
+;; In FPR templates, "<Rf>" will expand to "f" in TFmode and "R" otherwise.
+;; This is used to disable the memory alternative in TFmode patterns.
+(define_mode_attr Rf [(TF "f") (DF "R") (SF "R")])
+
+;; In GPR and P templates, a constraint like "<d0>" will expand to "d" in DImode
+;; and "0" in SImode. This allows to combine instructions of which the 31bit
+;; version only operates on one register.
+(define_mode_attr d0 [(DI "d") (SI "0")])
+
+;; In combination with d0 this allows to combine instructions of which the 31bit
+;; version only operates on one register. The DImode version needs an additional
+;; register for the assembler output.
+(define_mode_attr 1 [(DI "%1,") (SI "")])
+  
 ;; In SHIFT templates, a string like "s<lr>dl" will expand to "sldl" in 
 ;; 'ashift' and "srdl" in 'lshiftrt'.
 (define_code_attr lr [(ashift "l") (lshiftrt "r")])
 ;; in "RRE" for DImode and "RR" for SImode.
 (define_mode_attr E [(DI "E") (SI "")])
 
+;; This attribute handles differences in the instruction 'type' and makes RX<Y>
+;; to result in "RXY" for DImode and "RX" for SImode.
+(define_mode_attr Y [(DI "Y") (SI "")])
+
 ;; This attribute handles differences in the instruction 'type' and will result
 ;; in "RSE" for TImode and "RS" for DImode.
 (define_mode_attr TE [(TI "E") (DI "")])
 ;; and "lcr" in SImode.
 (define_mode_attr g [(DI "g") (SI "")])
 
+;; In GPR templates, a string like "sl<y>" will expand to "slg" in DImode
+;; and "sly" in SImode. This is useful because on 64bit the ..g instructions
+;; were enhanced with long displacements whereas 31bit instructions got a ..y
+;; variant for long displacements.
+(define_mode_attr y [(DI "g") (SI "y")])
+
 ;; In DP templates, a string like "cds<g>" will expand to "cdsg" in TImode
 ;; and "cds" in DImode.
 (define_mode_attr tg [(TI "g") (DI "")])
   [(set_attr "op_type" "RI")])
 
 
+;
 ; Load-and-Test instructions
+;
+
+; tst(di|si) instruction pattern(s).
 
 (define_insn "*tstdi_sign"
   [(set (reg CC_REGNUM)
   "ltgfr\t%2,%0"
   [(set_attr "op_type" "RRE")])
 
-(define_insn "*tstdi_extimm"
+(define_insn "*tst<mode>_extimm"
   [(set (reg CC_REGNUM)
-        (compare (match_operand:DI 0 "nonimmediate_operand" "d,m")
-                 (match_operand:DI 1 "const0_operand" "")))
-   (set (match_operand:DI 2 "register_operand" "=d,d")
+        (compare (match_operand:GPR 0 "nonimmediate_operand" "d,m")
+                 (match_operand:GPR 1 "const0_operand" "")))
+   (set (match_operand:GPR 2 "register_operand" "=d,d")
         (match_dup 0))]
-  "s390_match_ccmode(insn, CCSmode) && TARGET_64BIT && TARGET_EXTIMM"
+  "s390_match_ccmode(insn, CCSmode) && TARGET_EXTIMM"
   "@
-   ltgr\t%2,%0
-   ltg\t%2,%0"
-  [(set_attr "op_type" "RRE,RXY")])
+   lt<g>r\t%2,%0
+   lt<g>\t%2,%0"
+  [(set_attr "op_type" "RR<E>,RXY")])
 
-(define_insn "*tstdi_cconly_extimm"
+(define_insn "*tst<mode>_cconly_extimm"
   [(set (reg CC_REGNUM)
-        (compare (match_operand:DI 0 "nonimmediate_operand" "d,m")
-                 (match_operand:DI 1 "const0_operand" "")))
-   (clobber (match_scratch:DI 2 "=X,d"))]
-  "s390_match_ccmode(insn, CCSmode) && TARGET_64BIT && TARGET_EXTIMM"
+        (compare (match_operand:GPR 0 "nonimmediate_operand" "d,m")
+                 (match_operand:GPR 1 "const0_operand" "")))
+   (clobber (match_scratch:GPR 2 "=X,d"))]
+  "s390_match_ccmode(insn, CCSmode) && TARGET_EXTIMM"
   "@
-   ltgr\t%0,%0
-   ltg\t%2,%0"
-  [(set_attr "op_type" "RRE,RXY")])
+   lt<g>r\t%0,%0
+   lt<g>\t%2,%0"
+  [(set_attr "op_type" "RR<E>,RXY")])
 
 (define_insn "*tstdi"
   [(set (reg CC_REGNUM)
   "ltgr\t%2,%0"
   [(set_attr "op_type" "RRE")])
 
-(define_insn "*tstdi_cconly"
-  [(set (reg CC_REGNUM)
-        (compare (match_operand:DI 0 "register_operand" "d")
-                 (match_operand:DI 1 "const0_operand" "")))]
-  "s390_match_ccmode(insn, CCSmode) && TARGET_64BIT"
-  "ltgr\t%0,%0"
-  [(set_attr "op_type" "RRE")])
-
-(define_insn "*tstdi_cconly_31"
-  [(set (reg CC_REGNUM)
-        (compare (match_operand:DI 0 "register_operand" "d")
-                 (match_operand:DI 1 "const0_operand" "")))]
-  "s390_match_ccmode(insn, CCSmode) && !TARGET_64BIT"
-  "srda\t%0,0"
-  [(set_attr "op_type" "RS")
-   (set_attr "atype"   "reg")])
-
-(define_insn "*tstsi_extimm"
-  [(set (reg CC_REGNUM)
-        (compare (match_operand:SI 0 "nonimmediate_operand" "d,m")
-                 (match_operand:SI 1 "const0_operand" "")))
-   (set (match_operand:SI 2 "register_operand" "=d,d")
-        (match_dup 0))]
-  "s390_match_ccmode(insn, CCSmode) && TARGET_EXTIMM"
-  "@
-   ltr\t%2,%0
-   lt\t%2,%0"
-  [(set_attr "op_type" "RR,RXY")])
-
-(define_insn "*tstsi_cconly_extimm"
-  [(set (reg CC_REGNUM)
-        (compare (match_operand:SI 0 "nonimmediate_operand" "d,m")
-                 (match_operand:SI 1 "const0_operand" "")))
-   (clobber (match_scratch:SI 2 "=X,d"))]
-  "s390_match_ccmode(insn, CCSmode) && TARGET_EXTIMM"
-  "@
-   ltr\t%0,%0
-   lt\t%2,%0"
-  [(set_attr "op_type" "RR,RXY")])
-
 (define_insn "*tstsi"
   [(set (reg CC_REGNUM)
         (compare (match_operand:SI 0 "nonimmediate_operand" "d,Q,S")
    icmy\t%2,15,%S0"
   [(set_attr "op_type" "RR,RS,RSY")])
 
-(define_insn "*tstsi_cconly2"
+(define_insn "*tstdi_cconly_31"
   [(set (reg CC_REGNUM)
-        (compare (match_operand:SI 0 "register_operand" "d")
-                 (match_operand:SI 1 "const0_operand" "")))]
+        (compare (match_operand:DI 0 "register_operand" "d")
+                 (match_operand:DI 1 "const0_operand" "")))]
+  "s390_match_ccmode(insn, CCSmode) && !TARGET_64BIT"
+  "srda\t%0,0"
+  [(set_attr "op_type" "RS")
+   (set_attr "atype"   "reg")])
+
+(define_insn "*tst<mode>_cconly2"
+  [(set (reg CC_REGNUM)
+        (compare (match_operand:GPR 0 "register_operand" "d")
+                 (match_operand:GPR 1 "const0_operand" "")))]
   "s390_match_ccmode(insn, CCSmode)"
-  "ltr\t%0,%0"
-  [(set_attr "op_type" "RR")])
+  "lt<g>r\t%0,%0"
+  [(set_attr "op_type" "RR<E>")])
+
+; tst(hi|qi) instruction pattern(s).
 
 (define_insn "*tst<mode>CCT"
   [(set (reg CC_REGNUM)
    cgf\t%0,%1"
   [(set_attr "op_type" "RRE,RXY")])
 
-(define_insn "*cmpdi_ccs"
-  [(set (reg CC_REGNUM)
-       (compare (match_operand:DI 0 "register_operand" "d,d,d,d")
-                (match_operand:DI 1 "general_operand" "d,K,Os,m")))]
-  "s390_match_ccmode(insn, CCSmode) && TARGET_64BIT"
-  "@
-   cgr\t%0,%1
-   cghi\t%0,%h1
-   cgfi\t%0,%1
-   cg\t%0,%1"
-  [(set_attr "op_type" "RRE,RI,RIL,RXY")])
-
 (define_insn "*cmpsi_ccs_sign"
   [(set (reg CC_REGNUM)
         (compare (sign_extend:SI (match_operand:HI 1 "memory_operand" "R,T"))
    chy\t%0,%1"
   [(set_attr "op_type" "RX,RXY")])
 
-(define_insn "*cmpsi_ccs"
+(define_insn "*cmp<mode>_ccs"
   [(set (reg CC_REGNUM)
-        (compare (match_operand:SI 0 "register_operand" "d,d,d,d,d")
-                 (match_operand:SI 1 "general_operand" "d,K,Os,R,T")))]
+        (compare (match_operand:GPR 0 "register_operand" "d,d,d,d,d")
+                 (match_operand:GPR 1 "general_operand" "d,K,Os,R,T")))]
   "s390_match_ccmode(insn, CCSmode)"
   "@
-   cr\t%0,%1
-   chi\t%0,%h1
-   cfi\t%0,%1
-   c\t%0,%1
-   cy\t%0,%1"
-  [(set_attr "op_type" "RR,RI,RIL,RX,RXY")])
+   c<g>r\t%0,%1
+   c<g>hi\t%0,%h1
+   c<g>fi\t%0,%1
+   c<g>\t%0,%1
+   c<y>\t%0,%1"
+  [(set_attr "op_type" "RR<E>,RI,RIL,RX<Y>,RXY")])
 
 
 ; Compare (unsigned) instructions
         (compare (match_operand:FPR 0 "register_operand" "f")
                  (match_operand:FPR 1 "const0_operand" "")))]
   "s390_match_ccmode(insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "lt<de>br\t%0,%0"
+  "lt<xde>br\t%0,%0"
    [(set_attr "op_type" "RRE")
     (set_attr "type"  "fsimp<mode>")])
 
         (compare (match_operand:FPR 0 "register_operand" "f")
                  (match_operand:FPR 1 "const0_operand" "")))]
   "s390_match_ccmode(insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
-  "lt<de>r\t%0,%0"
-   [(set_attr "op_type" "RR")
+  "lt<xde>r\t%0,%0"
+   [(set_attr "op_type" "<RRe>")
     (set_attr "type"  "fsimp<mode>")])
 
 (define_insn "*cmp<mode>_ccs"
   [(set (reg CC_REGNUM)
         (compare (match_operand:FPR 0 "register_operand" "f,f")
-                 (match_operand:FPR 1 "general_operand" "f,R")))]
+                 (match_operand:FPR 1 "general_operand" "f,<Rf>")))]
   "s390_match_ccmode(insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   c<de>br\t%0,%1
-   c<de>b\t%0,%1"
+   c<xde>br\t%0,%1
+   c<xde>b\t%0,%1"
    [(set_attr "op_type" "RRE,RXE")
     (set_attr "type"  "fsimp<mode>")])
 
 (define_insn "*cmp<mode>_ccs_ibm"
   [(set (reg CC_REGNUM)
         (compare (match_operand:FPR 0 "register_operand" "f,f")
-                 (match_operand:FPR 1 "general_operand" "f,R")))]
+                 (match_operand:FPR 1 "general_operand" "f,<Rf>")))]
   "s390_match_ccmode(insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
   "@
-   c<de>r\t%0,%1
-   c<de>\t%0,%1"
-   [(set_attr "op_type" "RR,RX")
+   c<xde>r\t%0,%1
+   c<xde>\t%0,%1"
+   [(set_attr "op_type" "<RRe>,<RXe>")
     (set_attr "type"  "fsimp<mode>")])
 
 
    (set_attr "type" "lr,load,load,*")])
 
 ;
+; movtf instruction pattern(s).
+;
+
+(define_expand "movtf"
+  [(set (match_operand:TF 0 "nonimmediate_operand" "")
+        (match_operand:TF 1 "general_operand"       ""))]
+  ""
+  "")
+
+(define_insn "*movtf_64"
+  [(set (match_operand:TF 0 "nonimmediate_operand" "=f,f,f,o,d,QS,d,o,Q")
+        (match_operand:TF 1 "general_operand"       "G,f,o,f,QS,d,dm,d,Q"))]
+  "TARGET_64BIT"
+  "@
+   lzxr\t%0
+   lxr\t%0,%1
+   #
+   #
+   lmg\t%0,%N0,%S1
+   stmg\t%1,%N1,%S0
+   #
+   #
+   #"
+  [(set_attr "op_type" "RRE,RRE,*,*,RSY,RSY,*,*,*")
+   (set_attr "type"    "fsimptf,fsimptf,*,*,lm,stm,*,*,*")])
+
+(define_insn "*movtf_31"
+  [(set (match_operand:TF 0 "nonimmediate_operand" "=f,f,f,o,Q")
+        (match_operand:TF 1 "general_operand"       "G,f,o,f,Q"))]
+  "!TARGET_64BIT"
+  "@
+   lzxr\t%0
+   lxr\t%0,%1
+   #
+   #
+   #"
+  [(set_attr "op_type" "RRE,RRE,*,*,*")
+   (set_attr "type"    "fsimptf,fsimptf,*,*,*")])
+
+; TFmode in GPRs splitters
+
+(define_split
+  [(set (match_operand:TF 0 "nonimmediate_operand" "")
+        (match_operand:TF 1 "general_operand" ""))]
+  "TARGET_64BIT && reload_completed
+   && s390_split_ok_p (operands[0], operands[1], TFmode, 0)"
+  [(set (match_dup 2) (match_dup 4))
+   (set (match_dup 3) (match_dup 5))]
+{
+  operands[2] = operand_subword (operands[0], 0, 0, TFmode);
+  operands[3] = operand_subword (operands[0], 1, 0, TFmode);
+  operands[4] = operand_subword (operands[1], 0, 0, TFmode);
+  operands[5] = operand_subword (operands[1], 1, 0, TFmode);
+})
+
+(define_split
+  [(set (match_operand:TF 0 "nonimmediate_operand" "")
+        (match_operand:TF 1 "general_operand" ""))]
+  "TARGET_64BIT && reload_completed
+   && s390_split_ok_p (operands[0], operands[1], TFmode, 1)"
+  [(set (match_dup 2) (match_dup 4))
+   (set (match_dup 3) (match_dup 5))]
+{
+  operands[2] = operand_subword (operands[0], 1, 0, TFmode);
+  operands[3] = operand_subword (operands[0], 0, 0, TFmode);
+  operands[4] = operand_subword (operands[1], 1, 0, TFmode);
+  operands[5] = operand_subword (operands[1], 0, 0, TFmode);
+})
+
+(define_split
+  [(set (match_operand:TF 0 "register_operand" "")
+        (match_operand:TF 1 "memory_operand" ""))]
+  "TARGET_64BIT && reload_completed
+   && !FP_REG_P (operands[0])
+   && !s_operand (operands[1], VOIDmode)"
+  [(set (match_dup 0) (match_dup 1))]
+{
+  rtx addr = operand_subword (operands[0], 1, 0, DFmode);
+  s390_load_address (addr, XEXP (operands[1], 0));
+  operands[1] = replace_equiv_address (operands[1], addr);
+})
+
+; TFmode in FPRs splitters
+
+(define_split
+  [(set (match_operand:TF 0 "register_operand" "")
+        (match_operand:TF 1 "memory_operand" ""))]
+  "reload_completed && offsettable_memref_p (operands[1]) 
+   && FP_REG_P (operands[0])"
+  [(set (match_dup 2) (match_dup 4))
+   (set (match_dup 3) (match_dup 5))]
+{
+  operands[2] = simplify_gen_subreg (DFmode, operands[0], TFmode, 0);
+  operands[3] = simplify_gen_subreg (DFmode, operands[0], TFmode, 8);
+  operands[4] = adjust_address_nv (operands[1], DFmode, 0);
+  operands[5] = adjust_address_nv (operands[1], DFmode, 8);
+})
+
+(define_split
+  [(set (match_operand:TF 0 "memory_operand" "")
+        (match_operand:TF 1 "register_operand" ""))]
+  "reload_completed && offsettable_memref_p (operands[0])
+   && FP_REG_P (operands[1])"
+  [(set (match_dup 2) (match_dup 4))
+   (set (match_dup 3) (match_dup 5))]
+{
+  operands[2] = adjust_address_nv (operands[0], DFmode, 0);
+  operands[3] = adjust_address_nv (operands[0], DFmode, 8);
+  operands[4] = simplify_gen_subreg (DFmode, operands[1], TFmode, 0);
+  operands[5] = simplify_gen_subreg (DFmode, operands[1], TFmode, 8);
+})
+
+(define_expand "reload_outtf"
+  [(parallel [(match_operand:TF 0 "" "")
+              (match_operand:TF 1 "register_operand" "f")
+              (match_operand:SI 2 "register_operand" "=&a")])]
+  ""
+{
+  rtx addr = gen_lowpart (Pmode, operands[2]);
+
+  gcc_assert (MEM_P (operands[0]));
+  s390_load_address (addr, find_replacement (&XEXP (operands[0], 0)));
+  operands[0] = replace_equiv_address (operands[0], addr);
+  emit_move_insn (operands[0], operands[1]);
+  DONE;
+})
+
+(define_expand "reload_intf"
+  [(parallel [(match_operand:TF 0 "register_operand" "=f")
+              (match_operand:TF 1 "" "")
+              (match_operand:SI 2 "register_operand" "=&a")])]
+  ""
+{
+  rtx addr = gen_lowpart (Pmode, operands[2]);
+  gcc_assert (MEM_P (operands[1]));
+  s390_load_address (addr, find_replacement (&XEXP (operands[1], 0)));
+  operands[1] = replace_equiv_address (operands[1], addr);
+  emit_move_insn (operands[0], operands[1]);
+  DONE;
+})
+
+;
 ; movdf instruction pattern(s).
 ;
 
   [(parallel
     [(clobber (match_dup 1))
      (set (match_operand:BLK 0 "memory_operand" "")
-          (match_operand 2 "setmem_operand" ""))
+          (match_operand 2 "shift_count_or_setmem_operand" ""))
      (use (match_operand 1 "general_operand" ""))
      (use (match_dup 3))
      (clobber (reg:CC CC_REGNUM))])]
 (define_insn "*setmem_long"
   [(clobber (match_operand:<DBL> 0 "register_operand" "=d"))
    (set (mem:BLK (subreg:P (match_operand:<DBL> 3 "register_operand" "0") 0))
-        (match_operand 2 "setmem_operand" "Y"))
+        (match_operand 2 "shift_count_or_setmem_operand" "Y"))
    (use (match_dup 3))
    (use (match_operand:<DBL> 1 "register_operand" "d"))
    (clobber (reg:CC CC_REGNUM))]
   [(set_attr "length" "8")
    (set_attr "type" "vs")])
 
+(define_insn "*setmem_long_and"
+  [(clobber (match_operand:<DBL> 0 "register_operand" "=d"))
+   (set (mem:BLK (subreg:P (match_operand:<DBL> 3 "register_operand" "0") 0))
+        (and (match_operand 2 "shift_count_or_setmem_operand" "Y")
+            (match_operand 4 "const_int_operand"             "n")))
+   (use (match_dup 3))
+   (use (match_operand:<DBL> 1 "register_operand" "d"))
+   (clobber (reg:CC CC_REGNUM))]
+  "(INTVAL (operands[4]) & 255) == 255"
+  "mvcle\t%0,%1,%Y2\;jo\t.-4"
+  [(set_attr "length" "8")
+   (set_attr "type" "vs")])
 ;
 ; cmpmemM instruction pattern(s).
 ;
   [(set (zero_extract:P (match_operand:P 0 "register_operand" "+d")
                        (const_int 16)
                        (match_operand 1 "const_int_operand" "n"))
-       (match_operand:P 2 "const_int_operand" "K"))]
+       (match_operand:P 2 "const_int_operand" "n"))]
   "TARGET_ZARCH
    && INTVAL (operands[1]) >= 0
    && INTVAL (operands[1]) < BITS_PER_WORD
   [(set (zero_extract:P (match_operand:P 0 "register_operand" "+d")
                        (const_int 32)
                        (match_operand 1 "const_int_operand" "n"))
-       (match_operand:P 2 "const_int_operand" "Os"))]
+       (match_operand:P 2 "const_int_operand" "n"))]
   "TARGET_EXTIMM
    && INTVAL (operands[1]) >= 0
    && INTVAL (operands[1]) < BITS_PER_WORD
   [(set_attr "op_type" "RRE,RXY")])
 
 ;
-; extend(hi|qi)di2 instruction pattern(s).
+; extend(hi|qi)(si|di)2 instruction pattern(s).
 ;
 
-(define_expand "extend<mode>di2"
-  [(set (match_operand:DI 0 "register_operand" "")
-        (sign_extend:DI (match_operand:HQI 1 "nonimmediate_operand" "")))]
+(define_expand "extend<HQI:mode><DSI:mode>2"
+  [(set (match_operand:DSI 0 "register_operand" "")
+        (sign_extend:DSI (match_operand:HQI 1 "nonimmediate_operand" "")))]
   ""
 {
-  if (!TARGET_64BIT)
+  if (<DSI:MODE>mode == DImode && !TARGET_64BIT)
     {
       rtx tmp = gen_reg_rtx (SImode);
-      emit_insn (gen_extend<mode>si2 (tmp, operands[1]));
+      emit_insn (gen_extend<HQI:mode>si2 (tmp, operands[1]));
       emit_insn (gen_extendsidi2 (operands[0], tmp));
       DONE;
     }
   else if (!TARGET_EXTIMM)
     {
-      rtx bitcount = GEN_INT (GET_MODE_BITSIZE (DImode) - 
-                             GET_MODE_BITSIZE (<MODE>mode));
-      operands[1] = gen_lowpart (DImode, operands[1]);
-      emit_insn (gen_ashldi3 (operands[0], operands[1], bitcount));
-      emit_insn (gen_ashrdi3 (operands[0], operands[0], bitcount));
+      rtx bitcount = GEN_INT (GET_MODE_BITSIZE (<DSI:MODE>mode) -
+                             GET_MODE_BITSIZE (<HQI:MODE>mode));
+
+      operands[1] = gen_lowpart (<DSI:MODE>mode, operands[1]);
+      emit_insn (gen_ashl<DSI:mode>3 (operands[0], operands[1], bitcount));
+      emit_insn (gen_ashr<DSI:mode>3 (operands[0], operands[0], bitcount));
       DONE;
     }
 })
 
+;
+; extendhidi2 instruction pattern(s).
+;
+
 (define_insn "*extendhidi2_extimm"
   [(set (match_operand:DI 0 "register_operand" "=d,d")
         (sign_extend:DI (match_operand:HI 1 "nonimmediate_operand" "d,m")))]
   "lgh\t%0,%1"
   [(set_attr "op_type" "RXY")])
 
-(define_insn "*extendqidi2_extimm"
-  [(set (match_operand:DI 0 "register_operand" "=d,d")
-        (sign_extend:DI (match_operand:QI 1 "nonimmediate_operand" "d,m")))]
-  "TARGET_64BIT && TARGET_EXTIMM"
-  "@
-   lgbr\t%0,%1
-   lgb\t%0,%1"
-  [(set_attr "op_type" "RRE,RXY")])
-
-(define_insn "*extendqidi2"
-  [(set (match_operand:DI 0 "register_operand" "=d")
-        (sign_extend:DI (match_operand:QI 1 "memory_operand" "m")))]
-  "TARGET_64BIT && TARGET_LONG_DISPLACEMENT"
-  "lgb\t%0,%1"
-  [(set_attr "op_type" "RXY")])
-
-(define_insn_and_split "*extendqidi2_short_displ"
-  [(set (match_operand:DI 0 "register_operand" "=d")
-        (sign_extend:DI (match_operand:QI 1 "s_operand" "Q")))
-   (clobber (reg:CC CC_REGNUM))]
-  "TARGET_64BIT && !TARGET_LONG_DISPLACEMENT"
-  "#"
-  "&& reload_completed"
-  [(parallel
-    [(set (match_dup 0) (unspec:DI [(match_dup 1) (const_int 8)] UNSPEC_ICM))
-     (clobber (reg:CC CC_REGNUM))])
-   (parallel
-    [(set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 56)))
-     (clobber (reg:CC CC_REGNUM))])]
-{
-  operands[1] = adjust_address (operands[1], BLKmode, 0);
-  set_mem_size (operands[1], GEN_INT (GET_MODE_SIZE (QImode)));
-})
-
 ;
-; extend(hi|qi)si2 instruction pattern(s).
+; extendhisi2 instruction pattern(s).
 ;
 
-(define_expand "extend<mode>si2"
-  [(set (match_operand:SI 0 "register_operand" "")
-        (sign_extend:SI (match_operand:HQI 1 "nonimmediate_operand" "")))]
-  ""
-{
- if (!TARGET_EXTIMM)
-   {
-     rtx bitcount = GEN_INT (GET_MODE_BITSIZE(SImode) - 
-                            GET_MODE_BITSIZE(<MODE>mode));
-     operands[1] = gen_lowpart (SImode, operands[1]);
-     emit_insn (gen_ashlsi3 (operands[0], operands[1], bitcount));
-     emit_insn (gen_ashrsi3 (operands[0], operands[0], bitcount));
-     DONE;
-   }
-})
-
 (define_insn "*extendhisi2_extimm"
   [(set (match_operand:SI 0 "register_operand" "=d,d,d")
         (sign_extend:SI (match_operand:HI 1 "nonimmediate_operand" "d,R,T")))]
    lhy\t%0,%1"
   [(set_attr "op_type" "RX,RXY")])
 
-(define_insn "*extendqisi2_extimm"
-  [(set (match_operand:SI 0 "register_operand" "=d,d")
-        (sign_extend:SI (match_operand:QI 1 "nonimmediate_operand" "d,m")))]
+;
+; extendqi(si|di)2 instruction pattern(s).
+;
+
+(define_insn "*extendqi<mode>2_extimm"
+  [(set (match_operand:GPR 0 "register_operand" "=d,d")
+        (sign_extend:GPR (match_operand:QI 1 "nonimmediate_operand" "d,m")))]
   "TARGET_EXTIMM"
   "@
-   lbr\t%0,%1
-   lb\t%0,%1"
+   l<g>br\t%0,%1
+   l<g>b\t%0,%1"
   [(set_attr "op_type" "RRE,RXY")])
 
-(define_insn "*extendqisi2"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (sign_extend:SI (match_operand:QI 1 "memory_operand" "m")))]
-  "TARGET_LONG_DISPLACEMENT && !TARGET_EXTIMM"
-  "lb\t%0,%1"
+(define_insn "*extendqi<mode>2"
+  [(set (match_operand:GPR 0 "register_operand" "=d")
+        (sign_extend:GPR (match_operand:QI 1 "memory_operand" "m")))]
+  "!TARGET_EXTIMM && TARGET_LONG_DISPLACEMENT"
+  "l<g>b\t%0,%1"
   [(set_attr "op_type" "RXY")])
 
-(define_insn_and_split "*extendqisi2_short_displ"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (sign_extend:SI (match_operand:QI 1 "s_operand" "Q")))
+(define_insn_and_split "*extendqi<mode>2_short_displ"
+  [(set (match_operand:GPR 0 "register_operand" "=d")
+        (sign_extend:GPR (match_operand:QI 1 "s_operand" "Q")))
    (clobber (reg:CC CC_REGNUM))]
-  "!TARGET_LONG_DISPLACEMENT"
+  "!TARGET_EXTIMM && !TARGET_LONG_DISPLACEMENT"
   "#"
   "&& reload_completed"
   [(parallel
-    [(set (match_dup 0) (unspec:SI [(match_dup 1) (const_int 8)] UNSPEC_ICM))
+    [(set (match_dup 0) (unspec:GPR [(match_dup 1) (const_int 8)] UNSPEC_ICM))
      (clobber (reg:CC CC_REGNUM))])
    (parallel
-    [(set (match_dup 0) (ashiftrt:SI (match_dup 0) (const_int 24)))
+    [(set (match_dup 0) (ashiftrt:GPR (match_dup 0) (match_dup 2)))
      (clobber (reg:CC CC_REGNUM))])]
 {
   operands[1] = adjust_address (operands[1], BLKmode, 0);
   set_mem_size (operands[1], GEN_INT (GET_MODE_SIZE (QImode)));
+  operands[2] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode)
+                        - GET_MODE_BITSIZE (QImode));
 })
 
 ;
-; extendqihi2 instruction pattern(s).
-;
-
-
-;
 ; zero_extendsidi2 instruction pattern(s).
 ;
 
   [(set_attr "op_type" "RRE,RXY")])
 
 ;
-; zero_extend(hi|qi)di2 instruction pattern(s).
-;
-
-(define_expand "zero_extend<mode>di2"
-  [(set (match_operand:DI 0 "register_operand" "")
-        (zero_extend:DI (match_operand:HQI 1 "nonimmediate_operand" "")))]
-  ""
-{
-  if (!TARGET_64BIT)
-    {
-      rtx tmp = gen_reg_rtx (SImode);
-      emit_insn (gen_zero_extend<mode>si2 (tmp, operands[1]));
-      emit_insn (gen_zero_extendsidi2 (operands[0], tmp));
-      DONE;
-    }
-  else if (!TARGET_EXTIMM)
-    {
-      rtx bitcount = GEN_INT (GET_MODE_BITSIZE(DImode) - 
-                             GET_MODE_BITSIZE(<MODE>mode));
-      operands[1] = gen_lowpart (DImode, operands[1]);
-      emit_insn (gen_ashldi3 (operands[0], operands[1], bitcount));
-      emit_insn (gen_lshrdi3 (operands[0], operands[0], bitcount));
-      DONE;
-    }
-})
-
-(define_insn "*zero_extend<mode>di2_extimm"
-  [(set (match_operand:DI 0 "register_operand" "=d,d")
-        (zero_extend:DI (match_operand:HQI 1 "nonimmediate_operand" "d,m")))]
-  "TARGET_64BIT && TARGET_EXTIMM"
-  "@
-   llg<hc>r\t%0,%1
-   llg<hc>\t%0,%1"
-  [(set_attr "op_type" "RRE,RXY")])
-
-(define_insn "*zero_extend<mode>di2"
-  [(set (match_operand:DI 0 "register_operand" "=d")
-        (zero_extend:DI (match_operand:HQI 1 "memory_operand" "m")))]
-  "TARGET_64BIT  && !TARGET_EXTIMM"
-  "llg<hc>\t%0,%1"
-  [(set_attr "op_type" "RXY")])
-
-;
 ; LLGT-type instructions (zero-extend from 31 bit to 64 bit).
 ;
 
   "")
 
 ;
-; zero_extend(hi|qi)si2 instruction pattern(s).
+; zero_extend(hi|qi)(si|di)2 instruction pattern(s).
 ;
 
+(define_expand "zero_extend<mode>di2"
+  [(set (match_operand:DI 0 "register_operand" "")
+        (zero_extend:DI (match_operand:HQI 1 "nonimmediate_operand" "")))]
+  ""
+{
+  if (!TARGET_64BIT)
+    {
+      rtx tmp = gen_reg_rtx (SImode);
+      emit_insn (gen_zero_extend<mode>si2 (tmp, operands[1]));
+      emit_insn (gen_zero_extendsidi2 (operands[0], tmp));
+      DONE;
+    }
+  else if (!TARGET_EXTIMM)
+    {
+      rtx bitcount = GEN_INT (GET_MODE_BITSIZE(DImode) - 
+                             GET_MODE_BITSIZE(<MODE>mode));
+      operands[1] = gen_lowpart (DImode, operands[1]);
+      emit_insn (gen_ashldi3 (operands[0], operands[1], bitcount));
+      emit_insn (gen_lshrdi3 (operands[0], operands[0], bitcount));
+      DONE;
+    }
+})
+
 (define_expand "zero_extend<mode>si2"
   [(set (match_operand:SI 0 "register_operand" "")
         (zero_extend:SI (match_operand:HQI 1 "nonimmediate_operand" "")))]
       emit_insn (gen_andsi3 (operands[0], operands[1], 
                    GEN_INT ((1 << GET_MODE_BITSIZE(<MODE>mode)) - 1)));
       DONE;
-}
+    }
 })
 
-(define_insn "*zero_extend<mode>si2_extimm"
-  [(set (match_operand:SI 0 "register_operand" "=d,d")
-        (zero_extend:SI (match_operand:HQI 1 "nonimmediate_operand" "d,m")))]
+(define_insn "*zero_extend<HQI:mode><GPR:mode>2_extimm"
+  [(set (match_operand:GPR 0 "register_operand" "=d,d")
+        (zero_extend:GPR (match_operand:HQI 1 "nonimmediate_operand" "d,m")))]
   "TARGET_EXTIMM"
   "@
-   ll<hc>r\t%0,%1
-   ll<hc>\t%0,%1"
+   ll<g><hc>r\t%0,%1
+   ll<g><hc>\t%0,%1"
   [(set_attr "op_type" "RRE,RXY")])
 
-(define_insn "*zero_extend<mode>si2_64"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (zero_extend:SI (match_operand:HQI 1 "memory_operand" "m")))]
+(define_insn "*zero_extend<HQI:mode><GPR:mode>2"
+  [(set (match_operand:GPR 0 "register_operand" "=d")
+        (zero_extend:GPR (match_operand:HQI 1 "memory_operand" "m")))]
   "TARGET_ZARCH && !TARGET_EXTIMM"
   "llg<hc>\t%0,%1"
   [(set_attr "op_type" "RXY")])
   DONE;
 })
 
-(define_expand "fix_trunc<FPR:mode>di2"
+(define_expand "fix_trunc<mode>di2"
   [(set (match_operand:DI 0 "register_operand" "")
-        (fix:DI (match_operand:FPR 1 "nonimmediate_operand" "")))]
+        (fix:DI (match_operand:DSF 1 "nonimmediate_operand" "")))]
   "TARGET_64BIT && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
 {
-  operands[1] = force_reg (<FPR:MODE>mode, operands[1]);
-  emit_insn (gen_fix_trunc<FPR:mode>di2_ieee (operands[0], operands[1],
+  operands[1] = force_reg (<MODE>mode, operands[1]);
+  emit_insn (gen_fix_trunc<mode>di2_ieee (operands[0], operands[1],
       GEN_INT(5)));
   DONE;
 })
    (unspec:GPR [(match_operand:GPR 2 "immediate_operand" "K")] UNSPEC_ROUND)
    (clobber (reg:CC CC_REGNUM))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "c<GPR:gf><FPR:de>br\t%0,%h2,%1"
+  "c<GPR:gf><FPR:xde>br\t%0,%h2,%1"
   [(set_attr "op_type" "RRE")
    (set_attr "type"    "ftoi")])
 
 ;
+; fix_trunctf(si|di)2 instruction pattern(s).
+;
+
+(define_expand "fix_trunctf<mode>2"
+  [(parallel [(set (match_operand:GPR 0 "register_operand" "")
+                  (fix:GPR (match_operand:TF 1 "register_operand" "")))
+             (unspec:GPR [(const_int 5)] UNSPEC_ROUND)
+             (clobber (reg:CC CC_REGNUM))])]
+  "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "")
+
+;
 ; fix_truncdfsi2 instruction pattern(s).
 ;
 
 })
 
 ;
-; floatdi(df|sf)2 instruction pattern(s).
+; float(si|di)(tf|df|sf)2 instruction pattern(s).
 ;
 
 (define_insn "floatdi<mode>2"
   [(set (match_operand:FPR 0 "register_operand" "=f")
         (float:FPR (match_operand:DI 1 "register_operand" "d")))]
   "TARGET_64BIT && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "c<de>gbr\t%0,%1"
+  "c<xde>gbr\t%0,%1"
   [(set_attr "op_type" "RRE")
    (set_attr "type"    "itof" )])
 
+(define_insn "floatsi<mode>2_ieee"
+  [(set (match_operand:FPR 0 "register_operand" "=f")
+        (float:FPR (match_operand:SI 1 "register_operand" "d")))]
+  "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "c<xde>fbr\t%0,%1"
+  [(set_attr "op_type" "RRE")
+   (set_attr "type"   "itof" )])
+
+
 ;
-; floatsidf2 instruction pattern(s).
+; floatsi(tf|df)2 instruction pattern(s).
 ;
 
+(define_expand "floatsitf2"
+  [(set (match_operand:TF 0 "register_operand" "")
+        (float:TF (match_operand:SI 1 "register_operand" "")))]
+  "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "")
+
 (define_expand "floatsidf2"
   [(set (match_operand:DF 0 "register_operand" "")
         (float:DF (match_operand:SI 1 "register_operand" "")))]
     }
 })
 
-(define_insn "floatsidf2_ieee"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-        (float:DF (match_operand:SI 1 "register_operand" "d")))]
-  "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "cdfbr\t%0,%1"
-  [(set_attr "op_type" "RRE")
-   (set_attr "type"   "itof" )])
-
 (define_insn "floatsidf2_ibm"
   [(set (match_operand:DF 0 "register_operand" "=f")
         (float:DF (match_operand:SI 1 "register_operand" "d")))
     }
 })
 
-(define_insn "floatsisf2_ieee"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-        (float:SF (match_operand:SI 1 "register_operand" "d")))]
-  "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "cefbr\t%0,%1"
-  [(set_attr "op_type" "RRE")
-   (set_attr "type"    "itof" )])
-
 ;
 ; truncdfsf2 instruction pattern(s).
 ;
         (float_truncate:SF (match_operand:DF 1 "register_operand" "f")))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "ledbr\t%0,%1"
-  [(set_attr "op_type"  "RRE")])
+  [(set_attr "op_type"  "RRE")
+   (set_attr "type"   "ftruncdf")])
 
 (define_insn "truncdfsf2_ibm"
   [(set (match_operand:SF 0 "register_operand" "=f,f")
    (set_attr "type"   "floadsf")])
 
 ;
+; trunctfdf2 instruction pattern(s).
+;
+
+(define_expand "trunctfdf2"
+  [(parallel 
+    [(set (match_operand:DF 0 "register_operand" "")
+         (float_truncate:DF (match_operand:TF 1 "register_operand" "")))
+     (clobber (match_scratch:TF 2 "=f"))])]
+  "TARGET_HARD_FLOAT"
+  "")
+
+(define_insn "*trunctfdf2_ieee"
+  [(set (match_operand:DF 0 "register_operand" "=f")
+        (float_truncate:DF (match_operand:TF 1 "register_operand" "f")))
+   (clobber (match_scratch:TF 2 "=f"))]
+  "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "ldxbr\t%2,%1\;ldr\t%0,%2"
+  [(set_attr "length" "6")
+   (set_attr "type"   "ftrunctf")])   
+
+(define_insn "*trunctfdf2_ibm"
+  [(set (match_operand:DF 0 "register_operand" "=f")
+        (float_truncate:DF (match_operand:TF 1 "register_operand" "f")))
+   (clobber (match_scratch:TF 2 "=f"))]
+  "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
+  "ldxr\t%2,%1\;ldr\t%0,%2"
+  [(set_attr "length"  "4")
+   (set_attr "type"   "ftrunctf")])
+
+;
+; trunctfsf2 instruction pattern(s).
+;
+
+(define_expand "trunctfsf2"
+  [(parallel 
+    [(set (match_operand:SF 0 "register_operand" "=f")
+         (float_truncate:SF (match_operand:TF 1 "register_operand" "f")))
+     (clobber (match_scratch:TF 2 "=f"))])]
+  "TARGET_HARD_FLOAT"
+  "")
+
+(define_insn "*trunctfsf2_ieee"
+  [(set (match_operand:SF 0 "register_operand" "=f")
+        (float_truncate:SF (match_operand:TF 1 "register_operand" "f")))
+   (clobber (match_scratch:TF 2 "=f"))]
+  "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "lexbr\t%2,%1\;ler\t%0,%2"
+  [(set_attr "length"  "6")
+   (set_attr "type"   "ftrunctf")])
+
+(define_insn "*trunctfsf2_ibm"
+  [(set (match_operand:SF 0 "register_operand" "=f")
+        (float_truncate:SF (match_operand:TF 1 "register_operand" "f")))
+   (clobber (match_scratch:TF 2 "=f"))]
+  "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
+  "lexr\t%2,%1\;ler\t%0,%2"
+  [(set_attr "length"  "6")
+   (set_attr "type"   "ftrunctf")])
+
+;
 ; extendsfdf2 instruction pattern(s).
 ;
 
    ldebr\t%0,%1
    ldeb\t%0,%1"
   [(set_attr "op_type"  "RRE,RXE")
-   (set_attr "type"   "floadsf")])
+   (set_attr "type"   "fsimpsf, floadsf")])
 
 (define_insn "extendsfdf2_ibm"
   [(set (match_operand:DF 0 "register_operand" "=f,f")
   [(set_attr "length"   "4,6")
    (set_attr "type"     "floadsf")])
 
+;
+; extenddftf2 instruction pattern(s).
+;
+
+(define_expand "extenddftf2"
+  [(set (match_operand:TF 0 "register_operand" "")
+        (float_extend:TF (match_operand:DF 1 "nonimmediate_operand" "")))]
+  "TARGET_HARD_FLOAT"
+  "")
+
+(define_insn "*extenddftf2_ieee"
+  [(set (match_operand:TF 0 "register_operand" "=f,f")
+        (float_extend:TF (match_operand:DF 1 "nonimmediate_operand" "f,R")))]
+  "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "@
+   lxdbr\t%0,%1
+   lxdb\t%0,%1"
+  [(set_attr "op_type"  "RRE,RXE")
+   (set_attr "type"   "fsimptf, floadtf")])
+
+(define_insn "*extenddftf2_ibm"
+  [(set (match_operand:TF 0 "register_operand" "=f,f")
+        (float_extend:TF (match_operand:DF 1 "nonimmediate_operand" "f,R")))]
+  "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
+  "@
+   lxdr\t%0,%1
+   lxd\t%0,%1"
+  [(set_attr "op_type"  "RRE,RXE")
+   (set_attr "type"   "fsimptf, floadtf")])
+
+;
+; extendsftf2 instruction pattern(s).
+;
+
+(define_expand "extendsftf2"
+  [(set (match_operand:TF 0 "register_operand" "")
+        (float_extend:TF (match_operand:SF 1 "nonimmediate_operand" "")))]
+  "TARGET_HARD_FLOAT"
+  "")
+
+(define_insn "*extendsftf2_ieee"
+  [(set (match_operand:TF 0 "register_operand" "=f,f")
+        (float_extend:TF (match_operand:SF 1 "nonimmediate_operand" "f,R")))]
+  "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
+  "@
+   lxebr\t%0,%1
+   lxeb\t%0,%1"
+  [(set_attr "op_type"  "RRE,RXE")
+   (set_attr "type"   "fsimptf, floadtf")])
+
+(define_insn "*extendsftf2_ibm"
+  [(set (match_operand:TF 0 "register_operand" "=f,f")
+        (float_extend:TF (match_operand:SF 1 "nonimmediate_operand" "f,R")))]
+  "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
+  "@
+   lxer\t%0,%1
+   lxe\t%0,%1"
+  [(set_attr "op_type"  "RRE,RXE")
+   (set_attr "type"   "fsimptf, floadtf")])
+
 
 ;;
 ;; ARITHMETIC OPERATIONS
 ; adddi3 instruction pattern(s).
 ;
 
+(define_expand "adddi3"
+  [(parallel
+    [(set (match_operand:DI 0 "register_operand" "")
+          (plus:DI (match_operand:DI 1 "nonimmediate_operand" "")
+                   (match_operand:DI 2 "general_operand" "")))
+     (clobber (reg:CC CC_REGNUM))])]
+  ""
+  "")
+
 (define_insn "*adddi3_sign"
   [(set (match_operand:DI 0 "register_operand" "=d,d")
         (plus:DI (sign_extend:DI (match_operand:SI 2 "general_operand" "d,m"))
 
 (define_insn "*adddi3_zero"
   [(set (match_operand:DI 0 "register_operand" "=d,d")
-        (plus:DI (zero_extend:DI (match_operand:SI 2 "general_operand" "d,m"))
-                 (match_operand:DI 1 "register_operand" "0,0")))
-   (clobber (reg:CC CC_REGNUM))]
-  "TARGET_64BIT"
-  "@
-   algfr\t%0,%2
-   algf\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXY")])
-
-(define_insn "*adddi3_imm_cc"
-  [(set (reg CC_REGNUM)
-        (compare (plus:DI (match_operand:DI 1 "nonimmediate_operand" "0,0")
-                          (match_operand:DI 2 "const_int_operand" "K,Os"))
-                 (const_int 0)))
-   (set (match_operand:DI 0 "register_operand" "=d,d")
-        (plus:DI (match_dup 1) (match_dup 2)))]
-  "TARGET_64BIT
-   && s390_match_ccmode (insn, CCAmode)
-   && (CONST_OK_FOR_CONSTRAINT_P (INTVAL (operands[2]), 'K', \"K\")
-       || CONST_OK_FOR_CONSTRAINT_P (INTVAL (operands[2]), 'O', \"Os\"))"
-  "@
-   aghi\t%0,%h2
-   agfi\t%0,%2"
-  [(set_attr "op_type"  "RI,RIL")])
-
-(define_insn "*adddi3_carry1_cc"
-  [(set (reg CC_REGNUM)
-        (compare (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0,0,0")
-                          (match_operand:DI 2 "general_operand" "d,Op,On,m"))
-                 (match_dup 1)))
-   (set (match_operand:DI 0 "register_operand" "=d,d,d,d")
-        (plus:DI (match_dup 1) (match_dup 2)))]
-  "s390_match_ccmode (insn, CCL1mode) && TARGET_64BIT"
-  "@
-   algr\t%0,%2
-   algfi\t%0,%2
-   slgfi\t%0,%n2
-   alg\t%0,%2"
-  [(set_attr "op_type"  "RRE,RIL,RIL,RXY")])
-
-(define_insn "*adddi3_carry1_cconly"
-  [(set (reg CC_REGNUM)
-        (compare (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
-                          (match_operand:DI 2 "general_operand" "d,m"))
-                 (match_dup 1)))
-   (clobber (match_scratch:DI 0 "=d,d"))]
-  "s390_match_ccmode (insn, CCL1mode) && TARGET_64BIT"
-  "@
-   algr\t%0,%2
-   alg\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXY")])
-
-(define_insn "*adddi3_carry2_cc"
-  [(set (reg CC_REGNUM)
-        (compare (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0,0,0")
-                          (match_operand:DI 2 "general_operand" "d,Op,On,m"))
-                 (match_dup 2)))
-   (set (match_operand:DI 0 "register_operand" "=d,d,d,d")
-        (plus:DI (match_dup 1) (match_dup 2)))]
-  "s390_match_ccmode (insn, CCL1mode) && TARGET_64BIT"
-  "@
-   algr\t%0,%2
-   algfi\t%0,%2
-   slgfi\t%0,%n2
-   alg\t%0,%2"
-  [(set_attr "op_type"  "RRE,RIL,RIL,RXY")])
-
-(define_insn "*adddi3_carry2_cconly"
-  [(set (reg CC_REGNUM)
-        (compare (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
-                          (match_operand:DI 2 "general_operand" "d,m"))
-                 (match_dup 2)))
-   (clobber (match_scratch:DI 0 "=d,d"))]
-  "s390_match_ccmode (insn, CCL1mode) && TARGET_64BIT"
-  "@
-   algr\t%0,%2
-   alg\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXY")])
-
-(define_insn "*adddi3_cc"
-  [(set (reg CC_REGNUM)
-        (compare (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0,0,0")
-                          (match_operand:DI 2 "general_operand" "d,Op,On,m"))
-                 (const_int 0)))
-   (set (match_operand:DI 0 "register_operand" "=d,d,d,d")
-        (plus:DI (match_dup 1) (match_dup 2)))]
-  "s390_match_ccmode (insn, CCLmode) && TARGET_64BIT"
-  "@
-   algr\t%0,%2
-   algfi\t%0,%2
-   slgfi\t%0,%n2
-   alg\t%0,%2"
-  [(set_attr "op_type"  "RRE,RIL,RIL,RXY")])
-
-(define_insn "*adddi3_cconly"
-  [(set (reg CC_REGNUM)
-        (compare (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0")
-                          (match_operand:DI 2 "general_operand" "d,m"))
-                 (const_int 0)))
-   (clobber (match_scratch:DI 0 "=d,d"))]
-  "s390_match_ccmode (insn, CCLmode) && TARGET_64BIT"
-  "@
-   algr\t%0,%2
-   alg\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXY")])
-
-(define_insn "*adddi3_cconly2"
-  [(set (reg CC_REGNUM)
-        (compare (match_operand:DI 1 "nonimmediate_operand" "%0,0")
-                 (neg:SI (match_operand:DI 2 "general_operand" "d,m"))))
-   (clobber (match_scratch:DI 0 "=d,d"))]
-  "s390_match_ccmode(insn, CCLmode) && TARGET_64BIT"
-  "@
-   algr\t%0,%2
-   alg\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXY")])
-
-(define_insn "*adddi3_64"
-  [(set (match_operand:DI 0 "register_operand" "=d,d,d,d,d")
-        (plus:DI (match_operand:DI 1 "nonimmediate_operand" "%0,0,0,0,0")
-                 (match_operand:DI 2 "general_operand" "d,K,Op,On,m") ) )
+        (plus:DI (zero_extend:DI (match_operand:SI 2 "general_operand" "d,m"))
+                 (match_operand:DI 1 "register_operand" "0,0")))
    (clobber (reg:CC CC_REGNUM))]
   "TARGET_64BIT"
   "@
-   agr\t%0,%2
-   aghi\t%0,%h2
-   algfi\t%0,%2
-   slgfi\t%0,%n2
-   ag\t%0,%2"
-  [(set_attr "op_type"  "RRE,RI,RIL,RIL,RXY")])
+   algfr\t%0,%2
+   algf\t%0,%2"
+  [(set_attr "op_type"  "RRE,RXY")])
 
 (define_insn_and_split "*adddi3_31z"
   [(set (match_operand:DI 0 "register_operand" "=&d")
    operands[8] = operand_subword (operands[2], 1, 0, DImode);
    operands[9] = gen_label_rtx ();")
 
-(define_expand "adddi3"
+;
+; addsi3 instruction pattern(s).
+;
+
+(define_expand "addsi3"
   [(parallel
-    [(set (match_operand:DI 0 "register_operand" "")
-          (plus:DI (match_operand:DI 1 "nonimmediate_operand" "")
-                   (match_operand:DI 2 "general_operand" "")))
+    [(set (match_operand:SI 0 "register_operand" "")
+          (plus:SI (match_operand:SI 1 "nonimmediate_operand" "")
+                   (match_operand:SI 2 "general_operand" "")))
      (clobber (reg:CC CC_REGNUM))])]
   ""
   "")
 
+(define_insn "*addsi3_sign"
+  [(set (match_operand:SI 0 "register_operand" "=d,d")
+        (plus:SI (sign_extend:SI (match_operand:HI 2 "memory_operand" "R,T"))
+                 (match_operand:SI 1 "register_operand" "0,0")))
+   (clobber (reg:CC CC_REGNUM))]
+  ""
+  "@
+   ah\t%0,%2
+   ahy\t%0,%2"
+  [(set_attr "op_type"  "RX,RXY")])
+
 ;
-; addsi3 instruction pattern(s).
+; add(di|si)3 instruction pattern(s).
 ;
 
-(define_insn "*addsi3_imm_cc"
-  [(set (reg CC_REGNUM)
-        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "0,0")
-                          (match_operand:SI 2 "const_int_operand" "K,Os"))
-                 (const_int 0)))
-   (set (match_operand:SI 0 "register_operand" "=d,d")
-        (plus:SI (match_dup 1) (match_dup 2)))]
-  "s390_match_ccmode (insn, CCAmode)
-   && (CONST_OK_FOR_CONSTRAINT_P (INTVAL (operands[2]), 'K', \"K\")
-       || CONST_OK_FOR_CONSTRAINT_P (INTVAL (operands[2]), 'O', \"Os\"))
-   && INTVAL (operands[2]) != -((HOST_WIDE_INT)1 << 31)"
+(define_insn "*add<mode>3"
+  [(set (match_operand:GPR 0 "register_operand" "=d,d,d,d,d,d")
+        (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,0,0,0,0,0")
+                 (match_operand:GPR 2 "general_operand" "d,K,Op,On,R,T") ) )
+   (clobber (reg:CC CC_REGNUM))]
+  ""
   "@
-   ahi\t%0,%h2
-   afi\t%0,%2"
-  [(set_attr "op_type"  "RI,RIL")])
+   a<g>r\t%0,%2
+   a<g>hi\t%0,%h2
+   al<g>fi\t%0,%2
+   sl<g>fi\t%0,%n2
+   a<g>\t%0,%2
+   a<y>\t%0,%2"
+  [(set_attr "op_type"  "RR<E>,RI,RIL,RIL,RX<Y>,RXY")])
 
-(define_insn "*addsi3_carry1_cc"
+(define_insn "*add<mode>3_carry1_cc"
   [(set (reg CC_REGNUM)
-        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0,0")
-                          (match_operand:SI 2 "general_operand" "d,Os,R,T"))
+        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,0,0,0,0")
+                          (match_operand:GPR 2 "general_operand" "d,Op,On,R,T"))
                  (match_dup 1)))
-   (set (match_operand:SI 0 "register_operand" "=d,d,d,d")
-        (plus:SI (match_dup 1) (match_dup 2)))]
+   (set (match_operand:GPR 0 "register_operand" "=d,d,d,d,d")
+        (plus:GPR (match_dup 1) (match_dup 2)))]
   "s390_match_ccmode (insn, CCL1mode)"
   "@
-   alr\t%0,%2
-   alfi\t%0,%o2
-   al\t%0,%2
-   aly\t%0,%2"
-  [(set_attr "op_type"  "RR,RIL,RX,RXY")])
+   al<g>r\t%0,%2
+   al<g>fi\t%0,%2
+   sl<g>fi\t%0,%n2
+   al<g>\t%0,%2
+   al<y>\t%0,%2"
+  [(set_attr "op_type"  "RR<E>,RIL,RIL,RX<Y>,RXY")])
 
-(define_insn "*addsi3_carry1_cconly"
+(define_insn "*add<mode>3_carry1_cconly"
   [(set (reg CC_REGNUM)
-        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
-                          (match_operand:SI 2 "general_operand" "d,R,T"))
+        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,0,0")
+                          (match_operand:GPR 2 "general_operand" "d,R,T"))
                  (match_dup 1)))
-   (clobber (match_scratch:SI 0 "=d,d,d"))]
+   (clobber (match_scratch:GPR 0 "=d,d,d"))]
   "s390_match_ccmode (insn, CCL1mode)"
   "@
-   alr\t%0,%2
-   al\t%0,%2
-   aly\t%0,%2"
-  [(set_attr "op_type"  "RR,RX,RXY")])
+   al<g>r\t%0,%2
+   al<g>\t%0,%2
+   al<y>\t%0,%2"
+  [(set_attr "op_type"  "RR<E>,RX<Y>,RXY")])
 
-(define_insn "*addsi3_carry2_cc"
+(define_insn "*add<mode>3_carry2_cc"
   [(set (reg CC_REGNUM)
-        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0,0")
-                          (match_operand:SI 2 "general_operand" "d,Os,R,T"))
+        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,0,0,0,0")
+                          (match_operand:GPR 2 "general_operand" "d,Op,On,R,T"))
                  (match_dup 2)))
-   (set (match_operand:SI 0 "register_operand" "=d,d,d,d")
-        (plus:SI (match_dup 1) (match_dup 2)))]
+   (set (match_operand:GPR 0 "register_operand" "=d,d,d,d,d")
+        (plus:GPR (match_dup 1) (match_dup 2)))]
   "s390_match_ccmode (insn, CCL1mode)"
   "@
-   alr\t%0,%2
-   alfi\t%0,%o2
-   al\t%0,%2
-   aly\t%0,%2"
-  [(set_attr "op_type"  "RR,RIL,RX,RXY")])
+   al<g>r\t%0,%2
+   al<g>fi\t%0,%2
+   sl<g>fi\t%0,%n2
+   al<g>\t%0,%2
+   al<y>\t%0,%2"
+  [(set_attr "op_type"  "RR<E>,RIL,RIL,RX<Y>,RXY")])
 
-(define_insn "*addsi3_carry2_cconly"
+(define_insn "*add<mode>3_carry2_cconly"
   [(set (reg CC_REGNUM)
-        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
-                          (match_operand:SI 2 "general_operand" "d,R,T"))
+        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,0,0")
+                          (match_operand:GPR 2 "general_operand" "d,R,T"))
                  (match_dup 2)))
-   (clobber (match_scratch:SI 0 "=d,d,d"))]
+   (clobber (match_scratch:GPR 0 "=d,d,d"))]
   "s390_match_ccmode (insn, CCL1mode)"
   "@
-   alr\t%0,%2
-   al\t%0,%2
-   aly\t%0,%2"
-  [(set_attr "op_type"  "RR,RX,RXY")])
+   al<g>r\t%0,%2
+   al<g>\t%0,%2
+   al<y>\t%0,%2"
+  [(set_attr "op_type"  "RR<E>,RX<Y>,RXY")])
 
-(define_insn "*addsi3_cc"
+(define_insn "*add<mode>3_cc"
   [(set (reg CC_REGNUM)
-        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0,0")
-                          (match_operand:SI 2 "general_operand" "d,Os,R,T"))
+        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,0,0,0,0")
+                          (match_operand:GPR 2 "general_operand" "d,Op,On,R,T"))
                  (const_int 0)))
-   (set (match_operand:SI 0 "register_operand" "=d,d,d,d")
-        (plus:SI (match_dup 1) (match_dup 2)))]
+   (set (match_operand:GPR 0 "register_operand" "=d,d,d,d,d")
+        (plus:GPR (match_dup 1) (match_dup 2)))]
   "s390_match_ccmode (insn, CCLmode)"
   "@
-   alr\t%0,%2
-   alfi\t%0,%o2
-   al\t%0,%2
-   aly\t%0,%2"
-  [(set_attr "op_type"  "RR,RIL,RX,RXY")])
+   al<g>r\t%0,%2
+   al<g>fi\t%0,%2
+   sl<g>fi\t%0,%n2
+   al<g>\t%0,%2
+   al<y>\t%0,%2"
+  [(set_attr "op_type"  "RR<E>,RIL,RIL,RX<Y>,RXY")])
 
-(define_insn "*addsi3_cconly"
+(define_insn "*add<mode>3_cconly"
   [(set (reg CC_REGNUM)
-        (compare (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
-                          (match_operand:SI 2 "general_operand" "d,R,T"))
+        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "%0,0,0")
+                          (match_operand:GPR 2 "general_operand" "d,R,T"))
                  (const_int 0)))
-   (clobber (match_scratch:SI 0 "=d,d,d"))]
+   (clobber (match_scratch:GPR 0 "=d,d,d"))]
   "s390_match_ccmode (insn, CCLmode)"
   "@
-   alr\t%0,%2
-   al\t%0,%2
-   aly\t%0,%2"
-  [(set_attr "op_type"  "RR,RX,RXY")])
+   al<g>r\t%0,%2
+   al<g>\t%0,%2
+   al<y>\t%0,%2"
+  [(set_attr "op_type"  "RR<E>,RX<Y>,RXY")])
 
-(define_insn "*addsi3_cconly2"
+(define_insn "*add<mode>3_cconly2"
   [(set (reg CC_REGNUM)
-        (compare (match_operand:SI 1 "nonimmediate_operand" "%0,0,0")
-                 (neg:SI (match_operand:SI 2 "general_operand" "d,R,T"))))
-   (clobber (match_scratch:SI 0 "=d,d,d"))]
-  "s390_match_ccmode (insn, CCLmode)"
-  "@
-   alr\t%0,%2
-   al\t%0,%2
-   aly\t%0,%2"
-  [(set_attr "op_type"  "RR,RX,RXY")])
-
-(define_insn "*addsi3_sign"
-  [(set (match_operand:SI 0 "register_operand" "=d,d")
-        (plus:SI (sign_extend:SI (match_operand:HI 2 "memory_operand" "R,T"))
-                 (match_operand:SI 1 "register_operand" "0,0")))
-   (clobber (reg:CC CC_REGNUM))]
-  ""
+        (compare (match_operand:GPR 1 "nonimmediate_operand" "%0,0,0")
+                 (neg:GPR (match_operand:GPR 2 "general_operand" "d,R,T"))))
+   (clobber (match_scratch:GPR 0 "=d,d,d"))]
+  "s390_match_ccmode(insn, CCLmode)"
   "@
-   ah\t%0,%2
-   ahy\t%0,%2"
-  [(set_attr "op_type"  "RX,RXY")])
+   al<g>r\t%0,%2
+   al<g>\t%0,%2
+   al<y>\t%0,%2"
+  [(set_attr "op_type"  "RR<E>,RX<Y>,RXY")])
 
-(define_insn "addsi3"
-  [(set (match_operand:SI 0 "register_operand" "=d,d,d,d,d")
-        (plus:SI (match_operand:SI 1 "nonimmediate_operand" "%0,0,0,0,0")
-                 (match_operand:SI 2 "general_operand" "d,K,Os,R,T")))
-   (clobber (reg:CC CC_REGNUM))]
-  ""
+(define_insn "*add<mode>3_imm_cc"
+  [(set (reg CC_REGNUM)
+        (compare (plus:GPR (match_operand:GPR 1 "nonimmediate_operand" "0,0")
+                          (match_operand:GPR 2 "const_int_operand" "K,Os"))
+                 (const_int 0)))
+   (set (match_operand:GPR 0 "register_operand" "=d,d")
+        (plus:GPR (match_dup 1) (match_dup 2)))]
+  "s390_match_ccmode (insn, CCAmode)
+   && (CONST_OK_FOR_CONSTRAINT_P (INTVAL (operands[2]), 'K', \"K\")
+       || CONST_OK_FOR_CONSTRAINT_P (INTVAL (operands[2]), 'O', \"Os\"))
+   && INTVAL (operands[2]) != -((HOST_WIDE_INT)1 << (GET_MODE_BITSIZE(<MODE>mode) - 1))"
   "@
-   ar\t%0,%2
-   ahi\t%0,%h2
-   afi\t%0,%2
-   a\t%0,%2
-   ay\t%0,%2"
-  [(set_attr "op_type"  "RR,RI,RIL,RX,RXY")])
+   a<g>hi\t%0,%h2
+   a<g>fi\t%0,%2"
+  [(set_attr "op_type"  "RI,RIL")])
 
 ;
 ; add(df|sf)3 instruction pattern(s).
   [(parallel
     [(set (match_operand:FPR 0 "register_operand" "=f,f")
           (plus:FPR (match_operand:FPR 1 "nonimmediate_operand" "%0,0")
-                    (match_operand:FPR 2 "general_operand" "f,R")))
+                    (match_operand:FPR 2 "general_operand" "f,<Rf>")))
      (clobber (reg:CC CC_REGNUM))])]
   "TARGET_HARD_FLOAT"
   "")
 (define_insn "*add<mode>3"
   [(set (match_operand:FPR 0 "register_operand" "=f,f")
         (plus:FPR (match_operand:FPR 1 "nonimmediate_operand" "%0,0")
-                  (match_operand:FPR 2 "general_operand" "f,R")))
+                  (match_operand:FPR 2 "general_operand" "f,<Rf>")))
    (clobber (reg:CC CC_REGNUM))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   a<de>br\t%0,%2
-   a<de>b\t%0,%2"
+   a<xde>br\t%0,%2
+   a<xde>b\t%0,%2"
   [(set_attr "op_type"  "RRE,RXE")
    (set_attr "type"     "fsimp<mode>")])
 
 (define_insn "*add<mode>3_cc"
   [(set (reg CC_REGNUM)
        (compare (plus:FPR (match_operand:FPR 1 "nonimmediate_operand" "%0,0")
-                          (match_operand:FPR 2 "general_operand" "f,R"))
+                          (match_operand:FPR 2 "general_operand" "f,<Rf>"))
                 (match_operand:FPR 3 "const0_operand" "")))
    (set (match_operand:FPR 0 "register_operand" "=f,f")
        (plus:FPR (match_dup 1) (match_dup 2)))]
   "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   a<de>br\t%0,%2
-   a<de>b\t%0,%2"
+   a<xde>br\t%0,%2
+   a<xde>b\t%0,%2"
   [(set_attr "op_type"  "RRE,RXE")
    (set_attr "type"     "fsimp<mode>")])
 
 (define_insn "*add<mode>3_cconly"
   [(set (reg CC_REGNUM)
        (compare (plus:FPR (match_operand:FPR 1 "nonimmediate_operand" "%0,0")
-                          (match_operand:FPR 2 "general_operand" "f,R"))
+                          (match_operand:FPR 2 "general_operand" "f,<Rf>"))
                 (match_operand:FPR 3 "const0_operand" "")))
    (clobber (match_scratch:FPR 0 "=f,f"))]
   "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   a<de>br\t%0,%2
-   a<de>b\t%0,%2"
+   a<xde>br\t%0,%2
+   a<xde>b\t%0,%2"
   [(set_attr "op_type"  "RRE,RXE")
    (set_attr "type"     "fsimp<mode>")])
 
 (define_insn "*add<mode>3_ibm"
   [(set (match_operand:FPR 0 "register_operand" "=f,f")
         (plus:FPR (match_operand:FPR 1 "nonimmediate_operand" "%0,0")
-                  (match_operand:FPR 2 "general_operand" "f,R")))
+                  (match_operand:FPR 2 "general_operand" "f,<Rf>")))
    (clobber (reg:CC CC_REGNUM))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
   "@
-   a<de>r\t%0,%2
-   a<de>\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
+   a<xde>r\t%0,%2
+   a<xde>\t%0,%2"
+  [(set_attr "op_type"  "<RRe>,<RXe>")
    (set_attr "type"     "fsimp<mode>")])
 
 
 ; subdi3 instruction pattern(s).
 ;
 
+(define_expand "subdi3"
+  [(parallel
+    [(set (match_operand:DI 0 "register_operand" "")
+          (minus:DI (match_operand:DI 1 "register_operand" "")
+                    (match_operand:DI 2 "general_operand" "")))
+     (clobber (reg:CC CC_REGNUM))])]
+  ""
+  "")
+
 (define_insn "*subdi3_sign"
   [(set (match_operand:DI 0 "register_operand" "=d,d")
         (minus:DI (match_operand:DI 1 "register_operand" "0,0")
    slgf\t%0,%2"
   [(set_attr "op_type"  "RRE,RXY")])
 
-(define_insn "*subdi3_borrow_cc"
-  [(set (reg CC_REGNUM)
-        (compare (minus:DI (match_operand:DI 1 "register_operand" "0,0")
-                           (match_operand:DI 2 "general_operand" "d,m"))
-                 (match_dup 1)))
-   (set (match_operand:DI 0 "register_operand" "=d,d")
-        (minus:DI (match_dup 1) (match_dup 2)))]
-  "s390_match_ccmode (insn, CCL2mode) && TARGET_64BIT"
-  "@
-   slgr\t%0,%2
-   slg\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXY")])
-
-(define_insn "*subdi3_borrow_cconly"
-  [(set (reg CC_REGNUM)
-        (compare (minus:DI (match_operand:DI 1 "register_operand" "0,0")
-                           (match_operand:DI 2 "general_operand" "d,m"))
-                 (match_dup 1)))
-   (clobber (match_scratch:DI 0 "=d,d"))]
-  "s390_match_ccmode (insn, CCL2mode) && TARGET_64BIT"
-  "@
-   slgr\t%0,%2
-   slg\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXY")])
-
-(define_insn "*subdi3_cc"
-  [(set (reg CC_REGNUM)
-        (compare (minus:DI (match_operand:DI 1 "register_operand" "0,0")
-                           (match_operand:DI 2 "general_operand" "d,m"))
-                 (const_int 0)))
-   (set (match_operand:DI 0 "register_operand" "=d,d")
-        (minus:DI (match_dup 1) (match_dup 2)))]
-  "s390_match_ccmode (insn, CCLmode) && TARGET_64BIT"
-  "@
-   slgr\t%0,%2
-   slg\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXY")])
-
-(define_insn "*subdi3_cc2"
-  [(set (reg CC_REGNUM)
-        (compare (match_operand:DI 1 "register_operand" "0,0")
-                 (match_operand:DI 2 "general_operand" "d,m")))
-   (set (match_operand:DI 0 "register_operand" "=d,d")
-        (minus:DI (match_dup 1) (match_dup 2)))]
-  "s390_match_ccmode (insn, CCL3mode) && TARGET_64BIT"
-  "@
-   slgr\t%0,%2
-   slg\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXY")])
-
-(define_insn "*subdi3_cconly"
-  [(set (reg CC_REGNUM)
-        (compare (minus:DI (match_operand:DI 1 "register_operand" "0,0")
-                           (match_operand:DI 2 "general_operand" "d,m"))
-                 (const_int 0)))
-   (clobber (match_scratch:DI 0 "=d,d"))]
-  "s390_match_ccmode (insn, CCLmode) && TARGET_64BIT"
-  "@
-   slgr\t%0,%2
-   slg\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXY")])
-
-(define_insn "*subdi3_cconly2"
-  [(set (reg CC_REGNUM)
-        (compare (match_operand:DI 1 "register_operand" "0,0")
-                 (match_operand:DI 2 "general_operand" "d,m")))
-   (clobber (match_scratch:DI 0 "=d,d"))]
-  "s390_match_ccmode (insn, CCL3mode) && TARGET_64BIT"
-  "@
-   slgr\t%0,%2
-   slg\t%0,%2"
-  [(set_attr "op_type"  "RRE,RXY")])
-
-(define_insn "*subdi3_64"
-  [(set (match_operand:DI 0 "register_operand" "=d,d")
-        (minus:DI (match_operand:DI 1 "register_operand" "0,0")
-                  (match_operand:DI 2 "general_operand" "d,m") ) )
-   (clobber (reg:CC CC_REGNUM))]
-  "TARGET_64BIT"
-  "@
-   sgr\t%0,%2
-   sg\t%0,%2"
-  [(set_attr "op_type"  "RRE,RRE")])
-
 (define_insn_and_split "*subdi3_31z"
   [(set (match_operand:DI 0 "register_operand" "=&d")
         (minus:DI (match_operand:DI 1 "register_operand" "0")
    operands[8] = operand_subword (operands[2], 1, 0, DImode);
    operands[9] = gen_label_rtx ();")
 
-(define_expand "subdi3"
+;
+; subsi3 instruction pattern(s).
+;
+
+(define_expand "subsi3"
   [(parallel
-    [(set (match_operand:DI 0 "register_operand" "")
-          (minus:DI (match_operand:DI 1 "register_operand" "")
-                    (match_operand:DI 2 "general_operand" "")))
+    [(set (match_operand:SI 0 "register_operand" "")
+          (minus:SI (match_operand:SI 1 "register_operand" "")
+                    (match_operand:SI 2 "general_operand" "")))
      (clobber (reg:CC CC_REGNUM))])]
   ""
   "")
 
+(define_insn "*subsi3_sign"
+  [(set (match_operand:SI 0 "register_operand" "=d,d")
+        (minus:SI (match_operand:SI 1 "register_operand" "0,0")
+                  (sign_extend:SI (match_operand:HI 2 "memory_operand" "R,T"))))
+   (clobber (reg:CC CC_REGNUM))]
+  ""
+  "@
+   sh\t%0,%2
+   shy\t%0,%2"
+  [(set_attr "op_type"  "RX,RXY")])
+
 ;
-; subsi3 instruction pattern(s).
+; sub(di|si)3 instruction pattern(s).
 ;
 
-(define_insn "*subsi3_borrow_cc"
+(define_insn "*sub<mode>3"
+  [(set (match_operand:GPR 0 "register_operand" "=d,d,d")
+        (minus:GPR (match_operand:GPR 1 "register_operand" "0,0,0")
+                  (match_operand:GPR 2 "general_operand" "d,R,T") ) )
+   (clobber (reg:CC CC_REGNUM))]
+  ""
+  "@
+   s<g>r\t%0,%2
+   s<g>\t%0,%2
+   s<y>\t%0,%2"
+  [(set_attr "op_type"  "RR<E>,RX<Y>,RXY")])
+
+(define_insn "*sub<mode>3_borrow_cc"
   [(set (reg CC_REGNUM)
-        (compare (minus:SI (match_operand:SI 1 "register_operand" "0,0,0")
-                           (match_operand:SI 2 "general_operand" "d,R,T"))
+        (compare (minus:GPR (match_operand:GPR 1 "register_operand" "0,0,0")
+                           (match_operand:GPR 2 "general_operand" "d,R,T"))
                  (match_dup 1)))
-   (set (match_operand:SI 0 "register_operand" "=d,d,d")
-        (minus:SI (match_dup 1) (match_dup 2)))]
+   (set (match_operand:GPR 0 "register_operand" "=d,d,d")
+        (minus:GPR (match_dup 1) (match_dup 2)))]
   "s390_match_ccmode (insn, CCL2mode)"
   "@
-   slr\t%0,%2
-   sl\t%0,%2
-   sly\t%0,%2"
-  [(set_attr "op_type"  "RR,RX,RXY")])
+   sl<g>r\t%0,%2
+   sl<g>\t%0,%2
+   sl<y>\t%0,%2"
+  [(set_attr "op_type"  "RR<E>,RX<Y>,RXY")])
 
-(define_insn "*subsi3_borrow_cconly"
+(define_insn "*sub<mode>3_borrow_cconly"
   [(set (reg CC_REGNUM)
-        (compare (minus:SI (match_operand:SI 1 "register_operand" "0,0,0")
-                           (match_operand:SI 2 "general_operand" "d,R,T"))
+        (compare (minus:GPR (match_operand:GPR 1 "register_operand" "0,0,0")
+                           (match_operand:GPR 2 "general_operand" "d,R,T"))
                  (match_dup 1)))
-   (clobber (match_scratch:SI 0 "=d,d,d"))]
+   (clobber (match_scratch:GPR 0 "=d,d,d"))]
   "s390_match_ccmode (insn, CCL2mode)"
   "@
-   slr\t%0,%2
-   sl\t%0,%2
-   sly\t%0,%2"
-  [(set_attr "op_type"  "RR,RX,RXY")])
+   sl<g>r\t%0,%2
+   sl<g>\t%0,%2
+   sl<y>\t%0,%2"
+  [(set_attr "op_type"  "RR<E>,RX<Y>,RXY")])
 
-(define_insn "*subsi3_cc"
+(define_insn "*sub<mode>3_cc"
   [(set (reg CC_REGNUM)
-        (compare (minus:SI (match_operand:SI 1 "register_operand" "0,0,0")
-                           (match_operand:SI 2 "general_operand" "d,R,T"))
+        (compare (minus:GPR (match_operand:GPR 1 "register_operand" "0,0,0")
+                           (match_operand:GPR 2 "general_operand" "d,R,T"))
                  (const_int 0)))
-   (set (match_operand:SI 0 "register_operand" "=d,d,d")
-        (minus:SI (match_dup 1) (match_dup 2)))]
+   (set (match_operand:GPR 0 "register_operand" "=d,d,d")
+        (minus:GPR (match_dup 1) (match_dup 2)))]
   "s390_match_ccmode (insn, CCLmode)"
   "@
-   slr\t%0,%2
-   sl\t%0,%2
-   sly\t%0,%2"
-  [(set_attr "op_type"  "RR,RX,RXY")])
+   sl<g>r\t%0,%2
+   sl<g>\t%0,%2
+   sl<y>\t%0,%2"
+  [(set_attr "op_type"  "RR<E>,RX<Y>,RXY")])
 
-(define_insn "*subsi3_cc2"
+(define_insn "*sub<mode>3_cc2"
   [(set (reg CC_REGNUM)
-        (compare (match_operand:SI 1 "register_operand" "0,0,0")
-                 (match_operand:SI 2 "general_operand" "d,R,T")))
-   (set (match_operand:SI 0 "register_operand" "=d,d,d")
-        (minus:SI (match_dup 1) (match_dup 2)))]
+        (compare (match_operand:GPR 1 "register_operand" "0,0,0")
+                 (match_operand:GPR 2 "general_operand" "d,R,T")))
+   (set (match_operand:GPR 0 "register_operand" "=d,d,d")
+        (minus:GPR (match_dup 1) (match_dup 2)))]
   "s390_match_ccmode (insn, CCL3mode)"
   "@
-   slr\t%0,%2
-   sl\t%0,%2
-   sly\t%0,%2"
-  [(set_attr "op_type"  "RR,RX,RXY")])
+   sl<g>r\t%0,%2
+   sl<g>\t%0,%2
+   sl<y>\t%0,%2"
+  [(set_attr "op_type"  "RR<E>,RX<Y>,RXY")])
 
-(define_insn "*subsi3_cconly"
+(define_insn "*sub<mode>3_cconly"
   [(set (reg CC_REGNUM)
-        (compare (minus:SI (match_operand:SI 1 "register_operand" "0,0,0")
-                           (match_operand:SI 2 "general_operand" "d,R,T"))
+        (compare (minus:GPR (match_operand:GPR 1 "register_operand" "0,0,0")
+                           (match_operand:GPR 2 "general_operand" "d,R,T"))
                  (const_int 0)))
-   (clobber (match_scratch:SI 0 "=d,d,d"))]
+   (clobber (match_scratch:GPR 0 "=d,d,d"))]
   "s390_match_ccmode (insn, CCLmode)"
   "@
-   slr\t%0,%2
-   sl\t%0,%2
-   sly\t%0,%2"
-  [(set_attr "op_type"  "RR,RX,RXY")])
+   sl<g>r\t%0,%2
+   sl<g>\t%0,%2
+   sl<y>\t%0,%2"
+  [(set_attr "op_type"  "RR<E>,RX<Y>,RXY")])
 
-(define_insn "*subsi3_cconly2"
+(define_insn "*sub<mode>3_cconly2"
   [(set (reg CC_REGNUM)
-        (compare (match_operand:SI 1 "register_operand" "0,0,0")
-                 (match_operand:SI 2 "general_operand" "d,R,T")))
-   (clobber (match_scratch:SI 0 "=d,d,d"))]
+        (compare (match_operand:GPR 1 "register_operand" "0,0,0")
+                 (match_operand:GPR 2 "general_operand" "d,R,T")))
+   (clobber (match_scratch:GPR 0 "=d,d,d"))]
   "s390_match_ccmode (insn, CCL3mode)"
   "@
-   slr\t%0,%2
-   sl\t%0,%2
-   sly\t%0,%2"
-  [(set_attr "op_type"  "RR,RX,RXY")])
-
-(define_insn "*subsi3_sign"
-  [(set (match_operand:SI 0 "register_operand" "=d,d")
-        (minus:SI (match_operand:SI 1 "register_operand" "0,0")
-                  (sign_extend:SI (match_operand:HI 2 "memory_operand" "R,T"))))
-   (clobber (reg:CC CC_REGNUM))]
-  ""
-  "@
-   sh\t%0,%2
-   shy\t%0,%2"
-  [(set_attr "op_type"  "RX,RXY")])
-
-(define_insn "subsi3"
-  [(set (match_operand:SI 0 "register_operand" "=d,d,d")
-        (minus:SI (match_operand:SI 1 "register_operand" "0,0,0")
-                  (match_operand:SI 2 "general_operand" "d,R,T")))
-   (clobber (reg:CC CC_REGNUM))]
-  ""
-  "@
-   sr\t%0,%2
-   s\t%0,%2
-   sy\t%0,%2"
-  [(set_attr "op_type"  "RR,RX,RXY")])
-
+   sl<g>r\t%0,%2
+   sl<g>\t%0,%2
+   sl<y>\t%0,%2"
+  [(set_attr "op_type"  "RR<E>,RX<Y>,RXY")])
 
 ;
 ; sub(df|sf)3 instruction pattern(s).
 (define_insn "*sub<mode>3"
   [(set (match_operand:FPR 0 "register_operand" "=f,f")
         (minus:FPR (match_operand:FPR 1 "register_operand" "0,0")
-                   (match_operand:FPR 2 "general_operand" "f,R")))
+                   (match_operand:FPR 2 "general_operand" "f,<Rf>")))
    (clobber (reg:CC CC_REGNUM))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   s<de>br\t%0,%2
-   s<de>b\t%0,%2"
+   s<xde>br\t%0,%2
+   s<xde>b\t%0,%2"
   [(set_attr "op_type"  "RRE,RXE")
    (set_attr "type"     "fsimp<mode>")])
 
 (define_insn "*sub<mode>3_cc"
   [(set (reg CC_REGNUM)
        (compare (minus:FPR (match_operand:FPR 1 "nonimmediate_operand" "0,0")
-                           (match_operand:FPR 2 "general_operand" "f,R"))
+                           (match_operand:FPR 2 "general_operand" "f,<Rf>"))
                 (match_operand:FPR 3 "const0_operand" "")))
    (set (match_operand:FPR 0 "register_operand" "=f,f")
        (minus:FPR (match_dup 1) (match_dup 2)))]
   "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   s<de>br\t%0,%2
-   s<de>b\t%0,%2"
+   s<xde>br\t%0,%2
+   s<xde>b\t%0,%2"
   [(set_attr "op_type"  "RRE,RXE")
    (set_attr "type"     "fsimp<mode>")])
 
 (define_insn "*sub<mode>3_cconly"
   [(set (reg CC_REGNUM)
        (compare (minus:FPR (match_operand:FPR 1 "nonimmediate_operand" "0,0")
-                           (match_operand:FPR 2 "general_operand" "f,R"))
+                           (match_operand:FPR 2 "general_operand" "f,<Rf>"))
                 (match_operand:FPR 3 "const0_operand" "")))
    (clobber (match_scratch:FPR 0 "=f,f"))]
   "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   s<de>br\t%0,%2
-   s<de>b\t%0,%2"
+   s<xde>br\t%0,%2
+   s<xde>b\t%0,%2"
   [(set_attr "op_type"  "RRE,RXE")
    (set_attr "type"     "fsimp<mode>")])
 
 (define_insn "*sub<mode>3_ibm"
   [(set (match_operand:FPR 0 "register_operand" "=f,f")
         (minus:FPR (match_operand:FPR 1 "register_operand" "0,0")
-                   (match_operand:FPR 2 "general_operand" "f,R")))
+                   (match_operand:FPR 2 "general_operand" "f,<Rf>")))
    (clobber (reg:CC CC_REGNUM))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
   "@
-   s<de>r\t%0,%2
-   s<de>\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
+   s<xde>r\t%0,%2
+   s<xde>\t%0,%2"
+  [(set_attr "op_type"  "<RRe>,<RXe>")
    (set_attr "type"     "fsimp<mode>")])
 
 
 (define_expand "mul<mode>3"
   [(set (match_operand:FPR 0 "register_operand" "=f,f")
         (mult:FPR (match_operand:FPR 1 "nonimmediate_operand" "%0,0")
-                  (match_operand:FPR 2 "general_operand" "f,R")))]
+                  (match_operand:FPR 2 "general_operand" "f,<Rf>")))]
   "TARGET_HARD_FLOAT"
   "")
 
 (define_insn "*mul<mode>3"
   [(set (match_operand:FPR 0 "register_operand" "=f,f")
         (mult:FPR (match_operand:FPR 1 "nonimmediate_operand" "%0,0")
-                  (match_operand:FPR 2 "general_operand" "f,R")))]
+                  (match_operand:FPR 2 "general_operand" "f,<Rf>")))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   m<dee>br\t%0,%2
-   m<dee>b\t%0,%2"
+   m<xdee>br\t%0,%2
+   m<xdee>b\t%0,%2"
   [(set_attr "op_type"  "RRE,RXE")
    (set_attr "type"     "fmul<mode>")])
 
 (define_insn "*mul<mode>3_ibm"
   [(set (match_operand:FPR 0 "register_operand" "=f,f")
         (mult:FPR (match_operand:FPR 1 "nonimmediate_operand" "%0,0")
-                  (match_operand:FPR 2 "general_operand" "f,R")))]
+                  (match_operand:FPR 2 "general_operand" "f,<Rf>")))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
   "@
-   m<de>r\t%0,%2
-   m<de>\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
+   m<xde>r\t%0,%2
+   m<xde>\t%0,%2"
+  [(set_attr "op_type"  "<RRe>,<RXe>")
    (set_attr "type"     "fmul<mode>")])
 
 (define_insn "*fmadd<mode>"
-  [(set (match_operand:FPR 0 "register_operand" "=f,f")
-       (plus:FPR (mult:FPR (match_operand:FPR 1 "register_operand" "%f,f")
-                          (match_operand:FPR 2 "nonimmediate_operand"  "f,R"))
-                (match_operand:FPR 3 "register_operand" "0,0")))]
+  [(set (match_operand:DSF 0 "register_operand" "=f,f")
+       (plus:DSF (mult:DSF (match_operand:DSF 1 "register_operand" "%f,f")
+                           (match_operand:DSF 2 "nonimmediate_operand"  "f,R"))
+                (match_operand:DSF 3 "register_operand" "0,0")))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT && TARGET_FUSED_MADD"
   "@
-   ma<de>br\t%0,%1,%2
-   ma<de>b\t%0,%1,%2"
+   ma<xde>br\t%0,%1,%2
+   ma<xde>b\t%0,%1,%2"
   [(set_attr "op_type"  "RRE,RXE")
    (set_attr "type"     "fmul<mode>")])
 
 (define_insn "*fmsub<mode>"
-  [(set (match_operand:FPR 0 "register_operand" "=f,f")
-       (minus:FPR (mult:FPR (match_operand:FPR 1 "register_operand" "f,f")
-                           (match_operand:FPR 2 "nonimmediate_operand"  "f,R"))
-                (match_operand:FPR 3 "register_operand" "0,0")))]
+  [(set (match_operand:DSF 0 "register_operand" "=f,f")
+       (minus:DSF (mult:DSF (match_operand:DSF 1 "register_operand" "f,f")
+                            (match_operand:DSF 2 "nonimmediate_operand"  "f,R"))
+                (match_operand:DSF 3 "register_operand" "0,0")))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT && TARGET_FUSED_MADD"
   "@
-   ms<de>br\t%0,%1,%2
-   ms<de>b\t%0,%1,%2"
+   ms<xde>br\t%0,%1,%2
+   ms<xde>b\t%0,%1,%2"
   [(set_attr "op_type"  "RRE,RXE")
    (set_attr "type"     "fmul<mode>")])
 
 (define_expand "div<mode>3"
   [(set (match_operand:FPR 0 "register_operand" "=f,f")
         (div:FPR (match_operand:FPR 1 "register_operand" "0,0")
-                 (match_operand:FPR 2 "general_operand" "f,R")))]
+                 (match_operand:FPR 2 "general_operand" "f,<Rf>")))]
   "TARGET_HARD_FLOAT"
   "")
 
 (define_insn "*div<mode>3"
   [(set (match_operand:FPR 0 "register_operand" "=f,f")
         (div:FPR (match_operand:FPR 1 "register_operand" "0,0")
-                 (match_operand:FPR 2 "general_operand" "f,R")))]
+                 (match_operand:FPR 2 "general_operand" "f,<Rf>")))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   d<de>br\t%0,%2
-   d<de>b\t%0,%2"
+   d<xde>br\t%0,%2
+   d<xde>b\t%0,%2"
   [(set_attr "op_type"  "RRE,RXE")
    (set_attr "type"     "fdiv<mode>")])
 
 (define_insn "*div<mode>3_ibm"
   [(set (match_operand:FPR 0 "register_operand" "=f,f")
         (div:FPR (match_operand:FPR 1 "register_operand" "0,0")
-                 (match_operand:FPR 2 "general_operand" "f,R")))]
+                 (match_operand:FPR 2 "general_operand" "f,<Rf>")))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
   "@
-   d<de>r\t%0,%2
-   d<de>\t%0,%2"
-  [(set_attr "op_type"  "RR,RX")
+   d<xde>r\t%0,%2
+   d<xde>\t%0,%2"
+  [(set_attr "op_type"  "<RRe>,<RXe>")
    (set_attr "type"     "fdiv<mode>")])
 
 
    (set (match_operand:FPR 0 "register_operand" "=f")
         (neg:FPR (match_dup 1)))]
   "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "lc<de>br\t%0,%1"
+  "lc<xde>br\t%0,%1"
   [(set_attr "op_type"  "RRE")
    (set_attr "type"     "fsimp<mode>")])
   
                  (match_operand:FPR 2 "const0_operand" "")))
    (clobber (match_scratch:FPR 0 "=f"))]
   "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "lc<de>br\t%0,%1"
+  "lc<xde>br\t%0,%1"
   [(set_attr "op_type"  "RRE")
    (set_attr "type"     "fsimp<mode>")])
   
         (neg:FPR (match_operand:FPR 1 "register_operand" "f")))
    (clobber (reg:CC CC_REGNUM))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "lc<de>br\t%0,%1"
+  "lc<xde>br\t%0,%1"
   [(set_attr "op_type"  "RRE")
    (set_attr "type"     "fsimp<mode>")])
 
         (neg:FPR (match_operand:FPR 1 "register_operand" "f")))
    (clobber (reg:CC CC_REGNUM))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
-  "lc<de>r\t%0,%1"
-  [(set_attr "op_type"  "RR")
+  "lc<xde>r\t%0,%1"
+  [(set_attr "op_type"  "<RRe>")
    (set_attr "type"     "fsimp<mode>")])
 
 
    (set (match_operand:FPR 0 "register_operand" "=f")
         (abs:FPR (match_dup 1)))]
   "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "lp<de>br\t%0,%1"
+  "lp<xde>br\t%0,%1"
   [(set_attr "op_type"  "RRE")
    (set_attr "type"     "fsimp<mode>")])
   
                  (match_operand:FPR 2 "const0_operand" "")))
    (clobber (match_scratch:FPR 0 "=f"))]
   "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "lp<de>br\t%0,%1"
+  "lp<xde>br\t%0,%1"
   [(set_attr "op_type"  "RRE")
    (set_attr "type"     "fsimp<mode>")])
   
         (abs:FPR (match_operand:FPR 1 "register_operand" "f")))
    (clobber (reg:CC CC_REGNUM))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "lp<de>br\t%0,%1"
+  "lp<xde>br\t%0,%1"
   [(set_attr "op_type"  "RRE")
    (set_attr "type"     "fsimp<mode>")])
 
         (abs:FPR (match_operand:FPR 1 "register_operand" "f")))
    (clobber (reg:CC CC_REGNUM))]
   "TARGET_HARD_FLOAT && TARGET_IBM_FLOAT"
-  "lp<de>r\t%0,%1"
-  [(set_attr "op_type"  "RR")
+  "lp<xde>r\t%0,%1"
+  [(set_attr "op_type"  "<RRe>")
    (set_attr "type"     "fsimp<mode>")])
 
 ;;
    (set (match_operand:FPR 0 "register_operand" "=f")
         (neg:FPR (abs:FPR (match_dup 1))))]
   "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "ln<de>br\t%0,%1"
+  "ln<xde>br\t%0,%1"
   [(set_attr "op_type"  "RRE")
    (set_attr "type"     "fsimp<mode>")])
   
                  (match_operand:FPR 2 "const0_operand" "")))
    (clobber (match_scratch:FPR 0 "=f"))]
   "s390_match_ccmode (insn, CCSmode) && TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "ln<de>br\t%0,%1"
+  "ln<xde>br\t%0,%1"
   [(set_attr "op_type"  "RRE")
    (set_attr "type"     "fsimp<mode>")])
   
         (neg:FPR (abs:FPR (match_operand:FPR 1 "register_operand" "f"))))
    (clobber (reg:CC CC_REGNUM))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
-  "ln<de>br\t%0,%1"
+  "ln<xde>br\t%0,%1"
   [(set_attr "op_type"  "RRE")
    (set_attr "type"     "fsimp<mode>")])
 
 
 (define_insn "sqrt<mode>2"
   [(set (match_operand:FPR 0 "register_operand" "=f,f")
-       (sqrt:FPR (match_operand:FPR 1 "general_operand" "f,R")))]
+       (sqrt:FPR (match_operand:FPR 1 "general_operand" "f,<Rf>")))]
   "TARGET_HARD_FLOAT && TARGET_IEEE_FLOAT"
   "@
-   sq<de>br\t%0,%1
-   sq<de>b\t%0,%1"
+   sq<xde>br\t%0,%1
+   sq<xde>b\t%0,%1"
   [(set_attr "op_type" "RRE,RXE")
    (set_attr "type" "fsqrt<mode>")])
 
 (define_insn "rotl<mode>3"
   [(set (match_operand:GPR 0 "register_operand" "=d")
        (rotate:GPR (match_operand:GPR 1 "register_operand" "d")
-                   (match_operand:SI 2 "shift_count_operand" "Y")))]
+                   (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")))]
   "TARGET_CPU_ZARCH"
   "rll<g>\t%0,%1,%Y2"
   [(set_attr "op_type"  "RSE")
    (set_attr "atype"    "reg")])
 
+(define_insn "*rotl<mode>3_and"
+  [(set (match_operand:GPR 0 "register_operand" "=d")
+       (rotate:GPR (match_operand:GPR 1 "register_operand" "d")
+                   (and:SI (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")
+                           (match_operand:SI 3 "const_int_operand"   "n"))))]
+  "TARGET_CPU_ZARCH && (INTVAL (operands[3]) & 63) == 63"
+  "rll<g>\t%0,%1,%Y2"
+  [(set_attr "op_type"  "RSE")
+   (set_attr "atype"    "reg")])
+
 
 ;;
 ;;- Shift instructions.
 ;;
 
 ;
-; (ashl|lshr)di3 instruction pattern(s).
+; (ashl|lshr)(di|si)3 instruction pattern(s).
 ;
 
-(define_expand "<shift>di3"
-  [(set (match_operand:DI 0 "register_operand" "")
-        (SHIFT:DI (match_operand:DI 1 "register_operand" "")
-                  (match_operand:SI 2 "shift_count_operand" "")))]
+(define_expand "<shift><mode>3"
+  [(set (match_operand:DSI 0 "register_operand" "")
+        (SHIFT:DSI (match_operand:DSI 1 "register_operand" "")
+                   (match_operand:SI 2 "shift_count_or_setmem_operand" "")))]
   ""
   "")
 
 (define_insn "*<shift>di3_31"
   [(set (match_operand:DI 0 "register_operand" "=d")
         (SHIFT:DI (match_operand:DI 1 "register_operand" "0")
-                  (match_operand:SI 2 "shift_count_operand" "Y")))]
+                  (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")))]
   "!TARGET_64BIT"
   "s<lr>dl\t%0,%Y2"
   [(set_attr "op_type"  "RS")
    (set_attr "atype"    "reg")])
 
-(define_insn "*<shift>di3_64"
+(define_insn "*<shift><mode>3"
+  [(set (match_operand:GPR 0 "register_operand" "=d")
+        (SHIFT:GPR (match_operand:GPR 1 "register_operand" "<d0>")
+                   (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")))]
+  ""
+  "s<lr>l<g>\t%0,<1>%Y2"
+  [(set_attr "op_type"  "RS<E>")
+   (set_attr "atype"    "reg")])
+
+(define_insn "*<shift>di3_31_and"
   [(set (match_operand:DI 0 "register_operand" "=d")
-        (SHIFT:DI (match_operand:DI 1 "register_operand" "d")
-                  (match_operand:SI 2 "shift_count_operand" "Y")))]
-  "TARGET_64BIT"
-  "s<lr>lg\t%0,%1,%Y2"
-  [(set_attr "op_type"  "RSE")
+        (SHIFT:DI (match_operand:DI 1 "register_operand" "0")
+                  (and:SI (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")
+                         (match_operand:SI 3 "const_int_operand"   "n"))))]
+  "!TARGET_64BIT && (INTVAL (operands[3]) & 63) == 63"
+  "s<lr>dl\t%0,%Y2"
+  [(set_attr "op_type"  "RS")
+   (set_attr "atype"    "reg")])
+
+(define_insn "*<shift><mode>3_and"
+  [(set (match_operand:GPR 0 "register_operand" "=d")
+        (SHIFT:GPR (match_operand:GPR 1 "register_operand" "<d0>")
+                   (and:SI (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")
+                          (match_operand:SI 3 "const_int_operand"   "n"))))]
+  "(INTVAL (operands[3]) & 63) == 63"
+  "s<lr>l<g>\t%0,<1>%Y2"
+  [(set_attr "op_type"  "RS<E>")
    (set_attr "atype"    "reg")])
 
 ;
-; ashrdi3 instruction pattern(s).
+; ashr(di|si)3 instruction pattern(s).
 ;
 
-(define_expand "ashrdi3"
+(define_expand "ashr<mode>3"
   [(parallel
-    [(set (match_operand:DI 0 "register_operand" "")
-          (ashiftrt:DI (match_operand:DI 1 "register_operand" "")
-                       (match_operand:SI 2 "shift_count_operand" "")))
+    [(set (match_operand:DSI 0 "register_operand" "")
+          (ashiftrt:DSI (match_operand:DSI 1 "register_operand" "")
+                        (match_operand:SI 2 "shift_count_or_setmem_operand" "")))
      (clobber (reg:CC CC_REGNUM))])]
   ""
   "")
 (define_insn "*ashrdi3_cc_31"
   [(set (reg CC_REGNUM)
         (compare (ashiftrt:DI (match_operand:DI 1 "register_operand" "0")
-                              (match_operand:SI 2 "shift_count_operand" "Y"))
+                              (match_operand:SI 2 "shift_count_or_setmem_operand" "Y"))
                  (const_int 0)))
    (set (match_operand:DI 0 "register_operand" "=d")
         (ashiftrt:DI (match_dup 1) (match_dup 2)))]
 (define_insn "*ashrdi3_cconly_31"
   [(set (reg CC_REGNUM)
         (compare (ashiftrt:DI (match_operand:DI 1 "register_operand" "0")
-                              (match_operand:SI 2 "shift_count_operand" "Y"))
+                              (match_operand:SI 2 "shift_count_or_setmem_operand" "Y"))
                  (const_int 0)))
    (clobber (match_scratch:DI 0 "=d"))]
   "!TARGET_64BIT && s390_match_ccmode(insn, CCSmode)"
 (define_insn "*ashrdi3_31"
   [(set (match_operand:DI 0 "register_operand" "=d")
         (ashiftrt:DI (match_operand:DI 1 "register_operand" "0")
-                     (match_operand:SI 2 "shift_count_operand" "Y")))
+                     (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")))
    (clobber (reg:CC CC_REGNUM))]
   "!TARGET_64BIT"
   "srda\t%0,%Y2"
   [(set_attr "op_type"  "RS")
    (set_attr "atype"    "reg")])
 
-(define_insn "*ashrdi3_cc_64"
+(define_insn "*ashr<mode>3_cc"
   [(set (reg CC_REGNUM)
-        (compare (ashiftrt:DI (match_operand:DI 1 "register_operand" "d")
-                              (match_operand:SI 2 "shift_count_operand" "Y"))
+        (compare (ashiftrt:GPR (match_operand:GPR 1 "register_operand" "<d0>")
+                               (match_operand:SI 2 "shift_count_or_setmem_operand" "Y"))
                  (const_int 0)))
-   (set (match_operand:DI 0 "register_operand" "=d")
-        (ashiftrt:DI (match_dup 1) (match_dup 2)))]
-  "s390_match_ccmode(insn, CCSmode) && TARGET_64BIT"
-  "srag\t%0,%1,%Y2"
-  [(set_attr "op_type"  "RSE")
+   (set (match_operand:GPR 0 "register_operand" "=d")
+        (ashiftrt:GPR (match_dup 1) (match_dup 2)))]
+  "s390_match_ccmode(insn, CCSmode)"
+  "sra<g>\t%0,<1>%Y2"
+  [(set_attr "op_type"  "RS<E>")
    (set_attr "atype"    "reg")])
 
-(define_insn "*ashrdi3_cconly_64"
+(define_insn "*ashr<mode>3_cconly"
   [(set (reg CC_REGNUM)
-        (compare (ashiftrt:DI (match_operand:DI 1 "register_operand" "d")
-                              (match_operand:SI 2 "shift_count_operand" "Y"))
+        (compare (ashiftrt:GPR (match_operand:GPR 1 "register_operand" "<d0>")
+                               (match_operand:SI 2 "shift_count_or_setmem_operand" "Y"))
                  (const_int 0)))
-   (clobber (match_scratch:DI 0 "=d"))]
-  "s390_match_ccmode(insn, CCSmode) && TARGET_64BIT"
-  "srag\t%0,%1,%Y2"
-  [(set_attr "op_type"  "RSE")
+   (clobber (match_scratch:GPR 0 "=d"))]
+  "s390_match_ccmode(insn, CCSmode)"
+  "sra<g>\t%0,<1>%Y2"
+  [(set_attr "op_type"  "RS<E>")
    (set_attr "atype"    "reg")])
 
-(define_insn "*ashrdi3_64"
-  [(set (match_operand:DI 0 "register_operand" "=d")
-        (ashiftrt:DI (match_operand:DI 1 "register_operand" "d")
-                     (match_operand:SI 2 "shift_count_operand" "Y")))
+(define_insn "*ashr<mode>3"
+  [(set (match_operand:GPR 0 "register_operand" "=d")
+        (ashiftrt:GPR (match_operand:GPR 1 "register_operand" "<d0>")
+                      (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")))
    (clobber (reg:CC CC_REGNUM))]
-  "TARGET_64BIT"
-  "srag\t%0,%1,%Y2"
-  [(set_attr "op_type"  "RSE")
+  ""
+  "sra<g>\t%0,<1>%Y2"
+  [(set_attr "op_type"  "RS<E>")
    (set_attr "atype"    "reg")])
 
 
-;
-; (ashl|lshr)si3 instruction pattern(s).
-;
+; shift pattern with implicit ANDs
 
-(define_insn "<shift>si3"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (SHIFT:SI (match_operand:SI 1 "register_operand" "0")
-                  (match_operand:SI 2 "shift_count_operand" "Y")))]
-  ""
-  "s<lr>l\t%0,%Y2"
+(define_insn "*ashrdi3_cc_31_and"
+  [(set (reg CC_REGNUM)
+        (compare (ashiftrt:DI (match_operand:DI 1 "register_operand" "0")
+                              (and:SI (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")
+                                     (match_operand:SI 3 "const_int_operand"   "n")))
+                (const_int 0)))
+   (set (match_operand:DI 0 "register_operand" "=d")
+        (ashiftrt:DI (match_dup 1) (and:SI (match_dup 2) (match_dup 3))))]
+  "!TARGET_64BIT && s390_match_ccmode(insn, CCSmode)
+   && (INTVAL (operands[3]) & 63) == 63"
+  "srda\t%0,%Y2"
   [(set_attr "op_type"  "RS")
    (set_attr "atype"    "reg")])
 
-;
-; ashrsi3 instruction pattern(s).
-;
-
-(define_insn "*ashrsi3_cc"
+(define_insn "*ashrdi3_cconly_31_and"
   [(set (reg CC_REGNUM)
-        (compare (ashiftrt:SI (match_operand:SI 1 "register_operand" "0")
-                              (match_operand:SI 2 "shift_count_operand" "Y"))
+        (compare (ashiftrt:DI (match_operand:DI 1 "register_operand" "0")
+                              (and:SI (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")
+                                     (match_operand:SI 3 "const_int_operand"   "n")))
                  (const_int 0)))
-   (set (match_operand:SI 0 "register_operand" "=d")
-        (ashiftrt:SI (match_dup 1) (match_dup 2)))]
-  "s390_match_ccmode(insn, CCSmode)"
-  "sra\t%0,%Y2"
+   (clobber (match_scratch:DI 0 "=d"))]
+  "!TARGET_64BIT && s390_match_ccmode(insn, CCSmode)
+   && (INTVAL (operands[3]) & 63) == 63"
+  "srda\t%0,%Y2"
+  [(set_attr "op_type"  "RS")
+   (set_attr "atype"    "reg")])
+
+(define_insn "*ashrdi3_31_and"
+  [(set (match_operand:DI 0 "register_operand" "=d")
+        (ashiftrt:DI (match_operand:DI 1 "register_operand" "0")
+                     (and:SI (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")
+                            (match_operand:SI 3 "const_int_operand"   "n"))))
+   (clobber (reg:CC CC_REGNUM))]
+  "!TARGET_64BIT && (INTVAL (operands[3]) & 63) == 63"
+  "srda\t%0,%Y2"
   [(set_attr "op_type"  "RS")
    (set_attr "atype"    "reg")])
 
+(define_insn "*ashr<mode>3_cc_and"
+  [(set (reg CC_REGNUM)
+        (compare (ashiftrt:GPR (match_operand:GPR 1 "register_operand" "<d0>")
+                               (and:SI (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")
+                                      (match_operand:SI 3 "const_int_operand"   "n")))
+                (const_int 0)))
+   (set (match_operand:GPR 0 "register_operand" "=d")
+        (ashiftrt:GPR (match_dup 1) (and:SI (match_dup 2) (match_dup 3))))]
+  "s390_match_ccmode(insn, CCSmode) && (INTVAL (operands[3]) & 63) == 63"
+  "sra<g>\t%0,<1>%Y2"
+  [(set_attr "op_type"  "RS<E>")
+   (set_attr "atype"    "reg")])
 
-(define_insn "*ashrsi3_cconly"
+(define_insn "*ashr<mode>3_cconly_and"
   [(set (reg CC_REGNUM)
-        (compare (ashiftrt:SI (match_operand:SI 1 "register_operand" "0")
-                              (match_operand:SI 2 "shift_count_operand" "Y"))
+        (compare (ashiftrt:GPR (match_operand:GPR 1 "register_operand" "<d0>")
+                               (and:SI (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")
+                                      (match_operand:SI 3 "const_int_operand"   "n")))
                  (const_int 0)))
-   (clobber (match_scratch:SI 0 "=d"))]
-  "s390_match_ccmode(insn, CCSmode)"
-  "sra\t%0,%Y2"
-  [(set_attr "op_type"  "RS")
+   (clobber (match_scratch:GPR 0 "=d"))]
+  "s390_match_ccmode(insn, CCSmode) && (INTVAL (operands[3]) & 63) == 63"
+  "sra<g>\t%0,<1>%Y2"
+  [(set_attr "op_type"  "RS<E>")
    (set_attr "atype"    "reg")])
 
-(define_insn "ashrsi3"
-  [(set (match_operand:SI 0 "register_operand" "=d")
-        (ashiftrt:SI (match_operand:SI 1 "register_operand" "0")
-                     (match_operand:SI 2 "shift_count_operand" "Y")))
+(define_insn "*ashr<mode>3_and"
+  [(set (match_operand:GPR 0 "register_operand" "=d")
+        (ashiftrt:GPR (match_operand:GPR 1 "register_operand" "<d0>")
+                      (and:SI (match_operand:SI 2 "shift_count_or_setmem_operand" "Y")
+                             (match_operand:SI 3 "const_int_operand"   "n"))))
    (clobber (reg:CC CC_REGNUM))]
-  ""
-  "sra\t%0,%Y2"
-  [(set_attr "op_type"  "RS")
+  "(INTVAL (operands[3]) & 63) == 63"
+  "sra<g>\t%0,<1>%Y2"
+  [(set_attr "op_type"  "RS<E>")
    (set_attr "atype"    "reg")])
 
 
          (compare:CCZ1 (match_dup 1) (match_dup 2)))])]
   "")
 
+(define_expand "sync_compare_and_swap<mode>"
+  [(parallel
+    [(set (match_operand:HQI 0 "register_operand" "")
+         (match_operand:HQI 1 "memory_operand" ""))
+     (set (match_dup 1)
+         (unspec_volatile:HQI
+           [(match_dup 1)
+            (match_operand:HQI 2 "general_operand" "")
+            (match_operand:HQI 3 "general_operand" "")]
+           UNSPECV_CAS))
+     (set (reg:CCZ1 CC_REGNUM)
+         (compare:CCZ1 (match_dup 1) (match_dup 2)))])]
+  ""
+  "s390_expand_cs_hqi (<MODE>mode, operands[0], operands[1], 
+                      operands[2], operands[3]); DONE;")
+
 (define_expand "sync_compare_and_swap_cc<mode>"
   [(parallel
     [(set (match_operand:TDSI 0 "register_operand" "")
    (set_attr "type"   "sem")])
 
 
+;
+; Other atomic instruction patterns.
+;
+
+(define_expand "sync_lock_test_and_set<mode>"
+  [(match_operand:HQI 0 "register_operand")
+   (match_operand:HQI 1 "memory_operand")
+   (match_operand:HQI 2 "general_operand")]
+  ""
+  "s390_expand_atomic (<MODE>mode, SET, operands[0], operands[1], 
+                      operands[2], false); DONE;")
+
+(define_expand "sync_<atomic><mode>"
+  [(set (match_operand:HQI 0 "memory_operand")
+       (ATOMIC:HQI (match_dup 0)
+                   (match_operand:HQI 1 "general_operand")))]
+  ""
+  "s390_expand_atomic (<MODE>mode, <CODE>, NULL_RTX, operands[0], 
+                      operands[1], false); DONE;")
+
+(define_expand "sync_old_<atomic><mode>"
+  [(set (match_operand:HQI 0 "register_operand")
+       (match_operand:HQI 1 "memory_operand"))
+   (set (match_dup 1)
+       (ATOMIC:HQI (match_dup 1)
+                   (match_operand:HQI 2 "general_operand")))]
+  ""
+  "s390_expand_atomic (<MODE>mode, <CODE>, operands[0], operands[1], 
+                      operands[2], false); DONE;")
+
+(define_expand "sync_new_<atomic><mode>"
+  [(set (match_operand:HQI 0 "register_operand")
+       (ATOMIC:HQI (match_operand:HQI 1 "memory_operand")
+                   (match_operand:HQI 2 "general_operand"))) 
+   (set (match_dup 1) (ATOMIC:HQI (match_dup 1) (match_dup 2)))]
+  ""
+  "s390_expand_atomic (<MODE>mode, <CODE>, operands[0], operands[1], 
+                      operands[2], true); DONE;")
+
 ;;
 ;;- Miscellaneous instructions.
 ;;