OSDN Git Service

PR target/5362
[pf3gnuchains/gcc-fork.git] / gcc / config / ia64 / ia64.md
index 01e4d73..b4894ab 100644 (file)
@@ -1,5 +1,6 @@
 ;; IA-64 Machine description template
-;; Copyright (C) 1999, 2000, 2001, 2002, 2003 Free Software Foundation, Inc.
+;; Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008
+;; Free Software Foundation, Inc.
 ;; Contributed by James E. Wilson <wilson@cygnus.com> and
 ;;               David Mosberger <davidm@hpl.hp.com>.
 
@@ -7,7 +8,7 @@
 
 ;; GCC is free software; you can redistribute it and/or modify
 ;; it under the terms of the GNU General Public License as published by
-;; the Free Software Foundation; either version 2, or (at your option)
+;; the Free Software Foundation; either version 3, or (at your option)
 ;; any later version.
 
 ;; GCC is distributed in the hope that it will be useful,
@@ -16,9 +17,8 @@
 ;; GNU General Public License for more details.
 
 ;; You should have received a copy of the GNU General Public License
-;; along with GCC; see the file COPYING.  If not, write to
-;; the Free Software Foundation, 59 Temple Place - Suite 330,
-;; Boston, MA 02111-1307, USA.
+;; along with GCC; see the file COPYING3.  If not see
+;; <http://www.gnu.org/licenses/>.
 
 ;;- See file "rtl.def" for documentation on define_insn, match_*, et. al.
 
@@ -55,6 +55,7 @@
    (UNSPEC_DTPREL              2)
    (UNSPEC_LTOFF_TPREL         3)
    (UNSPEC_TPREL               4)
+   (UNSPEC_DTPMOD              5)
 
    (UNSPEC_LD_BASE             9)
    (UNSPEC_GR_SPILL            10)
    (UNSPEC_RET_ADDR            26)
    (UNSPEC_SETF_EXP             27)
    (UNSPEC_FR_SQRT_RECIP_APPROX 28)
+   (UNSPEC_SHRP                        29)
+   (UNSPEC_COPYSIGN            30)
+   (UNSPEC_VECT_EXTR           31)
+   (UNSPEC_LDA                  40)
+   (UNSPEC_LDS                  41)
+   (UNSPEC_LDS_A                42)
+   (UNSPEC_LDSA                 43)
+   (UNSPEC_LDCCLR               44)
+   (UNSPEC_LDCNC                45)
+   (UNSPEC_CHKACLR              46)
+   (UNSPEC_CHKANC               47)
+   (UNSPEC_CHKS                 48)
+   (UNSPEC_FR_RECIP_APPROX_RES  49)
+   (UNSPEC_FR_SQRT_RECIP_APPROX_RES 50)
   ])
 
 (define_constants
    (UNSPECV_PSAC_ALL           5)      ; pred.safe_across_calls
    (UNSPECV_PSAC_NORMAL                6)
    (UNSPECV_SETJMP_RECEIVER    7)
+   (UNSPECV_GOTO_RECEIVER      8)
   ])
+
+(include "predicates.md")
+(include "constraints.md")
 \f
 ;; ::::::::::::::::::::
 ;; ::
 ;; which emit instruction that can go in any slot (e.g. nop).
 
 (define_attr "itanium_class" "unknown,ignore,stop_bit,br,fcmp,fcvtfx,fld,
-       fmac,fmisc,frar_i,frar_m,frbr,frfr,frpr,ialu,icmp,ilog,ishf,ld,
-       chk_s,long_i,mmmul,mmshf,mmshfi,rse_m,scall,sem,stf,st,syst_m0,
-       syst_m,tbit,toar_i,toar_m,tobr,tofr,topr,xmpy,xtd,nop,nop_b,nop_f,
-       nop_i,nop_m,nop_x,lfetch,pre_cycle"
+       fldp,fmac,fmisc,frar_i,frar_m,frbr,frfr,frpr,ialu,icmp,ilog,ishf,
+       ld,chk_s_i,chk_s_f,chk_a,long_i,mmalua,mmmul,mmshf,mmshfi,rse_m,scall,sem,stf,
+        st,syst_m0, syst_m,tbit,toar_i,toar_m,tobr,tofr,topr,xmpy,xtd,nop,
+        nop_b,nop_f,nop_i,nop_m,nop_x,lfetch,pre_cycle"
   (const_string "unknown"))
 
-;; chk_s has an I and an M form; use type A for convenience.
+;; chk_s_i has an I and an M form; use type A for convenience.
 (define_attr "type" "unknown,A,I,M,F,B,L,X,S"
-  (cond [(eq_attr "itanium_class" "ld,st,fld,stf,sem,nop_m") (const_string "M")
+  (cond [(eq_attr "itanium_class" "ld,st,fld,fldp,stf,sem,nop_m") (const_string "M")
         (eq_attr "itanium_class" "rse_m,syst_m,syst_m0") (const_string "M")
         (eq_attr "itanium_class" "frar_m,toar_m,frfr,tofr") (const_string "M")
         (eq_attr "itanium_class" "lfetch") (const_string "M")
-        (eq_attr "itanium_class" "chk_s,ialu,icmp,ilog") (const_string "A")
+         (eq_attr "itanium_class" "chk_s_f,chk_a") (const_string "M")
+        (eq_attr "itanium_class" "chk_s_i,ialu,icmp,ilog,mmalua")
+          (const_string "A")
         (eq_attr "itanium_class" "fmisc,fmac,fcmp,xmpy") (const_string "F")
         (eq_attr "itanium_class" "fcvtfx,nop_f") (const_string "F")
         (eq_attr "itanium_class" "frar_i,toar_i,frbr,tobr") (const_string "I")
 
 (define_attr "predicable" "no,yes" (const_string "yes"))
 
-\f
+;; Empty.  True iff this insn does not generate any code.
+
+(define_attr "empty" "no,yes" (const_string "no"))
+
+;; True iff this insn must be the first insn of an instruction group.
+;; This is true for the alloc instruction, and will also be true of others
+;; when we have full intrinsics support.
+
+(define_attr "first_insn" "no,yes" (const_string "no"))
+
+(define_attr "data_speculative" "no,yes" (const_string "no"))
 
+(define_attr "control_speculative" "no,yes" (const_string "no"))
+
+(define_attr "check_load" "no,yes" (const_string "no"))
+
+(define_attr "speculable1" "no,yes" (const_string "no"))
+
+(define_attr "speculable2" "no,yes" (const_string "no"))
+\f
 ;; DFA descriptions of ia64 processors used for insn scheduling and
 ;; bundling.
 
 
 (automata_option "w")
 
-;;(automata_option "no-minimization")
-
-
 (include "itanium1.md")
 (include "itanium2.md")
 
    (set_attr "predicable" "no")])
 
 (define_insn "movbi"
-  [(set (match_operand:BI 0 "nonimmediate_operand" "=c,c,?c,?*r, c,*r,*r,*m,*r")
-       (match_operand:BI 1 "move_operand"         " O,n, c,  c,*r, n,*m,*r,*r"))]
+  [(set (match_operand:BI 0 "destination_operand" "=c,c,?c,?*r, c,*r,*r,*m,*r")
+       (match_operand:BI 1 "move_operand"        " O,n, c,  c,*r, n,*m,*r,*r"))]
   ""
   "@
    cmp.ne %0, %I0 = r0, r0
    ld1%O1 %0 = %1%P1
    st1%Q0 %0 = %1%P0
    mov %0 = %1"
-  [(set_attr "itanium_class" "icmp,icmp,unknown,unknown,tbit,ialu,ld,st,ialu")])
+  [(set_attr "itanium_class" "icmp,icmp,unknown,unknown,tbit,ialu,ld,st,ialu")
+   (set_attr "speculable1"   "yes")
+   (set_attr "speculable2"   "no,  no,  no,     no,     no,  no, yes,no,no")])
 
 (define_split
   [(set (match_operand:BI 0 "register_operand" "")
   operands[1] = op1;
 })
 
-(define_insn "*movqi_internal"
+(define_insn "movqi_internal"
   [(set (match_operand:QI 0 "destination_operand" "=r,r,r, m, r,*f,*f")
        (match_operand:QI 1 "move_operand"        "rO,J,m,rO,*f,rO,*f"))]
   "ia64_move_ok (operands[0], operands[1])"
    getf.sig %0 = %1
    setf.sig %0 = %r1
    mov %0 = %1"
-  [(set_attr "itanium_class" "ialu,ialu,ld,st,frfr,tofr,fmisc")])
+  [(set_attr "itanium_class" "ialu,ialu,ld,st,frfr,tofr,fmisc")
+   (set_attr "speculable1"   "yes")
+   (set_attr "speculable2"   "no,  no, yes,no,no,  no,  no")])
 
 (define_expand "movhi"
   [(set (match_operand:HI 0 "general_operand" "")
   operands[1] = op1;
 })
 
-(define_insn "*movhi_internal"
+(define_insn "movhi_internal"
   [(set (match_operand:HI 0 "destination_operand" "=r,r,r, m, r,*f,*f")
        (match_operand:HI 1 "move_operand"        "rO,J,m,rO,*f,rO,*f"))]
   "ia64_move_ok (operands[0], operands[1])"
    getf.sig %0 = %1
    setf.sig %0 = %r1
    mov %0 = %1"
-  [(set_attr "itanium_class" "ialu,ialu,ld,st,frfr,tofr,fmisc")])
+  [(set_attr "itanium_class" "ialu,ialu,ld,st,frfr,tofr,fmisc")
+   (set_attr "speculable1"   "yes")
+   (set_attr "speculable2"   "no,  no, yes,no,no,  no,  no")])
 
 (define_expand "movsi"
   [(set (match_operand:SI 0 "general_operand" "")
   operands[1] = op1;
 })
 
-(define_insn "*movsi_internal"
-  [(set (match_operand:SI 0 "destination_operand" "=r,r,r,r, m, r,*f,*f, r,*d")
-       (match_operand:SI 1 "move_operand"        "rO,J,i,m,rO,*f,rO,*f,*d,rK"))]
+(define_insn "movsi_internal"
+  [(set (match_operand:SI 0 "destination_operand" "=r,r,r,r,r, m, r,*f,*f, r,*d")
+       (match_operand:SI 1 "move_operand"        "rO,J,j,i,m,rO,*f,rO,*f,*d,rK"))]
   "ia64_move_ok (operands[0], operands[1])"
   "@
   mov %0 = %r1
   addl %0 = %1, r0
+  addp4 %0 = %1 - 0x100000000, r0
   movl %0 = %1
   ld4%O1 %0 = %1%P1
   st4%Q0 %0 = %r1%P0
   mov %0 = %1
   mov %0 = %r1"
   ;; frar_m, toar_m ??? why not frar_i and toar_i
-  [(set_attr "itanium_class" "ialu,ialu,long_i,ld,st,frfr,tofr,fmisc,frar_m,toar_m")])
+  [(set_attr "itanium_class" "ialu,ialu,ialu,long_i,ld,st,frfr,tofr,fmisc,frar_m,toar_m")
+   (set_attr "speculable1"   "yes")
+   (set_attr "speculable2"   "no,  no,  no,  no,   yes,no,no,  no,  no,   no,    no")])
 
 (define_expand "movdi"
   [(set (match_operand:DI 0 "general_operand" "")
   operands[1] = op1;
 })
 
-(define_insn "*movdi_internal"
+(define_insn "movdi_internal"
   [(set (match_operand:DI 0 "destination_operand"
-                   "=r,r,r,r, m, r,*f,*f,*f, Q, r,*b, r,*e, r,*d, r,*c")
+                   "=r,r,r,r,r, m, r,*f,*f,*f, Q, r,*b, r,*e, r,*d, r,*c")
        (match_operand:DI 1 "move_operand"
-                   "rO,JT,i,m,rO,*f,rO,*f, Q,*f,*b,rO,*e,rK,*d,rK,*c,rO"))]
+                   "rO,JT,j,i,m,rO,*f,rO,*f, Q,*f,*b,rO,*e,rK,*d,rK,*c,rO"))]
   "ia64_move_ok (operands[0], operands[1])"
 {
   static const char * const alt[] = {
     "%,mov %0 = %r1",
     "%,addl %0 = %1, r0",
+    "%,addp4 %0 = %1 - 0x100000000, r0",
     "%,movl %0 = %1",
     "%,ld8%O1 %0 = %1%P1",
     "%,st8%Q0 %0 = %r1%P0",
     "mov pr = %1, -1"
   };
 
-  if (which_alternative == 2 && ! TARGET_NO_PIC
-      && symbolic_operand (operands[1], VOIDmode))
-    abort ();
+  gcc_assert (which_alternative != 2 || TARGET_NO_PIC
+              || !symbolic_operand (operands[1], VOIDmode));
 
   return alt[which_alternative];
 }
-  [(set_attr "itanium_class" "ialu,ialu,long_i,ld,st,frfr,tofr,fmisc,fld,stf,frbr,tobr,frar_i,toar_i,frar_m,toar_m,frpr,topr")])
+  [(set_attr "itanium_class" "ialu,ialu,ialu,long_i,ld,st,frfr,tofr,fmisc,fld,stf,frbr,tobr,frar_i,toar_i,frar_m,toar_m,frpr,topr")
+   (set_attr "speculable1"   "yes")
+   (set_attr "speculable2"   "no,  no,  no,  no,   yes,no,no,  no,  no,   yes,no, no,  no,  no,    no,    no,    no,    no,  no")])
+
+(define_mode_iterator MODE [BI QI HI SI DI SF DF XF TI])
+(define_mode_iterator MODE_FOR_EXTEND [QI HI SI])
+
+(define_mode_attr output_a [
+  (BI "ld1.a %0 = %1%P1")
+  (QI "ld1.a %0 = %1%P1")
+  (HI "ld2.a %0 = %1%P1")
+  (SI "ld4.a %0 = %1%P1")
+  (DI
+   "@
+    ld8.a %0 = %1%P1
+    ldf8.a %0 = %1%P1")
+  (SF
+   "@
+    ldfs.a %0 = %1%P1
+    ld4.a %0 = %1%P1")
+  (DF
+   "@
+    ldfd.a %0 = %1%P1
+    ld8.a %0 = %1%P1")
+  (XF "ldfe.a %0 = %1%P1")
+  (TI "ldfp8.a %X0 = %1%P1")])
+
+(define_mode_attr output_s [
+  (BI "ld1.s %0 = %1%P1")
+  (QI "ld1.s %0 = %1%P1")
+  (HI "ld2.s %0 = %1%P1")
+  (SI "ld4.s %0 = %1%P1")
+  (DI
+   "@
+    ld8.s %0 = %1%P1
+    ldf8.s %0 = %1%P1")
+  (SF
+   "@
+    ldfs.s %0 = %1%P1
+    ld4.s %0 = %1%P1")
+  (DF
+   "@
+    ldfd.s %0 = %1%P1
+    ld8.s %0 = %1%P1")
+  (XF "ldfe.s %0 = %1%P1")
+  (TI "ldfp8.s %X0 = %1%P1")])
+
+(define_mode_attr output_sa [
+  (BI "ld1.sa %0 = %1%P1")
+  (QI "ld1.sa %0 = %1%P1")
+  (HI "ld2.sa %0 = %1%P1")
+  (SI "ld4.sa %0 = %1%P1")
+  (DI
+   "@
+    ld8.sa %0 = %1%P1
+    ldf8.sa %0 = %1%P1")
+  (SF
+   "@
+    ldfs.sa %0 = %1%P1
+    ld4.sa %0 = %1%P1")
+  (DF
+   "@
+    ldfd.sa %0 = %1%P1
+    ld8.sa %0 = %1%P1")
+  (XF "ldfe.sa %0 = %1%P1")
+  (TI "ldfp8.sa %X0 = %1%P1")])
+
+(define_mode_attr output_c_clr [
+  (BI "ld1.c.clr%O1 %0 = %1%P1")
+  (QI "ld1.c.clr%O1 %0 = %1%P1")
+  (HI "ld2.c.clr%O1 %0 = %1%P1")
+  (SI "ld4.c.clr%O1 %0 = %1%P1")
+  (DI
+   "@
+    ld8.c.clr%O1 %0 = %1%P1
+    ldf8.c.clr %0 = %1%P1")
+  (SF
+   "@
+    ldfs.c.clr %0 = %1%P1
+    ld4.c.clr%O1 %0 = %1%P1")
+  (DF
+   "@
+    ldfd.c.clr %0 = %1%P1
+    ld8.c.clr%O1 %0 = %1%P1")
+  (XF "ldfe.c.clr %0 = %1%P1")
+  (TI "ldfp8.c.clr %X0 = %1%P1")])
+
+(define_mode_attr output_c_nc [
+  (BI "ld1.c.nc%O1 %0 = %1%P1")
+  (QI "ld1.c.nc%O1 %0 = %1%P1")
+  (HI "ld2.c.nc%O1 %0 = %1%P1")
+  (SI "ld4.c.nc%O1 %0 = %1%P1")
+  (DI
+   "@
+    ld8.c.nc%O1 %0 = %1%P1
+    ldf8.c.nc %0 = %1%P1")
+  (SF
+   "@
+    ldfs.c.nc %0 = %1%P1
+    ld4.c.nc%O1 %0 = %1%P1")
+  (DF
+   "@
+    ldfd.c.nc %0 = %1%P1
+    ld8.c.nc%O1 %0 = %1%P1")
+  (XF "ldfe.c.nc %0 = %1%P1")
+  (TI "ldfp8.c.nc %X0 = %1%P1")])
+
+(define_mode_attr ld_reg_constr [(BI "=*r") (QI "=r") (HI "=r") (SI "=r") (DI "=r,*f") (SF "=f,*r") (DF "=f,*r") (XF "=f") (TI "=*x")])
+(define_mode_attr ldc_reg_constr [(BI "+*r") (QI "+r") (HI "+r") (SI "+r") (DI "+r,*f") (SF "+f,*r") (DF "+f,*r") (XF "+f") (TI "+*x")])
+(define_mode_attr chk_reg_constr [(BI "*r") (QI "r") (HI "r") (SI "r") (DI "r,*f") (SF "f,*r") (DF "f,*r") (XF "f") (TI "*x")])
+
+(define_mode_attr mem_constr [(BI "*m") (QI "m") (HI "m") (SI "m") (DI "m,Q") (SF "Q,m") (DF "Q,m") (XF "m") (TI "Q")])
+
+;; Define register predicate prefix.
+;; We can generate speculative loads only for general and fp registers - this
+;; is constrained in ia64.c: ia64_speculate_insn ().
+(define_mode_attr reg_pred_prefix [(BI "gr") (QI "gr") (HI "gr") (SI "gr") (DI "grfr") (SF "grfr") (DF "grfr") (XF "fr") (TI "fr")])
+
+(define_mode_attr ld_class [(BI "ld") (QI "ld") (HI "ld") (SI "ld") (DI "ld,fld") (SF "fld,ld") (DF "fld,ld") (XF "fld") (TI "fldp")])
+(define_mode_attr chka_class [(BI "chk_a") (QI "chk_a") (HI "chk_a") (SI "chk_a") (DI "chk_a,chk_a") (SF "chk_a,chk_a") (DF "chk_a,chk_a") (XF "chk_a") (TI "chk_a")])
+(define_mode_attr chks_class [(BI "chk_s_i") (QI "chk_s_i") (HI "chk_s_i") (SI "chk_s_i") (DI "chk_s_i,chk_s_f") (SF "chk_s_f,chk_s_i") (DF "chk_s_f,chk_s_i") (XF "chk_s_f") (TI "chk_s_i")])
+
+(define_mode_attr attr_yes [(BI "yes") (QI "yes") (HI "yes") (SI "yes") (DI "yes,yes") (SF "yes,yes") (DF "yes,yes") (XF "yes") (TI "yes")])
+
+(define_insn "mov<mode>_advanced"
+  [(set (match_operand:MODE 0 "<reg_pred_prefix>_register_operand" "<ld_reg_constr>")
+       (unspec:MODE [(match_operand:MODE 1 "memory_operand" "<mem_constr>")] UNSPEC_LDA))]
+  "ia64_move_ok (operands[0], operands[1])"
+  "<output_a>"
+  [(set_attr "itanium_class" "<ld_class>")
+   (set_attr "data_speculative" "<attr_yes>")])
+
+(define_insn "zero_extend<mode>di2_advanced"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (zero_extend:DI (unspec:MODE_FOR_EXTEND [(match_operand:MODE_FOR_EXTEND 1 "memory_operand" "<mem_constr>")] UNSPEC_LDA)))]
+  ""
+  "<output_a>"
+  [(set_attr "itanium_class" "<ld_class>")
+   (set_attr "data_speculative" "<attr_yes>")])
+
+(define_insn "mov<mode>_speculative"
+  [(set (match_operand:MODE 0 "<reg_pred_prefix>_register_operand" "<ld_reg_constr>")
+       (unspec:MODE [(match_operand:MODE 1 "memory_operand" "<mem_constr>")] UNSPEC_LDS))]
+  "ia64_move_ok (operands[0], operands[1])"
+  "<output_s>"
+  [(set_attr "itanium_class" "<ld_class>")
+   (set_attr "control_speculative" "<attr_yes>")])
+
+(define_insn "zero_extend<mode>di2_speculative"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (zero_extend:DI (unspec:MODE_FOR_EXTEND [(match_operand:MODE_FOR_EXTEND 1 "memory_operand" "<mem_constr>")] UNSPEC_LDS)))]
+  ""
+  "<output_s>"
+  [(set_attr "itanium_class" "<ld_class>")
+   (set_attr "control_speculative" "<attr_yes>")])
+
+(define_insn "mov<mode>_speculative_advanced"
+  [(set (match_operand:MODE 0 "<reg_pred_prefix>_register_operand" "<ld_reg_constr>")
+       (unspec:MODE [(match_operand:MODE 1 "memory_operand" "<mem_constr>")] UNSPEC_LDSA))]
+  "ia64_move_ok (operands[0], operands[1])"
+  "<output_sa>"
+  [(set_attr "itanium_class" "<ld_class>")
+   (set_attr "data_speculative" "<attr_yes>")
+   (set_attr "control_speculative" "<attr_yes>")])
+
+(define_insn "mov<mode>_speculative_a"
+  [(set (match_operand:MODE 0 "<reg_pred_prefix>_register_operand" "<ld_reg_constr>")
+       (unspec:MODE [(match_operand:MODE 1 "memory_operand" "<mem_constr>")] UNSPEC_LDS_A))]
+  "ia64_move_ok (operands[0], operands[1])"
+  "<output_sa>"
+  [(set_attr "itanium_class" "<ld_class>")
+   (set_attr "data_speculative" "<attr_yes>")
+   (set_attr "control_speculative" "<attr_yes>")])
+
+(define_insn "zero_extend<mode>di2_speculative_advanced"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (zero_extend:DI (unspec:MODE_FOR_EXTEND [(match_operand:MODE_FOR_EXTEND 1 "memory_operand" "<mem_constr>")] UNSPEC_LDSA)))]
+  ""
+  "<output_sa>"
+  [(set_attr "itanium_class" "<ld_class>")
+   (set_attr "data_speculative" "<attr_yes>")
+   (set_attr "control_speculative" "<attr_yes>")])
+
+(define_insn "zero_extend<mode>di2_speculative_a"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (zero_extend:DI (unspec:MODE_FOR_EXTEND [(match_operand:MODE_FOR_EXTEND 1 "memory_operand" "<mem_constr>")] UNSPEC_LDS_A)))]
+  ""
+  "<output_sa>"
+  [(set_attr "itanium_class" "<ld_class>")
+   (set_attr "data_speculative" "<attr_yes>")
+   (set_attr "control_speculative" "<attr_yes>")])
+
+(define_insn "mov<mode>_clr"
+  [(set (match_operand:MODE 0 "<reg_pred_prefix>_register_operand" "<ldc_reg_constr>")
+       (if_then_else:MODE (ne (unspec [(match_dup 0)] UNSPEC_LDCCLR) (const_int 0))
+                          (match_operand:MODE 1 "memory_operand" "<mem_constr>")
+                          (match_dup 0)))]
+  "ia64_move_ok (operands[0], operands[1])"
+  "<output_c_clr>"
+  [(set_attr "itanium_class" "<ld_class>")
+   (set_attr "check_load" "<attr_yes>")])
+
+(define_insn "mov<mode>_nc"
+  [(set (match_operand:MODE 0 "<reg_pred_prefix>_register_operand" "<ldc_reg_constr>")
+       (if_then_else:MODE (ne (unspec [(match_dup 0)] UNSPEC_LDCNC) (const_int 0))
+                          (match_operand:MODE 1 "memory_operand" "<mem_constr>")
+                          (match_dup 0)))]
+  "ia64_move_ok (operands[0], operands[1])"
+  "<output_c_nc>"
+  [(set_attr "itanium_class" "<ld_class>")
+   (set_attr "check_load" "<attr_yes>")])
+
+(define_insn "zero_extend<mode>di2_clr"
+  [(set (match_operand:DI 0 "gr_register_operand" "+r")
+       (if_then_else:DI (ne (unspec [(match_dup 0)] UNSPEC_LDCCLR) (const_int 0))
+                        (zero_extend:DI (match_operand:MODE_FOR_EXTEND 1 "memory_operand" "<mem_constr>"))
+                        (match_dup 0)))]
+  ""
+  "<output_c_clr>"
+  [(set_attr "itanium_class" "<ld_class>")
+   (set_attr "check_load" "<attr_yes>")])
+
+(define_insn "zero_extend<mode>di2_nc"
+  [(set (match_operand:DI 0 "gr_register_operand" "+r")
+       (if_then_else:DI (ne (unspec [(match_dup 0)] UNSPEC_LDCNC) (const_int 0))
+                        (zero_extend:DI (match_operand:MODE_FOR_EXTEND 1 "memory_operand" "<mem_constr>"))
+                        (match_dup 0)))]
+  ""
+  "<output_c_nc>"
+  [(set_attr "itanium_class" "<ld_class>")
+   (set_attr "check_load" "<attr_yes>")])
+
+(define_insn "advanced_load_check_clr_<mode>"
+  [(set (pc)
+        (if_then_else (ne (unspec [(match_operand:MODE 0 "<reg_pred_prefix>_register_operand" "<chk_reg_constr>")] UNSPEC_CHKACLR) (const_int 0))
+                      (pc)
+                      (label_ref (match_operand 1 "" ""))))]
+  ""
+  "chk.a.clr %0, %l1"
+  [(set_attr "itanium_class" "<chka_class>")])
+
+(define_insn "advanced_load_check_nc_<mode>"
+  [(set (pc)
+        (if_then_else (ne (unspec [(match_operand:MODE 0 "<reg_pred_prefix>_register_operand" "<chk_reg_constr>")] UNSPEC_CHKANC) (const_int 0))
+                      (pc)
+                      (label_ref (match_operand 1 "" ""))))]
+  ""
+  "chk.a.clr %0, %l1"
+  [(set_attr "itanium_class" "<chka_class>")])
+
+(define_insn "speculation_check_<mode>"
+  [(set (pc) 
+        (if_then_else (ne (unspec [(match_operand:MODE 0 "<reg_pred_prefix>_register_operand" "<chk_reg_constr>")] UNSPEC_CHKS) (const_int 0))
+                      (pc)
+                      (label_ref (match_operand 1 "" ""))))]
+  ""
+  "chk.s %0, %l1"
+  [(set_attr "itanium_class" "<chks_class>")])
 
 (define_split
   [(set (match_operand 0 "register_operand" "")
        (match_operand 1 "symbolic_operand" ""))]
-  "reload_completed && ! TARGET_NO_PIC"
+  "reload_completed"
   [(const_int 0)]
 {
-  ia64_expand_load_address (operands[0], operands[1]);
-  DONE;
+  if (ia64_expand_load_address (operands[0], operands[1]))
+    DONE;
+  else
+    FAIL;
 })
 
 (define_expand "load_fptr"
-  [(set (match_dup 2)
-       (plus:DI (reg:DI 1) (match_operand 1 "function_operand" "")))
-   (set (match_operand:DI 0 "register_operand" "") (match_dup 3))]
-  ""
+  [(set (match_operand:DI 0 "register_operand" "")
+       (plus:DI (match_dup 2) (match_operand 1 "function_operand" "")))
+   (set (match_dup 0) (match_dup 3))]
+  "reload_completed"
 {
-  operands[2] = no_new_pseudos ? operands[0] : gen_reg_rtx (DImode);
-  operands[3] = gen_rtx_MEM (DImode, operands[2]);
-  RTX_UNCHANGING_P (operands[3]) = 1;
+  operands[2] = pic_offset_table_rtx;
+  operands[3] = gen_const_mem (DImode, operands[0]);
 })
 
 (define_insn "*load_fptr_internal1"
   [(set (match_operand:DI 0 "register_operand" "=r")
        (plus:DI (reg:DI 1) (match_operand 1 "function_operand" "s")))]
-  ""
+  "reload_completed"
   "addl %0 = @ltoff(@fptr(%1)), gp"
   [(set_attr "itanium_class" "ialu")])
 
 (define_insn "load_gprel"
   [(set (match_operand:DI 0 "register_operand" "=r")
        (plus:DI (reg:DI 1) (match_operand 1 "sdata_symbolic_operand" "s")))]
-  ""
+  "reload_completed"
   "addl %0 = @gprel(%1), gp"
   [(set_attr "itanium_class" "ialu")])
 
-(define_insn "gprel64_offset"
+(define_insn "*gprel64_offset"
   [(set (match_operand:DI 0 "register_operand" "=r")
        (minus:DI (match_operand:DI 1 "symbolic_operand" "") (reg:DI 1)))]
-  ""
+  "reload_completed"
   "movl %0 = @gprel(%1)"
   [(set_attr "itanium_class" "long_i")])
 
 (define_expand "load_gprel64"
-  [(set (match_dup 2)
-       (minus:DI (match_operand:DI 1 "symbolic_operand" "") (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
-       (plus:DI (match_dup 3) (match_dup 2)))]
-  ""
+  [(set (match_operand:DI 0 "register_operand" "")
+       (minus:DI (match_operand:DI 1 "symbolic_operand" "") (match_dup 2)))
+   (set (match_dup 0)
+       (plus:DI (match_dup 2) (match_dup 0)))]
+  "reload_completed"
 {
-  operands[2] = no_new_pseudos ? operands[0] : gen_reg_rtx (DImode);
-  operands[3] = pic_offset_table_rtx;
+  operands[2] = pic_offset_table_rtx;
 })
 
 ;; This is used as a placeholder for the return address during early
   [(set (match_operand:DI 0 "register_operand" "=r")
        (plus:DI (high:DI (match_operand 1 "got_symbolic_operand" "s"))
                 (match_operand:DI 2 "register_operand" "a")))]
-  ""
+  "reload_completed"
 {
   if (HAVE_AS_LTOFFX_LDXMOV_RELOCS)
     return "%,addl %0 = @ltoffx(%1), %2";
   [(set (match_operand:DI 0 "register_operand" "=r")
        (lo_sum:DI (match_operand:DI 1 "register_operand" "r")
                   (match_operand 2 "got_symbolic_operand" "s")))]
-  ""
+  "reload_completed"
 {
   if (HAVE_AS_LTOFFX_LDXMOV_RELOCS)
     return "%,ld8.mov %0 = [%1], %2";
 }
   [(set_attr "itanium_class" "ld")])
 
-(define_insn "load_ltoff_dtpmod"
+(define_insn_and_split "load_dtpmod"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (plus:DI (reg:DI 1)
-                (unspec:DI [(match_operand:DI 1 "symbolic_operand" "")]
-                           UNSPEC_LTOFF_DTPMOD)))]
+       (unspec:DI [(match_operand 1 "tls_symbolic_operand" "")]
+                  UNSPEC_DTPMOD))]
   ""
-  "addl %0 = @ltoff(@dtpmod(%1)), gp"
-  [(set_attr "itanium_class" "ialu")])
+  "#"
+  "reload_completed"
+  [(set (match_dup 0)
+       (plus:DI (unspec:DI [(match_dup 1)] UNSPEC_LTOFF_DTPMOD)
+                (match_dup 2)))
+   (set (match_dup 0) (match_dup 3))]
+{
+  operands[2] = pic_offset_table_rtx;
+  operands[3] = gen_const_mem (DImode, operands[0]);
+})
 
-(define_insn "load_ltoff_dtprel"
+(define_insn "*load_ltoff_dtpmod"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (plus:DI (reg:DI 1)
-                (unspec:DI [(match_operand:DI 1 "symbolic_operand" "")]
-                           UNSPEC_LTOFF_DTPREL)))]
-  ""
-  "addl %0 = @ltoff(@dtprel(%1)), gp"
+       (plus:DI (unspec:DI [(match_operand 1 "tls_symbolic_operand" "")]
+                           UNSPEC_LTOFF_DTPMOD)
+                (match_operand:DI 2 "register_operand" "a")))]
+  "reload_completed"
+  "addl %0 = @ltoff(@dtpmod(%1)), %2"
   [(set_attr "itanium_class" "ialu")])
 
 (define_expand "load_dtprel"
   [(set (match_operand:DI 0 "register_operand" "")
-       (unspec:DI [(match_operand:DI 1 "symbolic_operand" "")]
+       (unspec:DI [(match_operand 1 "tls_symbolic_operand" "")]
                   UNSPEC_DTPREL))]
   ""
   "")
 
 (define_insn "*load_dtprel64"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (unspec:DI [(match_operand:DI 1 "symbolic_operand" "")]
+       (unspec:DI [(match_operand 1 "ld_tls_symbolic_operand" "")]
                   UNSPEC_DTPREL))]
   "TARGET_TLS64"
   "movl %0 = @dtprel(%1)"
 
 (define_insn "*load_dtprel22"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (unspec:DI [(match_operand:DI 1 "symbolic_operand" "")]
+       (unspec:DI [(match_operand 1 "ld_tls_symbolic_operand" "")]
                   UNSPEC_DTPREL))]
   ""
   "addl %0 = @dtprel(%1), r0"
   [(set_attr "itanium_class" "ialu")])
 
+(define_insn_and_split "*load_dtprel_gd"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (unspec:DI [(match_operand 1 "tls_symbolic_operand" "")]
+                  UNSPEC_DTPREL))]
+  ""
+  "#"
+  "reload_completed"
+  [(set (match_dup 0)
+       (plus:DI (unspec:DI [(match_dup 1)] UNSPEC_LTOFF_DTPREL)
+                (match_dup 2)))
+   (set (match_dup 0) (match_dup 3))]
+{
+  operands[2] = pic_offset_table_rtx;
+  operands[3] = gen_const_mem (DImode, operands[0]);
+})
+
+(define_insn "*load_ltoff_dtprel"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (plus:DI (unspec:DI [(match_operand 1 "tls_symbolic_operand" "")]
+                           UNSPEC_LTOFF_DTPREL)
+                (match_operand:DI 2 "register_operand" "a")))]
+  ""
+  "addl %0 = @ltoff(@dtprel(%1)), %2"
+  [(set_attr "itanium_class" "ialu")])
+
 (define_expand "add_dtprel"
   [(set (match_operand:DI 0 "register_operand" "")
-       (plus:DI (match_operand:DI 1 "register_operand" "")
-                (unspec:DI [(match_operand:DI 2 "symbolic_operand" "")]
-                           UNSPEC_DTPREL)))]
+       (plus:DI (unspec:DI [(match_operand 1 "ld_tls_symbolic_operand" "")]
+                           UNSPEC_DTPREL)
+                (match_operand:DI 2 "register_operand" "")))]
   "!TARGET_TLS64"
   "")
 
 (define_insn "*add_dtprel14"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (plus:DI (match_operand:DI 1 "register_operand" "r")
-                (unspec:DI [(match_operand:DI 2 "symbolic_operand" "")]
-                           UNSPEC_DTPREL)))]
+       (plus:DI (unspec:DI [(match_operand 1 "ld_tls_symbolic_operand" "")]
+                           UNSPEC_DTPREL)
+                (match_operand:DI 2 "register_operand" "r")))]
   "TARGET_TLS14"
-  "adds %0 = @dtprel(%2), %1"
+  "adds %0 = @dtprel(%1), %2"
   [(set_attr "itanium_class" "ialu")])
 
 (define_insn "*add_dtprel22"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (plus:DI (match_operand:DI 1 "register_operand" "a")
-                (unspec:DI [(match_operand:DI 2 "symbolic_operand" "")]
-                           UNSPEC_DTPREL)))]
+       (plus:DI (unspec:DI [(match_operand 1 "ld_tls_symbolic_operand" "")]
+                           UNSPEC_DTPREL)
+                (match_operand:DI 2 "register_operand" "a")))]
   "TARGET_TLS22"
-  "addl %0 = @dtprel(%2), %1"
-  [(set_attr "itanium_class" "ialu")])
-
-(define_insn "load_ltoff_tprel"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (plus:DI (reg:DI 1)
-                (unspec:DI [(match_operand:DI 1 "symbolic_operand" "")]
-                           UNSPEC_LTOFF_TPREL)))]
-  ""
-  "addl %0 = @ltoff(@tprel(%1)), gp"
+  "addl %0 = @dtprel(%1), %2"
   [(set_attr "itanium_class" "ialu")])
 
 (define_expand "load_tprel"
   [(set (match_operand:DI 0 "register_operand" "")
-       (unspec:DI [(match_operand:DI 1 "symbolic_operand" "")]
+       (unspec:DI [(match_operand 1 "tls_symbolic_operand" "")]
                   UNSPEC_TPREL))]
   ""
   "")
 
 (define_insn "*load_tprel64"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (unspec:DI [(match_operand:DI 1 "symbolic_operand" "")]
+       (unspec:DI [(match_operand 1 "le_tls_symbolic_operand" "")]
                   UNSPEC_TPREL))]
   "TARGET_TLS64"
   "movl %0 = @tprel(%1)"
 
 (define_insn "*load_tprel22"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (unspec:DI [(match_operand:DI 1 "symbolic_operand" "")]
+       (unspec:DI [(match_operand 1 "le_tls_symbolic_operand" "")]
                   UNSPEC_TPREL))]
   ""
   "addl %0 = @tprel(%1), r0"
   [(set_attr "itanium_class" "ialu")])
 
+(define_insn_and_split "*load_tprel_ie"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (unspec:DI [(match_operand 1 "ie_tls_symbolic_operand" "")]
+                  UNSPEC_TPREL))]
+  ""
+  "#"
+  "reload_completed"
+  [(set (match_dup 0)
+       (plus:DI (unspec:DI [(match_dup 1)] UNSPEC_LTOFF_TPREL)
+                (match_dup 2)))
+   (set (match_dup 0) (match_dup 3))]
+{
+  operands[2] = pic_offset_table_rtx;
+  operands[3] = gen_const_mem (DImode, operands[0]);
+})
+
+(define_insn "*load_ltoff_tprel"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (plus:DI (unspec:DI [(match_operand 1 "ie_tls_symbolic_operand" "")]
+                           UNSPEC_LTOFF_TPREL)
+                (match_operand:DI 2 "register_operand" "a")))]
+  ""
+  "addl %0 = @ltoff(@tprel(%1)), %2"
+  [(set_attr "itanium_class" "ialu")])
+
 (define_expand "add_tprel"
   [(set (match_operand:DI 0 "register_operand" "")
-       (plus:DI (match_operand:DI 1 "register_operand" "")
-                (unspec:DI [(match_operand:DI 2 "symbolic_operand" "")]
-                           UNSPEC_TPREL)))]
+       (plus:DI (unspec:DI [(match_operand 1 "le_tls_symbolic_operand" "")]
+                           UNSPEC_TPREL)
+                (match_operand:DI 2 "register_operand" "")))]
   "!TARGET_TLS64"
   "")
 
 (define_insn "*add_tprel14"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (plus:DI (match_operand:DI 1 "register_operand" "r")
-                (unspec:DI [(match_operand:DI 2 "symbolic_operand" "")]
-                           UNSPEC_TPREL)))]
+       (plus:DI (unspec:DI [(match_operand 1 "le_tls_symbolic_operand" "")]
+                           UNSPEC_TPREL)
+                (match_operand:DI 2 "register_operand" "r")))]
   "TARGET_TLS14"
-  "adds %0 = @tprel(%2), %1"
+  "adds %0 = @tprel(%1), %2"
   [(set_attr "itanium_class" "ialu")])
 
 (define_insn "*add_tprel22"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (plus:DI (match_operand:DI 1 "register_operand" "a")
-                (unspec:DI [(match_operand:DI 2 "symbolic_operand" "")]
-                           UNSPEC_TPREL)))]
+       (plus:DI (unspec:DI [(match_operand 1 "le_tls_symbolic_operand" "")]
+                           UNSPEC_TPREL)
+                (match_operand:DI 2 "register_operand" "a")))]
   "TARGET_TLS22"
-  "addl %0 = @tprel(%2), %1"
+  "addl %0 = @tprel(%1), %2"
   [(set_attr "itanium_class" "ialu")])
 
 ;; With no offsettable memory references, we've got to have a scratch
-;; around to play with the second word.
+;; around to play with the second word.  However, in order to avoid a
+;; reload nightmare we lie, claim we don't need one, and fix it up
+;; in ia64_split_tmode_move.
 (define_expand "movti"
-  [(parallel [(set (match_operand:TI 0 "general_operand" "")
-                  (match_operand:TI 1 "general_operand" ""))
-             (clobber (match_scratch:DI 2 ""))])]
+  [(set (match_operand:TI 0 "general_operand" "")
+       (match_operand:TI 1 "general_operand" ""))]
   ""
 {
   rtx op1 = ia64_expand_move (operands[0], operands[1]);
   operands[1] = op1;
 })
 
-(define_insn_and_split "*movti_internal"
-  [(set (match_operand:TI 0 "nonimmediate_operand" "=r,r,m")
-       (match_operand:TI 1 "general_operand"      "ri,m,r"))
-   (clobber (match_scratch:DI 2 "=X,&r,&r"))]
+(define_insn_and_split "movti_internal"
+  [(set (match_operand:TI 0 "destination_operand" "=r,   *fm,*x,*f,  Q")
+       (match_operand:TI 1 "general_operand"     "r*fim,r,  Q, *fOQ,*f"))]
   "ia64_move_ok (operands[0], operands[1])"
-  "#"
-  "reload_completed"
-  [(const_int 0)]
-{
-  rtx adj1, adj2, in[2], out[2], insn;
-  int first;
-
-  adj1 = ia64_split_timode (in, operands[1], operands[2]);
-  adj2 = ia64_split_timode (out, operands[0], operands[2]);
-
-  first = 0;
-  if (reg_overlap_mentioned_p (out[0], in[1]))
-    {
-      if (reg_overlap_mentioned_p (out[1], in[0]))
-       abort ();
-      first = 1;
-    }
-
-  if (adj1 && adj2)
-    abort ();
-  if (adj1)
-    emit_insn (adj1);
-  if (adj2)
-    emit_insn (adj2);
-  insn = emit_insn (gen_rtx_SET (VOIDmode, out[first], in[first]));
-  if (GET_CODE (out[first]) == MEM
-      && GET_CODE (XEXP (out[first], 0)) == POST_MODIFY)
-    REG_NOTES (insn) = gen_rtx_EXPR_LIST (REG_INC,
-                                         XEXP (XEXP (out[first], 0), 0),
-                                         REG_NOTES (insn));
-  insn = emit_insn (gen_rtx_SET (VOIDmode, out[!first], in[!first]));
-  if (GET_CODE (out[!first]) == MEM
-      && GET_CODE (XEXP (out[!first], 0)) == POST_MODIFY)
-    REG_NOTES (insn) = gen_rtx_EXPR_LIST (REG_INC,
-                                         XEXP (XEXP (out[!first], 0), 0),
-                                         REG_NOTES (insn));
-  DONE;
-}
-  [(set_attr "itanium_class" "unknown")
-   (set_attr "predicable" "no")])
-
-;; ??? SSA creates these.  Can't allow memories since we don't have
-;; the scratch register.  Fortunately combine will know how to add
-;; the clobber and scratch.
-(define_insn_and_split "*movti_internal_reg"
-  [(set (match_operand:TI 0 "register_operand"  "=r")
-       (match_operand:TI 1 "nonmemory_operand" "ri"))]
-  ""
-  "#"
-  "reload_completed"
+  "@
+   #
+   #
+   ldfp8 %X0 = %1%P1
+   #
+   #"
+  "reload_completed && !ia64_load_pair_ok(operands[0], operands[1])"
   [(const_int 0)]
 {
-  rtx in[2], out[2];
-  int first;
-
-  ia64_split_timode (in, operands[1], NULL_RTX);
-  ia64_split_timode (out, operands[0], NULL_RTX);
-
-  first = 0;
-  if (reg_overlap_mentioned_p (out[0], in[1]))
-    {
-      if (reg_overlap_mentioned_p (out[1], in[0]))
-       abort ();
-      first = 1;
-    }
-
-  emit_insn (gen_rtx_SET (VOIDmode, out[first], in[first]));
-  emit_insn (gen_rtx_SET (VOIDmode, out[!first], in[!first]));
+  ia64_split_tmode_move (operands);
   DONE;
 }
-  [(set_attr "itanium_class" "unknown")
-   (set_attr "predicable" "no")])
-
-(define_expand "reload_inti"
-  [(parallel [(set (match_operand:TI 0 "register_operand" "=r")
-                  (match_operand:TI 1 "" "m"))
-             (clobber (match_operand:TI 2 "register_operand" "=&r"))])]
-  ""
-{
-  unsigned int s_regno = REGNO (operands[2]);
-  if (s_regno == REGNO (operands[0]))
-    s_regno += 1;
-  operands[2] = gen_rtx_REG (DImode, s_regno);
-})
-
-(define_expand "reload_outti"
-  [(parallel [(set (match_operand:TI 0 "" "=m")
-                  (match_operand:TI 1 "register_operand" "r"))
-             (clobber (match_operand:TI 2 "register_operand" "=&r"))])]
-  ""
-{
-  unsigned int s_regno = REGNO (operands[2]);
-  if (s_regno == REGNO (operands[1]))
-    s_regno += 1;
-  operands[2] = gen_rtx_REG (DImode, s_regno);
-})
+  [(set_attr "itanium_class" "unknown,unknown,fldp,unknown,unknown")
+   (set_attr "speculable1"   "yes")
+   (set_attr "speculable2"   "no,     no,     yes, no,     no")])
 
 ;; Floating Point Moves
 ;;
   operands[1] = op1;
 })
 
-(define_insn "*movsf_internal"
+(define_insn "movsf_internal"
   [(set (match_operand:SF 0 "destination_operand" "=f,f, Q,*r, f,*r,*r, m")
        (match_operand:SF 1 "general_operand"     "fG,Q,fG,fG,*r,*r, m,*r"))]
   "ia64_move_ok (operands[0], operands[1])"
    mov %0 = %1
    ld4%O1 %0 = %1%P1
    st4%Q0 %0 = %1%P0"
-  [(set_attr "itanium_class" "fmisc,fld,stf,frfr,tofr,ialu,ld,st")])
+  [(set_attr "itanium_class" "fmisc,fld,stf,frfr,tofr,ialu,ld,st")
+   (set_attr "speculable1"   "yes")
+   (set_attr "speculable2"   "no,   yes,no, no,  no,  no, yes,no")])
 
 (define_expand "movdf"
   [(set (match_operand:DF 0 "general_operand" "")
   operands[1] = op1;
 })
 
-(define_insn "*movdf_internal"
+(define_insn "movdf_internal"
   [(set (match_operand:DF 0 "destination_operand" "=f,f, Q,*r, f,*r,*r, m")
        (match_operand:DF 1 "general_operand"     "fG,Q,fG,fG,*r,*r, m,*r"))]
   "ia64_move_ok (operands[0], operands[1])"
    mov %0 = %1
    ld8%O1 %0 = %1%P1
    st8%Q0 %0 = %1%P0"
-  [(set_attr "itanium_class" "fmisc,fld,stf,frfr,tofr,ialu,ld,st")])
+  [(set_attr "itanium_class" "fmisc,fld,stf,frfr,tofr,ialu,ld,st")
+   (set_attr "speculable1"   "yes")
+   (set_attr "speculable2"   "no,   yes,no, no,  no,  no, yes,no")])
 
 ;; With no offsettable memory references, we've got to have a scratch
 ;; around to play with the second word if the variable winds up in GRs.
        (match_operand:XF 1 "general_operand" ""))]
   ""
 {
-  /* We must support XFmode loads into general registers for stdarg/vararg
-     and unprototyped calls.  We split them into DImode loads for convenience.
-     We don't need XFmode stores from general regs, because a stdarg/vararg
-     routine does a block store to memory of unnamed arguments.  */
-  if (GET_CODE (operands[0]) == REG
-      && GR_REGNO_P (REGNO (operands[0])))
-    {
-      /* We're hoping to transform everything that deals with XFmode
-        quantities and GR registers early in the compiler.  */
-      if (no_new_pseudos)
-       abort ();
-
-      /* Struct to register can just use TImode instead.  */
-      if ((GET_CODE (operands[1]) == SUBREG
-          && GET_MODE (SUBREG_REG (operands[1])) == TImode)
-         || (GET_CODE (operands[1]) == REG
-             && GR_REGNO_P (REGNO (operands[1]))))
-       {
-         emit_move_insn (gen_rtx_REG (TImode, REGNO (operands[0])),
-                         SUBREG_REG (operands[1]));
-         DONE;
-       }
-
-      if (GET_CODE (operands[1]) == CONST_DOUBLE)
-       {
-         emit_move_insn (gen_rtx_REG (DImode, REGNO (operands[0])),
-                         operand_subword (operands[1], 0, 0, XFmode));
-         emit_move_insn (gen_rtx_REG (DImode, REGNO (operands[0]) + 1),
-                         operand_subword (operands[1], 1, 0, XFmode));
-         DONE;
-       }
-
-      /* If the quantity is in a register not known to be GR, spill it.  */
-      if (register_operand (operands[1], XFmode))
-       operands[1] = spill_xfmode_operand (operands[1], 1);
-
-      if (GET_CODE (operands[1]) == MEM)
-       {
-         rtx out[2];
-
-         out[WORDS_BIG_ENDIAN] = gen_rtx_REG (DImode, REGNO (operands[0]));
-         out[!WORDS_BIG_ENDIAN] = gen_rtx_REG (DImode, REGNO (operands[0])+1);
-
-         emit_move_insn (out[0], adjust_address (operands[1], DImode, 0));
-         emit_move_insn (out[1], adjust_address (operands[1], DImode, 8));
-         DONE;
-       }
-
-      abort ();
-    }
-
-  if (! reload_in_progress && ! reload_completed)
-    {
-      operands[0] = spill_xfmode_operand (operands[0], 0);
-      operands[1] = spill_xfmode_operand (operands[1], 0);
-
-      if (! ia64_move_ok (operands[0], operands[1]))
-       operands[1] = force_reg (XFmode, operands[1]);
-    }
+  if (ia64_expand_movxf_movrf (XFmode, operands))
+    DONE;
 })
 
 ;; ??? There's no easy way to mind volatile acquire/release semantics.
 
-(define_insn "*movxf_internal"
-  [(set (match_operand:XF 0 "destination_xfmode_operand" "=f,f, m")
-       (match_operand:XF 1 "general_xfmode_operand"     "fG,m,fG"))]
+(define_insn "movxf_internal"
+  [(set (match_operand:XF 0 "destination_operand" "=f,f, m")
+       (match_operand:XF 1 "general_operand"     "fG,m,fG"))]
   "ia64_move_ok (operands[0], operands[1])"
   "@
    mov %0 = %F1
    ldfe %0 = %1%P1
    stfe %0 = %F1%P0"
+  [(set_attr "itanium_class" "fmisc,fld,stf")
+   (set_attr "speculable1"   "yes")
+   (set_attr "speculable2"   "no,   yes,no")])
+
+;; Same as for movxf, but for RFmode.
+(define_expand "movrf"
+  [(set (match_operand:RF 0 "general_operand" "")
+       (match_operand:RF 1 "general_operand" ""))]
+  ""
+{
+  if (ia64_expand_movxf_movrf (RFmode, operands))
+    DONE;
+})
+
+(define_insn "*movrf_internal"
+  [(set (match_operand:RF 0 "destination_operand" "=f,f, m")
+       (match_operand:RF 1 "general_operand"     "fG,m,fG"))]
+  "ia64_move_ok (operands[0], operands[1])"
+  "@
+   mov %0 = %F1
+   ldf.fill %0 = %1%P1
+   stf.spill %0 = %F1%P0"
   [(set_attr "itanium_class" "fmisc,fld,stf")])
+
+;; Better code generation via insns that deal with TFmode register pairs
+;; directly.  Same concerns apply as for TImode.
+(define_expand "movtf"
+  [(set (match_operand:TF 0 "general_operand" "")
+       (match_operand:TF 1 "general_operand" ""))]
+  ""
+{
+  rtx op1 = ia64_expand_move (operands[0], operands[1]);
+  if (!op1)
+    DONE;
+  operands[1] = op1;
+})
+
+(define_insn_and_split "*movtf_internal"
+  [(set (match_operand:TF 0 "destination_operand"  "=r,r,m")
+       (match_operand:TF 1 "general_operand"      "ri,m,r"))]
+  "ia64_move_ok (operands[0], operands[1])"
+  "#"
+  "reload_completed"
+  [(const_int 0)]
+{
+  ia64_split_tmode_move (operands);
+  DONE;
+}
+  [(set_attr "itanium_class" "unknown")
+   (set_attr "predicable" "no")])
+
 \f
 ;; ::::::::::::::::::::
 ;; ::
   "@
    zxt1 %0 = %1
    ld1%O1 %0 = %1%P1"
-  [(set_attr "itanium_class" "xtd,ld")])
+  [(set_attr "itanium_class" "xtd,ld")
+   (set_attr "speculable1"   "yes")
+   (set_attr "speculable2"   "no, yes")])
 
 (define_insn "zero_extendhidi2"
   [(set (match_operand:DI 0 "gr_register_operand" "=r,r")
   "@
    zxt2 %0 = %1
    ld2%O1 %0 = %1%P1"
-  [(set_attr "itanium_class" "xtd,ld")])
+  [(set_attr "itanium_class" "xtd,ld")
+   (set_attr "speculable1"   "yes")
+   (set_attr "speculable2"   "no, yes")])
 
 (define_insn "zero_extendsidi2"
   [(set (match_operand:DI 0 "grfr_register_operand" "=r,r,?f")
          (match_operand:SI 1 "grfr_nonimmediate_operand" "r,m,f")))]
   ""
   "@
-   zxt4 %0 = %1
+   addp4 %0 = %1, r0
    ld4%O1 %0 = %1%P1
    fmix.r %0 = f0, %1"
-  [(set_attr "itanium_class" "xtd,ld,fmisc")])
+  [(set_attr "itanium_class" "ialu,ld,fmisc")
+   (set_attr "speculable1"   "yes")
+   (set_attr "speculable2"   "no, yes,no")])
 
 ;; Convert between floating point types of different sizes.
 
 (define_insn "extv"
   [(set (match_operand:DI 0 "gr_register_operand" "=r")
        (sign_extract:DI (match_operand:DI 1 "gr_register_operand" "r")
-                        (match_operand:DI 2 "const_int_operand" "n")
-                        (match_operand:DI 3 "const_int_operand" "n")))]
+                        (match_operand:DI 2 "extr_len_operand" "n")
+                        (match_operand:DI 3 "shift_count_operand" "M")))]
   ""
   "extr %0 = %1, %3, %2"
   [(set_attr "itanium_class" "ishf")])
 (define_insn "extzv"
   [(set (match_operand:DI 0 "gr_register_operand" "=r")
        (zero_extract:DI (match_operand:DI 1 "gr_register_operand" "r")
-                        (match_operand:DI 2 "const_int_operand" "n")
-                        (match_operand:DI 3 "const_int_operand" "n")))]
+                        (match_operand:DI 2 "extr_len_operand" "n")
+                        (match_operand:DI 3 "shift_count_operand" "M")))]
   ""
   "extr.u %0 = %1, %3, %2"
   [(set_attr "itanium_class" "ishf")])
   [(set_attr "itanium_class" "ishf")])
 
 ;; Combine doesn't like to create bit-field insertions into zero.
+(define_insn "*shladdp4_internal"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (and:DI (ashift:DI (match_operand:DI 1 "gr_register_operand" "r")
+                          (match_operand:DI 2 "shladd_log2_operand" "n"))
+               (match_operand:DI 3 "const_int_operand" "n")))]
+  "ia64_depz_field_mask (operands[3], operands[2]) + INTVAL (operands[2]) == 32"
+  "shladdp4 %0 = %1, %2, r0"
+  [(set_attr "itanium_class" "ialu")])
+
 (define_insn "*depz_internal"
   [(set (match_operand:DI 0 "gr_register_operand" "=r")
        (and:DI (ashift:DI (match_operand:DI 1 "gr_register_operand" "r")
-                          (match_operand:DI 2 "const_int_operand" "n"))
+                          (match_operand:DI 2 "const_int_operand" "M"))
                (match_operand:DI 3 "const_int_operand" "n")))]
-  "CONST_OK_FOR_M (INTVAL (operands[2]))
+  "satisfies_constraint_M (operands[2])
    && ia64_depz_field_mask (operands[3], operands[2]) > 0"
 {
   operands[3] = GEN_INT (ia64_depz_field_mask (operands[3], operands[2]));
                         (const_int 32) (const_int 0))
        (match_operand:DI 1 "register_operand" ""))
    (clobber (match_operand:DI 2 "register_operand" ""))]
-  "reload_completed"
-  [(set (match_dup 3) (ashift:DI (match_dup 1) (const_int 32)))
-   (set (zero_extract:DI (match_dup 0) (const_int 32) (const_int 0))
-       (lshiftrt:DI (match_dup 3) (const_int 32)))]
-  "operands[3] = operands[2];")
-
-(define_split
-  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "")
-                        (const_int 32) (const_int 0))
-       (match_operand:DI 1 "register_operand" ""))
-   (clobber (match_operand:DI 2 "register_operand" ""))]
-  "! reload_completed"
+  ""
   [(set (match_dup 3) (ashift:DI (match_dup 1) (const_int 32)))
    (set (zero_extract:DI (match_dup 0) (const_int 32) (const_int 0))
        (lshiftrt:DI (match_dup 3) (const_int 32)))]
 \f
 ;; ::::::::::::::::::::
 ;; ::
-;; :: 1 bit Integer arithmetic
+;; :: 1-bit Integer arithmetic
 ;; ::
 ;; ::::::::::::::::::::
 
        (and:BI (ne:BI (zero_extract:DI
                         (match_operand:DI 1 "gr_register_operand" "r")
                         (const_int 1)
-                        (match_operand:DI 2 "const_int_operand" "n"))
+                        (match_operand:DI 2 "shift_count_operand" "M"))
                       (const_int 0))
                (match_operand:BI 3 "register_operand" "0")))]
   ""
        (and:BI (eq:BI (zero_extract:DI
                         (match_operand:DI 1 "gr_register_operand" "r")
                         (const_int 1)
-                        (match_operand:DI 2 "const_int_operand" "n"))
+                        (match_operand:DI 2 "shift_count_operand" "M"))
                       (const_int 0))
                (match_operand:BI 3 "register_operand" "0")))]
   ""
        (ior:BI (ne:BI (zero_extract:DI
                         (match_operand:DI 1 "gr_register_operand" "r")
                         (const_int 1)
-                        (match_operand:DI 2 "const_int_operand" "n"))
+                        (match_operand:DI 2 "shift_count_operand" "M"))
                       (const_int 0))
                (match_operand:BI 3 "register_operand" "0")))]
   ""
        (ior:BI (eq:BI (zero_extract:DI
                         (match_operand:DI 1 "gr_register_operand" "r")
                         (const_int 1)
-                        (match_operand:DI 2 "const_int_operand" "n"))
+                        (match_operand:DI 2 "shift_count_operand" "M"))
                       (const_int 0))
                (match_operand:BI 3 "register_operand" "0")))]
   ""
 \f
 ;; ::::::::::::::::::::
 ;; ::
-;; :: 16 bit Integer arithmetic
+;; :: 16-bit Integer arithmetic
 ;; ::
 ;; ::::::::::::::::::::
 
 \f
 ;; ::::::::::::::::::::
 ;; ::
-;; :: 32 bit Integer arithmetic
+;; :: 32-bit Integer arithmetic
 ;; ::
 ;; ::::::::::::::::::::
 
                (match_operand:SI 2 "general_operand" "")))]
   "TARGET_INLINE_INT_DIV"
 {
-  rtx op1_xf, op2_xf, op0_xf, op0_di, twon34;
-  REAL_VALUE_TYPE twon34_r;
+  rtx op1_xf, op2_xf, op0_xf, op0_di, twon34, twon34_exp;
 
   op0_xf = gen_reg_rtx (XFmode);
   op0_di = gen_reg_rtx (DImode);
   expand_float (op2_xf, operands[2], 0);
 
   /* 2^-34 */
-  real_2expN (&twon34_r, -34);
-  twon34 = CONST_DOUBLE_FROM_REAL_VALUE (twon34_r, XFmode);
-  twon34 = force_reg (XFmode, twon34);
-
+  twon34_exp = gen_reg_rtx (DImode);
+  emit_move_insn (twon34_exp, GEN_INT (65501));
+  twon34 = gen_reg_rtx (XFmode);
+  emit_insn (gen_setf_exp_xf (twon34, twon34_exp));
+
+  emit_insn (gen_cond_trap (EQ, operands[2], CONST0_RTX (SImode),
+                           CONST1_RTX (SImode)));
+  
   emit_insn (gen_divsi3_internal (op0_xf, op1_xf, op2_xf, twon34));
 
   emit_insn (gen_fix_truncxfdi2_alts (op0_di, op0_xf, const1_rtx));
                 (match_operand:SI 2 "general_operand" "")))]
   "TARGET_INLINE_INT_DIV"
 {
-  rtx op1_xf, op2_xf, op0_xf, op0_di, twon34;
-  REAL_VALUE_TYPE twon34_r;
+  rtx op1_xf, op2_xf, op0_xf, op0_di, twon34, twon34_exp;
 
   op0_xf = gen_reg_rtx (XFmode);
   op0_di = gen_reg_rtx (DImode);
   expand_float (op2_xf, operands[2], 1);
 
   /* 2^-34 */
-  real_2expN (&twon34_r, -34);
-  twon34 = CONST_DOUBLE_FROM_REAL_VALUE (twon34_r, XFmode);
-  twon34 = force_reg (XFmode, twon34);
-
+  twon34_exp = gen_reg_rtx (DImode);
+  emit_move_insn (twon34_exp, GEN_INT (65501));
+  twon34 = gen_reg_rtx (XFmode);
+  emit_insn (gen_setf_exp_xf (twon34, twon34_exp));
+
+  emit_insn (gen_cond_trap (EQ, operands[2], CONST0_RTX (SImode),
+                           CONST1_RTX (SImode)));
+  
   emit_insn (gen_divsi3_internal (op0_xf, op1_xf, op2_xf, twon34));
 
   emit_insn (gen_fixuns_truncxfdi2_alts (op0_di, op0_xf, const1_rtx));
   "TARGET_INLINE_INT_DIV"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0) (div:XF (const_int 1) (match_dup 2)))
+  [(parallel [(set (match_dup 0) (unspec:XF [(const_int 1) (match_dup 2)]
+                                           UNSPEC_FR_RECIP_APPROX_RES))
              (set (match_dup 6) (unspec:BI [(match_dup 1) (match_dup 2)]
                                            UNSPEC_FR_RECIP_APPROX))
              (use (const_int 1))])
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 6) (const_int 0))
      (parallel [(set (match_dup 5)
-                    (plus:XF (neg:XF (mult:XF (match_dup 2) (match_dup 0)))
-                             (match_dup 7)))
+                    (minus:XF (match_dup 7)
+                              (mult:XF (match_dup 2) (match_dup 0))))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 6) (const_int 0))
      (parallel [(set (match_dup 4)
 \f
 ;; ::::::::::::::::::::
 ;; ::
-;; :: 64 bit Integer arithmetic
+;; :: 64-bit Integer arithmetic
 ;; ::
 ;; ::::::::::::::::::::
 
    (set (match_dup 0) (plus:DI (match_dup 0) (match_dup 4)))]
   "")
 
-;; ??? There are highpart multiply and add instructions, but we have no way
-;; to generate them.
-
 (define_insn "smuldi3_highpart"
   [(set (match_operand:DI 0 "fr_register_operand" "=f")
        (truncate:DI
   "popcnt %0 = %1"
   [(set_attr "itanium_class" "mmmul")])
 
+(define_insn "bswapdi2"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+        (bswap:DI (match_operand:DI 1 "gr_register_operand" "r")))]
+  ""
+  "mux1 %0 = %1, @rev"
+  [(set_attr "itanium_class" "mmshf")])
+
 (define_insn "*getf_exp_xf"
   [(set (match_operand:DI 0 "gr_register_operand" "=r")
        (unspec:DI [(match_operand:XF 1 "fr_register_operand" "f")]
   op2_xf = gen_reg_rtx (XFmode);
   expand_float (op2_xf, operands[2], 0);
 
-  if (TARGET_INLINE_INT_DIV_LAT)
+  emit_insn (gen_cond_trap (EQ, operands[2], CONST0_RTX (DImode),
+                           CONST1_RTX (DImode)));
+
+  if (TARGET_INLINE_INT_DIV == INL_MIN_LAT)
     emit_insn (gen_divdi3_internal_lat (op0_xf, op1_xf, op2_xf));
   else
     emit_insn (gen_divdi3_internal_thr (op0_xf, op1_xf, op2_xf));
   op2_xf = gen_reg_rtx (XFmode);
   expand_float (op2_xf, operands[2], 1);
 
-  if (TARGET_INLINE_INT_DIV_LAT)
+  emit_insn (gen_cond_trap (EQ, operands[2], CONST0_RTX (DImode),
+                           CONST1_RTX (DImode)));
+
+  if (TARGET_INLINE_INT_DIV == INL_MIN_LAT)
     emit_insn (gen_divdi3_internal_lat (op0_xf, op1_xf, op2_xf));
   else
     emit_insn (gen_divdi3_internal_thr (op0_xf, op1_xf, op2_xf));
    (clobber (match_scratch:XF 4 "=&f"))
    (clobber (match_scratch:XF 5 "=&f"))
    (clobber (match_scratch:BI 6 "=c"))]
-  "TARGET_INLINE_INT_DIV_LAT"
+  "TARGET_INLINE_INT_DIV == INL_MIN_LAT"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0) (div:XF (const_int 1) (match_dup 2)))
+  [(parallel [(set (match_dup 0) (unspec:XF [(const_int 1) (match_dup 2)]
+                                           UNSPEC_FR_RECIP_APPROX_RES))
              (set (match_dup 6) (unspec:BI [(match_dup 1) (match_dup 2)]
                                            UNSPEC_FR_RECIP_APPROX))
              (use (const_int 1))])
    (cond_exec (ne (match_dup 6) (const_int 0))
      (parallel [(set (match_dup 3)
-                    (plus:XF (neg:XF (mult:XF (match_dup 2) (match_dup 0)))
-                             (match_dup 7)))
+                    (minus:XF (match_dup 7)
+                              (mult:XF (match_dup 2) (match_dup 0))))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 6) (const_int 0))
      (parallel [(set (match_dup 4) (mult:XF (match_dup 1) (match_dup 0)))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 6) (const_int 0))
      (parallel [(set (match_dup 4)
-                    (plus:XF (neg:XF (mult:XF (match_dup 2) (match_dup 3)))
-                             (match_dup 1)))
+                    (minus:XF (match_dup 1)
+                              (mult:XF (match_dup 2) (match_dup 3))))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 6) (const_int 0))
      (parallel [(set (match_dup 0)
    (clobber (match_scratch:XF 3 "=&f"))
    (clobber (match_scratch:XF 4 "=f"))
    (clobber (match_scratch:BI 5 "=c"))]
-  "TARGET_INLINE_INT_DIV_THR"
+  "TARGET_INLINE_INT_DIV == INL_MAX_THR"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0) (div:XF (const_int 1) (match_dup 2)))
+  [(parallel [(set (match_dup 0) (unspec:XF [(const_int 1) (match_dup 2)]
+                                           UNSPEC_FR_RECIP_APPROX_RES))
              (set (match_dup 5) (unspec:BI [(match_dup 1) (match_dup 2)] 
                                            UNSPEC_FR_RECIP_APPROX))
              (use (const_int 1))])
    (cond_exec (ne (match_dup 5) (const_int 0))
      (parallel [(set (match_dup 3)
-                    (plus:XF (neg:XF (mult:XF (match_dup 2) (match_dup 0)))
-                             (match_dup 6)))
+                    (minus:XF (match_dup 6)
+                              (mult:XF (match_dup 2) (match_dup 0))))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 5) (const_int 0))
      (parallel [(set (match_dup 0)
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 5) (const_int 0))
      (parallel [(set (match_dup 4)
-                    (plus:XF (neg:XF (mult:XF (match_dup 2) (match_dup 3)))
-                             (match_dup 1)))
+                    (minus:XF (match_dup 1)
+                              (mult:XF (match_dup 2) (match_dup 3))))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 5) (const_int 0))
      (parallel [(set (match_dup 0)
 \f
 ;; ::::::::::::::::::::
 ;; ::
-;; :: 32 bit floating point arithmetic
+;; :: 128-bit Integer arithmetic
+;; ::
+;; ::::::::::::::::::::
+
+(define_insn "addti3"
+  [(set (match_operand:TI 0 "gr_register_operand" "=&r")
+       (plus:TI (match_operand:TI 1 "gr_register_operand" "%r")
+                (match_operand:TI 2 "gr_reg_or_14bit_operand" "rI")))
+   (clobber (match_scratch:BI 3 "=&c"))]
+  ""
+  "#"
+  [(set_attr "itanium_class" "unknown")])
+
+(define_split
+  [(set (match_operand:TI 0 "register_operand" "")
+       (plus:TI (match_operand:TI 1 "register_operand" "")
+                (match_operand:TI 2 "register_operand" "")))
+   (clobber (match_scratch:BI 3 ""))]
+  "reload_completed"
+  [(set (match_dup 0) (plus:DI (match_dup 1) (match_dup 2)))
+   (set (match_dup 3) (ltu:BI (match_dup 0) (match_dup 1)))
+   (cond_exec (eq (match_dup 3) (const_int 0))
+             (set (match_dup 4) (plus:DI (match_dup 5) (match_dup 6))))
+   (cond_exec (ne (match_dup 3) (const_int 0))
+             (set (match_dup 4)
+                  (plus:DI (plus:DI (match_dup 5) (match_dup 6))
+                           (const_int 1))))]
+{
+  operands[4] = gen_highpart (DImode, operands[0]);
+  operands[0] = gen_lowpart (DImode, operands[0]);
+  operands[5] = gen_highpart (DImode, operands[1]);
+  operands[1] = gen_lowpart (DImode, operands[1]);
+  operands[6] = gen_highpart (DImode, operands[2]);
+  operands[2] = gen_lowpart (DImode, operands[2]);
+})
+
+(define_split
+  [(set (match_operand:TI 0 "register_operand" "")
+       (plus:TI (match_operand:TI 1 "register_operand" "")
+                (match_operand:TI 2 "immediate_operand" "")))
+   (clobber (match_scratch:BI 3 ""))]
+  "reload_completed"
+  [(set (match_dup 0) (plus:DI (match_dup 1) (match_dup 2)))
+   (set (match_dup 3) (ltu:BI (match_dup 0) (match_dup 1)))
+   (cond_exec (eq (match_dup 3) (const_int 0))
+             (set (match_dup 4)
+                  (plus:DI (match_dup 5) (match_dup 6))))
+   (cond_exec (ne (match_dup 3) (const_int 0))
+             (set (match_dup 4)
+                  (plus:DI (match_dup 5) (match_dup 7))))]
+{
+  operands[4] = gen_highpart (DImode, operands[0]);
+  operands[0] = gen_lowpart (DImode, operands[0]);
+  operands[5] = gen_highpart (DImode, operands[1]);
+  operands[1] = gen_lowpart (DImode, operands[1]);
+  operands[6] = INTVAL (operands[2]) < 0 ? constm1_rtx : const0_rtx;
+  operands[7] = INTVAL (operands[2]) < 0 ? const0_rtx : const1_rtx;
+})
+
+(define_insn "subti3"
+  [(set (match_operand:TI 0 "gr_register_operand" "=&r")
+       (minus:TI (match_operand:TI 1 "gr_reg_or_8bit_operand" "rK")
+                 (match_operand:TI 2 "gr_register_operand" "r")))
+   (clobber (match_scratch:BI 3 "=&c"))]
+  ""
+  "#"
+  [(set_attr "itanium_class" "unknown")])
+
+(define_split
+  [(set (match_operand:TI 0 "register_operand" "")
+       (minus:TI (match_operand:TI 1 "register_operand" "")
+                 (match_operand:TI 2 "register_operand" "")))
+   (clobber (match_scratch:BI 3 "=&c"))]
+  "reload_completed"
+  [(set (match_dup 0) (minus:DI (match_dup 1) (match_dup 2)))
+   (set (match_dup 3) (ltu:BI (match_dup 1) (match_dup 0)))
+   (cond_exec (eq (match_dup 3) (const_int 0))
+             (set (match_dup 4) (minus:DI (match_dup 5) (match_dup 6))))
+   (cond_exec (ne (match_dup 3) (const_int 0))
+             (set (match_dup 4)
+                  (plus:DI (not:DI (match_dup 6)) (match_dup 5))))]
+{
+  operands[4] = gen_highpart (DImode, operands[0]);
+  operands[0] = gen_lowpart (DImode, operands[0]);
+  operands[5] = gen_highpart (DImode, operands[1]);
+  operands[1] = gen_lowpart (DImode, operands[1]);
+  operands[6] = gen_highpart (DImode, operands[2]);
+  operands[2] = gen_lowpart (DImode, operands[2]);
+})
+
+(define_split
+  [(set (match_operand:TI 0 "register_operand" "")
+       (minus:TI (match_operand:TI 1 "immediate_operand" "")
+                 (match_operand:TI 2 "register_operand" "")))
+   (clobber (match_scratch:BI 3 "=&c"))]
+  "reload_completed && satisfies_constraint_K (operands[1])"
+  [(set (match_dup 0) (minus:DI (match_dup 1) (match_dup 2)))
+   (set (match_dup 3) (gtu:BI (match_dup 0) (match_dup 1)))
+   (cond_exec (ne (match_dup 3) (const_int 0))
+             (set (match_dup 4) (minus:DI (match_dup 6) (match_dup 5))))
+   (cond_exec (eq (match_dup 3) (const_int 0))
+             (set (match_dup 4) (minus:DI (match_dup 7) (match_dup 5))))]
+{
+  operands[4] = gen_highpart (DImode, operands[0]);
+  operands[0] = gen_lowpart (DImode, operands[0]);
+  operands[5] = gen_highpart (DImode, operands[2]);
+  operands[2] = gen_lowpart (DImode, operands[2]);
+  operands[6] = INTVAL (operands[1]) < 0 ? GEN_INT (-2) : constm1_rtx;
+  operands[7] = INTVAL (operands[1]) < 0 ? constm1_rtx : const0_rtx;
+})
+
+(define_expand "mulditi3"
+  [(set (match_operand:TI 0 "fr_register_operand" "")
+       (mult:TI (sign_extend:TI
+                  (match_operand:DI 1 "fr_register_operand" ""))
+                (sign_extend:TI
+                  (match_operand:DI 2 "fr_register_operand" ""))))]
+  ""
+  "")
+
+(define_insn_and_split "*mulditi3_internal"
+  [(set (match_operand:TI 0 "fr_register_operand" "=&f")
+       (mult:TI (sign_extend:TI
+                  (match_operand:DI 1 "fr_register_operand" "%f"))
+                (sign_extend:TI
+                  (match_operand:DI 2 "fr_register_operand" "f"))))]
+  ""
+  "#"
+  "reload_completed"
+  [(set (match_dup 0) (mult:DI (match_dup 1) (match_dup 2)))
+   (set (match_dup 3) (truncate:DI
+                       (lshiftrt:TI
+                         (mult:TI (sign_extend:TI (match_dup 1))
+                                  (sign_extend:TI (match_dup 2)))
+                         (const_int 64))))]
+{
+  operands[3] = gen_highpart (DImode, operands[0]);
+  operands[0] = gen_lowpart (DImode, operands[0]);
+}
+  [(set_attr "itanium_class" "unknown")])
+
+(define_expand "umulditi3"
+  [(set (match_operand:TI 0 "fr_register_operand" "")
+       (mult:TI (zero_extend:TI
+                  (match_operand:DI 1 "fr_register_operand" ""))
+                (zero_extend:TI
+                  (match_operand:DI 2 "fr_register_operand" ""))))]
+  ""
+  "")
+
+(define_insn_and_split "*umulditi3_internal"
+  [(set (match_operand:TI 0 "fr_register_operand" "=&f")
+       (mult:TI (zero_extend:TI
+                  (match_operand:DI 1 "fr_register_operand" "%f"))
+                (zero_extend:TI
+                  (match_operand:DI 2 "fr_register_operand" "f"))))]
+  ""
+  "#"
+  "reload_completed"
+  [(set (match_dup 0) (mult:DI (match_dup 1) (match_dup 2)))
+   (set (match_dup 3) (truncate:DI
+                       (lshiftrt:TI
+                         (mult:TI (zero_extend:TI (match_dup 1))
+                                  (zero_extend:TI (match_dup 2)))
+                         (const_int 64))))]
+{
+  operands[3] = gen_highpart (DImode, operands[0]);
+  operands[0] = gen_lowpart (DImode, operands[0]);
+}
+  [(set_attr "itanium_class" "unknown")])
+
+(define_insn_and_split "negti2"
+  [(set (match_operand:TI 0 "gr_register_operand" "=&r")
+       (neg:TI (match_operand:TI 1 "gr_register_operand" "r")))
+   (clobber (match_scratch:BI 2 "=&c"))]
+  ""
+  "#"
+  "reload_completed"
+  [(set (match_dup 2) (eq:BI (match_dup 1) (const_int 0)))
+   (set (match_dup 0) (minus:DI (const_int 0) (match_dup 1)))
+   (cond_exec (eq (match_dup 2) (const_int 0))
+             (set (match_dup 3) (minus:DI (const_int -1) (match_dup 4))))
+   (cond_exec (ne (match_dup 2) (const_int 0))
+             (set (match_dup 3) (minus:DI (const_int 0) (match_dup 4))))]
+{
+  operands[3] = gen_highpart (DImode, operands[0]);
+  operands[0] = gen_lowpart (DImode, operands[0]);
+  operands[4] = gen_highpart (DImode, operands[1]);
+  operands[1] = gen_lowpart (DImode, operands[1]);
+}
+  [(set_attr "itanium_class" "unknown")])
+\f
+;; ::::::::::::::::::::
+;; ::
+;; :: 32-bit floating point arithmetic
 ;; ::
 ;; ::::::::::::::::::::
 
   "fnegabs %0 = %1"
   [(set_attr "itanium_class" "fmisc")])
 
-(define_insn "minsf3"
+(define_insn "copysignsf3"
+  [(set (match_operand:SF 0 "register_operand" "=f")
+       (unspec:SF [(match_operand:SF 1 "fr_reg_or_fp01_operand" "fG")
+                   (match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")]
+                  UNSPEC_COPYSIGN))]
+  ""
+  "fmerge.s %0 = %F2, %F1"
+  [(set_attr "itanium_class" "fmisc")])
+
+(define_insn "*ncopysignsf3"
+  [(set (match_operand:SF 0 "register_operand" "=f")
+       (neg:SF (unspec:SF [(match_operand:SF 1 "fr_reg_or_fp01_operand" "fG")
+                           (match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")]
+                          UNSPEC_COPYSIGN)))]
+  ""
+  "fmerge.ns %0 = %F2, %F1"
+  [(set_attr "itanium_class" "fmisc")])
+
+(define_insn "sminsf3"
   [(set (match_operand:SF 0 "fr_register_operand" "=f")
        (smin:SF (match_operand:SF 1 "fr_register_operand" "f")
                 (match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")))]
   "fmin %0 = %1, %F2"
   [(set_attr "itanium_class" "fmisc")])
 
-(define_insn "maxsf3"
+(define_insn "smaxsf3"
   [(set (match_operand:SF 0 "fr_register_operand" "=f")
        (smax:SF (match_operand:SF 1 "fr_register_operand" "f")
                 (match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")))]
   "fnmpy.s %0 = %1, %2"
   [(set_attr "itanium_class" "fmac")])
 
-;; ??? Is it possible to canonicalize this as (minus (reg) (mult))?
-
 (define_insn "*nmaddsf4"
   [(set (match_operand:SF 0 "fr_register_operand" "=f")
-       (plus:SF (neg:SF (mult:SF
-                          (match_operand:SF 1 "fr_register_operand" "f")
-                          (match_operand:SF 2 "fr_register_operand" "f")))
-                (match_operand:SF 3 "fr_reg_or_fp01_operand" "fG")))]
+       (minus:SF (match_operand:SF 3 "fr_reg_or_fp01_operand" "fG") 
+                 (mult:SF (match_operand:SF 1 "fr_register_operand" "f")
+                          (match_operand:SF 2 "fr_register_operand" "f"))))]
   ""
   "fnma.s %0 = %1, %2, %F3"
   [(set_attr "itanium_class" "fmac")])
 
+(define_insn "*nmaddsf4_alts"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (minus:SF (match_operand:SF 3 "fr_reg_or_fp01_operand" "fG") 
+                 (mult:SF (match_operand:SF 1 "fr_register_operand" "f")
+                          (match_operand:SF 2 "fr_register_operand" "f"))))
+   (use (match_operand:SI 4 "const_int_operand" ""))]
+  ""
+  "fnma.s.s%4 %0 = %1, %2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
 (define_expand "divsf3"
   [(set (match_operand:SF 0 "fr_register_operand" "")
        (div:SF (match_operand:SF 1 "fr_register_operand" "")
   "TARGET_INLINE_FLOAT_DIV"
 {
   rtx insn;
-  if (TARGET_INLINE_FLOAT_DIV_LAT)
+  if (TARGET_INLINE_FLOAT_DIV == INL_MIN_LAT)
     insn = gen_divsf3_internal_lat (operands[0], operands[1], operands[2]);
   else
     insn = gen_divsf3_internal_thr (operands[0], operands[1], operands[2]);
    (clobber (match_scratch:XF 3 "=&f"))
    (clobber (match_scratch:XF 4 "=f"))
    (clobber (match_scratch:BI 5 "=c"))]
-  "TARGET_INLINE_FLOAT_DIV_LAT"
+  "TARGET_INLINE_FLOAT_DIV == INL_MIN_LAT"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 6) (div:XF (const_int 1) (match_dup 8)))
+  [(parallel [(set (match_dup 6) (unspec:XF [(const_int 1) (match_dup 8)]
+                                           UNSPEC_FR_RECIP_APPROX_RES))
              (set (match_dup 5) (unspec:BI [(match_dup 7) (match_dup 8)]
                                            UNSPEC_FR_RECIP_APPROX))
-             (use (const_int 1))])
+             (use (const_int 0))])
    (cond_exec (ne (match_dup 5) (const_int 0))
      (parallel [(set (match_dup 3) (mult:XF (match_dup 7) (match_dup 6)))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 5) (const_int 0))
-     (parallel [(set (match_dup 4)
-                    (plus:XF (neg:XF (mult:XF (match_dup 8) (match_dup 6)))
-                             (match_dup 10)))
-               (use (const_int 1))]))
-   (cond_exec (ne (match_dup 5) (const_int 0))
-     (parallel [(set (match_dup 3)
-                    (plus:XF (mult:XF (match_dup 4) (match_dup 3))
-                             (match_dup 3)))
-               (use (const_int 1))]))
-   (cond_exec (ne (match_dup 5) (const_int 0))
-     (parallel [(set (match_dup 4) (mult:XF (match_dup 4) (match_dup 4)))
-               (use (const_int 1))]))
-   (cond_exec (ne (match_dup 5) (const_int 0))
-     (parallel [(set (match_dup 3)
-                    (plus:XF (mult:XF (match_dup 4) (match_dup 3))
-                             (match_dup 3)))
-               (use (const_int 1))]))
-   (cond_exec (ne (match_dup 5) (const_int 0))
-     (parallel [(set (match_dup 4) (mult:XF (match_dup 4) (match_dup 4)))
-               (use (const_int 1))]))
-   (cond_exec (ne (match_dup 5) (const_int 0))
-     (parallel [(set (match_dup 9)
-                    (float_truncate:DF
-                      (plus:XF (mult:XF (match_dup 4) (match_dup 3))
-                             (match_dup 3))))
-               (use (const_int 1))]))
-   (cond_exec (ne (match_dup 5) (const_int 0))
-     (set (match_dup 0)
-         (float_truncate:SF (match_dup 6))))
-  ] 
-{
-  operands[6] = gen_rtx_REG (XFmode, REGNO (operands[0]));
-  operands[7] = gen_rtx_REG (XFmode, REGNO (operands[1]));
-  operands[8] = gen_rtx_REG (XFmode, REGNO (operands[2]));
-  operands[9] = gen_rtx_REG (DFmode, REGNO (operands[0]));
-  operands[10] = CONST1_RTX (XFmode);
-}
-  [(set_attr "predicable" "no")])
-
-(define_insn_and_split "divsf3_internal_thr"
-  [(set (match_operand:SF 0 "fr_register_operand" "=&f")
-       (div:SF (match_operand:SF 1 "fr_register_operand" "f")
-               (match_operand:SF 2 "fr_register_operand" "f")))
-   (clobber (match_scratch:XF 3 "=&f"))
-   (clobber (match_scratch:XF 4 "=f"))
-   (clobber (match_scratch:BI 5 "=c"))]
-  "TARGET_INLINE_FLOAT_DIV_THR"
-  "#"
-  "&& reload_completed"
-  [(parallel [(set (match_dup 6) (div:XF (const_int 1) (match_dup 8)))
-             (set (match_dup 5) (unspec:BI [(match_dup 7) (match_dup 8)]
-                                           UNSPEC_FR_RECIP_APPROX))
-             (use (const_int 1))])
+     (parallel [(set (match_dup 4)
+                    (minus:XF (match_dup 10)
+                              (mult:XF (match_dup 8) (match_dup 6))))
+               (use (const_int 1))]))
    (cond_exec (ne (match_dup 5) (const_int 0))
      (parallel [(set (match_dup 3)
-                    (plus:XF (neg:XF (mult:XF (match_dup 8) (match_dup 6)))
-                             (match_dup 10)))
+                    (plus:XF (mult:XF (match_dup 4) (match_dup 3))
+                             (match_dup 3)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 4) (mult:XF (match_dup 4) (match_dup 4)))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 5) (const_int 0))
      (parallel [(set (match_dup 3)
-                    (plus:XF (mult:XF (match_dup 3) (match_dup 3))
+                    (plus:XF (mult:XF (match_dup 4) (match_dup 3))
                              (match_dup 3)))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 5) (const_int 0))
-     (parallel [(set (match_dup 6)
-                    (plus:XF (mult:XF (match_dup 3) (match_dup 6))
-                             (match_dup 6)))
+     (parallel [(set (match_dup 4) (mult:XF (match_dup 4) (match_dup 4)))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 5) (const_int 0))
      (parallel [(set (match_dup 9)
-                    (float_truncate:SF
-                      (mult:XF (match_dup 7) (match_dup 6))))
-               (use (const_int 1))]))
-   (cond_exec (ne (match_dup 5) (const_int 0))
-     (parallel [(set (match_dup 4)
-                    (plus:XF (neg:XF (mult:XF (match_dup 8) (match_dup 3)))
-                             (match_dup 7)))
+                    (float_truncate:DF
+                      (plus:XF (mult:XF (match_dup 4) (match_dup 3))
+                             (match_dup 3))))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 5) (const_int 0))
      (set (match_dup 0)
-         (float_truncate:SF
-           (plus:XF (mult:XF (match_dup 4) (match_dup 6))
-                             (match_dup 3)))))
+         (float_truncate:SF (match_dup 6))))
   ] 
 {
   operands[6] = gen_rtx_REG (XFmode, REGNO (operands[0]));
   operands[7] = gen_rtx_REG (XFmode, REGNO (operands[1]));
   operands[8] = gen_rtx_REG (XFmode, REGNO (operands[2]));
-  operands[9] = gen_rtx_REG (SFmode, REGNO (operands[3]));
+  operands[9] = gen_rtx_REG (DFmode, REGNO (operands[0]));
   operands[10] = CONST1_RTX (XFmode);
 }
   [(set_attr "predicable" "no")])
 (define_insn "*sqrt_approx"
   [(set (match_operand:XF 0 "fr_register_operand" "=f")
         (div:XF (const_int 1)
-                (sqrt:XF (match_operand:XF 2 "fr_register_operand" "f"))))
+               (unspec:XF [(match_operand:XF 2 "fr_register_operand" "f")]
+                          UNSPEC_FR_SQRT_RECIP_APPROX_RES)))
    (set (match_operand:BI 1 "register_operand" "=c")
         (unspec:BI [(match_dup 2)] UNSPEC_FR_SQRT_RECIP_APPROX))
    (use (match_operand:SI 3 "const_int_operand" "")) ]
   [(set_attr "itanium_class" "fmisc")
    (set_attr "predicable" "no")])
 
-(define_insn "*setf_exp_xf"
+(define_insn "setf_exp_xf"
   [(set (match_operand:XF 0 "fr_register_operand" "=f")
         (unspec:XF [(match_operand:DI 1 "register_operand" "r")]
                   UNSPEC_SETF_EXP))]
   "TARGET_INLINE_SQRT"
 {
   rtx insn;
-  if (TARGET_INLINE_SQRT_LAT)
 #if 0
+  if (TARGET_INLINE_SQRT == INL_MIN_LAT)
     insn = gen_sqrtsf2_internal_lat (operands[0], operands[1]);
+  else
 #else
-    abort ();
+  gcc_assert (TARGET_INLINE_SQRT != INL_MIN_LAT);
 #endif
-  else
-    insn = gen_sqrtsf2_internal_thr (operands[0], operands[1]);
+  insn = gen_sqrtsf2_internal_thr (operands[0], operands[1]);
   emit_insn (insn);
   DONE;
 })
    (clobber (match_scratch:XF 5 "=&f"))
    ;; Register p6 in optimization guide.
    (clobber (match_scratch:BI 6 "=c"))]
-  "TARGET_INLINE_SQRT_THR"
+  "TARGET_INLINE_SQRT == INL_MAX_THR"
   "#"
   "&& reload_completed"
   [ ;; exponent of +1/2 in r2
     ;; y0 = 1/sqrt(a) in f7
     (parallel [(set (match_dup 7)
                     (div:XF (const_int 1)
-                            (sqrt:XF (match_dup 8))))
+                           (unspec:XF [(match_dup 8)]
+                                      UNSPEC_FR_SQRT_RECIP_APPROX_RES)))
                (set (match_dup 6)
                     (unspec:BI [(match_dup 8)]
-                                 UNSPEC_FR_SQRT_RECIP_APPROX))
+                              UNSPEC_FR_SQRT_RECIP_APPROX))
                (use (const_int 0))])
     ;; Step 2
     ;; H0 = 1/2 * y0 in f9
     ;; d = 1/2 - S0 * H0 in f10
     (cond_exec (ne (match_dup 6) (const_int 0))
       (parallel [(set (match_dup 5)
-                      (plus:XF (neg:XF (mult:XF (match_dup 7) (match_dup 4)))
-                               (match_dup 3)))
+                      (minus:XF (match_dup 3)
+                               (mult:XF (match_dup 7) (match_dup 4))))
                  (use (const_int 1))]))
     ;; Step 5
     ;; d' = d + 1/2 * d in f8
     ;; d1 = a - S1 * S1 in f9
     (cond_exec (ne (match_dup 6) (const_int 0))
        (parallel [(set (match_dup 4)
-                       (plus:XF (neg:XF (mult:XF (match_dup 7) (match_dup 7)))
-                                (match_dup 8)))
+                       (minus:XF (match_dup 8)
+                                (mult:XF (match_dup 7) (match_dup 7))))
                   (use (const_int 1))]))
     ;; Step 10
     ;; S = S1 + d1 * H1 in f7
 \f
 ;; ::::::::::::::::::::
 ;; ::
-;; :: 64 bit floating point arithmetic
+;; :: 64-bit floating point arithmetic
 ;; ::
 ;; ::::::::::::::::::::
 
   "fnegabs %0 = %1"
   [(set_attr "itanium_class" "fmisc")])
 
-(define_insn "mindf3"
+(define_insn "copysigndf3"
+  [(set (match_operand:DF 0 "register_operand" "=f")
+       (unspec:DF [(match_operand:DF 1 "fr_reg_or_fp01_operand" "fG")
+                   (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")]
+                  UNSPEC_COPYSIGN))]
+  ""
+  "fmerge.s %0 = %F2, %F1"
+  [(set_attr "itanium_class" "fmisc")])
+
+(define_insn "*ncopysigndf3"
+  [(set (match_operand:DF 0 "register_operand" "=f")
+       (neg:DF (unspec:DF [(match_operand:DF 1 "fr_reg_or_fp01_operand" "fG")
+                           (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")]
+                          UNSPEC_COPYSIGN)))]
+  ""
+  "fmerge.ns %0 = %F2, %F1"
+  [(set_attr "itanium_class" "fmisc")])
+
+(define_insn "smindf3"
   [(set (match_operand:DF 0 "fr_register_operand" "=f")
        (smin:DF (match_operand:DF 1 "fr_register_operand" "f")
                 (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")))]
   "fmin %0 = %1, %F2"
   [(set_attr "itanium_class" "fmisc")])
 
-(define_insn "maxdf3"
+(define_insn "smaxdf3"
   [(set (match_operand:DF 0 "fr_register_operand" "=f")
        (smax:DF (match_operand:DF 1 "fr_register_operand" "f")
                 (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")))]
   "fnmpy.s %0 = %1, %2"
   [(set_attr "itanium_class" "fmac")])
 
-;; ??? Is it possible to canonicalize this as (minus (reg) (mult))?
-
 (define_insn "*nmadddf4"
   [(set (match_operand:DF 0 "fr_register_operand" "=f")
-       (plus:DF (neg:DF (mult:DF
-                          (match_operand:DF 1 "fr_register_operand" "f")
-                          (match_operand:DF 2 "fr_register_operand" "f")))
-                (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")))]
+       (minus:DF (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")
+                 (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+                          (match_operand:DF 2 "fr_register_operand" "f"))))]
   ""
   "fnma.d %0 = %1, %2, %F3"
   [(set_attr "itanium_class" "fmac")])
 
 (define_insn "*nmadddf4_alts"
   [(set (match_operand:DF 0 "fr_register_operand" "=f")
-       (plus:DF (neg:DF (mult:DF
-                          (match_operand:DF 1 "fr_register_operand" "f")
-                          (match_operand:DF 2 "fr_register_operand" "f")))
-                (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")))
+       (minus:DF (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")
+                 (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+                          (match_operand:DF 2 "fr_register_operand" "f"))))
    (use (match_operand:SI 4 "const_int_operand" ""))]
   ""
   "fnma.d.s%4 %0 = %1, %2, %F3"
   [(set_attr "itanium_class" "fmac")])
 
-(define_insn "*nmadddf4_trunc"
+(define_insn "*nmadddf4_truncsf"
   [(set (match_operand:SF 0 "fr_register_operand" "=f")
        (float_truncate:SF
-         (plus:DF (neg:DF (mult:DF
-                            (match_operand:DF 1 "fr_register_operand" "f")
-                            (match_operand:DF 2 "fr_register_operand" "f")))
-                  (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG"))))]
+       (minus:DF (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")
+                 (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+                          (match_operand:DF 2 "fr_register_operand" "f")))))]
   ""
   "fnma.s %0 = %1, %2, %F3"
   [(set_attr "itanium_class" "fmac")])
 
+(define_insn "*nmadddf4_truncsf_alts"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+       (minus:DF (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")
+                 (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+                          (match_operand:DF 2 "fr_register_operand" "f")))))
+   (use (match_operand:SI 4 "const_int_operand" ""))]
+  ""
+  "fnma.s.s%4 %0 = %1, %2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
 (define_expand "divdf3"
   [(set (match_operand:DF 0 "fr_register_operand" "")
        (div:DF (match_operand:DF 1 "fr_register_operand" "")
   "TARGET_INLINE_FLOAT_DIV"
 {
   rtx insn;
-  if (TARGET_INLINE_FLOAT_DIV_LAT)
+  if (TARGET_INLINE_FLOAT_DIV == INL_MIN_LAT)
     insn = gen_divdf3_internal_lat (operands[0], operands[1], operands[2]);
   else
     insn = gen_divdf3_internal_thr (operands[0], operands[1], operands[2]);
    (clobber (match_scratch:XF 4 "=&f"))
    (clobber (match_scratch:XF 5 "=&f"))
    (clobber (match_scratch:BI 6 "=c"))]
-  "TARGET_INLINE_FLOAT_DIV_LAT"
+  "TARGET_INLINE_FLOAT_DIV == INL_MIN_LAT"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 7) (div:XF (const_int 1) (match_dup 9)))
+  [(parallel [(set (match_dup 7) (unspec:XF [(const_int 1) (match_dup 9)]
+                                           UNSPEC_FR_RECIP_APPROX_RES))
              (set (match_dup 6) (unspec:BI [(match_dup 8) (match_dup 9)]
                                            UNSPEC_FR_RECIP_APPROX))
-             (use (const_int 1))])
+             (use (const_int 0))])
    (cond_exec (ne (match_dup 6) (const_int 0))
      (parallel [(set (match_dup 3) (mult:XF (match_dup 8) (match_dup 7)))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 6) (const_int 0))
      (parallel [(set (match_dup 4)
-                    (plus:XF (neg:XF (mult:XF (match_dup 9) (match_dup 7)))
-                             (match_dup 12)))
+                    (minus:XF (match_dup 12)
+                              (mult:XF (match_dup 9) (match_dup 7))))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 6) (const_int 0))
      (parallel [(set (match_dup 3)
    (cond_exec (ne (match_dup 6) (const_int 0))
      (parallel [(set (match_dup 11)
                     (float_truncate:DF
-                      (plus:XF (neg:XF (mult:XF (match_dup 9) (match_dup 3)))
-                               (match_dup 8))))
+                      (minus:XF (match_dup 8)
+                                (mult:XF (match_dup 9) (match_dup 3)))))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 6) (const_int 0))
      (set (match_dup 0)
 }
   [(set_attr "predicable" "no")])
 
-(define_insn_and_split "divdf3_internal_thr"
-  [(set (match_operand:DF 0 "fr_register_operand" "=&f")
-       (div:DF (match_operand:DF 1 "fr_register_operand" "f")
-               (match_operand:DF 2 "fr_register_operand" "f")))
-   (clobber (match_scratch:XF 3 "=&f"))
-   (clobber (match_scratch:DF 4 "=f"))
-   (clobber (match_scratch:BI 5 "=c"))]
-  "TARGET_INLINE_FLOAT_DIV_THR"
-  "#"
-  "&& reload_completed"
-  [(parallel [(set (match_dup 6) (div:XF (const_int 1) (match_dup 8)))
-             (set (match_dup 5) (unspec:BI [(match_dup 7) (match_dup 8)]
-                                           UNSPEC_FR_RECIP_APPROX))
-             (use (const_int 1))])
-   (cond_exec (ne (match_dup 5) (const_int 0))
-     (parallel [(set (match_dup 3)
-                    (plus:XF (neg:XF (mult:XF (match_dup 8) (match_dup 6)))
-                             (match_dup 10)))
-               (use (const_int 1))]))
-   (cond_exec (ne (match_dup 5) (const_int 0))
-     (parallel [(set (match_dup 6)
-                    (plus:XF (mult:XF (match_dup 3) (match_dup 6))
-                             (match_dup 6)))
-               (use (const_int 1))]))
-   (cond_exec (ne (match_dup 5) (const_int 0))
-     (parallel [(set (match_dup 3)
-                    (mult:XF (match_dup 3) (match_dup 3)))
-               (use (const_int 1))]))
-   (cond_exec (ne (match_dup 5) (const_int 0))
-     (parallel [(set (match_dup 6)
-                    (plus:XF (mult:XF (match_dup 3) (match_dup 6))
-                             (match_dup 6)))
-               (use (const_int 1))]))
-   (cond_exec (ne (match_dup 5) (const_int 0))
-     (parallel [(set (match_dup 3)
-                    (mult:XF (match_dup 3) (match_dup 3)))
-               (use (const_int 1))]))
-   (cond_exec (ne (match_dup 5) (const_int 0))
-     (parallel [(set (match_dup 6)
-                    (plus:XF (mult:XF (match_dup 3) (match_dup 6))
-                             (match_dup 6)))
-               (use (const_int 1))]))
-   (cond_exec (ne (match_dup 5) (const_int 0))
-     (parallel [(set (match_dup 9)
-                    (float_truncate:DF
-                      (mult:XF (match_dup 7) (match_dup 3))))
-               (use (const_int 1))]))
-   (cond_exec (ne (match_dup 5) (const_int 0))
-     (parallel [(set (match_dup 4)
-                    (plus:DF (neg:DF (mult:DF (match_dup 2) (match_dup 9)))
-                             (match_dup 1)))
-               (use (const_int 1))]))
-   (cond_exec (ne (match_dup 5) (const_int 0))
-     (set (match_dup 0)
-         (plus:DF (mult:DF (match_dup 4) (match_dup 0))
-                           (match_dup 9))))
-  ] 
-{
-  operands[6] = gen_rtx_REG (XFmode, REGNO (operands[0]));
-  operands[7] = gen_rtx_REG (XFmode, REGNO (operands[1]));
-  operands[8] = gen_rtx_REG (XFmode, REGNO (operands[2]));
-  operands[9] = gen_rtx_REG (DFmode, REGNO (operands[3]));
-  operands[10] = CONST1_RTX (XFmode);
-}
-  [(set_attr "predicable" "no")])
-
 ;; Inline square root.
 
 (define_expand "sqrtdf2"
   "TARGET_INLINE_SQRT"
 {
   rtx insn;
-  if (TARGET_INLINE_SQRT_LAT)
 #if 0
+  if (TARGET_INLINE_SQRT == INL_MIN_LAT)
     insn = gen_sqrtdf2_internal_lat (operands[0], operands[1]);
+  else
 #else
-    abort ();
+  gcc_assert (TARGET_INLINE_SQRT != INL_MIN_LAT);
 #endif
-  else
-    insn = gen_sqrtdf2_internal_thr (operands[0], operands[1]);
+  insn = gen_sqrtdf2_internal_thr (operands[0], operands[1]);
   emit_insn (insn);
   DONE;
 })
    (clobber (match_scratch:XF 5 "=&f"))
    ;; Register p6 in optimization guide.
    (clobber (match_scratch:BI 6 "=c"))]
-  "TARGET_INLINE_SQRT_THR"
+  "TARGET_INLINE_SQRT == INL_MAX_THR"
   "#"
   "&& reload_completed"
   [ ;; exponent of +1/2 in r2
     ;; y0 = 1/sqrt(a) in f7
     (parallel [(set (match_dup 7)
                     (div:XF (const_int 1)
-                            (sqrt:XF (match_dup 8))))
+                           (unspec:XF [(match_dup 8)]
+                                      UNSPEC_FR_SQRT_RECIP_APPROX_RES)))
                (set (match_dup 6)
                     (unspec:BI [(match_dup 8)]
-                                 UNSPEC_FR_SQRT_RECIP_APPROX))
+                              UNSPEC_FR_SQRT_RECIP_APPROX))
                (use (const_int 0))])
     ;; Step 2
     ;; H0 = 1/2 * y0 in f8
     ;; r0 = 1/2 - G0 * H0 in f9
     (cond_exec (ne (match_dup 6) (const_int 0))
       (parallel [(set (match_dup 4)
-                      (plus:XF (neg:XF (mult:XF (match_dup 7) (match_dup 3)))
-                               (match_dup 5)))
+                      (minus:XF (match_dup 5)
+                               (mult:XF (match_dup 7) (match_dup 3))))
                  (use (const_int 1))]))
     ;; Step 5
     ;; H1 = H0 + r0 * H0 in f8
     ;; r1 = 1/2 - G1 * H1 in f9
     (cond_exec (ne (match_dup 6) (const_int 0))
       (parallel [(set (match_dup 4)
-                      (plus:XF (neg:XF (mult:XF (match_dup 7) (match_dup 3)))
-                               (match_dup 5)))
+                      (minus:XF (match_dup 5)
+                               (mult:XF (match_dup 7) (match_dup 3))))
                  (use (const_int 1))]))
     ;; Step 8
     ;; H2 = H1 + r1 * H1 in f8
     ;; d2 = a - G2 * G2 in f9
     (cond_exec (ne (match_dup 6) (const_int 0))
        (parallel [(set (match_dup 4)
-                       (plus:XF (neg:XF (mult:XF (match_dup 7) (match_dup 7)))
-                                (match_dup 8)))
+                       (minus:XF (match_dup 8)
+                                (mult:XF (match_dup 7) (match_dup 7))))
                   (use (const_int 1))]))
     ;; Step 11
     ;; G3 = G2 + d2 * H2 in f7
     ;; d3 = a - G3 * G3 in f9
     (cond_exec (ne (match_dup 6) (const_int 0))
        (parallel [(set (match_dup 4)
-                       (plus:XF (neg:XF (mult:XF (match_dup 7) (match_dup 7)))
-                                (match_dup 8)))
+                       (minus:XF (match_dup 8)
+                                (mult:XF (match_dup 7) (match_dup 7))))
                   (use (const_int 1))]))
     ;; Step 13
     ;; S = G3 + d3 * H2 in f7
 \f
 ;; ::::::::::::::::::::
 ;; ::
-;; :: 80 bit floating point arithmetic
+;; :: 80-bit floating point arithmetic
 ;; ::
 ;; ::::::::::::::::::::
 
   "fnegabs %0 = %F1"
   [(set_attr "itanium_class" "fmisc")])
 
-(define_insn "minxf3"
+(define_insn "copysignxf3"
+  [(set (match_operand:XF 0 "register_operand" "=f")
+       (unspec:XF [(match_operand:XF 1 "fr_reg_or_fp01_operand" "fG")
+                   (match_operand:XF 2 "fr_reg_or_fp01_operand" "fG")]
+                  UNSPEC_COPYSIGN))]
+  ""
+  "fmerge.s %0 = %F2, %F1"
+  [(set_attr "itanium_class" "fmisc")])
+
+(define_insn "*ncopysignxf3"
+  [(set (match_operand:XF 0 "register_operand" "=f")
+       (neg:XF (unspec:XF [(match_operand:XF 1 "fr_reg_or_fp01_operand" "fG")
+                           (match_operand:XF 2 "fr_reg_or_fp01_operand" "fG")]
+                          UNSPEC_COPYSIGN)))]
+  ""
+  "fmerge.ns %0 = %F2, %F1"
+  [(set_attr "itanium_class" "fmisc")])
+
+(define_insn "sminxf3"
   [(set (match_operand:XF 0 "fr_register_operand" "=f")
        (smin:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
                 (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")))]
   "fmin %0 = %F1, %F2"
   [(set_attr "itanium_class" "fmisc")])
 
-(define_insn "maxxf3"
+(define_insn "smaxxf3"
   [(set (match_operand:XF 0 "fr_register_operand" "=f")
        (smax:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
                 (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")))]
   "fnmpy.d %0 = %F1, %F2"
   [(set_attr "itanium_class" "fmac")])
 
-;; ??? Is it possible to canonicalize this as (minus (reg) (mult))?
-
 (define_insn "*nmaddxf4"
   [(set (match_operand:XF 0 "fr_register_operand" "=f")
-       (plus:XF (neg:XF (mult:XF
-                         (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
-                         (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")))
-                (match_operand:XF 3 "xfreg_or_fp01_operand" "fG")))]
+       (minus:XF (match_operand:XF 3 "xfreg_or_fp01_operand" "fG")
+                 (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                          (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")
+   )))]
   ""
   "fnma %0 = %F1, %F2, %F3"
   [(set_attr "itanium_class" "fmac")])
 (define_insn "*nmaddxf4_truncsf"
   [(set (match_operand:SF 0 "fr_register_operand" "=f")
        (float_truncate:SF
-         (plus:XF (neg:XF (mult:XF
-                           (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
-                           (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")))
-                  (match_operand:XF 3 "xfreg_or_fp01_operand" "fG"))))]
+         (minus:XF (match_operand:XF 3 "xfreg_or_fp01_operand" "fG") 
+                   (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                            (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")
+   ))))]
   ""
   "fnma.s %0 = %F1, %F2, %F3"
   [(set_attr "itanium_class" "fmac")])
 (define_insn "*nmaddxf4_truncdf"
   [(set (match_operand:DF 0 "fr_register_operand" "=f")
        (float_truncate:DF
-         (plus:XF (neg:XF (mult:XF
-                           (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
-                           (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")))
-                  (match_operand:XF 3 "xfreg_or_fp01_operand" "fG"))))]
+         (minus:XF (match_operand:XF 3 "xfreg_or_fp01_operand" "fG") 
+                   (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                            (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")
+   ))))]
   ""
   "fnma.d %0 = %F1, %F2, %F3"
   [(set_attr "itanium_class" "fmac")])
 
 (define_insn "*nmaddxf4_alts"
   [(set (match_operand:XF 0 "fr_register_operand" "=f")
-       (plus:XF (neg:XF (mult:XF
-                         (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
-                         (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")))
-                (match_operand:XF 3 "xfreg_or_fp01_operand" "fG")))
+       (minus:XF (match_operand:XF 3 "xfreg_or_fp01_operand" "fG")
+                 (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                          (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")
+   )))
    (use (match_operand:SI 4 "const_int_operand" ""))]
   ""
   "fnma.s%4 %0 = %F1, %F2, %F3"
   [(set_attr "itanium_class" "fmac")])
 
+(define_insn "*nmaddxf4_truncsf_alts"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+         (minus:XF (match_operand:XF 3 "xfreg_or_fp01_operand" "fG") 
+                   (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                            (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")
+   ))))
+   (use (match_operand:SI 4 "const_int_operand" ""))]
+  ""
+  "fnma.s.s%4 %0 = %F1, %F2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
 (define_insn "*nmaddxf4_truncdf_alts"
   [(set (match_operand:DF 0 "fr_register_operand" "=f")
        (float_truncate:DF
-         (plus:XF (neg:XF
-                    (mult:XF
-                      (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
-                      (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")))
-                (match_operand:XF 3 "xfreg_or_fp01_operand" "fG"))))
+         (minus:XF (match_operand:XF 3 "xfreg_or_fp01_operand" "fG") 
+                   (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                            (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")
+   ))))
    (use (match_operand:SI 4 "const_int_operand" ""))]
   ""
   "fnma.d.s%4 %0 = %F1, %F2, %F3"
   "TARGET_INLINE_FLOAT_DIV"
 {
   rtx insn;
-  if (TARGET_INLINE_FLOAT_DIV_LAT)
+  if (TARGET_INLINE_FLOAT_DIV == INL_MIN_LAT)
     insn = gen_divxf3_internal_lat (operands[0], operands[1], operands[2]);
   else
     insn = gen_divxf3_internal_thr (operands[0], operands[1], operands[2]);
    (clobber (match_scratch:XF 5 "=&f"))
    (clobber (match_scratch:XF 6 "=&f"))
    (clobber (match_scratch:BI 7 "=c"))]
-  "TARGET_INLINE_FLOAT_DIV_LAT"
+  "TARGET_INLINE_FLOAT_DIV == INL_MIN_LAT"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0) (div:XF (const_int 1) (match_dup 2)))
+  [(parallel [(set (match_dup 0) (unspec:XF [(const_int 1) (match_dup 2)]
+                                           UNSPEC_FR_RECIP_APPROX_RES))
              (set (match_dup 7) (unspec:BI [(match_dup 1) (match_dup 2)]
                                            UNSPEC_FR_RECIP_APPROX))
-             (use (const_int 1))])
+             (use (const_int 0))])
    (cond_exec (ne (match_dup 7) (const_int 0))
      (parallel [(set (match_dup 3)
-                    (plus:XF (neg:XF (mult:XF (match_dup 2) (match_dup 0)))
-                             (match_dup 8)))
+                    (minus:XF (match_dup 8)
+                              (mult:XF (match_dup 2) (match_dup 0))))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 7) (const_int 0))
      (parallel [(set (match_dup 4) (mult:XF (match_dup 1) (match_dup 0)))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 7) (const_int 0))
      (parallel [(set (match_dup 4)
-                    (plus:XF (neg:XF (mult:XF (match_dup 2) (match_dup 4)))
-                             (match_dup 1)))
+                    (minus:XF (match_dup 1)
+                              (mult:XF (match_dup 2) (match_dup 4))))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 7) (const_int 0))
      (parallel [(set (match_dup 3)
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 7) (const_int 0))
      (parallel [(set (match_dup 5)
-                    (plus:XF (neg:XF (mult:XF (match_dup 2) (match_dup 0)))
-                             (match_dup 8)))
+                    (minus:XF (match_dup 8)
+                              (mult:XF (match_dup 2) (match_dup 0))))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 7) (const_int 0))
      (parallel [(set (match_dup 0)
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 7) (const_int 0))
      (parallel [(set (match_dup 4)
-                    (plus:XF (neg:XF (mult:XF (match_dup 2) (match_dup 3)))
-                             (match_dup 1)))
+                    (minus:XF (match_dup 1)
+                              (mult:XF (match_dup 2) (match_dup 3))))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 7) (const_int 0))
      (set (match_dup 0)
    (clobber (match_scratch:XF 3 "=&f"))
    (clobber (match_scratch:XF 4 "=&f"))
    (clobber (match_scratch:BI 5 "=c"))]
-  "TARGET_INLINE_FLOAT_DIV_THR"
+  "TARGET_INLINE_FLOAT_DIV == INL_MAX_THR"
   "#"
   "&& reload_completed"
-  [(parallel [(set (match_dup 0) (div:XF (const_int 1) (match_dup 2)))
+  [(parallel [(set (match_dup 0) (unspec:XF [(const_int 1) (match_dup 2)]
+                                           UNSPEC_FR_RECIP_APPROX_RES))
              (set (match_dup 5) (unspec:BI [(match_dup 1) (match_dup 2)]
                                            UNSPEC_FR_RECIP_APPROX))
-             (use (const_int 1))])
+             (use (const_int 0))])
    (cond_exec (ne (match_dup 5) (const_int 0))
      (parallel [(set (match_dup 3)
-                    (plus:XF (neg:XF (mult:XF (match_dup 2) (match_dup 0)))
-                             (match_dup 6)))
+                    (minus:XF (match_dup 6)
+                              (mult:XF (match_dup 2) (match_dup 0))))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 5) (const_int 0))
      (parallel [(set (match_dup 4)
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 5) (const_int 0))
      (parallel [(set (match_dup 0)
-                    (plus:XF (neg:XF (mult:XF (match_dup 2) (match_dup 3)))
-                             (match_dup 6)))
+                    (minus:XF (match_dup 6)
+                              (mult:XF (match_dup 2) (match_dup 3))))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 5) (const_int 0))
      (parallel [(set (match_dup 0)
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 5) (const_int 0))
      (parallel [(set (match_dup 3)
-                    (plus:XF (neg:XF (mult:XF (match_dup 2) (match_dup 4)))
-                             (match_dup 1)))
+                    (minus:XF (match_dup 1)
+                              (mult:XF (match_dup 2) (match_dup 4))))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 5) (const_int 0))
      (parallel [(set (match_dup 3)
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 5) (const_int 0))
      (parallel [(set (match_dup 4)
-                    (plus:XF (neg:XF (mult:XF (match_dup 2) (match_dup 0)))
-                             (match_dup 6)))
+                    (minus:XF (match_dup 6)
+                              (mult:XF (match_dup 2) (match_dup 0))))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 5) (const_int 0))
      (parallel [(set (match_dup 0)
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 5) (const_int 0))
      (parallel [(set (match_dup 4)
-                    (plus:XF (neg:XF (mult:XF (match_dup 2) (match_dup 3)))
-                             (match_dup 1)))
+                    (minus:XF (match_dup 1)
+                              (mult:XF (match_dup 2) (match_dup 3))))
                (use (const_int 1))]))
    (cond_exec (ne (match_dup 5) (const_int 0))
      (set (match_dup 0)
   "TARGET_INLINE_SQRT"
 {
   rtx insn;
-  if (TARGET_INLINE_SQRT_LAT)
 #if 0
+  if (TARGET_INLINE_SQRT == INL_MIN_LAT)
     insn = gen_sqrtxf2_internal_lat (operands[0], operands[1]);
+  else
 #else
-    abort ();
+  gcc_assert (TARGET_INLINE_SQRT != INL_MIN_LAT);
 #endif
-  else
-    insn = gen_sqrtxf2_internal_thr (operands[0], operands[1]);
+  insn = gen_sqrtxf2_internal_thr (operands[0], operands[1]);
   emit_insn (insn);
   DONE;
 })
    (clobber (match_scratch:XF 6 "=&f"))
    ;; Register p6 in optimization guide.
    (clobber (match_scratch:BI 7 "=c"))]
-  "TARGET_INLINE_SQRT_THR"
+  "TARGET_INLINE_SQRT == INL_MAX_THR"
   "#"
   "&& reload_completed"
   [ ;; exponent of +1/2 in r2
     ;; y0 = 1/sqrt(a) in f7
     (parallel [(set (match_dup 8)
                     (div:XF (const_int 1)
-                            (sqrt:XF (match_dup 9))))
+                           (unspec:XF [(match_dup 9)]
+                                      UNSPEC_FR_SQRT_RECIP_APPROX_RES)))
                (set (match_dup 7)
                     (unspec:BI [(match_dup 9)]
-                                 UNSPEC_FR_SQRT_RECIP_APPROX))
+                              UNSPEC_FR_SQRT_RECIP_APPROX))
                (use (const_int 0))])
     ;; Step 2
     ;; H0 = 1/2 * y0 in f9
     ;; d0 = 1/2 - S0 * H0 in f10
     (cond_exec (ne (match_dup 7) (const_int 0))
       (parallel [(set (match_dup 5)
-                      (plus:XF (neg:XF (mult:XF (match_dup 8) (match_dup 4)))
-                               (match_dup 3)))
+                      (minus:XF (match_dup 3)
+                               (mult:XF (match_dup 8) (match_dup 4))))
                  (use (const_int 1))]))
     ;; Step 5
     ;; H1 = H0 + d0 * H0 in f9
     ;; d1 = 1/2 - S1 * H1 in f10
     (cond_exec (ne (match_dup 7) (const_int 0))
       (parallel [(set (match_dup 5)
-                      (plus:XF (neg:XF (mult:XF (match_dup 8) (match_dup 4)))
-                               (match_dup 3)))
+                      (minus:XF (match_dup 3)
+                               (mult:XF (match_dup 8) (match_dup 4))))
                  (use (const_int 1))]))
     ;; Step 8
     ;; H2 = H1 + d1 * H1 in f9
     ;; d2 = 1/2 - S2 * H2 in f10
     (cond_exec (ne (match_dup 7) (const_int 0))
        (parallel [(set (match_dup 5)
-                       (plus:XF (neg:XF (mult:XF (match_dup 8) (match_dup 4)))
-                                (match_dup 3)))
+                       (minus:XF (match_dup 3)
+                                (mult:XF (match_dup 8) (match_dup 4))))
                   (use (const_int 1))]))
     ;; Step 11
     ;; e2 = a - S2 * S2 in f8
     (cond_exec (ne (match_dup 7) (const_int 0))
        (parallel [(set (match_dup 3)
-                       (plus:XF (neg:XF (mult:XF (match_dup 8) (match_dup 8)))
-                                (match_dup 9)))
+                       (minus:XF (match_dup 9)
+                                (mult:XF (match_dup 8) (match_dup 8))))
                   (use (const_int 1))]))
     ;; Step 12
     ;; S3 = S2 + e2 * H2 in f7
     ;; e3 = a - S3 * S3 in f8
     (cond_exec (ne (match_dup 7) (const_int 0))
        (parallel [(set (match_dup 3)
-                       (plus:XF (neg:XF (mult:XF (match_dup 8) (match_dup 8)))
-                                (match_dup 9)))
+                       (minus:XF (match_dup 9)
+                                (mult:XF (match_dup 8) (match_dup 8))))
                   (use (const_int 1))]))
     ;; Step 15
     ;; S = S3 + e3 * H3 in f7
 
 (define_insn "*recip_approx"
   [(set (match_operand:XF 0 "fr_register_operand" "=f")
-       (div:XF (const_int 1)
-               (match_operand:XF 3 "fr_register_operand" "f")))
+       (unspec:XF [(const_int 1)
+                   (match_operand:XF 3 "fr_register_operand" "f")]
+                  UNSPEC_FR_RECIP_APPROX_RES))
    (set (match_operand:BI 1 "register_operand" "=c")
        (unspec:BI [(match_operand:XF 2 "fr_register_operand" "f")
                    (match_dup 3)] UNSPEC_FR_RECIP_APPROX))
 \f
 ;; ::::::::::::::::::::
 ;; ::
-;; :: 32 bit Integer Shifts and Rotates
+;; :: 32-bit Integer Shifts and Rotates
 ;; ::
 ;; ::::::::::::::::::::
 
        (rotate:SI (match_operand:SI 1 "gr_register_operand" "r")
                   (match_operand:SI 2 "shift_32bit_count_operand" "n")))]
   ""
-  "#"
-  "reload_completed"
+  "mux2 %0 = %1, 0xe1"
+  "reload_completed && INTVAL (operands[2]) != 16"
   [(set (match_dup 3)
        (ior:DI (zero_extend:DI (match_dup 1))
                (ashift:DI (zero_extend:DI (match_dup 1)) (const_int 32))))
 {
   operands[3] = gen_rtx_REG (DImode, REGNO (operands[0]));
   operands[2] = GEN_INT (32 - INTVAL (operands[2]));
-})
+}
+  [(set_attr "itanium_class" "mmshf")])
 \f
 ;; ::::::::::::::::::::
 ;; ::
-;; :: 64 bit Integer Shifts and Rotates
+;; :: 64-bit Integer Shifts and Rotates
 ;; ::
 ;; ::::::::::::::::::::
 
                          (match_operand:DI 3 "nonmemory_operand" "r"))
                 (match_operand:DI 4 "nonmemory_operand" "rI")))]
   "reload_in_progress"
-  "* abort ();"
+  "* gcc_unreachable ();"
   "reload_completed"
   [(set (match_dup 0) (plus:DI (mult:DI (match_dup 1) (match_dup 2))
                               (match_dup 3)))
 \f
 ;; ::::::::::::::::::::
 ;; ::
-;; :: 32 bit Integer Logical operations
+;; :: 128-bit Integer Shifts and Rotates
+;; ::
+;; ::::::::::::::::::::
+
+(define_expand "ashlti3"
+  [(set (match_operand:TI 0 "gr_register_operand" "")
+       (ashift:TI (match_operand:TI 1 "gr_register_operand" "")
+                  (match_operand:DI 2 "nonmemory_operand" "")))]
+  ""
+{
+  if (!dshift_count_operand (operands[2], DImode))
+    FAIL;
+})
+
+(define_insn_and_split "*ashlti3_internal"
+  [(set (match_operand:TI 0 "gr_register_operand" "=&r")
+       (ashift:TI (match_operand:TI 1 "gr_register_operand" "r")
+                  (match_operand:DI 2 "dshift_count_operand" "n")))]
+  ""
+  "#"
+  "reload_completed"
+  [(const_int 0)]
+{
+  HOST_WIDE_INT shift = INTVAL (operands[2]);
+  rtx rl = gen_lowpart (DImode, operands[0]);
+  rtx rh = gen_highpart (DImode, operands[0]);
+  rtx lo = gen_lowpart (DImode, operands[1]);
+  rtx shiftlo = GEN_INT (shift & 63);
+
+  if (shift & 64)
+    {
+      emit_move_insn (rl, const0_rtx);
+      if (shift & 63)
+       emit_insn (gen_ashldi3 (rh, lo, shiftlo));
+      else
+       emit_move_insn (rh, lo);
+    }
+  else
+    {
+      rtx hi = gen_highpart (DImode, operands[1]);
+
+      emit_insn (gen_shrp (rh, hi, lo, GEN_INT (-shift & 63)));
+      emit_insn (gen_ashldi3 (rl, lo, shiftlo));
+    }
+  DONE;
+})
+
+(define_expand "ashrti3"
+  [(set (match_operand:TI 0 "gr_register_operand" "")
+       (ashiftrt:TI (match_operand:TI 1 "gr_register_operand" "")
+                    (match_operand:DI 2 "nonmemory_operand" "")))]
+  ""
+{
+  if (!dshift_count_operand (operands[2], DImode))
+    FAIL;
+})
+
+(define_insn_and_split "*ashrti3_internal"
+  [(set (match_operand:TI 0 "gr_register_operand" "=&r")
+       (ashiftrt:TI (match_operand:TI 1 "gr_register_operand" "r")
+                    (match_operand:DI 2 "dshift_count_operand" "n")))]
+  ""
+  "#"
+  "reload_completed"
+  [(const_int 0)]
+{
+  HOST_WIDE_INT shift = INTVAL (operands[2]);
+  rtx rl = gen_lowpart (DImode, operands[0]);
+  rtx rh = gen_highpart (DImode, operands[0]);
+  rtx hi = gen_highpart (DImode, operands[1]);
+  rtx shiftlo = GEN_INT (shift & 63);
+
+  if (shift & 64)
+    {
+      if (shift & 63)
+       emit_insn (gen_ashrdi3 (rl, hi, shiftlo));
+      else
+       emit_move_insn (rl, hi);
+      emit_insn (gen_ashrdi3 (rh, hi, GEN_INT (63)));
+    }
+  else
+    {
+      rtx lo = gen_lowpart (DImode, operands[1]);
+
+      emit_insn (gen_shrp (rl, hi, lo, shiftlo));
+      emit_insn (gen_ashrdi3 (rh, hi, shiftlo));
+    }
+  DONE;
+})
+
+(define_expand "lshrti3"
+  [(set (match_operand:TI 0 "gr_register_operand" "")
+        (lshiftrt:TI (match_operand:TI 1 "gr_register_operand" "")
+                     (match_operand:DI 2 "nonmemory_operand" "")))]
+  ""
+{ 
+  if (!dshift_count_operand (operands[2], DImode))
+    FAIL;
+}) 
+
+(define_insn_and_split "*lshrti3_internal"
+  [(set (match_operand:TI 0 "gr_register_operand" "=&r")
+       (lshiftrt:TI (match_operand:TI 1 "gr_register_operand" "r")
+                    (match_operand:DI 2 "dshift_count_operand" "n")))]
+  ""
+  "#"
+  "reload_completed"
+  [(const_int 0)]
+{
+  HOST_WIDE_INT shift = INTVAL (operands[2]);
+  rtx rl = gen_lowpart (DImode, operands[0]);
+  rtx rh = gen_highpart (DImode, operands[0]);
+  rtx hi = gen_highpart (DImode, operands[1]);
+  rtx shiftlo = GEN_INT (shift & 63);
+
+  if (shift & 64)
+    {
+      if (shift & 63)
+       emit_insn (gen_lshrdi3 (rl, hi, shiftlo));
+      else
+       emit_move_insn (rl, hi);
+      emit_move_insn (rh, const0_rtx);
+    }
+  else
+    {
+      rtx lo = gen_lowpart (DImode, operands[1]);
+
+      emit_insn (gen_shrp (rl, hi, lo, shiftlo));
+      emit_insn (gen_lshrdi3 (rh, hi, shiftlo));
+    }
+  DONE;
+})
+
+(define_expand "rotlti3"
+  [(set (match_operand:TI 0 "gr_register_operand" "")
+       (rotate:TI (match_operand:TI 1 "gr_register_operand" "")
+                  (match_operand:DI 2 "nonmemory_operand" "")))]
+  ""
+{
+  if (! dshift_count_operand (operands[2], DImode))
+    FAIL;
+})
+
+(define_insn_and_split "*rotlti3_internal"
+  [(set (match_operand:TI 0 "gr_register_operand" "=&r")
+       (rotate:TI (match_operand:TI 1 "gr_register_operand" "r")
+                  (match_operand:DI 2 "dshift_count_operand" "n")))]
+  ""
+  "#"
+  "reload_completed"
+  [(const_int 0)]
+{
+  HOST_WIDE_INT count = INTVAL (operands[2]);
+  rtx rl = gen_lowpart (DImode, operands[0]);
+  rtx rh = gen_highpart (DImode, operands[0]);
+  rtx lo = gen_lowpart (DImode, operands[1]);
+  rtx hi = gen_highpart (DImode, operands[1]);
+  rtx countlo = GEN_INT (-count & 63);
+
+  if (count & 64)
+    {
+      if (count & 63)
+       {
+         emit_insn (gen_shrp (rl, hi, lo, countlo));
+         emit_insn (gen_shrp (rh, lo, hi, countlo));
+       }
+      else
+       {
+         emit_move_insn (rl, hi);
+         emit_move_insn (rh, lo);
+       }
+    }
+  else
+    {
+      emit_insn (gen_shrp (rl, lo, hi, countlo));
+      emit_insn (gen_shrp (rh, hi, lo, countlo));
+    }
+  DONE;
+}
+  [(set_attr "itanium_class" "unknown")])
+
+(define_insn "shrp"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (unspec:DI [(match_operand:DI 1 "gr_register_operand" "r")
+                   (match_operand:DI 2 "gr_register_operand" "r")
+                   (match_operand:DI 3 "shift_count_operand" "M")]
+                  UNSPEC_SHRP))]
+  ""
+  "shrp %0 = %1, %2, %3"
+  [(set_attr "itanium_class" "ishf")])
+\f
+;; ::::::::::::::::::::
+;; ::
+;; :: 32-bit Integer Logical operations
 ;; ::
 ;; ::::::::::::::::::::
 
 \f
 ;; ::::::::::::::::::::
 ;; ::
-;; :: 64 bit Integer Logical operations
+;; :: 64-bit Integer Logical operations
 ;; ::
 ;; ::::::::::::::::::::
 
   [(set (match_operand:BI 0 "register_operand" "=c")
        (eq:BI (zero_extract:DI (match_operand:DI 1 "gr_register_operand" "r")
                                (const_int 1)
-                               (match_operand:DI 2 "immediate_operand" "n"))
+                               (match_operand:DI 2 "shift_count_operand" "M"))
               (const_int 0)))]
   ""
   "tbit.z %0, %I0 = %1, %2"
   [(set (match_operand:BI 0 "register_operand" "=c")
        (ne:BI (zero_extract:DI (match_operand:DI 1 "gr_register_operand" "r")
                                (const_int 1)
-                               (match_operand:DI 2 "immediate_operand" "n"))
+                               (match_operand:DI 2 "shift_count_operand" "M"))
               (const_int 0)))]
   ""
   "tbit.nz %0, %I0 = %1, %2"
           "rim,rim,rim, rim, *f, *b,*d*e,*f,*b,*d*e,rO,*f,rOQ,rO,  rK")))]
   "ia64_move_ok (operands[0], operands[2])
    && ia64_move_ok (operands[0], operands[3])"
-  { abort (); }
+  { gcc_unreachable (); }
   [(set_attr "predicable" "no")])
 
 (define_split
                    "rim*f,rO,rO,0,0,0,rim*f,rO,rO")))]
   "ia64_move_ok (operands[0], operands[2])
    && ia64_move_ok (operands[0], operands[3])"
-  { abort (); }
+  { gcc_unreachable (); }
   [(set_attr "predicable" "no")])
 
 (define_insn "*abssi2_internal"
   [(set_attr "itanium_class" "br,scall")])
 
 (define_insn "call_value_nogp"
-  [(set (match_operand 0 "" "")
+  [(set (match_operand 0 "" "=X,X")
        (call (mem:DI (match_operand:DI 1 "call_operand" "?b,i"))
              (const_int 0)))
    (clobber (match_operand:DI 2 "register_operand" "=b,b"))]
 })
 
 (define_insn "call_value_gp"
-  [(set (match_operand 0 "" "")
+  [(set (match_operand 0 "" "=X,X")
        (call (mem:DI (match_operand:DI 1 "call_operand" "?r,i"))
              (const_int 1)))
    (clobber (match_operand:DI 2 "register_operand" "=b,b"))
       start_sequence ();
       set = single_set (last);
 
-      if (! rtx_equal_p (SET_DEST (set), op0)
-         || GET_CODE (SET_SRC (set)) != MEM)
-       abort ();
+      gcc_assert (rtx_equal_p (SET_DEST (set), op0)
+                 && GET_CODE (SET_SRC (set)) == MEM);
       addr = XEXP (SET_SRC (set), 0);
-      if (rtx_equal_p (addr, op0))
-       abort ();
+      gcc_assert (!rtx_equal_p (addr, op0));
     }
 
   /* Jump table elements are stored pc-relative.  That is, a displacement
   ""
   ""
   [(set_attr "itanium_class" "ignore")
-   (set_attr "predicable" "no")])
+   (set_attr "predicable" "no")
+   (set_attr "empty" "yes")])
 
 ;; Allocate a new register frame.
 
   ""
   "alloc %0 = ar.pfs, %1, %2, %3, %4"
   [(set_attr "itanium_class" "syst_m0")
-   (set_attr "predicable" "no")])
+   (set_attr "predicable" "no")
+   (set_attr "first_insn" "yes")])
 
 ;; Modifies ar.unat
 (define_expand "gr_spill"
   "operands[3] = gen_rtx_REG (DImode, AR_UNAT_REGNUM);")
 
 (define_insn "gr_spill_internal"
-  [(set (match_operand:DI 0 "memory_operand" "=m")
+  [(set (match_operand:DI 0 "destination_operand" "=m")
        (unspec:DI [(match_operand:DI 1 "register_operand" "r")
                    (match_operand:DI 2 "const_int_operand" "")]
                   UNSPEC_GR_SPILL))
   [(set_attr "itanium_class" "ld")])
 
 (define_insn "fr_spill"
-  [(set (match_operand:XF 0 "memory_operand" "=m")
+  [(set (match_operand:XF 0 "destination_operand" "=m")
        (unspec:XF [(match_operand:XF 1 "register_operand" "f")]
                   UNSPEC_FR_SPILL))]
   ""
   [(const_int 5)]
   ""
   ""
-  [(set_attr "itanium_class" "nop_x")])
+  [(set_attr "itanium_class" "nop_x")
+   (set_attr "empty" "yes")])
 
 ;; The following insn will be never generated.  It is used only by
 ;; insn scheduler to change state before advancing cycle.
   ""
   ";;"
   [(set_attr "itanium_class" "stop_bit")
-   (set_attr "predicable" "no")])
+   (set_attr "predicable" "no")
+   (set_attr "empty" "yes")])
 
 (define_expand "trap"
   [(trap_if (const_int 1) (const_int 0))]
   [(trap_if (const_int 1) (match_operand 0 "const_int_operand" ""))]
   ""
   "break %0"
-  [(set_attr "itanium_class" "chk_s")])
+  [(set_attr "itanium_class" "chk_s_i")])
 
 (define_expand "conditional_trap"
   [(trap_if (match_operand 0 "" "") (match_operand 1 "" ""))]
            (match_operand 2 "const_int_operand" ""))]
   ""
   "(%J0) break %2"
-  [(set_attr "itanium_class" "chk_s")
+  [(set_attr "itanium_class" "chk_s_i")
    (set_attr "predicable" "no")])
 
 (define_insn "break_f"
   int i = (INTVAL (operands[1]));
   int j = (INTVAL (operands[2]));
 
-  if (i != 0 && i != 1)
-    abort ();
-  if (j < 0 || j > 3)
-    abort ();
+  gcc_assert (i == 0 || i == 1);
+  gcc_assert (j >= 0 && j <= 3);
   return alt[i][j];
 }
   [(set_attr "itanium_class" "lfetch")])
   DONE;
 })
 
+(define_insn_and_split "nonlocal_goto_receiver"
+  [(unspec_volatile [(const_int 0)] UNSPECV_GOTO_RECEIVER)]
+  ""
+  "#"
+  "reload_completed"
+  [(const_int 0)]
+{
+  ia64_reload_gp ();
+  DONE;
+})
+
 (define_insn_and_split "builtin_setjmp_receiver"
   [(unspec_volatile [(match_operand:DI 0 "" "")] UNSPECV_SETJMP_RECEIVER)]
   ""
       emit_move_insn (sp, operands[2]);
       operands[2] = sp;
     }
-  emit_insn (gen_rtx_USE (VOIDmode, sp));
-  emit_insn (gen_rtx_USE (VOIDmode, bsp));
+  emit_use (sp);
+  emit_use (bsp);
 
   cfun->machine->ia64_eh_epilogue_sp = sp;
   cfun->machine->ia64_eh_epilogue_bsp = bsp;
 })
 
 \f
-;;; Intrinsics support.
-
-(define_expand "mf"
-  [(set (mem:BLK (match_dup 0))
-       (unspec:BLK [(mem:BLK (match_dup 0))] UNSPEC_MF))]
-  ""
-{
-  operands[0] = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (DImode));
-  MEM_VOLATILE_P (operands[0]) = 1;
-})
-
-(define_insn "*mf_internal"
-  [(set (match_operand:BLK 0 "" "")
-       (unspec:BLK [(match_operand:BLK 1 "" "")] UNSPEC_MF))]
-  ""
-  "mf"
-  [(set_attr "itanium_class" "syst_m")])
-
-(define_insn "fetchadd_acq_si"
-  [(set (match_operand:SI 0 "gr_register_operand" "=r")
-       (match_dup 1))
-   (set (match_operand:SI 1 "not_postinc_memory_operand" "+S")
-       (unspec:SI [(match_dup 1)
-                   (match_operand:SI 2 "fetchadd_operand" "n")]
-                  UNSPEC_FETCHADD_ACQ))]
-  ""
-  "fetchadd4.acq %0 = %1, %2"
-  [(set_attr "itanium_class" "sem")])
-
-(define_insn "fetchadd_acq_di"
-  [(set (match_operand:DI 0 "gr_register_operand" "=r")
-       (match_dup 1))
-   (set (match_operand:DI 1 "not_postinc_memory_operand" "+S")
-       (unspec:DI [(match_dup 1)
-                   (match_operand:DI 2 "fetchadd_operand" "n")]
-                  UNSPEC_FETCHADD_ACQ))]
-  ""
-  "fetchadd8.acq %0 = %1, %2"
-  [(set_attr "itanium_class" "sem")])
-
-(define_insn "cmpxchg_acq_si"
-  [(set (match_operand:SI 0 "gr_register_operand" "=r")
-       (match_dup 1))
-   (set (match_operand:SI 1 "not_postinc_memory_operand" "+S")
-        (unspec:SI [(match_dup 1)
-                    (match_operand:SI 2 "gr_register_operand" "r")
-                   (match_operand:DI 3 "ar_ccv_reg_operand" "")]
-                  UNSPEC_CMPXCHG_ACQ))]
-  ""
-  "cmpxchg4.acq %0 = %1, %2, %3"
-  [(set_attr "itanium_class" "sem")])
-
-(define_insn "cmpxchg_acq_di"
-  [(set (match_operand:DI 0 "gr_register_operand" "=r")
-       (match_dup 1))
-   (set (match_operand:DI 1 "not_postinc_memory_operand" "+S")
-        (unspec:DI [(match_dup 1)
-                    (match_operand:DI 2 "gr_register_operand" "r")
-                   (match_operand:DI 3 "ar_ccv_reg_operand" "")]
-                  UNSPEC_CMPXCHG_ACQ))]
-  ""
-  "cmpxchg8.acq %0 = %1, %2, %3"
-  [(set_attr "itanium_class" "sem")])
-
-(define_insn "xchgsi"
-  [(set (match_operand:SI 0 "gr_register_operand" "=r")
-        (match_operand:SI 1 "not_postinc_memory_operand" "+S"))
-   (set (match_dup 1)
-        (match_operand:SI 2 "gr_register_operand" "r"))]
-  ""
-  "xchg4 %0 = %1, %2"
-  [(set_attr "itanium_class" "sem")])
-
-(define_insn "xchgdi"
-  [(set (match_operand:DI 0 "gr_register_operand" "=r")
-        (match_operand:DI 1 "not_postinc_memory_operand" "+S"))
-   (set (match_dup 1)
-        (match_operand:DI 2 "gr_register_operand" "r"))]
-  ""
-  "xchg8 %0 = %1, %2"
-  [(set_attr "itanium_class" "sem")])
-\f
 ;; Predication.
 
 (define_cond_exec
   [(set_attr "itanium_class" "ignore")
    (set_attr "predicable" "no")])
 
-;; UNSPEC instruction definition to "swizzle" 32 bit pointer into 64 bit
+;; UNSPEC instruction definition to "swizzle" 32-bit pointer into 64-bit
 ;; pointer.  This is used by the HP-UX 32 bit mode.
 
 (define_insn "ptr_extend"
   "addp4_optimize_ok (operands[1], operands[2])"
   "addp4 %0 = %1, %2"
   [(set_attr "itanium_class" "ialu")])
+
+;;
+;; Get instruction pointer
+
+(define_insn "ip_value"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+        (pc))]
+ ""
+ "mov %0 = ip"
+  [(set_attr "itanium_class" "frbr")])
+
+;; Vector operations
+(include "vect.md")
+;; Atomic operations
+(include "sync.md")
+;; New division operations
+(include "div.md")