OSDN Git Service

2005-12-05 Jan Beulich <jbeulich@novell.com>
[pf3gnuchains/gcc-fork.git] / gcc / config / ia64 / ia64.md
index 0734936..afe3e53 100644 (file)
@@ -1,24 +1,25 @@
 ;; IA-64 Machine description template
-;; Copyright (C) 1999, 2000 Free Software Foundation, Inc.
+;; Copyright (C) 1999, 2000, 2001, 2002, 2003, 2004, 2005
+;; Free Software Foundation, Inc.
 ;; Contributed by James E. Wilson <wilson@cygnus.com> and
 ;;               David Mosberger <davidm@hpl.hp.com>.
 
-;; This file is part of GNU CC.
+;; This file is part of GCC.
 
-;; GNU CC is free software; you can redistribute it and/or modify
+;; GCC is free software; you can redistribute it and/or modify
 ;; it under the terms of the GNU General Public License as published by
 ;; the Free Software Foundation; either version 2, or (at your option)
 ;; any later version.
 
-;; GNU CC is distributed in the hope that it will be useful,
+;; GCC is distributed in the hope that it will be useful,
 ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 ;; GNU General Public License for more details.
 
 ;; You should have received a copy of the GNU General Public License
-;; along with GNU CC; see the file COPYING.  If not, write to
-;; the Free Software Foundation, 59 Temple Place - Suite 330,
-;; Boston, MA 02111-1307, USA.
+;; along with GCC; see the file COPYING.  If not, write to
+;; the Free Software Foundation, 51 Franklin Street, Fifth Floor,
+;; Boston, MA 02110-1301, USA.
 
 ;;- See file "rtl.def" for documentation on define_insn, match_*, et. al.
 
 
 ;; ??? Add function unit scheduling info for Itanium (TM) processor.
 
-;; ??? The explicit stop in the flushrs pattern is not ideal.  It
-;; would be better if rtx_needs_barrier took care of this, but this is
-;; something that can be fixed later.
-
-;; Unspec usage:
-;;
-;; unspec:
-;;     1       gr_spill
-;;     2       gr_restore
-;;     3       fr_spill
-;;     4       fr_restore
-;;     8       popcnt
-;;     12      mf
-;;     13      cmpxchg_acq
-;;     19      fetchadd_acq
-;;     20      bsp_value
-;;     21      flushrs
-;;
-;; unspec_volatile:
-;;     0       alloc
-;;     1       blockage
-;;     2       insn_group_barrier
-;;     5       set_bsp
-;;     7       pred.rel.mutex
+;; ??? Need a better way to describe alternate fp status registers.
+
+(define_constants
+  [; Relocations
+   (UNSPEC_LTOFF_DTPMOD                0)
+   (UNSPEC_LTOFF_DTPREL                1)
+   (UNSPEC_DTPREL              2)
+   (UNSPEC_LTOFF_TPREL         3)
+   (UNSPEC_TPREL               4)
+   (UNSPEC_DTPMOD              5)
+
+   (UNSPEC_LD_BASE             9)
+   (UNSPEC_GR_SPILL            10)
+   (UNSPEC_GR_RESTORE          11)
+   (UNSPEC_FR_SPILL            12)
+   (UNSPEC_FR_RESTORE          13)
+   (UNSPEC_FR_RECIP_APPROX     14)
+   (UNSPEC_PRED_REL_MUTEX      15)
+   (UNSPEC_GETF_EXP            16)
+   (UNSPEC_PIC_CALL            17)
+   (UNSPEC_MF                  18)
+   (UNSPEC_CMPXCHG_ACQ         19)
+   (UNSPEC_FETCHADD_ACQ                20)
+   (UNSPEC_BSP_VALUE           21)
+   (UNSPEC_FLUSHRS             22)
+   (UNSPEC_BUNDLE_SELECTOR     23)
+   (UNSPEC_ADDP4               24)
+   (UNSPEC_PROLOGUE_USE                25)
+   (UNSPEC_RET_ADDR            26)
+   (UNSPEC_SETF_EXP             27)
+   (UNSPEC_FR_SQRT_RECIP_APPROX 28)
+   (UNSPEC_SHRP                        29)
+   (UNSPEC_COPYSIGN            30)
+   (UNSPEC_VECT_EXTR           31)
+  ])
+
+(define_constants
+  [(UNSPECV_ALLOC              0)
+   (UNSPECV_BLOCKAGE           1)
+   (UNSPECV_INSN_GROUP_BARRIER 2)
+   (UNSPECV_BREAK              3)
+   (UNSPECV_SET_BSP            4)
+   (UNSPECV_PSAC_ALL           5)      ; pred.safe_across_calls
+   (UNSPECV_PSAC_NORMAL                6)
+   (UNSPECV_SETJMP_RECEIVER    7)
+  ])
+
+(include "predicates.md")
 \f
 ;; ::::::::::::::::::::
 ;; ::
 ;; ::
 ;; ::::::::::::::::::::
 
+;; Processor type.  This attribute must exactly match the processor_type
+;; enumeration in ia64.h.
+(define_attr "cpu" "itanium,itanium2" (const (symbol_ref "ia64_tune")))
+
 ;; Instruction type.  This primarily determines how instructions can be
 ;; packed in bundles, and secondarily affects scheduling to function units.
 
 ;; multiple instructions, patterns which emit 0 instructions, and patterns
 ;; which emit instruction that can go in any slot (e.g. nop).
 
-(define_attr "type" "unknown,A,I,M,F,B,L,S" (const_string "unknown"))
+(define_attr "itanium_class" "unknown,ignore,stop_bit,br,fcmp,fcvtfx,fld,
+       fldp,fmac,fmisc,frar_i,frar_m,frbr,frfr,frpr,ialu,icmp,ilog,ishf,
+       ld,chk_s,long_i,mmalua,mmmul,mmshf,mmshfi,rse_m,scall,sem,stf,
+        st,syst_m0, syst_m,tbit,toar_i,toar_m,tobr,tofr,topr,xmpy,xtd,nop,
+        nop_b,nop_f,nop_i,nop_m,nop_x,lfetch,pre_cycle"
+  (const_string "unknown"))
+
+;; chk_s has an I and an M form; use type A for convenience.
+(define_attr "type" "unknown,A,I,M,F,B,L,X,S"
+  (cond [(eq_attr "itanium_class" "ld,st,fld,fldp,stf,sem,nop_m") (const_string "M")
+        (eq_attr "itanium_class" "rse_m,syst_m,syst_m0") (const_string "M")
+        (eq_attr "itanium_class" "frar_m,toar_m,frfr,tofr") (const_string "M")
+        (eq_attr "itanium_class" "lfetch") (const_string "M")
+        (eq_attr "itanium_class" "chk_s,ialu,icmp,ilog,mmalua")
+          (const_string "A")
+        (eq_attr "itanium_class" "fmisc,fmac,fcmp,xmpy") (const_string "F")
+        (eq_attr "itanium_class" "fcvtfx,nop_f") (const_string "F")
+        (eq_attr "itanium_class" "frar_i,toar_i,frbr,tobr") (const_string "I")
+        (eq_attr "itanium_class" "frpr,topr,ishf,xtd,tbit") (const_string "I")
+        (eq_attr "itanium_class" "mmmul,mmshf,mmshfi,nop_i") (const_string "I")
+        (eq_attr "itanium_class" "br,scall,nop_b") (const_string "B")
+        (eq_attr "itanium_class" "stop_bit") (const_string "S")
+        (eq_attr "itanium_class" "nop_x") (const_string "X")
+        (eq_attr "itanium_class" "long_i") (const_string "L")]
+       (const_string "unknown")))
+
+(define_attr "itanium_requires_unit0" "no,yes"
+  (cond [(eq_attr "itanium_class" "syst_m0,sem,frfr,rse_m") (const_string "yes")
+        (eq_attr "itanium_class" "toar_m,frar_m") (const_string "yes")
+        (eq_attr "itanium_class" "frbr,tobr,mmmul") (const_string "yes")
+        (eq_attr "itanium_class" "tbit,ishf,topr,frpr") (const_string "yes")
+        (eq_attr "itanium_class" "toar_i,frar_i") (const_string "yes")
+        (eq_attr "itanium_class" "fmisc,fcmp") (const_string "yes")]
+       (const_string "no")))
 
 ;; Predication.  True iff this instruction can be predicated.
 
 (define_attr "predicable" "no,yes" (const_string "yes"))
 
-\f
-;; ::::::::::::::::::::
-;; ::
-;; :: Function Units
-;; ::
-;; ::::::::::::::::::::
-
-;; Each usage of a function units by a class of insns is specified with a
-;; `define_function_unit' expression, which looks like this:
-;; (define_function_unit NAME MULTIPLICITY SIMULTANEITY TEST READY-DELAY
-;;   ISSUE-DELAY [CONFLICT-LIST])
-
-;; This default scheduling info seeks to pack instructions into bundles
-;; efficiently to reduce code size, so we just list how many of each
-;; instruction type can go in a bundle.  ISSUE_RATE is set to 3.
-
-;; ??? Add scheduler ready-list hook (MD_SCHED_REORDER) that orders
-;; instructions, so that the next instruction can fill the next bundle slot.
-;; This really needs to know where the stop bits are though.
+;; Empty.  True iff this insn does not generate any code.
 
-;; ??? Use MD_SCHED_REORDER to put alloc first instead of using an unspec
-;; volatile.  Use ADJUST_PRIORITY to set the priority of alloc very high to
-;; make it schedule first.
+(define_attr "empty" "no,yes" (const_string "no"))
 
-;; ??? Modify the md_reorg code that emits stop bits so that instead of putting
-;; them in the last possible place, we put them in places where bundles allow
-;; them.  This should reduce code size, but may decrease performance if we end
-;; up with more stop bits than the minimum we need.
+;; True iff this insn must be the first insn of an instruction group.
+;; This is true for the alloc instruction, and will also be true of others
+;; when we have full intrinsics support.
 
-;; Alu instructions can execute on either the integer or memory function
-;; unit.  We indicate this by defining an alu function unit, and then marking
-;; it as busy everytime we issue a integer or memory type instruction.
-
-(define_function_unit "alu" 3 1 (eq_attr "type" "A,I,M") 1 0)
-
-(define_function_unit "integer" 2 1 (eq_attr "type" "I") 1 0)
-
-(define_function_unit "memory" 3 1 (eq_attr "type" "M") 1 0)
+(define_attr "first_insn" "no,yes" (const_string "no"))
+\f
+;; DFA descriptions of ia64 processors used for insn scheduling and
+;; bundling.
 
-(define_function_unit "floating_point" 1 1 (eq_attr "type" "F") 1 0)
+(automata_option "ndfa")
 
-(define_function_unit "branch" 3 1 (eq_attr "type" "B") 1 0)
+;; Uncomment the following line to output automata for debugging.
+;; (automata_option "v")
 
-;; ??? This isn't quite right, because we can only fit two insns in a bundle
-;; when using an L type instruction.  That isn't modeled currently.
+(automata_option "w")
 
-(define_function_unit "long_immediate" 1 1 (eq_attr "type" "L") 1 0)
+(include "itanium1.md")
+(include "itanium2.md")
 
 \f
 ;; ::::::::::::::::::::
 ;; ::
 ;; ::::::::::::::::::::
 
-(define_expand "movqi"
-  [(set (match_operand:QI 0 "general_operand" "")
-       (match_operand:QI 1 "general_operand" ""))]
-  ""
-  "
-{
-  if (! reload_in_progress && ! reload_completed
-      && ! ia64_move_ok (operands[0], operands[1]))
-    operands[1] = force_reg (QImode, operands[1]);
-}")
-
-;; Errata 72 implies that we cannot use predicated loads and stores
-;; on affected systems.  Reuse TARGET_A_STEP for convenience.
+;; Set of a single predicate register.  This is only used to implement
+;; pr-to-pr move and complement.
 
-;; ??? It would be convenient at this point if the cond_exec pattern
-;; expander understood non-constant conditions on attributes.  Failing
-;; that we have to replicate patterns.
-
-(define_insn "*movqicc_astep"
-  [(cond_exec
-     (match_operator 2 "predicate_operator"
-       [(match_operand:CC 3 "register_operand" "c,c,c,c,c")
-        (const_int 0)])
-     (set (match_operand:QI 0 "register_operand"  "=r,r, r,*f,*f")
-         (match_operand:QI 1 "nonmemory_operand" "rO,J,*f,rO,*f")))]
-  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
+(define_insn "*movcci"
+  [(set (match_operand:CCI 0 "register_operand" "=c,c,c")
+       (match_operand:CCI 1 "nonmemory_operand" "O,n,c"))]
+  ""
   "@
-   (%J2) mov %0 = %r1
-   (%J2) addl %0 = %1, r0
-   (%J2) getf.sig %0 = %1
-   (%J2) setf.sig %0 = %r1
-   (%J2) mov %0 = %1"
-  [(set_attr "type" "A,A,M,M,F")
+   cmp.ne %0, p0 = r0, r0
+   cmp.eq %0, p0 = r0, r0
+   (%1) cmp.eq.unc %0, p0 = r0, r0"
+  [(set_attr "itanium_class" "icmp")
    (set_attr "predicable" "no")])
 
-(define_insn "*movqi_internal_astep"
-  [(set (match_operand:QI 0 "destination_operand" "=r,r,r, m, r,*f,*f")
-       (match_operand:QI 1 "move_operand"        "rO,J,m,rO,*f,rO,*f"))]
-  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
+(define_insn "movbi"
+  [(set (match_operand:BI 0 "destination_operand" "=c,c,?c,?*r, c,*r,*r,*m,*r")
+       (match_operand:BI 1 "move_operand"        " O,n, c,  c,*r, n,*m,*r,*r"))]
+  ""
   "@
-   mov %0 = %r1
-   addl %0 = %1, r0
+   cmp.ne %0, %I0 = r0, r0
+   cmp.eq %0, %I0 = r0, r0
+   #
+   #
+   tbit.nz %0, %I0 = %1, 0
+   adds %0 = %1, r0
    ld1%O1 %0 = %1%P1
-   st1%Q0 %0 = %r1%P0
-   getf.sig %0 = %1
-   setf.sig %0 = %r1
+   st1%Q0 %0 = %1%P0
    mov %0 = %1"
-  [(set_attr "type" "A,A,M,M,M,M,F")
-   (set_attr "predicable" "no")])
+  [(set_attr "itanium_class" "icmp,icmp,unknown,unknown,tbit,ialu,ld,st,ialu")])
+
+(define_split
+  [(set (match_operand:BI 0 "register_operand" "")
+       (match_operand:BI 1 "register_operand" ""))]
+  "reload_completed
+   && GET_CODE (operands[0]) == REG && GR_REGNO_P (REGNO (operands[0]))
+   && GET_CODE (operands[1]) == REG && PR_REGNO_P (REGNO (operands[1]))"
+  [(cond_exec (ne (match_dup 1) (const_int 0))
+     (set (match_dup 0) (const_int 1)))
+   (cond_exec (eq (match_dup 1) (const_int 0))
+     (set (match_dup 0) (const_int 0)))]
+  "")
+
+(define_split
+  [(set (match_operand:BI 0 "register_operand" "")
+       (match_operand:BI 1 "register_operand" ""))]
+  "reload_completed
+   && GET_CODE (operands[0]) == REG && PR_REGNO_P (REGNO (operands[0]))
+   && GET_CODE (operands[1]) == REG && PR_REGNO_P (REGNO (operands[1]))"
+  [(set (match_dup 2) (match_dup 4))
+   (set (match_dup 3) (match_dup 5))
+   (set (match_dup 0) (unspec:BI [(match_dup 0)] UNSPEC_PRED_REL_MUTEX))]
+  "operands[2] = gen_rtx_REG (CCImode, REGNO (operands[0]));
+   operands[3] = gen_rtx_REG (CCImode, REGNO (operands[0]) + 1);
+   operands[4] = gen_rtx_REG (CCImode, REGNO (operands[1]));
+   operands[5] = gen_rtx_REG (CCImode, REGNO (operands[1]) + 1);")
+
+(define_expand "movqi"
+  [(set (match_operand:QI 0 "general_operand" "")
+       (match_operand:QI 1 "general_operand" ""))]
+  ""
+{
+  rtx op1 = ia64_expand_move (operands[0], operands[1]);
+  if (!op1)
+    DONE;
+  operands[1] = op1;
+})
 
 (define_insn "*movqi_internal"
   [(set (match_operand:QI 0 "destination_operand" "=r,r,r, m, r,*f,*f")
        (match_operand:QI 1 "move_operand"        "rO,J,m,rO,*f,rO,*f"))]
-  "! TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
+  "ia64_move_ok (operands[0], operands[1])"
   "@
    mov %0 = %r1
    addl %0 = %1, r0
    getf.sig %0 = %1
    setf.sig %0 = %r1
    mov %0 = %1"
-  [(set_attr "type" "A,A,M,M,M,M,F")])
+  [(set_attr "itanium_class" "ialu,ialu,ld,st,frfr,tofr,fmisc")])
 
 (define_expand "movhi"
   [(set (match_operand:HI 0 "general_operand" "")
        (match_operand:HI 1 "general_operand" ""))]
   ""
-  "
 {
-  if (! reload_in_progress && ! reload_completed
-      && ! ia64_move_ok (operands[0], operands[1]))
-    operands[1] = force_reg (HImode, operands[1]);
-}")
-
-;; Errata 72 workaround.
-(define_insn "*movhicc_astep"
-  [(cond_exec
-     (match_operator 2 "predicate_operator"
-       [(match_operand:CC 3 "register_operand" "c,c,c,c,c")
-        (const_int 0)])
-     (set (match_operand:HI 0 "register_operand"  "=r,r, r,*f,*f")
-         (match_operand:HI 1 "nonmemory_operand" "rO,J,*f,rO,*f")))]
-  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
-  "@
-   (%J2) mov %0 = %r1
-   (%J2) addl %0 = %1, r0
-   (%J2) getf.sig %0 = %1
-   (%J2) setf.sig %0 = %r1
-   (%J2) mov %0 = %1"
-  [(set_attr "type" "A,A,M,M,F")
-   (set_attr "predicable" "no")])
-
-(define_insn "*movhi_internal_astep"
-  [(set (match_operand:HI 0 "destination_operand" "=r,r,r, m, r,*f,*f")
-       (match_operand:HI 1 "move_operand"        "rO,J,m,rO,*f,rO,*f"))]
-  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
-  "@
-   mov %0 = %r1
-   addl %0 = %1, r0
-   ld2%O1 %0 = %1%P1
-   st2%Q0 %0 = %r1%P0
-   getf.sig %0 = %1
-   setf.sig %0 = %r1
-   mov %0 = %1"
-  [(set_attr "type" "A,A,M,M,M,M,F")
-   (set_attr "predicable" "no")])
+  rtx op1 = ia64_expand_move (operands[0], operands[1]);
+  if (!op1)
+    DONE;
+  operands[1] = op1;
+})
 
 (define_insn "*movhi_internal"
   [(set (match_operand:HI 0 "destination_operand" "=r,r,r, m, r,*f,*f")
        (match_operand:HI 1 "move_operand"        "rO,J,m,rO,*f,rO,*f"))]
-  "! TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
+  "ia64_move_ok (operands[0], operands[1])"
   "@
    mov %0 = %r1
    addl %0 = %1, r0
    getf.sig %0 = %1
    setf.sig %0 = %r1
    mov %0 = %1"
-  [(set_attr "type" "A,A,M,M,M,M,F")])
+  [(set_attr "itanium_class" "ialu,ialu,ld,st,frfr,tofr,fmisc")])
 
 (define_expand "movsi"
   [(set (match_operand:SI 0 "general_operand" "")
        (match_operand:SI 1 "general_operand" ""))]
   ""
-  "
 {
-  if (! reload_in_progress && ! reload_completed
-      && ! ia64_move_ok (operands[0], operands[1]))
-    operands[1] = force_reg (SImode, operands[1]);
-}")
-
-;; Errata 72 workaround.
-(define_insn "*movsicc_astep"
-  [(cond_exec
-     (match_operator 2 "predicate_operator"
-       [(match_operand:CC 3 "register_operand" "c,c,c,c,c,c,c,c")
-        (const_int 0)])
-     (set (match_operand:SI 0 "register_operand"  "=r,r,r, r,*f,*f, r,*d")
-         (match_operand:SI 1 "nonmemory_operand" "rO,J,i,*f,rO,*f,*d,rO")))]
-  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
-  "@
-   (%J2) mov %0 = %r1
-   (%J2) addl %0 = %1, r0
-   (%J2) movl %0 = %1
-   (%J2) getf.sig %0 = %1
-   (%J2) setf.sig %0 = %r1
-   (%J2) mov %0 = %1
-   (%J2) mov %0 = %1
-   (%J2) mov %0 = %r1"
-  [(set_attr "type" "A,A,L,M,M,F,M,M")
-   (set_attr "predicable" "no")])
-
-(define_insn "*movsi_internal_astep"
-  [(set (match_operand:SI 0 "destination_operand" "=r,r,r,r, m, r,*f,*f, r,*d")
-       (match_operand:SI 1 "move_operand"        "rO,J,i,m,rO,*f,rO,*f,*d,rO"))]
-  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
-  "@
-  mov %0 = %r1
-  addl %0 = %1, r0
-  movl %0 = %1
-  ld4%O1 %0 = %1%P1
-  st4%Q0 %0 = %r1%P0
-  getf.sig %0 = %1
-  setf.sig %0 = %r1
-  mov %0 = %1
-  mov %0 = %1
-  mov %0 = %r1"
-  [(set_attr "type" "A,A,L,M,M,M,M,F,M,M")
-   (set_attr "predicable" "no")])
+  rtx op1 = ia64_expand_move (operands[0], operands[1]);
+  if (!op1)
+    DONE;
+  operands[1] = op1;
+})
 
 (define_insn "*movsi_internal"
   [(set (match_operand:SI 0 "destination_operand" "=r,r,r,r, m, r,*f,*f, r,*d")
-       (match_operand:SI 1 "move_operand"        "rO,J,i,m,rO,*f,rO,*f,*d,rO"))]
-  "! TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
+       (match_operand:SI 1 "move_operand"        "rO,J,i,m,rO,*f,rO,*f,*d,rK"))]
+  "ia64_move_ok (operands[0], operands[1])"
   "@
   mov %0 = %r1
   addl %0 = %1, r0
   mov %0 = %1
   mov %0 = %1
   mov %0 = %r1"
-  [(set_attr "type" "A,A,L,M,M,M,M,F,M,M")])
+  ;; frar_m, toar_m ??? why not frar_i and toar_i
+  [(set_attr "itanium_class" "ialu,ialu,long_i,ld,st,frfr,tofr,fmisc,frar_m,toar_m")])
 
 (define_expand "movdi"
   [(set (match_operand:DI 0 "general_operand" "")
        (match_operand:DI 1 "general_operand" ""))]
   ""
-  "
-{
-  if (! reload_in_progress && ! reload_completed
-      && ! ia64_move_ok (operands[0], operands[1]))
-    operands[1] = force_reg (DImode, operands[1]);
-  if (! TARGET_NO_PIC && symbolic_operand (operands[1], DImode))
-    {
-      /* Before optimization starts, delay committing to any particular
-        type of PIC address load.  If this function gets deferred, we
-        may acquire information that changes the value of the
-        sdata_symbolic_operand predicate.  */
-      if (rtx_equal_function_value_matters)
-       emit_insn (gen_movdi_symbolic (operands[0], operands[1]));
-      else
-        ia64_expand_load_address (operands[0], operands[1]);
-      DONE;
-    }
-}")
-
-;; Errata 72 workaround.
-(define_insn ""
-  [(cond_exec
-     (match_operator 2 "predicate_operator"
-       [(match_operand:CC 3 "register_operand" "c,c,c,c,c,c,c,c,c,c")
-        (const_int 0)])
-     (set (match_operand:DI 0 "register_operand"
-                             "=r,r,r, r,*f,*f,   r,*b*e, r,*d")
-         (match_operand:DI 1 "nonmemory_operand"
-                             "rO,J,i,*f,rO,*f,*b*e,  rO,*d,rO")))]
-  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
-  "*
-{
-  static const char * const alt[] = {
-    \"(%J2) mov %0 = %r1\",
-    \"(%J2) addl %0 = %1, r0\",
-    \"(%J2) movl %0 = %1\",
-    \"(%J2) getf.sig %0 = %1\",
-    \"(%J2) setf.sig %0 = %r1\",
-    \"(%J2) mov %0 = %1\",
-    \"(%J2) mov %0 = %1\",
-    \"(%J2) mov %0 = %r1\",
-    \"(%J2) mov %0 = %1\",
-    \"(%J2) mov %0 = %r1\"
-  };
-
-  /* We use 'i' for alternative 2 despite possible PIC problems.
-
-     If we define LEGITIMATE_CONSTANT_P such that symbols are not
-     allowed, then the compiler dumps the data into constant memory
-     instead of letting us read the values from the GOT.  Similarly
-     if we use 'n' instead of 'i'.
-
-     Instead, we allow such insns through reload and then split them
-     afterward (even without optimization).  Therefore, we should
-     never get so far with a symbolic operand.  */
-
-  if (which_alternative == 2 && ! TARGET_NO_PIC
-      && symbolic_operand (operands[1], VOIDmode))
-    abort ();
-
-  return alt[which_alternative];
-}"
-  [(set_attr "type" "A,A,L,M,M,F,I,I,M,M")
-   (set_attr "predicable" "no")])
-
-;; This is used during early compilation to delay the decision on
-;; how to refer to a variable as long as possible.  This is especially
-;; important between initial rtl generation and optimization for
-;; deferred functions, since we may acquire additional information
-;; on the variables used in the meantime.
-
-(define_insn_and_split "movdi_symbolic"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (match_operand:DI 1 "symbolic_operand" "s"))
-   (use (reg:DI 1))]
-  ""
-  "* abort ();"
-  ""
-  [(const_int 0)]
-  "ia64_expand_load_address (operands[0], operands[1]); DONE;")
-
-(define_insn "*movdi_internal_astep"
-  [(set (match_operand:DI 0 "destination_operand"
-                   "=r,r,r,r, m, r,*f,*f,*f, Q,   r,*b*e, r,*d, r,*c")
-       (match_operand:DI 1 "move_operand"
-                   "rO,J,i,m,rO,*f,rO,*f, Q,*f,*b*e,  rO,*d,rO,*c,rO"))]
-  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
-  "*
 {
-  static const char * const alt[] = {
-    \"mov %0 = %r1\",
-    \"addl %0 = %1, r0\",
-    \"movl %0 = %1\",
-    \"ld8%O1 %0 = %1%P1\",
-    \"st8%Q0 %0 = %r1%P0\",
-    \"getf.sig %0 = %1\",
-    \"setf.sig %0 = %r1\",
-    \"mov %0 = %1\",
-    \"ldf8 %0 = %1%P1\",
-    \"stf8 %0 = %1%P0\",
-    \"mov %0 = %1\",
-    \"mov %0 = %r1\",
-    \"mov %0 = %1\",
-    \"mov %0 = %r1\",
-    \"mov %0 = pr\",
-    \"mov pr = %1, -1\"
-  };
-
-  if (which_alternative == 2 && ! TARGET_NO_PIC
-      && symbolic_operand (operands[1], VOIDmode))
-    abort ();
-
-  return alt[which_alternative];
-}"
-  [(set_attr "type" "A,A,L,M,M,M,M,F,M,M,I,I,M,M,I,I")
-   (set_attr "predicable" "no")])
+  rtx op1 = ia64_expand_move (operands[0], operands[1]);
+  if (!op1)
+    DONE;
+  operands[1] = op1;
+})
 
 (define_insn "*movdi_internal"
   [(set (match_operand:DI 0 "destination_operand"
-                   "=r,r,r,r, m, r,*f,*f,*f, Q,   r,*b*e, r,*d, r,*c")
+                   "=r,r,r,r, m, r,*f,*f,*f, Q, r,*b, r,*e, r,*d, r,*c")
        (match_operand:DI 1 "move_operand"
-                   "rO,J,i,m,rO,*f,rO,*f, Q,*f,*b*e,  rO,*d,rO,*c,rO"))]
-  "! TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
-  "*
+                   "rO,JT,i,m,rO,*f,rO,*f, Q,*f,*b,rO,*e,rK,*d,rK,*c,rO"))]
+  "ia64_move_ok (operands[0], operands[1])"
 {
   static const char * const alt[] = {
-    \"%,mov %0 = %r1\",
-    \"%,addl %0 = %1, r0\",
-    \"%,movl %0 = %1\",
-    \"%,ld8%O1 %0 = %1%P1\",
-    \"%,st8%Q0 %0 = %r1%P0\",
-    \"%,getf.sig %0 = %1\",
-    \"%,setf.sig %0 = %r1\",
-    \"%,mov %0 = %1\",
-    \"%,ldf8 %0 = %1%P1\",
-    \"%,stf8 %0 = %1%P0\",
-    \"%,mov %0 = %1\",
-    \"%,mov %0 = %r1\",
-    \"%,mov %0 = %1\",
-    \"%,mov %0 = %r1\",
-    \"mov %0 = pr\",
-    \"mov pr = %1, -1\"
+    "%,mov %0 = %r1",
+    "%,addl %0 = %1, r0",
+    "%,movl %0 = %1",
+    "%,ld8%O1 %0 = %1%P1",
+    "%,st8%Q0 %0 = %r1%P0",
+    "%,getf.sig %0 = %1",
+    "%,setf.sig %0 = %r1",
+    "%,mov %0 = %1",
+    "%,ldf8 %0 = %1%P1",
+    "%,stf8 %0 = %1%P0",
+    "%,mov %0 = %1",
+    "%,mov %0 = %r1",
+    "%,mov %0 = %1",
+    "%,mov %0 = %1",
+    "%,mov %0 = %1",
+    "%,mov %0 = %1",
+    "mov %0 = pr",
+    "mov pr = %1, -1"
   };
 
-  if (which_alternative == 2 && ! TARGET_NO_PIC
-      && symbolic_operand (operands[1], VOIDmode))
-    abort ();
+  gcc_assert (which_alternative != 2 || TARGET_NO_PIC
+              || !symbolic_operand (operands[1], VOIDmode));
 
   return alt[which_alternative];
-}"
-  [(set_attr "type" "A,A,L,M,M,M,M,F,M,M,I,I,M,M,I,I")])
+}
+  [(set_attr "itanium_class" "ialu,ialu,long_i,ld,st,frfr,tofr,fmisc,fld,stf,frbr,tobr,frar_i,toar_i,frar_m,toar_m,frpr,topr")])
 
 (define_split
-  [(set (match_operand:DI 0 "register_operand" "")
-       (match_operand:DI 1 "symbolic_operand" ""))]
-  "reload_completed && ! TARGET_NO_PIC"
+  [(set (match_operand 0 "register_operand" "")
+       (match_operand 1 "symbolic_operand" ""))]
+  "reload_completed"
   [(const_int 0)]
-  "
 {
-  ia64_expand_load_address (operands[0], operands[1]);
-  DONE;
-}")
+  if (ia64_expand_load_address (operands[0], operands[1]))
+    DONE;
+  else
+    FAIL;
+})
 
 (define_expand "load_fptr"
-  [(set (match_dup 2)
-       (plus:DI (reg:DI 1) (match_operand:DI 1 "function_operand" "")))
-   (set (match_operand:DI 0 "register_operand" "") (match_dup 3))]
-  ""
-  "
+  [(set (match_operand:DI 0 "register_operand" "")
+       (plus:DI (match_dup 2) (match_operand 1 "function_operand" "")))
+   (set (match_dup 0) (match_dup 3))]
+  "reload_completed"
 {
-  operands[2] = no_new_pseudos ? operands[0] : gen_reg_rtx (DImode);
-  operands[3] = gen_rtx_MEM (DImode, operands[2]);
-  RTX_UNCHANGING_P (operands[3]) = 1;
-}")
+  operands[2] = pic_offset_table_rtx;
+  operands[3] = gen_const_mem (DImode, operands[0]);
+})
 
 (define_insn "*load_fptr_internal1"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (plus:DI (reg:DI 1) (match_operand:DI 1 "function_operand" "s")))]
-  ""
+       (plus:DI (reg:DI 1) (match_operand 1 "function_operand" "s")))]
+  "reload_completed"
   "addl %0 = @ltoff(@fptr(%1)), gp"
-  [(set_attr "type" "A")])
+  [(set_attr "itanium_class" "ialu")])
 
 (define_insn "load_gprel"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (plus:DI (reg:DI 1) (match_operand:DI 1 "sdata_symbolic_operand" "s")))]
-  ""
+       (plus:DI (reg:DI 1) (match_operand 1 "sdata_symbolic_operand" "s")))]
+  "reload_completed"
   "addl %0 = @gprel(%1), gp"
-  [(set_attr "type" "A")])
+  [(set_attr "itanium_class" "ialu")])
 
-(define_insn "gprel64_offset"
+(define_insn "*gprel64_offset"
   [(set (match_operand:DI 0 "register_operand" "=r")
        (minus:DI (match_operand:DI 1 "symbolic_operand" "") (reg:DI 1)))]
-  ""
+  "reload_completed"
   "movl %0 = @gprel(%1)"
-  [(set_attr "type" "L")])
+  [(set_attr "itanium_class" "long_i")])
 
 (define_expand "load_gprel64"
-  [(set (match_dup 2)
-       (minus:DI (match_operand:DI 1 "symbolic_operand" "") (reg:DI 1)))
-   (set (match_operand:DI 0 "register_operand" "")
-       (plus:DI (reg:DI 1) (match_dup 2)))]
+  [(set (match_operand:DI 0 "register_operand" "")
+       (minus:DI (match_operand:DI 1 "symbolic_operand" "") (match_dup 2)))
+   (set (match_dup 0)
+       (plus:DI (match_dup 2) (match_dup 0)))]
+  "reload_completed"
+{
+  operands[2] = pic_offset_table_rtx;
+})
+
+;; This is used as a placeholder for the return address during early
+;; compilation.  We won't know where we've placed this until during
+;; reload, at which point it can wind up in b0, a general register,
+;; or memory.  The only safe destination under these conditions is a
+;; general register.
+
+(define_insn_and_split "*movdi_ret_addr"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (unspec:DI [(const_int 0)] UNSPEC_RET_ADDR))]
   ""
-  "
+  "#"
+  "reload_completed"
+  [(const_int 0)]
 {
-  operands[2] = no_new_pseudos ? operands[0] : gen_reg_rtx (DImode);
-}")
+  ia64_split_return_addr_rtx (operands[0]);
+  DONE;
+}
+  [(set_attr "itanium_class" "ialu")])
 
-(define_expand "load_symptr"
-  [(set (match_dup 2)
-       (plus:DI (reg:DI 1) (match_operand:DI 1 "got_symbolic_operand" "")))
-   (set (match_operand:DI 0 "register_operand" "") (match_dup 3))]
+(define_insn "*load_symptr_high"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (plus:DI (high:DI (match_operand 1 "got_symbolic_operand" "s"))
+                (match_operand:DI 2 "register_operand" "a")))]
+  "reload_completed"
+{
+  if (HAVE_AS_LTOFFX_LDXMOV_RELOCS)
+    return "%,addl %0 = @ltoffx(%1), %2";
+  else
+    return "%,addl %0 = @ltoff(%1), %2";
+}
+  [(set_attr "itanium_class" "ialu")])
+
+(define_insn "*load_symptr_low"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (lo_sum:DI (match_operand:DI 1 "register_operand" "r")
+                  (match_operand 2 "got_symbolic_operand" "s")))]
+  "reload_completed"
+{
+  if (HAVE_AS_LTOFFX_LDXMOV_RELOCS)
+    return "%,ld8.mov %0 = [%1], %2";
+  else
+    return "%,ld8 %0 = [%1]";
+}
+  [(set_attr "itanium_class" "ld")])
+
+(define_insn_and_split "load_dtpmod"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (unspec:DI [(match_operand 1 "tls_symbolic_operand" "")]
+                  UNSPEC_DTPMOD))]
   ""
-  "
+  "#"
+  "reload_completed"
+  [(set (match_dup 0)
+       (plus:DI (unspec:DI [(match_dup 1)] UNSPEC_LTOFF_DTPMOD)
+                (match_dup 2)))
+   (set (match_dup 0) (match_dup 3))]
 {
-  operands[2] = no_new_pseudos ? operands[0] : gen_reg_rtx (DImode);
-  operands[3] = gen_rtx_MEM (DImode, operands[2]);
-  RTX_UNCHANGING_P (operands[3]) = 1;
-}")
+  operands[2] = pic_offset_table_rtx;
+  operands[3] = gen_const_mem (DImode, operands[0]);
+})
 
-(define_insn "*load_symptr_internal1"
+(define_insn "*load_ltoff_dtpmod"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (plus:DI (reg:DI 1) (match_operand:DI 1 "got_symbolic_operand" "s")))]
+       (plus:DI (unspec:DI [(match_operand 1 "tls_symbolic_operand" "")]
+                           UNSPEC_LTOFF_DTPMOD)
+                (match_operand:DI 2 "register_operand" "a")))]
+  "reload_completed"
+  "addl %0 = @ltoff(@dtpmod(%1)), %2"
+  [(set_attr "itanium_class" "ialu")])
+
+(define_expand "load_dtprel"
+  [(set (match_operand:DI 0 "register_operand" "")
+       (unspec:DI [(match_operand 1 "tls_symbolic_operand" "")]
+                  UNSPEC_DTPREL))]
   ""
-  "addl %0 = @ltoff(%1), gp"
-  [(set_attr "type" "A")])
+  "")
 
-;; With no offsettable memory references, we've got to have a scratch
-;; around to play with the second word.
-(define_expand "movti"
-  [(parallel [(set (match_operand:TI 0 "general_operand" "")
-                  (match_operand:TI 1 "general_operand" ""))
-             (clobber (match_scratch:DI 2 ""))])]
+(define_insn "*load_dtprel64"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (unspec:DI [(match_operand 1 "ld_tls_symbolic_operand" "")]
+                  UNSPEC_DTPREL))]
+  "TARGET_TLS64"
+  "movl %0 = @dtprel(%1)"
+  [(set_attr "itanium_class" "long_i")])
+
+(define_insn "*load_dtprel22"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (unspec:DI [(match_operand 1 "ld_tls_symbolic_operand" "")]
+                  UNSPEC_DTPREL))]
   ""
-  "
-{
-  if (! reload_in_progress && ! reload_completed
-      && ! ia64_move_ok (operands[0], operands[1]))
-    operands[1] = force_reg (TImode, operands[1]);
-}")
+  "addl %0 = @dtprel(%1), r0"
+  [(set_attr "itanium_class" "ialu")])
 
-(define_insn_and_split "*movti_internal"
-  [(set (match_operand:TI 0 "nonimmediate_operand" "=r,r,m")
-       (match_operand:TI 1 "general_operand"      "ri,m,r"))
-   (clobber (match_scratch:DI 2 "=X,&r,&r"))]
-  "ia64_move_ok (operands[0], operands[1])"
+(define_insn_and_split "*load_dtprel_gd"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (unspec:DI [(match_operand 1 "tls_symbolic_operand" "")]
+                  UNSPEC_DTPREL))]
+  ""
   "#"
   "reload_completed"
-  [(const_int 0)]
-  "
+  [(set (match_dup 0)
+       (plus:DI (unspec:DI [(match_dup 1)] UNSPEC_LTOFF_DTPREL)
+                (match_dup 2)))
+   (set (match_dup 0) (match_dup 3))]
 {
-  rtx adj1, adj2, in[2], out[2];
-  int first;
+  operands[2] = pic_offset_table_rtx;
+  operands[3] = gen_const_mem (DImode, operands[0]);
+})
 
-  adj1 = ia64_split_timode (in, operands[1], operands[2]);
-  adj2 = ia64_split_timode (out, operands[0], operands[2]);
+(define_insn "*load_ltoff_dtprel"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (plus:DI (unspec:DI [(match_operand 1 "tls_symbolic_operand" "")]
+                           UNSPEC_LTOFF_DTPREL)
+                (match_operand:DI 2 "register_operand" "a")))]
+  ""
+  "addl %0 = @ltoff(@dtprel(%1)), %2"
+  [(set_attr "itanium_class" "ialu")])
 
-  first = 0;
-  if (reg_overlap_mentioned_p (out[0], in[1]))
-    {
-      if (reg_overlap_mentioned_p (out[1], in[0]))
-       abort ();
-      first = 1;
-    }
+(define_expand "add_dtprel"
+  [(set (match_operand:DI 0 "register_operand" "")
+       (plus:DI (unspec:DI [(match_operand 1 "ld_tls_symbolic_operand" "")]
+                           UNSPEC_DTPREL)
+                (match_operand:DI 2 "register_operand" "")))]
+  "!TARGET_TLS64"
+  "")
 
-  if (adj1 && adj2)
-    abort ();
-  if (adj1)
-    emit_insn (adj1);
-  if (adj2)
-    emit_insn (adj2);
-  emit_insn (gen_rtx_SET (VOIDmode, out[first], in[first]));
-  emit_insn (gen_rtx_SET (VOIDmode, out[!first], in[!first]));
-  DONE;
-}"
-  [(set_attr "type" "unknown")
-   (set_attr "predicable" "no")])
+(define_insn "*add_dtprel14"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (plus:DI (unspec:DI [(match_operand 1 "ld_tls_symbolic_operand" "")]
+                           UNSPEC_DTPREL)
+                (match_operand:DI 2 "register_operand" "r")))]
+  "TARGET_TLS14"
+  "adds %0 = @dtprel(%1), %2"
+  [(set_attr "itanium_class" "ialu")])
+
+(define_insn "*add_dtprel22"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (plus:DI (unspec:DI [(match_operand 1 "ld_tls_symbolic_operand" "")]
+                           UNSPEC_DTPREL)
+                (match_operand:DI 2 "register_operand" "a")))]
+  "TARGET_TLS22"
+  "addl %0 = @dtprel(%1), %2"
+  [(set_attr "itanium_class" "ialu")])
+
+(define_expand "load_tprel"
+  [(set (match_operand:DI 0 "register_operand" "")
+       (unspec:DI [(match_operand 1 "tls_symbolic_operand" "")]
+                  UNSPEC_TPREL))]
+  ""
+  "")
+
+(define_insn "*load_tprel64"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (unspec:DI [(match_operand 1 "le_tls_symbolic_operand" "")]
+                  UNSPEC_TPREL))]
+  "TARGET_TLS64"
+  "movl %0 = @tprel(%1)"
+  [(set_attr "itanium_class" "long_i")])
+
+(define_insn "*load_tprel22"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (unspec:DI [(match_operand 1 "le_tls_symbolic_operand" "")]
+                  UNSPEC_TPREL))]
+  ""
+  "addl %0 = @tprel(%1), r0"
+  [(set_attr "itanium_class" "ialu")])
 
-;; ??? SSA creates these.  Can't allow memories since we don't have
-;; the scratch register.  Fortunately combine will know how to add
-;; the clobber and scratch.
-(define_insn_and_split "*movti_internal_reg"
-  [(set (match_operand:TI 0 "register_operand"  "=r")
-       (match_operand:TI 1 "nonmemory_operand" "ri"))]
+(define_insn_and_split "*load_tprel_ie"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (unspec:DI [(match_operand 1 "ie_tls_symbolic_operand" "")]
+                  UNSPEC_TPREL))]
   ""
   "#"
   "reload_completed"
-  [(const_int 0)]
-  "
+  [(set (match_dup 0)
+       (plus:DI (unspec:DI [(match_dup 1)] UNSPEC_LTOFF_TPREL)
+                (match_dup 2)))
+   (set (match_dup 0) (match_dup 3))]
 {
-  rtx in[2], out[2];
-  int first;
+  operands[2] = pic_offset_table_rtx;
+  operands[3] = gen_const_mem (DImode, operands[0]);
+})
 
-  ia64_split_timode (in, operands[1], NULL_RTX);
-  ia64_split_timode (out, operands[0], NULL_RTX);
+(define_insn "*load_ltoff_tprel"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (plus:DI (unspec:DI [(match_operand 1 "ie_tls_symbolic_operand" "")]
+                           UNSPEC_LTOFF_TPREL)
+                (match_operand:DI 2 "register_operand" "a")))]
+  ""
+  "addl %0 = @ltoff(@tprel(%1)), %2"
+  [(set_attr "itanium_class" "ialu")])
 
-  first = 0;
-  if (reg_overlap_mentioned_p (out[0], in[1]))
-    {
-      if (reg_overlap_mentioned_p (out[1], in[0]))
-       abort ();
-      first = 1;
-    }
+(define_expand "add_tprel"
+  [(set (match_operand:DI 0 "register_operand" "")
+       (plus:DI (unspec:DI [(match_operand 1 "le_tls_symbolic_operand" "")]
+                           UNSPEC_TPREL)
+                (match_operand:DI 2 "register_operand" "")))]
+  "!TARGET_TLS64"
+  "")
 
-  emit_insn (gen_rtx_SET (VOIDmode, out[first], in[first]));
-  emit_insn (gen_rtx_SET (VOIDmode, out[!first], in[!first]));
-  DONE;
-}"
-  [(set_attr "type" "unknown")
-   (set_attr "predicable" "no")])
+(define_insn "*add_tprel14"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (plus:DI (unspec:DI [(match_operand 1 "le_tls_symbolic_operand" "")]
+                           UNSPEC_TPREL)
+                (match_operand:DI 2 "register_operand" "r")))]
+  "TARGET_TLS14"
+  "adds %0 = @tprel(%1), %2"
+  [(set_attr "itanium_class" "ialu")])
+
+(define_insn "*add_tprel22"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+       (plus:DI (unspec:DI [(match_operand 1 "le_tls_symbolic_operand" "")]
+                           UNSPEC_TPREL)
+                (match_operand:DI 2 "register_operand" "a")))]
+  "TARGET_TLS22"
+  "addl %0 = @tprel(%1), %2"
+  [(set_attr "itanium_class" "ialu")])
 
-(define_expand "reload_inti"
-  [(parallel [(set (match_operand:TI 0 "register_operand" "=r")
-                  (match_operand:TI 1 "" "m"))
-             (clobber (match_operand:DI 2 "register_operand" "=&r"))])]
+;; With no offsettable memory references, we've got to have a scratch
+;; around to play with the second word.  However, in order to avoid a
+;; reload nightmare we lie, claim we don't need one, and fix it up
+;; in ia64_split_tmode_move.
+(define_expand "movti"
+  [(set (match_operand:TI 0 "general_operand" "")
+       (match_operand:TI 1 "general_operand" ""))]
   ""
-  "
 {
-  /* ??? Should now be enforced by tweeks to push_secondary_reload.  */
-  if (reg_overlap_mentioned_p (operands[2], operands[0])
-      || reg_overlap_mentioned_p (operands[2], operands[1]))
-    abort ();
-}")
-
-(define_expand "reload_outti"
-  [(parallel [(set (match_operand:TI 0 "" "=m")
-                  (match_operand:TI 1 "register_operand" "r"))
-             (clobber (match_operand:DI 2 "register_operand" "=&r"))])]
-  ""
-  "
+  rtx op1 = ia64_expand_move (operands[0], operands[1]);
+  if (!op1)
+    DONE;
+  operands[1] = op1;
+})
+
+(define_insn_and_split "*movti_internal"
+  [(set (match_operand:TI 0 "destination_operand" "=r,   *fm,*x,*f,  Q")
+       (match_operand:TI 1 "general_operand"     "r*fim,r,  Q, *fOQ,*f"))]
+  "ia64_move_ok (operands[0], operands[1])"
+  "@
+   #
+   #
+   ldfp8 %X0 = %1%P1
+   #
+   #"
+  "reload_completed && !ia64_load_pair_ok(operands[0], operands[1])"
+  [(const_int 0)]
 {
-  /* ??? Should now be enforced by tweeks to push_secondary_reload.  */
-  if (reg_overlap_mentioned_p (operands[2], operands[0])
-      || reg_overlap_mentioned_p (operands[2], operands[1]))
-    abort ();
-}")
+  ia64_split_tmode_move (operands);
+  DONE;
+}
+  [(set_attr "itanium_class" "unknown,unknown,fldp,unknown,unknown")])
 
 ;; Floating Point Moves
 ;;
 ;; Note - Patterns for SF mode moves are compulsory, but
-;; patterns for DF are optional, as GCC can synthesise them.
+;; patterns for DF are optional, as GCC can synthesize them.
 
 (define_expand "movsf"
   [(set (match_operand:SF 0 "general_operand" "")
        (match_operand:SF 1 "general_operand" ""))]
   ""
-  "
 {
-  if (! reload_in_progress && ! reload_completed
-      && ! ia64_move_ok (operands[0], operands[1]))
-    operands[1] = force_reg (SFmode, operands[1]);
-}")
-
-;; Errata 72 workaround.
-(define_insn "*movsfcc_astep"
-  [(cond_exec
-     (match_operator 2 "predicate_operator"
-       [(match_operand:CC 3 "register_operand" "c,c,c,c")
-        (const_int 0)])
-     (set (match_operand:SF 0 "register_operand"  "=f,*r, f,*r")
-         (match_operand:SF 1 "nonmemory_operand" "fG,fG,*r,*r")))]
-  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
-  "@
-  (%J2) mov %0 = %F1
-  (%J2) getf.s %0 = %F1
-  (%J2) setf.s %0 = %1
-  (%J2) mov %0 = %1"
-  [(set_attr "type" "F,M,M,A")
-   (set_attr "predicable" "no")])
-
-(define_insn "*movsf_internal_astep"
-  [(set (match_operand:SF 0 "destination_operand" "=f,f, Q,*r, f,*r,*r, m")
-       (match_operand:SF 1 "general_operand"     "fG,Q,fG,fG,*r,*r, m,*r"))]
-  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
-  "@
-  mov %0 = %F1
-  ldfs %0 = %1%P1
-  stfs %0 = %F1%P0
-  getf.s %0 = %F1
-  setf.s %0 = %1
-  mov %0 = %1
-  ld4%O1 %0 = %1%P1
-  st4%Q0 %0 = %1%P0"
-  [(set_attr "type" "F,M,M,M,M,A,M,M")
-   (set_attr "predicable" "no")])
+  rtx op1 = ia64_expand_move (operands[0], operands[1]);
+  if (!op1)
+    DONE;
+  operands[1] = op1;
+})
 
 (define_insn "*movsf_internal"
   [(set (match_operand:SF 0 "destination_operand" "=f,f, Q,*r, f,*r,*r, m")
        (match_operand:SF 1 "general_operand"     "fG,Q,fG,fG,*r,*r, m,*r"))]
-  "! TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
+  "ia64_move_ok (operands[0], operands[1])"
   "@
-  mov %0 = %F1
-  ldfs %0 = %1%P1
-  stfs %0 = %F1%P0
-  getf.s %0 = %F1
-  setf.s %0 = %1
-  mov %0 = %1
-  ld4%O1 %0 = %1%P1
-  st4%Q0 %0 = %1%P0"
-  [(set_attr "type" "F,M,M,M,M,A,M,M")])
+   mov %0 = %F1
+   ldfs %0 = %1%P1
+   stfs %0 = %F1%P0
+   getf.s %0 = %F1
+   setf.s %0 = %1
+   mov %0 = %1
+   ld4%O1 %0 = %1%P1
+   st4%Q0 %0 = %1%P0"
+  [(set_attr "itanium_class" "fmisc,fld,stf,frfr,tofr,ialu,ld,st")])
 
 (define_expand "movdf"
   [(set (match_operand:DF 0 "general_operand" "")
        (match_operand:DF 1 "general_operand" ""))]
   ""
-  "
 {
-  if (! reload_in_progress && ! reload_completed
-      && ! ia64_move_ok (operands[0], operands[1]))
-    operands[1] = force_reg (DFmode, operands[1]);
-}")
-
-;; Errata 72 workaround.
-(define_insn "*movdfcc_astep"
-  [(cond_exec
-     (match_operator 2 "predicate_operator"
-       [(match_operand:CC 3 "register_operand" "c,c,c,c")
-        (const_int 0)])
-     (set (match_operand:DF 0 "register_operand"  "=f,*r, f,*r")
-         (match_operand:DF 1 "nonmemory_operand" "fG,fG,*r,*r")))]
-  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
-  "@
-  (%J2) mov %0 = %F1
-  (%J2) getf.d %0 = %F1
-  (%J2) setf.d %0 = %1
-  (%J2) mov %0 = %1"
-  [(set_attr "type" "F,M,M,A")
-   (set_attr "predicable" "no")])
-
-(define_insn "*movdf_internal_astep"
-  [(set (match_operand:DF 0 "destination_operand" "=f,f, Q,*r, f,*r,*r, m")
-       (match_operand:DF 1 "general_operand"     "fG,Q,fG,fG,*r,*r, m,*r"))]
-  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
-  "@
-  mov %0 = %F1
-  ldfd %0 = %1%P1
-  stfd %0 = %F1%P0
-  getf.d %0 = %F1
-  setf.d %0 = %1
-  mov %0 = %1
-  ld8%O1 %0 = %1%P1
-  st8%Q0 %0 = %1%P0"
-  [(set_attr "type" "F,M,M,M,M,A,M,M")
-   (set_attr "predicable" "no")])
+  rtx op1 = ia64_expand_move (operands[0], operands[1]);
+  if (!op1)
+    DONE;
+  operands[1] = op1;
+})
 
 (define_insn "*movdf_internal"
   [(set (match_operand:DF 0 "destination_operand" "=f,f, Q,*r, f,*r,*r, m")
        (match_operand:DF 1 "general_operand"     "fG,Q,fG,fG,*r,*r, m,*r"))]
-  "! TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
+  "ia64_move_ok (operands[0], operands[1])"
   "@
-  mov %0 = %F1
-  ldfd %0 = %1%P1
-  stfd %0 = %F1%P0
-  getf.d %0 = %F1
-  setf.d %0 = %1
-  mov %0 = %1
-  ld8%O1 %0 = %1%P1
-  st8%Q0 %0 = %1%P0"
-  [(set_attr "type" "F,M,M,M,M,A,M,M")])
+   mov %0 = %F1
+   ldfd %0 = %1%P1
+   stfd %0 = %F1%P0
+   getf.d %0 = %F1
+   setf.d %0 = %1
+   mov %0 = %1
+   ld8%O1 %0 = %1%P1
+   st8%Q0 %0 = %1%P0"
+  [(set_attr "itanium_class" "fmisc,fld,stf,frfr,tofr,ialu,ld,st")])
 
 ;; With no offsettable memory references, we've got to have a scratch
 ;; around to play with the second word if the variable winds up in GRs.
-(define_expand "movtf"
-  [(set (match_operand:TF 0 "general_operand" "")
-       (match_operand:TF 1 "general_operand" ""))]
+(define_expand "movxf"
+  [(set (match_operand:XF 0 "general_operand" "")
+       (match_operand:XF 1 "general_operand" ""))]
   ""
-  "
 {
-  /* We must support TFmode loads into general registers for stdarg/vararg
-     and unprototyped calls.  We split them into DImode loads for convenience.
-     We don't need TFmode stores from general regs, because a stdarg/vararg
-     routine does a block store to memory of unnamed arguments.  */
-  if (GET_CODE (operands[0]) == REG
-      && GR_REGNO_P (REGNO (operands[0])))
-    {
-      /* We're hoping to transform everything that deals with TFmode
-        quantities and GR registers early in the compiler.  */
-      if (no_new_pseudos)
-       abort ();
-
-      /* Struct to register can just use TImode instead.  */
-      if ((GET_CODE (operands[1]) == SUBREG
-          && GET_MODE (SUBREG_REG (operands[1])) == TImode)
-         || (GET_CODE (operands[1]) == REG
-             && GR_REGNO_P (REGNO (operands[1]))))
-       {
-         emit_move_insn (gen_rtx_REG (TImode, REGNO (operands[0])),
-                         SUBREG_REG (operands[1]));
-         DONE;
-       }
-
-      if (GET_CODE (operands[1]) == CONST_DOUBLE)
-       {
-         emit_move_insn (gen_rtx_REG (DImode, REGNO (operands[0])),
-                         operand_subword (operands[1], 0, 0, DImode));
-         emit_move_insn (gen_rtx_REG (DImode, REGNO (operands[0]) + 1),
-                         operand_subword (operands[1], 1, 0, DImode));
-         DONE;
-       }
-
-      /* If the quantity is in a register not known to be GR, spill it.  */
-      if (register_operand (operands[1], TFmode))
-       operands[1] = spill_tfmode_operand (operands[1], 1);
-
-      if (GET_CODE (operands[1]) == MEM)
-       {
-         rtx out[2];
-
-         out[WORDS_BIG_ENDIAN] = gen_rtx_REG (DImode, REGNO (operands[0]));
-         out[!WORDS_BIG_ENDIAN] = gen_rtx_REG (DImode, REGNO (operands[0])+1);
-
-         emit_move_insn (out[0], change_address (operands[1], DImode, NULL));
-         emit_move_insn (out[1],
-                         change_address (operands[1], DImode,
-                                         plus_constant (XEXP (operands[1], 0),
-                                                        8)));
-         DONE;
-       }
-
-      abort ();
-    }
-
-  if (! reload_in_progress && ! reload_completed)
-    {
-      operands[0] = spill_tfmode_operand (operands[0], 0);
-      operands[1] = spill_tfmode_operand (operands[1], 0);
-
-      if (! ia64_move_ok (operands[0], operands[1]))
-       operands[1] = force_reg (TFmode, operands[1]);
-    }
-}")
+  if (ia64_expand_movxf_movrf (XFmode, operands))
+    DONE;
+})
 
 ;; ??? There's no easy way to mind volatile acquire/release semantics.
 
-;; Errata 72 workaround.
-(define_insn "*movtfcc_astep"
-  [(cond_exec
-     (match_operator 2 "predicate_operator"
-       [(match_operand:CC 3 "register_operand" "c")
-        (const_int 0)])
-     (set (match_operand:TF 0 "register_operand"  "=f")
-         (match_operand:TF 1 "nonmemory_operand" "fG")))]
-  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
-  "(%J2) mov %0 = %F1"
-  [(set_attr "type" "F")
-   (set_attr "predicable" "no")])
+(define_insn "*movxf_internal"
+  [(set (match_operand:XF 0 "destination_operand" "=f,f, m")
+       (match_operand:XF 1 "general_operand"     "fG,m,fG"))]
+  "ia64_move_ok (operands[0], operands[1])"
+  "@
+   mov %0 = %F1
+   ldfe %0 = %1%P1
+   stfe %0 = %F1%P0"
+  [(set_attr "itanium_class" "fmisc,fld,stf")])
 
-(define_insn "*movtf_internal_astep"
-  [(set (match_operand:TF 0 "destination_tfmode_operand" "=f,f, m")
-       (match_operand:TF 1 "general_tfmode_operand"     "fG,m,fG"))]
-  "TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
+;; Same as for movxf, but for RFmode.
+(define_expand "movrf"
+  [(set (match_operand:RF 0 "general_operand" "")
+       (match_operand:RF 1 "general_operand" ""))]
+  ""
+{
+  if (ia64_expand_movxf_movrf (RFmode, operands))
+    DONE;
+})
+
+(define_insn "*movrf_internal"
+  [(set (match_operand:RF 0 "destination_operand" "=f,f, m")
+       (match_operand:RF 1 "general_operand"     "fG,m,fG"))]
+  "ia64_move_ok (operands[0], operands[1])"
   "@
-  mov %0 = %F1
-  ldfe %0 = %1%P1
-  stfe %0 = %F1%P0"
-  [(set_attr "type" "F,M,M")
+   mov %0 = %F1
+   ldf.fill %0 = %1%P1
+   stf.spill %0 = %F1%P0"
+  [(set_attr "itanium_class" "fmisc,fld,stf")])
+
+;; Better code generation via insns that deal with TFmode register pairs
+;; directly.  Same concerns apply as for TImode.
+(define_expand "movtf"
+  [(set (match_operand:TF 0 "general_operand" "")
+       (match_operand:TF 1 "general_operand" ""))]
+  ""
+{
+  rtx op1 = ia64_expand_move (operands[0], operands[1]);
+  if (!op1)
+    DONE;
+  operands[1] = op1;
+})
+
+(define_insn_and_split "*movtf_internal"
+  [(set (match_operand:TF 0 "destination_operand"  "=r,r,m")
+       (match_operand:TF 1 "general_operand"      "ri,m,r"))]
+  "ia64_move_ok (operands[0], operands[1])"
+  "#"
+  "reload_completed"
+  [(const_int 0)]
+{
+  ia64_split_tmode_move (operands);
+  DONE;
+}
+  [(set_attr "itanium_class" "unknown")
    (set_attr "predicable" "no")])
 
-(define_insn "*movtf_internal"
-  [(set (match_operand:TF 0 "destination_tfmode_operand" "=f,f, m")
-       (match_operand:TF 1 "general_tfmode_operand"     "fG,m,fG"))]
-  "! TARGET_A_STEP && ia64_move_ok (operands[0], operands[1])"
-  "@
-  mov %0 = %F1
-  ldfe %0 = %1%P1
-  stfe %0 = %F1%P0"
-  [(set_attr "type" "F,M,M")])
 \f
 ;; ::::::::::::::::::::
 ;; ::
 ;; Signed conversions from a smaller integer to a larger integer
 
 (define_insn "extendqidi2"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (sign_extend:DI (match_operand:QI 1 "register_operand" "r")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (sign_extend:DI (match_operand:QI 1 "gr_register_operand" "r")))]
   ""
   "sxt1 %0 = %1"
-  [(set_attr "type" "I")])
+  [(set_attr "itanium_class" "xtd")])
 
 (define_insn "extendhidi2"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (sign_extend:DI (match_operand:HI 1 "register_operand" "r")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (sign_extend:DI (match_operand:HI 1 "gr_register_operand" "r")))]
   ""
   "sxt2 %0 = %1"
-  [(set_attr "type" "I")])
+  [(set_attr "itanium_class" "xtd")])
 
 (define_insn "extendsidi2"
-  [(set (match_operand:DI 0 "register_operand" "=r,*f")
-       (sign_extend:DI (match_operand:SI 1 "register_operand" "r,*f")))]
+  [(set (match_operand:DI 0 "grfr_register_operand" "=r,?f")
+       (sign_extend:DI (match_operand:SI 1 "grfr_register_operand" "r,f")))]
   ""
   "@
    sxt4 %0 = %1
-   fsxt.r %0 = %1, %1%B0"
-  [(set_attr "type" "I,F")])
+   fsxt.r %0 = %1, %1"
+  [(set_attr "itanium_class" "xtd,fmisc")])
 
 ;; Unsigned conversions from a smaller integer to a larger integer
 
 (define_insn "zero_extendqidi2"
-  [(set (match_operand:DI 0 "register_operand" "=r,r")
-       (zero_extend:DI (match_operand:QI 1 "nonimmediate_operand" "r,m")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r,r")
+       (zero_extend:DI (match_operand:QI 1 "gr_nonimmediate_operand" "r,m")))]
   ""
   "@
    zxt1 %0 = %1
    ld1%O1 %0 = %1%P1"
-  [(set_attr "type" "I,M")])
+  [(set_attr "itanium_class" "xtd,ld")])
 
 (define_insn "zero_extendhidi2"
-  [(set (match_operand:DI 0 "register_operand" "=r,r")
-       (zero_extend:DI (match_operand:HI 1 "nonimmediate_operand" "r,m")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r,r")
+       (zero_extend:DI (match_operand:HI 1 "gr_nonimmediate_operand" "r,m")))]
   ""
   "@
    zxt2 %0 = %1
    ld2%O1 %0 = %1%P1"
-  [(set_attr "type" "I,M")])
+  [(set_attr "itanium_class" "xtd,ld")])
 
 (define_insn "zero_extendsidi2"
-  [(set (match_operand:DI 0 "register_operand" "=r,r,*f")
-       (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,*f")))]
+  [(set (match_operand:DI 0 "grfr_register_operand" "=r,r,?f")
+       (zero_extend:DI
+         (match_operand:SI 1 "grfr_nonimmediate_operand" "r,m,f")))]
   ""
   "@
-   zxt4 %0 = %1
+   addp4 %0 = %1, r0
    ld4%O1 %0 = %1%P1
-   fsxt.r %0 = f1, %1%B0"
-  [(set_attr "type" "I,M,F")])
+   fmix.r %0 = f0, %1"
+  [(set_attr "itanium_class" "ialu,ld,fmisc")])
 
 ;; Convert between floating point types of different sizes.
 
+;; At first glance, it would appear that emitting fnorm for an extending
+;; conversion is unnecessary.  However, the stf and getf instructions work
+;; correctly only if the input is properly rounded for its type.  In
+;; particular, we get the wrong result for getf.d/stfd if the input is a
+;; denorm single.  Since we don't know what the next instruction will be, we
+;; have to emit an fnorm.
+
 ;; ??? Optimization opportunity here.  Get rid of the insn altogether
 ;; when we can.  Should probably use a scheme like has been proposed
 ;; for ia32 in dealing with operands that match unary operators.  This
-;; would let combine merge the thing into adjacent insns.
+;; would let combine merge the thing into adjacent insns.  See also how the
+;; mips port handles SIGN_EXTEND as operands to integer arithmetic insns via
+;; se_register_operand.
 
-(define_insn_and_split "extendsfdf2"
-  [(set (match_operand:DF 0 "register_operand" "=f,f")
-       (float_extend:DF (match_operand:SF 1 "register_operand" "0,f")))]
+(define_insn "extendsfdf2"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (float_extend:DF (match_operand:SF 1 "fr_register_operand" "f")))]
   ""
-  "mov %0 = %1"
-  "reload_completed"
-  [(set (match_dup 0) (float_extend:DF (match_dup 1)))]
-  "if (true_regnum (operands[0]) == true_regnum (operands[1])) DONE;"
-  [(set_attr "type" "F")])
+  "fnorm.d %0 = %1"
+  [(set_attr "itanium_class" "fmac")])
 
-(define_insn_and_split "extendsftf2"
-  [(set (match_operand:TF 0 "register_operand" "=f,f")
-       (float_extend:TF (match_operand:SF 1 "register_operand" "0,f")))]
+(define_insn "extendsfxf2"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (float_extend:XF (match_operand:SF 1 "fr_register_operand" "f")))]
   ""
-  "mov %0 = %1"
-  "reload_completed"
-  [(set (match_dup 0) (float_extend:TF (match_dup 1)))]
-  "if (true_regnum (operands[0]) == true_regnum (operands[1])) DONE;"
-  [(set_attr "type" "F")])
+  "fnorm %0 = %1"
+  [(set_attr "itanium_class" "fmac")])
 
-(define_insn_and_split "extenddftf2"
-  [(set (match_operand:TF 0 "register_operand" "=f,f")
-       (float_extend:TF (match_operand:DF 1 "register_operand" "0,f")))]
+(define_insn "extenddfxf2"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (float_extend:XF (match_operand:DF 1 "fr_register_operand" "f")))]
   ""
-  "mov %0 = %1"
-  "reload_completed"
-  [(set (match_dup 0) (float_extend:TF (match_dup 1)))]
-  "if (true_regnum (operands[0]) == true_regnum (operands[1])) DONE;"
-  [(set_attr "type" "F")])
+  "fnorm %0 = %1"
+  [(set_attr "itanium_class" "fmac")])
 
 (define_insn "truncdfsf2"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-       (float_truncate:SF (match_operand:DF 1 "register_operand" "f")))]
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF (match_operand:DF 1 "fr_register_operand" "f")))]
   ""
-  "fnorm.s %0 = %1%B0"
-  [(set_attr "type" "F")])
+  "fnorm.s %0 = %1"
+  [(set_attr "itanium_class" "fmac")])
 
-(define_insn "trunctfsf2"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-       (float_truncate:SF (match_operand:TF 1 "register_operand" "f")))]
+(define_insn "truncxfsf2"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF (match_operand:XF 1 "fr_register_operand" "f")))]
   ""
-  "fnorm.s %0 = %1%B0"
-  [(set_attr "type" "F")])
+  "fnorm.s %0 = %1"
+  [(set_attr "itanium_class" "fmac")])
 
-(define_insn "trunctfdf2"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-       (float_truncate:DF (match_operand:TF 1 "register_operand" "f")))]
+(define_insn "truncxfdf2"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (float_truncate:DF (match_operand:XF 1 "fr_register_operand" "f")))]
   ""
-  "fnorm.d %0 = %1%B0"
-  [(set_attr "type" "F")])
+  "fnorm.d %0 = %1"
+  [(set_attr "itanium_class" "fmac")])
 
 ;; Convert between signed integer types and floating point.
 
-(define_insn "floatditf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (float:TF (match_operand:DI 1 "register_operand" "f")))]
+(define_insn "floatdixf2"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (float:XF (match_operand:DI 1 "fr_register_operand" "f")))]
   ""
   "fcvt.xf %0 = %1"
-  [(set_attr "type" "F")])
+  [(set_attr "itanium_class" "fcvtfx")])
 
 (define_insn "fix_truncsfdi2"
-  [(set (match_operand:DI 0 "register_operand" "=f")
-       (fix:DI (match_operand:SF 1 "register_operand" "f")))]
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
+       (fix:DI (match_operand:SF 1 "fr_register_operand" "f")))]
   ""
-  "fcvt.fx.trunc %0 = %1%B0"
-  [(set_attr "type" "F")])
+  "fcvt.fx.trunc %0 = %1"
+  [(set_attr "itanium_class" "fcvtfx")])
 
 (define_insn "fix_truncdfdi2"
-  [(set (match_operand:DI 0 "register_operand" "=f")
-       (fix:DI (match_operand:DF 1 "register_operand" "f")))]
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
+       (fix:DI (match_operand:DF 1 "fr_register_operand" "f")))]
+  ""
+  "fcvt.fx.trunc %0 = %1"
+  [(set_attr "itanium_class" "fcvtfx")])
+
+(define_insn "fix_truncxfdi2"
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
+       (fix:DI (match_operand:XF 1 "fr_register_operand" "f")))]
   ""
-  "fcvt.fx.trunc %0 = %1%B0"
-  [(set_attr "type" "F")])
+  "fcvt.fx.trunc %0 = %1"
+  [(set_attr "itanium_class" "fcvtfx")])
 
-(define_insn "fix_trunctfdi2"
-  [(set (match_operand:DI 0 "register_operand" "=f")
-       (fix:DI (match_operand:TF 1 "register_operand" "f")))]
+(define_insn "fix_truncxfdi2_alts"
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
+       (fix:DI (match_operand:XF 1 "fr_register_operand" "f")))
+   (use (match_operand:SI 2 "const_int_operand" ""))]
   ""
-  "fcvt.fx.trunc %0 = %1%B0"
-  [(set_attr "type" "F")])
+  "fcvt.fx.trunc.s%2 %0 = %1"
+  [(set_attr "itanium_class" "fcvtfx")])
 
 ;; Convert between unsigned integer types and floating point.
 
 (define_insn "floatunsdisf2"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-       (unsigned_float:SF (match_operand:DI 1 "register_operand" "f")))]
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (unsigned_float:SF (match_operand:DI 1 "fr_register_operand" "f")))]
   ""
-  "fcvt.xuf.s %0 = %1%B0"
-  [(set_attr "type" "F")])
+  "fcvt.xuf.s %0 = %1"
+  [(set_attr "itanium_class" "fcvtfx")])
 
 (define_insn "floatunsdidf2"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-       (unsigned_float:DF (match_operand:DI 1 "register_operand" "f")))]
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (unsigned_float:DF (match_operand:DI 1 "fr_register_operand" "f")))]
   ""
-  "fcvt.xuf.d %0 = %1%B0"
-  [(set_attr "type" "F")])
+  "fcvt.xuf.d %0 = %1"
+  [(set_attr "itanium_class" "fcvtfx")])
 
-(define_insn "floatunsditf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (unsigned_float:TF (match_operand:DI 1 "register_operand" "f")))]
+(define_insn "floatunsdixf2"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (unsigned_float:XF (match_operand:DI 1 "fr_register_operand" "f")))]
   ""
-  "fcvt.xuf %0 = %1%B0"
-  [(set_attr "type" "F")])
+  "fcvt.xuf %0 = %1"
+  [(set_attr "itanium_class" "fcvtfx")])
 
 (define_insn "fixuns_truncsfdi2"
-  [(set (match_operand:DI 0 "register_operand" "=f")
-       (unsigned_fix:DI (match_operand:SF 1 "register_operand" "f")))]
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
+       (unsigned_fix:DI (match_operand:SF 1 "fr_register_operand" "f")))]
   ""
-  "fcvt.fxu.trunc %0 = %1%B0"
-  [(set_attr "type" "F")])
+  "fcvt.fxu.trunc %0 = %1"
+  [(set_attr "itanium_class" "fcvtfx")])
 
 (define_insn "fixuns_truncdfdi2"
-  [(set (match_operand:DI 0 "register_operand" "=f")
-       (unsigned_fix:DI (match_operand:DF 1 "register_operand" "f")))]
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
+       (unsigned_fix:DI (match_operand:DF 1 "fr_register_operand" "f")))]
+  ""
+  "fcvt.fxu.trunc %0 = %1"
+  [(set_attr "itanium_class" "fcvtfx")])
+
+(define_insn "fixuns_truncxfdi2"
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
+       (unsigned_fix:DI (match_operand:XF 1 "fr_register_operand" "f")))]
   ""
-  "fcvt.fxu.trunc %0 = %1%B0"
-  [(set_attr "type" "F")])
+  "fcvt.fxu.trunc %0 = %1"
+  [(set_attr "itanium_class" "fcvtfx")])
 
-(define_insn "fixuns_trunctfdi2"
-  [(set (match_operand:DI 0 "register_operand" "=f")
-       (unsigned_fix:DI (match_operand:TF 1 "register_operand" "f")))]
+(define_insn "fixuns_truncxfdi2_alts"
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
+       (unsigned_fix:DI (match_operand:XF 1 "fr_register_operand" "f")))
+   (use (match_operand:SI 2 "const_int_operand" ""))]
   ""
-  "fcvt.fxu.trunc %0 = %1%B0"
-  [(set_attr "type" "F")])
+  "fcvt.fxu.trunc.s%2 %0 = %1"
+  [(set_attr "itanium_class" "fcvtfx")])
 \f
 ;; ::::::::::::::::::::
 ;; ::
 ;; ::
 ;; ::::::::::::::::::::
 
-;; ??? It would be useful to have SImode versions of the extract and insert
-;; patterns.
-
 (define_insn "extv"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (sign_extract:DI (match_operand:DI 1 "register_operand" "r")
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (sign_extract:DI (match_operand:DI 1 "gr_register_operand" "r")
                         (match_operand:DI 2 "const_int_operand" "n")
                         (match_operand:DI 3 "const_int_operand" "n")))]
   ""
   "extr %0 = %1, %3, %2"
-  [(set_attr "type" "I")])
+  [(set_attr "itanium_class" "ishf")])
 
 (define_insn "extzv"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (zero_extract:DI (match_operand:DI 1 "register_operand" "r")
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (zero_extract:DI (match_operand:DI 1 "gr_register_operand" "r")
                         (match_operand:DI 2 "const_int_operand" "n")
                         (match_operand:DI 3 "const_int_operand" "n")))]
   ""
   "extr.u %0 = %1, %3, %2"
-  [(set_attr "type" "I")])
+  [(set_attr "itanium_class" "ishf")])
 
 ;; Insert a bit field.
 ;; Can have 3 operands, source1 (inserter), source2 (insertee), dest.
 ;; where source2 != dest.
 
 (define_expand "insv"
-  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "")
+  [(set (zero_extract:DI (match_operand:DI 0 "gr_register_operand" "")
                         (match_operand:DI 1 "const_int_operand" "")
                         (match_operand:DI 2 "const_int_operand" ""))
        (match_operand:DI 3 "nonmemory_operand" ""))]
   ""
-  "
 {
   int width = INTVAL (operands[1]);
   int shift = INTVAL (operands[2]);
       operands[2] = GEN_INT (shift);
 #endif
     }
-}")
+})
 
 (define_insn "*insv_internal"
-  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+r")
+  [(set (zero_extract:DI (match_operand:DI 0 "gr_register_operand" "+r")
                         (match_operand:DI 1 "const_int_operand" "n")
                         (match_operand:DI 2 "const_int_operand" "n"))
        (match_operand:DI 3 "nonmemory_operand" "rP"))]
-  "(register_operand (operands[3], DImode) && INTVAL (operands[1]) <= 16)
+  "(gr_register_operand (operands[3], DImode) && INTVAL (operands[1]) <= 16)
    || operands[3] == const0_rtx || operands[3] == constm1_rtx"
   "dep %0 = %3, %0, %2, %1"
-  [(set_attr "type" "I")])
+  [(set_attr "itanium_class" "ishf")])
+
+;; Combine doesn't like to create bit-field insertions into zero.
+(define_insn "*shladdp4_internal"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (and:DI (ashift:DI (match_operand:DI 1 "gr_register_operand" "r")
+                          (match_operand:DI 2 "shladd_log2_operand" "n"))
+               (match_operand:DI 3 "const_int_operand" "n")))]
+  "ia64_depz_field_mask (operands[3], operands[2]) + INTVAL (operands[2]) == 32"
+  "shladdp4 %0 = %1, %2, r0"
+  [(set_attr "itanium_class" "ialu")])
+
+(define_insn "*depz_internal"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (and:DI (ashift:DI (match_operand:DI 1 "gr_register_operand" "r")
+                          (match_operand:DI 2 "const_int_operand" "n"))
+               (match_operand:DI 3 "const_int_operand" "n")))]
+  "CONST_OK_FOR_M (INTVAL (operands[2]))
+   && ia64_depz_field_mask (operands[3], operands[2]) > 0"
+{
+  operands[3] = GEN_INT (ia64_depz_field_mask (operands[3], operands[2]));
+  return "%,dep.z %0 = %1, %2, %3";
+}
+  [(set_attr "itanium_class" "ishf")])
 
 (define_insn "shift_mix4left"
-  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+r")
+  [(set (zero_extract:DI (match_operand:DI 0 "gr_register_operand" "+r")
                         (const_int 32) (const_int 0))
-       (match_operand:DI 1 "register_operand" "r"))
-   (clobber (match_operand:DI 2 "register_operand" "=r"))]
+       (match_operand:DI 1 "gr_register_operand" "r"))
+   (clobber (match_operand:DI 2 "gr_register_operand" "=r"))]
   ""
   "#"
-  [(set_attr "type" "unknown")])
-
-;; ??? Need to emit an instruction group barrier here because this gets split
-;; after md_reorg.
-
-(define_split
-  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "")
-                        (const_int 32) (const_int 0))
-       (match_operand:DI 1 "register_operand" ""))
-   (clobber (match_operand:DI 2 "register_operand" ""))]
-  "reload_completed"
-  [(set (match_dup 3) (ashift:DI (match_dup 1) (const_int 32)))
-   (unspec_volatile [(const_int 0)] 2)
-   (set (zero_extract:DI (match_dup 0) (const_int 32) (const_int 0))
-       (lshiftrt:DI (match_dup 3) (const_int 32)))]
-  "operands[3] = operands[2];")
+  [(set_attr "itanium_class" "unknown")])
 
 (define_split
   [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "")
                         (const_int 32) (const_int 0))
        (match_operand:DI 1 "register_operand" ""))
    (clobber (match_operand:DI 2 "register_operand" ""))]
-  "! reload_completed"
+  ""
   [(set (match_dup 3) (ashift:DI (match_dup 1) (const_int 32)))
    (set (zero_extract:DI (match_dup 0) (const_int 32) (const_int 0))
        (lshiftrt:DI (match_dup 3) (const_int 32)))]
   "operands[3] = operands[2];")
 
 (define_insn "*mix4left"
-  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+r")
+  [(set (zero_extract:DI (match_operand:DI 0 "gr_register_operand" "+r")
                         (const_int 32) (const_int 0))
-       (lshiftrt:DI (match_operand:DI 1 "register_operand" "r")
+       (lshiftrt:DI (match_operand:DI 1 "gr_register_operand" "r")
                     (const_int 32)))]
   ""
   "mix4.l %0 = %0, %r1"
-  [(set_attr "type" "I")])
+  [(set_attr "itanium_class" "mmshf")])
 
 (define_insn "mix4right"
-  [(set (zero_extract:DI (match_operand:DI 0 "register_operand" "+r")
+  [(set (zero_extract:DI (match_operand:DI 0 "gr_register_operand" "+r")
                         (const_int 32) (const_int 32))
-       (match_operand:DI 1 "reg_or_0_operand" "rO"))]
+       (match_operand:DI 1 "gr_reg_or_0_operand" "rO"))]
   ""
   "mix4.r %0 = %r1, %0"
-  [(set_attr "type" "I")])
+  [(set_attr "itanium_class" "mmshf")])
 
 ;; This is used by the rotrsi3 pattern.
 
 (define_insn "*mix4right_3op"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (ior:DI (zero_extend:DI (match_operand:SI 1 "register_operand" "r"))
-               (ashift:DI (zero_extend:DI (match_operand:SI 2 "register_operand" "r"))
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (ior:DI (zero_extend:DI (match_operand:SI 1 "gr_register_operand" "r"))
+               (ashift:DI (zero_extend:DI
+                            (match_operand:SI 2 "gr_register_operand" "r"))
                           (const_int 32))))]
   ""
   "mix4.r %0 = %2, %1"
-  [(set_attr "type" "I")])
+  [(set_attr "itanium_class" "mmshf")])
+
+\f
+;; ::::::::::::::::::::
+;; ::
+;; :: 1 bit Integer arithmetic
+;; ::
+;; ::::::::::::::::::::
+
+(define_insn_and_split "andbi3"
+  [(set (match_operand:BI 0 "register_operand" "=c,c,r")
+       (and:BI (match_operand:BI 1 "register_operand" "%0,0,r")
+               (match_operand:BI 2 "register_operand" "c,r,r")))]
+  ""
+  "@
+   #
+   tbit.nz.and.orcm %0, %I0 = %2, 0
+   and %0 = %2, %1"
+  "reload_completed
+   && GET_CODE (operands[0]) == REG && PR_REGNO_P (REGNO (operands[0]))
+   && GET_CODE (operands[2]) == REG && PR_REGNO_P (REGNO (operands[2]))"
+  [(cond_exec (eq (match_dup 2) (const_int 0))
+     (set (match_dup 0) (and:BI (ne:BI (const_int 0) (const_int 0))
+                               (match_dup 0))))]
+  ""
+  [(set_attr "itanium_class" "unknown,tbit,ilog")])
+
+(define_insn_and_split "*andcmbi3"
+  [(set (match_operand:BI 0 "register_operand" "=c,c,r")
+       (and:BI (not:BI (match_operand:BI 1 "register_operand" "c,r,r"))
+               (match_operand:BI 2 "register_operand" "0,0,r")))]
+  ""
+  "@
+   #
+   tbit.z.and.orcm %0, %I0 = %1, 0
+   andcm %0 = %2, %1"
+  "reload_completed
+   && GET_CODE (operands[0]) == REG && PR_REGNO_P (REGNO (operands[0]))
+   && GET_CODE (operands[1]) == REG && PR_REGNO_P (REGNO (operands[1]))"
+  [(cond_exec (ne (match_dup 1) (const_int 0))
+     (set (match_dup 0) (and:BI (ne:BI (const_int 0) (const_int 0))
+                               (match_dup 0))))]
+  ""
+  [(set_attr "itanium_class" "unknown,tbit,ilog")])
+
+(define_insn_and_split "iorbi3"
+  [(set (match_operand:BI 0 "register_operand" "=c,c,r")
+       (ior:BI (match_operand:BI 1 "register_operand" "%0,0,r")
+               (match_operand:BI 2 "register_operand" "c,r,r")))]
+  ""
+  "@
+   #
+   tbit.nz.or.andcm %0, %I0 = %2, 0
+   or %0 = %2, %1"
+  "reload_completed
+   && GET_CODE (operands[0]) == REG && PR_REGNO_P (REGNO (operands[0]))
+   && GET_CODE (operands[2]) == REG && PR_REGNO_P (REGNO (operands[2]))"
+  [(cond_exec (ne (match_dup 2) (const_int 0))
+     (set (match_dup 0) (ior:BI (eq:BI (const_int 0) (const_int 0))
+                               (match_dup 0))))]
+  ""
+  [(set_attr "itanium_class" "unknown,tbit,ilog")])
+
+(define_insn_and_split "*iorcmbi3"
+  [(set (match_operand:BI 0 "register_operand" "=c,c")
+       (ior:BI (not:BI (match_operand:BI 1 "register_operand" "c,r"))
+               (match_operand:BI 2 "register_operand" "0,0")))]
+  ""
+  "@
+   #
+   tbit.z.or.andcm %0, %I0 = %1, 0"
+  "reload_completed
+   && GET_CODE (operands[0]) == REG && PR_REGNO_P (REGNO (operands[0]))
+   && GET_CODE (operands[1]) == REG && PR_REGNO_P (REGNO (operands[1]))"
+  [(cond_exec (eq (match_dup 1) (const_int 0))
+     (set (match_dup 0) (ior:BI (eq:BI (const_int 0) (const_int 0))
+                               (match_dup 0))))]
+  ""
+  [(set_attr "itanium_class" "unknown,tbit")])
+
+(define_insn "one_cmplbi2"
+  [(set (match_operand:BI 0 "register_operand" "=c,r,c,&c")
+       (not:BI (match_operand:BI 1 "register_operand" "r,r,0,c")))
+   (clobber (match_scratch:BI 2 "=X,X,c,X"))]
+  ""
+  "@
+   tbit.z %0, %I0 = %1, 0
+   xor %0 = 1, %1
+   #
+   #"
+  [(set_attr "itanium_class" "tbit,ilog,unknown,unknown")])
+
+(define_split
+  [(set (match_operand:BI 0 "register_operand" "")
+       (not:BI (match_operand:BI 1 "register_operand" "")))
+   (clobber (match_scratch:BI 2 ""))]
+  "reload_completed
+   && GET_CODE (operands[0]) == REG && PR_REGNO_P (REGNO (operands[0]))
+   && rtx_equal_p (operands[0], operands[1])"
+  [(set (match_dup 4) (match_dup 3))
+   (set (match_dup 0) (const_int 1))
+   (cond_exec (ne (match_dup 2) (const_int 0))
+     (set (match_dup 0) (const_int 0)))
+   (set (match_dup 0) (unspec:BI [(match_dup 0)] UNSPEC_PRED_REL_MUTEX))]
+  "operands[3] = gen_rtx_REG (CCImode, REGNO (operands[1]));
+   operands[4] = gen_rtx_REG (CCImode, REGNO (operands[2]));")
+
+(define_split
+  [(set (match_operand:BI 0 "register_operand" "")
+       (not:BI (match_operand:BI 1 "register_operand" "")))
+   (clobber (match_scratch:BI 2 ""))]
+  "reload_completed
+   && GET_CODE (operands[0]) == REG && PR_REGNO_P (REGNO (operands[0]))
+   && GET_CODE (operands[1]) == REG && PR_REGNO_P (REGNO (operands[1]))
+   && ! rtx_equal_p (operands[0], operands[1])"
+  [(cond_exec (ne (match_dup 1) (const_int 0))
+     (set (match_dup 0) (const_int 0)))
+   (cond_exec (eq (match_dup 1) (const_int 0))
+     (set (match_dup 0) (const_int 1)))
+   (set (match_dup 0) (unspec:BI [(match_dup 0)] UNSPEC_PRED_REL_MUTEX))]
+  "")
+
+(define_insn "*cmpsi_and_0"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (and:BI (match_operator:BI 4 "predicate_operator"
+                 [(match_operand:SI 2 "gr_reg_or_0_operand" "rO")
+                  (match_operand:SI 3 "gr_reg_or_8bit_operand" "rK")])
+               (match_operand:BI 1 "register_operand" "0")))]
+  ""
+  "cmp4.%C4.and.orcm %0, %I0 = %3, %r2"
+  [(set_attr "itanium_class" "icmp")])
+
+(define_insn "*cmpsi_and_1"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (and:BI (match_operator:BI 3 "signed_inequality_operator"
+                 [(match_operand:SI 2 "gr_register_operand" "r")
+                  (const_int 0)])
+               (match_operand:BI 1 "register_operand" "0")))]
+  ""
+  "cmp4.%C3.and.orcm %0, %I0 = r0, %2"
+  [(set_attr "itanium_class" "icmp")])
+
+(define_insn "*cmpsi_andnot_0"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (and:BI (not:BI (match_operator:BI 4 "predicate_operator"
+                        [(match_operand:SI 2 "gr_reg_or_0_operand" "rO")
+                         (match_operand:SI 3 "gr_reg_or_8bit_operand" "rK")]))
+               (match_operand:BI 1 "register_operand" "0")))]
+  ""
+  "cmp4.%C4.or.andcm %I0, %0 = %3, %r2"
+  [(set_attr "itanium_class" "icmp")])
+
+(define_insn "*cmpsi_andnot_1"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (and:BI (not:BI (match_operator:BI 3 "signed_inequality_operator"
+                         [(match_operand:SI 2 "gr_register_operand" "r")
+                          (const_int 0)]))
+               (match_operand:BI 1 "register_operand" "0")))]
+  ""
+  "cmp4.%C3.or.andcm %I0, %0 = r0, %2"
+  [(set_attr "itanium_class" "icmp")])
+
+(define_insn "*cmpdi_and_0"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (and:BI (match_operator:BI 4 "predicate_operator"
+                 [(match_operand:DI 2 "gr_register_operand" "r")
+                  (match_operand:DI 3 "gr_reg_or_8bit_operand" "rK")])
+               (match_operand:BI 1 "register_operand" "0")))]
+  ""
+  "cmp.%C4.and.orcm %0, %I0 = %3, %2"
+  [(set_attr "itanium_class" "icmp")])
+
+(define_insn "*cmpdi_and_1"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (and:BI (match_operator:BI 3 "signed_inequality_operator"
+                 [(match_operand:DI 2 "gr_register_operand" "r")
+                  (const_int 0)])
+               (match_operand:BI 1 "register_operand" "0")))]
+  ""
+  "cmp.%C3.and.orcm %0, %I0 = r0, %2"
+  [(set_attr "itanium_class" "icmp")])
+
+(define_insn "*cmpdi_andnot_0"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (and:BI (not:BI (match_operator:BI 4 "predicate_operator"
+                        [(match_operand:DI 2 "gr_register_operand" "r")
+                         (match_operand:DI 3 "gr_reg_or_8bit_operand" "rK")]))
+               (match_operand:BI 1 "register_operand" "0")))]
+  ""
+  "cmp.%C4.or.andcm %I0, %0 = %3, %2"
+  [(set_attr "itanium_class" "icmp")])
+
+(define_insn "*cmpdi_andnot_1"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (and:BI (not:BI (match_operator:BI 3 "signed_inequality_operator"
+                         [(match_operand:DI 2 "gr_register_operand" "r")
+                          (const_int 0)]))
+               (match_operand:BI 1 "register_operand" "0")))]
+  ""
+  "cmp.%C3.or.andcm %I0, %0 = r0, %2"
+  [(set_attr "itanium_class" "icmp")])
+
+(define_insn "*tbit_and_0"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (and:BI (ne:BI (and:DI (match_operand:DI 1 "gr_register_operand" "r")
+                              (const_int 1))
+                      (const_int 0))
+               (match_operand:BI 2 "register_operand" "0")))]
+  ""
+  "tbit.nz.and.orcm %0, %I0 = %1, 0"
+  [(set_attr "itanium_class" "tbit")])
+
+(define_insn "*tbit_and_1"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (and:BI (eq:BI (and:DI (match_operand:DI 1 "gr_register_operand" "r")
+                              (const_int 1))
+                      (const_int 0))
+               (match_operand:BI 2 "register_operand" "0")))]
+  ""
+  "tbit.z.and.orcm %0, %I0 = %1, 0"
+  [(set_attr "itanium_class" "tbit")])
+
+(define_insn "*tbit_and_2"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (and:BI (ne:BI (zero_extract:DI
+                        (match_operand:DI 1 "gr_register_operand" "r")
+                        (const_int 1)
+                        (match_operand:DI 2 "const_int_operand" "n"))
+                      (const_int 0))
+               (match_operand:BI 3 "register_operand" "0")))]
+  ""
+  "tbit.nz.and.orcm %0, %I0 = %1, %2"
+  [(set_attr "itanium_class" "tbit")])
+
+(define_insn "*tbit_and_3"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (and:BI (eq:BI (zero_extract:DI
+                        (match_operand:DI 1 "gr_register_operand" "r")
+                        (const_int 1)
+                        (match_operand:DI 2 "const_int_operand" "n"))
+                      (const_int 0))
+               (match_operand:BI 3 "register_operand" "0")))]
+  ""
+  "tbit.z.and.orcm %0, %I0 = %1, %2"
+  [(set_attr "itanium_class" "tbit")])
+
+(define_insn "*cmpsi_or_0"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (ior:BI (match_operator:BI 4 "predicate_operator"
+                 [(match_operand:SI 2 "gr_reg_or_0_operand" "rO")
+                  (match_operand:SI 3 "gr_reg_or_8bit_operand" "rK")])
+               (match_operand:BI 1 "register_operand" "0")))]
+  ""
+  "cmp4.%C4.or.andcm %0, %I0 = %3, %r2"
+  [(set_attr "itanium_class" "icmp")])
+
+(define_insn "*cmpsi_or_1"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (ior:BI (match_operator:BI 3 "signed_inequality_operator"
+                 [(match_operand:SI 2 "gr_register_operand" "r")
+                  (const_int 0)])
+               (match_operand:BI 1 "register_operand" "0")))]
+  ""
+  "cmp4.%C3.or.andcm %0, %I0 = r0, %2"
+  [(set_attr "itanium_class" "icmp")])
+
+(define_insn "*cmpsi_orcm_0"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (ior:BI (not:BI (match_operator:BI 4 "predicate_operator"
+                        [(match_operand:SI 2 "gr_reg_or_0_operand" "rO")
+                         (match_operand:SI 3 "gr_reg_or_8bit_operand" "rK")]))
+               (match_operand:BI 1 "register_operand" "0")))]
+  ""
+  "cmp4.%C4.and.orcm %I0, %0 = %3, %r2"
+  [(set_attr "itanium_class" "icmp")])
+
+(define_insn "*cmpsi_orcm_1"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (ior:BI (not:BI (match_operator:BI 3 "signed_inequality_operator"
+                         [(match_operand:SI 2 "gr_register_operand" "r")
+                          (const_int 0)]))
+               (match_operand:BI 1 "register_operand" "0")))]
+  ""
+  "cmp4.%C3.and.orcm %I0, %0 = r0, %2"
+  [(set_attr "itanium_class" "icmp")])
+
+(define_insn "*cmpdi_or_0"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (ior:BI (match_operator:BI 4 "predicate_operator"
+                 [(match_operand:DI 2 "gr_register_operand" "r")
+                  (match_operand:DI 3 "gr_reg_or_8bit_operand" "rK")])
+               (match_operand:BI 1 "register_operand" "0")))]
+  ""
+  "cmp.%C4.or.andcm %0, %I0 = %3, %2"
+  [(set_attr "itanium_class" "icmp")])
+
+(define_insn "*cmpdi_or_1"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (ior:BI (match_operator:BI 3 "signed_inequality_operator"
+                 [(match_operand:DI 2 "gr_register_operand" "r")
+                  (const_int 0)])
+               (match_operand:BI 1 "register_operand" "0")))]
+  ""
+  "cmp.%C3.or.andcm %0, %I0 = r0, %2"
+  [(set_attr "itanium_class" "icmp")])
+
+(define_insn "*cmpdi_orcm_0"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (ior:BI (not:BI (match_operator:BI 4 "predicate_operator"
+                        [(match_operand:DI 2 "gr_register_operand" "r")
+                         (match_operand:DI 3 "gr_reg_or_8bit_operand" "rK")]))
+               (match_operand:BI 1 "register_operand" "0")))]
+  ""
+  "cmp.%C4.and.orcm %I0, %0 = %3, %2"
+  [(set_attr "itanium_class" "icmp")])
+
+(define_insn "*cmpdi_orcm_1"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (ior:BI (not:BI (match_operator:BI 3 "signed_inequality_operator"
+                         [(match_operand:DI 2 "gr_register_operand" "r")
+                          (const_int 0)]))
+               (match_operand:BI 1 "register_operand" "0")))]
+  ""
+  "cmp.%C3.and.orcm %I0, %0 = r0, %2"
+  [(set_attr "itanium_class" "icmp")])
+
+(define_insn "*tbit_or_0"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (ior:BI (ne:BI (and:DI (match_operand:DI 1 "gr_register_operand" "r")
+                              (const_int 1))
+                      (const_int 0))
+               (match_operand:BI 2 "register_operand" "0")))]
+  ""
+  "tbit.nz.or.andcm %0, %I0 = %1, 0"
+  [(set_attr "itanium_class" "tbit")])
+
+(define_insn "*tbit_or_1"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (ior:BI (eq:BI (and:DI (match_operand:DI 1 "gr_register_operand" "r")
+                              (const_int 1))
+                      (const_int 0))
+               (match_operand:BI 2 "register_operand" "0")))]
+  ""
+  "tbit.z.or.andcm %0, %I0 = %1, 0"
+  [(set_attr "itanium_class" "tbit")])
+
+(define_insn "*tbit_or_2"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (ior:BI (ne:BI (zero_extract:DI
+                        (match_operand:DI 1 "gr_register_operand" "r")
+                        (const_int 1)
+                        (match_operand:DI 2 "const_int_operand" "n"))
+                      (const_int 0))
+               (match_operand:BI 3 "register_operand" "0")))]
+  ""
+  "tbit.nz.or.andcm %0, %I0 = %1, %2"
+  [(set_attr "itanium_class" "tbit")])
+
+(define_insn "*tbit_or_3"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (ior:BI (eq:BI (zero_extract:DI
+                        (match_operand:DI 1 "gr_register_operand" "r")
+                        (const_int 1)
+                        (match_operand:DI 2 "const_int_operand" "n"))
+                      (const_int 0))
+               (match_operand:BI 3 "register_operand" "0")))]
+  ""
+  "tbit.z.or.andcm %0, %I0 = %1, %2"
+  [(set_attr "itanium_class" "tbit")])
+
+;; Transform test of and/or of setcc into parallel comparisons.
+
+(define_split
+  [(set (match_operand:BI 0 "register_operand" "")
+       (ne:BI (and:DI (ne:DI (match_operand:BI 2 "register_operand" "")
+                             (const_int 0))
+                      (match_operand:DI 3 "register_operand" ""))
+              (const_int 0)))]
+  ""
+  [(set (match_dup 0)
+       (and:BI (ne:BI (and:DI (match_dup 3) (const_int 1)) (const_int 0))
+               (match_dup 2)))]
+  "")
+
+(define_split
+  [(set (match_operand:BI 0 "register_operand" "")
+       (eq:BI (and:DI (ne:DI (match_operand:BI 2 "register_operand" "")
+                             (const_int 0))
+                      (match_operand:DI 3 "register_operand" ""))
+              (const_int 0)))]
+  ""
+  [(set (match_dup 0)
+       (and:BI (ne:BI (and:DI (match_dup 3) (const_int 1)) (const_int 0))
+               (match_dup 2)))
+   (parallel [(set (match_dup 0) (not:BI (match_dup 0)))
+             (clobber (scratch))])]
+  "")
+
+(define_split
+  [(set (match_operand:BI 0 "register_operand" "")
+       (ne:BI (ior:DI (ne:DI (match_operand:BI 2 "register_operand" "")
+                             (const_int 0))
+                      (match_operand:DI 3 "register_operand" ""))
+              (const_int 0)))]
+  ""
+  [(set (match_dup 0) 
+       (ior:BI (ne:BI (match_dup 3) (const_int 0))
+               (match_dup 2)))]
+  "")
+
+(define_split
+  [(set (match_operand:BI 0 "register_operand" "")
+       (eq:BI (ior:DI (ne:DI (match_operand:BI 2 "register_operand" "")
+                             (const_int 0))
+                      (match_operand:DI 3 "register_operand" ""))
+              (const_int 0)))]
+  ""
+  [(set (match_dup 0) 
+       (ior:BI (ne:BI (match_dup 3) (const_int 0))
+               (match_dup 2)))
+   (parallel [(set (match_dup 0) (not:BI (match_dup 0)))
+             (clobber (scratch))])]
+  "")
+
+;; ??? Incredibly hackish.  Either need four proper patterns with all
+;; the alternatives, or rely on sched1 to split the insn and hope that
+;; nothing bad happens to the comparisons in the meantime.
+;;
+;; Alternately, adjust combine to allow 2->2 and 3->3 splits, assuming
+;; that we're doing height reduction.
+;
+;(define_insn_and_split ""
+;  [(set (match_operand:BI 0 "register_operand" "=c")
+;      (and:BI (and:BI (match_operator:BI 1 "comparison_operator"
+;                        [(match_operand 2 "" "")
+;                         (match_operand 3 "" "")])
+;                      (match_operator:BI 4 "comparison_operator"
+;                        [(match_operand 5 "" "")
+;                         (match_operand 6 "" "")]))
+;              (match_dup 0)))]
+;  "flag_schedule_insns"
+;  "#"
+;  ""
+;  [(set (match_dup 0) (and:BI (match_dup 1) (match_dup 0)))
+;   (set (match_dup 0) (and:BI (match_dup 4) (match_dup 0)))]
+;  "")
+;
+;(define_insn_and_split ""
+;  [(set (match_operand:BI 0 "register_operand" "=c")
+;      (ior:BI (ior:BI (match_operator:BI 1 "comparison_operator"
+;                        [(match_operand 2 "" "")
+;                         (match_operand 3 "" "")])
+;                      (match_operator:BI 4 "comparison_operator"
+;                        [(match_operand 5 "" "")
+;                         (match_operand 6 "" "")]))
+;              (match_dup 0)))]
+;  "flag_schedule_insns"
+;  "#"
+;  ""
+;  [(set (match_dup 0) (ior:BI (match_dup 1) (match_dup 0)))
+;   (set (match_dup 0) (ior:BI (match_dup 4) (match_dup 0)))]
+;  "")
+;
+;(define_split
+;  [(set (match_operand:BI 0 "register_operand" "")
+;      (and:BI (and:BI (match_operator:BI 1 "comparison_operator"
+;                        [(match_operand 2 "" "")
+;                         (match_operand 3 "" "")])
+;                      (match_operand:BI 7 "register_operand" ""))
+;              (and:BI (match_operator:BI 4 "comparison_operator"
+;                        [(match_operand 5 "" "")
+;                         (match_operand 6 "" "")])
+;                      (match_operand:BI 8 "register_operand" ""))))]
+;  ""
+;  [(set (match_dup 0) (and:BI (match_dup 7) (match_dup 8)))
+;   (set (match_dup 0) (and:BI (and:BI (match_dup 1) (match_dup 4))
+;                            (match_dup 0)))]
+;  "")
+;
+;(define_split
+;  [(set (match_operand:BI 0 "register_operand" "")
+;      (ior:BI (ior:BI (match_operator:BI 1 "comparison_operator"
+;                        [(match_operand 2 "" "")
+;                         (match_operand 3 "" "")])
+;                      (match_operand:BI 7 "register_operand" ""))
+;              (ior:BI (match_operator:BI 4 "comparison_operator"
+;                        [(match_operand 5 "" "")
+;                         (match_operand 6 "" "")])
+;                      (match_operand:BI 8 "register_operand" ""))))]
+;  ""
+;  [(set (match_dup 0) (ior:BI (match_dup 7) (match_dup 8)))
+;   (set (match_dup 0) (ior:BI (ior:BI (match_dup 1) (match_dup 4))
+;                            (match_dup 0)))]
+;  "")
+
+;; Try harder to avoid predicate copies by duplicating compares.
+;; Note that we'll have already split the predicate copy, which
+;; is kind of a pain, but oh well.
+
+(define_peephole2
+  [(set (match_operand:BI 0 "register_operand" "")
+       (match_operand:BI 1 "comparison_operator" ""))
+   (set (match_operand:CCI 2 "register_operand" "")
+       (match_operand:CCI 3 "register_operand" ""))
+   (set (match_operand:CCI 4 "register_operand" "")
+       (match_operand:CCI 5 "register_operand" ""))
+   (set (match_operand:BI 6 "register_operand" "")
+       (unspec:BI [(match_dup 6)] UNSPEC_PRED_REL_MUTEX))]
+  "REGNO (operands[3]) == REGNO (operands[0])
+   && REGNO (operands[4]) == REGNO (operands[0]) + 1
+   && REGNO (operands[4]) == REGNO (operands[2]) + 1
+   && REGNO (operands[6]) == REGNO (operands[2])"
+  [(set (match_dup 0) (match_dup 1))
+   (set (match_dup 6) (match_dup 7))]
+  "operands[7] = copy_rtx (operands[1]);")
+\f
+;; ::::::::::::::::::::
+;; ::
+;; :: 16 bit Integer arithmetic
+;; ::
+;; ::::::::::::::::::::
+
+(define_insn "mulhi3"
+  [(set (match_operand:HI 0 "gr_register_operand" "=r")
+       (mult:HI (match_operand:HI 1 "gr_register_operand" "r")
+                (match_operand:HI 2 "gr_register_operand" "r")))]
+  ""
+  "pmpy2.r %0 = %1, %2"
+  [(set_attr "itanium_class" "mmmul")])
 
 \f
 ;; ::::::::::::::::::::
 ;; ::::::::::::::::::::
 
 (define_insn "addsi3"
-  [(set (match_operand:SI 0 "register_operand" "=r,r,r")
-       (plus:SI (match_operand:SI 1 "register_operand" "%r,r,a")
-                (match_operand:SI 2 "reg_or_22bit_operand" "r,I,J")))]
+  [(set (match_operand:SI 0 "gr_register_operand" "=r,r,r")
+       (plus:SI (match_operand:SI 1 "gr_register_operand" "%r,r,a")
+                (match_operand:SI 2 "gr_reg_or_22bit_operand" "r,I,J")))]
   ""
   "@
-  add %0 = %1, %2
-  adds %0 = %2, %1
-  addl %0 = %2, %1"
-  [(set_attr "type" "A")])
+   add %0 = %1, %2
+   adds %0 = %2, %1
+   addl %0 = %2, %1"
+  [(set_attr "itanium_class" "ialu")])
 
 (define_insn "*addsi3_plus1"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-       (plus:SI (plus:SI (match_operand:SI 1 "register_operand" "r")
-                         (match_operand:SI 2 "register_operand" "r"))
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+       (plus:SI (plus:SI (match_operand:SI 1 "gr_register_operand" "r")
+                         (match_operand:SI 2 "gr_register_operand" "r"))
                 (const_int 1)))]
   ""
   "add %0 = %1, %2, 1"
-  [(set_attr "type" "A")])
+  [(set_attr "itanium_class" "ialu")])
 
 (define_insn "*addsi3_plus1_alt"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-       (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+       (plus:SI (mult:SI (match_operand:SI 1 "gr_register_operand" "r")
                          (const_int 2))
                 (const_int 1)))]
   ""
   "add %0 = %1, %1, 1"
-  [(set_attr "type" "A")])
+  [(set_attr "itanium_class" "ialu")])
 
 (define_insn "*addsi3_shladd"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-       (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r")
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+       (plus:SI (mult:SI (match_operand:SI 1 "gr_register_operand" "r")
                          (match_operand:SI 2 "shladd_operand" "n"))
-                (match_operand:SI 3 "register_operand" "r")))]
+                (match_operand:SI 3 "gr_register_operand" "r")))]
   ""
   "shladd %0 = %1, %S2, %3"
-  [(set_attr "type" "A")])
+  [(set_attr "itanium_class" "ialu")])
 
 (define_insn "subsi3"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-       (minus:SI (match_operand:SI 1 "reg_or_8bit_operand" "rK")
-                 (match_operand:SI 2 "register_operand" "r")))]
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+       (minus:SI (match_operand:SI 1 "gr_reg_or_8bit_operand" "rK")
+                 (match_operand:SI 2 "gr_register_operand" "r")))]
   ""
   "sub %0 = %1, %2"
-  [(set_attr "type" "A")])
+  [(set_attr "itanium_class" "ialu")])
 
 (define_insn "*subsi3_minus1"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-       (plus:SI (not:SI (match_operand:SI 1 "register_operand" "r"))
-                (match_operand:SI 2 "register_operand" "r")))]
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+       (plus:SI (not:SI (match_operand:SI 1 "gr_register_operand" "r"))
+                (match_operand:SI 2 "gr_register_operand" "r")))]
   ""
   "sub %0 = %2, %1, 1"
-  [(set_attr "type" "A")])
+  [(set_attr "itanium_class" "ialu")])
 
 ;; ??? Could add maddsi3 patterns patterned after the madddi3 patterns.
 
 (define_insn "mulsi3"
-  [(set (match_operand:SI 0 "register_operand" "=f")
-       (mult:SI (match_operand:SI 1 "register_operand" "f")
-                (match_operand:SI 2 "nonmemory_operand" "f")))]
+  [(set (match_operand:SI 0 "fr_register_operand" "=f")
+       (mult:SI (match_operand:SI 1 "grfr_register_operand" "f")
+                (match_operand:SI 2 "grfr_register_operand" "f")))]
+  ""
+  "xmpy.l %0 = %1, %2"
+  [(set_attr "itanium_class" "xmpy")])
+
+(define_insn "maddsi4"
+  [(set (match_operand:SI 0 "fr_register_operand" "=f")
+       (plus:SI (mult:SI (match_operand:SI 1 "grfr_register_operand" "f")
+                         (match_operand:SI 2 "grfr_register_operand" "f"))
+                (match_operand:SI 3 "grfr_register_operand" "f")))]
   ""
-  "xma.l %0 = %1, %2, f0%B0"
-  [(set_attr "type" "F")])
+  "xma.l %0 = %1, %2, %3"
+  [(set_attr "itanium_class" "xmpy")])
 
 (define_insn "negsi2"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-       (neg:SI (match_operand:SI 1 "register_operand" "r")))]
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+       (neg:SI (match_operand:SI 1 "gr_register_operand" "r")))]
   ""
   "sub %0 = r0, %1"
-  [(set_attr "type" "A")])
+  [(set_attr "itanium_class" "ialu")])
 
 (define_expand "abssi2"
   [(set (match_dup 2)
-       (ge:CC (match_operand:SI 1 "register_operand" "") (const_int 0)))
-   (set (match_operand:SI 0 "register_operand" "")
-       (if_then_else:SI (eq:CC (match_dup 2) (const_int 0))
+       (ge:BI (match_operand:SI 1 "gr_register_operand" "") (const_int 0)))
+   (set (match_operand:SI 0 "gr_register_operand" "")
+       (if_then_else:SI (eq (match_dup 2) (const_int 0))
                         (neg:SI (match_dup 1))
                         (match_dup 1)))]
   ""
-  "
-{
-  operands[2] = gen_reg_rtx (CCmode);
-}")
+  { operands[2] = gen_reg_rtx (BImode); })
 
 (define_expand "sminsi3"
   [(set (match_dup 3)
-       (ge:CC (match_operand:SI 1 "register_operand" "")
-              (match_operand:SI 2 "register_operand" "")))
-   (set (match_operand:SI 0 "register_operand" "")
-       (if_then_else:SI (ne:CC (match_dup 3) (const_int 0))
+       (ge:BI (match_operand:SI 1 "gr_register_operand" "")
+              (match_operand:SI 2 "gr_register_operand" "")))
+   (set (match_operand:SI 0 "gr_register_operand" "")
+       (if_then_else:SI (ne (match_dup 3) (const_int 0))
                         (match_dup 2) (match_dup 1)))]
   ""
-  "
-{
-  operands[3] = gen_reg_rtx (CCmode);
-}")
+  { operands[3] = gen_reg_rtx (BImode); })
 
 (define_expand "smaxsi3"
   [(set (match_dup 3)
-       (ge:CC (match_operand:SI 1 "register_operand" "")
-              (match_operand:SI 2 "register_operand" "")))
-   (set (match_operand:SI 0 "register_operand" "")
-       (if_then_else:SI (ne:CC (match_dup 3) (const_int 0))
+       (ge:BI (match_operand:SI 1 "gr_register_operand" "")
+              (match_operand:SI 2 "gr_register_operand" "")))
+   (set (match_operand:SI 0 "gr_register_operand" "")
+       (if_then_else:SI (ne (match_dup 3) (const_int 0))
                         (match_dup 1) (match_dup 2)))]
   ""
-  "
-{
-  operands[3] = gen_reg_rtx (CCmode);
-}")
+  { operands[3] = gen_reg_rtx (BImode); })
 
 (define_expand "uminsi3"
   [(set (match_dup 3)
-       (geu:CC (match_operand:SI 1 "register_operand" "")
-               (match_operand:SI 2 "register_operand" "")))
-   (set (match_operand:SI 0 "register_operand" "")
-       (if_then_else:SI (ne:CC (match_dup 3) (const_int 0))
+       (geu:BI (match_operand:SI 1 "gr_register_operand" "")
+               (match_operand:SI 2 "gr_register_operand" "")))
+   (set (match_operand:SI 0 "gr_register_operand" "")
+       (if_then_else:SI (ne (match_dup 3) (const_int 0))
                         (match_dup 2) (match_dup 1)))]
   ""
-  "
-{
-  operands[3] = gen_reg_rtx (CCmode);
-}")
+  { operands[3] = gen_reg_rtx (BImode); })
 
 (define_expand "umaxsi3"
   [(set (match_dup 3)
-       (geu:CC (match_operand:SI 1 "register_operand" "")
-               (match_operand:SI 2 "register_operand" "")))
-   (set (match_operand:SI 0 "register_operand" "")
-       (if_then_else:SI (ne:CC (match_dup 3) (const_int 0))
+       (geu:BI (match_operand:SI 1 "gr_register_operand" "")
+               (match_operand:SI 2 "gr_register_operand" "")))
+   (set (match_operand:SI 0 "gr_register_operand" "")
+       (if_then_else:SI (ne (match_dup 3) (const_int 0))
                         (match_dup 1) (match_dup 2)))]
   ""
-  "
+  { operands[3] = gen_reg_rtx (BImode); })
+
+(define_expand "divsi3"
+  [(set (match_operand:SI 0 "register_operand" "")
+       (div:SI (match_operand:SI 1 "general_operand" "")
+               (match_operand:SI 2 "general_operand" "")))]
+  "TARGET_INLINE_INT_DIV"
+{
+  rtx op1_xf, op2_xf, op0_xf, op0_di, twon34, twon34_exp;
+
+  op0_xf = gen_reg_rtx (XFmode);
+  op0_di = gen_reg_rtx (DImode);
+
+  if (CONSTANT_P (operands[1]))
+    operands[1] = force_reg (SImode, operands[1]);
+  op1_xf = gen_reg_rtx (XFmode);
+  expand_float (op1_xf, operands[1], 0);
+
+  if (CONSTANT_P (operands[2]))
+    operands[2] = force_reg (SImode, operands[2]);
+  op2_xf = gen_reg_rtx (XFmode);
+  expand_float (op2_xf, operands[2], 0);
+
+  /* 2^-34 */
+  twon34_exp = gen_reg_rtx (DImode);
+  emit_move_insn (twon34_exp, GEN_INT (65501));
+  twon34 = gen_reg_rtx (XFmode);
+  emit_insn (gen_setf_exp_xf (twon34, twon34_exp));
+
+  emit_insn (gen_cond_trap (EQ, operands[2], CONST0_RTX (SImode),
+                           CONST1_RTX (SImode)));
+  
+  emit_insn (gen_divsi3_internal (op0_xf, op1_xf, op2_xf, twon34));
+
+  emit_insn (gen_fix_truncxfdi2_alts (op0_di, op0_xf, const1_rtx));
+  emit_move_insn (operands[0], gen_lowpart (SImode, op0_di));
+  DONE;
+})
+
+(define_expand "modsi3"
+  [(set (match_operand:SI 0 "register_operand" "")
+       (mod:SI (match_operand:SI 1 "general_operand" "")
+               (match_operand:SI 2 "general_operand" "")))]
+  "TARGET_INLINE_INT_DIV"
+{
+  rtx op2_neg, op1_di, div;
+
+  div = gen_reg_rtx (SImode);
+  emit_insn (gen_divsi3 (div, operands[1], operands[2]));
+
+  op2_neg = expand_unop (SImode, neg_optab, operands[2], NULL_RTX, 0);
+
+  /* This is a trick to get us to reuse the value that we're sure to
+     have already copied to the FP regs.  */
+  op1_di = gen_reg_rtx (DImode);
+  convert_move (op1_di, operands[1], 0);
+
+  emit_insn (gen_maddsi4 (operands[0], div, op2_neg,
+                         gen_lowpart (SImode, op1_di)));
+  DONE;
+})
+
+(define_expand "udivsi3"
+  [(set (match_operand:SI 0 "register_operand" "")
+       (udiv:SI (match_operand:SI 1 "general_operand" "")
+                (match_operand:SI 2 "general_operand" "")))]
+  "TARGET_INLINE_INT_DIV"
+{
+  rtx op1_xf, op2_xf, op0_xf, op0_di, twon34, twon34_exp;
+
+  op0_xf = gen_reg_rtx (XFmode);
+  op0_di = gen_reg_rtx (DImode);
+
+  if (CONSTANT_P (operands[1]))
+    operands[1] = force_reg (SImode, operands[1]);
+  op1_xf = gen_reg_rtx (XFmode);
+  expand_float (op1_xf, operands[1], 1);
+
+  if (CONSTANT_P (operands[2]))
+    operands[2] = force_reg (SImode, operands[2]);
+  op2_xf = gen_reg_rtx (XFmode);
+  expand_float (op2_xf, operands[2], 1);
+
+  /* 2^-34 */
+  twon34_exp = gen_reg_rtx (DImode);
+  emit_move_insn (twon34_exp, GEN_INT (65501));
+  twon34 = gen_reg_rtx (XFmode);
+  emit_insn (gen_setf_exp_xf (twon34, twon34_exp));
+
+  emit_insn (gen_cond_trap (EQ, operands[2], CONST0_RTX (SImode),
+                           CONST1_RTX (SImode)));
+  
+  emit_insn (gen_divsi3_internal (op0_xf, op1_xf, op2_xf, twon34));
+
+  emit_insn (gen_fixuns_truncxfdi2_alts (op0_di, op0_xf, const1_rtx));
+  emit_move_insn (operands[0], gen_lowpart (SImode, op0_di));
+  DONE;
+})
+
+(define_expand "umodsi3"
+  [(set (match_operand:SI 0 "register_operand" "")
+       (umod:SI (match_operand:SI 1 "general_operand" "")
+                (match_operand:SI 2 "general_operand" "")))]
+  "TARGET_INLINE_INT_DIV"
 {
-  operands[3] = gen_reg_rtx (CCmode);
-}")
+  rtx op2_neg, op1_di, div;
+
+  div = gen_reg_rtx (SImode);
+  emit_insn (gen_udivsi3 (div, operands[1], operands[2]));
+
+  op2_neg = expand_unop (SImode, neg_optab, operands[2], NULL_RTX, 0);
+
+  /* This is a trick to get us to reuse the value that we're sure to
+     have already copied to the FP regs.  */
+  op1_di = gen_reg_rtx (DImode);
+  convert_move (op1_di, operands[1], 1);
 
+  emit_insn (gen_maddsi4 (operands[0], div, op2_neg,
+                         gen_lowpart (SImode, op1_di)));
+  DONE;
+})
+
+(define_insn_and_split "divsi3_internal"
+  [(set (match_operand:XF 0 "fr_register_operand" "=&f")
+       (float:XF (div:SI (match_operand:XF 1 "fr_register_operand" "f")
+                         (match_operand:XF 2 "fr_register_operand" "f"))))
+   (clobber (match_scratch:XF 4 "=&f"))
+   (clobber (match_scratch:XF 5 "=&f"))
+   (clobber (match_scratch:BI 6 "=c"))
+   (use (match_operand:XF 3 "fr_register_operand" "f"))]
+  "TARGET_INLINE_INT_DIV"
+  "#"
+  "&& reload_completed"
+  [(parallel [(set (match_dup 0) (div:XF (const_int 1) (match_dup 2)))
+             (set (match_dup 6) (unspec:BI [(match_dup 1) (match_dup 2)]
+                                           UNSPEC_FR_RECIP_APPROX))
+             (use (const_int 1))])
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 4) (mult:XF (match_dup 1) (match_dup 0)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 5)
+                    (minus:XF (match_dup 7)
+                              (mult:XF (match_dup 2) (match_dup 0))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 4)
+                    (plus:XF (mult:XF (match_dup 5) (match_dup 4))
+                             (match_dup 4)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 5)
+                    (plus:XF (mult:XF (match_dup 5) (match_dup 5))
+                             (match_dup 3)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 0)
+                    (plus:XF (mult:XF (match_dup 5) (match_dup 4))
+                             (match_dup 4)))
+               (use (const_int 1))]))
+  ] 
+  "operands[7] = CONST1_RTX (XFmode);"
+  [(set_attr "predicable" "no")])
 \f
 ;; ::::::::::::::::::::
 ;; ::
 ;; ::::::::::::::::::::
 
 (define_insn "adddi3"
-  [(set (match_operand:DI 0 "register_operand" "=r,r,r")
-       (plus:DI (match_operand:DI 1 "register_operand" "%r,r,a")
-                (match_operand:DI 2 "reg_or_22bit_operand" "r,I,J")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r,r,r")
+       (plus:DI (match_operand:DI 1 "gr_register_operand" "%r,r,a")
+                (match_operand:DI 2 "gr_reg_or_22bit_operand" "r,I,J")))]
   ""
   "@
-  add %0 = %1, %2
-  adds %0 = %2, %1
-  addl %0 = %2, %1"
-  [(set_attr "type" "A")])
+   add %0 = %1, %2
+   adds %0 = %2, %1
+   addl %0 = %2, %1"
+  [(set_attr "itanium_class" "ialu")])
 
 (define_insn "*adddi3_plus1"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (plus:DI (plus:DI (match_operand:DI 1 "register_operand" "r")
-                         (match_operand:DI 2 "register_operand" "r"))
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (plus:DI (plus:DI (match_operand:DI 1 "gr_register_operand" "r")
+                         (match_operand:DI 2 "gr_register_operand" "r"))
                 (const_int 1)))]
   ""
   "add %0 = %1, %2, 1"
-  [(set_attr "type" "A")])
+  [(set_attr "itanium_class" "ialu")])
 
 ;; This has some of the same problems as shladd.  We let the shladd
 ;; eliminator hack handle it, which results in the 1 being forced into
 ;; a register, but not more ugliness here.
 (define_insn "*adddi3_plus1_alt"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (plus:DI (mult:DI (match_operand:DI 1 "register_operand" "r")
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (plus:DI (mult:DI (match_operand:DI 1 "gr_register_operand" "r")
                          (const_int 2))
                 (const_int 1)))]
   ""
   "add %0 = %1, %1, 1"
-  [(set_attr "type" "A")])
+  [(set_attr "itanium_class" "ialu")])
 
 (define_insn "subdi3"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (minus:DI (match_operand:DI 1 "reg_or_8bit_operand" "rK")
-                 (match_operand:DI 2 "register_operand" "r")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (minus:DI (match_operand:DI 1 "gr_reg_or_8bit_operand" "rK")
+                 (match_operand:DI 2 "gr_register_operand" "r")))]
   ""
   "sub %0 = %1, %2"
-  [(set_attr "type" "A")])
+  [(set_attr "itanium_class" "ialu")])
 
 (define_insn "*subdi3_minus1"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (plus:DI (not:DI (match_operand:DI 1 "register_operand" "r"))
-                (match_operand:DI 2 "register_operand" "r")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (plus:DI (not:DI (match_operand:DI 1 "gr_register_operand" "r"))
+                (match_operand:DI 2 "gr_register_operand" "r")))]
   ""
   "sub %0 = %2, %1, 1"
-  [(set_attr "type" "A")])
+  [(set_attr "itanium_class" "ialu")])
 
+;; ??? Use grfr instead of fr because of virtual register elimination
+;; and silly test cases multiplying by the frame pointer.
 (define_insn "muldi3"
-  [(set (match_operand:DI 0 "register_operand" "=f")
-       (mult:DI (match_operand:DI 1 "register_operand" "f")
-                (match_operand:DI 2 "register_operand" "f")))]
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
+       (mult:DI (match_operand:DI 1 "grfr_register_operand" "f")
+                (match_operand:DI 2 "grfr_register_operand" "f")))]
   ""
-  "xma.l %0 = %1, %2, f0%B0"
-  [(set_attr "type" "F")])
+  "xmpy.l %0 = %1, %2"
+  [(set_attr "itanium_class" "xmpy")])
 
 ;; ??? If operand 3 is an eliminable reg, then register elimination causes the
 ;; same problem that we have with shladd below.  Unfortunately, this case is
 
 ;; ??? Maybe we should change how adds are canonicalized.
 
-(define_insn "*madddi3"
-  [(set (match_operand:DI 0 "register_operand" "=f")
-       (plus:DI (mult:DI (match_operand:DI 1 "register_operand" "f")
-                         (match_operand:DI 2 "register_operand" "f"))
-                (match_operand:DI 3 "register_operand" "f")))
+(define_insn "madddi4"
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
+       (plus:DI (mult:DI (match_operand:DI 1 "grfr_register_operand" "f")
+                         (match_operand:DI 2 "grfr_register_operand" "f"))
+                (match_operand:DI 3 "grfr_register_operand" "f")))
    (clobber (match_scratch:DI 4 "=X"))]
   ""
-  "xma.l %0 = %1, %2, %3%B0"
-  [(set_attr "type" "F")])
+  "xma.l %0 = %1, %2, %3"
+  [(set_attr "itanium_class" "xmpy")])
 
 ;; This can be created by register elimination if operand3 of shladd is an
 ;; eliminable register or has reg_equiv_constant set.
 
 ;; We have to use nonmemory_operand for operand 4, to ensure that the
 ;; validate_changes call inside eliminate_regs will always succeed.  If it
-;; doesn't succeed, then this remain a madddi3 pattern, and will be reloaded
+;; doesn't succeed, then this remain a madddi4 pattern, and will be reloaded
 ;; incorrectly.
 
-(define_insn "*madddi3_elim"
+(define_insn "*madddi4_elim"
   [(set (match_operand:DI 0 "register_operand" "=&r")
        (plus:DI (plus:DI (mult:DI (match_operand:DI 1 "register_operand" "f")
                                   (match_operand:DI 2 "register_operand" "f"))
    (clobber (match_scratch:DI 5 "=f"))]
   "reload_in_progress"
   "#"
-  [(set_attr "type" "unknown")])
-
-;; ??? Need to emit an instruction group barrier here because this gets split
-;; after md_reorg.
+  [(set_attr "itanium_class" "unknown")])
 
 (define_split
   [(set (match_operand:DI 0 "register_operand" "")
        (plus:DI (plus:DI (mult:DI (match_operand:DI 1 "register_operand" "")
                                   (match_operand:DI 2 "register_operand" ""))
                          (match_operand:DI 3 "register_operand" ""))
-                (match_operand:DI 4 "reg_or_14bit_operand" "")))
+                (match_operand:DI 4 "gr_reg_or_14bit_operand" "")))
    (clobber (match_scratch:DI 5 ""))]
   "reload_completed"
   [(parallel [(set (match_dup 5) (plus:DI (mult:DI (match_dup 1) (match_dup 2))
                                          (match_dup 3)))
              (clobber (match_dup 0))])
-   (unspec_volatile [(const_int 0)] 2)
    (set (match_dup 0) (match_dup 5))
-   (unspec_volatile [(const_int 0)] 2)
    (set (match_dup 0) (plus:DI (match_dup 0) (match_dup 4)))]
   "")
 
-;; ??? There are highpart multiply and add instructions, but we have no way
-;; to generate them.
-
 (define_insn "smuldi3_highpart"
-  [(set (match_operand:DI 0 "register_operand" "=f")
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
        (truncate:DI
         (lshiftrt:TI
-         (mult:TI (sign_extend:TI (match_operand:DI 1 "register_operand" "f"))
-                  (sign_extend:TI (match_operand:DI 2 "register_operand" "f")))
+         (mult:TI (sign_extend:TI
+                    (match_operand:DI 1 "fr_register_operand" "f"))
+                  (sign_extend:TI
+                    (match_operand:DI 2 "fr_register_operand" "f")))
          (const_int 64))))]
   ""
-  "xma.h %0 = %1, %2, f0%B0"
-  [(set_attr "type" "F")])
+  "xmpy.h %0 = %1, %2"
+  [(set_attr "itanium_class" "xmpy")])
 
 (define_insn "umuldi3_highpart"
-  [(set (match_operand:DI 0 "register_operand" "=f")
+  [(set (match_operand:DI 0 "fr_register_operand" "=f")
        (truncate:DI
         (lshiftrt:TI
-         (mult:TI (zero_extend:TI (match_operand:DI 1 "register_operand" "f"))
-                  (zero_extend:TI (match_operand:DI 2 "register_operand" "f")))
+         (mult:TI (zero_extend:TI
+                    (match_operand:DI 1 "fr_register_operand" "f"))
+                  (zero_extend:TI
+                    (match_operand:DI 2 "fr_register_operand" "f")))
          (const_int 64))))]
   ""
-  "xma.hu %0 = %1, %2, f0%B0"
-  [(set_attr "type" "F")])
+  "xmpy.hu %0 = %1, %2"
+  [(set_attr "itanium_class" "xmpy")])
 
 (define_insn "negdi2"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (neg:DI (match_operand:DI 1 "register_operand" "r")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (neg:DI (match_operand:DI 1 "gr_register_operand" "r")))]
   ""
   "sub %0 = r0, %1"
-  [(set_attr "type" "A")])
+  [(set_attr "itanium_class" "ialu")])
 
 (define_expand "absdi2"
   [(set (match_dup 2)
-       (ge:CC (match_operand:DI 1 "register_operand" "") (const_int 0)))
-   (set (match_operand:DI 0 "register_operand" "")
-       (if_then_else:DI (eq:CC (match_dup 2) (const_int 0))
+       (ge:BI (match_operand:DI 1 "gr_register_operand" "") (const_int 0)))
+   (set (match_operand:DI 0 "gr_register_operand" "")
+       (if_then_else:DI (eq (match_dup 2) (const_int 0))
                         (neg:DI (match_dup 1))
                         (match_dup 1)))]
   ""
-  "
-{
-  operands[2] = gen_reg_rtx (CCmode);
-}")
+  { operands[2] = gen_reg_rtx (BImode); })
 
 (define_expand "smindi3"
   [(set (match_dup 3)
-       (ge:CC (match_operand:DI 1 "register_operand" "")
-              (match_operand:DI 2 "register_operand" "")))
-   (set (match_operand:DI 0 "register_operand" "")
-       (if_then_else:DI (ne:CC (match_dup 3) (const_int 0))
+       (ge:BI (match_operand:DI 1 "gr_register_operand" "")
+              (match_operand:DI 2 "gr_register_operand" "")))
+   (set (match_operand:DI 0 "gr_register_operand" "")
+       (if_then_else:DI (ne (match_dup 3) (const_int 0))
                         (match_dup 2) (match_dup 1)))]
   ""
-  "
-{
-  operands[3] = gen_reg_rtx (CCmode);
-}")
+  { operands[3] = gen_reg_rtx (BImode); })
 
 (define_expand "smaxdi3"
   [(set (match_dup 3)
-       (ge:CC (match_operand:DI 1 "register_operand" "")
-              (match_operand:DI 2 "register_operand" "")))
-   (set (match_operand:DI 0 "register_operand" "")
-       (if_then_else:DI (ne:CC (match_dup 3) (const_int 0))
+       (ge:BI (match_operand:DI 1 "gr_register_operand" "")
+              (match_operand:DI 2 "gr_register_operand" "")))
+   (set (match_operand:DI 0 "gr_register_operand" "")
+       (if_then_else:DI (ne (match_dup 3) (const_int 0))
                         (match_dup 1) (match_dup 2)))]
   ""
-  "
-{
-  operands[3] = gen_reg_rtx (CCmode);
-}")
+  { operands[3] = gen_reg_rtx (BImode); })
 
 (define_expand "umindi3"
   [(set (match_dup 3)
-       (geu:CC (match_operand:DI 1 "register_operand" "")
-               (match_operand:DI 2 "register_operand" "")))
-   (set (match_operand:DI 0 "register_operand" "")
-       (if_then_else:DI (ne:CC (match_dup 3) (const_int 0))
+       (geu:BI (match_operand:DI 1 "gr_register_operand" "")
+               (match_operand:DI 2 "gr_register_operand" "")))
+   (set (match_operand:DI 0 "gr_register_operand" "")
+       (if_then_else:DI (ne (match_dup 3) (const_int 0))
                         (match_dup 2) (match_dup 1)))]
   ""
-  "
-{
-  operands[3] = gen_reg_rtx (CCmode);
-}")
+  { operands[3] = gen_reg_rtx (BImode); })
 
 (define_expand "umaxdi3"
   [(set (match_dup 3)
-       (geu:CC (match_operand:DI 1 "register_operand" "")
-               (match_operand:DI 2 "register_operand" "")))
-   (set (match_operand:DI 0 "register_operand" "")
-       (if_then_else:DI (ne:CC (match_dup 3) (const_int 0))
+       (geu:BI (match_operand:DI 1 "gr_register_operand" "")
+               (match_operand:DI 2 "gr_register_operand" "")))
+   (set (match_operand:DI 0 "gr_register_operand" "")
+       (if_then_else:DI (ne (match_dup 3) (const_int 0))
                         (match_dup 1) (match_dup 2)))]
   ""
-  "
-{
-  operands[3] = gen_reg_rtx (CCmode);
-}")
+  { operands[3] = gen_reg_rtx (BImode); })
 
 (define_expand "ffsdi2"
   [(set (match_dup 6)
-       (eq:CC (match_operand:DI 1 "register_operand" "") (const_int 0)))
+       (eq:BI (match_operand:DI 1 "gr_register_operand" "") (const_int 0)))
    (set (match_dup 2) (plus:DI (match_dup 1) (const_int -1)))
    (set (match_dup 5) (const_int 0))
    (set (match_dup 3) (xor:DI (match_dup 1) (match_dup 2)))
-   (set (match_dup 4) (unspec:DI [(match_dup 3)] 8))
-   (set (match_operand:DI 0 "register_operand" "")
-       (if_then_else:DI (ne:CC (match_dup 6) (const_int 0))
+   (set (match_dup 4) (popcount:DI (match_dup 3)))
+   (set (match_operand:DI 0 "gr_register_operand" "")
+       (if_then_else:DI (ne (match_dup 6) (const_int 0))
                         (match_dup 5) (match_dup 4)))]
   ""
-  "
 {
   operands[2] = gen_reg_rtx (DImode);
   operands[3] = gen_reg_rtx (DImode);
   operands[4] = gen_reg_rtx (DImode);
   operands[5] = gen_reg_rtx (DImode);
-  operands[6] = gen_reg_rtx (CCmode);
-}")
+  operands[6] = gen_reg_rtx (BImode);
+})
 
-(define_insn "*popcnt"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (unspec:DI [(match_operand:DI 1 "register_operand" "r")] 8))]
+(define_expand "ctzdi2"
+  [(set (match_dup 2) (plus:DI (match_operand:DI 1 "gr_register_operand" "")
+                              (const_int -1)))
+   (set (match_dup 3) (not:DI (match_dup 1)))
+   (set (match_dup 4) (and:DI (match_dup 2) (match_dup 3)))
+   (set (match_operand:DI 0 "gr_register_operand" "")
+       (popcount:DI (match_dup 4)))]
   ""
-  "popcnt %0 = %1"
-  [(set_attr "type" "I")])
-
-\f
-;; ::::::::::::::::::::
-;; ::
-;; :: 32 bit floating point arithmetic
-;; ::
-;; ::::::::::::::::::::
+{
+  operands[2] = gen_reg_rtx (DImode);
+  operands[3] = gen_reg_rtx (DImode);
+  operands[4] = gen_reg_rtx (DImode);
+})
 
-(define_insn "addsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-       (plus:SF (match_operand:SF 1 "register_operand" "%f")
-                (match_operand:SF 2 "reg_or_fp01_operand" "fG")))]
+;; Note the computation here is op0 = 63 - (exp - 0xffff).
+(define_expand "clzdi2"
+  [(set (match_dup 2)
+       (unsigned_float:XF (match_operand:DI 1 "fr_register_operand" "")))
+   (set (match_dup 3)
+       (unspec:DI [(match_dup 2)] UNSPEC_GETF_EXP))
+   (set (match_dup 4) (const_int 65598))
+   (set (match_operand:DI 0 "gr_register_operand" "")
+       (minus:DI (match_dup 4) (match_dup 3)))]
   ""
-  "fadd.s %0 = %1, %F2%B0"
-  [(set_attr "type" "F")])
+{
+  operands[2] = gen_reg_rtx (XFmode);
+  operands[3] = gen_reg_rtx (DImode);
+  operands[4] = gen_reg_rtx (DImode);
+})
 
-(define_insn "subsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-       (minus:SF (match_operand:SF 1 "reg_or_fp01_operand" "fG")
-                 (match_operand:SF 2 "reg_or_fp01_operand" "fG")))]
+(define_insn "popcountdi2"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (popcount:DI (match_operand:DI 1 "gr_register_operand" "r")))]
   ""
-  "fsub.s %0 = %F1, %F2%B0"
-  [(set_attr "type" "F")])
+  "popcnt %0 = %1"
+  [(set_attr "itanium_class" "mmmul")])
 
-(define_insn "mulsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-       (mult:SF (match_operand:SF 1 "register_operand" "%f")
-                (match_operand:SF 2 "register_operand" "f")))]
+(define_insn "*getf_exp_xf"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (unspec:DI [(match_operand:XF 1 "fr_register_operand" "f")]
+                  UNSPEC_GETF_EXP))]
   ""
-  "fmpy.s %0 = %1, %2%B0"
-  [(set_attr "type" "F")])
+  "getf.exp %0 = %1"
+  [(set_attr "itanium_class" "frfr")])
 
-(define_insn "abssf2"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-       (abs:SF (match_operand:SF 1 "register_operand" "f")))]
-  ""
-  "fabs %0 = %1%B0"
-  [(set_attr "type" "F")])
+(define_expand "divdi3"
+  [(set (match_operand:DI 0 "register_operand" "")
+       (div:DI (match_operand:DI 1 "general_operand" "")
+               (match_operand:DI 2 "general_operand" "")))]
+  "TARGET_INLINE_INT_DIV"
+{
+  rtx op1_xf, op2_xf, op0_xf;
 
-(define_insn "negsf2"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-       (neg:SF (match_operand:SF 1 "register_operand" "f")))]
-  ""
-  "fneg %0 = %1%B0"
-  [(set_attr "type" "F")])
+  op0_xf = gen_reg_rtx (XFmode);
 
-(define_insn "*nabssf2"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-       (neg:SF (abs:SF (match_operand:SF 1 "register_operand" "f"))))]
-  ""
-  "fnegabs %0 = %1%B0"
-  [(set_attr "type" "F")])
+  if (CONSTANT_P (operands[1]))
+    operands[1] = force_reg (DImode, operands[1]);
+  op1_xf = gen_reg_rtx (XFmode);
+  expand_float (op1_xf, operands[1], 0);
 
-(define_insn "minsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-       (smin:SF (match_operand:SF 1 "register_operand" "f")
-                (match_operand:SF 2 "reg_or_fp01_operand" "fG")))]
-  ""
-  "fmin %0 = %1, %F2%B0"
-  [(set_attr "type" "F")])
+  if (CONSTANT_P (operands[2]))
+    operands[2] = force_reg (DImode, operands[2]);
+  op2_xf = gen_reg_rtx (XFmode);
+  expand_float (op2_xf, operands[2], 0);
 
-(define_insn "maxsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-       (smax:SF (match_operand:SF 1 "register_operand" "f")
-                (match_operand:SF 2 "reg_or_fp01_operand" "fG")))]
-  ""
-  "fmax %0 = %1, %F2%B0"
-  [(set_attr "type" "F")])
+  emit_insn (gen_cond_trap (EQ, operands[2], CONST0_RTX (DImode),
+                           CONST1_RTX (DImode)));
 
-(define_insn "*maddsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-       (plus:SF (mult:SF (match_operand:SF 1 "register_operand" "f")
-                         (match_operand:SF 2 "register_operand" "f"))
-                (match_operand:SF 3 "reg_or_fp01_operand" "fG")))]
+  if (TARGET_INLINE_INT_DIV == INL_MIN_LAT)
+    emit_insn (gen_divdi3_internal_lat (op0_xf, op1_xf, op2_xf));
+  else
+    emit_insn (gen_divdi3_internal_thr (op0_xf, op1_xf, op2_xf));
+
+  emit_insn (gen_fix_truncxfdi2_alts (operands[0], op0_xf, const1_rtx));
+  DONE;
+})
+
+(define_expand "moddi3"
+  [(set (match_operand:DI 0 "register_operand" "")
+       (mod:SI (match_operand:DI 1 "general_operand" "")
+               (match_operand:DI 2 "general_operand" "")))]
+  "TARGET_INLINE_INT_DIV"
+{
+  rtx op2_neg, div;
+
+  div = gen_reg_rtx (DImode);
+  emit_insn (gen_divdi3 (div, operands[1], operands[2]));
+
+  op2_neg = expand_unop (DImode, neg_optab, operands[2], NULL_RTX, 0);
+
+  emit_insn (gen_madddi4 (operands[0], div, op2_neg, operands[1]));
+  DONE;
+})
+
+(define_expand "udivdi3"
+  [(set (match_operand:DI 0 "register_operand" "")
+       (udiv:DI (match_operand:DI 1 "general_operand" "")
+                (match_operand:DI 2 "general_operand" "")))]
+  "TARGET_INLINE_INT_DIV"
+{
+  rtx op1_xf, op2_xf, op0_xf;
+
+  op0_xf = gen_reg_rtx (XFmode);
+
+  if (CONSTANT_P (operands[1]))
+    operands[1] = force_reg (DImode, operands[1]);
+  op1_xf = gen_reg_rtx (XFmode);
+  expand_float (op1_xf, operands[1], 1);
+
+  if (CONSTANT_P (operands[2]))
+    operands[2] = force_reg (DImode, operands[2]);
+  op2_xf = gen_reg_rtx (XFmode);
+  expand_float (op2_xf, operands[2], 1);
+
+  emit_insn (gen_cond_trap (EQ, operands[2], CONST0_RTX (DImode),
+                           CONST1_RTX (DImode)));
+
+  if (TARGET_INLINE_INT_DIV == INL_MIN_LAT)
+    emit_insn (gen_divdi3_internal_lat (op0_xf, op1_xf, op2_xf));
+  else
+    emit_insn (gen_divdi3_internal_thr (op0_xf, op1_xf, op2_xf));
+
+  emit_insn (gen_fixuns_truncxfdi2_alts (operands[0], op0_xf, const1_rtx));
+  DONE;
+})
+
+(define_expand "umoddi3"
+  [(set (match_operand:DI 0 "register_operand" "")
+       (umod:DI (match_operand:DI 1 "general_operand" "")
+                (match_operand:DI 2 "general_operand" "")))]
+  "TARGET_INLINE_INT_DIV"
+{
+  rtx op2_neg, div;
+
+  div = gen_reg_rtx (DImode);
+  emit_insn (gen_udivdi3 (div, operands[1], operands[2]));
+
+  op2_neg = expand_unop (DImode, neg_optab, operands[2], NULL_RTX, 0);
+
+  emit_insn (gen_madddi4 (operands[0], div, op2_neg, operands[1]));
+  DONE;
+})
+
+(define_insn_and_split "divdi3_internal_lat"
+  [(set (match_operand:XF 0 "fr_register_operand" "=&f")
+       (float:XF (div:SI (match_operand:XF 1 "fr_register_operand" "f")
+                         (match_operand:XF 2 "fr_register_operand" "f"))))
+   (clobber (match_scratch:XF 3 "=&f"))
+   (clobber (match_scratch:XF 4 "=&f"))
+   (clobber (match_scratch:XF 5 "=&f"))
+   (clobber (match_scratch:BI 6 "=c"))]
+  "TARGET_INLINE_INT_DIV == INL_MIN_LAT"
+  "#"
+  "&& reload_completed"
+  [(parallel [(set (match_dup 0) (div:XF (const_int 1) (match_dup 2)))
+             (set (match_dup 6) (unspec:BI [(match_dup 1) (match_dup 2)]
+                                           UNSPEC_FR_RECIP_APPROX))
+             (use (const_int 1))])
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (minus:XF (match_dup 7)
+                              (mult:XF (match_dup 2) (match_dup 0))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 4) (mult:XF (match_dup 1) (match_dup 0)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 5) (mult:XF (match_dup 3) (match_dup 3)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 4)
+                    (plus:XF (mult:XF (match_dup 3) (match_dup 4))
+                             (match_dup 4)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 0)
+                    (plus:XF (mult:XF (match_dup 3) (match_dup 0))
+                             (match_dup 0)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (plus:XF (mult:XF (match_dup 5) (match_dup 4))
+                             (match_dup 4)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 0)
+                    (plus:XF (mult:XF (match_dup 5) (match_dup 0))
+                             (match_dup 0)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 4)
+                    (minus:XF (match_dup 1)
+                              (mult:XF (match_dup 2) (match_dup 3))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 0)
+                    (plus:XF (mult:XF (match_dup 4) (match_dup 0))
+                             (match_dup 3)))
+               (use (const_int 1))]))
+  ] 
+  "operands[7] = CONST1_RTX (XFmode);"
+  [(set_attr "predicable" "no")])
+
+(define_insn_and_split "divdi3_internal_thr"
+  [(set (match_operand:XF 0 "fr_register_operand" "=&f")
+       (float:XF (div:SI (match_operand:XF 1 "fr_register_operand" "f")
+                         (match_operand:XF 2 "fr_register_operand" "f"))))
+   (clobber (match_scratch:XF 3 "=&f"))
+   (clobber (match_scratch:XF 4 "=f"))
+   (clobber (match_scratch:BI 5 "=c"))]
+  "TARGET_INLINE_INT_DIV == INL_MAX_THR"
+  "#"
+  "&& reload_completed"
+  [(parallel [(set (match_dup 0) (div:XF (const_int 1) (match_dup 2)))
+             (set (match_dup 5) (unspec:BI [(match_dup 1) (match_dup 2)] 
+                                           UNSPEC_FR_RECIP_APPROX))
+             (use (const_int 1))])
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (minus:XF (match_dup 6)
+                              (mult:XF (match_dup 2) (match_dup 0))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 0)
+                    (plus:XF (mult:XF (match_dup 3) (match_dup 0))
+                             (match_dup 0)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 3) (mult:XF (match_dup 3) (match_dup 3)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 0)
+                    (plus:XF (mult:XF (match_dup 3) (match_dup 0))
+                             (match_dup 0)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 3) (mult:XF (match_dup 0) (match_dup 1)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 4)
+                    (minus:XF (match_dup 1)
+                              (mult:XF (match_dup 2) (match_dup 3))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 0)
+                    (plus:XF (mult:XF (match_dup 4) (match_dup 0))
+                             (match_dup 3)))
+               (use (const_int 1))]))
+  ] 
+  "operands[6] = CONST1_RTX (XFmode);"
+  [(set_attr "predicable" "no")])
+\f
+;; ::::::::::::::::::::
+;; ::
+;; :: 128 bit Integer arithmetic
+;; ::
+;; ::::::::::::::::::::
+
+(define_insn "addti3"
+  [(set (match_operand:TI 0 "gr_register_operand" "=&r")
+       (plus:TI (match_operand:TI 1 "gr_register_operand" "%r")
+                (match_operand:TI 2 "gr_reg_or_14bit_operand" "rI")))
+   (clobber (match_scratch:BI 3 "=&c"))]
   ""
-  "fma.s %0 = %1, %2, %F3%B0"
-  [(set_attr "type" "F")])
+  "#"
+  [(set_attr "itanium_class" "unknown")])
 
-(define_insn "*msubsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-       (minus:SF (mult:SF (match_operand:SF 1 "register_operand" "f")
-                          (match_operand:SF 2 "register_operand" "f"))
-                 (match_operand:SF 3 "reg_or_fp01_operand" "fG")))]
+(define_split
+  [(set (match_operand:TI 0 "register_operand" "")
+       (plus:TI (match_operand:TI 1 "register_operand" "")
+                (match_operand:TI 2 "register_operand" "")))
+   (clobber (match_scratch:BI 3 ""))]
+  "reload_completed"
+  [(set (match_dup 0) (plus:DI (match_dup 1) (match_dup 2)))
+   (set (match_dup 3) (ltu:BI (match_dup 0) (match_dup 1)))
+   (cond_exec (eq (match_dup 3) (const_int 0))
+             (set (match_dup 4) (plus:DI (match_dup 5) (match_dup 6))))
+   (cond_exec (ne (match_dup 3) (const_int 0))
+             (set (match_dup 4)
+                  (plus:DI (plus:DI (match_dup 5) (match_dup 6))
+                           (const_int 1))))]
+{
+  operands[4] = gen_highpart (DImode, operands[0]);
+  operands[0] = gen_lowpart (DImode, operands[0]);
+  operands[5] = gen_highpart (DImode, operands[1]);
+  operands[1] = gen_lowpart (DImode, operands[1]);
+  operands[6] = gen_highpart (DImode, operands[2]);
+  operands[2] = gen_lowpart (DImode, operands[2]);
+})
+
+(define_split
+  [(set (match_operand:TI 0 "register_operand" "")
+       (plus:TI (match_operand:TI 1 "register_operand" "")
+                (match_operand:TI 2 "immediate_operand" "")))
+   (clobber (match_scratch:BI 3 ""))]
+  "reload_completed"
+  [(set (match_dup 0) (plus:DI (match_dup 1) (match_dup 2)))
+   (set (match_dup 3) (ltu:BI (match_dup 0) (match_dup 1)))
+   (cond_exec (eq (match_dup 3) (const_int 0))
+             (set (match_dup 4)
+                  (plus:DI (match_dup 5) (match_dup 6))))
+   (cond_exec (ne (match_dup 3) (const_int 0))
+             (set (match_dup 4)
+                  (plus:DI (match_dup 5) (match_dup 7))))]
+{
+  operands[4] = gen_highpart (DImode, operands[0]);
+  operands[0] = gen_lowpart (DImode, operands[0]);
+  operands[5] = gen_highpart (DImode, operands[1]);
+  operands[1] = gen_lowpart (DImode, operands[1]);
+  operands[6] = INTVAL (operands[2]) < 0 ? constm1_rtx : const0_rtx;
+  operands[7] = INTVAL (operands[2]) < 0 ? const0_rtx : const1_rtx;
+})
+
+(define_insn "subti3"
+  [(set (match_operand:TI 0 "gr_register_operand" "=&r")
+       (minus:TI (match_operand:TI 1 "gr_reg_or_8bit_operand" "rK")
+                 (match_operand:TI 2 "gr_register_operand" "r")))
+   (clobber (match_scratch:BI 3 "=&c"))]
   ""
-  "fms.s %0 = %1, %2, %F3%B0"
-  [(set_attr "type" "F")])
+  "#"
+  [(set_attr "itanium_class" "unknown")])
 
-(define_insn "*nmulsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-       (neg:SF (mult:SF (match_operand:SF 1 "register_operand" "f")
-                        (match_operand:SF 2 "register_operand" "f"))))]
+(define_split
+  [(set (match_operand:TI 0 "register_operand" "")
+       (minus:TI (match_operand:TI 1 "register_operand" "")
+                 (match_operand:TI 2 "register_operand" "")))
+   (clobber (match_scratch:BI 3 "=&c"))]
+  "reload_completed"
+  [(set (match_dup 0) (minus:DI (match_dup 1) (match_dup 2)))
+   (set (match_dup 3) (ltu:BI (match_dup 1) (match_dup 0)))
+   (cond_exec (eq (match_dup 3) (const_int 0))
+             (set (match_dup 4) (minus:DI (match_dup 5) (match_dup 6))))
+   (cond_exec (ne (match_dup 3) (const_int 0))
+             (set (match_dup 4)
+                  (plus:DI (not:DI (match_dup 6)) (match_dup 5))))]
+{
+  operands[4] = gen_highpart (DImode, operands[0]);
+  operands[0] = gen_lowpart (DImode, operands[0]);
+  operands[5] = gen_highpart (DImode, operands[1]);
+  operands[1] = gen_lowpart (DImode, operands[1]);
+  operands[6] = gen_highpart (DImode, operands[2]);
+  operands[2] = gen_lowpart (DImode, operands[2]);
+})
+
+(define_split
+  [(set (match_operand:TI 0 "register_operand" "")
+       (minus:TI (match_operand:TI 1 "immediate_operand" "")
+                 (match_operand:TI 2 "register_operand" "")))
+   (clobber (match_scratch:BI 3 "=&c"))]
+  "reload_completed && CONST_OK_FOR_K (INTVAL (operands[1]))"
+  [(set (match_dup 0) (minus:DI (match_dup 1) (match_dup 2)))
+   (set (match_dup 3) (gtu:BI (match_dup 0) (match_dup 1)))
+   (cond_exec (ne (match_dup 3) (const_int 0))
+             (set (match_dup 4) (minus:DI (match_dup 6) (match_dup 5))))
+   (cond_exec (eq (match_dup 3) (const_int 0))
+             (set (match_dup 4) (minus:DI (match_dup 7) (match_dup 5))))]
+{
+  operands[4] = gen_highpart (DImode, operands[0]);
+  operands[0] = gen_lowpart (DImode, operands[0]);
+  operands[5] = gen_highpart (DImode, operands[2]);
+  operands[2] = gen_lowpart (DImode, operands[2]);
+  operands[6] = INTVAL (operands[1]) < 0 ? GEN_INT (-2) : constm1_rtx;
+  operands[7] = INTVAL (operands[1]) < 0 ? constm1_rtx : const0_rtx;
+})
+
+(define_expand "mulditi3"
+  [(set (match_operand:TI 0 "fr_register_operand" "")
+       (mult:TI (sign_extend:TI
+                  (match_operand:DI 1 "fr_register_operand" ""))
+                (sign_extend:TI
+                  (match_operand:DI 2 "fr_register_operand" ""))))]
   ""
-  "fnmpy.s %0 = %1, %2%B0"
-  [(set_attr "type" "F")])
+  "")
 
-;; ??? Is it possible to canonicalize this as (minus (reg) (mult))?
+(define_insn_and_split "*mulditi3_internal"
+  [(set (match_operand:TI 0 "fr_register_operand" "=&f")
+       (mult:TI (sign_extend:TI
+                  (match_operand:DI 1 "fr_register_operand" "%f"))
+                (sign_extend:TI
+                  (match_operand:DI 2 "fr_register_operand" "f"))))]
+  ""
+  "#"
+  "reload_completed"
+  [(set (match_dup 0) (mult:DI (match_dup 1) (match_dup 2)))
+   (set (match_dup 3) (truncate:DI
+                       (lshiftrt:TI
+                         (mult:TI (sign_extend:TI (match_dup 1))
+                                  (sign_extend:TI (match_dup 2)))
+                         (const_int 64))))]
+{
+  operands[3] = gen_highpart (DImode, operands[0]);
+  operands[0] = gen_lowpart (DImode, operands[0]);
+}
+  [(set_attr "itanium_class" "unknown")])
+
+(define_expand "umulditi3"
+  [(set (match_operand:TI 0 "fr_register_operand" "")
+       (mult:TI (zero_extend:TI
+                  (match_operand:DI 1 "fr_register_operand" ""))
+                (zero_extend:TI
+                  (match_operand:DI 2 "fr_register_operand" ""))))]
+  ""
+  "")
 
-(define_insn "*nmaddsf3"
-  [(set (match_operand:SF 0 "register_operand" "=f")
-       (plus:SF (neg:SF (mult:SF (match_operand:SF 1 "register_operand" "f")
-                                 (match_operand:SF 2 "register_operand" "f")))
-                (match_operand:SF 3 "reg_or_fp01_operand" "fG")))]
+(define_insn_and_split "*umulditi3_internal"
+  [(set (match_operand:TI 0 "fr_register_operand" "=&f")
+       (mult:TI (zero_extend:TI
+                  (match_operand:DI 1 "fr_register_operand" "%f"))
+                (zero_extend:TI
+                  (match_operand:DI 2 "fr_register_operand" "f"))))]
   ""
-  "fnma.s %0 = %1, %2, %F3%B0"
-  [(set_attr "type" "F")])
+  "#"
+  "reload_completed"
+  [(set (match_dup 0) (mult:DI (match_dup 1) (match_dup 2)))
+   (set (match_dup 3) (truncate:DI
+                       (lshiftrt:TI
+                         (mult:TI (zero_extend:TI (match_dup 1))
+                                  (zero_extend:TI (match_dup 2)))
+                         (const_int 64))))]
+{
+  operands[3] = gen_highpart (DImode, operands[0]);
+  operands[0] = gen_lowpart (DImode, operands[0]);
+}
+  [(set_attr "itanium_class" "unknown")])
 
+(define_insn_and_split "negti2"
+  [(set (match_operand:TI 0 "gr_register_operand" "=&r")
+       (neg:TI (match_operand:TI 1 "gr_register_operand" "r")))
+   (clobber (match_scratch:BI 2 "=&c"))]
+  ""
+  "#"
+  "reload_completed"
+  [(set (match_dup 2) (eq:BI (match_dup 1) (const_int 0)))
+   (set (match_dup 0) (minus:DI (const_int 0) (match_dup 1)))
+   (cond_exec (eq (match_dup 2) (const_int 0))
+             (set (match_dup 3) (minus:DI (const_int -1) (match_dup 4))))
+   (cond_exec (ne (match_dup 2) (const_int 0))
+             (set (match_dup 3) (minus:DI (const_int 0) (match_dup 4))))]
+{
+  operands[3] = gen_highpart (DImode, operands[0]);
+  operands[0] = gen_lowpart (DImode, operands[0]);
+  operands[4] = gen_highpart (DImode, operands[1]);
+  operands[1] = gen_lowpart (DImode, operands[1]);
+}
+  [(set_attr "itanium_class" "unknown")])
 \f
 ;; ::::::::::::::::::::
 ;; ::
-;; :: 64 bit floating point arithmetic
+;; :: 32 bit floating point arithmetic
 ;; ::
 ;; ::::::::::::::::::::
 
-(define_insn "adddf3"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-       (plus:DF (match_operand:DF 1 "register_operand" "%f")
-                (match_operand:DF 2 "reg_or_fp01_operand" "fG")))]
+(define_insn "addsf3"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (plus:SF (match_operand:SF 1 "fr_register_operand" "%f")
+                (match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")))]
   ""
-  "fadd.d %0 = %1, %F2%B0"
-  [(set_attr "type" "F")])
+  "fadd.s %0 = %1, %F2"
+  [(set_attr "itanium_class" "fmac")])
 
-(define_insn "subdf3"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-       (minus:DF (match_operand:DF 1 "reg_or_fp01_operand" "fG")
-                 (match_operand:DF 2 "reg_or_fp01_operand" "fG")))]
+(define_insn "subsf3"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (minus:SF (match_operand:SF 1 "fr_reg_or_fp01_operand" "fG")
+                 (match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")))]
   ""
-  "fsub.d %0 = %F1, %F2%B0"
-  [(set_attr "type" "F")])
+  "fsub.s %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmac")])
 
-(define_insn "muldf3"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-       (mult:DF (match_operand:DF 1 "register_operand" "f")
-                (match_operand:DF 2 "register_operand" "f")))]
+(define_insn "mulsf3"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (mult:SF (match_operand:SF 1 "fr_register_operand" "%f")
+                (match_operand:SF 2 "fr_register_operand" "f")))]
   ""
-  "fmpy.d %0 = %1, %2%B0"
-  [(set_attr "type" "F")])
+  "fmpy.s %0 = %1, %2"
+  [(set_attr "itanium_class" "fmac")])
 
-(define_insn "absdf2"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-       (abs:DF (match_operand:DF 1 "register_operand" "f")))]
+(define_insn "abssf2"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (abs:SF (match_operand:SF 1 "fr_register_operand" "f")))]
   ""
-  "fabs %0 = %1%B0"
-  [(set_attr "type" "F")])
+  "fabs %0 = %1"
+  [(set_attr "itanium_class" "fmisc")])
 
-(define_insn "negdf2"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-       (neg:DF (match_operand:DF 1 "register_operand" "f")))]
+(define_insn "negsf2"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (neg:SF (match_operand:SF 1 "fr_register_operand" "f")))]
   ""
-  "fneg %0 = %1%B0"
-  [(set_attr "type" "F")])
+  "fneg %0 = %1"
+  [(set_attr "itanium_class" "fmisc")])
 
-(define_insn "*nabsdf2"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-       (neg:DF (abs:DF (match_operand:DF 1 "register_operand" "f"))))]
+(define_insn "*nabssf2"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (neg:SF (abs:SF (match_operand:SF 1 "fr_register_operand" "f"))))]
   ""
-  "fnegabs %0 = %1%B0"
-  [(set_attr "type" "F")])
+  "fnegabs %0 = %1"
+  [(set_attr "itanium_class" "fmisc")])
 
-(define_insn "mindf3"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-       (smin:DF (match_operand:DF 1 "register_operand" "f")
-                (match_operand:DF 2 "reg_or_fp01_operand" "fG")))]
+(define_insn "copysignsf3"
+  [(set (match_operand:SF 0 "register_operand" "=f")
+       (unspec:SF [(match_operand:SF 1 "fr_reg_or_fp01_operand" "fG")
+                   (match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")]
+                  UNSPEC_COPYSIGN))]
   ""
-  "fmin %0 = %1, %F2%B0"
-  [(set_attr "type" "F")])
+  "fmerge.s %0 = %F2, %F1"
+  [(set_attr "itanium_class" "fmisc")])
 
-(define_insn "maxdf3"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-       (smax:DF (match_operand:DF 1 "register_operand" "f")
-                (match_operand:DF 2 "reg_or_fp01_operand" "fG")))]
+(define_insn "*ncopysignsf3"
+  [(set (match_operand:SF 0 "register_operand" "=f")
+       (neg:SF (unspec:SF [(match_operand:SF 1 "fr_reg_or_fp01_operand" "fG")
+                           (match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")]
+                          UNSPEC_COPYSIGN)))]
   ""
-  "fmax %0 = %1, %F2%B0"
-  [(set_attr "type" "F")])
+  "fmerge.ns %0 = %F2, %F1"
+  [(set_attr "itanium_class" "fmisc")])
 
-(define_insn "*madddf3"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-       (plus:DF (mult:DF (match_operand:DF 1 "register_operand" "f")
-                         (match_operand:DF 2 "register_operand" "f"))
-                (match_operand:DF 3 "reg_or_fp01_operand" "fG")))]
+(define_insn "sminsf3"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (smin:SF (match_operand:SF 1 "fr_register_operand" "f")
+                (match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")))]
   ""
-  "fma.d %0 = %1, %2, %F3%B0"
-  [(set_attr "type" "F")])
+  "fmin %0 = %1, %F2"
+  [(set_attr "itanium_class" "fmisc")])
 
-(define_insn "*msubdf3"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-       (minus:DF (mult:DF (match_operand:DF 1 "register_operand" "f")
-                          (match_operand:DF 2 "register_operand" "f"))
-                 (match_operand:DF 3 "reg_or_fp01_operand" "fG")))]
+(define_insn "smaxsf3"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (smax:SF (match_operand:SF 1 "fr_register_operand" "f")
+                (match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")))]
   ""
-  "fms.d %0 = %1, %2, %F3%B0"
-  [(set_attr "type" "F")])
+  "fmax %0 = %1, %F2"
+  [(set_attr "itanium_class" "fmisc")])
 
-(define_insn "*nmuldf3"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-       (neg:DF (mult:DF (match_operand:DF 1 "register_operand" "f")
-                        (match_operand:DF 2 "register_operand" "f"))))]
+(define_insn "*maddsf4"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (plus:SF (mult:SF (match_operand:SF 1 "fr_register_operand" "f")
+                         (match_operand:SF 2 "fr_register_operand" "f"))
+                (match_operand:SF 3 "fr_reg_or_fp01_operand" "fG")))]
   ""
-  "fnmpy.d %0 = %1, %2%B0"
-  [(set_attr "type" "F")])
+  "fma.s %0 = %1, %2, %F3"
+  [(set_attr "itanium_class" "fmac")])
 
-;; ??? Is it possible to canonicalize this as (minus (reg) (mult))?
+(define_insn "*msubsf4"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (minus:SF (mult:SF (match_operand:SF 1 "fr_register_operand" "f")
+                          (match_operand:SF 2 "fr_register_operand" "f"))
+                 (match_operand:SF 3 "fr_reg_or_fp01_operand" "fG")))]
+  ""
+  "fms.s %0 = %1, %2, %F3"
+  [(set_attr "itanium_class" "fmac")])
 
-(define_insn "*nmadddf3"
-  [(set (match_operand:DF 0 "register_operand" "=f")
-       (plus:DF (neg:DF (mult:DF (match_operand:DF 1 "register_operand" "f")
-                                 (match_operand:DF 2 "register_operand" "f")))
-                (match_operand:DF 3 "reg_or_fp01_operand" "fG")))]
+(define_insn "*nmulsf3"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (neg:SF (mult:SF (match_operand:SF 1 "fr_register_operand" "f")
+                        (match_operand:SF 2 "fr_register_operand" "f"))))]
+  ""
+  "fnmpy.s %0 = %1, %2"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*nmaddsf4"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (minus:SF (match_operand:SF 3 "fr_reg_or_fp01_operand" "fG") 
+                 (mult:SF (match_operand:SF 1 "fr_register_operand" "f")
+                          (match_operand:SF 2 "fr_register_operand" "f"))))]
+  ""
+  "fnma.s %0 = %1, %2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*nmaddsf4_alts"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (minus:SF (match_operand:SF 3 "fr_reg_or_fp01_operand" "fG") 
+                 (mult:SF (match_operand:SF 1 "fr_register_operand" "f")
+                          (match_operand:SF 2 "fr_register_operand" "f"))))
+   (use (match_operand:SI 4 "const_int_operand" ""))]
+  ""
+  "fnma.s.s%4 %0 = %1, %2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_expand "divsf3"
+  [(set (match_operand:SF 0 "fr_register_operand" "")
+       (div:SF (match_operand:SF 1 "fr_register_operand" "")
+               (match_operand:SF 2 "fr_register_operand" "")))]
+  "TARGET_INLINE_FLOAT_DIV"
+{
+  rtx insn;
+  if (TARGET_INLINE_FLOAT_DIV == INL_MIN_LAT)
+    insn = gen_divsf3_internal_lat (operands[0], operands[1], operands[2]);
+  else
+    insn = gen_divsf3_internal_thr (operands[0], operands[1], operands[2]);
+  emit_insn (insn);
+  DONE;
+})
+
+(define_insn_and_split "divsf3_internal_lat"
+  [(set (match_operand:SF 0 "fr_register_operand" "=&f")
+       (div:SF (match_operand:SF 1 "fr_register_operand" "f")
+               (match_operand:SF 2 "fr_register_operand" "f")))
+   (clobber (match_scratch:XF 3 "=&f"))
+   (clobber (match_scratch:XF 4 "=f"))
+   (clobber (match_scratch:BI 5 "=c"))]
+  "TARGET_INLINE_FLOAT_DIV == INL_MIN_LAT"
+  "#"
+  "&& reload_completed"
+  [(parallel [(set (match_dup 6) (div:XF (const_int 1) (match_dup 8)))
+             (set (match_dup 5) (unspec:BI [(match_dup 7) (match_dup 8)]
+                                           UNSPEC_FR_RECIP_APPROX))
+             (use (const_int 0))])
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 3) (mult:XF (match_dup 7) (match_dup 6)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 4)
+                    (minus:XF (match_dup 10)
+                              (mult:XF (match_dup 8) (match_dup 6))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (plus:XF (mult:XF (match_dup 4) (match_dup 3))
+                             (match_dup 3)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 4) (mult:XF (match_dup 4) (match_dup 4)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (plus:XF (mult:XF (match_dup 4) (match_dup 3))
+                             (match_dup 3)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 4) (mult:XF (match_dup 4) (match_dup 4)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 9)
+                    (float_truncate:DF
+                      (plus:XF (mult:XF (match_dup 4) (match_dup 3))
+                             (match_dup 3))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (set (match_dup 0)
+         (float_truncate:SF (match_dup 6))))
+  ] 
+{
+  operands[6] = gen_rtx_REG (XFmode, REGNO (operands[0]));
+  operands[7] = gen_rtx_REG (XFmode, REGNO (operands[1]));
+  operands[8] = gen_rtx_REG (XFmode, REGNO (operands[2]));
+  operands[9] = gen_rtx_REG (DFmode, REGNO (operands[0]));
+  operands[10] = CONST1_RTX (XFmode);
+}
+  [(set_attr "predicable" "no")])
+
+(define_insn_and_split "divsf3_internal_thr"
+  [(set (match_operand:SF 0 "fr_register_operand" "=&f")
+       (div:SF (match_operand:SF 1 "fr_register_operand" "f")
+               (match_operand:SF 2 "fr_register_operand" "f")))
+   (clobber (match_scratch:XF 3 "=&f"))
+   (clobber (match_scratch:XF 4 "=f"))
+   (clobber (match_scratch:BI 5 "=c"))]
+  "TARGET_INLINE_FLOAT_DIV == INL_MAX_THR"
+  "#"
+  "&& reload_completed"
+  [(parallel [(set (match_dup 6) (div:XF (const_int 1) (match_dup 8)))
+             (set (match_dup 5) (unspec:BI [(match_dup 7) (match_dup 8)]
+                                           UNSPEC_FR_RECIP_APPROX))
+             (use (const_int 0))])
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (minus:XF (match_dup 10)
+                              (mult:XF (match_dup 8) (match_dup 6))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (plus:XF (mult:XF (match_dup 3) (match_dup 3))
+                             (match_dup 3)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 6)
+                    (plus:XF (mult:XF (match_dup 3) (match_dup 6))
+                             (match_dup 6)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 9)
+                    (float_truncate:SF
+                      (mult:XF (match_dup 7) (match_dup 6))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 4)
+                    (minus:XF (match_dup 7)
+                              (mult:XF (match_dup 8) (match_dup 3))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (set (match_dup 0)
+         (float_truncate:SF
+           (plus:XF (mult:XF (match_dup 4) (match_dup 6))
+                             (match_dup 3)))))
+  ] 
+{
+  operands[6] = gen_rtx_REG (XFmode, REGNO (operands[0]));
+  operands[7] = gen_rtx_REG (XFmode, REGNO (operands[1]));
+  operands[8] = gen_rtx_REG (XFmode, REGNO (operands[2]));
+  operands[9] = gen_rtx_REG (SFmode, REGNO (operands[3]));
+  operands[10] = CONST1_RTX (XFmode);
+}
+  [(set_attr "predicable" "no")])
+
+;; Inline square root.
+
+(define_insn "*sqrt_approx"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+        (div:XF (const_int 1)
+                (sqrt:XF (match_operand:XF 2 "fr_register_operand" "f"))))
+   (set (match_operand:BI 1 "register_operand" "=c")
+        (unspec:BI [(match_dup 2)] UNSPEC_FR_SQRT_RECIP_APPROX))
+   (use (match_operand:SI 3 "const_int_operand" "")) ]
+  ""
+  "frsqrta.s%3 %0, %1 = %2"
+  [(set_attr "itanium_class" "fmisc")
+   (set_attr "predicable" "no")])
+
+(define_insn "setf_exp_xf"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+        (unspec:XF [(match_operand:DI 1 "register_operand" "r")]
+                  UNSPEC_SETF_EXP))]
   ""
-  "fnma.d %0 = %1, %2, %F3%B0"
-  [(set_attr "type" "F")])
+  "setf.exp %0 = %1"
+  [(set_attr "itanium_class" "frfr")])
+
+(define_expand "sqrtsf2"
+  [(set (match_operand:SF 0 "fr_register_operand" "=&f")
+       (sqrt:SF (match_operand:SF 1 "fr_register_operand" "f")))]
+  "TARGET_INLINE_SQRT"
+{
+  rtx insn;
+#if 0
+  if (TARGET_INLINE_SQRT == INL_MIN_LAT)
+    insn = gen_sqrtsf2_internal_lat (operands[0], operands[1]);
+  else
+#else
+  gcc_assert (TARGET_INLINE_SQRT != INL_MIN_LAT);
+#endif
+  insn = gen_sqrtsf2_internal_thr (operands[0], operands[1]);
+  emit_insn (insn);
+  DONE;
+})
+
+;; Latency-optimized square root.
+;; FIXME: Implement.
+
+;; Throughput-optimized square root.
+
+(define_insn_and_split "sqrtsf2_internal_thr"
+  [(set (match_operand:SF 0 "fr_register_operand" "=&f")
+       (sqrt:SF (match_operand:SF 1 "fr_register_operand" "f")))
+   ;; Register r2 in optimization guide.
+   (clobber (match_scratch:DI 2 "=r"))
+   ;; Register f8 in optimization guide
+   (clobber (match_scratch:XF 3 "=&f"))
+   ;; Register f9 in optimization guide
+   (clobber (match_scratch:XF 4 "=&f"))
+   ;; Register f10 in optimization guide
+   (clobber (match_scratch:XF 5 "=&f"))
+   ;; Register p6 in optimization guide.
+   (clobber (match_scratch:BI 6 "=c"))]
+  "TARGET_INLINE_SQRT == INL_MAX_THR"
+  "#"
+  "&& reload_completed"
+  [ ;; exponent of +1/2 in r2
+    (set (match_dup 2) (const_int 65534))
+    ;; +1/2 in f8
+    (set (match_dup 3) 
+         (unspec:XF [(match_dup 2)] UNSPEC_SETF_EXP))
+    ;; Step 1
+    ;; y0 = 1/sqrt(a) in f7
+    (parallel [(set (match_dup 7)
+                    (div:XF (const_int 1)
+                            (sqrt:XF (match_dup 8))))
+               (set (match_dup 6)
+                    (unspec:BI [(match_dup 8)]
+                                 UNSPEC_FR_SQRT_RECIP_APPROX))
+               (use (const_int 0))])
+    ;; Step 2
+    ;; H0 = 1/2 * y0 in f9
+    (cond_exec (ne (match_dup 6) (const_int 0))
+      (parallel [(set (match_dup 4)
+                      (plus:XF (mult:XF (match_dup 3) (match_dup 7))
+                               (match_dup 9)))
+                 (use (const_int 1))]))
+    ;; Step 3
+    ;; S0 = a * y0 in f7
+    (cond_exec (ne (match_dup 6) (const_int 0))
+      (parallel [(set (match_dup 7)
+                      (plus:XF (mult:XF (match_dup 8) (match_dup 7))
+                               (match_dup 9)))
+                 (use (const_int 1))]))
+    ;; Step 4
+    ;; d = 1/2 - S0 * H0 in f10
+    (cond_exec (ne (match_dup 6) (const_int 0))
+      (parallel [(set (match_dup 5)
+                      (minus:XF (match_dup 3)
+                               (mult:XF (match_dup 7) (match_dup 4))))
+                 (use (const_int 1))]))
+    ;; Step 5
+    ;; d' = d + 1/2 * d in f8
+    (cond_exec (ne (match_dup 6) (const_int 0))
+       (parallel [(set (match_dup 3)
+                       (plus:XF (mult:XF (match_dup 3) (match_dup 5))
+                                (match_dup 5)))
+                  (use (const_int 1))]))
+    ;; Step 6
+    ;; e = d + d * d' in f8
+    (cond_exec (ne (match_dup 6) (const_int 0))
+       (parallel [(set (match_dup 3)
+                       (plus:XF (mult:XF (match_dup 5) (match_dup 3))
+                                (match_dup 5)))
+                  (use (const_int 1))]))
+    ;; Step 7
+    ;; S1 = S0 + e * S0 in f7
+    (cond_exec (ne (match_dup 6) (const_int 0))
+      (parallel [(set (match_dup 0)
+                     (float_truncate:SF
+                        (plus:XF (mult:XF (match_dup 3) (match_dup 7))
+                                 (match_dup 7))))
+                 (use (const_int 1))]))
+    ;; Step 8
+    ;; H1 = H0 + e * H0 in f8
+    (cond_exec (ne (match_dup 6) (const_int 0))
+       (parallel [(set (match_dup 3)
+                       (plus:XF (mult:XF (match_dup 3) (match_dup 4))
+                                (match_dup 4)))
+                  (use (const_int 1))]))
+    ;; Step 9 
+    ;; d1 = a - S1 * S1 in f9
+    (cond_exec (ne (match_dup 6) (const_int 0))
+       (parallel [(set (match_dup 4)
+                       (minus:XF (match_dup 8)
+                                (mult:XF (match_dup 7) (match_dup 7))))
+                  (use (const_int 1))]))
+    ;; Step 10
+    ;; S = S1 + d1 * H1 in f7
+    (cond_exec (ne (match_dup 6) (const_int 0))
+       (parallel [(set (match_dup 0)
+                       (float_truncate:SF
+                         (plus:XF (mult:XF (match_dup 4) (match_dup 3))
+                                  (match_dup 7))))
+                  (use (const_int 0))]))]
+{
+  /* Generate 82-bit versions of the input and output operands.  */
+  operands[7] = gen_rtx_REG (XFmode, REGNO (operands[0]));
+  operands[8] = gen_rtx_REG (XFmode, REGNO (operands[1]));
+  /* Generate required floating-point constants.  */
+  operands[9] = CONST0_RTX (XFmode);
+}
+  [(set_attr "predicable" "no")])
 \f
 ;; ::::::::::::::::::::
 ;; ::
-;; :: 80 bit floating point arithmetic
+;; :: 64 bit floating point arithmetic
 ;; ::
 ;; ::::::::::::::::::::
 
-(define_insn "addtf3"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (plus:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
-                (match_operand:TF 2 "tfreg_or_fp01_operand" "fG")))]
+(define_insn "adddf3"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (plus:DF (match_operand:DF 1 "fr_register_operand" "%f")
+                (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")))]
   ""
-  "fadd %0 = %F1, %F2%B0"
-  [(set_attr "type" "F")])
+  "fadd.d %0 = %1, %F2"
+  [(set_attr "itanium_class" "fmac")])
 
-(define_insn "subtf3"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (minus:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
-                 (match_operand:TF 2 "tfreg_or_fp01_operand" "fG")))]
+(define_insn "*adddf3_trunc"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+         (plus:DF (match_operand:DF 1 "fr_register_operand" "%f")
+                  (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG"))))]
   ""
-  "fsub %0 = %F1, %F2%B0"
-  [(set_attr "type" "F")])
+  "fadd.s %0 = %1, %F2"
+  [(set_attr "itanium_class" "fmac")])
 
-(define_insn "multf3"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (mult:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
-                (match_operand:TF 2 "tfreg_or_fp01_operand" "fG")))]
+(define_insn "subdf3"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (minus:DF (match_operand:DF 1 "fr_reg_or_fp01_operand" "fG")
+                 (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")))]
   ""
-  "fmpy %0 = %F1, %F2%B0"
-  [(set_attr "type" "F")])
+  "fsub.d %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmac")])
 
-(define_insn "abstf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (abs:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")))]
+(define_insn "*subdf3_trunc"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+         (minus:DF (match_operand:DF 1 "fr_reg_or_fp01_operand" "fG")
+                   (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG"))))]
   ""
-  "fabs %0 = %F1%B0"
-  [(set_attr "type" "F")])
+  "fsub.s %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmac")])
 
-(define_insn "negtf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (neg:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")))]
+(define_insn "muldf3"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+                (match_operand:DF 2 "fr_register_operand" "f")))]
   ""
-  "fneg %0 = %F1%B0"
-  [(set_attr "type" "F")])
+  "fmpy.d %0 = %1, %2"
+  [(set_attr "itanium_class" "fmac")])
 
-(define_insn "*nabstf2"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (neg:TF (abs:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG"))))]
+(define_insn "*muldf3_trunc"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+         (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+                  (match_operand:DF 2 "fr_register_operand" "f"))))]
   ""
-  "fnegabs %0 = %F1%B0"
-  [(set_attr "type" "F")])
+  "fmpy.s %0 = %1, %2"
+  [(set_attr "itanium_class" "fmac")])
 
-(define_insn "mintf3"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (smin:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
-                (match_operand:TF 2 "tfreg_or_fp01_operand" "fG")))]
+(define_insn "absdf2"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (abs:DF (match_operand:DF 1 "fr_register_operand" "f")))]
   ""
-  "fmin %0 = %F1, %F2%B0"
-  [(set_attr "type" "F")])
+  "fabs %0 = %1"
+  [(set_attr "itanium_class" "fmisc")])
 
-(define_insn "maxtf3"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (smax:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
-                (match_operand:TF 2 "tfreg_or_fp01_operand" "fG")))]
+(define_insn "negdf2"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (neg:DF (match_operand:DF 1 "fr_register_operand" "f")))]
   ""
-  "fmax %0 = %F1, %F2%B0"
-  [(set_attr "type" "F")])
+  "fneg %0 = %1"
+  [(set_attr "itanium_class" "fmisc")])
 
-(define_insn "*maddtf3"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (plus:TF (mult:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
-                         (match_operand:TF 2 "tfreg_or_fp01_operand" "fG"))
-                (match_operand:TF 3 "tfreg_or_fp01_operand" "fG")))]
+(define_insn "*nabsdf2"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (neg:DF (abs:DF (match_operand:DF 1 "fr_register_operand" "f"))))]
   ""
-  "fma %0 = %F1, %F2, %F3%B0"
-  [(set_attr "type" "F")])
+  "fnegabs %0 = %1"
+  [(set_attr "itanium_class" "fmisc")])
 
-(define_insn "*msubtf3"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (minus:TF (mult:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
-                          (match_operand:TF 2 "tfreg_or_fp01_operand" "fG"))
-                 (match_operand:TF 3 "tfreg_or_fp01_operand" "fG")))]
+(define_insn "copysigndf3"
+  [(set (match_operand:DF 0 "register_operand" "=f")
+       (unspec:DF [(match_operand:DF 1 "fr_reg_or_fp01_operand" "fG")
+                   (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")]
+                  UNSPEC_COPYSIGN))]
   ""
-  "fms %0 = %F1, %F2, %F3%B0"
-  [(set_attr "type" "F")])
+  "fmerge.s %0 = %F2, %F1"
+  [(set_attr "itanium_class" "fmisc")])
 
-(define_insn "*nmultf3"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (neg:TF (mult:TF (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
-                        (match_operand:TF 2 "tfreg_or_fp01_operand" "fG"))))]
-  ""
-  "fnmpy %0 = %F1, %F2%B0"
-  [(set_attr "type" "F")])
+(define_insn "*ncopysigndf3"
+  [(set (match_operand:DF 0 "register_operand" "=f")
+       (neg:DF (unspec:DF [(match_operand:DF 1 "fr_reg_or_fp01_operand" "fG")
+                           (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")]
+                          UNSPEC_COPYSIGN)))]
+  ""
+  "fmerge.ns %0 = %F2, %F1"
+  [(set_attr "itanium_class" "fmisc")])
+
+(define_insn "smindf3"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (smin:DF (match_operand:DF 1 "fr_register_operand" "f")
+                (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")))]
+  ""
+  "fmin %0 = %1, %F2"
+  [(set_attr "itanium_class" "fmisc")])
+
+(define_insn "smaxdf3"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (smax:DF (match_operand:DF 1 "fr_register_operand" "f")
+                (match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")))]
+  ""
+  "fmax %0 = %1, %F2"
+  [(set_attr "itanium_class" "fmisc")])
+
+(define_insn "*madddf4"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (plus:DF (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+                         (match_operand:DF 2 "fr_register_operand" "f"))
+                (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")))]
+  ""
+  "fma.d %0 = %1, %2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*madddf4_trunc"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+         (plus:DF (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+                           (match_operand:DF 2 "fr_register_operand" "f"))
+                  (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG"))))]
+  ""
+  "fma.s %0 = %1, %2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*msubdf4"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (minus:DF (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+                          (match_operand:DF 2 "fr_register_operand" "f"))
+                 (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")))]
+  ""
+  "fms.d %0 = %1, %2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*msubdf4_trunc"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+         (minus:DF (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+                            (match_operand:DF 2 "fr_register_operand" "f"))
+                   (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG"))))]
+  ""
+  "fms.s %0 = %1, %2, %F3"
+  [(set_attr "itanium_class" "fmac")])
 
-;; ??? Is it possible to canonicalize this as (minus (reg) (mult))?
+(define_insn "*nmuldf3"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (neg:DF (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+                        (match_operand:DF 2 "fr_register_operand" "f"))))]
+  ""
+  "fnmpy.d %0 = %1, %2"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*nmuldf3_trunc"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+         (neg:DF (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+                          (match_operand:DF 2 "fr_register_operand" "f")))))]
+  ""
+  "fnmpy.s %0 = %1, %2"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*nmadddf4"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (minus:DF (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")
+                 (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+                          (match_operand:DF 2 "fr_register_operand" "f"))))]
+  ""
+  "fnma.d %0 = %1, %2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*nmadddf4_alts"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (minus:DF (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")
+                 (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+                          (match_operand:DF 2 "fr_register_operand" "f"))))
+   (use (match_operand:SI 4 "const_int_operand" ""))]
+  ""
+  "fnma.d.s%4 %0 = %1, %2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*nmadddf4_truncsf"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+       (minus:DF (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")
+                 (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+                          (match_operand:DF 2 "fr_register_operand" "f")))))]
+  ""
+  "fnma.s %0 = %1, %2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*nmadddf4_truncsf_alts"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+       (minus:DF (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")
+                 (mult:DF (match_operand:DF 1 "fr_register_operand" "f")
+                          (match_operand:DF 2 "fr_register_operand" "f")))))
+   (use (match_operand:SI 4 "const_int_operand" ""))]
+  ""
+  "fnma.s.s%4 %0 = %1, %2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_expand "divdf3"
+  [(set (match_operand:DF 0 "fr_register_operand" "")
+       (div:DF (match_operand:DF 1 "fr_register_operand" "")
+               (match_operand:DF 2 "fr_register_operand" "")))]
+  "TARGET_INLINE_FLOAT_DIV"
+{
+  rtx insn;
+  if (TARGET_INLINE_FLOAT_DIV == INL_MIN_LAT)
+    insn = gen_divdf3_internal_lat (operands[0], operands[1], operands[2]);
+  else
+    insn = gen_divdf3_internal_thr (operands[0], operands[1], operands[2]);
+  emit_insn (insn);
+  DONE;
+})
+
+(define_insn_and_split "divdf3_internal_lat"
+  [(set (match_operand:DF 0 "fr_register_operand" "=&f")
+       (div:DF (match_operand:DF 1 "fr_register_operand" "f")
+               (match_operand:DF 2 "fr_register_operand" "f")))
+   (clobber (match_scratch:XF 3 "=&f"))
+   (clobber (match_scratch:XF 4 "=&f"))
+   (clobber (match_scratch:XF 5 "=&f"))
+   (clobber (match_scratch:BI 6 "=c"))]
+  "TARGET_INLINE_FLOAT_DIV == INL_MIN_LAT"
+  "#"
+  "&& reload_completed"
+  [(parallel [(set (match_dup 7) (div:XF (const_int 1) (match_dup 9)))
+             (set (match_dup 6) (unspec:BI [(match_dup 8) (match_dup 9)]
+                                           UNSPEC_FR_RECIP_APPROX))
+             (use (const_int 0))])
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 3) (mult:XF (match_dup 8) (match_dup 7)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 4)
+                    (minus:XF (match_dup 12)
+                              (mult:XF (match_dup 9) (match_dup 7))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (plus:XF (mult:XF (match_dup 4) (match_dup 3))
+                             (match_dup 3)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 5) (mult:XF (match_dup 4) (match_dup 4)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 7)
+                    (plus:XF (mult:XF (match_dup 4) (match_dup 7))
+                             (match_dup 7)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (plus:XF (mult:XF (match_dup 5) (match_dup 3))
+                             (match_dup 3)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 4) (mult:XF (match_dup 5) (match_dup 5)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 7)
+                    (plus:XF (mult:XF (match_dup 5) (match_dup 7))
+                             (match_dup 7)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 10)
+                    (float_truncate:DF
+                      (plus:XF (mult:XF (match_dup 4) (match_dup 3))
+                             (match_dup 3))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 7)
+                    (plus:XF (mult:XF (match_dup 4) (match_dup 7))
+                             (match_dup 7)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (parallel [(set (match_dup 11)
+                    (float_truncate:DF
+                      (minus:XF (match_dup 8)
+                                (mult:XF (match_dup 9) (match_dup 3)))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 6) (const_int 0))
+     (set (match_dup 0)
+         (float_truncate:DF (plus:XF (mult:XF (match_dup 5) (match_dup 7))
+                             (match_dup 3)))))
+  ] 
+{
+  operands[7] = gen_rtx_REG (XFmode, REGNO (operands[0]));
+  operands[8] = gen_rtx_REG (XFmode, REGNO (operands[1]));
+  operands[9] = gen_rtx_REG (XFmode, REGNO (operands[2]));
+  operands[10] = gen_rtx_REG (DFmode, REGNO (operands[3]));
+  operands[11] = gen_rtx_REG (DFmode, REGNO (operands[5]));
+  operands[12] = CONST1_RTX (XFmode);
+}
+  [(set_attr "predicable" "no")])
 
-(define_insn "*nmaddtf3"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (plus:TF (neg:TF (mult:TF
-                         (match_operand:TF 1 "tfreg_or_fp01_operand" "fG")
-                         (match_operand:TF 2 "tfreg_or_fp01_operand" "fG")))
-                (match_operand:TF 3 "tfreg_or_fp01_operand" "fG")))]
-  ""
-  "fnma %0 = %F1, %F2, %F3%B0"
-  [(set_attr "type" "F")])
+(define_insn_and_split "divdf3_internal_thr"
+  [(set (match_operand:DF 0 "fr_register_operand" "=&f")
+       (div:DF (match_operand:DF 1 "fr_register_operand" "f")
+               (match_operand:DF 2 "fr_register_operand" "f")))
+   (clobber (match_scratch:XF 3 "=&f"))
+   (clobber (match_scratch:DF 4 "=f"))
+   (clobber (match_scratch:BI 5 "=c"))]
+  "TARGET_INLINE_FLOAT_DIV == INL_MAX_THR"
+  "#"
+  "&& reload_completed"
+  [(parallel [(set (match_dup 6) (div:XF (const_int 1) (match_dup 8)))
+             (set (match_dup 5) (unspec:BI [(match_dup 7) (match_dup 8)]
+                                           UNSPEC_FR_RECIP_APPROX))
+             (use (const_int 0))])
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (minus:XF (match_dup 10)
+                              (mult:XF (match_dup 8) (match_dup 6))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 6)
+                    (plus:XF (mult:XF (match_dup 3) (match_dup 6))
+                             (match_dup 6)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (mult:XF (match_dup 3) (match_dup 3)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 6)
+                    (plus:XF (mult:XF (match_dup 3) (match_dup 6))
+                             (match_dup 6)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (mult:XF (match_dup 3) (match_dup 3)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 6)
+                    (plus:XF (mult:XF (match_dup 3) (match_dup 6))
+                             (match_dup 6)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 9)
+                    (float_truncate:DF
+                      (mult:XF (match_dup 7) (match_dup 6))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 4)
+                    (minus:DF (match_dup 1)
+                              (mult:DF (match_dup 2) (match_dup 9))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (set (match_dup 0)
+         (plus:DF (mult:DF (match_dup 4) (match_dup 0))
+                           (match_dup 9))))
+  ] 
+{
+  operands[6] = gen_rtx_REG (XFmode, REGNO (operands[0]));
+  operands[7] = gen_rtx_REG (XFmode, REGNO (operands[1]));
+  operands[8] = gen_rtx_REG (XFmode, REGNO (operands[2]));
+  operands[9] = gen_rtx_REG (DFmode, REGNO (operands[3]));
+  operands[10] = CONST1_RTX (XFmode);
+}
+  [(set_attr "predicable" "no")])
+
+;; Inline square root.
+
+(define_expand "sqrtdf2"
+  [(set (match_operand:DF 0 "fr_register_operand" "=&f")
+       (sqrt:DF (match_operand:DF 1 "fr_register_operand" "f")))]
+  "TARGET_INLINE_SQRT"
+{
+  rtx insn;
+#if 0
+  if (TARGET_INLINE_SQRT == INL_MIN_LAT)
+    insn = gen_sqrtdf2_internal_lat (operands[0], operands[1]);
+  else
+#else
+  gcc_assert (TARGET_INLINE_SQRT != INL_MIN_LAT);
+#endif
+  insn = gen_sqrtdf2_internal_thr (operands[0], operands[1]);
+  emit_insn (insn);
+  DONE;
+})
+
+;; Latency-optimized square root.
+;; FIXME: Implement.
+
+;; Throughput-optimized square root.
+
+(define_insn_and_split "sqrtdf2_internal_thr"
+  [(set (match_operand:DF 0 "fr_register_operand" "=&f")
+       (sqrt:DF (match_operand:DF 1 "fr_register_operand" "f")))
+   ;; Register r2 in optimization guide.
+   (clobber (match_scratch:DI 2 "=r"))
+   ;; Register f8 in optimization guide
+   (clobber (match_scratch:XF 3 "=&f"))
+   ;; Register f9 in optimization guide
+   (clobber (match_scratch:XF 4 "=&f"))
+   ;; Register f10 in optimization guide
+   (clobber (match_scratch:XF 5 "=&f"))
+   ;; Register p6 in optimization guide.
+   (clobber (match_scratch:BI 6 "=c"))]
+  "TARGET_INLINE_SQRT == INL_MAX_THR"
+  "#"
+  "&& reload_completed"
+  [ ;; exponent of +1/2 in r2
+    (set (match_dup 2) (const_int 65534))
+    ;; +1/2 in f10
+    (set (match_dup 5) 
+         (unspec:XF [(match_dup 2)] UNSPEC_SETF_EXP))
+    ;; Step 1
+    ;; y0 = 1/sqrt(a) in f7
+    (parallel [(set (match_dup 7)
+                    (div:XF (const_int 1)
+                            (sqrt:XF (match_dup 8))))
+               (set (match_dup 6)
+                    (unspec:BI [(match_dup 8)]
+                                 UNSPEC_FR_SQRT_RECIP_APPROX))
+               (use (const_int 0))])
+    ;; Step 2
+    ;; H0 = 1/2 * y0 in f8
+    (cond_exec (ne (match_dup 6) (const_int 0))
+      (parallel [(set (match_dup 3)
+                      (plus:XF (mult:XF (match_dup 5) (match_dup 7))
+                               (match_dup 9)))
+                 (use (const_int 1))]))
+    ;; Step 3
+    ;; G0 = a * y0 in f7
+    (cond_exec (ne (match_dup 6) (const_int 0))
+      (parallel [(set (match_dup 7)
+                      (plus:XF (mult:XF (match_dup 8) (match_dup 7))
+                               (match_dup 9)))
+                 (use (const_int 1))]))
+    ;; Step 4
+    ;; r0 = 1/2 - G0 * H0 in f9
+    (cond_exec (ne (match_dup 6) (const_int 0))
+      (parallel [(set (match_dup 4)
+                      (minus:XF (match_dup 5)
+                               (mult:XF (match_dup 7) (match_dup 3))))
+                 (use (const_int 1))]))
+    ;; Step 5
+    ;; H1 = H0 + r0 * H0 in f8
+    (cond_exec (ne (match_dup 6) (const_int 0))
+       (parallel [(set (match_dup 3)
+                       (plus:XF (mult:XF (match_dup 4) (match_dup 3))
+                                (match_dup 3)))
+                  (use (const_int 1))]))
+    ;; Step 6
+    ;; G1 = G0 + r0 * G0 in f7
+    (cond_exec (ne (match_dup 6) (const_int 0))
+       (parallel [(set (match_dup 7)
+                       (plus:XF (mult:XF (match_dup 4) (match_dup 7))
+                                (match_dup 7)))
+                  (use (const_int 1))]))
+    ;; Step 7
+    ;; r1 = 1/2 - G1 * H1 in f9
+    (cond_exec (ne (match_dup 6) (const_int 0))
+      (parallel [(set (match_dup 4)
+                      (minus:XF (match_dup 5)
+                               (mult:XF (match_dup 7) (match_dup 3))))
+                 (use (const_int 1))]))
+    ;; Step 8
+    ;; H2 = H1 + r1 * H1 in f8
+    (cond_exec (ne (match_dup 6) (const_int 0))
+       (parallel [(set (match_dup 3)
+                       (plus:XF (mult:XF (match_dup 4) (match_dup 3))
+                                (match_dup 3)))
+                  (use (const_int 1))]))
+    ;; Step 9 
+    ;; G2 = G1 + r1 * G1 in f7
+    (cond_exec (ne (match_dup 6) (const_int 0))
+       (parallel [(set (match_dup 7)
+                       (plus:XF (mult:XF (match_dup 4) (match_dup 7))
+                                (match_dup 7)))
+                  (use (const_int 1))]))
+    ;; Step 10
+    ;; d2 = a - G2 * G2 in f9
+    (cond_exec (ne (match_dup 6) (const_int 0))
+       (parallel [(set (match_dup 4)
+                       (minus:XF (match_dup 8)
+                                (mult:XF (match_dup 7) (match_dup 7))))
+                  (use (const_int 1))]))
+    ;; Step 11
+    ;; G3 = G2 + d2 * H2 in f7
+    (cond_exec (ne (match_dup 6) (const_int 0))
+       (parallel [(set (match_dup 7)
+                       (plus:XF (mult:XF (match_dup 4) (match_dup 3))
+                                (match_dup 7)))
+                  (use (const_int 1))]))
+    ;; Step 12
+    ;; d3 = a - G3 * G3 in f9
+    (cond_exec (ne (match_dup 6) (const_int 0))
+       (parallel [(set (match_dup 4)
+                       (minus:XF (match_dup 8)
+                                (mult:XF (match_dup 7) (match_dup 7))))
+                  (use (const_int 1))]))
+    ;; Step 13
+    ;; S = G3 + d3 * H2 in f7
+    (cond_exec (ne (match_dup 6) (const_int 0))
+       (parallel [(set (match_dup 0)
+                       (float_truncate:DF
+                         (plus:XF (mult:XF (match_dup 4) (match_dup 3))
+                                  (match_dup 7))))
+                  (use (const_int 0))]))]
+{
+  /* Generate 82-bit versions of the input and output operands.  */
+  operands[7] = gen_rtx_REG (XFmode, REGNO (operands[0]));
+  operands[8] = gen_rtx_REG (XFmode, REGNO (operands[1]));
+  /* Generate required floating-point constants.  */
+  operands[9] = CONST0_RTX (XFmode);
+}
+  [(set_attr "predicable" "no")])
 \f
 ;; ::::::::::::::::::::
 ;; ::
-;; :: 32 bit Integer Shifts and Rotates
+;; :: 80 bit floating point arithmetic
 ;; ::
 ;; ::::::::::::::::::::
 
-;; There is no sign-extend form of dep, so we only get 32 bits of valid result
-;; instead of 64 like the patterns below.
+(define_insn "addxf3"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (plus:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")))]
+  ""
+  "fadd %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*addxf3_truncsf"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+         (plus:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                  (match_operand:XF 2 "xfreg_or_fp01_operand" "fG"))))]
+  ""
+  "fadd.s %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*addxf3_truncdf"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (float_truncate:DF
+         (plus:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                  (match_operand:XF 2 "xfreg_or_fp01_operand" "fG"))))]
+  ""
+  "fadd.d %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "subxf3"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (minus:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                 (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")))]
+  ""
+  "fsub %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*subxf3_truncsf"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+         (minus:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                   (match_operand:XF 2 "xfreg_or_fp01_operand" "fG"))))]
+  ""
+  "fsub.s %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*subxf3_truncdf"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (float_truncate:DF
+         (minus:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                   (match_operand:XF 2 "xfreg_or_fp01_operand" "fG"))))]
+  ""
+  "fsub.d %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "mulxf3"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")))]
+  ""
+  "fmpy %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*mulxf3_truncsf"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+         (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                  (match_operand:XF 2 "xfreg_or_fp01_operand" "fG"))))]
+  ""
+  "fmpy.s %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*mulxf3_truncdf"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (float_truncate:DF
+         (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                  (match_operand:XF 2 "xfreg_or_fp01_operand" "fG"))))]
+  ""
+  "fmpy.d %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*mulxf3_alts"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")))
+   (use (match_operand:SI 3 "const_int_operand" ""))]
+  ""
+  "fmpy.s%3 %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*mulxf3_truncsf_alts"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+         (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                  (match_operand:XF 2 "xfreg_or_fp01_operand" "fG"))))
+   (use (match_operand:SI 3 "const_int_operand" ""))]
+  ""
+  "fmpy.s.s%3 %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*mulxf3_truncdf_alts"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (float_truncate:DF
+         (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                  (match_operand:XF 2 "xfreg_or_fp01_operand" "fG"))))
+   (use (match_operand:SI 3 "const_int_operand" ""))]
+  ""
+  "fmpy.d.s%3 %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "absxf2"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (abs:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")))]
+  ""
+  "fabs %0 = %F1"
+  [(set_attr "itanium_class" "fmisc")])
+
+(define_insn "negxf2"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (neg:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")))]
+  ""
+  "fneg %0 = %F1"
+  [(set_attr "itanium_class" "fmisc")])
+
+(define_insn "*nabsxf2"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (neg:XF (abs:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG"))))]
+  ""
+  "fnegabs %0 = %F1"
+  [(set_attr "itanium_class" "fmisc")])
+
+(define_insn "copysignxf3"
+  [(set (match_operand:XF 0 "register_operand" "=f")
+       (unspec:XF [(match_operand:XF 1 "fr_reg_or_fp01_operand" "fG")
+                   (match_operand:XF 2 "fr_reg_or_fp01_operand" "fG")]
+                  UNSPEC_COPYSIGN))]
+  ""
+  "fmerge.s %0 = %F2, %F1"
+  [(set_attr "itanium_class" "fmisc")])
+
+(define_insn "*ncopysignxf3"
+  [(set (match_operand:XF 0 "register_operand" "=f")
+       (neg:XF (unspec:XF [(match_operand:XF 1 "fr_reg_or_fp01_operand" "fG")
+                           (match_operand:XF 2 "fr_reg_or_fp01_operand" "fG")]
+                          UNSPEC_COPYSIGN)))]
+  ""
+  "fmerge.ns %0 = %F2, %F1"
+  [(set_attr "itanium_class" "fmisc")])
+
+(define_insn "sminxf3"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (smin:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")))]
+  ""
+  "fmin %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmisc")])
+
+(define_insn "smaxxf3"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (smax:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")))]
+  ""
+  "fmax %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmisc")])
+
+(define_insn "*maddxf4"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (plus:XF (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                         (match_operand:XF 2 "xfreg_or_fp01_operand" "fG"))
+                (match_operand:XF 3 "xfreg_or_fp01_operand" "fG")))]
+  ""
+  "fma %0 = %F1, %F2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*maddxf4_truncsf"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+         (plus:XF (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                           (match_operand:XF 2 "xfreg_or_fp01_operand" "fG"))
+                  (match_operand:XF 3 "xfreg_or_fp01_operand" "fG"))))]
+  ""
+  "fma.s %0 = %F1, %F2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*maddxf4_truncdf"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (float_truncate:DF
+         (plus:XF (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                           (match_operand:XF 2 "xfreg_or_fp01_operand" "fG"))
+                  (match_operand:XF 3 "xfreg_or_fp01_operand" "fG"))))]
+  ""
+  "fma.d %0 = %F1, %F2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*maddxf4_alts"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (plus:XF (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                         (match_operand:XF 2 "xfreg_or_fp01_operand" "fG"))
+                (match_operand:XF 3 "xfreg_or_fp01_operand" "fG")))
+   (use (match_operand:SI 4 "const_int_operand" ""))]
+  ""
+  "fma.s%4 %0 = %F1, %F2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*maddxf4_alts_truncsf"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+         (plus:XF (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                           (match_operand:XF 2 "xfreg_or_fp01_operand" "fG"))
+                  (match_operand:XF 3 "xfreg_or_fp01_operand" "fG"))))
+   (use (match_operand:SI 4 "const_int_operand" ""))]
+  ""
+  "fma.s.s%4 %0 = %F1, %F2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*maddxf4_alts_truncdf"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (float_truncate:DF
+         (plus:XF (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                           (match_operand:XF 2 "xfreg_or_fp01_operand" "fG"))
+                  (match_operand:XF 3 "xfreg_or_fp01_operand" "fG"))))
+   (use (match_operand:SI 4 "const_int_operand" ""))]
+  ""
+  "fma.d.s%4 %0 = %F1, %F2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*msubxf4"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (minus:XF (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                          (match_operand:XF 2 "xfreg_or_fp01_operand" "fG"))
+                 (match_operand:XF 3 "xfreg_or_fp01_operand" "fG")))]
+  ""
+  "fms %0 = %F1, %F2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*msubxf4_truncsf"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+         (minus:XF (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                            (match_operand:XF 2 "xfreg_or_fp01_operand" "fG"))
+                   (match_operand:XF 3 "xfreg_or_fp01_operand" "fG"))))]
+  ""
+  "fms.s %0 = %F1, %F2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*msubxf4_truncdf"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (float_truncate:DF
+         (minus:XF (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                            (match_operand:XF 2 "xfreg_or_fp01_operand" "fG"))
+                   (match_operand:XF 3 "xfreg_or_fp01_operand" "fG"))))]
+  ""
+  "fms.d %0 = %F1, %F2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*nmulxf3"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (neg:XF (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                        (match_operand:XF 2 "xfreg_or_fp01_operand" "fG"))))]
+  ""
+  "fnmpy %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*nmulxf3_truncsf"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+         (neg:XF (mult:XF
+                   (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                   (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")))))]
+  ""
+  "fnmpy.s %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*nmulxf3_truncdf"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (float_truncate:DF
+         (neg:XF (mult:XF
+                   (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                   (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")))))]
+  ""
+  "fnmpy.d %0 = %F1, %F2"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*nmaddxf4"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (minus:XF (match_operand:XF 3 "xfreg_or_fp01_operand" "fG")
+                 (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                          (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")
+   )))]
+  ""
+  "fnma %0 = %F1, %F2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*nmaddxf4_truncsf"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+         (minus:XF (match_operand:XF 3 "xfreg_or_fp01_operand" "fG") 
+                   (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                            (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")
+   ))))]
+  ""
+  "fnma.s %0 = %F1, %F2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*nmaddxf4_truncdf"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (float_truncate:DF
+         (minus:XF (match_operand:XF 3 "xfreg_or_fp01_operand" "fG") 
+                   (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                            (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")
+   ))))]
+  ""
+  "fnma.d %0 = %F1, %F2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*nmaddxf4_alts"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (minus:XF (match_operand:XF 3 "xfreg_or_fp01_operand" "fG")
+                 (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                          (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")
+   )))
+   (use (match_operand:SI 4 "const_int_operand" ""))]
+  ""
+  "fnma.s%4 %0 = %F1, %F2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*nmaddxf4_truncsf_alts"
+  [(set (match_operand:SF 0 "fr_register_operand" "=f")
+       (float_truncate:SF
+         (minus:XF (match_operand:XF 3 "xfreg_or_fp01_operand" "fG") 
+                   (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                            (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")
+   ))))
+   (use (match_operand:SI 4 "const_int_operand" ""))]
+  ""
+  "fnma.s.s%4 %0 = %F1, %F2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_insn "*nmaddxf4_truncdf_alts"
+  [(set (match_operand:DF 0 "fr_register_operand" "=f")
+       (float_truncate:DF
+         (minus:XF (match_operand:XF 3 "xfreg_or_fp01_operand" "fG") 
+                   (mult:XF (match_operand:XF 1 "xfreg_or_fp01_operand" "fG")
+                            (match_operand:XF 2 "xfreg_or_fp01_operand" "fG")
+   ))))
+   (use (match_operand:SI 4 "const_int_operand" ""))]
+  ""
+  "fnma.d.s%4 %0 = %F1, %F2, %F3"
+  [(set_attr "itanium_class" "fmac")])
+
+(define_expand "divxf3"
+  [(set (match_operand:XF 0 "fr_register_operand" "")
+       (div:XF (match_operand:XF 1 "fr_register_operand" "")
+               (match_operand:XF 2 "fr_register_operand" "")))]
+  "TARGET_INLINE_FLOAT_DIV"
+{
+  rtx insn;
+  if (TARGET_INLINE_FLOAT_DIV == INL_MIN_LAT)
+    insn = gen_divxf3_internal_lat (operands[0], operands[1], operands[2]);
+  else
+    insn = gen_divxf3_internal_thr (operands[0], operands[1], operands[2]);
+  emit_insn (insn);
+  DONE;
+})
+
+(define_insn_and_split "divxf3_internal_lat"
+  [(set (match_operand:XF 0 "fr_register_operand" "=&f")
+       (div:XF (match_operand:XF 1 "fr_register_operand" "f")
+               (match_operand:XF 2 "fr_register_operand" "f")))
+   (clobber (match_scratch:XF 3 "=&f"))
+   (clobber (match_scratch:XF 4 "=&f"))
+   (clobber (match_scratch:XF 5 "=&f"))
+   (clobber (match_scratch:XF 6 "=&f"))
+   (clobber (match_scratch:BI 7 "=c"))]
+  "TARGET_INLINE_FLOAT_DIV == INL_MIN_LAT"
+  "#"
+  "&& reload_completed"
+  [(parallel [(set (match_dup 0) (div:XF (const_int 1) (match_dup 2)))
+             (set (match_dup 7) (unspec:BI [(match_dup 1) (match_dup 2)]
+                                           UNSPEC_FR_RECIP_APPROX))
+             (use (const_int 0))])
+   (cond_exec (ne (match_dup 7) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (minus:XF (match_dup 8)
+                              (mult:XF (match_dup 2) (match_dup 0))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 7) (const_int 0))
+     (parallel [(set (match_dup 4) (mult:XF (match_dup 1) (match_dup 0)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 7) (const_int 0))
+     (parallel [(set (match_dup 5) (mult:XF (match_dup 3) (match_dup 3)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 7) (const_int 0))
+     (parallel [(set (match_dup 6)
+                    (plus:XF (mult:XF (match_dup 3) (match_dup 3))
+                             (match_dup 3)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 7) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (plus:XF (mult:XF (match_dup 5) (match_dup 5))
+                             (match_dup 3)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 7) (const_int 0))
+     (parallel [(set (match_dup 5)
+                    (plus:XF (mult:XF (match_dup 6) (match_dup 0))
+                             (match_dup 0)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 7) (const_int 0))
+     (parallel [(set (match_dup 0)
+                    (plus:XF (mult:XF (match_dup 5) (match_dup 3))
+                             (match_dup 0)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 7) (const_int 0))
+     (parallel [(set (match_dup 4)
+                    (minus:XF (match_dup 1)
+                              (mult:XF (match_dup 2) (match_dup 4))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 7) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (plus:XF (mult:XF (match_dup 3) (match_dup 0))
+                             (match_dup 4)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 7) (const_int 0))
+     (parallel [(set (match_dup 5)
+                    (minus:XF (match_dup 8)
+                              (mult:XF (match_dup 2) (match_dup 0))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 7) (const_int 0))
+     (parallel [(set (match_dup 0)
+                    (plus:XF (mult:XF (match_dup 4) (match_dup 0))
+                             (match_dup 0)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 7) (const_int 0))
+     (parallel [(set (match_dup 4)
+                    (minus:XF (match_dup 1)
+                              (mult:XF (match_dup 2) (match_dup 3))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 7) (const_int 0))
+     (set (match_dup 0)
+         (plus:XF (mult:XF (match_dup 4) (match_dup 0))
+                  (match_dup 3))))
+  ] 
+  "operands[8] = CONST1_RTX (XFmode);"
+  [(set_attr "predicable" "no")])
 
-;; Using a predicate that accepts only constants doesn't work, because optabs
-;; will load the operand into a register and call the pattern if the predicate
-;; did not accept it on the first try.  So we use nonmemory_operand and then
-;; verify that we have an appropriate constant in the expander.
+(define_insn_and_split "divxf3_internal_thr"
+  [(set (match_operand:XF 0 "fr_register_operand" "=&f")
+       (div:XF (match_operand:XF 1 "fr_register_operand" "f")
+               (match_operand:XF 2 "fr_register_operand" "f")))
+   (clobber (match_scratch:XF 3 "=&f"))
+   (clobber (match_scratch:XF 4 "=&f"))
+   (clobber (match_scratch:BI 5 "=c"))]
+  "TARGET_INLINE_FLOAT_DIV == INL_MAX_THR"
+  "#"
+  "&& reload_completed"
+  [(parallel [(set (match_dup 0) (div:XF (const_int 1) (match_dup 2)))
+             (set (match_dup 5) (unspec:BI [(match_dup 1) (match_dup 2)]
+                                           UNSPEC_FR_RECIP_APPROX))
+             (use (const_int 0))])
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (minus:XF (match_dup 6)
+                              (mult:XF (match_dup 2) (match_dup 0))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 4)
+                    (plus:XF (mult:XF (match_dup 3) (match_dup 0))
+                             (match_dup 0)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 3) (mult:XF (match_dup 3) (match_dup 3)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (plus:XF (mult:XF (match_dup 3) (match_dup 4))
+                             (match_dup 4)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 4) (mult:XF (match_dup 1) (match_dup 0)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 0)
+                    (minus:XF (match_dup 6)
+                              (mult:XF (match_dup 2) (match_dup 3))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 0)
+                    (plus:XF (mult:XF (match_dup 0) (match_dup 3))
+                             (match_dup 3)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (minus:XF (match_dup 1)
+                              (mult:XF (match_dup 2) (match_dup 4))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 3)
+                    (plus:XF (mult:XF (match_dup 3) (match_dup 0))
+                             (match_dup 4)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 4)
+                    (minus:XF (match_dup 6)
+                              (mult:XF (match_dup 2) (match_dup 0))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 0)
+                    (plus:XF (mult:XF (match_dup 4) (match_dup 0))
+                             (match_dup 0)))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (parallel [(set (match_dup 4)
+                    (minus:XF (match_dup 1)
+                              (mult:XF (match_dup 2) (match_dup 3))))
+               (use (const_int 1))]))
+   (cond_exec (ne (match_dup 5) (const_int 0))
+     (set (match_dup 0)
+         (plus:XF (mult:XF (match_dup 4) (match_dup 0))
+                  (match_dup 3))))
+  ] 
+  "operands[6] = CONST1_RTX (XFmode);"
+  [(set_attr "predicable" "no")])
+
+;; Inline square root.
+
+(define_expand "sqrtxf2"
+  [(set (match_operand:XF 0 "fr_register_operand" "=&f")
+       (sqrt:XF (match_operand:XF 1 "fr_register_operand" "f")))]
+  "TARGET_INLINE_SQRT"
+{
+  rtx insn;
+#if 0
+  if (TARGET_INLINE_SQRT == INL_MIN_LAT)
+    insn = gen_sqrtxf2_internal_lat (operands[0], operands[1]);
+  else
+#else
+  gcc_assert (TARGET_INLINE_SQRT != INL_MIN_LAT);
+#endif
+  insn = gen_sqrtxf2_internal_thr (operands[0], operands[1]);
+  emit_insn (insn);
+  DONE;
+})
+
+;; Latency-optimized square root.
+;; FIXME: Implement.
+
+;; Throughput-optimized square root.
+
+(define_insn_and_split "sqrtxf2_internal_thr"
+  [(set (match_operand:XF 0 "fr_register_operand" "=&f")
+       (sqrt:XF (match_operand:XF 1 "fr_register_operand" "f")))
+   ;; Register r2 in optimization guide.
+   (clobber (match_scratch:DI 2 "=r"))
+   ;; Register f8 in optimization guide
+   (clobber (match_scratch:XF 3 "=&f"))
+   ;; Register f9 in optimization guide
+   (clobber (match_scratch:XF 4 "=&f"))
+   ;; Register f10 in optimization guide
+   (clobber (match_scratch:XF 5 "=&f"))
+   ;; Register f11 in optimization guide
+   (clobber (match_scratch:XF 6 "=&f"))
+   ;; Register p6 in optimization guide.
+   (clobber (match_scratch:BI 7 "=c"))]
+  "TARGET_INLINE_SQRT == INL_MAX_THR"
+  "#"
+  "&& reload_completed"
+  [ ;; exponent of +1/2 in r2
+    (set (match_dup 2) (const_int 65534))
+    ;; +1/2 in f8.  The Intel manual mistakenly specifies f10.
+    (set (match_dup 3) 
+         (unspec:XF [(match_dup 2)] UNSPEC_SETF_EXP))
+    ;; Step 1
+    ;; y0 = 1/sqrt(a) in f7
+    (parallel [(set (match_dup 8)
+                    (div:XF (const_int 1)
+                            (sqrt:XF (match_dup 9))))
+               (set (match_dup 7)
+                    (unspec:BI [(match_dup 9)]
+                                 UNSPEC_FR_SQRT_RECIP_APPROX))
+               (use (const_int 0))])
+    ;; Step 2
+    ;; H0 = 1/2 * y0 in f9
+    (cond_exec (ne (match_dup 7) (const_int 0))
+      (parallel [(set (match_dup 4)
+                      (plus:XF (mult:XF (match_dup 3) (match_dup 8))
+                               (match_dup 10)))
+                 (use (const_int 1))]))
+    ;; Step 3
+    ;; S0 = a * y0 in f7
+    (cond_exec (ne (match_dup 7) (const_int 0))
+      (parallel [(set (match_dup 8)
+                      (plus:XF (mult:XF (match_dup 9) (match_dup 8))
+                               (match_dup 10)))
+                 (use (const_int 1))]))
+    ;; Step 4
+    ;; d0 = 1/2 - S0 * H0 in f10
+    (cond_exec (ne (match_dup 7) (const_int 0))
+      (parallel [(set (match_dup 5)
+                      (minus:XF (match_dup 3)
+                               (mult:XF (match_dup 8) (match_dup 4))))
+                 (use (const_int 1))]))
+    ;; Step 5
+    ;; H1 = H0 + d0 * H0 in f9
+    (cond_exec (ne (match_dup 7) (const_int 0))
+       (parallel [(set (match_dup 4)
+                       (plus:XF (mult:XF (match_dup 5) (match_dup 4))
+                                (match_dup 4)))
+                  (use (const_int 1))]))
+    ;; Step 6
+    ;; S1 = S0 + d0 * S0 in f7
+    (cond_exec (ne (match_dup 7) (const_int 0))
+       (parallel [(set (match_dup 8)
+                       (plus:XF (mult:XF (match_dup 5) (match_dup 8))
+                                (match_dup 8)))
+                  (use (const_int 1))]))
+    ;; Step 7
+    ;; d1 = 1/2 - S1 * H1 in f10
+    (cond_exec (ne (match_dup 7) (const_int 0))
+      (parallel [(set (match_dup 5)
+                      (minus:XF (match_dup 3)
+                               (mult:XF (match_dup 8) (match_dup 4))))
+                 (use (const_int 1))]))
+    ;; Step 8
+    ;; H2 = H1 + d1 * H1 in f9
+    (cond_exec (ne (match_dup 7) (const_int 0))
+       (parallel [(set (match_dup 4)
+                       (plus:XF (mult:XF (match_dup 5) (match_dup 4))
+                                (match_dup 4)))
+                  (use (const_int 1))]))
+    ;; Step 9 
+    ;; S2 = S1 + d1 * S1 in f7
+    (cond_exec (ne (match_dup 7) (const_int 0))
+       (parallel [(set (match_dup 8)
+                       (plus:XF (mult:XF (match_dup 5) (match_dup 8))
+                                (match_dup 8)))
+                  (use (const_int 1))]))
+    ;; Step 10
+    ;; d2 = 1/2 - S2 * H2 in f10
+    (cond_exec (ne (match_dup 7) (const_int 0))
+       (parallel [(set (match_dup 5)
+                       (minus:XF (match_dup 3)
+                                (mult:XF (match_dup 8) (match_dup 4))))
+                  (use (const_int 1))]))
+    ;; Step 11
+    ;; e2 = a - S2 * S2 in f8
+    (cond_exec (ne (match_dup 7) (const_int 0))
+       (parallel [(set (match_dup 3)
+                       (minus:XF (match_dup 9)
+                                (mult:XF (match_dup 8) (match_dup 8))))
+                  (use (const_int 1))]))
+    ;; Step 12
+    ;; S3 = S2 + e2 * H2 in f7
+    (cond_exec (ne (match_dup 7) (const_int 0))
+       (parallel [(set (match_dup 8)
+                       (plus:XF (mult:XF (match_dup 3) (match_dup 4))
+                                (match_dup 8)))
+                  (use (const_int 1))]))
+    ;; Step 13
+    ;; H3 = H2 + d2 * H2 in f9
+    (cond_exec (ne (match_dup 7) (const_int 0))
+       (parallel [(set (match_dup 4)
+                       (plus:XF (mult:XF (match_dup 5) (match_dup 4))
+                                (match_dup 4)))
+                  (use (const_int 1))]))
+    ;; Step 14
+    ;; e3 = a - S3 * S3 in f8
+    (cond_exec (ne (match_dup 7) (const_int 0))
+       (parallel [(set (match_dup 3)
+                       (minus:XF (match_dup 9)
+                                (mult:XF (match_dup 8) (match_dup 8))))
+                  (use (const_int 1))]))
+    ;; Step 15
+    ;; S = S3 + e3 * H3 in f7
+    (cond_exec (ne (match_dup 7) (const_int 0))
+       (parallel [(set (match_dup 0)
+                       (plus:XF (mult:XF (match_dup 3) (match_dup 4))
+                                (match_dup 8)))
+                  (use (const_int 0))]))]
+{
+  /* Generate 82-bit versions of the input and output operands.  */
+  operands[8] = gen_rtx_REG (XFmode, REGNO (operands[0]));
+  operands[9] = gen_rtx_REG (XFmode, REGNO (operands[1]));
+  /* Generate required floating-point constants.  */
+  operands[10] = CONST0_RTX (XFmode);
+}
+  [(set_attr "predicable" "no")])
+
+;; ??? frcpa works like cmp.foo.unc.
+
+(define_insn "*recip_approx"
+  [(set (match_operand:XF 0 "fr_register_operand" "=f")
+       (div:XF (const_int 1)
+               (match_operand:XF 3 "fr_register_operand" "f")))
+   (set (match_operand:BI 1 "register_operand" "=c")
+       (unspec:BI [(match_operand:XF 2 "fr_register_operand" "f")
+                   (match_dup 3)] UNSPEC_FR_RECIP_APPROX))
+   (use (match_operand:SI 4 "const_int_operand" ""))]
+  ""
+  "frcpa.s%4 %0, %1 = %2, %3"
+  [(set_attr "itanium_class" "fmisc")
+   (set_attr "predicable" "no")])
+\f
+;; ::::::::::::::::::::
+;; ::
+;; :: 32 bit Integer Shifts and Rotates
+;; ::
+;; ::::::::::::::::::::
 
 (define_expand "ashlsi3"
-  [(set (match_operand:SI 0 "register_operand" "")
-       (ashift:SI (match_operand:SI 1 "register_operand" "")
-                  (match_operand:SI 2 "nonmemory_operand" "")))]
+  [(set (match_operand:SI 0 "gr_register_operand" "")
+       (ashift:SI (match_operand:SI 1 "gr_register_operand" "")
+                  (match_operand:SI 2 "gr_reg_or_5bit_operand" "")))]
   ""
-  "
 {
-  if (! shift_32bit_count_operand (operands[2], SImode))
-    FAIL;
-}")
+  if (GET_CODE (operands[2]) != CONST_INT)
+    {
+      /* Why oh why didn't Intel arrange for SHIFT_COUNT_TRUNCATED?  Now
+        we've got to get rid of stray bits outside the SImode register.  */
+      rtx subshift = gen_reg_rtx (DImode);
+      emit_insn (gen_zero_extendsidi2 (subshift, operands[2]));
+      operands[2] = subshift;
+    }
+})
 
 (define_insn "*ashlsi3_internal"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-       (ashift:SI (match_operand:SI 1 "register_operand" "r")
-                  (match_operand:SI 2 "shift_32bit_count_operand" "n")))]
+  [(set (match_operand:SI 0 "gr_register_operand" "=r,r,r")
+       (ashift:SI (match_operand:SI 1 "gr_register_operand" "r,r,r")
+                  (match_operand:DI 2 "gr_reg_or_5bit_operand" "R,n,r")))]
   ""
-  "dep.z %0 = %1, %2, %E2"
-  [(set_attr "type" "I")])
-
-;; This is really an extract, but this is how combine canonicalizes the
-;; operation.
+  "@
+   shladd %0 = %1, %2, r0
+   dep.z %0 = %1, %2, %E2
+   shl %0 = %1, %2"
+  [(set_attr "itanium_class" "ialu,ishf,mmshf")])
 
 (define_expand "ashrsi3"
-  [(set (match_dup 3)
-       (ashiftrt:DI (sign_extend:DI
-                     (match_operand:SI 1 "register_operand" ""))
-                    (match_operand:DI 2 "nonmemory_operand" "")))
-   (set (match_operand:SI 0 "register_operand" "") (match_dup 4))]
+  [(set (match_operand:SI 0 "gr_register_operand" "")
+       (ashiftrt:SI (match_operand:SI 1 "gr_register_operand" "")
+                    (match_operand:SI 2 "gr_reg_or_5bit_operand" "")))]
   ""
-  "
 {
-  if (! shift_32bit_count_operand (operands[2], SImode))
-    FAIL;
-
-  operands[3] = gen_reg_rtx (DImode);
-  operands[4] = gen_lowpart (SImode, operands[3]);
-}")
-
-(define_insn "*ashrsi3_internal"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (ashiftrt:DI (sign_extend:DI
-                     (match_operand:SI 1 "register_operand" "r"))
-                    (match_operand:DI 2 "shift_32bit_count_operand" "n")))]
-  ""
-  "extr %0 = %1, %2, %E2"
-  [(set_attr "type" "I")])
-
-;; This is really an extract, but this is how combine canonicalizes the
-;; operation.
+  rtx subtarget = gen_reg_rtx (DImode);
+  if (GET_CODE (operands[2]) == CONST_INT)
+    emit_insn (gen_extv (subtarget, gen_lowpart (DImode, operands[1]),
+                        GEN_INT (32 - INTVAL (operands[2])), operands[2]));
+  else
+    {
+      rtx subshift = gen_reg_rtx (DImode);
+      emit_insn (gen_extendsidi2 (subtarget, operands[1]));
+      emit_insn (gen_zero_extendsidi2 (subshift, operands[2]));
+      emit_insn (gen_ashrdi3 (subtarget, subtarget, subshift));
+    }
+  emit_move_insn (gen_lowpart (DImode, operands[0]), subtarget);
+  DONE;
+})
 
 (define_expand "lshrsi3"
-  [(set (match_dup 3)
-       (lshiftrt:DI (zero_extend:DI
-                     (match_operand:SI 1 "register_operand" ""))
-                    (match_operand:DI 2 "nonmemory_operand" "")))
-   (set (match_operand:SI 0 "register_operand" "") (match_dup 4))]
+  [(set (match_operand:SI 0 "gr_register_operand" "")
+       (lshiftrt:SI (match_operand:SI 1 "gr_register_operand" "")
+                    (match_operand:SI 2 "gr_reg_or_5bit_operand" "")))]
   ""
-  "
 {
-  if (! shift_32bit_count_operand (operands[2], SImode))
-    FAIL;
-
-  operands[3] = gen_reg_rtx (DImode);
-  operands[4] = gen_lowpart (SImode, operands[3]);
-}")
-
-(define_insn "*lshrsi3_internal"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (lshiftrt:DI (zero_extend:DI
-                     (match_operand:SI 1 "register_operand" "r"))
-                    (match_operand:DI 2 "shift_32bit_count_operand" "n")))]
-  ""
-  "extr.u %0 = %1, %2, %E2"
-  [(set_attr "type" "I")])
+  rtx subtarget = gen_reg_rtx (DImode);
+  if (GET_CODE (operands[2]) == CONST_INT)
+    emit_insn (gen_extzv (subtarget, gen_lowpart (DImode, operands[1]),
+                         GEN_INT (32 - INTVAL (operands[2])), operands[2]));
+  else
+    {
+      rtx subshift = gen_reg_rtx (DImode);
+      emit_insn (gen_zero_extendsidi2 (subtarget, operands[1]));
+      emit_insn (gen_zero_extendsidi2 (subshift, operands[2]));
+      emit_insn (gen_lshrdi3 (subtarget, subtarget, subshift));
+    }
+  emit_move_insn (gen_lowpart (DImode, operands[0]), subtarget);
+  DONE;
+})
 
 ;; Use mix4.r/shr to implement rotrsi3.  We only get 32 bits of valid result
-;; here, instead of 64 like the patterns above.
+;; here, instead of 64 like the patterns above.  Keep the pattern together
+;; until after combine; otherwise it won't get matched often.
 
 (define_expand "rotrsi3"
+  [(set (match_operand:SI 0 "gr_register_operand" "")
+       (rotatert:SI (match_operand:SI 1 "gr_register_operand" "")
+                    (match_operand:SI 2 "gr_reg_or_5bit_operand" "")))]
+  ""
+{
+  if (GET_MODE (operands[2]) != VOIDmode)
+    {
+      rtx tmp = gen_reg_rtx (DImode);
+      emit_insn (gen_zero_extendsidi2 (tmp, operands[2]));
+      operands[2] = tmp;
+    }
+})
+
+(define_insn_and_split "*rotrsi3_internal"
+  [(set (match_operand:SI 0 "gr_register_operand" "=&r")
+       (rotatert:SI (match_operand:SI 1 "gr_register_operand" "r")
+                    (match_operand:DI 2 "gr_reg_or_5bit_operand" "rM")))]
+  ""
+  "#"
+  "reload_completed"
   [(set (match_dup 3)
-       (ior:DI (zero_extend:DI (match_operand:SI 1 "register_operand" ""))
+       (ior:DI (zero_extend:DI (match_dup 1))
                (ashift:DI (zero_extend:DI (match_dup 1)) (const_int 32))))
    (set (match_dup 3)
-       (lshiftrt:DI (match_dup 3)
-                    (match_operand:DI 2 "nonmemory_operand" "")))
-   (set (match_operand:SI 0 "register_operand" "") (match_dup 4))]
+       (lshiftrt:DI (match_dup 3) (match_dup 2)))]
+  "operands[3] = gen_rtx_REG (DImode, REGNO (operands[0]));")
+
+(define_expand "rotlsi3"
+  [(set (match_operand:SI 0 "gr_register_operand" "")
+       (rotate:SI (match_operand:SI 1 "gr_register_operand" "")
+                  (match_operand:SI 2 "gr_reg_or_5bit_operand" "")))]
   ""
-  "
 {
   if (! shift_32bit_count_operand (operands[2], SImode))
-    FAIL;
-
-  operands[3] = gen_reg_rtx (DImode);
-  operands[4] = gen_lowpart (SImode, operands[3]);
-}")
+    {
+      rtx tmp = gen_reg_rtx (SImode);
+      emit_insn (gen_subsi3 (tmp, GEN_INT (32), operands[2]));
+      emit_insn (gen_rotrsi3 (operands[0], operands[1], tmp));
+      DONE;
+    }
+})
 
+(define_insn_and_split "*rotlsi3_internal"
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+       (rotate:SI (match_operand:SI 1 "gr_register_operand" "r")
+                  (match_operand:SI 2 "shift_32bit_count_operand" "n")))]
+  ""
+  "mux2 %0 = %1, 0xe1"
+  "reload_completed && INTVAL (operands[2]) != 16"
+  [(set (match_dup 3)
+       (ior:DI (zero_extend:DI (match_dup 1))
+               (ashift:DI (zero_extend:DI (match_dup 1)) (const_int 32))))
+   (set (match_dup 3)
+       (lshiftrt:DI (match_dup 3) (match_dup 2)))]
+{
+  operands[3] = gen_rtx_REG (DImode, REGNO (operands[0]));
+  operands[2] = GEN_INT (32 - INTVAL (operands[2]));
+}
+  [(set_attr "itanium_class" "mmshf")])
 \f
 ;; ::::::::::::::::::::
 ;; ::
 ;; ::::::::::::::::::::
 
 (define_insn "ashldi3"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (ashift:DI (match_operand:DI 1 "register_operand" "r")
-                  (match_operand:DI 2 "reg_or_6bit_operand" "rM")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r,r,r")
+       (ashift:DI (match_operand:DI 1 "gr_register_operand" "r,r,r")
+                  (match_operand:DI 2 "gr_reg_or_6bit_operand" "R,r,rM")))]
   ""
-  "shl %0 = %1, %2"
-  [(set_attr "type" "I")])
+  "@
+   shladd %0 = %1, %2, r0
+   shl %0 = %1, %2
+   shl %0 = %1, %2"
+  [(set_attr "itanium_class" "ialu,mmshf,mmshfi")])
 
 ;; ??? Maybe combine this with the multiply and add instruction?
 
 (define_insn "*shladd"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (plus:DI (mult:DI (match_operand:DI 1 "register_operand" "r")
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (plus:DI (mult:DI (match_operand:DI 1 "gr_register_operand" "r")
                          (match_operand:DI 2 "shladd_operand" "n"))
-                (match_operand:DI 3 "register_operand" "r")))]
+                (match_operand:DI 3 "gr_register_operand" "r")))]
   ""
   "shladd %0 = %1, %S2, %3"
-  [(set_attr "type" "A")])
+  [(set_attr "itanium_class" "ialu")])
 
 ;; This can be created by register elimination if operand3 of shladd is an
 ;; eliminable register or has reg_equiv_constant set.
 ;; doesn't succeed, then this remain a shladd pattern, and will be reloaded
 ;; incorrectly.
 
-(define_insn_and_split "*shladd_elim"
-  [(set (match_operand:DI 0 "register_operand" "=&r")
-       (plus:DI (plus:DI (mult:DI (match_operand:DI 1 "register_operand" "r")
-                                  (match_operand:DI 2 "shladd_operand" "n"))
-                         (match_operand:DI 3 "nonmemory_operand" "r"))
-                (match_operand:DI 4 "nonmemory_operand" "rI")))]
-  "reload_in_progress"
-  "* abort ();"
-  "reload_completed"
-  [(set (match_dup 0) (plus:DI (mult:DI (match_dup 1) (match_dup 2))
-                              (match_dup 3)))
-   (set (match_dup 0) (plus:DI (match_dup 0) (match_dup 4)))]
+(define_insn_and_split "*shladd_elim"
+  [(set (match_operand:DI 0 "gr_register_operand" "=&r")
+       (plus:DI (plus:DI (mult:DI (match_operand:DI 1 "gr_register_operand" "r")
+                                  (match_operand:DI 2 "shladd_operand" "n"))
+                         (match_operand:DI 3 "nonmemory_operand" "r"))
+                (match_operand:DI 4 "nonmemory_operand" "rI")))]
+  "reload_in_progress"
+  "* gcc_unreachable ();"
+  "reload_completed"
+  [(set (match_dup 0) (plus:DI (mult:DI (match_dup 1) (match_dup 2))
+                              (match_dup 3)))
+   (set (match_dup 0) (plus:DI (match_dup 0) (match_dup 4)))]
+  ""
+  [(set_attr "itanium_class" "unknown")])
+
+(define_insn "ashrdi3"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r,r")
+       (ashiftrt:DI (match_operand:DI 1 "gr_register_operand" "r,r")
+                    (match_operand:DI 2 "gr_reg_or_6bit_operand" "r,rM")))]
+  ""
+  "@
+   shr %0 = %1, %2
+   shr %0 = %1, %2"
+  [(set_attr "itanium_class" "mmshf,mmshfi")])
+
+(define_insn "lshrdi3"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r,r")
+       (lshiftrt:DI (match_operand:DI 1 "gr_register_operand" "r,r")
+                    (match_operand:DI 2 "gr_reg_or_6bit_operand" "r,rM")))]
+  ""
+  "@
+   shr.u %0 = %1, %2
+   shr.u %0 = %1, %2"
+  [(set_attr "itanium_class" "mmshf,mmshfi")])
+
+;; Using a predicate that accepts only constants doesn't work, because optabs
+;; will load the operand into a register and call the pattern if the predicate
+;; did not accept it on the first try.  So we use nonmemory_operand and then
+;; verify that we have an appropriate constant in the expander.
+
+(define_expand "rotrdi3"
+  [(set (match_operand:DI 0 "gr_register_operand" "")
+       (rotatert:DI (match_operand:DI 1 "gr_register_operand" "")
+                    (match_operand:DI 2 "nonmemory_operand" "")))]
+  ""
+{
+  if (! shift_count_operand (operands[2], DImode))
+    FAIL;
+})
+
+(define_insn "*rotrdi3_internal"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (rotatert:DI (match_operand:DI 1 "gr_register_operand" "r")
+                    (match_operand:DI 2 "shift_count_operand" "M")))]
+  ""
+  "shrp %0 = %1, %1, %2"
+  [(set_attr "itanium_class" "ishf")])
+
+(define_expand "rotldi3"
+  [(set (match_operand:DI 0 "gr_register_operand" "")
+       (rotate:DI (match_operand:DI 1 "gr_register_operand" "")
+                  (match_operand:DI 2 "nonmemory_operand" "")))]
+  ""
+{
+  if (! shift_count_operand (operands[2], DImode))
+    FAIL;
+})
+
+(define_insn "*rotldi3_internal"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (rotate:DI (match_operand:DI 1 "gr_register_operand" "r")
+                  (match_operand:DI 2 "shift_count_operand" "M")))]
+  ""
+  "shrp %0 = %1, %1, %e2"
+  [(set_attr "itanium_class" "ishf")])
+\f
+;; ::::::::::::::::::::
+;; ::
+;; :: 128 bit Integer Shifts and Rotates
+;; ::
+;; ::::::::::::::::::::
+
+(define_expand "ashlti3"
+  [(set (match_operand:TI 0 "gr_register_operand" "")
+       (ashift:TI (match_operand:TI 1 "gr_register_operand" "")
+                  (match_operand:DI 2 "nonmemory_operand" "")))]
+  ""
+{
+  if (!dshift_count_operand (operands[2], DImode))
+    FAIL;
+})
+
+(define_insn_and_split "*ashlti3_internal"
+  [(set (match_operand:TI 0 "gr_register_operand" "=&r")
+       (ashift:TI (match_operand:TI 1 "gr_register_operand" "r")
+                  (match_operand:DI 2 "dshift_count_operand" "n")))]
+  ""
+  "#"
+  "reload_completed"
+  [(const_int 0)]
+{
+  HOST_WIDE_INT shift = INTVAL (operands[2]);
+  rtx rl = gen_lowpart (DImode, operands[0]);
+  rtx rh = gen_highpart (DImode, operands[0]);
+  rtx lo = gen_lowpart (DImode, operands[1]);
+  rtx shiftlo = GEN_INT (shift & 63);
+
+  if (shift & 64)
+    {
+      emit_move_insn (rl, const0_rtx);
+      if (shift & 63)
+       emit_insn (gen_ashldi3 (rh, lo, shiftlo));
+      else
+       emit_move_insn (rh, lo);
+    }
+  else
+    {
+      rtx hi = gen_highpart (DImode, operands[1]);
+
+      emit_insn (gen_shrp (rh, hi, lo, GEN_INT (-shift & 63)));
+      emit_insn (gen_ashldi3 (rl, lo, shiftlo));
+    }
+  DONE;
+})
+
+(define_expand "ashrti3"
+  [(set (match_operand:TI 0 "gr_register_operand" "")
+       (ashiftrt:TI (match_operand:TI 1 "gr_register_operand" "")
+                    (match_operand:DI 2 "nonmemory_operand" "")))]
+  ""
+{
+  if (!dshift_count_operand (operands[2], DImode))
+    FAIL;
+})
+
+(define_insn_and_split "*ashrti3_internal"
+  [(set (match_operand:TI 0 "gr_register_operand" "=&r")
+       (ashiftrt:TI (match_operand:TI 1 "gr_register_operand" "r")
+                    (match_operand:DI 2 "dshift_count_operand" "n")))]
+  ""
+  "#"
+  "reload_completed"
+  [(const_int 0)]
+{
+  HOST_WIDE_INT shift = INTVAL (operands[2]);
+  rtx rl = gen_lowpart (DImode, operands[0]);
+  rtx rh = gen_highpart (DImode, operands[0]);
+  rtx hi = gen_highpart (DImode, operands[1]);
+  rtx shiftlo = GEN_INT (shift & 63);
+
+  if (shift & 64)
+    {
+      if (shift & 63)
+       emit_insn (gen_ashrdi3 (rl, hi, shiftlo));
+      else
+       emit_move_insn (rl, hi);
+      emit_insn (gen_ashrdi3 (rh, hi, GEN_INT (63)));
+    }
+  else
+    {
+      rtx lo = gen_lowpart (DImode, operands[1]);
+
+      emit_insn (gen_shrp (rl, hi, lo, shiftlo));
+      emit_insn (gen_ashrdi3 (rh, hi, shiftlo));
+    }
+  DONE;
+})
+
+(define_expand "lshrti3"
+  [(set (match_operand:TI 0 "gr_register_operand" "")
+        (lshiftrt:TI (match_operand:TI 1 "gr_register_operand" "")
+                     (match_operand:DI 2 "nonmemory_operand" "")))]
   ""
-  [(set_attr "type" "unknown")])
+{ 
+  if (!dshift_count_operand (operands[2], DImode))
+    FAIL;
+}) 
 
-(define_insn "ashrdi3"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (ashiftrt:DI (match_operand:DI 1 "register_operand" "r")
-                    (match_operand:DI 2 "reg_or_6bit_operand" "rM")))]
+(define_insn_and_split "*lshrti3_internal"
+  [(set (match_operand:TI 0 "gr_register_operand" "=&r")
+       (lshiftrt:TI (match_operand:TI 1 "gr_register_operand" "r")
+                    (match_operand:DI 2 "dshift_count_operand" "n")))]
   ""
-  "shr %0 = %1, %2"
-  [(set_attr "type" "I")])
+  "#"
+  "reload_completed"
+  [(const_int 0)]
+{
+  HOST_WIDE_INT shift = INTVAL (operands[2]);
+  rtx rl = gen_lowpart (DImode, operands[0]);
+  rtx rh = gen_highpart (DImode, operands[0]);
+  rtx hi = gen_highpart (DImode, operands[1]);
+  rtx shiftlo = GEN_INT (shift & 63);
 
-(define_insn "lshrdi3"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (lshiftrt:DI (match_operand:DI 1 "register_operand" "r")
-                    (match_operand:DI 2 "reg_or_6bit_operand" "rM")))]
-  ""
-  "shr.u %0 = %1, %2"
-  [(set_attr "type" "I")])
+  if (shift & 64)
+    {
+      if (shift & 63)
+       emit_insn (gen_lshrdi3 (rl, hi, shiftlo));
+      else
+       emit_move_insn (rl, hi);
+      emit_move_insn (rh, const0_rtx);
+    }
+  else
+    {
+      rtx lo = gen_lowpart (DImode, operands[1]);
 
-;; Using a predicate that accepts only constants doesn't work, because optabs
-;; will load the operand into a register and call the pattern if the predicate
-;; did not accept it on the first try.  So we use nonmemory_operand and then
-;; verify that we have an appropriate constant in the expander.
+      emit_insn (gen_shrp (rl, hi, lo, shiftlo));
+      emit_insn (gen_lshrdi3 (rh, hi, shiftlo));
+    }
+  DONE;
+})
 
-(define_expand "rotrdi3"
-  [(set (match_operand:DI 0 "register_operand" "")
-       (rotatert:DI (match_operand:DI 1 "register_operand" "")
-                    (match_operand:DI 2 "nonmemory_operand" "")))]
+(define_expand "rotlti3"
+  [(set (match_operand:TI 0 "gr_register_operand" "")
+       (rotate:TI (match_operand:TI 1 "gr_register_operand" "")
+                  (match_operand:DI 2 "nonmemory_operand" "")))]
   ""
-  "
 {
-  if (! shift_count_operand (operands[2], DImode))
+  if (! dshift_count_operand (operands[2], DImode))
     FAIL;
-}")
+})
 
-(define_insn "*rotrdi3_internal"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (rotatert:DI (match_operand:DI 1 "register_operand" "r")
-                    (match_operand:DI 2 "shift_count_operand" "M")))]
+(define_insn_and_split "*rotlti3_internal"
+  [(set (match_operand:TI 0 "gr_register_operand" "=&r")
+       (rotate:TI (match_operand:TI 1 "gr_register_operand" "r")
+                  (match_operand:DI 2 "dshift_count_operand" "n")))]
   ""
-  "shrp %0 = %1, %1, %2"
-  [(set_attr "type" "I")])
-
+  "#"
+  "reload_completed"
+  [(const_int 0)]
+{
+  HOST_WIDE_INT count = INTVAL (operands[2]);
+  rtx rl = gen_lowpart (DImode, operands[0]);
+  rtx rh = gen_highpart (DImode, operands[0]);
+  rtx lo = gen_lowpart (DImode, operands[1]);
+  rtx hi = gen_highpart (DImode, operands[1]);
+  rtx countlo = GEN_INT (-count & 63);
+
+  if (count & 64)
+    {
+      if (count & 63)
+       {
+         emit_insn (gen_shrp (rl, hi, lo, countlo));
+         emit_insn (gen_shrp (rh, lo, hi, countlo));
+       }
+      else
+       {
+         emit_move_insn (rl, hi);
+         emit_move_insn (rh, lo);
+       }
+    }
+  else
+    {
+      emit_insn (gen_shrp (rl, lo, hi, countlo));
+      emit_insn (gen_shrp (rh, hi, lo, countlo));
+    }
+  DONE;
+}
+  [(set_attr "itanium_class" "unknown")])
+
+(define_insn "shrp"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (unspec:DI [(match_operand:DI 1 "gr_register_operand" "r")
+                   (match_operand:DI 2 "gr_register_operand" "r")
+                   (match_operand:DI 3 "shift_count_operand" "M")]
+                  UNSPEC_SHRP))]
+  ""
+  "shrp %0 = %1, %2, %3"
+  [(set_attr "itanium_class" "ishf")])
 \f
 ;; ::::::::::::::::::::
 ;; ::
 ;; one_cmplsi2 pattern.
 
 (define_insn "one_cmplsi2"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-       (not:SI (match_operand:SI 1 "register_operand" "r")))]
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+       (not:SI (match_operand:SI 1 "gr_register_operand" "r")))]
   ""
   "andcm %0 = -1, %1"
-  [(set_attr "type" "A")])
+  [(set_attr "itanium_class" "ilog")])
 \f
 ;; ::::::::::::::::::::
 ;; ::
 ;; ::::::::::::::::::::
 
 (define_insn "anddi3"
-  [(set (match_operand:DI 0 "register_operand" "=r,*f")
-       (and:DI (match_operand:DI 1 "register_operand" "%r,*f")
-               (match_operand:DI 2 "reg_or_8bit_operand" "rK,*f")))]
+  [(set (match_operand:DI 0 "grfr_register_operand" "=r,*f")
+       (and:DI (match_operand:DI 1 "grfr_register_operand" "%r,*f")
+               (match_operand:DI 2 "grfr_reg_or_8bit_operand" "rK,*f")))]
   ""
   "@
    and %0 = %2, %1
-   fand %0 = %2, %1%B0"
-  [(set_attr "type" "A,F")])
+   fand %0 = %2, %1"
+  [(set_attr "itanium_class" "ilog,fmisc")])
 
 (define_insn "*andnot"
-  [(set (match_operand:DI 0 "register_operand" "=r,*f")
-       (and:DI (not:DI (match_operand:DI 1 "register_operand" "r,*f"))
-               (match_operand:DI 2 "reg_or_8bit_operand" "rK,*f")))]
+  [(set (match_operand:DI 0 "grfr_register_operand" "=r,*f")
+       (and:DI (not:DI (match_operand:DI 1 "grfr_register_operand" "r,*f"))
+               (match_operand:DI 2 "grfr_reg_or_8bit_operand" "rK,*f")))]
   ""
   "@
    andcm %0 = %2, %1
-   fandcm %0 = %2, %1%B0"
-  [(set_attr "type" "A,F")])
+   fandcm %0 = %2, %1"
+  [(set_attr "itanium_class" "ilog,fmisc")])
 
 (define_insn "iordi3"
-  [(set (match_operand:DI 0 "register_operand" "=r,*f")
-       (ior:DI (match_operand:DI 1 "register_operand" "%r,*f")
-               (match_operand:DI 2 "reg_or_8bit_operand" "rK,*f")))]
+  [(set (match_operand:DI 0 "grfr_register_operand" "=r,*f")
+       (ior:DI (match_operand:DI 1 "grfr_register_operand" "%r,*f")
+               (match_operand:DI 2 "grfr_reg_or_8bit_operand" "rK,*f")))]
   ""
   "@
    or %0 = %2, %1
-   for %0 = %2, %1%B0"
-  [(set_attr "type" "A,F")])
+   for %0 = %2, %1"
+  [(set_attr "itanium_class" "ilog,fmisc")])
 
 (define_insn "xordi3"
-  [(set (match_operand:DI 0 "register_operand" "=r,*f")
-       (xor:DI (match_operand:DI 1 "register_operand" "%r,*f")
-               (match_operand:DI 2 "reg_or_8bit_operand" "rK,*f")))]
+  [(set (match_operand:DI 0 "grfr_register_operand" "=r,*f")
+       (xor:DI (match_operand:DI 1 "grfr_register_operand" "%r,*f")
+               (match_operand:DI 2 "grfr_reg_or_8bit_operand" "rK,*f")))]
   ""
   "@
    xor %0 = %2, %1
-   fxor %0 = %2, %1%B0"
-  [(set_attr "type" "A,F")])
+   fxor %0 = %2, %1"
+  [(set_attr "itanium_class" "ilog,fmisc")])
 
 (define_insn "one_cmpldi2"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (not:DI (match_operand:DI 1 "register_operand" "r")))]
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (not:DI (match_operand:DI 1 "gr_register_operand" "r")))]
   ""
   "andcm %0 = -1, %1"
-  [(set_attr "type" "A")])
+  [(set_attr "itanium_class" "ilog")])
 \f
 ;; ::::::::::::::::::::
 ;; ::
 ;; ::
 ;; ::::::::::::::::::::
 
+(define_expand "cmpbi"
+  [(set (cc0)
+        (compare (match_operand:BI 0 "register_operand" "")
+                (match_operand:BI 1 "const_int_operand" "")))]
+  ""
+{
+  ia64_compare_op0 = operands[0];
+  ia64_compare_op1 = operands[1];
+  DONE;
+})
+
 (define_expand "cmpsi"
   [(set (cc0)
-        (compare (match_operand:SI 0 "register_operand" "")
-                (match_operand:SI 1 "reg_or_8bit_and_adjusted_operand" "")))]
+        (compare (match_operand:SI 0 "gr_register_operand" "")
+                (match_operand:SI 1 "gr_reg_or_8bit_and_adjusted_operand" "")))]
   ""
-  "
 {
   ia64_compare_op0 = operands[0];
   ia64_compare_op1 = operands[1];
   DONE;
-}")
+})
 
 (define_expand "cmpdi"
   [(set (cc0)
-        (compare (match_operand:DI 0 "register_operand" "")
-                (match_operand:DI 1 "reg_or_8bit_and_adjusted_operand" "")))]
+        (compare (match_operand:DI 0 "gr_register_operand" "")
+                (match_operand:DI 1 "gr_reg_or_8bit_and_adjusted_operand" "")))]
   ""
-  "
 {
   ia64_compare_op0 = operands[0];
   ia64_compare_op1 = operands[1];
   DONE;
-}")
+})
 
 (define_expand "cmpsf"
   [(set (cc0)
-        (compare (match_operand:SF 0 "reg_or_fp01_operand" "")
-                (match_operand:SF 1 "reg_or_fp01_operand" "")))]
+        (compare (match_operand:SF 0 "fr_reg_or_fp01_operand" "")
+                (match_operand:SF 1 "fr_reg_or_fp01_operand" "")))]
   ""
-  "
 {
   ia64_compare_op0 = operands[0];
   ia64_compare_op1 = operands[1];
   DONE;
-}")
+})
 
 (define_expand "cmpdf"
   [(set (cc0)
-        (compare (match_operand:DF 0 "reg_or_fp01_operand" "")
-                (match_operand:DF 1 "reg_or_fp01_operand" "")))]
+        (compare (match_operand:DF 0 "fr_reg_or_fp01_operand" "")
+                (match_operand:DF 1 "fr_reg_or_fp01_operand" "")))]
   ""
-  "
 {
   ia64_compare_op0 = operands[0];
   ia64_compare_op1 = operands[1];
   DONE;
-}")
+})
 
-(define_expand "cmptf"
+(define_expand "cmpxf"
   [(set (cc0)
-        (compare (match_operand:TF 0 "tfreg_or_fp01_operand" "")
-                (match_operand:TF 1 "tfreg_or_fp01_operand" "")))]
+        (compare (match_operand:XF 0 "xfreg_or_fp01_operand" "")
+                (match_operand:XF 1 "xfreg_or_fp01_operand" "")))]
   ""
-  "
 {
   ia64_compare_op0 = operands[0];
   ia64_compare_op1 = operands[1];
   DONE;
-}")
+})
+
+(define_expand "cmptf"
+  [(set (cc0)
+        (compare (match_operand:TF 0 "gr_register_operand" "")
+                (match_operand:TF 1 "gr_register_operand" "")))]
+  "TARGET_HPUX"
+{
+  ia64_compare_op0 = operands[0];
+  ia64_compare_op1 = operands[1];
+  DONE;
+})
 
 (define_insn "*cmpsi_normal"
-  [(set (match_operand:CC 0 "register_operand" "=c")
-       (match_operator:CC 1 "normal_comparison_operator"
-                          [(match_operand:SI 2 "register_operand" "r")
-                           (match_operand:SI 3 "reg_or_8bit_operand" "rK")]))]
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (match_operator:BI 1 "normal_comparison_operator"
+          [(match_operand:SI 2 "gr_register_operand" "r")
+           (match_operand:SI 3 "gr_reg_or_8bit_operand" "rK")]))]
   ""
   "cmp4.%C1 %0, %I0 = %3, %2"
-  [(set_attr "type" "A")])
+  [(set_attr "itanium_class" "icmp")])
+
+;; We use %r3 because it is possible for us to match a 0, and two of the
+;; unsigned comparisons don't accept immediate operands of zero.
 
 (define_insn "*cmpsi_adjusted"
-  [(set (match_operand:CC 0 "register_operand" "=c")
-       (match_operator:CC 1 "adjusted_comparison_operator"
-                          [(match_operand:SI 2 "register_operand" "r")
-                           (match_operand:SI 3 "reg_or_8bit_adjusted_operand"
-                                             "rL")]))]
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (match_operator:BI 1 "adjusted_comparison_operator"
+          [(match_operand:SI 2 "gr_register_operand" "r")
+           (match_operand:SI 3 "gr_reg_or_8bit_adjusted_operand" "rL")]))]
   ""
-  "cmp4.%C1 %0, %I0 = %3, %2"
-  [(set_attr "type" "A")])
+  "cmp4.%C1 %0, %I0 = %r3, %2"
+  [(set_attr "itanium_class" "icmp")])
 
 (define_insn "*cmpdi_normal"
-  [(set (match_operand:CC 0 "register_operand" "=c")
-       (match_operator:CC 1 "normal_comparison_operator"
-                          [(match_operand:DI 2 "register_operand" "r")
-                           (match_operand:DI 3 "reg_or_8bit_operand" "rK")]))]
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (match_operator:BI 1 "normal_comparison_operator"
+          [(match_operand:DI 2 "gr_reg_or_0_operand" "rO")
+           (match_operand:DI 3 "gr_reg_or_8bit_operand" "rK")]))]
   ""
-  "cmp.%C1 %0, %I0 = %3, %2"
-  [(set_attr "type" "A")])
+  "cmp.%C1 %0, %I0 = %3, %r2"
+  [(set_attr "itanium_class" "icmp")])
+
+;; We use %r3 because it is possible for us to match a 0, and two of the
+;; unsigned comparisons don't accept immediate operands of zero.
 
 (define_insn "*cmpdi_adjusted"
-  [(set (match_operand:CC 0 "register_operand" "=c")
-       (match_operator:CC 1 "adjusted_comparison_operator"
-                          [(match_operand:DI 2 "register_operand" "r")
-                           (match_operand:DI 3 "reg_or_8bit_adjusted_operand"
-                                             "rL")]))]
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (match_operator:BI 1 "adjusted_comparison_operator"
+          [(match_operand:DI 2 "gr_register_operand" "r")
+           (match_operand:DI 3 "gr_reg_or_8bit_adjusted_operand" "rL")]))]
   ""
-  "cmp.%C1 %0, %I0 = %3, %2"
-  [(set_attr "type" "A")])
+  "cmp.%C1 %0, %I0 = %r3, %2"
+  [(set_attr "itanium_class" "icmp")])
 
 (define_insn "*cmpsf_internal"
-  [(set (match_operand:CC 0 "register_operand" "=c")
-       (match_operator:CC 1 "comparison_operator"
-                          [(match_operand:SF 2 "reg_or_fp01_operand" "fG")
-                           (match_operand:SF 3 "reg_or_fp01_operand" "fG")]))]
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (match_operator:BI 1 "comparison_operator"
+          [(match_operand:SF 2 "fr_reg_or_fp01_operand" "fG")
+           (match_operand:SF 3 "fr_reg_or_fp01_operand" "fG")]))]
   ""
   "fcmp.%D1 %0, %I0 = %F2, %F3"
-  [(set_attr "type" "F")])
+  [(set_attr "itanium_class" "fcmp")])
 
 (define_insn "*cmpdf_internal"
-  [(set (match_operand:CC 0 "register_operand" "=c")
-       (match_operator:CC 1 "comparison_operator"
-                          [(match_operand:DF 2 "reg_or_fp01_operand" "fG")
-                           (match_operand:DF 3 "reg_or_fp01_operand" "fG")]))]
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (match_operator:BI 1 "comparison_operator"
+          [(match_operand:DF 2 "fr_reg_or_fp01_operand" "fG")
+           (match_operand:DF 3 "fr_reg_or_fp01_operand" "fG")]))]
   ""
   "fcmp.%D1 %0, %I0 = %F2, %F3"
-  [(set_attr "type" "F")])
+  [(set_attr "itanium_class" "fcmp")])
 
-(define_insn "*cmptf_internal"
-  [(set (match_operand:CC 0 "register_operand" "=c")
-       (match_operator:CC 1 "comparison_operator"
-                  [(match_operand:TF 2 "tfreg_or_fp01_operand" "fG")
-                   (match_operand:TF 3 "tfreg_or_fp01_operand" "fG")]))]
+(define_insn "*cmpxf_internal"
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (match_operator:BI 1 "comparison_operator"
+                  [(match_operand:XF 2 "xfreg_or_fp01_operand" "fG")
+                   (match_operand:XF 3 "xfreg_or_fp01_operand" "fG")]))]
   ""
   "fcmp.%D1 %0, %I0 = %F2, %F3"
-  [(set_attr "type" "F")])
+  [(set_attr "itanium_class" "fcmp")])
 
 ;; ??? Can this pattern be generated?
 
 (define_insn "*bit_zero"
-  [(set (match_operand:CC 0 "register_operand" "=c")
-       (eq:CC (zero_extract:DI (match_operand:DI 1 "register_operand" "r")
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (eq:BI (zero_extract:DI (match_operand:DI 1 "gr_register_operand" "r")
                                (const_int 1)
                                (match_operand:DI 2 "immediate_operand" "n"))
               (const_int 0)))]
   ""
   "tbit.z %0, %I0 = %1, %2"
-  [(set_attr "type" "I")])
+  [(set_attr "itanium_class" "tbit")])
 
 (define_insn "*bit_one"
-  [(set (match_operand:CC 0 "register_operand" "=c")
-       (ne:CC (zero_extract:DI (match_operand:DI 1 "register_operand" "r")
+  [(set (match_operand:BI 0 "register_operand" "=c")
+       (ne:BI (zero_extract:DI (match_operand:DI 1 "gr_register_operand" "r")
                                (const_int 1)
                                (match_operand:DI 2 "immediate_operand" "n"))
               (const_int 0)))]
   ""
   "tbit.nz %0, %I0 = %1, %2"
-  [(set_attr "type" "I")])
-
-;; ??? We also need this if we run out of PR regs and need to spill some.
-
-;; ??? We need this if a CCmode value does not get allocated to a hard
-;; register.  This happens if we cse/gcse a CCmode value across a call, and the
-;; function has a nonlocal goto.  This is because global does not allocate
-;; call crossing pseudos to hard registers when current_function_has_
-;; nonlocal_goto is true.  This is relatively common for C++ programs that
-;; use exceptions.  See ia64_secondary_reload_class.
-
-;; We use a define_expand here so that cse/gcse/combine can't accidentally
-;; create movcc insns.  If this was a named define_insn, we would not be able
-;; to make it conditional on reload.
-
-(define_expand "movcc"
-  [(set (match_operand:CC 0 "nonimmediate_operand" "")
-       (match_operand:CC 1 "move_operand" ""))]
-  ""
-  "
-{
-  if (! reload_in_progress && ! reload_completed)
-    FAIL;
-}")
-
-(define_insn "*movcc_internal"
-  [(set (match_operand:CC 0 "nonimmediate_operand" "=r,c,r,m")
-       (match_operand:CC 1 "move_operand" "c,r,m,r"))]
-  "reload_in_progress || reload_completed"
-  "@
-   #
-   cmp4.ne %0, %I0 = %1, r0
-   ld4%O1 %0 = %1%P1
-   st4%Q0 %0 = %1%P0"
-  [(set_attr "type" "unknown,A,M,M")])
-
-(define_split
-  [(set (match_operand:CC 0 "register_operand" "")
-       (match_operand:CC 1 "register_operand" ""))]
-  "reload_completed
-   && GET_CODE (operands[0]) == REG && GR_REGNO_P (REGNO (operands[0]))
-   && GET_CODE (operands[1]) == REG && PR_REGNO_P (REGNO (operands[1]))"
-  [(set (match_dup 2)
-       (if_then_else:DI (ne:CC (match_dup 1) (const_int 0))
-                        (const_int 1)
-                        (match_dup 2)))
-   (set (match_dup 2)
-       (if_then_else:DI (ne:CC (match_dup 1) (const_int 0))
-                        (match_dup 2)
-                        (const_int 0)))]
-  "operands[2] = gen_rtx_SUBREG (DImode, operands[0], 0);")
-
+  [(set_attr "itanium_class" "tbit")])
 \f
 ;; ::::::::::::::::::::
 ;; ::
 ;; ::::::::::::::::::::
 
 (define_expand "beq"
-  [(set (match_dup 1)
-       (eq:CC (match_dup 2)
-              (match_dup 3)))
-   (set (pc)
-       (if_then_else (ne:CC (match_dup 1)
-                            (const_int 0))
+  [(set (pc)
+       (if_then_else (match_dup 1)
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (EQ, VOIDmode);")
 
 (define_expand "bne"
-  [(set (match_dup 1)
-       (ne:CC (match_dup 2)
-              (match_dup 3)))
-   (set (pc)
-       (if_then_else (ne:CC (match_dup 1)
-                            (const_int 0))
+  [(set (pc)
+       (if_then_else (match_dup 1)
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (NE, VOIDmode);")
 
 (define_expand "blt"
-  [(set (match_dup 1)
-       (lt:CC (match_dup 2)
-              (match_dup 3)))
-   (set (pc)
-       (if_then_else (ne:CC (match_dup 1)
-                            (const_int 0))
+  [(set (pc)
+       (if_then_else (match_dup 1)
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (LT, VOIDmode);")
 
 (define_expand "ble"
-  [(set (match_dup 1)
-       (le:CC (match_dup 2)
-              (match_dup 3)))
-   (set (pc)
-       (if_then_else (ne:CC (match_dup 1)
-                            (const_int 0))
+  [(set (pc)
+       (if_then_else (match_dup 1)
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (LE, VOIDmode);")
 
 (define_expand "bgt"
-  [(set (match_dup 1)
-       (gt:CC (match_dup 2)
-              (match_dup 3)))
-   (set (pc)
-       (if_then_else (ne:CC (match_dup 1)
-                            (const_int 0))
+  [(set (pc)
+       (if_then_else (match_dup 1)
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (GT, VOIDmode);")
 
 (define_expand "bge"
-  [(set (match_dup 1)
-       (ge:CC (match_dup 2)
-              (match_dup 3)))
-   (set (pc)
-       (if_then_else (ne:CC (match_dup 1)
-                            (const_int 0))
+  [(set (pc)
+       (if_then_else (match_dup 1)
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (GE, VOIDmode);")
 
 (define_expand "bltu"
-  [(set (match_dup 1)
-       (ltu:CC (match_dup 2)
-               (match_dup 3)))
-   (set (pc)
-       (if_then_else (ne:CC (match_dup 1)
-                            (const_int 0))
+  [(set (pc)
+       (if_then_else (match_dup 1)
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (LTU, VOIDmode);")
 
 (define_expand "bleu"
-  [(set (match_dup 1)
-       (leu:CC (match_dup 2)
-               (match_dup 3)))
-   (set (pc)
-       (if_then_else (ne:CC (match_dup 1)
-                            (const_int 0))
+  [(set (pc)
+       (if_then_else (match_dup 1)
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (LEU, VOIDmode);")
 
 (define_expand "bgtu"
-  [(set (match_dup 1)
-       (gtu:CC (match_dup 2)
-               (match_dup 3)))
-   (set (pc)
-       (if_then_else (ne:CC (match_dup 1)
-                            (const_int 0))
+  [(set (pc)
+       (if_then_else (match_dup 1)
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (GTU, VOIDmode);")
 
 (define_expand "bgeu"
-  [(set (match_dup 1)
-       (geu:CC (match_dup 2)
-               (match_dup 3)))
-   (set (pc)
-       (if_then_else (ne:CC (match_dup 1)
-                            (const_int 0))
+  [(set (pc)
+       (if_then_else (match_dup 1)
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (GEU, VOIDmode);")
 
 (define_expand "bunordered"
-  [(set (match_dup 1)
-       (unordered:CC (match_dup 2)
-                     (match_dup 3)))
-   (set (pc)
-       (if_then_else (ne:CC (match_dup 1)
-                            (const_int 0))
+  [(set (pc)
+       (if_then_else (match_dup 1)
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (UNORDERED, VOIDmode);")
 
 (define_expand "bordered"
-  [(set (match_dup 1)
-       (ordered:CC (match_dup 2)
-                     (match_dup 3)))
-   (set (pc)
-       (if_then_else (ne:CC (match_dup 1)
-                            (const_int 0))
+  [(set (pc)
+       (if_then_else (match_dup 1)
                      (label_ref (match_operand 0 "" ""))
                      (pc)))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (ORDERED, VOIDmode);")
 
 (define_insn "*br_true"
   [(set (pc)
        (if_then_else (match_operator 0 "predicate_operator"
-                       [(match_operand:CC 1 "register_operand" "c")
+                       [(match_operand:BI 1 "register_operand" "c")
                         (const_int 0)])
                      (label_ref (match_operand 2 "" ""))
                      (pc)))]
   ""
   "(%J0) br.cond%+ %l2"
-  [(set_attr "type" "B")
+  [(set_attr "itanium_class" "br")
    (set_attr "predicable" "no")])
 
 (define_insn "*br_false"
   [(set (pc)
        (if_then_else (match_operator 0 "predicate_operator"
-                       [(match_operand:CC 1 "register_operand" "c")
+                       [(match_operand:BI 1 "register_operand" "c")
                         (const_int 0)])
                      (pc)
                      (label_ref (match_operand 2 "" ""))))]
   ""
   "(%j0) br.cond%+ %l2"
-  [(set_attr "type" "B")
+  [(set_attr "itanium_class" "br")
    (set_attr "predicable" "no")])
 \f
 ;; ::::::::::::::::::::
    (use (match_operand 3 "" ""))       ; loop level
    (use (match_operand 4 "" ""))]      ; label
   ""
-  "
 {
   /* Only use cloop on innermost loops.  */
   if (INTVAL (operands[3]) > 1)
   emit_jump_insn (gen_doloop_end_internal (gen_rtx_REG (DImode, AR_LC_REGNUM),
                                           operands[4]));
   DONE;
-}")
+})
 
 (define_insn "doloop_end_internal"
   [(set (pc) (if_then_else (ne (match_operand:DI 0 "ar_lc_reg_operand" "")
                (label_ref (match_operand 1 "" ""))
                (pc)))
    (set (match_dup 0) (if_then_else:DI (ne (match_dup 0) (const_int 0))
-                        (match_dup 0)
-                        (plus:DI (match_dup 0) (const_int -1))))]
+                        (plus:DI (match_dup 0) (const_int -1))
+                        (match_dup 0)))]
   ""
   "br.cloop.sptk.few %l1"
-  [(set_attr "type" "B")
+  [(set_attr "itanium_class" "br")
    (set_attr "predicable" "no")])
 \f
 ;; ::::::::::::::::::::
 ;; ::::::::::::::::::::
 
 (define_expand "seq"
-  [(set (match_dup 1)
-       (eq:CC (match_dup 2)
-              (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
-       (ne:DI (match_dup 1) (const_int 0)))]
+  [(set (match_operand:DI 0 "gr_register_operand" "") (match_dup 1))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (EQ, DImode);")
 
 (define_expand "sne"
-  [(set (match_dup 1)
-       (ne:CC (match_dup 2)
-              (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
-       (ne:DI (match_dup 1) (const_int 0)))]
+  [(set (match_operand:DI 0 "gr_register_operand" "") (match_dup 1))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (NE, DImode);")
 
 (define_expand "slt"
-  [(set (match_dup 1)
-       (lt:CC (match_dup 2)
-              (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
-       (ne:DI (match_dup 1) (const_int 0)))]
+  [(set (match_operand:DI 0 "gr_register_operand" "") (match_dup 1))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (LT, DImode);")
 
 (define_expand "sle"
-  [(set (match_dup 1)
-       (le:CC (match_dup 2)
-              (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
-       (ne:DI (match_dup 1) (const_int 0)))]
+  [(set (match_operand:DI 0 "gr_register_operand" "") (match_dup 1))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (LE, DImode);")
 
 (define_expand "sgt"
-  [(set (match_dup 1)
-       (gt:CC (match_dup 2)
-              (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
-       (ne:DI (match_dup 1) (const_int 0)))]
+  [(set (match_operand:DI 0 "gr_register_operand" "") (match_dup 1))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (GT, DImode);")
 
 (define_expand "sge"
-  [(set (match_dup 1)
-       (ge:CC (match_dup 2)
-              (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
-       (ne:DI (match_dup 1) (const_int 0)))]
+  [(set (match_operand:DI 0 "gr_register_operand" "") (match_dup 1))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (GE, DImode);")
 
 (define_expand "sltu"
-  [(set (match_dup 1)
-       (ltu:CC (match_dup 2)
-               (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
-       (ne:DI (match_dup 1) (const_int 0)))]
+  [(set (match_operand:DI 0 "gr_register_operand" "") (match_dup 1))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (LTU, DImode);")
 
 (define_expand "sleu"
-  [(set (match_dup 1)
-       (leu:CC (match_dup 2)
-               (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
-       (ne:DI (match_dup 1) (const_int 0)))]
+  [(set (match_operand:DI 0 "gr_register_operand" "") (match_dup 1))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (LEU, DImode);")
 
 (define_expand "sgtu"
-  [(set (match_dup 1)
-       (gtu:CC (match_dup 2)
-               (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
-       (ne:DI (match_dup 1) (const_int 0)))]
+  [(set (match_operand:DI 0 "gr_register_operand" "") (match_dup 1))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (GTU, DImode);")
 
 (define_expand "sgeu"
-  [(set (match_dup 1)
-       (geu:CC (match_dup 2)
-               (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
-       (ne:DI (match_dup 1) (const_int 0)))]
+  [(set (match_operand:DI 0 "gr_register_operand" "") (match_dup 1))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (GEU, DImode);")
 
 (define_expand "sunordered"
-  [(set (match_dup 1)
-       (unordered:CC (match_dup 2)
-                     (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
-       (ne:DI (match_dup 1) (const_int 0)))]
+  [(set (match_operand:DI 0 "gr_register_operand" "") (match_dup 1))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (UNORDERED, DImode);")
 
 (define_expand "sordered"
-  [(set (match_dup 1)
-       (ordered:CC (match_dup 2)
-                     (match_dup 3)))
-   (set (match_operand:DI 0 "register_operand" "")
-       (ne:DI (match_dup 1) (const_int 0)))]
+  [(set (match_operand:DI 0 "gr_register_operand" "") (match_dup 1))]
   ""
-  "
-{
-  operands[1] = gen_reg_rtx (CCmode);
-  operands[2] = ia64_compare_op0;
-  operands[3] = ia64_compare_op1;
-}")
+  "operands[1] = ia64_expand_compare (ORDERED, DImode);")
 
 ;; Don't allow memory as destination here, because cmov/cmov/st is more
 ;; efficient than mov/mov/cst/cst.
 
-(define_insn "*sne_internal"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (ne:DI (match_operand:CC 1 "register_operand" "c")
+(define_insn_and_split "*sne_internal"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (ne:DI (match_operand:BI 1 "register_operand" "c")
               (const_int 0)))]
   ""
   "#"
-  [(set_attr "type" "unknown")])
-
-(define_split
-  [(set (match_operand:DI 0 "register_operand" "")
-       (ne:DI (match_operand:CC 1 "register_operand" "")
-              (const_int 0)))]
   "reload_completed"
-  [(set (match_dup 0)
-       (if_then_else:DI (ne:CC (match_dup 1) (const_int 0))
-                        (const_int 1)
-                        (match_dup 0)))
-   (set (match_dup 0)
-       (if_then_else:DI (ne:CC (match_dup 1) (const_int 0))
-                        (match_dup 0)
-                        (const_int 0)))]
-  "")
-
-;; ??? Unknown if this can be matched.
+  [(cond_exec (ne (match_dup 1) (const_int 0))
+     (set (match_dup 0) (const_int 1)))
+   (cond_exec (eq (match_dup 1) (const_int 0))
+     (set (match_dup 0) (const_int 0)))]
+  ""
+  [(set_attr "itanium_class" "unknown")])
 
-(define_insn "*seq_internal"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-       (eq:DI (match_operand:CC 1 "register_operand" "c")
+(define_insn_and_split "*seq_internal"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+       (eq:DI (match_operand:BI 1 "register_operand" "c")
               (const_int 0)))]
   ""
   "#"
-  [(set_attr "type" "unknown")])
-
-;; ??? Unknown if this can be matched.
-
-(define_split
-  [(set (match_operand:DI 0 "register_operand" "")
-       (eq:DI (match_operand:CC 1 "register_operand" "")
-              (const_int 0)))]
   "reload_completed"
-  [(set (match_dup 0)
-       (if_then_else:DI (eq:CC (match_dup 1) (const_int 0))
-                        (const_int 1)
-                        (match_dup 0)))
-   (set (match_dup 0)
-       (if_then_else:DI (eq:CC (match_dup 1) (const_int 0))
-                        (match_dup 0)
-                        (const_int 0)))]
-  "")
-
+  [(cond_exec (ne (match_dup 1) (const_int 0))
+     (set (match_dup 0) (const_int 0)))
+   (cond_exec (eq (match_dup 1) (const_int 0))
+     (set (match_dup 0) (const_int 1)))]
+  ""
+  [(set_attr "itanium_class" "unknown")])
 \f
 ;; ::::::::::::::::::::
 ;; ::
 ;; DImode if_then_else patterns.
 ;;
 
-;; Errata 72 workaround.
-(define_insn "*cmovdi_internal_astep"
-  [(set (match_operand:DI 0 "nonimmediate_operand"
-               "=r,*f,Q,*b,r,*f,Q,*b,r,*f,Q,*b")
-       (if_then_else:DI
-         (match_operator:CC 4 "predicate_operator"
-           [(match_operand:CC 1 "register_operand"
-                              "c,c,c,c,c,c,c,c,c,c,c,c")
-            (const_int 0)])
-         (match_operand:DI 2 "general_operand"
-               "0,0,0,0,ri*f*b,rO,*f,r,ri*f*b,rO,*f,r")
-         (match_operand:DI 3 "general_operand"
-               "ri*f*b,rO,*f,r,0,0,0,0,ri*f*b,rO,*f,r")))]
-  "TARGET_A_STEP"
-  "* abort ();"
-  [(set_attr "predicable" "no")])
-
 (define_insn "*cmovdi_internal"
-  [(set (match_operand:DI 0 "nonimmediate_operand"
-               "=r,m,*f,Q,*b*d*e,r,m,*f,Q,*b*d*e,r,m,*f,Q,*b*d*e")
+  [(set (match_operand:DI 0 "destination_operand"
+          "= r,  r,  r,   r,  r,  r,   r, r, r,   r, m, Q, *f,*b,*d*e")
        (if_then_else:DI
-         (match_operator:CC 4 "predicate_operator"
-           [(match_operand:CC 1 "register_operand"
-                              "c,c,c,c,c,c,c,c,c,c,c,c,c,c,c")
+         (match_operator 4 "predicate_operator"
+           [(match_operand:BI 1 "register_operand"
+               "c,c,c,c,c,c,c,c,c,c,c,c,c,c,c")
             (const_int 0)])
-         (match_operand:DI 2 "general_operand"
-               "0,0,0,0,0,rim*f*b*d*e,rO,rOQ,*f,r,rim*f*b*d*e,rO,rOQ,*f,r")
-         (match_operand:DI 3 "general_operand"
-               "rim*f*b*d*e,rO,rOQ,*f,r,0,0,0,0,0,rim*f*b*d*e,rO,rOQ,*f,r")))]
-  "! TARGET_A_STEP"
-  "* abort ();"
+         (match_operand:DI 2 "move_operand"
+          "rim, *f, *b,*d*e,rim,rim, rim,*f,*b,*d*e,rO,*f,rOQ,rO,  rK")
+         (match_operand:DI 3 "move_operand"
+          "rim,rim,rim, rim, *f, *b,*d*e,*f,*b,*d*e,rO,*f,rOQ,rO,  rK")))]
+  "ia64_move_ok (operands[0], operands[2])
+   && ia64_move_ok (operands[0], operands[3])"
+  { gcc_unreachable (); }
   [(set_attr "predicable" "no")])
 
 (define_split
-  [(set (match_operand 0 "nonimmediate_operand" "")
+  [(set (match_operand 0 "destination_operand" "")
        (if_then_else
-         (match_operator:CC 4 "predicate_operator"
-           [(match_operand:CC 1 "register_operand" "")
+         (match_operator 4 "predicate_operator"
+           [(match_operand:BI 1 "register_operand" "")
             (const_int 0)])
-         (match_operand 2 "general_operand" "")
-         (match_operand 3 "general_operand" "")))]
+         (match_operand 2 "move_operand" "")
+         (match_operand 3 "move_operand" "")))]
   "reload_completed"
   [(const_int 0)]
-  "
 {
-  rtx tmp;
-  if (! rtx_equal_p (operands[0], operands[2]))
+  bool emitted_something = false;
+  rtx dest = operands[0];
+  rtx srct = operands[2];
+  rtx srcf = operands[3];
+  rtx cond = operands[4];
+
+  if (! rtx_equal_p (dest, srct))
     {
-      tmp = gen_rtx_SET (VOIDmode, operands[0], operands[2]);
-      tmp = gen_rtx_COND_EXEC (VOIDmode, operands[4], tmp);
-      emit_insn (tmp);
+      ia64_emit_cond_move (dest, srct, cond);
+      emitted_something = true;
     }
-  if (! rtx_equal_p (operands[0], operands[3]))
+  if (! rtx_equal_p (dest, srcf))
     {
-      tmp = gen_rtx_fmt_ee (GET_CODE (operands[4]) == NE ? EQ : NE,
-                           CCmode, operands[1], const0_rtx);
-      tmp = gen_rtx_COND_EXEC (VOIDmode, tmp,
-                              gen_rtx_SET (VOIDmode, operands[0],
-                                           operands[3]));
-      emit_insn (tmp);
+      cond = gen_rtx_fmt_ee (GET_CODE (cond) == NE ? EQ : NE,
+                            VOIDmode, operands[1], const0_rtx);
+      ia64_emit_cond_move (dest, srcf, cond);
+      emitted_something = true;
     }
+  if (! emitted_something)
+    emit_note (NOTE_INSN_DELETED);
   DONE;
-}")
+})
 
 ;; Absolute value pattern.
 
 (define_insn "*absdi2_internal"
-  [(set (match_operand:DI 0 "register_operand" "=r,r")
+  [(set (match_operand:DI 0 "gr_register_operand" "=r,r")
        (if_then_else:DI
-         (match_operator:CC 4 "predicate_operator"
-           [(match_operand:CC 1 "register_operand" "c,c")
+         (match_operator 4 "predicate_operator"
+           [(match_operand:BI 1 "register_operand" "c,c")
             (const_int 0)])
-         (neg:DI (match_operand:DI 2 "reg_or_22bit_operand" "rI,rI"))
-         (match_operand:DI 3 "reg_or_22bit_operand" "0,rI")))]
+         (neg:DI (match_operand:DI 2 "gr_reg_or_22bit_operand" "rI,rI"))
+         (match_operand:DI 3 "gr_reg_or_22bit_operand" "0,rI")))]
   ""
   "#"
-  [(set_attr "type" "A,unknown")
+  [(set_attr "itanium_class" "ialu,unknown")
    (set_attr "predicable" "no")])
 
 (define_split
   [(set (match_operand:DI 0 "register_operand" "")
        (if_then_else:DI
-         (match_operator:CC 4 "predicate_operator"
-           [(match_operand:CC 1 "register_operand" "c,c")
+         (match_operator 4 "predicate_operator"
+           [(match_operand:BI 1 "register_operand" "c,c")
             (const_int 0)])
-         (neg:DI (match_operand:DI 2 "reg_or_22bit_operand" ""))
-         (match_operand:DI 3 "reg_or_22bit_operand" "")))]
+         (neg:DI (match_operand:DI 2 "gr_reg_or_22bit_operand" ""))
+         (match_operand:DI 3 "gr_reg_or_22bit_operand" "")))]
   "reload_completed && rtx_equal_p (operands[0], operands[3])"
   [(cond_exec
      (match_dup 4)
 (define_split
   [(set (match_operand:DI 0 "register_operand" "")
        (if_then_else:DI
-         (match_operator:CC 4 "predicate_operator"
-           [(match_operand:CC 1 "register_operand" "c,c")
+         (match_operator 4 "predicate_operator"
+           [(match_operand:BI 1 "register_operand" "c,c")
             (const_int 0)])
-         (neg:DI (match_operand:DI 2 "reg_or_22bit_operand" ""))
-         (match_operand:DI 3 "reg_or_22bit_operand" "")))]
+         (neg:DI (match_operand:DI 2 "gr_reg_or_22bit_operand" ""))
+         (match_operand:DI 3 "gr_reg_or_22bit_operand" "")))]
   "reload_completed"
   [(cond_exec
      (match_dup 4)
    (cond_exec
      (match_dup 5)
      (set (match_dup 0) (match_dup 3)))]
-  "
 {
   operands[5] = gen_rtx_fmt_ee (GET_CODE (operands[4]) == NE ? EQ : NE,
-                               CCmode, operands[1], const0_rtx);
-}")
+                               VOIDmode, operands[1], const0_rtx);
+})
 
 ;;
 ;; SImode if_then_else patterns.
 ;;
 
-(define_insn "*cmovsi_internal_astep"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "=r,*f,r,*f,r,*f")
-       (if_then_else:SI
-         (match_operator:CC 4 "predicate_operator"
-           [(match_operand:CC 1 "register_operand" "c,c,c,c,c,c")
-            (const_int 0)])
-         (match_operand:SI 2 "general_operand"
-                   "0,0,ri*f,rO,ri*f,rO")
-         (match_operand:SI 3 "general_operand"
-                   "ri*f,rO,0,0,ri*f,rO")))]
-  "TARGET_A_STEP"
-  "* abort ();"
-  [(set_attr "predicable" "no")])
-
 (define_insn "*cmovsi_internal"
-  [(set (match_operand:SI 0 "nonimmediate_operand" "=r,m,*f,r,m,*f,r,m,*f")
+  [(set (match_operand:SI 0 "destination_operand" "=r,m,*f,r,m,*f,r,m,*f")
        (if_then_else:SI
-         (match_operator:CC 4 "predicate_operator"
-           [(match_operand:CC 1 "register_operand" "c,c,c,c,c,c,c,c,c")
+         (match_operator 4 "predicate_operator"
+           [(match_operand:BI 1 "register_operand" "c,c,c,c,c,c,c,c,c")
             (const_int 0)])
-         (match_operand:SI 2 "general_operand"
+         (match_operand:SI 2 "move_operand"
                    "0,0,0,rim*f,rO,rO,rim*f,rO,rO")
-         (match_operand:SI 3 "general_operand"
+         (match_operand:SI 3 "move_operand"
                    "rim*f,rO,rO,0,0,0,rim*f,rO,rO")))]
-  "! TARGET_A_STEP"
-  "* abort ();"
+  "ia64_move_ok (operands[0], operands[2])
+   && ia64_move_ok (operands[0], operands[3])"
+  { gcc_unreachable (); }
   [(set_attr "predicable" "no")])
 
 (define_insn "*abssi2_internal"
-  [(set (match_operand:SI 0 "register_operand" "=r,r")
+  [(set (match_operand:SI 0 "gr_register_operand" "=r,r")
        (if_then_else:SI
-         (match_operator:CC 4 "predicate_operator"
-           [(match_operand:CC 1 "register_operand" "c,c")
+         (match_operator 4 "predicate_operator"
+           [(match_operand:BI 1 "register_operand" "c,c")
             (const_int 0)])
-         (neg:SI (match_operand:SI 3 "reg_or_22bit_operand" "rI,rI"))
-         (match_operand:SI 2 "reg_or_22bit_operand" "0,rI")))]
+         (neg:SI (match_operand:SI 3 "gr_reg_or_22bit_operand" "rI,rI"))
+         (match_operand:SI 2 "gr_reg_or_22bit_operand" "0,rI")))]
   ""
   "#"
-  [(set_attr "type" "A,unknown")
+  [(set_attr "itanium_class" "ialu,unknown")
    (set_attr "predicable" "no")])
 
 (define_split
   [(set (match_operand:SI 0 "register_operand" "")
        (if_then_else:SI
-         (match_operator:CC 4 "predicate_operator"
-           [(match_operand:CC 1 "register_operand" "c,c")
+         (match_operator 4 "predicate_operator"
+           [(match_operand:BI 1 "register_operand" "c,c")
             (const_int 0)])
-         (neg:SI (match_operand:SI 2 "reg_or_22bit_operand" ""))
-         (match_operand:SI 3 "reg_or_22bit_operand" "")))]
+         (neg:SI (match_operand:SI 2 "gr_reg_or_22bit_operand" ""))
+         (match_operand:SI 3 "gr_reg_or_22bit_operand" "")))]
   "reload_completed && rtx_equal_p (operands[0], operands[3])"
   [(cond_exec
      (match_dup 4)
 (define_split
   [(set (match_operand:SI 0 "register_operand" "")
        (if_then_else:SI
-         (match_operator:CC 4 "predicate_operator"
-           [(match_operand:CC 1 "register_operand" "c,c")
+         (match_operator 4 "predicate_operator"
+           [(match_operand:BI 1 "register_operand" "c,c")
             (const_int 0)])
-         (neg:SI (match_operand:SI 2 "reg_or_22bit_operand" ""))
-         (match_operand:SI 3 "reg_or_22bit_operand" "")))]
+         (neg:SI (match_operand:SI 2 "gr_reg_or_22bit_operand" ""))
+         (match_operand:SI 3 "gr_reg_or_22bit_operand" "")))]
   "reload_completed"
   [(cond_exec
      (match_dup 4)
    (cond_exec
      (match_dup 5)
      (set (match_dup 0) (match_dup 3)))]
-  "
 {
   operands[5] = gen_rtx_fmt_ee (GET_CODE (operands[4]) == NE ? EQ : NE,
-                               CCmode, operands[1], const0_rtx);
-}")
+                               VOIDmode, operands[1], const0_rtx);
+})
+
+(define_insn_and_split "*cond_opsi2_internal"
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+       (match_operator:SI 5 "condop_operator"
+         [(if_then_else:SI
+            (match_operator 6 "predicate_operator"
+              [(match_operand:BI 1 "register_operand" "c")
+               (const_int 0)])
+            (match_operand:SI 2 "gr_register_operand" "r")
+            (match_operand:SI 3 "gr_register_operand" "r"))
+          (match_operand:SI 4 "gr_register_operand" "r")]))]
+  ""
+  "#"
+  "reload_completed"
+  [(cond_exec
+     (match_dup 6)
+     (set (match_dup 0) (match_op_dup:SI 5 [(match_dup 2) (match_dup 4)])))
+   (cond_exec
+     (match_dup 7)
+     (set (match_dup 0) (match_op_dup:SI 5 [(match_dup 3) (match_dup 4)])))]
+{
+  operands[7] = gen_rtx_fmt_ee (GET_CODE (operands[6]) == NE ? EQ : NE,
+                               VOIDmode, operands[1], const0_rtx);
+}
+  [(set_attr "itanium_class" "ialu")
+   (set_attr "predicable" "no")])
+
+
+(define_insn_and_split "*cond_opsi2_internal_b"
+  [(set (match_operand:SI 0 "gr_register_operand" "=r")
+       (match_operator:SI 5 "condop_operator"
+         [(match_operand:SI 4 "gr_register_operand" "r")
+          (if_then_else:SI
+            (match_operator 6 "predicate_operator"
+              [(match_operand:BI 1 "register_operand" "c")
+               (const_int 0)])
+            (match_operand:SI 2 "gr_register_operand" "r")
+            (match_operand:SI 3 "gr_register_operand" "r"))]))]
+  ""
+  "#"
+  "reload_completed"
+  [(cond_exec
+     (match_dup 6)
+     (set (match_dup 0) (match_op_dup:SI 5 [(match_dup 4) (match_dup 2)])))
+   (cond_exec
+     (match_dup 7)
+     (set (match_dup 0) (match_op_dup:SI 5 [(match_dup 4) (match_dup 3)])))]
+{
+  operands[7] = gen_rtx_fmt_ee (GET_CODE (operands[6]) == NE ? EQ : NE,
+                               VOIDmode, operands[1], const0_rtx);
+}
+  [(set_attr "itanium_class" "ialu")
+   (set_attr "predicable" "no")])
 
 \f
 ;; ::::::::::::::::::::
    (use (match_operand 2 "" ""))
    (use (match_operand 3 "" ""))]
   ""
-  "
 {
-  /* ??? Stripping off the MEM isn't correct.  Will lose alias info.  */
-  rtx addr = XEXP (operands[0], 0);
-  enum machine_mode mode = GET_MODE (addr);
-
-  if (TARGET_NO_PIC || TARGET_AUTO_PIC)
-    emit_call_insn (gen_call_internal (addr, operands[1],
-                                      gen_rtx_REG (DImode, R_BR (0))));
-
-  /* If this is an indirect call, then we have the address of a descriptor.  */
-  else if (! symbolic_operand (addr, mode))
-    emit_insn (gen_indirect_call_pic (addr, operands[1]));
-  else if (TARGET_CONST_GP)
-    emit_call_insn (gen_call_internal (addr, operands[1],
-                                      gen_rtx_REG (DImode, R_BR (0))));
-  else
-    emit_insn (gen_call_pic (addr, operands[1]));
-
+  ia64_expand_call (NULL_RTX, operands[0], operands[2], false);
   DONE;
-}")
-
-(define_expand "indirect_call_pic"
-  [(set (match_dup 2) (reg:DI 1))
-   (set (match_dup 3) (mem:DI (match_operand 0 "" "")))
-   (set (match_dup 4) (plus:DI (match_dup 0) (const_int 8)))
-   (set (reg:DI 1) (mem:DI (match_dup 4)))
-   (parallel [(call (mem:DI (match_dup 3)) (match_operand 1 "" ""))
-             (use (reg:DI 1))
-             (clobber (reg:DI 320))])
-   (set (reg:DI 1) (match_dup 2))]
-  ""
-  "
-{
-  operands[2] = ia64_gp_save_reg (0);
-  operands[3] = gen_reg_rtx (DImode);
-  operands[4] = gen_reg_rtx (DImode);
-}")
-
-;; ??? Saving/restoring the GP register is not needed if we are calling
-;; a function in the same module.
+})
 
-(define_expand "call_pic"
-  [(set (match_dup 2) (reg:DI 1))
-   (parallel [(call (mem:DI (match_operand 0 "" "")) (match_operand 1 "" ""))
-             (use (reg:DI 1))
-             (clobber (reg:DI 320))])
-   (set (reg:DI 1) (match_dup 2))]
+(define_expand "sibcall"
+  [(use (match_operand:DI 0 "" ""))
+   (use (match_operand 1 "" ""))
+   (use (match_operand 2 "" ""))
+   (use (match_operand 3 "" ""))]
   ""
-  "
 {
-  /* ??? Using setjmp_operand is an unsatisfying solution.  Should rethink.  */
-  operands[2] = ia64_gp_save_reg (setjmp_operand (XEXP (operands[0], 0),
-                                                 VOIDmode));
-}")
-
-(define_insn "call_internal"
-  [(call (mem:DI (match_operand:DI 0 "call_operand" "bi"))
-        (match_operand 1 "" ""))
-   (clobber (match_operand:DI 2 "register_operand" "=b"))]
-  ""
-  "br.call%+.many %2 = %0"
-  [(set_attr "type" "B")])
-
-(define_insn "*call_internal1"
-  [(call (mem:DI (match_operand:DI 0 "call_operand" "bi"))
-        (match_operand 1 "" ""))
-   (use (reg:DI 1))
-   (clobber (match_operand:DI 2 "register_operand" "=b"))]
-  ""
-  "br.call%+.many %2 = %0"
-  [(set_attr "type" "B")])
+  ia64_expand_call (NULL_RTX, operands[0], operands[2], true);
+  DONE;
+})
 
 ;; Subroutine call instruction returning a value.  Operand 0 is the hard
-;; register in which the value is returned.  There are three more operands, the
-;; same as the three operands of the `call' instruction (but with numbers
+;; register in which the value is returned.  There are three more operands,
+;; the same as the three operands of the `call' instruction (but with numbers
 ;; increased by one).
-
+;;
 ;; Subroutines that return `BLKmode' objects use the `call' insn.
 
 (define_expand "call_value"
    (use (match_operand 3 "" ""))
    (use (match_operand 4 "" ""))]
   ""
-  "
 {
-  /* ??? Stripping off the MEM isn't correct.  Will lose alias info.  */
-  rtx addr = XEXP (operands[1], 0);
-  enum machine_mode mode = GET_MODE (addr);
+  ia64_expand_call (operands[0], operands[1], operands[3], false);
+  DONE;
+})
 
-  if (TARGET_NO_PIC || TARGET_AUTO_PIC)
-    emit_call_insn (gen_call_value_internal (operands[0], addr, operands[2],
-                                            gen_rtx_REG (DImode, R_BR (0))));
+(define_expand "sibcall_value"
+  [(use (match_operand 0 "" ""))
+   (use (match_operand:DI 1 "" ""))
+   (use (match_operand 2 "" ""))
+   (use (match_operand 3 "" ""))
+   (use (match_operand 4 "" ""))]
+  ""
+{
+  ia64_expand_call (operands[0], operands[1], operands[3], true);
+  DONE;
+})
 
-  /* If this is an indirect call, then we have the address of a descriptor.  */
-  else if (! symbolic_operand (addr, mode))
-    {
-      /* This is for HFA returns.  */
-      if (GET_CODE (operands[0]) == PARALLEL)
-       emit_insn (gen_indirect_call_multiple_values_pic (operands[0], addr,
-                                                         operands[2]));
-      else
-       emit_insn (gen_indirect_call_value_pic (operands[0], addr,
-                                               operands[2]));
-    }
-  else if (TARGET_CONST_GP)
-    emit_call_insn (gen_call_value_internal (operands[0], addr, operands[2],
-                                            gen_rtx_REG (DImode, R_BR (0))));
-  /* This is for HFA returns.  */
-  else if (GET_CODE (operands[0]) == PARALLEL)
-    emit_insn (gen_call_multiple_values_pic (operands[0], addr, operands[2]));
-  else
-    emit_insn (gen_call_value_pic (operands[0], addr, operands[2]));
+;; Call subroutine returning any type.
 
-  DONE;
-}")
-
-(define_expand "indirect_call_value_pic"
-  [(set (match_dup 3) (reg:DI 1))
-   (set (match_dup 4) (mem:DI (match_operand 1 "" "")))
-   (set (match_dup 5) (plus:DI (match_dup 1) (const_int 8)))
-   (set (reg:DI 1) (mem:DI (match_dup 5)))
-   (parallel [(set (match_operand 0 "" "")
-                  (call (mem:DI (match_dup 4)) (match_operand 2 "" "")))
-             (use (reg:DI 1))
-             (clobber (reg:DI 320))])
-   (set (reg:DI 1) (match_dup 3))]
-  ""
-  "
-{
-  operands[3] = ia64_gp_save_reg (0);
-  operands[4] = gen_reg_rtx (DImode);
-  operands[5] = gen_reg_rtx (DImode);
-}")
-
-(define_expand "indirect_call_multiple_values_pic"
-  [(set (match_dup 3) (reg:DI 1))
-   (set (match_dup 4) (mem:DI (match_operand 1 "" "")))
-   (set (match_dup 5) (plus:DI (match_dup 1) (const_int 8)))
-   (set (reg:DI 1) (mem:DI (match_dup 5)))
-   (match_par_dup 6 [(set (match_operand 0 "" "")
-                         (call (mem:DI (match_dup 4))
-                               (match_operand 2 "" "")))
-                    (use (reg:DI 1))
-                    (clobber (reg:DI 320))])
-   (set (reg:DI 1) (match_dup 3))]
-  ""
-  "
+(define_expand "untyped_call"
+  [(parallel [(call (match_operand 0 "" "")
+                   (const_int 0))
+             (match_operand 1 "" "")
+             (match_operand 2 "" "")])]
+  ""
 {
-  int count;
   int i;
-  rtx call;
-
-  operands[3] = ia64_gp_save_reg (0);
-  operands[4] = gen_reg_rtx (DImode);
-  operands[5] = gen_reg_rtx (DImode);
 
-  /* This code is the same as the code in call_multiple_values_pic, except
-     that op3 was replaced with op6 and op1 was replaced with op4.  */
-  call = gen_rtx_CALL (VOIDmode, gen_rtx_MEM (DImode, operands[4]),
-                      operands[2]);
-
-  count = XVECLEN (operands[0], 0);
-  operands[6] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (count + 2));
-  
-  XVECEXP (operands[6], 0, 0)
-    = gen_rtx_SET (VOIDmode, XEXP (XVECEXP (operands[0], 0, 0), 0), call);
-
-  XVECEXP (operands[6], 0, 1)
-    = gen_rtx_USE (DImode, gen_rtx_REG (DImode, GR_REG (1)));
-  XVECEXP (operands[6], 0, 2)
-    = gen_rtx_CLOBBER (DImode, gen_rtx_REG (DImode, BR_REG (0)));
+  emit_call_insn (gen_call (operands[0], const0_rtx, NULL, const0_rtx));
 
-  for (i = 1; i < count; i++)
-    XVECEXP (operands[6], 0, i + 2)
-      = gen_rtx_SET (VOIDmode, XEXP (XVECEXP (operands[0], 0, i), 0), call);
+  for (i = 0; i < XVECLEN (operands[2], 0); i++)
+    {
+      rtx set = XVECEXP (operands[2], 0, i);
+      emit_move_insn (SET_DEST (set), SET_SRC (set));
+    }
 
-}")
+  /* The optimizer does not know that the call sets the function value
+     registers we stored in the result block.  We avoid problems by
+     claiming that all hard registers are used and clobbered at this
+     point.  */
+  emit_insn (gen_blockage ());
 
-;; ??? Saving/restoring the GP register is not needed if we are calling
-;; a function in the same module.
+  DONE;
+})
 
-(define_expand "call_value_pic"
-  [(set (match_dup 3) (reg:DI 1))
-   (parallel [(set (match_operand 0 "" "")
-                  (call (mem:DI (match_operand 1 "" ""))
-                        (match_operand 2 "" "")))
-             (use (reg:DI 1))
-             (clobber (reg:DI 320))])
-   (set (reg:DI 1) (match_dup 3))]
+(define_insn "call_nogp"
+  [(call (mem:DI (match_operand:DI 0 "call_operand" "?b,i"))
+        (const_int 0))
+   (clobber (match_operand:DI 1 "register_operand" "=b,b"))]
   ""
-  "
-{
-  /* ??? Using setjmp_operand is an unsatisfying solution.  Should rethink.  */
-  operands[3] = ia64_gp_save_reg (setjmp_operand (XEXP (operands[1], 0),
-                                                 VOIDmode));
-}")
-
-;; ??? Saving/restoring the GP register is not needed if we are calling
-;; a function in the same module.
-
-(define_expand "call_multiple_values_pic"
-  [(set (match_dup 4) (reg:DI 1))
-   (match_par_dup 3 [(set (match_operand 0 "" "")
-                         (call (mem:DI (match_operand 1 "" ""))
-                               (match_operand 2 "" "")))
-                    (use (reg:DI 1))
-                    (clobber (reg:DI 320))])
-   (set (reg:DI 1) (match_dup 4))]
-  ""
-  "
-{
-  int count;
-  int i;
-  rtx call;
-
-  operands[4] = ia64_gp_save_reg (0);
-
-  call = gen_rtx_CALL (VOIDmode, gen_rtx_MEM (DImode, operands[1]),
-                      operands[2]);
+  "br.call%+.many %1 = %0"
+  [(set_attr "itanium_class" "br,scall")])
 
-  count = XVECLEN (operands[0], 0);
-  operands[3] = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (count + 2));
-  
-  XVECEXP (operands[3], 0, 0)
-    = gen_rtx_SET (VOIDmode, XEXP (XVECEXP (operands[0], 0, 0), 0), call);
-
-  XVECEXP (operands[3], 0, 1)
-    = gen_rtx_USE (DImode, gen_rtx_REG (DImode, GR_REG (1)));
-  XVECEXP (operands[3], 0, 2)
-    = gen_rtx_CLOBBER (DImode, gen_rtx_REG (DImode, BR_REG (0)));
-
-  for (i = 1; i < count; i++)
-    XVECEXP (operands[3], 0, i + 2)
-      = gen_rtx_SET (VOIDmode, XEXP (XVECEXP (operands[0], 0, i), 0), call);
-}")
-
-(define_insn "call_value_internal"
-  [(set (match_operand 0 "register_operand" "=rf")
-       (call (mem:DI (match_operand:DI 1 "call_operand" "bi"))
-             (match_operand 2 "" "")))
-   (clobber (match_operand:DI 3 "register_operand" "=b"))]
-  ""
-  "br.call%+.many %3 = %1"
-  [(set_attr "type" "B")])
-
-(define_insn "*call_value_internal1"
-  [(set (match_operand 0 "register_operand" "=rf")
-       (call (mem:DI (match_operand:DI 1 "call_operand" "bi"))
-             (match_operand 2 "" "")))
-   (use (reg:DI 1))
-   (clobber (match_operand:DI 3 "register_operand" "=b"))]
-  ""
-  "br.call%+.many %3 = %1"
-  [(set_attr "type" "B")])
-
-(define_insn "*call_multiple_values_internal1"
-  [(match_parallel 0 "call_multiple_values_operation"
-                  [(set (match_operand 1 "register_operand" "=rf")
-                        (call (mem:DI (match_operand:DI 2 "call_operand" "bi"))
-                              (match_operand 3 "" "")))
-                   (use (reg:DI 1))
-                   (clobber (match_operand:DI 4 "register_operand" "=b"))])]
-  ""
-  "br.call%+.many %4 = %2"
-  [(set_attr "type" "B")])
+(define_insn "call_value_nogp"
+  [(set (match_operand 0 "" "=X,X")
+       (call (mem:DI (match_operand:DI 1 "call_operand" "?b,i"))
+             (const_int 0)))
+   (clobber (match_operand:DI 2 "register_operand" "=b,b"))]
+  ""
+  "br.call%+.many %2 = %1"
+  [(set_attr "itanium_class" "br,scall")])
 
-;; Call subroutine returning any type.
+(define_insn "sibcall_nogp"
+  [(call (mem:DI (match_operand:DI 0 "call_operand" "?b,i"))
+        (const_int 0))]
+  ""
+  "br%+.many %0"
+  [(set_attr "itanium_class" "br,scall")])
 
-(define_expand "untyped_call"
-  [(parallel [(call (match_operand 0 "" "")
-                   (const_int 0))
-             (match_operand 1 "" "")
-             (match_operand 2 "" "")])]
+(define_insn "call_gp"
+  [(call (mem:DI (match_operand:DI 0 "call_operand" "?r,i"))
+        (const_int 1))
+   (clobber (match_operand:DI 1 "register_operand" "=b,b"))
+   (clobber (match_scratch:DI 2 "=&r,X"))
+   (clobber (match_scratch:DI 3 "=b,X"))]
   ""
-  "
+  "#"
+  [(set_attr "itanium_class" "br,scall")])
+
+;; Irritatingly, we don't have access to INSN within the split body.
+;; See commentary in ia64_split_call as to why these aren't peep2.
+(define_split
+  [(call (mem (match_operand 0 "call_operand" ""))
+        (const_int 1))
+   (clobber (match_operand:DI 1 "register_operand" ""))
+   (clobber (match_scratch:DI 2 ""))
+   (clobber (match_scratch:DI 3 ""))]
+  "reload_completed && find_reg_note (insn, REG_NORETURN, NULL_RTX)"
+  [(const_int 0)]
 {
-  int i;
+  ia64_split_call (NULL_RTX, operands[0], operands[1], operands[2],
+                  operands[3], true, false);
+  DONE;
+})
 
-  emit_call_insn (gen_call (operands[0], const0_rtx, NULL, const0_rtx));
+(define_split
+  [(call (mem (match_operand 0 "call_operand" ""))
+        (const_int 1))
+   (clobber (match_operand:DI 1 "register_operand" ""))
+   (clobber (match_scratch:DI 2 ""))
+   (clobber (match_scratch:DI 3 ""))]
+  "reload_completed"
+  [(const_int 0)]
+{
+  ia64_split_call (NULL_RTX, operands[0], operands[1], operands[2],
+                  operands[3], false, false);
+  DONE;
+})
 
-  for (i = 0; i < XVECLEN (operands[2], 0); i++)
-    {
-      rtx set = XVECEXP (operands[2], 0, i);
-      emit_move_insn (SET_DEST (set), SET_SRC (set));
-    }
+(define_insn "call_value_gp"
+  [(set (match_operand 0 "" "=X,X")
+       (call (mem:DI (match_operand:DI 1 "call_operand" "?r,i"))
+             (const_int 1)))
+   (clobber (match_operand:DI 2 "register_operand" "=b,b"))
+   (clobber (match_scratch:DI 3 "=&r,X"))
+   (clobber (match_scratch:DI 4 "=b,X"))]
+  ""
+  "#"
+  [(set_attr "itanium_class" "br,scall")])
 
-  /* The optimizer does not know that the call sets the function value
-     registers we stored in the result block.  We avoid problems by
-     claiming that all hard registers are used and clobbered at this
-     point.  */
-  emit_insn (gen_blockage ());
+(define_split
+  [(set (match_operand 0 "" "")
+       (call (mem:DI (match_operand:DI 1 "call_operand" ""))
+             (const_int 1)))
+   (clobber (match_operand:DI 2 "register_operand" ""))
+   (clobber (match_scratch:DI 3 ""))
+   (clobber (match_scratch:DI 4 ""))]
+  "reload_completed && find_reg_note (insn, REG_NORETURN, NULL_RTX)"
+  [(const_int 0)]
+{
+  ia64_split_call (operands[0], operands[1], operands[2], operands[3],
+                  operands[4], true, false);
+  DONE;
+})
+
+(define_split
+  [(set (match_operand 0 "" "")
+       (call (mem:DI (match_operand:DI 1 "call_operand" ""))
+             (const_int 1)))
+   (clobber (match_operand:DI 2 "register_operand" ""))
+   (clobber (match_scratch:DI 3 ""))
+   (clobber (match_scratch:DI 4 ""))]
+  "reload_completed"
+  [(const_int 0)]
+{
+  ia64_split_call (operands[0], operands[1], operands[2], operands[3],
+                  operands[4], false, false);
+  DONE;
+})
 
+(define_insn_and_split "sibcall_gp"
+  [(call (mem:DI (match_operand:DI 0 "call_operand" "?r,i"))
+        (const_int 1))
+   (clobber (match_scratch:DI 1 "=&r,X"))
+   (clobber (match_scratch:DI 2 "=b,X"))]
+  ""
+  "#"
+  "reload_completed"
+  [(const_int 0)]
+{
+  ia64_split_call (NULL_RTX, operands[0], NULL_RTX, operands[1],
+                  operands[2], true, true);
   DONE;
-}")
+}
+  [(set_attr "itanium_class" "br")])
 
 (define_insn "return_internal"
   [(return)
    (use (match_operand:DI 0 "register_operand" "b"))]
   ""
   "br.ret.sptk.many %0"
-  [(set_attr "type" "B")])
+  [(set_attr "itanium_class" "br")])
 
 (define_insn "return"
   [(return)]
   "ia64_direct_return ()"
   "br.ret.sptk.many rp"
-  [(set_attr "type" "B")])
+  [(set_attr "itanium_class" "br")])
 
 (define_insn "*return_true"
   [(set (pc)
        (if_then_else (match_operator 0 "predicate_operator"
-                       [(match_operand:CC 1 "register_operand" "c")
+                       [(match_operand:BI 1 "register_operand" "c")
                         (const_int 0)])
                      (return)
                      (pc)))]
   "ia64_direct_return ()"
   "(%J0) br.ret%+.many rp"
-  [(set_attr "type" "B")
+  [(set_attr "itanium_class" "br")
    (set_attr "predicable" "no")])
 
 (define_insn "*return_false"
   [(set (pc)
        (if_then_else (match_operator 0 "predicate_operator"
-                       [(match_operand:CC 1 "register_operand" "c")
+                       [(match_operand:BI 1 "register_operand" "c")
                         (const_int 0)])
                      (pc)
                      (return)))]
   "ia64_direct_return ()"
   "(%j0) br.ret%+.many rp"
-  [(set_attr "type" "B")
+  [(set_attr "itanium_class" "br")
    (set_attr "predicable" "no")])
 
 (define_insn "jump"
   [(set (pc) (label_ref (match_operand 0 "" "")))]
   ""
   "br %l0"
-  [(set_attr "type" "B")])
+  [(set_attr "itanium_class" "br")])
 
 (define_insn "indirect_jump"
   [(set (pc) (match_operand:DI 0 "register_operand" "b"))]
   ""
   "br %0"
-  [(set_attr "type" "B")])
+  [(set_attr "itanium_class" "br")])
 
 (define_expand "tablejump"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand 1 "" "")]
+  [(parallel [(set (pc) (match_operand:DI 0 "memory_operand" ""))
+             (use (label_ref (match_operand 1 "" "")))])]
   ""
-  "
 {
-  rtx tmp1 = gen_reg_rtx (DImode);
-  rtx tmp2 = gen_reg_rtx (DImode);
+  rtx op0 = operands[0];
+  rtx addr;
+
+  /* ??? Bother -- do_tablejump is "helpful" and pulls the table
+     element into a register without bothering to see whether that
+     is necessary given the operand predicate.  Check for MEM just
+     in case someone fixes this.  */
+  if (GET_CODE (op0) == MEM)
+    addr = XEXP (op0, 0);
+  else
+    {
+      /* Otherwise, cheat and guess that the previous insn in the
+        stream was the memory load.  Grab the address from that.
+        Note we have to momentarily pop out of the sequence started
+        by the insn-emit wrapper in order to grab the last insn.  */
+      rtx last, set;
+
+      end_sequence ();
+      last = get_last_insn ();
+      start_sequence ();
+      set = single_set (last);
+
+      gcc_assert (rtx_equal_p (SET_DEST (set), op0)
+                 && GET_CODE (SET_SRC (set)) == MEM);
+      addr = XEXP (SET_SRC (set), 0);
+      gcc_assert (!rtx_equal_p (addr, op0));
+    }
 
-  emit_move_insn (tmp1, gen_rtx_LABEL_REF (Pmode, operands[1]));
-  emit_insn (gen_adddi3 (tmp2, operands[0], tmp1));
-  emit_jump_insn (gen_tablejump_internal (tmp2, operands[1]));
-  DONE;
-}")
+  /* Jump table elements are stored pc-relative.  That is, a displacement
+     from the entry to the label.  Thus to convert to an absolute address
+     we add the address of the memory from which the value is loaded.  */
+  operands[0] = expand_simple_binop (DImode, PLUS, op0, addr,
+                                    NULL_RTX, 1, OPTAB_DIRECT);
+})
 
-(define_insn "tablejump_internal"
+(define_insn "*tablejump_internal"
   [(set (pc) (match_operand:DI 0 "register_operand" "b"))
    (use (label_ref (match_operand 1 "" "")))]
   ""
   "br %0"
-  [(set_attr "type" "B")])
+  [(set_attr "itanium_class" "br")])
 
 \f
 ;; ::::::::::::::::::::
 (define_expand "prologue"
   [(const_int 1)]
   ""
-  "
 {
   ia64_expand_prologue ();
   DONE;
-}")
+})
 
 (define_expand "epilogue"
-  [(const_int 2)]
+  [(return)]
+  ""
+{
+  ia64_expand_epilogue (0);
+  DONE;
+})
+
+(define_expand "sibcall_epilogue"
+  [(return)]
   ""
-  "
 {
-  ia64_expand_epilogue ();
+  ia64_expand_epilogue (1);
   DONE;
-}")
+})
 
 ;; This prevents the scheduler from moving the SP decrement past FP-relative
 ;; stack accesses.  This is the same as adddi3 plus the extra set.
 (define_insn "prologue_allocate_stack"
   [(set (match_operand:DI 0 "register_operand" "=r,r,r")
        (plus:DI (match_operand:DI 1 "register_operand" "%r,r,a")
-                (match_operand:DI 2 "reg_or_22bit_operand" "r,I,J")))
-   (set (match_operand:DI 3 "register_operand" "=r,r,r")
+                (match_operand:DI 2 "gr_reg_or_22bit_operand" "r,I,J")))
+   (set (match_operand:DI 3 "register_operand" "+r,r,r")
        (match_dup 3))]
   ""
   "@
-  add %0 = %1, %2
-  adds %0 = %2, %1
-  addl %0 = %2, %1"
-  [(set_attr "type" "A")])
+   add %0 = %1, %2
+   adds %0 = %2, %1
+   addl %0 = %2, %1"
+  [(set_attr "itanium_class" "ialu")])
 
 ;; This prevents the scheduler from moving the SP restore past FP-relative
 ;; stack accesses.  This is similar to movdi plus the extra set.
    (set (match_dup 1) (match_dup 1))]
   ""
   "mov %0 = %1"
-  [(set_attr "type" "A")])
+  [(set_attr "itanium_class" "ialu")])
+
+;; As USE insns aren't meaningful after reload, this is used instead
+;; to prevent deleting instructions setting registers for EH handling
+(define_insn "prologue_use"
+  [(unspec:DI [(match_operand:DI 0 "register_operand" "")]
+             UNSPEC_PROLOGUE_USE)]
+  ""
+  ""
+  [(set_attr "itanium_class" "ignore")
+   (set_attr "predicable" "no")
+   (set_attr "empty" "yes")])
 
 ;; Allocate a new register frame.
 
 (define_insn "alloc"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (unspec_volatile:DI [(const_int 0)] 0))
+       (unspec_volatile:DI [(const_int 0)] UNSPECV_ALLOC))
    (use (match_operand:DI 1 "const_int_operand" "i"))
    (use (match_operand:DI 2 "const_int_operand" "i"))
    (use (match_operand:DI 3 "const_int_operand" "i"))
    (use (match_operand:DI 4 "const_int_operand" "i"))]
   ""
   "alloc %0 = ar.pfs, %1, %2, %3, %4"
-  [(set_attr "type" "M")
-   (set_attr "predicable" "no")])
+  [(set_attr "itanium_class" "syst_m0")
+   (set_attr "predicable" "no")
+   (set_attr "first_insn" "yes")])
 
 ;; Modifies ar.unat
 (define_expand "gr_spill"
   [(parallel [(set (match_operand:DI 0 "memory_operand" "=m")
                   (unspec:DI [(match_operand:DI 1 "register_operand" "r")
-                              (match_operand:DI 2 "const_int_operand" "")] 1))
+                              (match_operand:DI 2 "const_int_operand" "")]
+                             UNSPEC_GR_SPILL))
              (clobber (match_dup 3))])]
   ""
   "operands[3] = gen_rtx_REG (DImode, AR_UNAT_REGNUM);")
 
 (define_insn "gr_spill_internal"
-  [(set (match_operand:DI 0 "memory_operand" "=m")
+  [(set (match_operand:DI 0 "destination_operand" "=m")
        (unspec:DI [(match_operand:DI 1 "register_operand" "r")
-                   (match_operand:DI 2 "const_int_operand" "")] 1))
+                   (match_operand:DI 2 "const_int_operand" "")]
+                  UNSPEC_GR_SPILL))
    (clobber (match_operand:DI 3 "register_operand" ""))]
   ""
-  ".mem.offset %2, 0\;st8.spill %0 = %1%P0"
-  [(set_attr "type" "M")])
+{
+  /* Note that we use a C output pattern here to avoid the predicate
+     being automatically added before the .mem.offset directive.  */
+  return ".mem.offset %2, 0\;%,st8.spill %0 = %1%P0";
+}
+  [(set_attr "itanium_class" "st")])
 
 ;; Reads ar.unat
 (define_expand "gr_restore"
   [(parallel [(set (match_operand:DI 0 "register_operand" "=r")
                   (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
-                              (match_operand:DI 2 "const_int_operand" "")] 2))
+                              (match_operand:DI 2 "const_int_operand" "")]
+                             UNSPEC_GR_RESTORE))
              (use (match_dup 3))])]
   ""
   "operands[3] = gen_rtx_REG (DImode, AR_UNAT_REGNUM);")
 (define_insn "gr_restore_internal"
   [(set (match_operand:DI 0 "register_operand" "=r")
        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
-                   (match_operand:DI 2 "const_int_operand" "")] 2))
+                   (match_operand:DI 2 "const_int_operand" "")]
+                  UNSPEC_GR_RESTORE))
    (use (match_operand:DI 3 "register_operand" ""))]
   ""
-  ".mem.offset %2, 0\;ld8.fill %0 = %1%P1"
-  [(set_attr "type" "M")])
+  { return ".mem.offset %2, 0\;%,ld8.fill %0 = %1%P1"; }
+  [(set_attr "itanium_class" "ld")])
 
 (define_insn "fr_spill"
-  [(set (match_operand:TF 0 "memory_operand" "=m")
-       (unspec:TF [(match_operand:TF 1 "register_operand" "f")] 3))]
+  [(set (match_operand:XF 0 "destination_operand" "=m")
+       (unspec:XF [(match_operand:XF 1 "register_operand" "f")]
+                  UNSPEC_FR_SPILL))]
   ""
   "stf.spill %0 = %1%P0"
-  [(set_attr "type" "M")])
+  [(set_attr "itanium_class" "stf")])
 
 (define_insn "fr_restore"
-  [(set (match_operand:TF 0 "register_operand" "=f")
-       (unspec:TF [(match_operand:TF 1 "memory_operand" "m")] 4))]
+  [(set (match_operand:XF 0 "register_operand" "=f")
+       (unspec:XF [(match_operand:XF 1 "memory_operand" "m")]
+                  UNSPEC_FR_RESTORE))]
   ""
   "ldf.fill %0 = %1%P1"
-  [(set_attr "type" "M")])
+  [(set_attr "itanium_class" "fld")])
+
+;; ??? The explicit stop is not ideal.  It would be better if
+;; rtx_needs_barrier took care of this, but this is something that can be
+;; fixed later.  This avoids an RSE DV.
 
 (define_insn "bsp_value"
   [(set (match_operand:DI 0 "register_operand" "=r")
-       (unspec:DI [(const_int 0)] 20))]
+       (unspec:DI [(const_int 0)] UNSPEC_BSP_VALUE))]
   ""
-  "mov %0 = ar.bsp"
-  [(set_attr "type" "I")])
+  "*
+{
+  return \";;\;%,mov %0 = ar.bsp\";
+}"
+  [(set_attr "itanium_class" "frar_i")])
 
 (define_insn "set_bsp"
-  [(unspec_volatile [(const_int 0)] 5)
-   (use (match_operand:DI 0 "register_operand" "r"))]
-  ""
-  "flushrs\;mov r19=ar.rsc\;;;\;and r19=0x1c,r19\;;;\;mov ar.rsc=r19\;;;\;mov ar.bspstore=%0\;;;\;or r19=0x3,r19\;;;\;loadrs\;invala\;;;\;mov ar.rsc=r19"
-  [(set_attr "type" "unknown")
+  [(unspec_volatile [(match_operand:DI 0 "register_operand" "r")]
+                   UNSPECV_SET_BSP)]
+  ""
+  "flushrs
+       mov r19=ar.rsc
+       ;;
+       and r19=0x1c,r19
+       ;;
+       mov ar.rsc=r19
+       ;;
+       mov ar.bspstore=%0
+       ;;
+       or r19=0x3,r19
+       ;;
+       loadrs
+       invala
+       ;;
+       mov ar.rsc=r19"
+  [(set_attr "itanium_class" "unknown")
    (set_attr "predicable" "no")])
 
+;; ??? The explicit stops are not ideal.  It would be better if
+;; rtx_needs_barrier took care of this, but this is something that can be
+;; fixed later.  This avoids an RSE DV.
+
 (define_insn "flushrs"
-  [(unspec [(const_int 0)] 21)]
+  [(unspec [(const_int 0)] UNSPEC_FLUSHRS)]
   ""
-  ";;\;flushrs"
-  [(set_attr "type" "M")])
+  ";;\;flushrs\;;;"
+  [(set_attr "itanium_class" "rse_m")
+   (set_attr "predicable" "no")])
 \f
 ;; ::::::::::::::::::::
 ;; ::
 ;; ::
 ;; ::::::::::::::::::::
 
-;; ??? Emiting a NOP instruction isn't very useful.  This should probably
+;; ??? Emitting a NOP instruction isn't very useful.  This should probably
 ;; be emitting ";;" to force a break in the instruction packing.
 
 ;; No operation, needed in case the user uses -g but not -O.
   [(const_int 0)]
   ""
   "nop 0"
-  [(set_attr "type" "unknown")])
+  [(set_attr "itanium_class" "nop")])
+
+(define_insn "nop_m"
+  [(const_int 1)]
+  ""
+  "nop.m 0"
+  [(set_attr "itanium_class" "nop_m")])
+
+(define_insn "nop_i"
+  [(const_int 2)]
+  ""
+  "nop.i 0"
+  [(set_attr "itanium_class" "nop_i")])
+
+(define_insn "nop_f"
+  [(const_int 3)]
+  ""
+  "nop.f 0"
+  [(set_attr "itanium_class" "nop_f")])
+
+(define_insn "nop_b"
+  [(const_int 4)]
+  ""
+  "nop.b 0"
+  [(set_attr "itanium_class" "nop_b")])
+
+(define_insn "nop_x"
+  [(const_int 5)]
+  ""
+  ""
+  [(set_attr "itanium_class" "nop_x")
+   (set_attr "empty" "yes")])
+
+;; The following insn will be never generated.  It is used only by
+;; insn scheduler to change state before advancing cycle.
+(define_insn "pre_cycle"
+  [(const_int 6)]
+  ""
+  ""
+  [(set_attr "itanium_class" "pre_cycle")])
+
+(define_insn "bundle_selector"
+  [(unspec [(match_operand 0 "const_int_operand" "")] UNSPEC_BUNDLE_SELECTOR)]
+  ""
+  { return get_bundle_name (INTVAL (operands[0])); }
+  [(set_attr "itanium_class" "ignore")
+   (set_attr "predicable" "no")])
 
 ;; Pseudo instruction that prevents the scheduler from moving code above this
 ;; point.
 (define_insn "blockage"
-  [(unspec_volatile [(const_int 0)] 1)]
+  [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)]
   ""
   ""
-  [(set_attr "type" "unknown")
+  [(set_attr "itanium_class" "ignore")
    (set_attr "predicable" "no")])
 
 (define_insn "insn_group_barrier"
-  [(unspec_volatile [(const_int 0)] 2)]
+  [(unspec_volatile [(match_operand 0 "const_int_operand" "")]
+                   UNSPECV_INSN_GROUP_BARRIER)]
   ""
   ";;"
-  [(set_attr "type" "S")
+  [(set_attr "itanium_class" "stop_bit")
+   (set_attr "predicable" "no")
+   (set_attr "empty" "yes")])
+
+(define_expand "trap"
+  [(trap_if (const_int 1) (const_int 0))]
+  ""
+  "")
+
+;; ??? We don't have a match-any slot type.  Setting the type to unknown
+;; produces worse code that setting the slot type to A.
+
+(define_insn "*trap"
+  [(trap_if (const_int 1) (match_operand 0 "const_int_operand" ""))]
+  ""
+  "break %0"
+  [(set_attr "itanium_class" "chk_s")])
+
+(define_expand "conditional_trap"
+  [(trap_if (match_operand 0 "" "") (match_operand 1 "" ""))]
+  ""
+{
+  operands[0] = ia64_expand_compare (GET_CODE (operands[0]), VOIDmode);
+})
+
+(define_insn "*conditional_trap"
+  [(trap_if (match_operator 0 "predicate_operator"
+             [(match_operand:BI 1 "register_operand" "c")
+              (const_int 0)])  
+           (match_operand 2 "const_int_operand" ""))]
+  ""
+  "(%J0) break %2"
+  [(set_attr "itanium_class" "chk_s")
    (set_attr "predicable" "no")])
 
+(define_insn "break_f"
+  [(unspec_volatile [(const_int 0)] UNSPECV_BREAK)]
+  ""
+  "break.f 0"
+  [(set_attr "itanium_class" "nop_f")])
+
+(define_insn "prefetch"
+  [(prefetch (match_operand:DI 0 "address_operand" "p")
+            (match_operand:DI 1 "const_int_operand" "n")
+            (match_operand:DI 2 "const_int_operand" "n"))]
+  ""
+{
+  static const char * const alt[2][4] = {
+    {
+      "%,lfetch.nta [%0]",
+      "%,lfetch.nt1 [%0]",
+      "%,lfetch.nt2 [%0]",
+      "%,lfetch [%0]"
+    },
+    {
+      "%,lfetch.excl.nta [%0]",
+      "%,lfetch.excl.nt1 [%0]",
+      "%,lfetch.excl.nt2 [%0]",
+      "%,lfetch.excl [%0]"
+    }
+  };
+  int i = (INTVAL (operands[1]));
+  int j = (INTVAL (operands[2]));
+
+  gcc_assert (i == 0 || i == 1);
+  gcc_assert (j >= 0 && j <= 3);
+  return alt[i][j];
+}
+  [(set_attr "itanium_class" "lfetch")])
 \f
 ;; Non-local goto support.
 
   [(use (match_operand:OI 0 "memory_operand" ""))
    (use (match_operand:DI 1 "register_operand" ""))]
   ""
-  "
 {
   emit_library_call (gen_rtx_SYMBOL_REF (Pmode,
                                         \"__ia64_save_stack_nonlocal\"),
                     0, VOIDmode, 2, XEXP (operands[0], 0), Pmode,
                     operands[1], Pmode);
   DONE;
-}")
+})
 
 (define_expand "nonlocal_goto"
   [(use (match_operand 0 "general_operand" ""))
    (use (match_operand 2 "general_operand" ""))
    (use (match_operand 3 "general_operand" ""))]
   ""
-  "
 {
   emit_library_call (gen_rtx_SYMBOL_REF (Pmode, \"__ia64_nonlocal_goto\"),
-                    0, VOIDmode, 3,
+                    LCT_NORETURN, VOIDmode, 3,
                     operands[1], Pmode,
                     copy_to_reg (XEXP (operands[2], 0)), Pmode,
                     operands[3], Pmode);
   emit_barrier ();
   DONE;
-}")
-
-;; Restore the GP after the exception/longjmp.  The preceeding call will
-;; have tucked it away.
-(define_expand "exception_receiver"
-  [(set (reg:DI 1) (match_dup 0))]
-  ""
-  "operands[0] = ia64_gp_save_reg (0);")
-
-;; The rest of the setjmp processing happens with the nonlocal_goto expander.
-;; ??? This is not tested.
-(define_expand "builtin_setjmp_setup"
-  [(use (match_operand:DI 0 "" ""))]
-  ""
-  "
-{
-  emit_move_insn (ia64_gp_save_reg (0), gen_rtx_REG (DImode, GR_REG (1)));
-  DONE;
-}")
+})
 
-(define_expand "builtin_setjmp_receiver"
-  [(use (match_operand:DI 0 "" ""))]
+(define_insn_and_split "builtin_setjmp_receiver"
+  [(unspec_volatile [(match_operand:DI 0 "" "")] UNSPECV_SETJMP_RECEIVER)]
   ""
-  "
+  "#"
+  "reload_completed"
+  [(const_int 0)]
 {
-  emit_move_insn (gen_rtx_REG (DImode, GR_REG (1)), ia64_gp_save_reg (0));
+  ia64_reload_gp ();
   DONE;
-}")
+})
 
 (define_expand "eh_epilogue"
   [(use (match_operand:DI 0 "register_operand" "r"))
    (use (match_operand:DI 1 "register_operand" "r"))
    (use (match_operand:DI 2 "register_operand" "r"))]
   ""
-  "
 {
   rtx bsp = gen_rtx_REG (Pmode, 10);
   rtx sp = gen_rtx_REG (Pmode, 9);
 
   cfun->machine->ia64_eh_epilogue_sp = sp;
   cfun->machine->ia64_eh_epilogue_bsp = bsp;
-}")
+})
 \f
 ;; Builtin apply support.
 
   [(use (match_operand:DI 0 "register_operand" ""))
    (use (match_operand:OI 1 "memory_operand" ""))]
   ""
-  "
 {
   emit_library_call (gen_rtx_SYMBOL_REF (Pmode,
-                                        \"__ia64_restore_stack_nonlocal\"),
+                                        "__ia64_restore_stack_nonlocal"),
                     0, VOIDmode, 1,
                     copy_to_reg (XEXP (operands[1], 0)), Pmode);
   DONE;
-}")
+})
 
 \f
-;;; Intrinsics support.
-
-(define_insn "mf"
-  [(unspec [(match_operand:BLK 0 "memory_operand" "m")] 12)]
-  ""
-  "mf"
-  [(set_attr "type" "M")])
-
-(define_insn "fetchadd_acq_si"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
-                    (match_operand:SI 2 "fetchadd_operand" "n")] 19))]
-  ""
-  "fetchadd4.acq %0 = %1, %2"
-  [(set_attr "type" "M")])
-
-(define_insn "fetchadd_acq_di"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
-                    (match_operand:DI 2 "fetchadd_operand" "n")] 19))]
-  ""
-  "fetchadd8.acq %0 = %1, %2"
-  [(set_attr "type" "M")])
-
-(define_insn "cmpxchg_acq_si"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-        (unspec:SI [(match_operand:SI 1 "memory_operand" "m")
-                    (match_operand:SI 2 "register_operand" "r")
-                   (match_operand:SI 3 "ar_ccv_reg_operand" "")] 13))]
-  ""
-  "cmpxchg4.acq %0 = %1, %2, %3"
-  [(set_attr "type" "M")])
-
-(define_insn "cmpxchg_acq_di"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-        (unspec:DI [(match_operand:DI 1 "memory_operand" "m")
-                    (match_operand:DI 2 "register_operand" "r")
-                    (match_operand:DI 3 "ar_ccv_reg_operand" "")] 13))]
-  ""
-  "cmpxchg8.acq %0 = %1, %2, %3"
-  [(set_attr "type" "M")])
-
-(define_expand "val_compare_and_swap_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")
-   (match_operand:SI 3 "register_operand" "")]
-  ""
-  "
-{
-  rtx target = gen_rtx_MEM (BLKmode, gen_rtx_REG (DImode, GR_REG (1)));
-  rtx ccv = gen_rtx_REG (SImode, AR_CCV_REGNUM);
-  emit_move_insn (ccv, operands[2]);
-  emit_insn (gen_mf (target));
-  emit_insn (gen_cmpxchg_acq_si (operands[0], operands[1], operands[3], ccv));
-  DONE;
-}")
-
-(define_expand "val_compare_and_swap_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")
-   (match_operand:DI 3 "register_operand" "")]
-  ""
-  "
-{
-  rtx target = gen_rtx_MEM (BLKmode, gen_rtx_REG (DImode, GR_REG (1)));
-  rtx ccv = gen_rtx_REG (DImode, AR_CCV_REGNUM);
-  emit_move_insn (ccv, operands[2]);
-  emit_insn (gen_mf (target));
-  emit_insn (gen_cmpxchg_acq_di (operands[0], operands[1], operands[3], ccv));
-  DONE;
-}")
-
-(define_insn "xchgsi"
-  [(set (match_operand:SI 0 "register_operand" "=r")
-        (match_operand:SI 1 "memory_operand" "+m"))
-   (set (match_dup 1)
-        (match_operand:SI 2 "register_operand" "r"))]
-  ""
-  "xchg4 %0 = %1, %2"
-  [(set_attr "type" "M")])
-
-(define_insn "xchgdi"
-  [(set (match_operand:DI 0 "register_operand" "=r")
-        (match_operand:DI 1 "memory_operand" "+m"))
-   (set (match_dup 1)
-        (match_operand:DI 2 "register_operand" "r"))]
-  ""
-  "xchg8 %0 = %1, %2"
-  [(set_attr "type" "M")])
-
-(define_expand "lock_test_and_set_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  emit_insn (gen_xchgsi (operands[0], operands[1], operands[2]));
-  DONE;
-}")
-
-(define_expand "lock_test_and_set_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  emit_insn (gen_xchgdi (operands[0], operands[1], operands[2]));
-  DONE;
-}")
-
-(define_expand "fetch_and_add_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "nonmemory_operand" "")]
-  ""
-  "
-{
-  int x;
-
-  if (GET_CODE (operands[2]) == CONST_INT)
-    {
-      x = INTVAL(operands[2]);
-      if (x == -16 || x == -8 || x == -4 || x == -1 ||
-          x ==  16 || x ==  8 || x ==  4 || x ==  1)
-        {
-          emit_insn (gen_fetchadd_acq_si (operands[0], operands[1],
-                                         operands[2]));
-          DONE;
-        }
-    }
-
-  ia64_expand_fetch_and_op (IA64_ADD_OP, SImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_sub_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_SUB_OP, SImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_or_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_OR_OP, SImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_and_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_AND_OP, SImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_xor_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_XOR_OP, SImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_nand_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_NAND_OP, SImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_add_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "nonmemory_operand" "")]
-  ""
-  "
-{
-  int x;
-
-  if (GET_CODE (operands[2]) == CONST_INT)
-    {
-      x = INTVAL(operands[2]);
-      if (x == -16 || x == -8 || x == -4 || x == -1 ||
-          x ==  16 || x ==  8 || x ==  4 || x ==  1)
-        {
-          emit_insn (gen_fetchadd_acq_di (operands[0], operands[1],
-                                         operands[2]));
-          DONE;
-        }
-    }
-
-  ia64_expand_fetch_and_op (IA64_ADD_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_sub_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_SUB_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_or_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_OR_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_and_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_AND_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_xor_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_XOR_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "fetch_and_nand_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_fetch_and_op (IA64_NAND_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "add_and_fetch_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_ADD_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "sub_and_fetch_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_SUB_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "or_and_fetch_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_OR_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "and_and_fetch_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_AND_OP, DImode, operands);
-  DONE;
-}")
-
-(define_expand "xor_and_fetch_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_XOR_OP, DImode, operands);
-  DONE;
-}")
+;; Predication.
 
-(define_expand "nand_and_fetch_di"
-  [(match_operand:DI 0 "register_operand" "")
-   (match_operand:DI 1 "memory_operand" "")
-   (match_operand:DI 2 "register_operand" "")]
+(define_cond_exec
+  [(match_operator 0 "predicate_operator"
+     [(match_operand:BI 1 "register_operand" "c")
+      (const_int 0)])]
   ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_NAND_OP, DImode, operands);
-  DONE;
-}")
+  "(%J0)")
 
-(define_expand "add_and_fetch_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
+(define_insn "pred_rel_mutex"
+  [(set (match_operand:BI 0 "register_operand" "+c")
+       (unspec:BI [(match_dup 0)] UNSPEC_PRED_REL_MUTEX))]
   ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_ADD_OP, SImode, operands);
-  DONE;
-}")
+  ".pred.rel.mutex %0, %I0"
+  [(set_attr "itanium_class" "ignore")
+   (set_attr "predicable" "no")])
 
-(define_expand "sub_and_fetch_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
+(define_insn "safe_across_calls_all"
+  [(unspec_volatile [(const_int 0)] UNSPECV_PSAC_ALL)]
   ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_SUB_OP, SImode, operands);
-  DONE;
-}")
+  ".pred.safe_across_calls p1-p63"
+  [(set_attr "itanium_class" "ignore")
+   (set_attr "predicable" "no")])
 
-(define_expand "or_and_fetch_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
+(define_insn "safe_across_calls_normal"
+  [(unspec_volatile [(const_int 0)] UNSPECV_PSAC_NORMAL)]
   ""
-  "
 {
-  ia64_expand_op_and_fetch (IA64_OR_OP, SImode, operands);
-  DONE;
-}")
+  emit_safe_across_calls ();
+  return "";
+}
+  [(set_attr "itanium_class" "ignore")
+   (set_attr "predicable" "no")])
 
-(define_expand "and_and_fetch_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_AND_OP, SImode, operands);
-  DONE;
-}")
+;; UNSPEC instruction definition to "swizzle" 32 bit pointer into 64 bit
+;; pointer.  This is used by the HP-UX 32 bit mode.
 
-(define_expand "xor_and_fetch_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
+(define_insn "ptr_extend"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+        (unspec:DI [(match_operand:SI 1 "gr_register_operand" "r")]
+                  UNSPEC_ADDP4))]
   ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_XOR_OP, SImode, operands);
-  DONE;
-}")
+  "addp4 %0 = 0,%1"
+  [(set_attr "itanium_class" "ialu")])
 
-(define_expand "nand_and_fetch_si"
-  [(match_operand:SI 0 "register_operand" "")
-   (match_operand:SI 1 "memory_operand" "")
-   (match_operand:SI 2 "register_operand" "")]
-  ""
-  "
-{
-  ia64_expand_op_and_fetch (IA64_NAND_OP, SImode, operands);
-  DONE;
-}")
-\f
-;; Predication.
+;;
+;; Optimizations for ptr_extend
+
+(define_insn "ptr_extend_plus_imm"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+        (unspec:DI
+         [(plus:SI (match_operand:SI 1 "basereg_operand" "r")
+                   (match_operand:SI 2 "gr_reg_or_14bit_operand" "rI"))]
+         UNSPEC_ADDP4))]
+  "addp4_optimize_ok (operands[1], operands[2])"
+  "addp4 %0 = %2, %1"
+  [(set_attr "itanium_class" "ialu")])
+
+(define_insn "*ptr_extend_plus_2"
+  [(set (match_operand:DI 0 "gr_register_operand" "=r")
+        (unspec:DI
+         [(plus:SI (match_operand:SI 1 "gr_register_operand" "r")
+                   (match_operand:SI 2 "basereg_operand" "r"))]
+         UNSPEC_ADDP4))]
+  "addp4_optimize_ok (operands[1], operands[2])"
+  "addp4 %0 = %1, %2"
+  [(set_attr "itanium_class" "ialu")])
 
-(define_cond_exec
-  [(match_operator 0 "predicate_operator"
-     [(match_operand:CC 1 "register_operand" "c")
-      (const_int 0)])]
-  ""
-  "(%J0)")
+;;
+;; Get instruction pointer
 
-(define_insn "pred_rel_mutex"
-  [(unspec_volatile [(match_operand:CC 0 "register_operand" "c")] 7)]
-  ""
-  ".pred.rel.mutex %0, %I0"
-  [(set_attr "type" "unknown")
-   (set_attr "predicable" "no")])
+(define_insn "ip_value"
+  [(set (match_operand:DI 0 "register_operand" "=r")
+        (pc))]
+ ""
+ "mov %0 = ip"
+  [(set_attr "itanium_class" "ialu")])
+
+;; Vector operations
+(include "vect.md")
+;; Atomic operations
+(include "sync.md")