OSDN Git Service

* builtins.c (expand_builtin_compare_and_swap): If target is const0,
[pf3gnuchains/gcc-fork.git] / gcc / builtins.c
index e0afc90..9dc68cc 100644 (file)
@@ -75,11 +75,7 @@ const char * built_in_names[(int) END_BUILTINS] =
 
 /* Setup an array of _DECL trees, make sure each element is
    initialized to NULL_TREE.  */
-tree built_in_decls[(int) END_BUILTINS];
-/* Declarations used when constructing the builtin implicitly in the compiler.
-   It may be NULL_TREE when this is invalid (for instance runtime is not
-   required to implement the function call in all cases).  */
-tree implicit_built_in_decls[(int) END_BUILTINS];
+builtin_info_type builtin_info;
 
 static const char *c_getstr (tree);
 static rtx c_readstr (const char *, enum machine_mode);
@@ -227,6 +223,7 @@ static tree do_mpfr_bessel_n (tree, tree, tree,
                              const REAL_VALUE_TYPE *, bool);
 static tree do_mpfr_remquo (tree, tree, tree);
 static tree do_mpfr_lgamma_r (tree, tree, tree);
+static void expand_builtin_sync_synchronize (void);
 
 /* Return true if NAME starts with __builtin_ or __sync_.  */
 
@@ -237,6 +234,8 @@ is_builtin_name (const char *name)
     return true;
   if (strncmp (name, "__sync_", 7) == 0)
     return true;
+  if (strncmp (name, "__atomic_", 9) == 0)
+    return true;
   return false;
 }
 
@@ -1680,6 +1679,7 @@ expand_builtin_apply (rtx function, rtx arguments, rtx argsize)
   else
 #endif
     emit_stack_restore (SAVE_BLOCK, old_stack_level);
+  fixup_args_size_notes (call_insn, get_last_insn(), 0);
 
   OK_DEFER_POP;
 
@@ -1790,17 +1790,15 @@ expand_builtin_classify_type (tree exp)
   fcode = BUILT_IN_MATHFN##_R; fcodef = BUILT_IN_MATHFN##F_R ; \
   fcodel = BUILT_IN_MATHFN##L_R ; break;
 
-/* Return mathematic function equivalent to FN but operating directly
-   on TYPE, if available.  If IMPLICIT is true find the function in
-   implicit_built_in_decls[], otherwise use built_in_decls[].  If we
-   can't do the conversion, return zero.  */
+/* Return mathematic function equivalent to FN but operating directly on TYPE,
+   if available.  If IMPLICIT is true use the implicit builtin declaration,
+   otherwise use the explicit declaration.  If we can't do the conversion,
+   return zero.  */
 
 static tree
-mathfn_built_in_1 (tree type, enum built_in_function fn, bool implicit)
+mathfn_built_in_1 (tree type, enum built_in_function fn, bool implicit_p)
 {
-  tree const *const fn_arr
-    = implicit ? implicit_built_in_decls : built_in_decls;
-  enum built_in_function fcode, fcodef, fcodel;
+  enum built_in_function fcode, fcodef, fcodel, fcode2;
 
   switch (fn)
     {
@@ -1897,13 +1895,18 @@ mathfn_built_in_1 (tree type, enum built_in_function fn, bool implicit)
       }
 
   if (TYPE_MAIN_VARIANT (type) == double_type_node)
-    return fn_arr[fcode];
+    fcode2 = fcode;
   else if (TYPE_MAIN_VARIANT (type) == float_type_node)
-    return fn_arr[fcodef];
+    fcode2 = fcodef;
   else if (TYPE_MAIN_VARIANT (type) == long_double_type_node)
-    return fn_arr[fcodel];
+    fcode2 = fcodel;
   else
     return NULL_TREE;
+
+  if (implicit_p && !builtin_decl_implicit_p (fcode2))
+    return NULL_TREE;
+
+  return builtin_decl_explicit (fcode2);
 }
 
 /* Like mathfn_built_in_1(), but always use the implicit array.  */
@@ -2553,11 +2556,11 @@ expand_builtin_cexpi (tree exp, rtx target)
       rtx op1a, op2a;
 
       if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_CEXPIF)
-       fn = built_in_decls[BUILT_IN_SINCOSF];
+       fn = builtin_decl_explicit (BUILT_IN_SINCOSF);
       else if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_CEXPI)
-       fn = built_in_decls[BUILT_IN_SINCOS];
+       fn = builtin_decl_explicit (BUILT_IN_SINCOS);
       else if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_CEXPIL)
-       fn = built_in_decls[BUILT_IN_SINCOSL];
+       fn = builtin_decl_explicit (BUILT_IN_SINCOSL);
       else
        gcc_unreachable ();
 
@@ -2579,11 +2582,11 @@ expand_builtin_cexpi (tree exp, rtx target)
       tree ctype = build_complex_type (type);
 
       if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_CEXPIF)
-       fn = built_in_decls[BUILT_IN_CEXPF];
+       fn = builtin_decl_explicit (BUILT_IN_CEXPF);
       else if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_CEXPI)
-       fn = built_in_decls[BUILT_IN_CEXP];
+       fn = builtin_decl_explicit (BUILT_IN_CEXP);
       else if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_CEXPIL)
-       fn = built_in_decls[BUILT_IN_CEXPL];
+       fn = builtin_decl_explicit (BUILT_IN_CEXPL);
       else
        gcc_unreachable ();
 
@@ -3128,9 +3131,9 @@ expand_builtin_mempcpy_args (tree dest, tree src, tree len,
                             rtx target, enum machine_mode mode, int endp)
 {
     /* If return value is ignored, transform mempcpy into memcpy.  */
-  if (target == const0_rtx && implicit_built_in_decls[BUILT_IN_MEMCPY])
+  if (target == const0_rtx && builtin_decl_implicit_p (BUILT_IN_MEMCPY))
     {
-      tree fn = implicit_built_in_decls[BUILT_IN_MEMCPY];
+      tree fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
       tree result = build_call_nofold_loc (UNKNOWN_LOCATION, fn, 3,
                                           dest, src, len);
       return expand_expr (result, target, mode, EXPAND_NORMAL);
@@ -3291,9 +3294,9 @@ expand_builtin_stpcpy (tree exp, rtx target, enum machine_mode mode)
   src = CALL_EXPR_ARG (exp, 1);
 
   /* If return value is ignored, transform stpcpy into strcpy.  */
-  if (target == const0_rtx && implicit_built_in_decls[BUILT_IN_STRCPY])
+  if (target == const0_rtx && builtin_decl_implicit (BUILT_IN_STRCPY))
     {
-      tree fn = implicit_built_in_decls[BUILT_IN_STRCPY];
+      tree fn = builtin_decl_implicit (BUILT_IN_STRCPY);
       tree result = build_call_nofold_loc (loc, fn, 2, dst, src);
       return expand_expr (result, target, mode, EXPAND_NORMAL);
     }
@@ -3630,7 +3633,8 @@ expand_builtin_bzero (tree exp)
      calling bzero instead of memset.  */
 
   return expand_builtin_memset_args (dest, integer_zero_node,
-                                    fold_convert_loc (loc, sizetype, size),
+                                    fold_convert_loc (loc,
+                                                      size_type_node, size),
                                     const0_rtx, VOIDmode, exp);
 }
 
@@ -4224,11 +4228,10 @@ std_gimplify_va_arg_expr (tree valist, tree type, gimple_seq *pre_p,
                  fold_build_pointer_plus_hwi (valist_tmp, boundary - 1));
       gimplify_and_add (t, pre_p);
 
-      t = fold_convert (sizetype, valist_tmp);
       t = build2 (MODIFY_EXPR, TREE_TYPE (valist), valist_tmp,
-                 fold_convert (TREE_TYPE (valist),
-                               fold_build2 (BIT_AND_EXPR, sizetype, t,
-                                            size_int (-boundary))));
+                 fold_build2 (BIT_AND_EXPR, TREE_TYPE (valist),
+                              valist_tmp,
+                              build_int_cst (TREE_TYPE (valist), -boundary)));
       gimplify_and_add (t, pre_p);
     }
   else
@@ -4352,7 +4355,7 @@ gimplify_va_arg_expr (tree *expr_p, gimple_seq *pre_p, gimple_seq *post_p)
         expression to exit or longjmp.  */
       gimplify_and_add (valist, pre_p);
       t = build_call_expr_loc (loc,
-                              implicit_built_in_decls[BUILT_IN_TRAP], 0);
+                              builtin_decl_implicit (BUILT_IN_TRAP), 0);
       gimplify_and_add (t, pre_p);
 
       /* This is dead code, but go ahead and finish so that the
@@ -4515,20 +4518,33 @@ expand_builtin_alloca (tree exp, bool cannot_accumulate)
 {
   rtx op0;
   rtx result;
+  bool valid_arglist;
+  unsigned int align;
+  bool alloca_with_align = (DECL_FUNCTION_CODE (get_callee_fndecl (exp))
+                           == BUILT_IN_ALLOCA_WITH_ALIGN);
 
   /* Emit normal call if marked not-inlineable.  */
   if (CALL_CANNOT_INLINE_P (exp))
     return NULL_RTX;
 
-  if (!validate_arglist (exp, INTEGER_TYPE, VOID_TYPE))
+  valid_arglist
+    = (alloca_with_align
+       ? validate_arglist (exp, INTEGER_TYPE, INTEGER_TYPE, VOID_TYPE)
+       : validate_arglist (exp, INTEGER_TYPE, VOID_TYPE));
+
+  if (!valid_arglist)
     return NULL_RTX;
 
   /* Compute the argument.  */
   op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
 
+  /* Compute the alignment.  */
+  align = (alloca_with_align
+          ? TREE_INT_CST_LOW (CALL_EXPR_ARG (exp, 1))
+          : BIGGEST_ALIGNMENT);
+
   /* Allocate the desired space.  */
-  result = allocate_dynamic_stack_space (op0, 0, BIGGEST_ALIGNMENT,
-                                        cannot_accumulate);
+  result = allocate_dynamic_stack_space (op0, 0, align, cannot_accumulate);
   result = convert_memory_address (ptr_mode, result);
 
   return result;
@@ -5077,21 +5093,41 @@ get_builtin_sync_mem (tree loc, enum machine_mode mode)
   return mem;
 }
 
+/* Make sure an argument is in the right mode.
+   EXP is the tree argument. 
+   MODE is the mode it should be in.  */
+
+static rtx
+expand_expr_force_mode (tree exp, enum machine_mode mode)
+{
+  rtx val;
+  enum machine_mode old_mode;
+
+  val = expand_expr (exp, NULL_RTX, mode, EXPAND_NORMAL);
+  /* If VAL is promoted to a wider mode, convert it back to MODE.  Take care
+     of CONST_INTs, where we know the old_mode only from the call argument.  */
+
+  old_mode = GET_MODE (val);
+  if (old_mode == VOIDmode)
+    old_mode = TYPE_MODE (TREE_TYPE (exp));
+  val = convert_modes (mode, old_mode, val, 1);
+  return val;
+}
+
+
 /* Expand the __sync_xxx_and_fetch and __sync_fetch_and_xxx intrinsics.
    EXP is the CALL_EXPR.  CODE is the rtx code
    that corresponds to the arithmetic or logical operation from the name;
    an exception here is that NOT actually means NAND.  TARGET is an optional
    place for us to store the results; AFTER is true if this is the
-   fetch_and_xxx form.  IGNORE is true if we don't actually care about
-   the result of the operation at all.  */
+   fetch_and_xxx form.  */
 
 static rtx
 expand_builtin_sync_operation (enum machine_mode mode, tree exp,
                               enum rtx_code code, bool after,
-                              rtx target, bool ignore)
+                              rtx target)
 {
   rtx val, mem;
-  enum machine_mode old_mode;
   location_t loc = EXPR_LOCATION (exp);
 
   if (code == NOT && warn_sync_nand)
@@ -5108,11 +5144,10 @@ expand_builtin_sync_operation (enum machine_mode mode, tree exp,
        case BUILT_IN_SYNC_FETCH_AND_NAND_4:
        case BUILT_IN_SYNC_FETCH_AND_NAND_8:
        case BUILT_IN_SYNC_FETCH_AND_NAND_16:
-
          if (warned_f_a_n)
            break;
 
-         fndecl = implicit_built_in_decls[BUILT_IN_SYNC_FETCH_AND_NAND_N];
+         fndecl = builtin_decl_implicit (BUILT_IN_SYNC_FETCH_AND_NAND_N);
          inform (loc, "%qD changed semantics in GCC 4.4", fndecl);
          warned_f_a_n = true;
          break;
@@ -5122,11 +5157,10 @@ expand_builtin_sync_operation (enum machine_mode mode, tree exp,
        case BUILT_IN_SYNC_NAND_AND_FETCH_4:
        case BUILT_IN_SYNC_NAND_AND_FETCH_8:
        case BUILT_IN_SYNC_NAND_AND_FETCH_16:
-
          if (warned_n_a_f)
            break;
 
-         fndecl = implicit_built_in_decls[BUILT_IN_SYNC_NAND_AND_FETCH_N];
+        fndecl = builtin_decl_implicit (BUILT_IN_SYNC_NAND_AND_FETCH_N);
          inform (loc, "%qD changed semantics in GCC 4.4", fndecl);
          warned_n_a_f = true;
          break;
@@ -5138,19 +5172,10 @@ expand_builtin_sync_operation (enum machine_mode mode, tree exp,
 
   /* Expand the operands.  */
   mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
+  val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
 
-  val = expand_expr (CALL_EXPR_ARG (exp, 1), NULL_RTX, mode, EXPAND_NORMAL);
-  /* If VAL is promoted to a wider mode, convert it back to MODE.  Take care
-     of CONST_INTs, where we know the old_mode only from the call argument.  */
-  old_mode = GET_MODE (val);
-  if (old_mode == VOIDmode)
-    old_mode = TYPE_MODE (TREE_TYPE (CALL_EXPR_ARG (exp, 1)));
-  val = convert_modes (mode, old_mode, val, 1);
-
-  if (ignore)
-    return expand_sync_operation (mem, val, code);
-  else
-    return expand_sync_fetch_operation (mem, val, code, after, target);
+  return expand_atomic_fetch_op (target, mem, val, code, MEMMODEL_SEQ_CST,
+                                after);
 }
 
 /* Expand the __sync_val_compare_and_swap and __sync_bool_compare_and_swap
@@ -5163,34 +5188,27 @@ expand_builtin_compare_and_swap (enum machine_mode mode, tree exp,
                                 bool is_bool, rtx target)
 {
   rtx old_val, new_val, mem;
-  enum machine_mode old_mode;
+  rtx *pbool, *poval;
 
   /* Expand the operands.  */
   mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
+  old_val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
+  new_val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 2), mode);
 
+  pbool = poval = NULL;
+  if (target != const0_rtx)
+    {
+      if (is_bool)
+       pbool = ⌖
+      else
+       poval = ⌖
+    }
+  if (!expand_atomic_compare_and_swap (pbool, poval, mem, old_val, new_val,
+                                      false, MEMMODEL_SEQ_CST,
+                                      MEMMODEL_SEQ_CST))
+    return NULL_RTX;
 
-  old_val = expand_expr (CALL_EXPR_ARG (exp, 1), NULL_RTX,
-                        mode, EXPAND_NORMAL);
-  /* If VAL is promoted to a wider mode, convert it back to MODE.  Take care
-     of CONST_INTs, where we know the old_mode only from the call argument.  */
-  old_mode = GET_MODE (old_val);
-  if (old_mode == VOIDmode)
-    old_mode = TYPE_MODE (TREE_TYPE (CALL_EXPR_ARG (exp, 1)));
-  old_val = convert_modes (mode, old_mode, old_val, 1);
-
-  new_val = expand_expr (CALL_EXPR_ARG (exp, 2), NULL_RTX,
-                        mode, EXPAND_NORMAL);
-  /* If VAL is promoted to a wider mode, convert it back to MODE.  Take care
-     of CONST_INTs, where we know the old_mode only from the call argument.  */
-  old_mode = GET_MODE (new_val);
-  if (old_mode == VOIDmode)
-    old_mode = TYPE_MODE (TREE_TYPE (CALL_EXPR_ARG (exp, 2)));
-  new_val = convert_modes (mode, old_mode, new_val, 1);
-
-  if (is_bool)
-    return expand_bool_compare_and_swap (mem, old_val, new_val, target);
-  else
-    return expand_val_compare_and_swap (mem, old_val, new_val, target);
+  return target;
 }
 
 /* Expand the __sync_lock_test_and_set intrinsic.  Note that the most
@@ -5201,22 +5219,536 @@ expand_builtin_compare_and_swap (enum machine_mode mode, tree exp,
 
 static rtx
 expand_builtin_sync_lock_test_and_set (enum machine_mode mode, tree exp,
-                                 rtx target)
+                                      rtx target)
 {
   rtx val, mem;
-  enum machine_mode old_mode;
 
   /* Expand the operands.  */
   mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
-  val = expand_expr (CALL_EXPR_ARG (exp, 1), NULL_RTX, mode, EXPAND_NORMAL);
-  /* If VAL is promoted to a wider mode, convert it back to MODE.  Take care
-     of CONST_INTs, where we know the old_mode only from the call argument.  */
-  old_mode = GET_MODE (val);
-  if (old_mode == VOIDmode)
-    old_mode = TYPE_MODE (TREE_TYPE (CALL_EXPR_ARG (exp, 1)));
-  val = convert_modes (mode, old_mode, val, 1);
+  val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
+
+  return expand_atomic_exchange (target, mem, val, MEMMODEL_ACQUIRE, true);
+}
+
+/* Expand the __sync_lock_release intrinsic.  EXP is the CALL_EXPR.  */
+
+static void
+expand_builtin_sync_lock_release (enum machine_mode mode, tree exp)
+{
+  rtx mem;
+
+  /* Expand the operands.  */
+  mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
+
+  expand_atomic_store (mem, const0_rtx, MEMMODEL_RELEASE, true);
+}
+
+/* Given an integer representing an ``enum memmodel'', verify its
+   correctness and return the memory model enum.  */
+
+static enum memmodel
+get_memmodel (tree exp)
+{
+  rtx op;
+
+  /* If the parameter is not a constant, it's a run time value so we'll just
+     convert it to MEMMODEL_SEQ_CST to avoid annoying runtime checking.  */
+  if (TREE_CODE (exp) != INTEGER_CST)
+    return MEMMODEL_SEQ_CST;
+
+  op = expand_normal (exp);
+  if (INTVAL (op) < 0 || INTVAL (op) >= MEMMODEL_LAST)
+    {
+      warning (OPT_Winvalid_memory_model,
+              "invalid memory model argument to builtin");
+      return MEMMODEL_SEQ_CST;
+    }
+  return (enum memmodel) INTVAL (op);
+}
+
+/* Expand the __atomic_exchange intrinsic:
+       TYPE __atomic_exchange (TYPE *object, TYPE desired, enum memmodel)
+   EXP is the CALL_EXPR.
+   TARGET is an optional place for us to store the results.  */
+
+static rtx
+expand_builtin_atomic_exchange (enum machine_mode mode, tree exp, rtx target)
+{
+  rtx val, mem;
+  enum memmodel model;
+
+  model = get_memmodel (CALL_EXPR_ARG (exp, 2));
+  if (model == MEMMODEL_CONSUME)
+    {
+      error ("invalid memory model for %<__atomic_exchange%>");
+      return NULL_RTX;
+    }
+
+  if (!flag_inline_atomics)
+    return NULL_RTX;
+
+  /* Expand the operands.  */
+  mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
+  val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
+
+  return expand_atomic_exchange (target, mem, val, model, false);
+}
+
+/* Expand the __atomic_compare_exchange intrinsic:
+       bool __atomic_compare_exchange (TYPE *object, TYPE *expect, 
+                                       TYPE desired, BOOL weak, 
+                                       enum memmodel success,
+                                       enum memmodel failure)
+   EXP is the CALL_EXPR.
+   TARGET is an optional place for us to store the results.  */
+
+static rtx
+expand_builtin_atomic_compare_exchange (enum machine_mode mode, tree exp, 
+                                       rtx target)
+{
+  rtx expect, desired, mem, oldval;
+  enum memmodel success, failure;
+  tree weak;
+  bool is_weak;
+
+  success = get_memmodel (CALL_EXPR_ARG (exp, 4));
+  failure = get_memmodel (CALL_EXPR_ARG (exp, 5));
 
-  return expand_sync_lock_test_and_set (mem, val, target);
+  if (failure == MEMMODEL_RELEASE || failure == MEMMODEL_ACQ_REL)
+    {
+      error ("invalid failure memory model for %<__atomic_compare_exchange%>");
+      return NULL_RTX;
+    }
+
+  if (failure > success)
+    {
+      error ("failure memory model cannot be stronger than success "
+            "memory model for %<__atomic_compare_exchange%>");
+      return NULL_RTX;
+    }
+  
+  if (!flag_inline_atomics)
+    return NULL_RTX;
+
+  /* Expand the operands.  */
+  mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
+
+  expect = expand_normal (CALL_EXPR_ARG (exp, 1));
+  expect = convert_memory_address (Pmode, expect);
+  desired = expand_expr_force_mode (CALL_EXPR_ARG (exp, 2), mode);
+
+  weak = CALL_EXPR_ARG (exp, 3);
+  is_weak = false;
+  if (host_integerp (weak, 0) && tree_low_cst (weak, 0) != 0)
+    is_weak = true;
+
+  oldval = copy_to_reg (gen_rtx_MEM (mode, expect));
+
+  if (!expand_atomic_compare_and_swap ((target == const0_rtx ? NULL : &target),
+                                      &oldval, mem, oldval, desired,
+                                      is_weak, success, failure))
+    return NULL_RTX;
+
+  emit_move_insn (gen_rtx_MEM (mode, expect), oldval);
+  return target;
+}
+
+/* Expand the __atomic_load intrinsic:
+       TYPE __atomic_load (TYPE *object, enum memmodel)
+   EXP is the CALL_EXPR.
+   TARGET is an optional place for us to store the results.  */
+
+static rtx
+expand_builtin_atomic_load (enum machine_mode mode, tree exp, rtx target)
+{
+  rtx mem;
+  enum memmodel model;
+
+  model = get_memmodel (CALL_EXPR_ARG (exp, 1));
+  if (model == MEMMODEL_RELEASE
+      || model == MEMMODEL_ACQ_REL)
+    {
+      error ("invalid memory model for %<__atomic_load%>");
+      return NULL_RTX;
+    }
+
+  if (!flag_inline_atomics)
+    return NULL_RTX;
+
+  /* Expand the operand.  */
+  mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
+
+  return expand_atomic_load (target, mem, model);
+}
+
+
+/* Expand the __atomic_store intrinsic:
+       void __atomic_store (TYPE *object, TYPE desired, enum memmodel)
+   EXP is the CALL_EXPR.
+   TARGET is an optional place for us to store the results.  */
+
+static rtx
+expand_builtin_atomic_store (enum machine_mode mode, tree exp)
+{
+  rtx mem, val;
+  enum memmodel model;
+
+  model = get_memmodel (CALL_EXPR_ARG (exp, 2));
+  if (model != MEMMODEL_RELAXED
+      && model != MEMMODEL_SEQ_CST
+      && model != MEMMODEL_RELEASE)
+    {
+      error ("invalid memory model for %<__atomic_store%>");
+      return NULL_RTX;
+    }
+
+  if (!flag_inline_atomics)
+    return NULL_RTX;
+
+  /* Expand the operands.  */
+  mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
+  val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
+
+  return expand_atomic_store (mem, val, model, false);
+}
+
+/* Expand the __atomic_fetch_XXX intrinsic:
+       TYPE __atomic_fetch_XXX (TYPE *object, TYPE val, enum memmodel)
+   EXP is the CALL_EXPR.
+   TARGET is an optional place for us to store the results.
+   CODE is the operation, PLUS, MINUS, ADD, XOR, or IOR.
+   FETCH_AFTER is true if returning the result of the operation.
+   FETCH_AFTER is false if returning the value before the operation.
+   IGNORE is true if the result is not used.
+   EXT_CALL is the correct builtin for an external call if this cannot be
+   resolved to an instruction sequence.  */
+
+static rtx
+expand_builtin_atomic_fetch_op (enum machine_mode mode, tree exp, rtx target,
+                               enum rtx_code code, bool fetch_after,
+                               bool ignore, enum built_in_function ext_call)
+{
+  rtx val, mem, ret;
+  enum memmodel model;
+  tree fndecl;
+  tree addr;
+
+  model = get_memmodel (CALL_EXPR_ARG (exp, 2));
+
+  /* Expand the operands.  */
+  mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
+  val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
+
+  /* Only try generating instructions if inlining is turned on.  */
+  if (flag_inline_atomics)
+    {
+      ret = expand_atomic_fetch_op (target, mem, val, code, model, fetch_after);
+      if (ret)
+       return ret;
+    }
+
+  /* Return if a different routine isn't needed for the library call.  */
+  if (ext_call == BUILT_IN_NONE)
+    return NULL_RTX;
+
+  /* Change the call to the specified function.  */
+  fndecl = get_callee_fndecl (exp);
+  addr = CALL_EXPR_FN (exp);
+  STRIP_NOPS (addr);
+
+  gcc_assert (TREE_OPERAND (addr, 0) == fndecl);
+  TREE_OPERAND (addr, 0) = builtin_decl_explicit(ext_call);
+
+  /* Expand the call here so we can emit trailing code.  */
+  ret = expand_call (exp, target, ignore);
+
+  /* Replace the original function just in case it matters.  */
+  TREE_OPERAND (addr, 0) = fndecl;
+
+  /* Then issue the arithmetic correction to return the right result.  */
+  if (!ignore)
+    {
+      if (code == NOT)
+       {
+         ret = expand_simple_binop (mode, AND, ret, val, NULL_RTX, true,
+                                    OPTAB_LIB_WIDEN);
+         ret = expand_simple_unop (mode, NOT, ret, target, true);
+       }
+      else
+       ret = expand_simple_binop (mode, code, ret, val, target, true,
+                                  OPTAB_LIB_WIDEN);
+    }
+  return ret;
+}
+
+
+/* Expand an atomic clear operation.
+       void _atomic_clear (BOOL *obj, enum memmodel)
+   EXP is the call expression.  */
+
+static rtx
+expand_builtin_atomic_clear (tree exp) 
+{
+  enum machine_mode mode;
+  rtx mem, ret;
+  enum memmodel model;
+
+  mode = mode_for_size (BOOL_TYPE_SIZE, MODE_INT, 0);
+  mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
+  model = get_memmodel (CALL_EXPR_ARG (exp, 1));
+
+  if (model == MEMMODEL_ACQUIRE || model == MEMMODEL_ACQ_REL)
+    {
+      error ("invalid memory model for %<__atomic_store%>");
+      return const0_rtx;
+    }
+
+  /* Try issuing an __atomic_store, and allow fallback to __sync_lock_release.
+     Failing that, a store is issued by __atomic_store.  The only way this can
+     fail is if the bool type is larger than a word size.  Unlikely, but
+     handle it anyway for completeness.  Assume a single threaded model since
+     there is no atomic support in this case, and no barriers are required.  */
+  ret = expand_atomic_store (mem, const0_rtx, model, true);
+  if (!ret)
+    emit_move_insn (mem, const0_rtx);
+  return const0_rtx;
+}
+
+/* Expand an atomic test_and_set operation.
+       bool _atomic_test_and_set (BOOL *obj, enum memmodel)
+   EXP is the call expression.  */
+
+static rtx
+expand_builtin_atomic_test_and_set (tree exp)
+{
+  rtx mem, ret;
+  enum memmodel model;
+  enum machine_mode mode;
+
+  mode = mode_for_size (BOOL_TYPE_SIZE, MODE_INT, 0);
+  mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
+  model = get_memmodel (CALL_EXPR_ARG (exp, 1));
+
+  /* Try issuing an exchange.  If it is lock free, or if there is a limited
+     functionality __sync_lock_test_and_set, this will utilize it.  */
+  ret = expand_atomic_exchange (NULL_RTX, mem, const1_rtx, model, true);
+  if (ret)
+    return ret;
+
+  /* Otherwise, there is no lock free support for test and set.  Simply
+     perform a load and a store.  Since this presumes a non-atomic architecture,
+     also assume single threadedness and don't issue barriers either. */
+
+  ret = gen_reg_rtx (mode);
+  emit_move_insn (ret, mem);
+  emit_move_insn (mem, const1_rtx);
+  return ret;
+}
+
+
+/* Return true if (optional) argument ARG1 of size ARG0 is always lock free on
+   this architecture.  If ARG1 is NULL, use typical alignment for size ARG0.  */
+
+static tree
+fold_builtin_atomic_always_lock_free (tree arg0, tree arg1)
+{
+  int size;
+  enum machine_mode mode;
+  unsigned int mode_align, type_align;
+
+  if (TREE_CODE (arg0) != INTEGER_CST)
+    return NULL_TREE;
+
+  size = INTVAL (expand_normal (arg0)) * BITS_PER_UNIT;
+  mode = mode_for_size (size, MODE_INT, 0);
+  mode_align = GET_MODE_ALIGNMENT (mode);
+
+  if (TREE_CODE (arg1) == INTEGER_CST && INTVAL (expand_normal (arg1)) == 0)
+    type_align = mode_align;
+  else
+    {
+      tree ttype = TREE_TYPE (arg1);
+
+      /* This function is usually invoked and folded immediately by the front
+        end before anything else has a chance to look at it.  The pointer
+        parameter at this point is usually cast to a void *, so check for that
+        and look past the cast.  */
+      if (TREE_CODE (arg1) == NOP_EXPR && POINTER_TYPE_P (ttype)
+         && VOID_TYPE_P (TREE_TYPE (ttype)))
+       arg1 = TREE_OPERAND (arg1, 0);
+
+      ttype = TREE_TYPE (arg1);
+      gcc_assert (POINTER_TYPE_P (ttype));
+
+      /* Get the underlying type of the object.  */
+      ttype = TREE_TYPE (ttype);
+      type_align = TYPE_ALIGN (ttype);
+    }
+
+  /* If the object has smaller alignment, the the lock free routines cannot
+     be used.  */
+  if (type_align < mode_align)
+    return integer_zero_node;
+
+  /* Check if a compare_and_swap pattern exists for the mode which represents
+     the required size.  The pattern is not allowed to fail, so the existence
+     of the pattern indicates support is present.  */
+  if (can_compare_and_swap_p (mode, true))
+    return integer_one_node;
+  else
+    return integer_zero_node;
+}
+
+/* Return true if the parameters to call EXP represent an object which will
+   always generate lock free instructions.  The first argument represents the
+   size of the object, and the second parameter is a pointer to the object 
+   itself.  If NULL is passed for the object, then the result is based on 
+   typical alignment for an object of the specified size.  Otherwise return 
+   false.  */
+
+static rtx
+expand_builtin_atomic_always_lock_free (tree exp)
+{
+  tree size;
+  tree arg0 = CALL_EXPR_ARG (exp, 0);
+  tree arg1 = CALL_EXPR_ARG (exp, 1);
+
+  if (TREE_CODE (arg0) != INTEGER_CST)
+    {
+      error ("non-constant argument 1 to __atomic_always_lock_free");
+      return const0_rtx;
+    }
+
+  size = fold_builtin_atomic_always_lock_free (arg0, arg1);
+  if (size == integer_one_node)
+    return const1_rtx;
+  return const0_rtx;
+}
+
+/* Return a one or zero if it can be determined that object ARG1 of size ARG 
+   is lock free on this architecture.  */
+
+static tree
+fold_builtin_atomic_is_lock_free (tree arg0, tree arg1)
+{
+  if (!flag_inline_atomics)
+    return NULL_TREE;
+  
+  /* If it isn't always lock free, don't generate a result.  */
+  if (fold_builtin_atomic_always_lock_free (arg0, arg1) == integer_one_node)
+    return integer_one_node;
+
+  return NULL_TREE;
+}
+
+/* Return true if the parameters to call EXP represent an object which will
+   always generate lock free instructions.  The first argument represents the
+   size of the object, and the second parameter is a pointer to the object 
+   itself.  If NULL is passed for the object, then the result is based on 
+   typical alignment for an object of the specified size.  Otherwise return 
+   NULL*/
+
+static rtx
+expand_builtin_atomic_is_lock_free (tree exp)
+{
+  tree size;
+  tree arg0 = CALL_EXPR_ARG (exp, 0);
+  tree arg1 = CALL_EXPR_ARG (exp, 1);
+
+  if (!INTEGRAL_TYPE_P (TREE_TYPE (arg0)))
+    {
+      error ("non-integer argument 1 to __atomic_is_lock_free");
+      return NULL_RTX;
+    }
+
+  if (!flag_inline_atomics)
+    return NULL_RTX; 
+
+  /* If the value is known at compile time, return the RTX for it.  */
+  size = fold_builtin_atomic_is_lock_free (arg0, arg1);
+  if (size == integer_one_node)
+    return const1_rtx;
+
+  return NULL_RTX;
+}
+
+/* This routine will either emit the mem_thread_fence pattern or issue a 
+   sync_synchronize to generate a fence for memory model MEMMODEL.  */
+
+#ifndef HAVE_mem_thread_fence
+# define HAVE_mem_thread_fence 0
+# define gen_mem_thread_fence(x) (gcc_unreachable (), NULL_RTX)
+#endif
+
+void
+expand_builtin_mem_thread_fence (enum memmodel model)
+{
+  if (HAVE_mem_thread_fence)
+    emit_insn (gen_mem_thread_fence (GEN_INT (model)));
+  else if (model != MEMMODEL_RELAXED)
+    expand_builtin_sync_synchronize ();
+}
+
+/* Expand the __atomic_thread_fence intrinsic:
+       void __atomic_thread_fence (enum memmodel)
+   EXP is the CALL_EXPR.  */
+
+static void
+expand_builtin_atomic_thread_fence (tree exp)
+{
+  enum memmodel model;
+  
+  model = get_memmodel (CALL_EXPR_ARG (exp, 0));
+  expand_builtin_mem_thread_fence (model);
+}
+
+/* This routine will either emit the mem_signal_fence pattern or issue a 
+   sync_synchronize to generate a fence for memory model MEMMODEL.  */
+
+#ifndef HAVE_mem_signal_fence
+# define HAVE_mem_signal_fence 0
+# define gen_mem_signal_fence(x) (gcc_unreachable (), NULL_RTX)
+#endif
+
+static void
+expand_builtin_mem_signal_fence (enum memmodel model)
+{
+  if (HAVE_mem_signal_fence)
+    emit_insn (gen_mem_signal_fence (GEN_INT (model)));
+  else if (model != MEMMODEL_RELAXED)
+    {
+      rtx asm_op, clob;
+
+      /* By default targets are coherent between a thread and the signal
+        handler running on the same thread.  Thus this really becomes a
+        compiler barrier, in that stores must not be sunk past
+        (or raised above) a given point.  */
+
+      /* Generate asm volatile("" : : : "memory") as the memory barrier.  */
+      asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, empty_string, empty_string, 0,
+                                    rtvec_alloc (0), rtvec_alloc (0),
+                                    rtvec_alloc (0), UNKNOWN_LOCATION);
+      MEM_VOLATILE_P (asm_op) = 1;
+
+      clob = gen_rtx_SCRATCH (VOIDmode);
+      clob = gen_rtx_MEM (BLKmode, clob);
+      clob = gen_rtx_CLOBBER (VOIDmode, clob);
+
+      emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
+    }
+}
+
+/* Expand the __atomic_signal_fence intrinsic:
+       void __atomic_signal_fence (enum memmodel)
+   EXP is the CALL_EXPR.  */
+
+static void
+expand_builtin_atomic_signal_fence (tree exp)
+{
+  enum memmodel model;
+
+  model = get_memmodel (CALL_EXPR_ARG (exp, 0));
+  expand_builtin_mem_signal_fence (model);
 }
 
 /* Expand the __sync_synchronize intrinsic.  */
@@ -5251,33 +5783,6 @@ expand_builtin_sync_synchronize (void)
   expand_asm_stmt (x);
 }
 
-/* Expand the __sync_lock_release intrinsic.  EXP is the CALL_EXPR.  */
-
-static void
-expand_builtin_sync_lock_release (enum machine_mode mode, tree exp)
-{
-  struct expand_operand ops[2];
-  enum insn_code icode;
-  rtx mem;
-
-  /* Expand the operands.  */
-  mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
-
-  /* If there is an explicit operation in the md file, use it.  */
-  icode = direct_optab_handler (sync_lock_release_optab, mode);
-  if (icode != CODE_FOR_nothing)
-    {
-      create_fixed_operand (&ops[0], mem);
-      create_input_operand (&ops[1], const0_rtx, mode);
-      if (maybe_expand_insn (icode, 2, ops))
-       return;
-    }
-
-  /* Otherwise we can implement this operation by emitting a barrier
-     followed by a store of zero.  */
-  expand_builtin_sync_synchronize ();
-  emit_move_insn (mem, const0_rtx);
-}
 \f
 /* Expand an expression EXP that calls a built-in function,
    with result going to TARGET if that's convenient
@@ -5303,6 +5808,7 @@ expand_builtin (tree exp, rtx target, rtx subtarget, enum machine_mode mode,
       && !called_as_built_in (fndecl)
       && DECL_ASSEMBLER_NAME_SET_P (fndecl)
       && fcode != BUILT_IN_ALLOCA
+      && fcode != BUILT_IN_ALLOCA_WITH_ALIGN
       && fcode != BUILT_IN_FREE)
     return expand_call (exp, target, ignore);
 
@@ -5558,6 +6064,7 @@ expand_builtin (tree exp, rtx target, rtx subtarget, enum machine_mode mode,
        return XEXP (DECL_RTL (DECL_RESULT (current_function_decl)), 0);
 
     case BUILT_IN_ALLOCA:
+    case BUILT_IN_ALLOCA_WITH_ALIGN:
       /* If the allocation stems from the declaration of a variable-sized
         object, it cannot accumulate.  */
       target = expand_builtin_alloca (exp, CALL_ALLOCA_FOR_VAR_P (exp));
@@ -5876,8 +6383,7 @@ expand_builtin (tree exp, rtx target, rtx subtarget, enum machine_mode mode,
     case BUILT_IN_SYNC_FETCH_AND_ADD_8:
     case BUILT_IN_SYNC_FETCH_AND_ADD_16:
       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_FETCH_AND_ADD_1);
-      target = expand_builtin_sync_operation (mode, exp, PLUS,
-                                             false, target, ignore);
+      target = expand_builtin_sync_operation (mode, exp, PLUS, false, target);
       if (target)
        return target;
       break;
@@ -5888,8 +6394,7 @@ expand_builtin (tree exp, rtx target, rtx subtarget, enum machine_mode mode,
     case BUILT_IN_SYNC_FETCH_AND_SUB_8:
     case BUILT_IN_SYNC_FETCH_AND_SUB_16:
       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_FETCH_AND_SUB_1);
-      target = expand_builtin_sync_operation (mode, exp, MINUS,
-                                             false, target, ignore);
+      target = expand_builtin_sync_operation (mode, exp, MINUS, false, target);
       if (target)
        return target;
       break;
@@ -5900,8 +6405,7 @@ expand_builtin (tree exp, rtx target, rtx subtarget, enum machine_mode mode,
     case BUILT_IN_SYNC_FETCH_AND_OR_8:
     case BUILT_IN_SYNC_FETCH_AND_OR_16:
       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_FETCH_AND_OR_1);
-      target = expand_builtin_sync_operation (mode, exp, IOR,
-                                             false, target, ignore);
+      target = expand_builtin_sync_operation (mode, exp, IOR, false, target);
       if (target)
        return target;
       break;
@@ -5912,8 +6416,7 @@ expand_builtin (tree exp, rtx target, rtx subtarget, enum machine_mode mode,
     case BUILT_IN_SYNC_FETCH_AND_AND_8:
     case BUILT_IN_SYNC_FETCH_AND_AND_16:
       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_FETCH_AND_AND_1);
-      target = expand_builtin_sync_operation (mode, exp, AND,
-                                             false, target, ignore);
+      target = expand_builtin_sync_operation (mode, exp, AND, false, target);
       if (target)
        return target;
       break;
@@ -5924,8 +6427,7 @@ expand_builtin (tree exp, rtx target, rtx subtarget, enum machine_mode mode,
     case BUILT_IN_SYNC_FETCH_AND_XOR_8:
     case BUILT_IN_SYNC_FETCH_AND_XOR_16:
       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_FETCH_AND_XOR_1);
-      target = expand_builtin_sync_operation (mode, exp, XOR,
-                                             false, target, ignore);
+      target = expand_builtin_sync_operation (mode, exp, XOR, false, target);
       if (target)
        return target;
       break;
@@ -5936,8 +6438,7 @@ expand_builtin (tree exp, rtx target, rtx subtarget, enum machine_mode mode,
     case BUILT_IN_SYNC_FETCH_AND_NAND_8:
     case BUILT_IN_SYNC_FETCH_AND_NAND_16:
       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_FETCH_AND_NAND_1);
-      target = expand_builtin_sync_operation (mode, exp, NOT,
-                                             false, target, ignore);
+      target = expand_builtin_sync_operation (mode, exp, NOT, false, target);
       if (target)
        return target;
       break;
@@ -5948,8 +6449,7 @@ expand_builtin (tree exp, rtx target, rtx subtarget, enum machine_mode mode,
     case BUILT_IN_SYNC_ADD_AND_FETCH_8:
     case BUILT_IN_SYNC_ADD_AND_FETCH_16:
       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_ADD_AND_FETCH_1);
-      target = expand_builtin_sync_operation (mode, exp, PLUS,
-                                             true, target, ignore);
+      target = expand_builtin_sync_operation (mode, exp, PLUS, true, target);
       if (target)
        return target;
       break;
@@ -5960,8 +6460,7 @@ expand_builtin (tree exp, rtx target, rtx subtarget, enum machine_mode mode,
     case BUILT_IN_SYNC_SUB_AND_FETCH_8:
     case BUILT_IN_SYNC_SUB_AND_FETCH_16:
       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_SUB_AND_FETCH_1);
-      target = expand_builtin_sync_operation (mode, exp, MINUS,
-                                             true, target, ignore);
+      target = expand_builtin_sync_operation (mode, exp, MINUS, true, target);
       if (target)
        return target;
       break;
@@ -5972,8 +6471,7 @@ expand_builtin (tree exp, rtx target, rtx subtarget, enum machine_mode mode,
     case BUILT_IN_SYNC_OR_AND_FETCH_8:
     case BUILT_IN_SYNC_OR_AND_FETCH_16:
       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_OR_AND_FETCH_1);
-      target = expand_builtin_sync_operation (mode, exp, IOR,
-                                             true, target, ignore);
+      target = expand_builtin_sync_operation (mode, exp, IOR, true, target);
       if (target)
        return target;
       break;
@@ -5984,8 +6482,7 @@ expand_builtin (tree exp, rtx target, rtx subtarget, enum machine_mode mode,
     case BUILT_IN_SYNC_AND_AND_FETCH_8:
     case BUILT_IN_SYNC_AND_AND_FETCH_16:
       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_AND_AND_FETCH_1);
-      target = expand_builtin_sync_operation (mode, exp, AND,
-                                             true, target, ignore);
+      target = expand_builtin_sync_operation (mode, exp, AND, true, target);
       if (target)
        return target;
       break;
@@ -5996,8 +6493,7 @@ expand_builtin (tree exp, rtx target, rtx subtarget, enum machine_mode mode,
     case BUILT_IN_SYNC_XOR_AND_FETCH_8:
     case BUILT_IN_SYNC_XOR_AND_FETCH_16:
       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_XOR_AND_FETCH_1);
-      target = expand_builtin_sync_operation (mode, exp, XOR,
-                                             true, target, ignore);
+      target = expand_builtin_sync_operation (mode, exp, XOR, true, target);
       if (target)
        return target;
       break;
@@ -6008,8 +6504,7 @@ expand_builtin (tree exp, rtx target, rtx subtarget, enum machine_mode mode,
     case BUILT_IN_SYNC_NAND_AND_FETCH_8:
     case BUILT_IN_SYNC_NAND_AND_FETCH_16:
       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_NAND_AND_FETCH_1);
-      target = expand_builtin_sync_operation (mode, exp, NOT,
-                                             true, target, ignore);
+      target = expand_builtin_sync_operation (mode, exp, NOT, true, target);
       if (target)
        return target;
       break;
@@ -6067,6 +6562,242 @@ expand_builtin (tree exp, rtx target, rtx subtarget, enum machine_mode mode,
       expand_builtin_sync_synchronize ();
       return const0_rtx;
 
+    case BUILT_IN_ATOMIC_EXCHANGE_1:
+    case BUILT_IN_ATOMIC_EXCHANGE_2:
+    case BUILT_IN_ATOMIC_EXCHANGE_4:
+    case BUILT_IN_ATOMIC_EXCHANGE_8:
+    case BUILT_IN_ATOMIC_EXCHANGE_16:
+      mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_EXCHANGE_1);
+      target = expand_builtin_atomic_exchange (mode, exp, target);
+      if (target)
+       return target;
+      break;
+
+    case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_1:
+    case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_2:
+    case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_4:
+    case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_8:
+    case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_16:
+      mode = 
+         get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_COMPARE_EXCHANGE_1);
+      target = expand_builtin_atomic_compare_exchange (mode, exp, target);
+      if (target)
+       return target;
+      break;
+
+    case BUILT_IN_ATOMIC_LOAD_1:
+    case BUILT_IN_ATOMIC_LOAD_2:
+    case BUILT_IN_ATOMIC_LOAD_4:
+    case BUILT_IN_ATOMIC_LOAD_8:
+    case BUILT_IN_ATOMIC_LOAD_16:
+      mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_LOAD_1);
+      target = expand_builtin_atomic_load (mode, exp, target);
+      if (target)
+       return target;
+      break;
+
+    case BUILT_IN_ATOMIC_STORE_1:
+    case BUILT_IN_ATOMIC_STORE_2:
+    case BUILT_IN_ATOMIC_STORE_4:
+    case BUILT_IN_ATOMIC_STORE_8:
+    case BUILT_IN_ATOMIC_STORE_16:
+      mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_STORE_1);
+      target = expand_builtin_atomic_store (mode, exp);
+      if (target)
+       return const0_rtx;
+      break;
+
+    case BUILT_IN_ATOMIC_ADD_FETCH_1:
+    case BUILT_IN_ATOMIC_ADD_FETCH_2:
+    case BUILT_IN_ATOMIC_ADD_FETCH_4:
+    case BUILT_IN_ATOMIC_ADD_FETCH_8:
+    case BUILT_IN_ATOMIC_ADD_FETCH_16:
+      {
+       enum built_in_function lib;
+       mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_ADD_FETCH_1);
+       lib = (enum built_in_function)((int)BUILT_IN_ATOMIC_FETCH_ADD_1 + 
+                                      (fcode - BUILT_IN_ATOMIC_ADD_FETCH_1));
+       target = expand_builtin_atomic_fetch_op (mode, exp, target, PLUS, true,
+                                                ignore, lib);
+       if (target)
+         return target;
+       break;
+      }
+    case BUILT_IN_ATOMIC_SUB_FETCH_1:
+    case BUILT_IN_ATOMIC_SUB_FETCH_2:
+    case BUILT_IN_ATOMIC_SUB_FETCH_4:
+    case BUILT_IN_ATOMIC_SUB_FETCH_8:
+    case BUILT_IN_ATOMIC_SUB_FETCH_16:
+      {
+       enum built_in_function lib;
+       mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_SUB_FETCH_1);
+       lib = (enum built_in_function)((int)BUILT_IN_ATOMIC_FETCH_SUB_1 + 
+                                      (fcode - BUILT_IN_ATOMIC_SUB_FETCH_1));
+       target = expand_builtin_atomic_fetch_op (mode, exp, target, MINUS, true,
+                                                ignore, lib);
+       if (target)
+         return target;
+       break;
+      }
+    case BUILT_IN_ATOMIC_AND_FETCH_1:
+    case BUILT_IN_ATOMIC_AND_FETCH_2:
+    case BUILT_IN_ATOMIC_AND_FETCH_4:
+    case BUILT_IN_ATOMIC_AND_FETCH_8:
+    case BUILT_IN_ATOMIC_AND_FETCH_16:
+      {
+       enum built_in_function lib;
+       mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_AND_FETCH_1);
+       lib = (enum built_in_function)((int)BUILT_IN_ATOMIC_FETCH_AND_1 + 
+                                      (fcode - BUILT_IN_ATOMIC_AND_FETCH_1));
+       target = expand_builtin_atomic_fetch_op (mode, exp, target, AND, true,
+                                                ignore, lib);
+       if (target)
+         return target;
+       break;
+      }
+    case BUILT_IN_ATOMIC_NAND_FETCH_1:
+    case BUILT_IN_ATOMIC_NAND_FETCH_2:
+    case BUILT_IN_ATOMIC_NAND_FETCH_4:
+    case BUILT_IN_ATOMIC_NAND_FETCH_8:
+    case BUILT_IN_ATOMIC_NAND_FETCH_16:
+      {
+       enum built_in_function lib;
+       mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_NAND_FETCH_1);
+       lib = (enum built_in_function)((int)BUILT_IN_ATOMIC_FETCH_NAND_1 + 
+                                      (fcode - BUILT_IN_ATOMIC_NAND_FETCH_1));
+       target = expand_builtin_atomic_fetch_op (mode, exp, target, NOT, true,
+                                                ignore, lib);
+       if (target)
+         return target;
+       break;
+      }
+    case BUILT_IN_ATOMIC_XOR_FETCH_1:
+    case BUILT_IN_ATOMIC_XOR_FETCH_2:
+    case BUILT_IN_ATOMIC_XOR_FETCH_4:
+    case BUILT_IN_ATOMIC_XOR_FETCH_8:
+    case BUILT_IN_ATOMIC_XOR_FETCH_16:
+      {
+       enum built_in_function lib;
+       mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_XOR_FETCH_1);
+       lib = (enum built_in_function)((int)BUILT_IN_ATOMIC_FETCH_XOR_1 + 
+                                      (fcode - BUILT_IN_ATOMIC_XOR_FETCH_1));
+       target = expand_builtin_atomic_fetch_op (mode, exp, target, XOR, true,
+                                                ignore, lib);
+       if (target)
+         return target;
+       break;
+      }
+    case BUILT_IN_ATOMIC_OR_FETCH_1:
+    case BUILT_IN_ATOMIC_OR_FETCH_2:
+    case BUILT_IN_ATOMIC_OR_FETCH_4:
+    case BUILT_IN_ATOMIC_OR_FETCH_8:
+    case BUILT_IN_ATOMIC_OR_FETCH_16:
+      {
+       enum built_in_function lib;
+       mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_OR_FETCH_1);
+       lib = (enum built_in_function)((int)BUILT_IN_ATOMIC_FETCH_OR_1 + 
+                                      (fcode - BUILT_IN_ATOMIC_OR_FETCH_1));
+       target = expand_builtin_atomic_fetch_op (mode, exp, target, IOR, true,
+                                                ignore, lib);
+       if (target)
+         return target;
+       break;
+      }
+    case BUILT_IN_ATOMIC_FETCH_ADD_1:
+    case BUILT_IN_ATOMIC_FETCH_ADD_2:
+    case BUILT_IN_ATOMIC_FETCH_ADD_4:
+    case BUILT_IN_ATOMIC_FETCH_ADD_8:
+    case BUILT_IN_ATOMIC_FETCH_ADD_16:
+      mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_FETCH_ADD_1);
+      target = expand_builtin_atomic_fetch_op (mode, exp, target, PLUS, false,
+                                              ignore, BUILT_IN_NONE);
+      if (target)
+       return target;
+      break;
+    case BUILT_IN_ATOMIC_FETCH_SUB_1:
+    case BUILT_IN_ATOMIC_FETCH_SUB_2:
+    case BUILT_IN_ATOMIC_FETCH_SUB_4:
+    case BUILT_IN_ATOMIC_FETCH_SUB_8:
+    case BUILT_IN_ATOMIC_FETCH_SUB_16:
+      mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_FETCH_SUB_1);
+      target = expand_builtin_atomic_fetch_op (mode, exp, target, MINUS, false,
+                                              ignore, BUILT_IN_NONE);
+      if (target)
+       return target;
+      break;
+
+    case BUILT_IN_ATOMIC_FETCH_AND_1:
+    case BUILT_IN_ATOMIC_FETCH_AND_2:
+    case BUILT_IN_ATOMIC_FETCH_AND_4:
+    case BUILT_IN_ATOMIC_FETCH_AND_8:
+    case BUILT_IN_ATOMIC_FETCH_AND_16:
+      mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_FETCH_AND_1);
+      target = expand_builtin_atomic_fetch_op (mode, exp, target, AND, false,
+                                              ignore, BUILT_IN_NONE);
+      if (target)
+       return target;
+      break;
+  
+    case BUILT_IN_ATOMIC_FETCH_NAND_1:
+    case BUILT_IN_ATOMIC_FETCH_NAND_2:
+    case BUILT_IN_ATOMIC_FETCH_NAND_4:
+    case BUILT_IN_ATOMIC_FETCH_NAND_8:
+    case BUILT_IN_ATOMIC_FETCH_NAND_16:
+      mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_FETCH_NAND_1);
+      target = expand_builtin_atomic_fetch_op (mode, exp, target, NOT, false,
+                                              ignore, BUILT_IN_NONE);
+      if (target)
+       return target;
+      break;
+    case BUILT_IN_ATOMIC_FETCH_XOR_1:
+    case BUILT_IN_ATOMIC_FETCH_XOR_2:
+    case BUILT_IN_ATOMIC_FETCH_XOR_4:
+    case BUILT_IN_ATOMIC_FETCH_XOR_8:
+    case BUILT_IN_ATOMIC_FETCH_XOR_16:
+      mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_FETCH_XOR_1);
+      target = expand_builtin_atomic_fetch_op (mode, exp, target, XOR, false,
+                                              ignore, BUILT_IN_NONE);
+      if (target)
+       return target;
+      break;
+    case BUILT_IN_ATOMIC_FETCH_OR_1:
+    case BUILT_IN_ATOMIC_FETCH_OR_2:
+    case BUILT_IN_ATOMIC_FETCH_OR_4:
+    case BUILT_IN_ATOMIC_FETCH_OR_8:
+    case BUILT_IN_ATOMIC_FETCH_OR_16:
+      mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_FETCH_OR_1);
+      target = expand_builtin_atomic_fetch_op (mode, exp, target, IOR, false,
+                                              ignore, BUILT_IN_NONE);
+      if (target)
+       return target;
+      break;
+
+    case BUILT_IN_ATOMIC_TEST_AND_SET:
+      return expand_builtin_atomic_test_and_set (exp);
+
+    case BUILT_IN_ATOMIC_CLEAR:
+      return expand_builtin_atomic_clear (exp);
+    case BUILT_IN_ATOMIC_ALWAYS_LOCK_FREE:
+      return expand_builtin_atomic_always_lock_free (exp);
+
+    case BUILT_IN_ATOMIC_IS_LOCK_FREE:
+      target = expand_builtin_atomic_is_lock_free (exp);
+      if (target)
+        return target;
+      break;
+
+    case BUILT_IN_ATOMIC_THREAD_FENCE:
+      expand_builtin_atomic_thread_fence (exp);
+      return const0_rtx;
+
+    case BUILT_IN_ATOMIC_SIGNAL_FENCE:
+      expand_builtin_atomic_signal_fence (exp);
+      return const0_rtx;
+
     case BUILT_IN_OBJECT_SIZE:
       return expand_builtin_object_size (exp);
 
@@ -6095,7 +6826,8 @@ expand_builtin (tree exp, rtx target, rtx subtarget, enum machine_mode mode,
       break;
 
     case BUILT_IN_FREE:
-      maybe_emit_free_warning (exp);
+      if (warn_free_nonheap_object)
+       maybe_emit_free_warning (exp);
       break;
 
     default:   /* just do library call, if unknown builtin */
@@ -6230,7 +6962,7 @@ build_builtin_expect_predicate (location_t loc, tree pred, tree expected)
 {
   tree fn, arg_types, pred_type, expected_type, call_expr, ret_type;
 
-  fn = built_in_decls[BUILT_IN_EXPECT];
+  fn = builtin_decl_explicit (BUILT_IN_EXPECT);
   arg_types = TYPE_ARG_TYPES (TREE_TYPE (fn));
   ret_type = TREE_TYPE (TREE_TYPE (fn));
   pred_type = TREE_VALUE (arg_types);
@@ -7968,7 +8700,7 @@ fold_builtin_bzero (location_t loc, tree dest, tree size, bool ignore)
      calling bzero instead of memset.  */
 
   return fold_builtin_memset (loc, dest, integer_zero_node,
-                             fold_convert_loc (loc, sizetype, size),
+                             fold_convert_loc (loc, size_type_node, size),
                              void_type_node, ignore);
 }
 
@@ -8022,7 +8754,7 @@ fold_builtin_memory_op (location_t loc, tree dest, tree src,
                  && (MIN (src_align, dest_align) / BITS_PER_UNIT
                      >= (unsigned HOST_WIDE_INT) tree_low_cst (len, 1))))
            {
-             tree fn = implicit_built_in_decls[BUILT_IN_MEMCPY];
+             tree fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
              if (!fn)
                return NULL_TREE;
               return build_call_expr_loc (loc, fn, 3, dest, src, len);
@@ -8081,7 +8813,7 @@ fold_builtin_memory_op (location_t loc, tree dest, tree src,
              else
                return NULL_TREE;
 
-             fn = implicit_built_in_decls[BUILT_IN_MEMCPY];
+             fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
              if (!fn)
                return NULL_TREE;
              return build_call_expr_loc (loc, fn, 3, dest, src, len);
@@ -8100,7 +8832,7 @@ fold_builtin_memory_op (location_t loc, tree dest, tree src,
              if (!refs_may_alias_p_1 (&destr, &srcr, false))
                {
                  tree fn;
-                 fn = implicit_built_in_decls[BUILT_IN_MEMCPY];
+                 fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
                  if (!fn)
                    return NULL_TREE;
                  return build_call_expr_loc (loc, fn, 3, dest, src, len);
@@ -8275,7 +9007,7 @@ fold_builtin_strcpy (location_t loc, tree fndecl, tree dest, tree src, tree len)
   if (optimize_function_for_size_p (cfun))
     return NULL_TREE;
 
-  fn = implicit_built_in_decls[BUILT_IN_MEMCPY];
+  fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
   if (!fn)
     return NULL_TREE;
 
@@ -8286,7 +9018,8 @@ fold_builtin_strcpy (location_t loc, tree fndecl, tree dest, tree src, tree len)
        return NULL_TREE;
     }
 
-  len = size_binop_loc (loc, PLUS_EXPR, len, ssize_int (1));
+  len = fold_convert_loc (loc, size_type_node, len);
+  len = size_binop_loc (loc, PLUS_EXPR, len, build_int_cst (size_type_node, 1));
   return fold_convert_loc (loc, TREE_TYPE (TREE_TYPE (fndecl)),
                           build_call_expr_loc (loc, fn, 3, dest, src, len));
 }
@@ -8313,11 +9046,13 @@ fold_builtin_stpcpy (location_t loc, tree fndecl, tree dest, tree src)
       && !integer_zerop (len))
     return NULL_TREE;
 
-  fn = implicit_built_in_decls[BUILT_IN_MEMCPY];
+  fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
   if (!fn)
     return NULL_TREE;
 
-  lenp1 = size_binop_loc (loc, PLUS_EXPR, len, ssize_int (1));
+  lenp1 = size_binop_loc (loc, PLUS_EXPR,
+                         fold_convert_loc (loc, size_type_node, len),
+                         build_int_cst (size_type_node, 1));
   /* We use dest twice in building our expression.  Save it from
      multiple expansions.  */
   dest = builtin_save_expr (dest);
@@ -8370,9 +9105,11 @@ fold_builtin_strncpy (location_t loc, tree fndecl, tree dest,
     return NULL_TREE;
 
   /* OK transform into builtin memcpy.  */
-  fn = implicit_built_in_decls[BUILT_IN_MEMCPY];
+  fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
   if (!fn)
     return NULL_TREE;
+
+  len = fold_convert_loc (loc, size_type_node, len);
   return fold_convert_loc (loc, TREE_TYPE (TREE_TYPE (fndecl)),
                           build_call_expr_loc (loc, fn, 3, dest, src, len));
 }
@@ -8406,7 +9143,7 @@ fold_builtin_memchr (location_t loc, tree arg1, tree arg2, tree len, tree type)
          if (target_char_cast (arg2, &c))
            return NULL_TREE;
 
-         r = (char *) memchr (p1, c, tree_low_cst (len, 1));
+         r = (const char *) memchr (p1, c, tree_low_cst (len, 1));
 
          if (r == NULL)
            return build_int_cst (TREE_TYPE (arg1), 0);
@@ -9191,7 +9928,7 @@ fold_builtin_interclass_mathfn (location_t loc, tree fndecl, tree arg)
     CASE_FLT_FN (BUILT_IN_ISINF):
       {
        /* isinf(x) -> isgreater(fabs(x),DBL_MAX).  */
-       tree const isgr_fn = built_in_decls[BUILT_IN_ISGREATER];
+       tree const isgr_fn = builtin_decl_explicit (BUILT_IN_ISGREATER);
        tree const type = TREE_TYPE (arg);
        REAL_VALUE_TYPE r;
        char buf[128];
@@ -9207,7 +9944,7 @@ fold_builtin_interclass_mathfn (location_t loc, tree fndecl, tree arg)
     case BUILT_IN_ISFINITE:
       {
        /* isfinite(x) -> islessequal(fabs(x),DBL_MAX).  */
-       tree const isle_fn = built_in_decls[BUILT_IN_ISLESSEQUAL];
+       tree const isle_fn = builtin_decl_explicit (BUILT_IN_ISLESSEQUAL);
        tree const type = TREE_TYPE (arg);
        REAL_VALUE_TYPE r;
        char buf[128];
@@ -9230,8 +9967,8 @@ fold_builtin_interclass_mathfn (location_t loc, tree fndecl, tree arg)
       {
        /* isnormal(x) -> isgreaterequal(fabs(x),DBL_MIN) &
           islessequal(fabs(x),DBL_MAX).  */
-       tree const isle_fn = built_in_decls[BUILT_IN_ISLESSEQUAL];
-       tree const isge_fn = built_in_decls[BUILT_IN_ISGREATEREQUAL];
+       tree const isle_fn = builtin_decl_explicit (BUILT_IN_ISLESSEQUAL);
+       tree const isge_fn = builtin_decl_explicit (BUILT_IN_ISGREATEREQUAL);
        tree const type = TREE_TYPE (arg);
        REAL_VALUE_TYPE rmax, rmin;
        char buf[128];
@@ -9292,7 +10029,7 @@ fold_builtin_classify (location_t loc, tree fndecl, tree arg, int builtin_index)
           1.  So e.g. "if (isinf_sign(x))" would be folded to just
           "if (isinf(x) ? 1 : 0)" which becomes "if (isinf(x))". */
        tree signbit_fn = mathfn_built_in_1 (TREE_TYPE (arg), BUILT_IN_SIGNBIT, 0);
-       tree isinf_fn = built_in_decls[BUILT_IN_ISINF];
+       tree isinf_fn = builtin_decl_explicit (BUILT_IN_ISINF);
        tree tmp = NULL_TREE;
 
        arg = builtin_save_expr (arg);
@@ -10015,7 +10752,7 @@ fold_builtin_2 (location_t loc, tree fndecl, tree arg0, tree arg1, bool ignore)
     case BUILT_IN_STPCPY:
       if (ignore)
        {
-         tree fn = implicit_built_in_decls[BUILT_IN_STRCPY];
+         tree fn = builtin_decl_implicit (BUILT_IN_STRCPY);
          if (!fn)
            break;
 
@@ -10100,6 +10837,12 @@ fold_builtin_2 (location_t loc, tree fndecl, tree arg0, tree arg1, bool ignore)
       return fold_builtin_fprintf (loc, fndecl, arg0, arg1, NULL_TREE,
                                   ignore, fcode);
 
+    case BUILT_IN_ATOMIC_ALWAYS_LOCK_FREE:
+      return fold_builtin_atomic_always_lock_free (arg0, arg1);
+
+    case BUILT_IN_ATOMIC_IS_LOCK_FREE:
+      return fold_builtin_atomic_is_lock_free (arg0, arg1);
+
     default:
       break;
     }
@@ -10353,7 +11096,7 @@ fold_builtin_varargs (location_t loc, tree fndecl, tree exp,
    been inlined, otherwise e.g. -D_FORTIFY_SOURCE checking
    might not be performed.  */
 
-static bool
+bool
 avoid_folding_inline_builtin (tree fndecl)
 {
   return (DECL_DECLARED_INLINE_P (fndecl)
@@ -10815,7 +11558,7 @@ fold_builtin_strstr (location_t loc, tree s1, tree s2, tree type)
       if (p2[1] != '\0')
        return NULL_TREE;
 
-      fn = implicit_built_in_decls[BUILT_IN_STRCHR];
+      fn = builtin_decl_implicit (BUILT_IN_STRCHR);
       if (!fn)
        return NULL_TREE;
 
@@ -10935,7 +11678,7 @@ fold_builtin_strrchr (location_t loc, tree s1, tree s2, tree type)
       if (! integer_zerop (s2))
        return NULL_TREE;
 
-      fn = implicit_built_in_decls[BUILT_IN_STRCHR];
+      fn = builtin_decl_implicit (BUILT_IN_STRCHR);
       if (!fn)
        return NULL_TREE;
 
@@ -10999,7 +11742,7 @@ fold_builtin_strpbrk (location_t loc, tree s1, tree s2, tree type)
       if (p2[1] != '\0')
        return NULL_TREE;  /* Really call strpbrk.  */
 
-      fn = implicit_built_in_decls[BUILT_IN_STRCHR];
+      fn = builtin_decl_implicit (BUILT_IN_STRCHR);
       if (!fn)
        return NULL_TREE;
 
@@ -11046,8 +11789,8 @@ fold_builtin_strcat (location_t loc ATTRIBUTE_UNUSED, tree dst, tree src)
        {
          /* See if we can store by pieces into (dst + strlen(dst)).  */
          tree newdst, call;
-         tree strlen_fn = implicit_built_in_decls[BUILT_IN_STRLEN];
-         tree strcpy_fn = implicit_built_in_decls[BUILT_IN_STRCPY];
+         tree strlen_fn = builtin_decl_implicit (BUILT_IN_STRLEN);
+         tree strcpy_fn = builtin_decl_implicit (BUILT_IN_STRCPY);
 
          if (!strlen_fn || !strcpy_fn)
            return NULL_TREE;
@@ -11120,7 +11863,7 @@ fold_builtin_strncat (location_t loc, tree dst, tree src, tree len)
       if (TREE_CODE (len) == INTEGER_CST && p
          && compare_tree_int (len, strlen (p)) >= 0)
        {
-         tree fn = implicit_built_in_decls[BUILT_IN_STRCAT];
+         tree fn = builtin_decl_implicit (BUILT_IN_STRCAT);
 
          /* If the replacement _DECL isn't initialized, don't do the
             transformation.  */
@@ -11225,7 +11968,7 @@ fold_builtin_strcspn (location_t loc, tree s1, tree s2)
       /* If the second argument is "", return __builtin_strlen(s1).  */
       if (p2 && *p2 == '\0')
        {
-         tree fn = implicit_built_in_decls[BUILT_IN_STRLEN];
+         tree fn = builtin_decl_implicit (BUILT_IN_STRLEN);
 
          /* If the replacement _DECL isn't initialized, don't do the
             transformation.  */
@@ -11251,10 +11994,12 @@ fold_builtin_fputs (location_t loc, tree arg0, tree arg1,
 {
   /* If we're using an unlocked function, assume the other unlocked
      functions exist explicitly.  */
-  tree const fn_fputc = unlocked ? built_in_decls[BUILT_IN_FPUTC_UNLOCKED]
-    : implicit_built_in_decls[BUILT_IN_FPUTC];
-  tree const fn_fwrite = unlocked ? built_in_decls[BUILT_IN_FWRITE_UNLOCKED]
-    : implicit_built_in_decls[BUILT_IN_FWRITE];
+  tree const fn_fputc = (unlocked
+                        ? builtin_decl_explicit (BUILT_IN_FPUTC_UNLOCKED)
+                        : builtin_decl_implicit (BUILT_IN_FPUTC));
+  tree const fn_fwrite = (unlocked
+                         ? builtin_decl_explicit (BUILT_IN_FWRITE_UNLOCKED)
+                         : builtin_decl_implicit (BUILT_IN_FWRITE));
 
   /* If the return value is used, don't do the transformation.  */
   if (!ignore)
@@ -11448,7 +12193,7 @@ fold_builtin_sprintf (location_t loc, tree dest, tree fmt,
   /* If the format doesn't contain % args or %%, use strcpy.  */
   if (strchr (fmt_str, target_percent) == NULL)
     {
-      tree fn = implicit_built_in_decls[BUILT_IN_STRCPY];
+      tree fn = builtin_decl_implicit (BUILT_IN_STRCPY);
 
       if (!fn)
        return NULL_TREE;
@@ -11468,7 +12213,7 @@ fold_builtin_sprintf (location_t loc, tree dest, tree fmt,
   else if (fmt_str && strcmp (fmt_str, target_percent_s) == 0)
     {
       tree fn;
-      fn = implicit_built_in_decls[BUILT_IN_STRCPY];
+      fn = builtin_decl_implicit (BUILT_IN_STRCPY);
 
       if (!fn)
        return NULL_TREE;
@@ -11490,7 +12235,7 @@ fold_builtin_sprintf (location_t loc, tree dest, tree fmt,
   if (call && retval)
     {
       retval = fold_convert_loc
-       (loc, TREE_TYPE (TREE_TYPE (implicit_built_in_decls[BUILT_IN_SPRINTF])),
+       (loc, TREE_TYPE (TREE_TYPE (builtin_decl_implicit (BUILT_IN_SPRINTF))),
         retval);
       return build2 (COMPOUND_EXPR, TREE_TYPE (retval), call, retval);
     }
@@ -11543,7 +12288,7 @@ fold_builtin_snprintf (location_t loc, tree dest, tree destsize, tree fmt,
   /* If the format doesn't contain % args or %%, use strcpy.  */
   if (strchr (fmt_str, target_percent) == NULL)
     {
-      tree fn = implicit_built_in_decls[BUILT_IN_STRCPY];
+      tree fn = builtin_decl_implicit (BUILT_IN_STRCPY);
       size_t len = strlen (fmt_str);
 
       /* Don't optimize snprintf (buf, 4, "abc", ptr++).  */
@@ -11575,7 +12320,7 @@ fold_builtin_snprintf (location_t loc, tree dest, tree destsize, tree fmt,
   /* If the format is "%s", use strcpy if the result isn't used.  */
   else if (fmt_str && strcmp (fmt_str, target_percent_s) == 0)
     {
-      tree fn = implicit_built_in_decls[BUILT_IN_STRCPY];
+      tree fn = builtin_decl_implicit (BUILT_IN_STRCPY);
       unsigned HOST_WIDE_INT origlen;
 
       /* Don't crash on snprintf (str1, cst, "%s").  */
@@ -11610,7 +12355,7 @@ fold_builtin_snprintf (location_t loc, tree dest, tree destsize, tree fmt,
 
   if (call && retval)
     {
-      tree fn = built_in_decls[BUILT_IN_SNPRINTF];
+      tree fn = builtin_decl_explicit (BUILT_IN_SNPRINTF);
       retval = fold_convert_loc (loc, TREE_TYPE (TREE_TYPE (fn)), retval);
       return build2 (COMPOUND_EXPR, TREE_TYPE (retval), call, retval);
     }
@@ -11698,16 +12443,16 @@ expand_builtin_memory_chk (tree exp, rtx target, enum machine_mode mode,
       switch (fcode)
        {
        case BUILT_IN_MEMCPY_CHK:
-         fn = built_in_decls[BUILT_IN_MEMCPY];
+         fn = builtin_decl_explicit (BUILT_IN_MEMCPY);
          break;
        case BUILT_IN_MEMPCPY_CHK:
-         fn = built_in_decls[BUILT_IN_MEMPCPY];
+         fn = builtin_decl_explicit (BUILT_IN_MEMPCPY);
          break;
        case BUILT_IN_MEMMOVE_CHK:
-         fn = built_in_decls[BUILT_IN_MEMMOVE];
+         fn = builtin_decl_explicit (BUILT_IN_MEMMOVE);
          break;
        case BUILT_IN_MEMSET_CHK:
-         fn = built_in_decls[BUILT_IN_MEMSET];
+         fn = builtin_decl_explicit (BUILT_IN_MEMSET);
          break;
        default:
          break;
@@ -11759,7 +12504,7 @@ expand_builtin_memory_chk (tree exp, rtx target, enum machine_mode mode,
             normal __memcpy_chk.  */
          if (readonly_data_expr (src))
            {
-             tree fn = built_in_decls[BUILT_IN_MEMCPY_CHK];
+             tree fn = builtin_decl_explicit (BUILT_IN_MEMCPY_CHK);
              if (!fn)
                return NULL_RTX;
              fn = build_call_nofold_loc (EXPR_LOCATION (exp), fn, 4,
@@ -11914,11 +12659,11 @@ maybe_emit_free_warning (tree exp)
     return;
 
   if (SSA_VAR_P (arg))
-    warning_at (tree_nonartificial_location (exp),
-               0, "%Kattempt to free a non-heap object %qD", exp, arg);
+    warning_at (tree_nonartificial_location (exp), OPT_Wfree_nonheap_object,
+               "%Kattempt to free a non-heap object %qD", exp, arg);
   else
-    warning_at (tree_nonartificial_location (exp),
-               0, "%Kattempt to free a non-heap object", exp);
+    warning_at (tree_nonartificial_location (exp), OPT_Wfree_nonheap_object,
+               "%Kattempt to free a non-heap object", exp);
 }
 
 /* Fold a call to __builtin_object_size with arguments PTR and OST,
@@ -12023,7 +12768,7 @@ fold_builtin_memory_chk (location_t loc, tree fndecl,
                {
                  /* (void) __mempcpy_chk () can be optimized into
                     (void) __memcpy_chk ().  */
-                 fn = built_in_decls[BUILT_IN_MEMCPY_CHK];
+                 fn = builtin_decl_explicit (BUILT_IN_MEMCPY_CHK);
                  if (!fn)
                    return NULL_TREE;
 
@@ -12045,16 +12790,16 @@ fold_builtin_memory_chk (location_t loc, tree fndecl,
   switch (fcode)
     {
     case BUILT_IN_MEMCPY_CHK:
-      fn = built_in_decls[BUILT_IN_MEMCPY];
+      fn = builtin_decl_explicit (BUILT_IN_MEMCPY);
       break;
     case BUILT_IN_MEMPCPY_CHK:
-      fn = built_in_decls[BUILT_IN_MEMPCPY];
+      fn = builtin_decl_explicit (BUILT_IN_MEMPCPY);
       break;
     case BUILT_IN_MEMMOVE_CHK:
-      fn = built_in_decls[BUILT_IN_MEMMOVE];
+      fn = builtin_decl_explicit (BUILT_IN_MEMMOVE);
       break;
     case BUILT_IN_MEMSET_CHK:
-      fn = built_in_decls[BUILT_IN_MEMSET];
+      fn = builtin_decl_explicit (BUILT_IN_MEMSET);
       break;
     default:
       break;
@@ -12109,7 +12854,7 @@ fold_builtin_stxcpy_chk (location_t loc, tree fndecl, tree dest,
 
                  /* If return value of __stpcpy_chk is ignored,
                     optimize into __strcpy_chk.  */
-                 fn = built_in_decls[BUILT_IN_STRCPY_CHK];
+                 fn = builtin_decl_explicit (BUILT_IN_STRCPY_CHK);
                  if (!fn)
                    return NULL_TREE;
 
@@ -12121,11 +12866,13 @@ fold_builtin_stxcpy_chk (location_t loc, tree fndecl, tree dest,
 
              /* If c_strlen returned something, but not a constant,
                 transform __strcpy_chk into __memcpy_chk.  */
-             fn = built_in_decls[BUILT_IN_MEMCPY_CHK];
+             fn = builtin_decl_explicit (BUILT_IN_MEMCPY_CHK);
              if (!fn)
                return NULL_TREE;
 
-             len = size_binop_loc (loc, PLUS_EXPR, len, ssize_int (1));
+             len = fold_convert_loc (loc, size_type_node, len);
+             len = size_binop_loc (loc, PLUS_EXPR, len,
+                                   build_int_cst (size_type_node, 1));
              return fold_convert_loc (loc, TREE_TYPE (TREE_TYPE (fndecl)),
                                       build_call_expr_loc (loc, fn, 4,
                                                        dest, src, len, size));
@@ -12139,8 +12886,8 @@ fold_builtin_stxcpy_chk (location_t loc, tree fndecl, tree dest,
     }
 
   /* If __builtin_st{r,p}cpy_chk is used, assume st{r,p}cpy is available.  */
-  fn = built_in_decls[fcode == BUILT_IN_STPCPY_CHK
-                     ? BUILT_IN_STPCPY : BUILT_IN_STRCPY];
+  fn = builtin_decl_explicit (fcode == BUILT_IN_STPCPY_CHK
+                             ? BUILT_IN_STPCPY : BUILT_IN_STRCPY);
   if (!fn)
     return NULL_TREE;
 
@@ -12184,7 +12931,7 @@ fold_builtin_strncpy_chk (location_t loc, tree dest, tree src,
     }
 
   /* If __builtin_strncpy_chk is used, assume strncpy is available.  */
-  fn = built_in_decls[BUILT_IN_STRNCPY];
+  fn = builtin_decl_explicit (BUILT_IN_STRNCPY);
   if (!fn)
     return NULL_TREE;
 
@@ -12215,7 +12962,7 @@ fold_builtin_strcat_chk (location_t loc, tree fndecl, tree dest,
     return NULL_TREE;
 
   /* If __builtin_strcat_chk is used, assume strcat is available.  */
-  fn = built_in_decls[BUILT_IN_STRCAT];
+  fn = builtin_decl_explicit (BUILT_IN_STRCAT);
   if (!fn)
     return NULL_TREE;
 
@@ -12257,7 +13004,7 @@ fold_builtin_strncat_chk (location_t loc, tree fndecl,
          && ! tree_int_cst_lt (len, src_len))
        {
          /* If LEN >= strlen (SRC), optimize into __strcat_chk.  */
-         fn = built_in_decls[BUILT_IN_STRCAT_CHK];
+         fn = builtin_decl_explicit (BUILT_IN_STRCAT_CHK);
          if (!fn)
            return NULL_TREE;
 
@@ -12267,7 +13014,7 @@ fold_builtin_strncat_chk (location_t loc, tree fndecl,
     }
 
   /* If __builtin_strncat_chk is used, assume strncat is available.  */
-  fn = built_in_decls[BUILT_IN_STRNCAT];
+  fn = builtin_decl_explicit (BUILT_IN_STRNCAT);
   if (!fn)
     return NULL_TREE;
 
@@ -12358,8 +13105,8 @@ fold_builtin_sprintf_chk_1 (location_t loc, int nargs, tree *args,
     }
 
   /* If __builtin_{,v}sprintf_chk is used, assume {,v}sprintf is available.  */
-  fn = built_in_decls[fcode == BUILT_IN_VSPRINTF_CHK
-                     ? BUILT_IN_VSPRINTF : BUILT_IN_SPRINTF];
+  fn = builtin_decl_explicit (fcode == BUILT_IN_VSPRINTF_CHK
+                             ? BUILT_IN_VSPRINTF : BUILT_IN_SPRINTF);
   if (!fn)
     return NULL_TREE;
 
@@ -12447,8 +13194,8 @@ fold_builtin_snprintf_chk_1 (location_t loc, int nargs, tree *args,
 
   /* If __builtin_{,v}snprintf_chk is used, assume {,v}snprintf is
      available.  */
-  fn = built_in_decls[fcode == BUILT_IN_VSNPRINTF_CHK
-                     ? BUILT_IN_VSNPRINTF : BUILT_IN_SNPRINTF];
+  fn = builtin_decl_explicit (fcode == BUILT_IN_VSNPRINTF_CHK
+                             ? BUILT_IN_VSNPRINTF : BUILT_IN_SNPRINTF);
   if (!fn)
     return NULL_TREE;
 
@@ -12502,13 +13249,13 @@ fold_builtin_printf (location_t loc, tree fndecl, tree fmt,
     {
       /* If we're using an unlocked function, assume the other
         unlocked functions exist explicitly.  */
-      fn_putchar = built_in_decls[BUILT_IN_PUTCHAR_UNLOCKED];
-      fn_puts = built_in_decls[BUILT_IN_PUTS_UNLOCKED];
+      fn_putchar = builtin_decl_explicit (BUILT_IN_PUTCHAR_UNLOCKED);
+      fn_puts = builtin_decl_explicit (BUILT_IN_PUTS_UNLOCKED);
     }
   else
     {
-      fn_putchar = implicit_built_in_decls[BUILT_IN_PUTCHAR];
-      fn_puts = implicit_built_in_decls[BUILT_IN_PUTS];
+      fn_putchar = builtin_decl_implicit (BUILT_IN_PUTCHAR);
+      fn_puts = builtin_decl_implicit (BUILT_IN_PUTS);
     }
 
   if (!init_target_chars ())
@@ -12653,13 +13400,13 @@ fold_builtin_fprintf (location_t loc, tree fndecl, tree fp,
     {
       /* If we're using an unlocked function, assume the other
         unlocked functions exist explicitly.  */
-      fn_fputc = built_in_decls[BUILT_IN_FPUTC_UNLOCKED];
-      fn_fputs = built_in_decls[BUILT_IN_FPUTS_UNLOCKED];
+      fn_fputc = builtin_decl_explicit (BUILT_IN_FPUTC_UNLOCKED);
+      fn_fputs = builtin_decl_explicit (BUILT_IN_FPUTS_UNLOCKED);
     }
   else
     {
-      fn_fputc = implicit_built_in_decls[BUILT_IN_FPUTC];
-      fn_fputs = implicit_built_in_decls[BUILT_IN_FPUTS];
+      fn_fputc = builtin_decl_implicit (BUILT_IN_FPUTC);
+      fn_fputs = builtin_decl_implicit (BUILT_IN_FPUTS);
     }
 
   if (!init_target_chars ())
@@ -13461,7 +14208,7 @@ fold_call_stmt (gimple stmt, bool ignore)
   return NULL_TREE;
 }
 
-/* Look up the function in built_in_decls that corresponds to DECL
+/* Look up the function in builtin_decl that corresponds to DECL
    and set ASMSPEC as its user assembler name.  DECL must be a
    function decl that declares a builtin.  */
 
@@ -13473,7 +14220,7 @@ set_builtin_user_assembler_name (tree decl, const char *asmspec)
              && DECL_BUILT_IN_CLASS (decl) == BUILT_IN_NORMAL
              && asmspec != 0);
 
-  builtin = built_in_decls [DECL_FUNCTION_CODE (decl)];
+  builtin = builtin_decl_explicit (DECL_FUNCTION_CODE (decl));
   set_user_assembler_name (builtin, asmspec);
   switch (DECL_FUNCTION_CODE (decl))
     {
@@ -13559,6 +14306,7 @@ is_inexpensive_builtin (tree decl)
       {
       case BUILT_IN_ABS:
       case BUILT_IN_ALLOCA:
+      case BUILT_IN_ALLOCA_WITH_ALIGN:
       case BUILT_IN_BSWAP32:
       case BUILT_IN_BSWAP64:
       case BUILT_IN_CLZ: