+ else
+ /* Otherwise there is no atomic load, leave the library call. */
+ return NULL_RTX;
+ }
+
+ /* Otherwise assume loads are atomic, and emit the proper barriers. */
+ if (!target || target == const0_rtx)
+ target = gen_reg_rtx (mode);
+
+ /* Emit the appropriate barrier before the load. */
+ expand_mem_thread_fence (model);
+
+ emit_move_insn (target, mem);
+
+ /* For SEQ_CST, also emit a barrier after the load. */
+ if (model == MEMMODEL_SEQ_CST)
+ expand_mem_thread_fence (model);
+
+ return target;
+}
+
+/* This function expands the atomic store operation:
+ Atomically store VAL in MEM.
+ MEMMODEL is the memory model variant to use.
+ USE_RELEASE is true if __sync_lock_release can be used as a fall back.
+ function returns const0_rtx if a pattern was emitted. */
+
+rtx
+expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
+{
+ enum machine_mode mode = GET_MODE (mem);
+ enum insn_code icode;
+ struct expand_operand ops[3];
+
+ /* If the target supports the store directly, great. */
+ icode = direct_optab_handler (atomic_store_optab, mode);
+ if (icode != CODE_FOR_nothing)
+ {
+ create_fixed_operand (&ops[0], mem);
+ create_input_operand (&ops[1], val, mode);
+ create_integer_operand (&ops[2], model);
+ if (maybe_expand_insn (icode, 3, ops))
+ return const0_rtx;
+ }
+
+ /* If using __sync_lock_release is a viable alternative, try it. */
+ if (use_release)
+ {
+ icode = direct_optab_handler (sync_lock_release_optab, mode);
+ if (icode != CODE_FOR_nothing)
+ {
+ create_fixed_operand (&ops[0], mem);
+ create_input_operand (&ops[1], const0_rtx, mode);
+ if (maybe_expand_insn (icode, 2, ops))
+ {
+ /* lock_release is only a release barrier. */
+ if (model == MEMMODEL_SEQ_CST)
+ expand_mem_thread_fence (model);
+ return const0_rtx;
+ }
+ }
+ }
+
+ /* If the size of the object is greater than word size on this target,
+ a default store will not be atomic, Try a mem_exchange and throw away
+ the result. If that doesn't work, don't do anything. */
+ if (GET_MODE_PRECISION(mode) > BITS_PER_WORD)
+ {
+ rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
+ if (!target)
+ target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem, val);
+ if (target)
+ return const0_rtx;
+ else
+ return NULL_RTX;
+ }
+
+ /* If there is no mem_store, default to a move with barriers */
+ if (model == MEMMODEL_SEQ_CST || model == MEMMODEL_RELEASE)
+ expand_mem_thread_fence (model);
+
+ emit_move_insn (mem, val);
+
+ /* For SEQ_CST, also emit a barrier after the load. */
+ if (model == MEMMODEL_SEQ_CST)
+ expand_mem_thread_fence (model);
+
+ return const0_rtx;
+}
+
+
+/* Structure containing the pointers and values required to process the
+ various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
+
+struct atomic_op_functions
+{
+ direct_optab mem_fetch_before;
+ direct_optab mem_fetch_after;
+ direct_optab mem_no_result;
+ optab fetch_before;
+ optab fetch_after;
+ direct_optab no_result;
+ enum rtx_code reverse_code;
+};
+
+
+/* Fill in structure pointed to by OP with the various optab entries for an
+ operation of type CODE. */
+
+static void
+get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code)
+{
+ gcc_assert (op!= NULL);
+
+ /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
+ in the source code during compilation, and the optab entries are not
+ computable until runtime. Fill in the values at runtime. */
+ switch (code)
+ {
+ case PLUS:
+ op->mem_fetch_before = atomic_fetch_add_optab;
+ op->mem_fetch_after = atomic_add_fetch_optab;
+ op->mem_no_result = atomic_add_optab;
+ op->fetch_before = sync_old_add_optab;
+ op->fetch_after = sync_new_add_optab;
+ op->no_result = sync_add_optab;
+ op->reverse_code = MINUS;
+ break;
+ case MINUS:
+ op->mem_fetch_before = atomic_fetch_sub_optab;
+ op->mem_fetch_after = atomic_sub_fetch_optab;
+ op->mem_no_result = atomic_sub_optab;
+ op->fetch_before = sync_old_sub_optab;
+ op->fetch_after = sync_new_sub_optab;
+ op->no_result = sync_sub_optab;
+ op->reverse_code = PLUS;
+ break;
+ case XOR:
+ op->mem_fetch_before = atomic_fetch_xor_optab;
+ op->mem_fetch_after = atomic_xor_fetch_optab;
+ op->mem_no_result = atomic_xor_optab;
+ op->fetch_before = sync_old_xor_optab;
+ op->fetch_after = sync_new_xor_optab;
+ op->no_result = sync_xor_optab;
+ op->reverse_code = XOR;
+ break;
+ case AND:
+ op->mem_fetch_before = atomic_fetch_and_optab;
+ op->mem_fetch_after = atomic_and_fetch_optab;
+ op->mem_no_result = atomic_and_optab;
+ op->fetch_before = sync_old_and_optab;
+ op->fetch_after = sync_new_and_optab;
+ op->no_result = sync_and_optab;
+ op->reverse_code = UNKNOWN;
+ break;
+ case IOR:
+ op->mem_fetch_before = atomic_fetch_or_optab;
+ op->mem_fetch_after = atomic_or_fetch_optab;
+ op->mem_no_result = atomic_or_optab;
+ op->fetch_before = sync_old_ior_optab;
+ op->fetch_after = sync_new_ior_optab;
+ op->no_result = sync_ior_optab;
+ op->reverse_code = UNKNOWN;
+ break;
+ case NOT:
+ op->mem_fetch_before = atomic_fetch_nand_optab;
+ op->mem_fetch_after = atomic_nand_fetch_optab;
+ op->mem_no_result = atomic_nand_optab;
+ op->fetch_before = sync_old_nand_optab;
+ op->fetch_after = sync_new_nand_optab;
+ op->no_result = sync_nand_optab;
+ op->reverse_code = UNKNOWN;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+}
+
+/* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
+ using memory order MODEL. If AFTER is true the operation needs to return
+ the value of *MEM after the operation, otherwise the previous value.
+ TARGET is an optional place to place the result. The result is unused if
+ it is const0_rtx.
+ Return the result if there is a better sequence, otherwise NULL_RTX. */
+
+static rtx
+maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
+ enum memmodel model, bool after)
+{
+ /* If the value is prefetched, or not used, it may be possible to replace
+ the sequence with a native exchange operation. */
+ if (!after || target == const0_rtx)
+ {
+ /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
+ if (code == AND && val == const0_rtx)
+ {
+ if (target == const0_rtx)
+ target = gen_reg_rtx (GET_MODE (mem));
+ return maybe_emit_atomic_exchange (target, mem, val, model);
+ }
+
+ /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
+ if (code == IOR && val == constm1_rtx)
+ {
+ if (target == const0_rtx)
+ target = gen_reg_rtx (GET_MODE (mem));
+ return maybe_emit_atomic_exchange (target, mem, val, model);
+ }
+ }
+
+ return NULL_RTX;
+}
+
+/* Try to emit an instruction for a specific operation varaition.
+ OPTAB contains the OP functions.
+ TARGET is an optional place to return the result. const0_rtx means unused.
+ MEM is the memory location to operate on.
+ VAL is the value to use in the operation.
+ USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
+ MODEL is the memory model, if used.
+ AFTER is true if the returned result is the value after the operation. */
+
+static rtx
+maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem,
+ rtx val, bool use_memmodel, enum memmodel model, bool after)
+{
+ enum machine_mode mode = GET_MODE (mem);
+ struct expand_operand ops[4];
+ enum insn_code icode;
+ int op_counter = 0;
+ int num_ops;
+
+ /* Check to see if there is a result returned. */
+ if (target == const0_rtx)
+ {
+ if (use_memmodel)
+ {
+ icode = direct_optab_handler (optab->mem_no_result, mode);
+ create_integer_operand (&ops[2], model);
+ num_ops = 3;
+ }
+ else
+ {
+ icode = direct_optab_handler (optab->no_result, mode);
+ num_ops = 2;
+ }
+ }
+ /* Otherwise, we need to generate a result. */
+ else
+ {
+ if (use_memmodel)
+ {
+ icode = direct_optab_handler (after ? optab->mem_fetch_after
+ : optab->mem_fetch_before, mode);
+ create_integer_operand (&ops[3], model);
+ num_ops = 4;
+ }
+ else
+ {
+ icode = optab_handler (after ? optab->fetch_after
+ : optab->fetch_before, mode);
+ num_ops = 3;
+ }
+ create_output_operand (&ops[op_counter++], target, mode);
+ }
+ if (icode == CODE_FOR_nothing)
+ return NULL_RTX;
+
+ create_fixed_operand (&ops[op_counter++], mem);
+ /* VAL may have been promoted to a wider mode. Shrink it if so. */
+ create_convert_operand_to (&ops[op_counter++], val, mode, true);
+
+ if (maybe_expand_insn (icode, num_ops, ops))
+ return (target == const0_rtx ? const0_rtx : ops[0].value);
+
+ return NULL_RTX;
+}
+
+
+/* This function expands an atomic fetch_OP or OP_fetch operation:
+ TARGET is an option place to stick the return value. const0_rtx indicates
+ the result is unused.
+ atomically fetch MEM, perform the operation with VAL and return it to MEM.
+ CODE is the operation being performed (OP)
+ MEMMODEL is the memory model variant to use.
+ AFTER is true to return the result of the operation (OP_fetch).
+ AFTER is false to return the value before the operation (fetch_OP). */
+rtx
+expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
+ enum memmodel model, bool after)
+{
+ enum machine_mode mode = GET_MODE (mem);
+ struct atomic_op_functions optab;
+ rtx result;
+ bool unused_result = (target == const0_rtx);
+
+ get_atomic_op_for_code (&optab, code);
+
+ /* Check to see if there are any better instructions. */
+ result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
+ if (result)
+ return result;
+
+ /* Check for the case where the result isn't used and try those patterns. */
+ if (unused_result)
+ {
+ /* Try the memory model variant first. */
+ result = maybe_emit_op (&optab, target, mem, val, true, model, true);
+ if (result)
+ return result;
+
+ /* Next try the old style withuot a memory model. */
+ result = maybe_emit_op (&optab, target, mem, val, false, model, true);
+ if (result)
+ return result;
+
+ /* There is no no-result pattern, so try patterns with a result. */
+ target = NULL_RTX;
+ }
+
+ /* Try the __atomic version. */
+ result = maybe_emit_op (&optab, target, mem, val, true, model, after);
+ if (result)
+ return result;
+
+ /* Try the older __sync version. */
+ result = maybe_emit_op (&optab, target, mem, val, false, model, after);
+ if (result)
+ return result;
+
+ /* If the fetch value can be calculated from the other variation of fetch,
+ try that operation. */
+ if (after || unused_result || optab.reverse_code != UNKNOWN)
+ {
+ /* Try the __atomic version, then the older __sync version. */
+ result = maybe_emit_op (&optab, target, mem, val, true, model, !after);
+ if (!result)
+ result = maybe_emit_op (&optab, target, mem, val, false, model, !after);
+
+ if (result)
+ {
+ /* If the result isn't used, no need to do compensation code. */
+ if (unused_result)
+ return result;
+
+ /* Issue compensation code. Fetch_after == fetch_before OP val.
+ Fetch_before == after REVERSE_OP val. */
+ if (!after)
+ code = optab.reverse_code;
+ if (code == NOT)
+ {
+ result = expand_simple_binop (mode, AND, result, val, NULL_RTX,
+ true, OPTAB_LIB_WIDEN);
+ result = expand_simple_unop (mode, NOT, result, target, true);
+ }
+ else
+ result = expand_simple_binop (mode, code, result, val, target,
+ true, OPTAB_LIB_WIDEN);
+ return result;
+ }
+ }
+
+ /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
+ if (!can_compare_and_swap_p (mode, false))
+ {
+ rtx libfunc;
+ bool fixup = false;
+
+ libfunc = optab_libfunc (after ? optab.fetch_after
+ : optab.fetch_before, mode);
+ if (libfunc == NULL
+ && (after || unused_result || optab.reverse_code != UNKNOWN))
+ {
+ fixup = true;
+ if (!after)
+ code = optab.reverse_code;
+ libfunc = optab_libfunc (after ? optab.fetch_before
+ : optab.fetch_after, mode);
+ }
+ if (libfunc != NULL)
+ {
+ rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
+ result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
+ 2, addr, ptr_mode, val, mode);
+
+ if (!unused_result && fixup)
+ result = expand_simple_binop (mode, code, result, val, target,
+ true, OPTAB_LIB_WIDEN);
+ return result;
+ }
+ }
+
+ /* If nothing else has succeeded, default to a compare and swap loop. */
+ if (can_compare_and_swap_p (mode, true))
+ {
+ rtx insn;
+ rtx t0 = gen_reg_rtx (mode), t1;
+
+ start_sequence ();
+
+ /* If the result is used, get a register for it. */
+ if (!unused_result)
+ {
+ if (!target || !register_operand (target, mode))
+ target = gen_reg_rtx (mode);
+ /* If fetch_before, copy the value now. */
+ if (!after)
+ emit_move_insn (target, t0);
+ }
+ else
+ target = const0_rtx;
+
+ t1 = t0;
+ if (code == NOT)
+ {
+ t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX,
+ true, OPTAB_LIB_WIDEN);
+ t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true);
+ }
+ else
+ t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true,
+ OPTAB_LIB_WIDEN);
+
+ /* For after, copy the value now. */
+ if (!unused_result && after)
+ emit_move_insn (target, t1);
+ insn = get_insns ();
+ end_sequence ();
+
+ if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn))
+ return target;