{
rtx temp = emit_move_insn (target, xtarget);
- set_unique_reg_note (temp,
- REG_EQUAL,
- gen_rtx_fmt_ee (binoptab->code, mode,
- copy_rtx (xop0),
- copy_rtx (xop1)));
+ set_dst_reg_note (temp, REG_EQUAL,
+ gen_rtx_fmt_ee (binoptab->code, mode,
+ copy_rtx (xop0),
+ copy_rtx (xop1)),
+ target);
}
else
target = xtarget;
if (optab_handler (mov_optab, mode) != CODE_FOR_nothing)
{
temp = emit_move_insn (target ? target : product, product);
- set_unique_reg_note (temp,
- REG_EQUAL,
- gen_rtx_fmt_ee (MULT, mode,
- copy_rtx (op0),
- copy_rtx (op1)));
+ set_dst_reg_note (temp,
+ REG_EQUAL,
+ gen_rtx_fmt_ee (MULT, mode,
+ copy_rtx (op0),
+ copy_rtx (op1)),
+ target ? target : product);
}
return product;
}
gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
target = lowpart_subreg_maybe_copy (mode, temp, imode);
- set_unique_reg_note (get_last_insn (), REG_EQUAL,
- gen_rtx_fmt_e (code, mode, copy_rtx (op0)));
+ set_dst_reg_note (get_last_insn (), REG_EQUAL,
+ gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
+ target);
}
return target;
}
last = emit_move_insn (target, result);
- if (optab_handler (mov_optab, GET_MODE (target)) != CODE_FOR_nothing)
- set_unique_reg_note (last, REG_EQUAL, copy_rtx (equiv));
+ set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
if (final_dest != target)
emit_move_insn (final_dest, target);
{
/* Make a place for a REG_NOTE and add it. */
insn = emit_move_insn (to, to);
- set_unique_reg_note (insn,
- REG_EQUAL,
- gen_rtx_fmt_e (UNSIGNED_FIX,
- GET_MODE (to),
- copy_rtx (from)));
+ set_dst_reg_note (insn, REG_EQUAL,
+ gen_rtx_fmt_e (UNSIGNED_FIX, GET_MODE (to),
+ copy_rtx (from)),
+ to);
}
return;
buf[len + 2] = '\0';
mode = QImode;
- for (i = 1; i < max; i *= 2)
+ for (i = 1; i <= max; i *= 2)
{
buf[len + 1] = '0' + i;
set_optab_libfunc (tab, mode, buf);
init_sync_libfuncs_1 (sync_old_add_optab, "__sync_fetch_and_add", max);
init_sync_libfuncs_1 (sync_old_sub_optab, "__sync_fetch_and_sub", max);
- init_sync_libfuncs_1 (sync_old_ior_optab, "__sync_fetch_and_ior", max);
+ init_sync_libfuncs_1 (sync_old_ior_optab, "__sync_fetch_and_or", max);
init_sync_libfuncs_1 (sync_old_and_optab, "__sync_fetch_and_and", max);
init_sync_libfuncs_1 (sync_old_xor_optab, "__sync_fetch_and_xor", max);
init_sync_libfuncs_1 (sync_old_nand_optab, "__sync_fetch_and_nand", max);
init_sync_libfuncs_1 (sync_new_add_optab, "__sync_add_and_fetch", max);
init_sync_libfuncs_1 (sync_new_sub_optab, "__sync_sub_and_fetch", max);
- init_sync_libfuncs_1 (sync_new_ior_optab, "__sync_ior_and_fetch", max);
+ init_sync_libfuncs_1 (sync_new_ior_optab, "__sync_or_and_fetch", max);
init_sync_libfuncs_1 (sync_new_and_optab, "__sync_and_and_fetch", max);
init_sync_libfuncs_1 (sync_new_xor_optab, "__sync_xor_and_fetch", max);
init_sync_libfuncs_1 (sync_new_nand_optab, "__sync_nand_and_fetch", max);
break;
case VEC_INTERLEAVE_HIGH_EXPR:
- alt = nelt / 2;
- /* FALLTHRU */
case VEC_INTERLEAVE_LOW_EXPR:
+ if ((BYTES_BIG_ENDIAN != 0) ^ (code == VEC_INTERLEAVE_HIGH_EXPR))
+ alt = nelt / 2;
for (i = 0; i < nelt / 2; ++i)
{
data[i * 2] = i + alt;
}
/* If the input is a constant, expand it specially. */
- if (CONSTANT_P (sel))
+ gcc_assert (GET_MODE_CLASS (GET_MODE (sel)) == MODE_VECTOR_INT);
+ if (GET_CODE (sel) == CONST_VECTOR)
{
icode = direct_optab_handler (vec_perm_const_optab, mode);
if (icode != CODE_FOR_nothing)
{
unsigned int j, this_e;
- this_e = INTVAL (XVECEXP (sel, 0, i));
+ this_e = INTVAL (CONST_VECTOR_ELT (sel, i));
this_e &= 2 * e - 1;
this_e *= u;
return false;
}
+/* Return true if an atomic exchange can be performed. */
+
+bool
+can_atomic_exchange_p (enum machine_mode mode, bool allow_libcall)
+{
+ enum insn_code icode;
+
+ /* Check for __atomic_exchange. */
+ icode = direct_optab_handler (atomic_exchange_optab, mode);
+ if (icode != CODE_FOR_nothing)
+ return true;
+
+ /* Don't check __sync_test_and_set, as on some platforms that
+ has reduced functionality. Targets that really do support
+ a proper exchange should simply be updated to the __atomics. */
+
+ return can_compare_and_swap_p (mode, allow_libcall);
+}
+
+
/* Helper function to find the MODE_CC set in a sync_compare_and_swap
pattern. */
}
-/* This function expands the atomic exchange operation:
- atomically store VAL in MEM and return the previous value in MEM.
-
- MEMMODEL is the memory model variant to use.
- TARGET is an optional place to stick the return value.
- USE_TEST_AND_SET indicates whether __sync_lock_test_and_set should be used
- as a fall back if the atomic_exchange pattern does not exist. */
-
-rtx
-expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model,
- bool use_test_and_set)
+/* This function tries to emit an atomic_exchange intruction. VAL is written
+ to *MEM using memory model MODEL. The previous contents of *MEM are returned,
+ using TARGET if possible. */
+
+static rtx
+maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
{
enum machine_mode mode = GET_MODE (mem);
enum insn_code icode;
return ops[0].value;
}
- /* Legacy sync_lock_test_and_set works the same, but is only defined as an
- acquire barrier. If the pattern exists, and the memory model is stronger
- than acquire, add a release barrier before the instruction.
- The barrier is not needed if sync_lock_test_and_set doesn't exist since
- it will expand into a compare-and-swap loop.
+ return NULL_RTX;
+}
+
+/* This function tries to implement an atomic exchange operation using
+ __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
+ The previous contents of *MEM are returned, using TARGET if possible.
+ Since this instructionn is an acquire barrier only, stronger memory
+ models may require additional barriers to be emitted. */
+
+static rtx
+maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val,
+ enum memmodel model)
+{
+ enum machine_mode mode = GET_MODE (mem);
+ enum insn_code icode;
+ rtx last_insn = get_last_insn ();
+
+ icode = optab_handler (sync_lock_test_and_set_optab, mode);
- Some targets have non-compliant test_and_sets, so it would be incorrect
- to emit a test_and_set in place of an __atomic_exchange. The test_and_set
- builtin shares this expander since exchange can always replace the
- test_and_set. */
+ /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
+ exists, and the memory model is stronger than acquire, add a release
+ barrier before the instruction. */
- if (use_test_and_set)
+ if (model == MEMMODEL_SEQ_CST
+ || model == MEMMODEL_RELEASE
+ || model == MEMMODEL_ACQ_REL)
+ expand_mem_thread_fence (model);
+
+ if (icode != CODE_FOR_nothing)
{
- icode = optab_handler (sync_lock_test_and_set_optab, mode);
+ struct expand_operand ops[3];
+ create_output_operand (&ops[0], target, mode);
+ create_fixed_operand (&ops[1], mem);
+ /* VAL may have been promoted to a wider mode. Shrink it if so. */
+ create_convert_operand_to (&ops[2], val, mode, true);
+ if (maybe_expand_insn (icode, 3, ops))
+ return ops[0].value;
+ }
- if (icode != CODE_FOR_nothing)
+ /* If an external test-and-set libcall is provided, use that instead of
+ any external compare-and-swap that we might get from the compare-and-
+ swap-loop expansion later. */
+ if (!can_compare_and_swap_p (mode, false))
+ {
+ rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
+ if (libfunc != NULL)
{
- struct expand_operand ops[3];
- rtx last_insn = get_last_insn ();
-
- if (model == MEMMODEL_SEQ_CST
- || model == MEMMODEL_RELEASE
- || model == MEMMODEL_ACQ_REL)
- expand_builtin_mem_thread_fence (model);
-
- create_output_operand (&ops[0], target, mode);
- create_fixed_operand (&ops[1], mem);
- /* VAL may have been promoted to a wider mode. Shrink it if so. */
- create_convert_operand_to (&ops[2], val, mode, true);
- if (maybe_expand_insn (icode, 3, ops))
- return ops[0].value;
-
- delete_insns_since (last_insn);
+ rtx addr;
+
+ addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
+ return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
+ mode, 2, addr, ptr_mode,
+ val, mode);
}
+ }
- /* If an external test-and-set libcall is provided, use that instead of
- any external compare-and-swap that we might get from the compare-and-
- swap-loop expansion below. */
- if (!can_compare_and_swap_p (mode, false))
- {
- rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
- if (libfunc != NULL)
- {
- rtx addr;
+ /* If the test_and_set can't be emitted, eliminate any barrier that might
+ have been emitted. */
+ delete_insns_since (last_insn);
+ return NULL_RTX;
+}
- if (model == MEMMODEL_SEQ_CST
- || model == MEMMODEL_RELEASE
- || model == MEMMODEL_ACQ_REL)
- expand_builtin_mem_thread_fence (model);
+/* This function tries to implement an atomic exchange operation using a
+ compare_and_swap loop. VAL is written to *MEM. The previous contents of
+ *MEM are returned, using TARGET if possible. No memory model is required
+ since a compare_and_swap loop is seq-cst. */
- addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
- return emit_library_call_value (libfunc, target, LCT_NORMAL,
- mode, 2, addr, ptr_mode,
- val, mode);
- }
- }
- }
+static rtx
+maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val)
+{
+ enum machine_mode mode = GET_MODE (mem);
- /* Otherwise, use a compare-and-swap loop for the exchange. */
if (can_compare_and_swap_p (mode, true))
{
if (!target || !register_operand (target, mode))
return NULL_RTX;
}
+#ifndef HAVE_atomic_test_and_set
+#define HAVE_atomic_test_and_set 0
+#define gen_atomic_test_and_set(x,y,z) (gcc_unreachable (), NULL_RTX)
+#endif
+
+/* This function expands the legacy _sync_lock test_and_set operation which is
+ generally an atomic exchange. Some limited targets only allow the
+ constant 1 to be stored. This is an ACQUIRE operation.
+
+ TARGET is an optional place to stick the return value.
+ MEM is where VAL is stored. */
+
+rtx
+expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val)
+{
+ rtx ret;
+
+ /* Try an atomic_exchange first. */
+ ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_ACQUIRE);
+
+ if (!ret)
+ ret = maybe_emit_sync_lock_test_and_set (target, mem, val,
+ MEMMODEL_ACQUIRE);
+ if (!ret)
+ ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
+
+ /* If there are no other options, try atomic_test_and_set if the value
+ being stored is 1. */
+ if (!ret && val == const1_rtx && HAVE_atomic_test_and_set)
+ {
+ ret = gen_atomic_test_and_set (target, mem, GEN_INT (MEMMODEL_ACQUIRE));
+ emit_insn (ret);
+ }
+
+ return ret;
+}
+
+/* This function expands the atomic test_and_set operation:
+ atomically store a boolean TRUE into MEM and return the previous value.
+
+ MEMMODEL is the memory model variant to use.
+ TARGET is an optional place to stick the return value. */
+
+rtx
+expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
+{
+ enum machine_mode mode = GET_MODE (mem);
+ rtx ret = NULL_RTX;
+
+ if (target == NULL_RTX)
+ target = gen_reg_rtx (mode);
+
+ if (HAVE_atomic_test_and_set)
+ {
+ ret = gen_atomic_test_and_set (target, mem, GEN_INT (MEMMODEL_ACQUIRE));
+ emit_insn (ret);
+ return ret;
+ }
+
+ /* If there is no test and set, try exchange, then a compare_and_swap loop,
+ then __sync_test_and_set. */
+ ret = maybe_emit_atomic_exchange (target, mem, const1_rtx, model);
+
+ if (!ret)
+ ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, const1_rtx);
+
+ if (!ret)
+ ret = maybe_emit_sync_lock_test_and_set (target, mem, const1_rtx, model);
+
+ if (ret)
+ return ret;
+
+ /* Failing all else, assume a single threaded environment and simply perform
+ the operation. */
+ emit_move_insn (target, mem);
+ emit_move_insn (mem, const1_rtx);
+ return target;
+}
+
+/* This function expands the atomic exchange operation:
+ atomically store VAL in MEM and return the previous value in MEM.
+
+ MEMMODEL is the memory model variant to use.
+ TARGET is an optional place to stick the return value. */
+
+rtx
+expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
+{
+ rtx ret;
+
+ ret = maybe_emit_atomic_exchange (target, mem, val, model);
+
+ /* Next try a compare-and-swap loop for the exchange. */
+ if (!ret)
+ ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
+
+ return ret;
+}
+
/* This function expands the atomic compare exchange operation:
*PTARGET_BOOL is an optional place to store the boolean success/failure.
if (libfunc != NULL)
{
rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
- target_oval = emit_library_call_value (libfunc, target_oval, LCT_NORMAL,
+ target_oval = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
mode, 3, addr, ptr_mode,
expected, mode, desired, mode);
return true;
}
+/* Generate asm volatile("" : : : "memory") as the memory barrier. */
+
+static void
+expand_asm_memory_barrier (void)
+{
+ rtx asm_op, clob;
+
+ asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, empty_string, empty_string, 0,
+ rtvec_alloc (0), rtvec_alloc (0),
+ rtvec_alloc (0), UNKNOWN_LOCATION);
+ MEM_VOLATILE_P (asm_op) = 1;
+
+ clob = gen_rtx_SCRATCH (VOIDmode);
+ clob = gen_rtx_MEM (BLKmode, clob);
+ clob = gen_rtx_CLOBBER (VOIDmode, clob);
+
+ emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
+}
+
+/* This routine will either emit the mem_thread_fence pattern or issue a
+ sync_synchronize to generate a fence for memory model MEMMODEL. */
+
+#ifndef HAVE_mem_thread_fence
+# define HAVE_mem_thread_fence 0
+# define gen_mem_thread_fence(x) (gcc_unreachable (), NULL_RTX)
+#endif
+#ifndef HAVE_memory_barrier
+# define HAVE_memory_barrier 0
+# define gen_memory_barrier() (gcc_unreachable (), NULL_RTX)
+#endif
+
+void
+expand_mem_thread_fence (enum memmodel model)
+{
+ if (HAVE_mem_thread_fence)
+ emit_insn (gen_mem_thread_fence (GEN_INT (model)));
+ else if (model != MEMMODEL_RELAXED)
+ {
+ if (HAVE_memory_barrier)
+ emit_insn (gen_memory_barrier ());
+ else if (synchronize_libfunc != NULL_RTX)
+ emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode, 0);
+ else
+ expand_asm_memory_barrier ();
+ }
+}
+
+/* This routine will either emit the mem_signal_fence pattern or issue a
+ sync_synchronize to generate a fence for memory model MEMMODEL. */
+
+#ifndef HAVE_mem_signal_fence
+# define HAVE_mem_signal_fence 0
+# define gen_mem_signal_fence(x) (gcc_unreachable (), NULL_RTX)
+#endif
+
+void
+expand_mem_signal_fence (enum memmodel model)
+{
+ if (HAVE_mem_signal_fence)
+ emit_insn (gen_mem_signal_fence (GEN_INT (model)));
+ else if (model != MEMMODEL_RELAXED)
+ {
+ /* By default targets are coherent between a thread and the signal
+ handler running on the same thread. Thus this really becomes a
+ compiler barrier, in that stores must not be sunk past
+ (or raised above) a given point. */
+ expand_asm_memory_barrier ();
+ }
+}
+
/* This function expands the atomic load operation:
return the atomically loaded value in MEM.
target = gen_reg_rtx (mode);
/* Emit the appropriate barrier before the load. */
- expand_builtin_mem_thread_fence (model);
+ expand_mem_thread_fence (model);
emit_move_insn (target, mem);
/* For SEQ_CST, also emit a barrier after the load. */
if (model == MEMMODEL_SEQ_CST)
- expand_builtin_mem_thread_fence (model);
+ expand_mem_thread_fence (model);
return target;
}
{
/* lock_release is only a release barrier. */
if (model == MEMMODEL_SEQ_CST)
- expand_builtin_mem_thread_fence (model);
+ expand_mem_thread_fence (model);
return const0_rtx;
}
}
the result. If that doesn't work, don't do anything. */
if (GET_MODE_PRECISION(mode) > BITS_PER_WORD)
{
- rtx target = expand_atomic_exchange (NULL_RTX, mem, val, model, false);
+ rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
+ if (!target)
+ target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem, val);
if (target)
return const0_rtx;
else
/* If there is no mem_store, default to a move with barriers */
if (model == MEMMODEL_SEQ_CST || model == MEMMODEL_RELEASE)
- expand_builtin_mem_thread_fence (model);
+ expand_mem_thread_fence (model);
emit_move_insn (mem, val);
/* For SEQ_CST, also emit a barrier after the load. */
if (model == MEMMODEL_SEQ_CST)
- expand_builtin_mem_thread_fence (model);
+ expand_mem_thread_fence (model);
return const0_rtx;
}
}
}
+/* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
+ using memory order MODEL. If AFTER is true the operation needs to return
+ the value of *MEM after the operation, otherwise the previous value.
+ TARGET is an optional place to place the result. The result is unused if
+ it is const0_rtx.
+ Return the result if there is a better sequence, otherwise NULL_RTX. */
+
+static rtx
+maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
+ enum memmodel model, bool after)
+{
+ /* If the value is prefetched, or not used, it may be possible to replace
+ the sequence with a native exchange operation. */
+ if (!after || target == const0_rtx)
+ {
+ /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
+ if (code == AND && val == const0_rtx)
+ {
+ if (target == const0_rtx)
+ target = gen_reg_rtx (GET_MODE (mem));
+ return maybe_emit_atomic_exchange (target, mem, val, model);
+ }
+
+ /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
+ if (code == IOR && val == constm1_rtx)
+ {
+ if (target == const0_rtx)
+ target = gen_reg_rtx (GET_MODE (mem));
+ return maybe_emit_atomic_exchange (target, mem, val, model);
+ }
+ }
+
+ return NULL_RTX;
+}
+
/* Try to emit an instruction for a specific operation varaition.
OPTAB contains the OP functions.
TARGET is an optional place to return the result. const0_rtx means unused.
get_atomic_op_for_code (&optab, code);
+ /* Check to see if there are any better instructions. */
+ result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
+ if (result)
+ return result;
+
/* Check for the case where the result isn't used and try those patterns. */
if (unused_result)
{
{
/* If the result isn't used, no need to do compensation code. */
if (unused_result)
- return target;
+ return result;
/* Issue compensation code. Fetch_after == fetch_before OP val.
Fetch_before == after REVERSE_OP val. */
if (!after)
code = optab.reverse_code;
- result = expand_simple_binop (mode, code, result, val, target, true,
- OPTAB_LIB_WIDEN);
+ if (code == NOT)
+ {
+ result = expand_simple_binop (mode, AND, result, val, NULL_RTX,
+ true, OPTAB_LIB_WIDEN);
+ result = expand_simple_unop (mode, NOT, result, target, true);
+ }
+ else
+ result = expand_simple_binop (mode, code, result, val, target,
+ true, OPTAB_LIB_WIDEN);
return result;
}
}
result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
2, addr, ptr_mode, val, mode);
- if (unused_result)
- return target;
- if (fixup)
+ if (!unused_result && fixup)
result = expand_simple_binop (mode, code, result, val, target,
true, OPTAB_LIB_WIDEN);
return result;
return true;
/* If the operand is a memory whose address has no side effects,
- try forcing the address into a register. The check for side
- effects is important because force_reg cannot handle things
- like auto-modified addresses. */
- if (insn_data[(int) icode].operand[opno].allows_mem
- && MEM_P (op->value)
- && !side_effects_p (XEXP (op->value, 0)))
- {
- rtx addr, mem, last;
-
- last = get_last_insn ();
- addr = force_reg (Pmode, XEXP (op->value, 0));
- mem = replace_equiv_address (op->value, addr);
- if (insn_operand_matches (icode, opno, mem))
+ try forcing the address into a non-virtual pseudo register.
+ The check for side effects is important because copy_to_mode_reg
+ cannot handle things like auto-modified addresses. */
+ if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value))
+ {
+ rtx addr, mem;
+
+ mem = op->value;
+ addr = XEXP (mem, 0);
+ if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER)
+ && !side_effects_p (addr))
{
- op->value = mem;
- return true;
+ rtx last;
+ enum machine_mode mode;
+
+ last = get_last_insn ();
+ mode = targetm.addr_space.address_mode (MEM_ADDR_SPACE (mem));
+ mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr));
+ if (insn_operand_matches (icode, opno, mem))
+ {
+ op->value = mem;
+ return true;
+ }
+ delete_insns_since (last);
}
- delete_insns_since (last);
}
return false;
OSDN Git Service
