+ if (exit_bb)
+ {
+ si = gsi_last_bb (exit_bb);
+ gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN);
+ gsi_remove (&si, true);
+ single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
+ }
+}
+
+/* A subroutine of expand_omp_atomic. Attempt to implement the atomic
+ operation as a __sync_fetch_and_op builtin. INDEX is log2 of the
+ size of the data type, and thus usable to find the index of the builtin
+ decl. Returns false if the expression is not of the proper form. */
+
+static bool
+expand_omp_atomic_fetch_op (basic_block load_bb,
+ tree addr, tree loaded_val,
+ tree stored_val, int index)
+{
+ enum built_in_function base;
+ tree decl, itype, call;
+ enum insn_code *optab;
+ tree rhs;
+ basic_block store_bb = single_succ (load_bb);
+ gimple_stmt_iterator gsi;
+ gimple stmt;
+ location_t loc;
+
+ /* We expect to find the following sequences:
+
+ load_bb:
+ GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
+
+ store_bb:
+ val = tmp OP something; (or: something OP tmp)
+ GIMPLE_OMP_STORE (val)
+
+ ???FIXME: Allow a more flexible sequence.
+ Perhaps use data flow to pick the statements.
+
+ */
+
+ gsi = gsi_after_labels (store_bb);
+ stmt = gsi_stmt (gsi);
+ loc = gimple_location (stmt);
+ if (!is_gimple_assign (stmt))
+ return false;
+ gsi_next (&gsi);
+ if (gimple_code (gsi_stmt (gsi)) != GIMPLE_OMP_ATOMIC_STORE)
+ return false;
+
+ if (!operand_equal_p (gimple_assign_lhs (stmt), stored_val, 0))
+ return false;
+
+ /* Check for one of the supported fetch-op operations. */
+ switch (gimple_assign_rhs_code (stmt))
+ {
+ case PLUS_EXPR:
+ case POINTER_PLUS_EXPR:
+ base = BUILT_IN_FETCH_AND_ADD_N;
+ optab = sync_add_optab;
+ break;
+ case MINUS_EXPR:
+ base = BUILT_IN_FETCH_AND_SUB_N;
+ optab = sync_add_optab;
+ break;
+ case BIT_AND_EXPR:
+ base = BUILT_IN_FETCH_AND_AND_N;
+ optab = sync_and_optab;
+ break;
+ case BIT_IOR_EXPR:
+ base = BUILT_IN_FETCH_AND_OR_N;
+ optab = sync_ior_optab;
+ break;
+ case BIT_XOR_EXPR:
+ base = BUILT_IN_FETCH_AND_XOR_N;
+ optab = sync_xor_optab;
+ break;
+ default:
+ return false;
+ }
+ /* Make sure the expression is of the proper form. */
+ if (operand_equal_p (gimple_assign_rhs1 (stmt), loaded_val, 0))
+ rhs = gimple_assign_rhs2 (stmt);
+ else if (commutative_tree_code (gimple_assign_rhs_code (stmt))
+ && operand_equal_p (gimple_assign_rhs2 (stmt), loaded_val, 0))
+ rhs = gimple_assign_rhs1 (stmt);
+ else
+ return false;
+
+ decl = built_in_decls[base + index + 1];
+ itype = TREE_TYPE (TREE_TYPE (decl));
+
+ if (optab[TYPE_MODE (itype)] == CODE_FOR_nothing)
+ return false;
+
+ gsi = gsi_last_bb (load_bb);
+ gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_LOAD);
+ call = build_call_expr_loc (loc,
+ decl, 2, addr,
+ fold_convert_loc (loc, itype, rhs));
+ call = fold_convert_loc (loc, void_type_node, call);
+ force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT);
+ gsi_remove (&gsi, true);
+
+ gsi = gsi_last_bb (store_bb);
+ gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE);
+ gsi_remove (&gsi, true);
+ gsi = gsi_last_bb (store_bb);
+ gsi_remove (&gsi, true);
+
+ if (gimple_in_ssa_p (cfun))
+ update_ssa (TODO_update_ssa_no_phi);
+
+ return true;
+}
+
+/* A subroutine of expand_omp_atomic. Implement the atomic operation as:
+
+ oldval = *addr;
+ repeat:
+ newval = rhs; // with oldval replacing *addr in rhs
+ oldval = __sync_val_compare_and_swap (addr, oldval, newval);
+ if (oldval != newval)
+ goto repeat;
+
+ INDEX is log2 of the size of the data type, and thus usable to find the
+ index of the builtin decl. */
+
+static bool
+expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb,
+ tree addr, tree loaded_val, tree stored_val,
+ int index)
+{
+ tree loadedi, storedi, initial, new_storedi, old_vali;
+ tree type, itype, cmpxchg, iaddr;
+ gimple_stmt_iterator si;
+ basic_block loop_header = single_succ (load_bb);
+ gimple phi, stmt;
+ edge e;
+
+ cmpxchg = built_in_decls[BUILT_IN_VAL_COMPARE_AND_SWAP_N + index + 1];
+ type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
+ itype = TREE_TYPE (TREE_TYPE (cmpxchg));
+
+ if (sync_compare_and_swap[TYPE_MODE (itype)] == CODE_FOR_nothing)
+ return false;
+
+ /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
+ si = gsi_last_bb (load_bb);
+ gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
+
+ /* For floating-point values, we'll need to view-convert them to integers
+ so that we can perform the atomic compare and swap. Simplify the
+ following code by always setting up the "i"ntegral variables. */
+ if (!INTEGRAL_TYPE_P (type) && !POINTER_TYPE_P (type))
+ {
+ tree iaddr_val;
+
+ iaddr = create_tmp_var (build_pointer_type_for_mode (itype, ptr_mode,
+ true), NULL);
+ iaddr_val
+ = force_gimple_operand_gsi (&si,
+ fold_convert (TREE_TYPE (iaddr), addr),
+ false, NULL_TREE, true, GSI_SAME_STMT);
+ stmt = gimple_build_assign (iaddr, iaddr_val);
+ gsi_insert_before (&si, stmt, GSI_SAME_STMT);
+ loadedi = create_tmp_var (itype, NULL);
+ if (gimple_in_ssa_p (cfun))
+ {
+ add_referenced_var (iaddr);
+ add_referenced_var (loadedi);
+ loadedi = make_ssa_name (loadedi, NULL);
+ }
+ }
+ else
+ {
+ iaddr = addr;
+ loadedi = loaded_val;
+ }
+
+ initial = force_gimple_operand_gsi (&si, build_fold_indirect_ref (iaddr),
+ true, NULL_TREE, true, GSI_SAME_STMT);
+
+ /* Move the value to the LOADEDI temporary. */
+ if (gimple_in_ssa_p (cfun))
+ {
+ gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header)));
+ phi = create_phi_node (loadedi, loop_header);
+ SSA_NAME_DEF_STMT (loadedi) = phi;
+ SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, single_succ_edge (load_bb)),
+ initial);
+ }
+ else
+ gsi_insert_before (&si,
+ gimple_build_assign (loadedi, initial),
+ GSI_SAME_STMT);
+ if (loadedi != loaded_val)
+ {
+ gimple_stmt_iterator gsi2;
+ tree x;
+
+ x = build1 (VIEW_CONVERT_EXPR, type, loadedi);
+ gsi2 = gsi_start_bb (loop_header);
+ if (gimple_in_ssa_p (cfun))
+ {
+ gimple stmt;
+ x = force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
+ true, GSI_SAME_STMT);
+ stmt = gimple_build_assign (loaded_val, x);
+ gsi_insert_before (&gsi2, stmt, GSI_SAME_STMT);
+ }
+ else
+ {
+ x = build2 (MODIFY_EXPR, TREE_TYPE (loaded_val), loaded_val, x);
+ force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE,
+ true, GSI_SAME_STMT);
+ }
+ }
+ gsi_remove (&si, true);
+
+ si = gsi_last_bb (store_bb);
+ gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
+
+ if (iaddr == addr)
+ storedi = stored_val;
+ else
+ storedi =
+ force_gimple_operand_gsi (&si,
+ build1 (VIEW_CONVERT_EXPR, itype,
+ stored_val), true, NULL_TREE, true,
+ GSI_SAME_STMT);
+
+ /* Build the compare&swap statement. */
+ new_storedi = build_call_expr (cmpxchg, 3, iaddr, loadedi, storedi);
+ new_storedi = force_gimple_operand_gsi (&si,
+ fold_convert (TREE_TYPE (loadedi),
+ new_storedi),
+ true, NULL_TREE,
+ true, GSI_SAME_STMT);
+
+ if (gimple_in_ssa_p (cfun))
+ old_vali = loadedi;
+ else
+ {
+ old_vali = create_tmp_var (TREE_TYPE (loadedi), NULL);
+ if (gimple_in_ssa_p (cfun))
+ add_referenced_var (old_vali);
+ stmt = gimple_build_assign (old_vali, loadedi);
+ gsi_insert_before (&si, stmt, GSI_SAME_STMT);
+
+ stmt = gimple_build_assign (loadedi, new_storedi);
+ gsi_insert_before (&si, stmt, GSI_SAME_STMT);
+ }
+
+ /* Note that we always perform the comparison as an integer, even for
+ floating point. This allows the atomic operation to properly
+ succeed even with NaNs and -0.0. */
+ stmt = gimple_build_cond_empty
+ (build2 (NE_EXPR, boolean_type_node,
+ new_storedi, old_vali));
+ gsi_insert_before (&si, stmt, GSI_SAME_STMT);
+
+ /* Update cfg. */
+ e = single_succ_edge (store_bb);
+ e->flags &= ~EDGE_FALLTHRU;
+ e->flags |= EDGE_FALSE_VALUE;
+
+ e = make_edge (store_bb, loop_header, EDGE_TRUE_VALUE);
+
+ /* Copy the new value to loadedi (we already did that before the condition
+ if we are not in SSA). */
+ if (gimple_in_ssa_p (cfun))
+ {
+ phi = gimple_seq_first_stmt (phi_nodes (loop_header));
+ SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e), new_storedi);
+ }
+
+ /* Remove GIMPLE_OMP_ATOMIC_STORE. */
+ gsi_remove (&si, true);
+
+ if (gimple_in_ssa_p (cfun))
+ update_ssa (TODO_update_ssa_no_phi);
+
+ return true;
+}
+
+/* A subroutine of expand_omp_atomic. Implement the atomic operation as:
+
+ GOMP_atomic_start ();
+ *addr = rhs;
+ GOMP_atomic_end ();
+
+ The result is not globally atomic, but works so long as all parallel
+ references are within #pragma omp atomic directives. According to
+ responses received from omp@openmp.org, appears to be within spec.
+ Which makes sense, since that's how several other compilers handle
+ this situation as well.
+ LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
+ expanding. STORED_VAL is the operand of the matching
+ GIMPLE_OMP_ATOMIC_STORE.
+
+ We replace
+ GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
+ loaded_val = *addr;
+
+ and replace
+ GIMPLE_OMP_ATOMIC_ATORE (stored_val) with
+ *addr = stored_val;
+*/
+
+static bool
+expand_omp_atomic_mutex (basic_block load_bb, basic_block store_bb,
+ tree addr, tree loaded_val, tree stored_val)
+{
+ gimple_stmt_iterator si;
+ gimple stmt;
+ tree t;
+
+ si = gsi_last_bb (load_bb);
+ gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD);
+
+ t = built_in_decls[BUILT_IN_GOMP_ATOMIC_START];
+ t = build_function_call_expr (UNKNOWN_LOCATION, t, 0);
+ force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
+
+ stmt = gimple_build_assign (loaded_val, build_fold_indirect_ref (addr));
+ gsi_insert_before (&si, stmt, GSI_SAME_STMT);
+ gsi_remove (&si, true);
+
+ si = gsi_last_bb (store_bb);
+ gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE);
+
+ stmt = gimple_build_assign (build_fold_indirect_ref (unshare_expr (addr)),
+ stored_val);
+ gsi_insert_before (&si, stmt, GSI_SAME_STMT);
+
+ t = built_in_decls[BUILT_IN_GOMP_ATOMIC_END];
+ t = build_function_call_expr (UNKNOWN_LOCATION, t, 0);
+ force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT);
+ gsi_remove (&si, true);
+
+ if (gimple_in_ssa_p (cfun))
+ update_ssa (TODO_update_ssa_no_phi);
+ return true;
+}
+
+/* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
+ using expand_omp_atomic_fetch_op. If it failed, we try to
+ call expand_omp_atomic_pipeline, and if it fails too, the
+ ultimate fallback is wrapping the operation in a mutex
+ (expand_omp_atomic_mutex). REGION is the atomic region built
+ by build_omp_regions_1(). */
+
+static void
+expand_omp_atomic (struct omp_region *region)
+{
+ basic_block load_bb = region->entry, store_bb = region->exit;
+ gimple load = last_stmt (load_bb), store = last_stmt (store_bb);
+ tree loaded_val = gimple_omp_atomic_load_lhs (load);
+ tree addr = gimple_omp_atomic_load_rhs (load);
+ tree stored_val = gimple_omp_atomic_store_val (store);
+ tree type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
+ HOST_WIDE_INT index;
+
+ /* Make sure the type is one of the supported sizes. */
+ index = tree_low_cst (TYPE_SIZE_UNIT (type), 1);
+ index = exact_log2 (index);
+ if (index >= 0 && index <= 4)
+ {
+ unsigned int align = TYPE_ALIGN_UNIT (type);
+
+ /* __sync builtins require strict data alignment. */
+ if (exact_log2 (align) >= index)
+ {
+ /* When possible, use specialized atomic update functions. */
+ if ((INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type))
+ && store_bb == single_succ (load_bb))
+ {
+ if (expand_omp_atomic_fetch_op (load_bb, addr,
+ loaded_val, stored_val, index))
+ return;
+ }
+
+ /* If we don't have specialized __sync builtins, try and implement
+ as a compare and swap loop. */
+ if (expand_omp_atomic_pipeline (load_bb, store_bb, addr,
+ loaded_val, stored_val, index))
+ return;
+ }
+ }
+
+ /* The ultimate fallback is wrapping the operation in a mutex. */
+ expand_omp_atomic_mutex (load_bb, store_bb, addr, loaded_val, stored_val);