1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005, 2006, 2007 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
30 #include "tree-gimple.h"
31 #include "tree-inline.h"
32 #include "langhooks.h"
33 #include "diagnostic.h"
34 #include "tree-flow.h"
40 #include "tree-pass.h"
43 #include "splay-tree.h"
47 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
48 phases. The first phase scans the function looking for OMP statements
49 and then for variables that must be replaced to satisfy data sharing
50 clauses. The second phase expands code for the constructs, as well as
51 re-gimplifying things when variables have been replaced with complex
54 Final code generation is done by pass_expand_omp. The flowgraph is
55 scanned for parallel regions which are then moved to a new
56 function, to be invoked by the thread library. */
58 /* Context structure. Used to store information about each parallel
59 directive in the code. */
61 typedef struct omp_context
63 /* This field must be at the beginning, as we do "inheritance": Some
64 callback functions for tree-inline.c (e.g., omp_copy_decl)
65 receive a copy_body_data pointer that is up-casted to an
66 omp_context pointer. */
69 /* The tree of contexts corresponding to the encountered constructs. */
70 struct omp_context *outer;
73 /* Map variables to fields in a structure that allows communication
74 between sending and receiving threads. */
80 /* A chain of variables to add to the top-level block surrounding the
81 construct. In the case of a parallel, this is in the child function. */
84 /* What to do with variables with implicitly determined sharing
86 enum omp_clause_default_kind default_kind;
88 /* Nesting depth of this context. Used to beautify error messages re
89 invalid gotos. The outermost ctx is depth 1, with depth 0 being
90 reserved for the main body of the function. */
93 /* True if this parallel directive is nested within another. */
98 /* A structure describing the main elements of a parallel loop. */
102 tree v, n1, n2, step, chunk_size, for_stmt;
103 enum tree_code cond_code;
105 bool have_nowait, have_ordered;
106 enum omp_clause_schedule_kind sched_kind;
110 static splay_tree all_contexts;
111 static int parallel_nesting_level;
112 struct omp_region *root_omp_region;
114 static void scan_omp (tree *, omp_context *);
115 static void lower_omp (tree *, omp_context *);
116 static tree lookup_decl_in_outer_ctx (tree, omp_context *);
117 static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *);
119 /* Find an OpenMP clause of type KIND within CLAUSES. */
122 find_omp_clause (tree clauses, enum tree_code kind)
124 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
125 if (OMP_CLAUSE_CODE (clauses) == kind)
131 /* Return true if CTX is for an omp parallel. */
134 is_parallel_ctx (omp_context *ctx)
136 return TREE_CODE (ctx->stmt) == OMP_PARALLEL;
140 /* Return true if REGION is a combined parallel+workshare region. */
143 is_combined_parallel (struct omp_region *region)
145 return region->is_combined_parallel;
149 /* Extract the header elements of parallel loop FOR_STMT and store
153 extract_omp_for_data (tree for_stmt, struct omp_for_data *fd)
157 fd->for_stmt = for_stmt;
160 t = OMP_FOR_INIT (for_stmt);
161 gcc_assert (TREE_CODE (t) == GIMPLE_MODIFY_STMT);
162 fd->v = GIMPLE_STMT_OPERAND (t, 0);
163 gcc_assert (SSA_VAR_P (fd->v));
164 gcc_assert (TREE_CODE (TREE_TYPE (fd->v)) == INTEGER_TYPE);
165 var = TREE_CODE (fd->v) == SSA_NAME ? SSA_NAME_VAR (fd->v) : fd->v;
166 fd->n1 = GIMPLE_STMT_OPERAND (t, 1);
168 t = OMP_FOR_COND (for_stmt);
169 fd->cond_code = TREE_CODE (t);
170 gcc_assert (TREE_OPERAND (t, 0) == var);
171 fd->n2 = TREE_OPERAND (t, 1);
172 switch (fd->cond_code)
178 fd->n2 = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->n2), fd->n2,
179 build_int_cst (TREE_TYPE (fd->n2), 1));
180 fd->cond_code = LT_EXPR;
183 fd->n2 = fold_build2 (MINUS_EXPR, TREE_TYPE (fd->n2), fd->n2,
184 build_int_cst (TREE_TYPE (fd->n2), 1));
185 fd->cond_code = GT_EXPR;
191 t = OMP_FOR_INCR (fd->for_stmt);
192 gcc_assert (TREE_CODE (t) == GIMPLE_MODIFY_STMT);
193 gcc_assert (GIMPLE_STMT_OPERAND (t, 0) == var);
194 t = GIMPLE_STMT_OPERAND (t, 1);
195 gcc_assert (TREE_OPERAND (t, 0) == var);
196 switch (TREE_CODE (t))
199 fd->step = TREE_OPERAND (t, 1);
202 fd->step = TREE_OPERAND (t, 1);
203 fd->step = fold_build1 (NEGATE_EXPR, TREE_TYPE (fd->step), fd->step);
209 fd->have_nowait = fd->have_ordered = false;
210 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC;
211 fd->chunk_size = NULL_TREE;
213 for (t = OMP_FOR_CLAUSES (for_stmt); t ; t = OMP_CLAUSE_CHAIN (t))
214 switch (OMP_CLAUSE_CODE (t))
216 case OMP_CLAUSE_NOWAIT:
217 fd->have_nowait = true;
219 case OMP_CLAUSE_ORDERED:
220 fd->have_ordered = true;
222 case OMP_CLAUSE_SCHEDULE:
223 fd->sched_kind = OMP_CLAUSE_SCHEDULE_KIND (t);
224 fd->chunk_size = OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t);
230 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME)
231 gcc_assert (fd->chunk_size == NULL);
232 else if (fd->chunk_size == NULL)
234 /* We only need to compute a default chunk size for ordered
235 static loops and dynamic loops. */
236 if (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC || fd->have_ordered)
237 fd->chunk_size = (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC)
238 ? integer_zero_node : integer_one_node;
243 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
244 is the immediate dominator of PAR_ENTRY_BB, return true if there
245 are no data dependencies that would prevent expanding the parallel
246 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
248 When expanding a combined parallel+workshare region, the call to
249 the child function may need additional arguments in the case of
250 OMP_FOR regions. In some cases, these arguments are computed out
251 of variables passed in from the parent to the child via 'struct
252 .omp_data_s'. For instance:
254 #pragma omp parallel for schedule (guided, i * 4)
259 # BLOCK 2 (PAR_ENTRY_BB)
261 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
263 # BLOCK 3 (WS_ENTRY_BB)
264 .omp_data_i = &.omp_data_o;
265 D.1667 = .omp_data_i->i;
267 #pragma omp for schedule (guided, D.1598)
269 When we outline the parallel region, the call to the child function
270 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
271 that value is computed *after* the call site. So, in principle we
272 cannot do the transformation.
274 To see whether the code in WS_ENTRY_BB blocks the combined
275 parallel+workshare call, we collect all the variables used in the
276 OMP_FOR header check whether they appear on the LHS of any
277 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
280 FIXME. If we had the SSA form built at this point, we could merely
281 hoist the code in block 3 into block 2 and be done with it. But at
282 this point we don't have dataflow information and though we could
283 hack something up here, it is really not worth the aggravation. */
286 workshare_safe_to_combine_p (basic_block par_entry_bb, basic_block ws_entry_bb)
288 struct omp_for_data fd;
289 tree par_stmt, ws_stmt;
291 par_stmt = last_stmt (par_entry_bb);
292 ws_stmt = last_stmt (ws_entry_bb);
294 if (TREE_CODE (ws_stmt) == OMP_SECTIONS)
297 gcc_assert (TREE_CODE (ws_stmt) == OMP_FOR);
299 extract_omp_for_data (ws_stmt, &fd);
301 /* FIXME. We give up too easily here. If any of these arguments
302 are not constants, they will likely involve variables that have
303 been mapped into fields of .omp_data_s for sharing with the child
304 function. With appropriate data flow, it would be possible to
306 if (!is_gimple_min_invariant (fd.n1)
307 || !is_gimple_min_invariant (fd.n2)
308 || !is_gimple_min_invariant (fd.step)
309 || (fd.chunk_size && !is_gimple_min_invariant (fd.chunk_size)))
316 /* Collect additional arguments needed to emit a combined
317 parallel+workshare call. WS_STMT is the workshare directive being
321 get_ws_args_for (tree ws_stmt)
325 if (TREE_CODE (ws_stmt) == OMP_FOR)
327 struct omp_for_data fd;
330 extract_omp_for_data (ws_stmt, &fd);
335 t = fold_convert (long_integer_type_node, fd.chunk_size);
336 ws_args = tree_cons (NULL, t, ws_args);
339 t = fold_convert (long_integer_type_node, fd.step);
340 ws_args = tree_cons (NULL, t, ws_args);
342 t = fold_convert (long_integer_type_node, fd.n2);
343 ws_args = tree_cons (NULL, t, ws_args);
345 t = fold_convert (long_integer_type_node, fd.n1);
346 ws_args = tree_cons (NULL, t, ws_args);
350 else if (TREE_CODE (ws_stmt) == OMP_SECTIONS)
352 /* Number of sections is equal to the number of edges from the
353 OMP_SECTIONS_SWITCH statement, except for the one to the exit
354 of the sections region. */
355 basic_block bb = single_succ (bb_for_stmt (ws_stmt));
356 t = build_int_cst (unsigned_type_node, EDGE_COUNT (bb->succs) - 1);
357 t = tree_cons (NULL, t, NULL);
365 /* Discover whether REGION is a combined parallel+workshare region. */
368 determine_parallel_type (struct omp_region *region)
370 basic_block par_entry_bb, par_exit_bb;
371 basic_block ws_entry_bb, ws_exit_bb;
373 if (region == NULL || region->inner == NULL
374 || region->exit == NULL || region->inner->exit == NULL
375 || region->inner->cont == NULL)
378 /* We only support parallel+for and parallel+sections. */
379 if (region->type != OMP_PARALLEL
380 || (region->inner->type != OMP_FOR
381 && region->inner->type != OMP_SECTIONS))
384 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
385 WS_EXIT_BB -> PAR_EXIT_BB. */
386 par_entry_bb = region->entry;
387 par_exit_bb = region->exit;
388 ws_entry_bb = region->inner->entry;
389 ws_exit_bb = region->inner->exit;
391 if (single_succ (par_entry_bb) == ws_entry_bb
392 && single_succ (ws_exit_bb) == par_exit_bb
393 && workshare_safe_to_combine_p (par_entry_bb, ws_entry_bb)
394 && (OMP_PARALLEL_COMBINED (last_stmt (par_entry_bb))
395 || (last_and_only_stmt (ws_entry_bb)
396 && last_and_only_stmt (par_exit_bb))))
398 tree ws_stmt = last_stmt (ws_entry_bb);
400 if (region->inner->type == OMP_FOR)
402 /* If this is a combined parallel loop, we need to determine
403 whether or not to use the combined library calls. There
404 are two cases where we do not apply the transformation:
405 static loops and any kind of ordered loop. In the first
406 case, we already open code the loop so there is no need
407 to do anything else. In the latter case, the combined
408 parallel loop call would still need extra synchronization
409 to implement ordered semantics, so there would not be any
410 gain in using the combined call. */
411 tree clauses = OMP_FOR_CLAUSES (ws_stmt);
412 tree c = find_omp_clause (clauses, OMP_CLAUSE_SCHEDULE);
414 || OMP_CLAUSE_SCHEDULE_KIND (c) == OMP_CLAUSE_SCHEDULE_STATIC
415 || find_omp_clause (clauses, OMP_CLAUSE_ORDERED))
417 region->is_combined_parallel = false;
418 region->inner->is_combined_parallel = false;
423 region->is_combined_parallel = true;
424 region->inner->is_combined_parallel = true;
425 region->ws_args = get_ws_args_for (ws_stmt);
430 /* Return true if EXPR is variable sized. */
433 is_variable_sized (const_tree expr)
435 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr)));
438 /* Return true if DECL is a reference type. */
441 is_reference (tree decl)
443 return lang_hooks.decls.omp_privatize_by_reference (decl);
446 /* Lookup variables in the decl or field splay trees. The "maybe" form
447 allows for the variable form to not have been entered, otherwise we
448 assert that the variable must have been entered. */
451 lookup_decl (tree var, omp_context *ctx)
454 n = (tree *) pointer_map_contains (ctx->cb.decl_map, var);
459 maybe_lookup_decl (tree var, omp_context *ctx)
462 n = (tree *) pointer_map_contains (ctx->cb.decl_map, var);
463 return n ? *n : NULL_TREE;
467 lookup_field (tree var, omp_context *ctx)
470 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
471 return (tree) n->value;
475 maybe_lookup_field (tree var, omp_context *ctx)
478 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var);
479 return n ? (tree) n->value : NULL_TREE;
482 /* Return true if DECL should be copied by pointer. SHARED_P is true
483 if DECL is to be shared. */
486 use_pointer_for_field (const_tree decl, bool shared_p)
488 if (AGGREGATE_TYPE_P (TREE_TYPE (decl)))
491 /* We can only use copy-in/copy-out semantics for shared variables
492 when we know the value is not accessible from an outer scope. */
495 /* ??? Trivially accessible from anywhere. But why would we even
496 be passing an address in this case? Should we simply assert
497 this to be false, or should we have a cleanup pass that removes
498 these from the list of mappings? */
499 if (TREE_STATIC (decl) || DECL_EXTERNAL (decl))
502 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
503 without analyzing the expression whether or not its location
504 is accessible to anyone else. In the case of nested parallel
505 regions it certainly may be. */
506 if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl))
509 /* Do not use copy-in/copy-out for variables that have their
511 if (TREE_ADDRESSABLE (decl))
518 /* Create a new VAR_DECL and copy information from VAR to it. */
521 copy_var_decl (tree var, tree name, tree type)
523 tree copy = build_decl (VAR_DECL, name, type);
525 TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (var);
526 TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (var);
527 DECL_GIMPLE_REG_P (copy) = DECL_GIMPLE_REG_P (var);
528 DECL_NO_TBAA_P (copy) = DECL_NO_TBAA_P (var);
529 DECL_ARTIFICIAL (copy) = DECL_ARTIFICIAL (var);
530 DECL_IGNORED_P (copy) = DECL_IGNORED_P (var);
531 DECL_CONTEXT (copy) = DECL_CONTEXT (var);
532 TREE_USED (copy) = 1;
533 DECL_SEEN_IN_BIND_EXPR_P (copy) = 1;
538 /* Construct a new automatic decl similar to VAR. */
541 omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx)
543 tree copy = copy_var_decl (var, name, type);
545 DECL_CONTEXT (copy) = current_function_decl;
546 TREE_CHAIN (copy) = ctx->block_vars;
547 ctx->block_vars = copy;
553 omp_copy_decl_1 (tree var, omp_context *ctx)
555 return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx);
558 /* Build tree nodes to access the field for VAR on the receiver side. */
561 build_receiver_ref (tree var, bool by_ref, omp_context *ctx)
563 tree x, field = lookup_field (var, ctx);
565 /* If the receiver record type was remapped in the child function,
566 remap the field into the new record type. */
567 x = maybe_lookup_field (field, ctx);
571 x = build_fold_indirect_ref (ctx->receiver_decl);
572 x = build3 (COMPONENT_REF, TREE_TYPE (field), x, field, NULL);
574 x = build_fold_indirect_ref (x);
579 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
580 of a parallel, this is a component reference; for workshare constructs
581 this is some variable. */
584 build_outer_var_ref (tree var, omp_context *ctx)
588 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx)))
590 else if (is_variable_sized (var))
592 x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0);
593 x = build_outer_var_ref (x, ctx);
594 x = build_fold_indirect_ref (x);
596 else if (is_parallel_ctx (ctx))
598 bool by_ref = use_pointer_for_field (var, false);
599 x = build_receiver_ref (var, by_ref, ctx);
602 x = lookup_decl (var, ctx->outer);
603 else if (is_reference (var))
604 /* This can happen with orphaned constructs. If var is reference, it is
605 possible it is shared and as such valid. */
610 if (is_reference (var))
611 x = build_fold_indirect_ref (x);
616 /* Build tree nodes to access the field for VAR on the sender side. */
619 build_sender_ref (tree var, omp_context *ctx)
621 tree field = lookup_field (var, ctx);
622 return build3 (COMPONENT_REF, TREE_TYPE (field),
623 ctx->sender_decl, field, NULL);
626 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
629 install_var_field (tree var, bool by_ref, omp_context *ctx)
633 gcc_assert (!splay_tree_lookup (ctx->field_map, (splay_tree_key) var));
635 type = TREE_TYPE (var);
637 type = build_pointer_type (type);
639 field = build_decl (FIELD_DECL, DECL_NAME (var), type);
641 /* Remember what variable this field was created for. This does have a
642 side effect of making dwarf2out ignore this member, so for helpful
643 debugging we clear it later in delete_omp_context. */
644 DECL_ABSTRACT_ORIGIN (field) = var;
646 insert_field_into_struct (ctx->record_type, field);
648 splay_tree_insert (ctx->field_map, (splay_tree_key) var,
649 (splay_tree_value) field);
653 install_var_local (tree var, omp_context *ctx)
655 tree new_var = omp_copy_decl_1 (var, ctx);
656 insert_decl_map (&ctx->cb, var, new_var);
660 /* Adjust the replacement for DECL in CTX for the new context. This means
661 copying the DECL_VALUE_EXPR, and fixing up the type. */
664 fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug)
668 new_decl = lookup_decl (decl, ctx);
670 TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), &ctx->cb);
672 if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug)
673 && DECL_HAS_VALUE_EXPR_P (decl))
675 tree ve = DECL_VALUE_EXPR (decl);
676 walk_tree (&ve, copy_body_r, &ctx->cb, NULL);
677 SET_DECL_VALUE_EXPR (new_decl, ve);
678 DECL_HAS_VALUE_EXPR_P (new_decl) = 1;
681 if (!TREE_CONSTANT (DECL_SIZE (new_decl)))
683 size = remap_decl (DECL_SIZE (decl), &ctx->cb);
684 if (size == error_mark_node)
685 size = TYPE_SIZE (TREE_TYPE (new_decl));
686 DECL_SIZE (new_decl) = size;
688 size = remap_decl (DECL_SIZE_UNIT (decl), &ctx->cb);
689 if (size == error_mark_node)
690 size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl));
691 DECL_SIZE_UNIT (new_decl) = size;
695 /* The callback for remap_decl. Search all containing contexts for a
696 mapping of the variable; this avoids having to duplicate the splay
697 tree ahead of time. We know a mapping doesn't already exist in the
698 given context. Create new mappings to implement default semantics. */
701 omp_copy_decl (tree var, copy_body_data *cb)
703 omp_context *ctx = (omp_context *) cb;
706 if (TREE_CODE (var) == LABEL_DECL)
708 new_var = create_artificial_label ();
709 DECL_CONTEXT (new_var) = current_function_decl;
710 insert_decl_map (&ctx->cb, var, new_var);
714 while (!is_parallel_ctx (ctx))
719 new_var = maybe_lookup_decl (var, ctx);
724 if (is_global_var (var) || decl_function_context (var) != ctx->cb.src_fn)
727 return error_mark_node;
731 /* Return the parallel region associated with STMT. */
733 /* Debugging dumps for parallel regions. */
734 void dump_omp_region (FILE *, struct omp_region *, int);
735 void debug_omp_region (struct omp_region *);
736 void debug_all_omp_regions (void);
738 /* Dump the parallel region tree rooted at REGION. */
741 dump_omp_region (FILE *file, struct omp_region *region, int indent)
743 fprintf (file, "%*sbb %d: %s\n", indent, "", region->entry->index,
744 tree_code_name[region->type]);
747 dump_omp_region (file, region->inner, indent + 4);
751 fprintf (file, "%*sbb %d: OMP_CONTINUE\n", indent, "",
752 region->cont->index);
756 fprintf (file, "%*sbb %d: OMP_RETURN\n", indent, "",
757 region->exit->index);
759 fprintf (file, "%*s[no exit marker]\n", indent, "");
762 dump_omp_region (file, region->next, indent);
766 debug_omp_region (struct omp_region *region)
768 dump_omp_region (stderr, region, 0);
772 debug_all_omp_regions (void)
774 dump_omp_region (stderr, root_omp_region, 0);
778 /* Create a new parallel region starting at STMT inside region PARENT. */
781 new_omp_region (basic_block bb, enum tree_code type, struct omp_region *parent)
783 struct omp_region *region = xcalloc (1, sizeof (*region));
785 region->outer = parent;
791 /* This is a nested region. Add it to the list of inner
792 regions in PARENT. */
793 region->next = parent->inner;
794 parent->inner = region;
798 /* This is a toplevel region. Add it to the list of toplevel
799 regions in ROOT_OMP_REGION. */
800 region->next = root_omp_region;
801 root_omp_region = region;
807 /* Release the memory associated with the region tree rooted at REGION. */
810 free_omp_region_1 (struct omp_region *region)
812 struct omp_region *i, *n;
814 for (i = region->inner; i ; i = n)
817 free_omp_region_1 (i);
823 /* Release the memory for the entire omp region tree. */
826 free_omp_regions (void)
828 struct omp_region *r, *n;
829 for (r = root_omp_region; r ; r = n)
832 free_omp_region_1 (r);
834 root_omp_region = NULL;
838 /* Create a new context, with OUTER_CTX being the surrounding context. */
841 new_omp_context (tree stmt, omp_context *outer_ctx)
843 omp_context *ctx = XCNEW (omp_context);
845 splay_tree_insert (all_contexts, (splay_tree_key) stmt,
846 (splay_tree_value) ctx);
851 ctx->outer = outer_ctx;
852 ctx->cb = outer_ctx->cb;
853 ctx->cb.block = NULL;
854 ctx->depth = outer_ctx->depth + 1;
858 ctx->cb.src_fn = current_function_decl;
859 ctx->cb.dst_fn = current_function_decl;
860 ctx->cb.src_node = cgraph_node (current_function_decl);
861 ctx->cb.dst_node = ctx->cb.src_node;
862 ctx->cb.src_cfun = cfun;
863 ctx->cb.copy_decl = omp_copy_decl;
864 ctx->cb.eh_region = -1;
865 ctx->cb.transform_call_graph_edges = CB_CGE_MOVE;
869 ctx->cb.decl_map = pointer_map_create ();
874 /* Destroy a omp_context data structures. Called through the splay tree
875 value delete callback. */
878 delete_omp_context (splay_tree_value value)
880 omp_context *ctx = (omp_context *) value;
882 pointer_map_destroy (ctx->cb.decl_map);
885 splay_tree_delete (ctx->field_map);
887 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
888 it produces corrupt debug information. */
889 if (ctx->record_type)
892 for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t))
893 DECL_ABSTRACT_ORIGIN (t) = NULL;
899 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
903 fixup_child_record_type (omp_context *ctx)
905 tree f, type = ctx->record_type;
907 /* ??? It isn't sufficient to just call remap_type here, because
908 variably_modified_type_p doesn't work the way we expect for
909 record types. Testing each field for whether it needs remapping
910 and creating a new record by hand works, however. */
911 for (f = TYPE_FIELDS (type); f ; f = TREE_CHAIN (f))
912 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn))
916 tree name, new_fields = NULL;
918 type = lang_hooks.types.make_type (RECORD_TYPE);
919 name = DECL_NAME (TYPE_NAME (ctx->record_type));
920 name = build_decl (TYPE_DECL, name, type);
921 TYPE_NAME (type) = name;
923 for (f = TYPE_FIELDS (ctx->record_type); f ; f = TREE_CHAIN (f))
925 tree new_f = copy_node (f);
926 DECL_CONTEXT (new_f) = type;
927 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &ctx->cb);
928 TREE_CHAIN (new_f) = new_fields;
931 /* Arrange to be able to look up the receiver field
932 given the sender field. */
933 splay_tree_insert (ctx->field_map, (splay_tree_key) f,
934 (splay_tree_value) new_f);
936 TYPE_FIELDS (type) = nreverse (new_fields);
940 TREE_TYPE (ctx->receiver_decl) = build_pointer_type (type);
943 /* Instantiate decls as necessary in CTX to satisfy the data sharing
944 specified by CLAUSES. */
947 scan_sharing_clauses (tree clauses, omp_context *ctx)
950 bool scan_array_reductions = false;
952 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
956 switch (OMP_CLAUSE_CODE (c))
958 case OMP_CLAUSE_PRIVATE:
959 decl = OMP_CLAUSE_DECL (c);
960 if (!is_variable_sized (decl))
961 install_var_local (decl, ctx);
964 case OMP_CLAUSE_SHARED:
965 gcc_assert (is_parallel_ctx (ctx));
966 decl = OMP_CLAUSE_DECL (c);
967 gcc_assert (!is_variable_sized (decl));
968 by_ref = use_pointer_for_field (decl, true);
969 /* Global variables don't need to be copied,
970 the receiver side will use them directly. */
971 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
973 if (! TREE_READONLY (decl)
974 || TREE_ADDRESSABLE (decl)
976 || is_reference (decl))
978 install_var_field (decl, by_ref, ctx);
979 install_var_local (decl, ctx);
982 /* We don't need to copy const scalar vars back. */
983 OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE);
986 case OMP_CLAUSE_LASTPRIVATE:
987 /* Let the corresponding firstprivate clause create
989 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
993 case OMP_CLAUSE_FIRSTPRIVATE:
994 case OMP_CLAUSE_REDUCTION:
995 decl = OMP_CLAUSE_DECL (c);
997 if (is_variable_sized (decl))
999 else if (is_parallel_ctx (ctx)
1000 && ! is_global_var (maybe_lookup_decl_in_outer_ctx (decl,
1003 by_ref = use_pointer_for_field (decl, false);
1004 install_var_field (decl, by_ref, ctx);
1006 install_var_local (decl, ctx);
1009 case OMP_CLAUSE_COPYPRIVATE:
1011 scan_omp (&OMP_CLAUSE_DECL (c), ctx->outer);
1014 case OMP_CLAUSE_COPYIN:
1015 decl = OMP_CLAUSE_DECL (c);
1016 by_ref = use_pointer_for_field (decl, false);
1017 install_var_field (decl, by_ref, ctx);
1020 case OMP_CLAUSE_DEFAULT:
1021 ctx->default_kind = OMP_CLAUSE_DEFAULT_KIND (c);
1025 case OMP_CLAUSE_NUM_THREADS:
1026 case OMP_CLAUSE_SCHEDULE:
1028 scan_omp (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer);
1031 case OMP_CLAUSE_NOWAIT:
1032 case OMP_CLAUSE_ORDERED:
1040 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1042 switch (OMP_CLAUSE_CODE (c))
1044 case OMP_CLAUSE_LASTPRIVATE:
1045 /* Let the corresponding firstprivate clause create
1047 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1051 case OMP_CLAUSE_PRIVATE:
1052 case OMP_CLAUSE_FIRSTPRIVATE:
1053 case OMP_CLAUSE_REDUCTION:
1054 decl = OMP_CLAUSE_DECL (c);
1055 if (is_variable_sized (decl))
1056 install_var_local (decl, ctx);
1057 fixup_remapped_decl (decl, ctx,
1058 OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE
1059 && OMP_CLAUSE_PRIVATE_DEBUG (c));
1060 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1061 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1062 scan_array_reductions = true;
1065 case OMP_CLAUSE_SHARED:
1066 decl = OMP_CLAUSE_DECL (c);
1067 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
1068 fixup_remapped_decl (decl, ctx, false);
1071 case OMP_CLAUSE_COPYPRIVATE:
1072 case OMP_CLAUSE_COPYIN:
1073 case OMP_CLAUSE_DEFAULT:
1075 case OMP_CLAUSE_NUM_THREADS:
1076 case OMP_CLAUSE_SCHEDULE:
1077 case OMP_CLAUSE_NOWAIT:
1078 case OMP_CLAUSE_ORDERED:
1086 if (scan_array_reductions)
1087 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
1088 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
1089 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1091 scan_omp (&OMP_CLAUSE_REDUCTION_INIT (c), ctx);
1092 scan_omp (&OMP_CLAUSE_REDUCTION_MERGE (c), ctx);
1096 /* Create a new name for omp child function. Returns an identifier. */
1098 static GTY(()) unsigned int tmp_ompfn_id_num;
1101 create_omp_child_function_name (void)
1103 tree name = DECL_ASSEMBLER_NAME (current_function_decl);
1104 size_t len = IDENTIFIER_LENGTH (name);
1105 char *tmp_name, *prefix;
1107 prefix = alloca (len + sizeof ("_omp_fn"));
1108 memcpy (prefix, IDENTIFIER_POINTER (name), len);
1109 strcpy (prefix + len, "_omp_fn");
1110 #ifndef NO_DOT_IN_LABEL
1112 #elif !defined NO_DOLLAR_IN_LABEL
1115 ASM_FORMAT_PRIVATE_NAME (tmp_name, prefix, tmp_ompfn_id_num++);
1116 return get_identifier (tmp_name);
1119 /* Build a decl for the omp child function. It'll not contain a body
1120 yet, just the bare decl. */
1123 create_omp_child_function (omp_context *ctx)
1125 tree decl, type, name, t;
1127 name = create_omp_child_function_name ();
1128 type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE);
1130 decl = build_decl (FUNCTION_DECL, name, type);
1131 decl = lang_hooks.decls.pushdecl (decl);
1133 ctx->cb.dst_fn = decl;
1135 TREE_STATIC (decl) = 1;
1136 TREE_USED (decl) = 1;
1137 DECL_ARTIFICIAL (decl) = 1;
1138 DECL_IGNORED_P (decl) = 0;
1139 TREE_PUBLIC (decl) = 0;
1140 DECL_UNINLINABLE (decl) = 1;
1141 DECL_EXTERNAL (decl) = 0;
1142 DECL_CONTEXT (decl) = NULL_TREE;
1143 DECL_INITIAL (decl) = make_node (BLOCK);
1145 t = build_decl (RESULT_DECL, NULL_TREE, void_type_node);
1146 DECL_ARTIFICIAL (t) = 1;
1147 DECL_IGNORED_P (t) = 1;
1148 DECL_RESULT (decl) = t;
1150 t = build_decl (PARM_DECL, get_identifier (".omp_data_i"), ptr_type_node);
1151 DECL_ARTIFICIAL (t) = 1;
1152 DECL_ARG_TYPE (t) = ptr_type_node;
1153 DECL_CONTEXT (t) = current_function_decl;
1155 DECL_ARGUMENTS (decl) = t;
1156 ctx->receiver_decl = t;
1158 /* Allocate memory for the function structure. The call to
1159 allocate_struct_function clobbers CFUN, so we need to restore
1161 push_struct_function (decl);
1162 DECL_SOURCE_LOCATION (decl) = EXPR_LOCATION (ctx->stmt);
1163 cfun->function_end_locus = EXPR_LOCATION (ctx->stmt);
1168 /* Scan an OpenMP parallel directive. */
1171 scan_omp_parallel (tree *stmt_p, omp_context *outer_ctx)
1176 /* Ignore parallel directives with empty bodies, unless there
1177 are copyin clauses. */
1179 && empty_body_p (OMP_PARALLEL_BODY (*stmt_p))
1180 && find_omp_clause (OMP_CLAUSES (*stmt_p), OMP_CLAUSE_COPYIN) == NULL)
1182 *stmt_p = build_empty_stmt ();
1186 ctx = new_omp_context (*stmt_p, outer_ctx);
1187 if (parallel_nesting_level > 1)
1188 ctx->is_nested = true;
1189 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
1190 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED;
1191 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
1192 name = create_tmp_var_name (".omp_data_s");
1193 name = build_decl (TYPE_DECL, name, ctx->record_type);
1194 TYPE_NAME (ctx->record_type) = name;
1195 create_omp_child_function (ctx);
1196 OMP_PARALLEL_FN (*stmt_p) = ctx->cb.dst_fn;
1198 scan_sharing_clauses (OMP_PARALLEL_CLAUSES (*stmt_p), ctx);
1199 scan_omp (&OMP_PARALLEL_BODY (*stmt_p), ctx);
1201 if (TYPE_FIELDS (ctx->record_type) == NULL)
1202 ctx->record_type = ctx->receiver_decl = NULL;
1205 layout_type (ctx->record_type);
1206 fixup_child_record_type (ctx);
1211 /* Scan an OpenMP loop directive. */
1214 scan_omp_for (tree *stmt_p, omp_context *outer_ctx)
1220 ctx = new_omp_context (stmt, outer_ctx);
1222 scan_sharing_clauses (OMP_FOR_CLAUSES (stmt), ctx);
1224 scan_omp (&OMP_FOR_PRE_BODY (stmt), ctx);
1225 scan_omp (&OMP_FOR_INIT (stmt), ctx);
1226 scan_omp (&OMP_FOR_COND (stmt), ctx);
1227 scan_omp (&OMP_FOR_INCR (stmt), ctx);
1228 scan_omp (&OMP_FOR_BODY (stmt), ctx);
1231 /* Scan an OpenMP sections directive. */
1234 scan_omp_sections (tree *stmt_p, omp_context *outer_ctx)
1240 ctx = new_omp_context (stmt, outer_ctx);
1241 scan_sharing_clauses (OMP_SECTIONS_CLAUSES (stmt), ctx);
1242 scan_omp (&OMP_SECTIONS_BODY (stmt), ctx);
1245 /* Scan an OpenMP single directive. */
1248 scan_omp_single (tree *stmt_p, omp_context *outer_ctx)
1250 tree stmt = *stmt_p;
1254 ctx = new_omp_context (stmt, outer_ctx);
1255 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
1256 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
1257 name = create_tmp_var_name (".omp_copy_s");
1258 name = build_decl (TYPE_DECL, name, ctx->record_type);
1259 TYPE_NAME (ctx->record_type) = name;
1261 scan_sharing_clauses (OMP_SINGLE_CLAUSES (stmt), ctx);
1262 scan_omp (&OMP_SINGLE_BODY (stmt), ctx);
1264 if (TYPE_FIELDS (ctx->record_type) == NULL)
1265 ctx->record_type = NULL;
1267 layout_type (ctx->record_type);
1271 /* Check OpenMP nesting restrictions. */
1273 check_omp_nesting_restrictions (tree t, omp_context *ctx)
1275 switch (TREE_CODE (t))
1280 for (; ctx != NULL; ctx = ctx->outer)
1281 switch (TREE_CODE (ctx->stmt))
1288 warning (0, "work-sharing region may not be closely nested inside "
1289 "of work-sharing, critical, ordered or master region");
1298 for (; ctx != NULL; ctx = ctx->outer)
1299 switch (TREE_CODE (ctx->stmt))
1304 warning (0, "master region may not be closely nested inside "
1305 "of work-sharing region");
1314 for (; ctx != NULL; ctx = ctx->outer)
1315 switch (TREE_CODE (ctx->stmt))
1318 warning (0, "ordered region may not be closely nested inside "
1319 "of critical region");
1322 if (find_omp_clause (OMP_CLAUSES (ctx->stmt),
1323 OMP_CLAUSE_ORDERED) == NULL)
1324 warning (0, "ordered region must be closely nested inside "
1325 "a loop region with an ordered clause");
1334 for (; ctx != NULL; ctx = ctx->outer)
1335 if (TREE_CODE (ctx->stmt) == OMP_CRITICAL
1336 && OMP_CRITICAL_NAME (t) == OMP_CRITICAL_NAME (ctx->stmt))
1338 warning (0, "critical region may not be nested inside a critical "
1339 "region with the same name");
1349 /* Callback for walk_stmts used to scan for OpenMP directives at TP. */
1352 scan_omp_1 (tree *tp, int *walk_subtrees, void *data)
1354 struct walk_stmt_info *wi = data;
1355 omp_context *ctx = wi->info;
1358 if (EXPR_HAS_LOCATION (t))
1359 input_location = EXPR_LOCATION (t);
1361 /* Check the OpenMP nesting restrictions. */
1362 if (OMP_DIRECTIVE_P (t) && ctx != NULL)
1363 check_omp_nesting_restrictions (t, ctx);
1366 switch (TREE_CODE (t))
1369 parallel_nesting_level++;
1370 scan_omp_parallel (tp, ctx);
1371 parallel_nesting_level--;
1375 scan_omp_for (tp, ctx);
1379 scan_omp_sections (tp, ctx);
1383 scan_omp_single (tp, ctx);
1390 ctx = new_omp_context (*tp, ctx);
1391 scan_omp (&OMP_BODY (*tp), ctx);
1399 for (var = BIND_EXPR_VARS (t); var ; var = TREE_CHAIN (var))
1400 insert_decl_map (&ctx->cb, var, var);
1409 *tp = remap_decl (t, &ctx->cb);
1413 if (ctx && TYPE_P (t))
1414 *tp = remap_type (t, &ctx->cb);
1415 else if (!DECL_P (t))
1424 /* Scan all the statements starting at STMT_P. CTX contains context
1425 information about the OpenMP directives and clauses found during
1429 scan_omp (tree *stmt_p, omp_context *ctx)
1431 location_t saved_location;
1432 struct walk_stmt_info wi;
1434 memset (&wi, 0, sizeof (wi));
1435 wi.callback = scan_omp_1;
1437 wi.want_bind_expr = (ctx != NULL);
1438 wi.want_locations = true;
1440 saved_location = input_location;
1441 walk_stmts (&wi, stmt_p);
1442 input_location = saved_location;
1445 /* Re-gimplification and code generation routines. */
1447 /* Build a call to GOMP_barrier. */
1450 build_omp_barrier (void)
1452 return build_call_expr (built_in_decls[BUILT_IN_GOMP_BARRIER], 0);
1455 /* If a context was created for STMT when it was scanned, return it. */
1457 static omp_context *
1458 maybe_lookup_ctx (tree stmt)
1461 n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt);
1462 return n ? (omp_context *) n->value : NULL;
1466 /* Find the mapping for DECL in CTX or the immediately enclosing
1467 context that has a mapping for DECL.
1469 If CTX is a nested parallel directive, we may have to use the decl
1470 mappings created in CTX's parent context. Suppose that we have the
1471 following parallel nesting (variable UIDs showed for clarity):
1474 #omp parallel shared(iD.1562) -> outer parallel
1475 iD.1562 = iD.1562 + 1;
1477 #omp parallel shared (iD.1562) -> inner parallel
1478 iD.1562 = iD.1562 - 1;
1480 Each parallel structure will create a distinct .omp_data_s structure
1481 for copying iD.1562 in/out of the directive:
1483 outer parallel .omp_data_s.1.i -> iD.1562
1484 inner parallel .omp_data_s.2.i -> iD.1562
1486 A shared variable mapping will produce a copy-out operation before
1487 the parallel directive and a copy-in operation after it. So, in
1488 this case we would have:
1491 .omp_data_o.1.i = iD.1562;
1492 #omp parallel shared(iD.1562) -> outer parallel
1493 .omp_data_i.1 = &.omp_data_o.1
1494 .omp_data_i.1->i = .omp_data_i.1->i + 1;
1496 .omp_data_o.2.i = iD.1562; -> **
1497 #omp parallel shared(iD.1562) -> inner parallel
1498 .omp_data_i.2 = &.omp_data_o.2
1499 .omp_data_i.2->i = .omp_data_i.2->i - 1;
1502 ** This is a problem. The symbol iD.1562 cannot be referenced
1503 inside the body of the outer parallel region. But since we are
1504 emitting this copy operation while expanding the inner parallel
1505 directive, we need to access the CTX structure of the outer
1506 parallel directive to get the correct mapping:
1508 .omp_data_o.2.i = .omp_data_i.1->i
1510 Since there may be other workshare or parallel directives enclosing
1511 the parallel directive, it may be necessary to walk up the context
1512 parent chain. This is not a problem in general because nested
1513 parallelism happens only rarely. */
1516 lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
1521 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
1522 t = maybe_lookup_decl (decl, up);
1524 gcc_assert (!ctx->is_nested || t || is_global_var (decl));
1526 return t ? t : decl;
1530 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
1531 in outer contexts. */
1534 maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx)
1539 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer)
1540 t = maybe_lookup_decl (decl, up);
1542 return t ? t : decl;
1546 /* Construct the initialization value for reduction CLAUSE. */
1549 omp_reduction_init (tree clause, tree type)
1551 switch (OMP_CLAUSE_REDUCTION_CODE (clause))
1558 case TRUTH_ORIF_EXPR:
1559 case TRUTH_XOR_EXPR:
1561 return fold_convert (type, integer_zero_node);
1564 case TRUTH_AND_EXPR:
1565 case TRUTH_ANDIF_EXPR:
1567 return fold_convert (type, integer_one_node);
1570 return fold_convert (type, integer_minus_one_node);
1573 if (SCALAR_FLOAT_TYPE_P (type))
1575 REAL_VALUE_TYPE max, min;
1576 if (HONOR_INFINITIES (TYPE_MODE (type)))
1579 real_arithmetic (&min, NEGATE_EXPR, &max, NULL);
1582 real_maxval (&min, 1, TYPE_MODE (type));
1583 return build_real (type, min);
1587 gcc_assert (INTEGRAL_TYPE_P (type));
1588 return TYPE_MIN_VALUE (type);
1592 if (SCALAR_FLOAT_TYPE_P (type))
1594 REAL_VALUE_TYPE max;
1595 if (HONOR_INFINITIES (TYPE_MODE (type)))
1598 real_maxval (&max, 0, TYPE_MODE (type));
1599 return build_real (type, max);
1603 gcc_assert (INTEGRAL_TYPE_P (type));
1604 return TYPE_MAX_VALUE (type);
1612 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
1613 from the receiver (aka child) side and initializers for REFERENCE_TYPE
1614 private variables. Initialization statements go in ILIST, while calls
1615 to destructors go in DLIST. */
1618 lower_rec_input_clauses (tree clauses, tree *ilist, tree *dlist,
1621 tree_stmt_iterator diter;
1622 tree c, dtor, copyin_seq, x, ptr;
1623 bool copyin_by_ref = false;
1624 bool lastprivate_firstprivate = false;
1627 *dlist = alloc_stmt_list ();
1628 diter = tsi_start (*dlist);
1631 /* Do all the fixed sized types in the first pass, and the variable sized
1632 types in the second pass. This makes sure that the scalar arguments to
1633 the variable sized types are processed before we use them in the
1634 variable sized operations. */
1635 for (pass = 0; pass < 2; ++pass)
1637 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
1639 enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c);
1645 case OMP_CLAUSE_PRIVATE:
1646 if (OMP_CLAUSE_PRIVATE_DEBUG (c))
1649 case OMP_CLAUSE_SHARED:
1650 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL)
1652 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c)));
1655 case OMP_CLAUSE_FIRSTPRIVATE:
1656 case OMP_CLAUSE_COPYIN:
1657 case OMP_CLAUSE_REDUCTION:
1659 case OMP_CLAUSE_LASTPRIVATE:
1660 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1662 lastprivate_firstprivate = true;
1671 new_var = var = OMP_CLAUSE_DECL (c);
1672 if (c_kind != OMP_CLAUSE_COPYIN)
1673 new_var = lookup_decl (var, ctx);
1675 if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN)
1680 else if (is_variable_sized (var))
1682 /* For variable sized types, we need to allocate the
1683 actual storage here. Call alloca and store the
1684 result in the pointer decl that we created elsewhere. */
1688 ptr = DECL_VALUE_EXPR (new_var);
1689 gcc_assert (TREE_CODE (ptr) == INDIRECT_REF);
1690 ptr = TREE_OPERAND (ptr, 0);
1691 gcc_assert (DECL_P (ptr));
1693 x = TYPE_SIZE_UNIT (TREE_TYPE (new_var));
1694 x = build_call_expr (built_in_decls[BUILT_IN_ALLOCA], 1, x);
1695 x = fold_convert (TREE_TYPE (ptr), x);
1696 x = build_gimple_modify_stmt (ptr, x);
1697 gimplify_and_add (x, ilist);
1699 else if (is_reference (var))
1701 /* For references that are being privatized for Fortran,
1702 allocate new backing storage for the new pointer
1703 variable. This allows us to avoid changing all the
1704 code that expects a pointer to something that expects
1705 a direct variable. Note that this doesn't apply to
1706 C++, since reference types are disallowed in data
1707 sharing clauses there, except for NRV optimized
1712 x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var)));
1713 if (TREE_CONSTANT (x))
1715 const char *name = NULL;
1716 if (DECL_NAME (var))
1717 name = IDENTIFIER_POINTER (DECL_NAME (new_var));
1719 x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)),
1721 gimple_add_tmp_var (x);
1722 x = build_fold_addr_expr_with_type (x, TREE_TYPE (new_var));
1726 x = build_call_expr (built_in_decls[BUILT_IN_ALLOCA], 1, x);
1727 x = fold_convert (TREE_TYPE (new_var), x);
1730 x = build_gimple_modify_stmt (new_var, x);
1731 gimplify_and_add (x, ilist);
1733 new_var = build_fold_indirect_ref (new_var);
1735 else if (c_kind == OMP_CLAUSE_REDUCTION
1736 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1744 switch (OMP_CLAUSE_CODE (c))
1746 case OMP_CLAUSE_SHARED:
1747 /* Shared global vars are just accessed directly. */
1748 if (is_global_var (new_var))
1750 /* Set up the DECL_VALUE_EXPR for shared variables now. This
1751 needs to be delayed until after fixup_child_record_type so
1752 that we get the correct type during the dereference. */
1753 by_ref = use_pointer_for_field (var, true);
1754 x = build_receiver_ref (var, by_ref, ctx);
1755 SET_DECL_VALUE_EXPR (new_var, x);
1756 DECL_HAS_VALUE_EXPR_P (new_var) = 1;
1758 /* ??? If VAR is not passed by reference, and the variable
1759 hasn't been initialized yet, then we'll get a warning for
1760 the store into the omp_data_s structure. Ideally, we'd be
1761 able to notice this and not store anything at all, but
1762 we're generating code too early. Suppress the warning. */
1764 TREE_NO_WARNING (var) = 1;
1767 case OMP_CLAUSE_LASTPRIVATE:
1768 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
1772 case OMP_CLAUSE_PRIVATE:
1773 x = lang_hooks.decls.omp_clause_default_ctor (c, new_var);
1775 gimplify_and_add (x, ilist);
1779 x = lang_hooks.decls.omp_clause_dtor (c, new_var);
1783 gimplify_stmt (&dtor);
1784 tsi_link_before (&diter, dtor, TSI_SAME_STMT);
1788 case OMP_CLAUSE_FIRSTPRIVATE:
1789 x = build_outer_var_ref (var, ctx);
1790 x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x);
1791 gimplify_and_add (x, ilist);
1795 case OMP_CLAUSE_COPYIN:
1796 by_ref = use_pointer_for_field (var, false);
1797 x = build_receiver_ref (var, by_ref, ctx);
1798 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x);
1799 append_to_statement_list (x, ©in_seq);
1800 copyin_by_ref |= by_ref;
1803 case OMP_CLAUSE_REDUCTION:
1804 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1806 gimplify_and_add (OMP_CLAUSE_REDUCTION_INIT (c), ilist);
1807 OMP_CLAUSE_REDUCTION_INIT (c) = NULL;
1811 x = omp_reduction_init (c, TREE_TYPE (new_var));
1812 gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE);
1813 x = build_gimple_modify_stmt (new_var, x);
1814 gimplify_and_add (x, ilist);
1824 /* The copyin sequence is not to be executed by the main thread, since
1825 that would result in self-copies. Perhaps not visible to scalars,
1826 but it certainly is to C++ operator=. */
1829 x = build_call_expr (built_in_decls[BUILT_IN_OMP_GET_THREAD_NUM], 0);
1830 x = build2 (NE_EXPR, boolean_type_node, x,
1831 build_int_cst (TREE_TYPE (x), 0));
1832 x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL);
1833 gimplify_and_add (x, ilist);
1836 /* If any copyin variable is passed by reference, we must ensure the
1837 master thread doesn't modify it before it is copied over in all
1838 threads. Similarly for variables in both firstprivate and
1839 lastprivate clauses we need to ensure the lastprivate copying
1840 happens after firstprivate copying in all threads. */
1841 if (copyin_by_ref || lastprivate_firstprivate)
1842 gimplify_and_add (build_omp_barrier (), ilist);
1846 /* Generate code to implement the LASTPRIVATE clauses. This is used for
1847 both parallel and workshare constructs. PREDICATE may be NULL if it's
1851 lower_lastprivate_clauses (tree clauses, tree predicate, tree *stmt_list,
1854 tree sub_list, x, c;
1856 /* Early exit if there are no lastprivate clauses. */
1857 clauses = find_omp_clause (clauses, OMP_CLAUSE_LASTPRIVATE);
1858 if (clauses == NULL)
1860 /* If this was a workshare clause, see if it had been combined
1861 with its parallel. In that case, look for the clauses on the
1862 parallel statement itself. */
1863 if (is_parallel_ctx (ctx))
1867 if (ctx == NULL || !is_parallel_ctx (ctx))
1870 clauses = find_omp_clause (OMP_PARALLEL_CLAUSES (ctx->stmt),
1871 OMP_CLAUSE_LASTPRIVATE);
1872 if (clauses == NULL)
1876 sub_list = alloc_stmt_list ();
1878 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
1882 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LASTPRIVATE)
1885 var = OMP_CLAUSE_DECL (c);
1886 new_var = lookup_decl (var, ctx);
1888 x = build_outer_var_ref (var, ctx);
1889 if (is_reference (var))
1890 new_var = build_fold_indirect_ref (new_var);
1891 x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var);
1892 append_to_statement_list (x, &sub_list);
1896 x = build3 (COND_EXPR, void_type_node, predicate, sub_list, NULL);
1900 gimplify_and_add (x, stmt_list);
1904 /* Generate code to implement the REDUCTION clauses. */
1907 lower_reduction_clauses (tree clauses, tree *stmt_list, omp_context *ctx)
1909 tree sub_list = NULL, x, c;
1912 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
1913 update in that case, otherwise use a lock. */
1914 for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c))
1915 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
1917 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1919 /* Never use OMP_ATOMIC for array reductions. */
1929 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
1931 tree var, ref, new_var;
1932 enum tree_code code;
1934 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
1937 var = OMP_CLAUSE_DECL (c);
1938 new_var = lookup_decl (var, ctx);
1939 if (is_reference (var))
1940 new_var = build_fold_indirect_ref (new_var);
1941 ref = build_outer_var_ref (var, ctx);
1942 code = OMP_CLAUSE_REDUCTION_CODE (c);
1944 /* reduction(-:var) sums up the partial results, so it acts
1945 identically to reduction(+:var). */
1946 if (code == MINUS_EXPR)
1951 tree addr = build_fold_addr_expr (ref);
1953 addr = save_expr (addr);
1954 ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr);
1955 x = fold_build2 (code, TREE_TYPE (ref), ref, new_var);
1956 x = build2 (OMP_ATOMIC, void_type_node, addr, x);
1957 gimplify_and_add (x, stmt_list);
1961 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
1963 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
1965 if (is_reference (var))
1966 ref = build_fold_addr_expr (ref);
1967 SET_DECL_VALUE_EXPR (placeholder, ref);
1968 DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
1969 gimplify_and_add (OMP_CLAUSE_REDUCTION_MERGE (c), &sub_list);
1970 OMP_CLAUSE_REDUCTION_MERGE (c) = NULL;
1971 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
1975 x = build2 (code, TREE_TYPE (ref), ref, new_var);
1976 ref = build_outer_var_ref (var, ctx);
1977 x = build_gimple_modify_stmt (ref, x);
1978 append_to_statement_list (x, &sub_list);
1982 x = build_call_expr (built_in_decls[BUILT_IN_GOMP_ATOMIC_START], 0);
1983 gimplify_and_add (x, stmt_list);
1985 gimplify_and_add (sub_list, stmt_list);
1987 x = build_call_expr (built_in_decls[BUILT_IN_GOMP_ATOMIC_END], 0);
1988 gimplify_and_add (x, stmt_list);
1992 /* Generate code to implement the COPYPRIVATE clauses. */
1995 lower_copyprivate_clauses (tree clauses, tree *slist, tree *rlist,
2000 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
2005 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE)
2008 var = OMP_CLAUSE_DECL (c);
2009 by_ref = use_pointer_for_field (var, false);
2011 ref = build_sender_ref (var, ctx);
2012 x = lookup_decl_in_outer_ctx (var, ctx);
2013 x = by_ref ? build_fold_addr_expr (x) : x;
2014 x = build_gimple_modify_stmt (ref, x);
2015 gimplify_and_add (x, slist);
2017 ref = build_receiver_ref (var, by_ref, ctx);
2018 if (is_reference (var))
2020 ref = build_fold_indirect_ref (ref);
2021 var = build_fold_indirect_ref (var);
2023 x = lang_hooks.decls.omp_clause_assign_op (c, var, ref);
2024 gimplify_and_add (x, rlist);
2029 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
2030 and REDUCTION from the sender (aka parent) side. */
2033 lower_send_clauses (tree clauses, tree *ilist, tree *olist, omp_context *ctx)
2037 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
2039 tree val, ref, x, var;
2040 bool by_ref, do_in = false, do_out = false;
2042 switch (OMP_CLAUSE_CODE (c))
2044 case OMP_CLAUSE_FIRSTPRIVATE:
2045 case OMP_CLAUSE_COPYIN:
2046 case OMP_CLAUSE_LASTPRIVATE:
2047 case OMP_CLAUSE_REDUCTION:
2053 val = OMP_CLAUSE_DECL (c);
2054 var = lookup_decl_in_outer_ctx (val, ctx);
2056 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN
2057 && is_global_var (var))
2059 if (is_variable_sized (val))
2061 by_ref = use_pointer_for_field (val, false);
2063 switch (OMP_CLAUSE_CODE (c))
2065 case OMP_CLAUSE_FIRSTPRIVATE:
2066 case OMP_CLAUSE_COPYIN:
2070 case OMP_CLAUSE_LASTPRIVATE:
2071 if (by_ref || is_reference (val))
2073 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
2081 case OMP_CLAUSE_REDUCTION:
2083 do_out = !(by_ref || is_reference (val));
2092 ref = build_sender_ref (val, ctx);
2093 x = by_ref ? build_fold_addr_expr (var) : var;
2094 x = build_gimple_modify_stmt (ref, x);
2095 gimplify_and_add (x, ilist);
2100 ref = build_sender_ref (val, ctx);
2101 x = build_gimple_modify_stmt (var, ref);
2102 gimplify_and_add (x, olist);
2107 /* Generate code to implement SHARED from the sender (aka parent) side.
2108 This is trickier, since OMP_PARALLEL_CLAUSES doesn't list things that
2109 got automatically shared. */
2112 lower_send_shared_vars (tree *ilist, tree *olist, omp_context *ctx)
2114 tree var, ovar, nvar, f, x;
2116 if (ctx->record_type == NULL)
2119 for (f = TYPE_FIELDS (ctx->record_type); f ; f = TREE_CHAIN (f))
2121 ovar = DECL_ABSTRACT_ORIGIN (f);
2122 nvar = maybe_lookup_decl (ovar, ctx);
2123 if (!nvar || !DECL_HAS_VALUE_EXPR_P (nvar))
2126 /* If CTX is a nested parallel directive. Find the immediately
2127 enclosing parallel or workshare construct that contains a
2128 mapping for OVAR. */
2129 var = lookup_decl_in_outer_ctx (ovar, ctx);
2131 if (use_pointer_for_field (ovar, true))
2133 x = build_sender_ref (ovar, ctx);
2134 var = build_fold_addr_expr (var);
2135 x = build_gimple_modify_stmt (x, var);
2136 gimplify_and_add (x, ilist);
2140 x = build_sender_ref (ovar, ctx);
2141 x = build_gimple_modify_stmt (x, var);
2142 gimplify_and_add (x, ilist);
2144 x = build_sender_ref (ovar, ctx);
2145 x = build_gimple_modify_stmt (var, x);
2146 gimplify_and_add (x, olist);
2151 /* Build the function calls to GOMP_parallel_start etc to actually
2152 generate the parallel operation. REGION is the parallel region
2153 being expanded. BB is the block where to insert the code. WS_ARGS
2154 will be set if this is a call to a combined parallel+workshare
2155 construct, it contains the list of additional arguments needed by
2156 the workshare construct. */
2159 expand_parallel_call (struct omp_region *region, basic_block bb,
2160 tree entry_stmt, tree ws_args)
2162 tree t, t1, t2, val, cond, c, clauses;
2163 block_stmt_iterator si;
2166 clauses = OMP_PARALLEL_CLAUSES (entry_stmt);
2168 /* Determine what flavor of GOMP_parallel_start we will be
2170 start_ix = BUILT_IN_GOMP_PARALLEL_START;
2171 if (is_combined_parallel (region))
2173 switch (region->inner->type)
2176 start_ix = BUILT_IN_GOMP_PARALLEL_LOOP_STATIC_START
2177 + region->inner->sched_kind;
2180 start_ix = BUILT_IN_GOMP_PARALLEL_SECTIONS_START;
2187 /* By default, the value of NUM_THREADS is zero (selected at run time)
2188 and there is no conditional. */
2190 val = build_int_cst (unsigned_type_node, 0);
2192 c = find_omp_clause (clauses, OMP_CLAUSE_IF);
2194 cond = OMP_CLAUSE_IF_EXPR (c);
2196 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_THREADS);
2198 val = OMP_CLAUSE_NUM_THREADS_EXPR (c);
2200 /* Ensure 'val' is of the correct type. */
2201 val = fold_convert (unsigned_type_node, val);
2203 /* If we found the clause 'if (cond)', build either
2204 (cond != 0) or (cond ? val : 1u). */
2207 block_stmt_iterator si;
2209 cond = gimple_boolify (cond);
2211 if (integer_zerop (val))
2212 val = fold_build2 (EQ_EXPR, unsigned_type_node, cond,
2213 build_int_cst (TREE_TYPE (cond), 0));
2216 basic_block cond_bb, then_bb, else_bb;
2217 edge e, e_then, e_else;
2218 tree t, tmp_then, tmp_else, tmp_join, tmp_var;
2220 tmp_var = create_tmp_var (TREE_TYPE (val), NULL);
2221 if (gimple_in_ssa_p (cfun))
2223 tmp_then = make_ssa_name (tmp_var, NULL_TREE);
2224 tmp_else = make_ssa_name (tmp_var, NULL_TREE);
2225 tmp_join = make_ssa_name (tmp_var, NULL_TREE);
2234 e = split_block (bb, NULL);
2239 then_bb = create_empty_bb (cond_bb);
2240 else_bb = create_empty_bb (then_bb);
2241 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
2242 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb);
2244 t = build3 (COND_EXPR, void_type_node,
2245 cond, NULL_TREE, NULL_TREE);
2247 si = bsi_start (cond_bb);
2248 bsi_insert_after (&si, t, BSI_CONTINUE_LINKING);
2250 si = bsi_start (then_bb);
2251 t = build_gimple_modify_stmt (tmp_then, val);
2252 if (gimple_in_ssa_p (cfun))
2253 SSA_NAME_DEF_STMT (tmp_then) = t;
2254 bsi_insert_after (&si, t, BSI_CONTINUE_LINKING);
2256 si = bsi_start (else_bb);
2257 t = build_gimple_modify_stmt (tmp_else,
2258 build_int_cst (unsigned_type_node, 1));
2259 if (gimple_in_ssa_p (cfun))
2260 SSA_NAME_DEF_STMT (tmp_else) = t;
2261 bsi_insert_after (&si, t, BSI_CONTINUE_LINKING);
2263 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
2264 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE);
2265 e_then = make_edge (then_bb, bb, EDGE_FALLTHRU);
2266 e_else = make_edge (else_bb, bb, EDGE_FALLTHRU);
2268 if (gimple_in_ssa_p (cfun))
2270 tree phi = create_phi_node (tmp_join, bb);
2271 SSA_NAME_DEF_STMT (tmp_join) = phi;
2272 add_phi_arg (phi, tmp_then, e_then);
2273 add_phi_arg (phi, tmp_else, e_else);
2279 si = bsi_start (bb);
2280 val = force_gimple_operand_bsi (&si, val, true, NULL_TREE,
2281 false, BSI_CONTINUE_LINKING);
2285 t = OMP_PARALLEL_DATA_ARG (entry_stmt);
2287 t1 = null_pointer_node;
2289 t1 = build_fold_addr_expr (t);
2290 t2 = build_fold_addr_expr (OMP_PARALLEL_FN (entry_stmt));
2294 tree args = tree_cons (NULL, t2,
2295 tree_cons (NULL, t1,
2296 tree_cons (NULL, val, ws_args)));
2297 t = build_function_call_expr (built_in_decls[start_ix], args);
2300 t = build_call_expr (built_in_decls[start_ix], 3, t2, t1, val);
2302 force_gimple_operand_bsi (&si, t, true, NULL_TREE,
2303 false, BSI_CONTINUE_LINKING);
2305 t = OMP_PARALLEL_DATA_ARG (entry_stmt);
2307 t = null_pointer_node;
2309 t = build_fold_addr_expr (t);
2310 t = build_call_expr (OMP_PARALLEL_FN (entry_stmt), 1, t);
2311 force_gimple_operand_bsi (&si, t, true, NULL_TREE,
2312 false, BSI_CONTINUE_LINKING);
2314 t = build_call_expr (built_in_decls[BUILT_IN_GOMP_PARALLEL_END], 0);
2315 force_gimple_operand_bsi (&si, t, true, NULL_TREE,
2316 false, BSI_CONTINUE_LINKING);
2320 /* If exceptions are enabled, wrap *STMT_P in a MUST_NOT_THROW catch
2321 handler. This prevents programs from violating the structured
2322 block semantics with throws. */
2325 maybe_catch_exception (tree *stmt_p)
2329 if (!flag_exceptions)
2332 if (lang_protect_cleanup_actions)
2333 t = lang_protect_cleanup_actions ();
2335 t = build_call_expr (built_in_decls[BUILT_IN_TRAP], 0);
2336 f = build2 (EH_FILTER_EXPR, void_type_node, NULL, NULL);
2337 EH_FILTER_MUST_NOT_THROW (f) = 1;
2338 gimplify_and_add (t, &EH_FILTER_FAILURE (f));
2340 t = build2 (TRY_CATCH_EXPR, void_type_node, *stmt_p, NULL);
2341 append_to_statement_list (f, &TREE_OPERAND (t, 1));
2344 append_to_statement_list (t, stmt_p);
2347 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
2350 list2chain (tree list)
2354 for (t = list; t; t = TREE_CHAIN (t))
2356 tree var = TREE_VALUE (t);
2358 TREE_CHAIN (var) = TREE_VALUE (TREE_CHAIN (t));
2360 TREE_CHAIN (var) = NULL_TREE;
2363 return list ? TREE_VALUE (list) : NULL_TREE;
2367 /* Remove barriers in REGION->EXIT's block. Note that this is only
2368 valid for OMP_PARALLEL regions. Since the end of a parallel region
2369 is an implicit barrier, any workshare inside the OMP_PARALLEL that
2370 left a barrier at the end of the OMP_PARALLEL region can now be
2374 remove_exit_barrier (struct omp_region *region)
2376 block_stmt_iterator si;
2377 basic_block exit_bb;
2382 exit_bb = region->exit;
2384 /* If the parallel region doesn't return, we don't have REGION->EXIT
2389 /* The last insn in the block will be the parallel's OMP_RETURN. The
2390 workshare's OMP_RETURN will be in a preceding block. The kinds of
2391 statements that can appear in between are extremely limited -- no
2392 memory operations at all. Here, we allow nothing at all, so the
2393 only thing we allow to precede this OMP_RETURN is a label. */
2394 si = bsi_last (exit_bb);
2395 gcc_assert (TREE_CODE (bsi_stmt (si)) == OMP_RETURN);
2397 if (!bsi_end_p (si) && TREE_CODE (bsi_stmt (si)) != LABEL_EXPR)
2400 FOR_EACH_EDGE (e, ei, exit_bb->preds)
2402 si = bsi_last (e->src);
2406 if (TREE_CODE (t) == OMP_RETURN)
2407 OMP_RETURN_NOWAIT (t) = 1;
2412 remove_exit_barriers (struct omp_region *region)
2414 if (region->type == OMP_PARALLEL)
2415 remove_exit_barrier (region);
2419 region = region->inner;
2420 remove_exit_barriers (region);
2421 while (region->next)
2423 region = region->next;
2424 remove_exit_barriers (region);
2429 /* Expand the OpenMP parallel directive starting at REGION. */
2432 expand_omp_parallel (struct omp_region *region)
2434 basic_block entry_bb, exit_bb, new_bb;
2435 struct function *child_cfun;
2436 tree child_fn, block, t, ws_args;
2437 block_stmt_iterator si;
2441 entry_stmt = last_stmt (region->entry);
2442 child_fn = OMP_PARALLEL_FN (entry_stmt);
2443 child_cfun = DECL_STRUCT_FUNCTION (child_fn);
2445 entry_bb = region->entry;
2446 exit_bb = region->exit;
2448 if (is_combined_parallel (region))
2449 ws_args = region->ws_args;
2451 ws_args = NULL_TREE;
2453 if (child_cfun->cfg)
2455 /* Due to inlining, it may happen that we have already outlined
2456 the region, in which case all we need to do is make the
2457 sub-graph unreachable and emit the parallel call. */
2458 edge entry_succ_e, exit_succ_e;
2459 block_stmt_iterator si;
2461 entry_succ_e = single_succ_edge (entry_bb);
2463 si = bsi_last (entry_bb);
2464 gcc_assert (TREE_CODE (bsi_stmt (si)) == OMP_PARALLEL);
2465 bsi_remove (&si, true);
2470 exit_succ_e = single_succ_edge (exit_bb);
2471 make_edge (new_bb, exit_succ_e->dest, EDGE_FALLTHRU);
2473 remove_edge_and_dominated_blocks (entry_succ_e);
2477 /* If the parallel region needs data sent from the parent
2478 function, then the very first statement (except possible
2479 tree profile counter updates) of the parallel body
2480 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
2481 &.OMP_DATA_O is passed as an argument to the child function,
2482 we need to replace it with the argument as seen by the child
2485 In most cases, this will end up being the identity assignment
2486 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
2487 a function call that has been inlined, the original PARM_DECL
2488 .OMP_DATA_I may have been converted into a different local
2489 variable. In which case, we need to keep the assignment. */
2490 if (OMP_PARALLEL_DATA_ARG (entry_stmt))
2492 basic_block entry_succ_bb = single_succ (entry_bb);
2493 block_stmt_iterator si;
2494 tree parcopy_stmt = NULL_TREE, arg, narg;
2496 for (si = bsi_start (entry_succ_bb); ; bsi_next (&si))
2500 gcc_assert (!bsi_end_p (si));
2501 stmt = bsi_stmt (si);
2502 if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
2505 arg = GIMPLE_STMT_OPERAND (stmt, 1);
2507 if (TREE_CODE (arg) == ADDR_EXPR
2508 && TREE_OPERAND (arg, 0)
2509 == OMP_PARALLEL_DATA_ARG (entry_stmt))
2511 parcopy_stmt = stmt;
2516 gcc_assert (parcopy_stmt != NULL_TREE);
2517 arg = DECL_ARGUMENTS (child_fn);
2519 if (!gimple_in_ssa_p (cfun))
2521 if (GIMPLE_STMT_OPERAND (parcopy_stmt, 0) == arg)
2522 bsi_remove (&si, true);
2524 GIMPLE_STMT_OPERAND (parcopy_stmt, 1) = arg;
2528 /* If we are in ssa form, we must load the value from the default
2529 definition of the argument. That should not be defined now,
2530 since the argument is not used uninitialized. */
2531 gcc_assert (gimple_default_def (cfun, arg) == NULL);
2532 narg = make_ssa_name (arg, build_empty_stmt ());
2533 set_default_def (arg, narg);
2534 GIMPLE_STMT_OPERAND (parcopy_stmt, 1) = narg;
2535 update_stmt (parcopy_stmt);
2539 /* Declare local variables needed in CHILD_CFUN. */
2540 block = DECL_INITIAL (child_fn);
2541 BLOCK_VARS (block) = list2chain (child_cfun->unexpanded_var_list);
2542 DECL_SAVED_TREE (child_fn) = bb_stmt_list (single_succ (entry_bb));
2544 /* Reset DECL_CONTEXT on function arguments. */
2545 for (t = DECL_ARGUMENTS (child_fn); t; t = TREE_CHAIN (t))
2546 DECL_CONTEXT (t) = child_fn;
2548 /* Split ENTRY_BB at OMP_PARALLEL so that it can be moved to the
2550 si = bsi_last (entry_bb);
2552 gcc_assert (t && TREE_CODE (t) == OMP_PARALLEL);
2553 bsi_remove (&si, true);
2554 e = split_block (entry_bb, t);
2556 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
2558 /* Convert OMP_RETURN into a RETURN_EXPR. */
2561 si = bsi_last (exit_bb);
2562 gcc_assert (!bsi_end_p (si)
2563 && TREE_CODE (bsi_stmt (si)) == OMP_RETURN);
2564 t = build1 (RETURN_EXPR, void_type_node, NULL);
2565 bsi_insert_after (&si, t, BSI_SAME_STMT);
2566 bsi_remove (&si, true);
2569 /* Move the parallel region into CHILD_CFUN. */
2571 if (gimple_in_ssa_p (cfun))
2573 push_cfun (child_cfun);
2575 init_ssa_operands ();
2576 cfun->gimple_df->in_ssa_p = true;
2579 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb);
2581 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU;
2583 /* Inform the callgraph about the new function. */
2584 DECL_STRUCT_FUNCTION (child_fn)->curr_properties
2585 = cfun->curr_properties;
2586 cgraph_add_new_function (child_fn, true);
2588 /* Fix the callgraph edges for child_cfun. Those for cfun will be
2589 fixed in a following pass. */
2590 push_cfun (child_cfun);
2591 rebuild_cgraph_edges ();
2595 /* Emit a library call to launch the children threads. */
2596 expand_parallel_call (region, new_bb, entry_stmt, ws_args);
2597 update_ssa (TODO_update_ssa_only_virtuals);
2601 /* A subroutine of expand_omp_for. Generate code for a parallel
2602 loop with any schedule. Given parameters:
2604 for (V = N1; V cond N2; V += STEP) BODY;
2606 where COND is "<" or ">", we generate pseudocode
2608 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
2609 if (more) goto L0; else goto L3;
2616 if (V cond iend) goto L1; else goto L2;
2618 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
2621 If this is a combined omp parallel loop, instead of the call to
2622 GOMP_loop_foo_start, we call GOMP_loop_foo_next. */
2625 expand_omp_for_generic (struct omp_region *region,
2626 struct omp_for_data *fd,
2627 enum built_in_function start_fn,
2628 enum built_in_function next_fn)
2630 tree type, istart0, iend0, iend, phi;
2631 tree t, vmain, vback;
2632 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb;
2633 basic_block l2_bb = NULL, l3_bb = NULL;
2634 block_stmt_iterator si;
2635 bool in_combined_parallel = is_combined_parallel (region);
2636 bool broken_loop = region->cont == NULL;
2639 gcc_assert (!broken_loop || !in_combined_parallel);
2641 type = TREE_TYPE (fd->v);
2643 istart0 = create_tmp_var (long_integer_type_node, ".istart0");
2644 iend0 = create_tmp_var (long_integer_type_node, ".iend0");
2645 TREE_ADDRESSABLE (istart0) = 1;
2646 TREE_ADDRESSABLE (iend0) = 1;
2647 if (gimple_in_ssa_p (cfun))
2649 add_referenced_var (istart0);
2650 add_referenced_var (iend0);
2653 entry_bb = region->entry;
2654 cont_bb = region->cont;
2655 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
2656 gcc_assert (broken_loop
2657 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
2658 l0_bb = split_edge (FALLTHRU_EDGE (entry_bb));
2659 l1_bb = single_succ (l0_bb);
2662 l2_bb = create_empty_bb (cont_bb);
2663 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l1_bb);
2664 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
2668 l3_bb = BRANCH_EDGE (entry_bb)->dest;
2669 exit_bb = region->exit;
2671 si = bsi_last (entry_bb);
2672 gcc_assert (TREE_CODE (bsi_stmt (si)) == OMP_FOR);
2673 if (in_combined_parallel)
2675 /* In a combined parallel loop, emit a call to
2676 GOMP_loop_foo_next. */
2677 t = build_call_expr (built_in_decls[next_fn], 2,
2678 build_fold_addr_expr (istart0),
2679 build_fold_addr_expr (iend0));
2683 tree t0, t1, t2, t3, t4;
2684 /* If this is not a combined parallel loop, emit a call to
2685 GOMP_loop_foo_start in ENTRY_BB. */
2686 t4 = build_fold_addr_expr (iend0);
2687 t3 = build_fold_addr_expr (istart0);
2688 t2 = fold_convert (long_integer_type_node, fd->step);
2689 t1 = fold_convert (long_integer_type_node, fd->n2);
2690 t0 = fold_convert (long_integer_type_node, fd->n1);
2693 t = fold_convert (long_integer_type_node, fd->chunk_size);
2694 t = build_call_expr (built_in_decls[start_fn], 6,
2695 t0, t1, t2, t, t3, t4);
2698 t = build_call_expr (built_in_decls[start_fn], 5,
2699 t0, t1, t2, t3, t4);
2701 t = force_gimple_operand_bsi (&si, t, true, NULL_TREE,
2702 true, BSI_SAME_STMT);
2703 t = build3 (COND_EXPR, void_type_node, t, NULL_TREE, NULL_TREE);
2704 bsi_insert_after (&si, t, BSI_SAME_STMT);
2706 /* V may be used outside of the loop (e.g., to handle lastprivate clause).
2707 If this is the case, its value is undefined if the loop is not entered
2708 at all. To handle this case, set its initial value to N1. */
2709 if (gimple_in_ssa_p (cfun))
2711 e = find_edge (entry_bb, l3_bb);
2712 for (phi = phi_nodes (l3_bb); phi; phi = PHI_CHAIN (phi))
2713 if (PHI_ARG_DEF_FROM_EDGE (phi, e) == fd->v)
2714 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e), fd->n1);
2718 t = build_gimple_modify_stmt (fd->v, fd->n1);
2719 bsi_insert_before (&si, t, BSI_SAME_STMT);
2722 /* Remove the OMP_FOR statement. */
2723 bsi_remove (&si, true);
2725 /* Iteration setup for sequential loop goes in L0_BB. */
2726 si = bsi_start (l0_bb);
2727 t = fold_convert (type, istart0);
2728 t = force_gimple_operand_bsi (&si, t, false, NULL_TREE,
2729 false, BSI_CONTINUE_LINKING);
2730 t = build_gimple_modify_stmt (fd->v, t);
2731 bsi_insert_after (&si, t, BSI_CONTINUE_LINKING);
2732 if (gimple_in_ssa_p (cfun))
2733 SSA_NAME_DEF_STMT (fd->v) = t;
2735 t = fold_convert (type, iend0);
2736 iend = force_gimple_operand_bsi (&si, t, true, NULL_TREE,
2737 false, BSI_CONTINUE_LINKING);
2741 /* Code to control the increment and predicate for the sequential
2742 loop goes in the CONT_BB. */
2743 si = bsi_last (cont_bb);
2745 gcc_assert (TREE_CODE (t) == OMP_CONTINUE);
2746 vmain = TREE_OPERAND (t, 1);
2747 vback = TREE_OPERAND (t, 0);
2749 t = fold_build2 (PLUS_EXPR, type, vmain, fd->step);
2750 t = force_gimple_operand_bsi (&si, t, false, NULL_TREE,
2751 true, BSI_SAME_STMT);
2752 t = build_gimple_modify_stmt (vback, t);
2753 bsi_insert_before (&si, t, BSI_SAME_STMT);
2754 if (gimple_in_ssa_p (cfun))
2755 SSA_NAME_DEF_STMT (vback) = t;
2757 t = build2 (fd->cond_code, boolean_type_node, vback, iend);
2758 t = build3 (COND_EXPR, void_type_node, t, NULL_TREE, NULL_TREE);
2759 bsi_insert_before (&si, t, BSI_SAME_STMT);
2761 /* Remove OMP_CONTINUE. */
2762 bsi_remove (&si, true);
2764 /* Emit code to get the next parallel iteration in L2_BB. */
2765 si = bsi_start (l2_bb);
2767 t = build_call_expr (built_in_decls[next_fn], 2,
2768 build_fold_addr_expr (istart0),
2769 build_fold_addr_expr (iend0));
2770 t = force_gimple_operand_bsi (&si, t, true, NULL_TREE,
2771 false, BSI_CONTINUE_LINKING);
2772 t = build3 (COND_EXPR, void_type_node, t, NULL_TREE, NULL_TREE);
2773 bsi_insert_after (&si, t, BSI_CONTINUE_LINKING);
2776 /* Add the loop cleanup function. */
2777 si = bsi_last (exit_bb);
2778 if (OMP_RETURN_NOWAIT (bsi_stmt (si)))
2779 t = built_in_decls[BUILT_IN_GOMP_LOOP_END_NOWAIT];
2781 t = built_in_decls[BUILT_IN_GOMP_LOOP_END];
2782 t = build_call_expr (t, 0);
2783 bsi_insert_after (&si, t, BSI_SAME_STMT);
2784 bsi_remove (&si, true);
2786 /* Connect the new blocks. */
2787 find_edge (entry_bb, l0_bb)->flags = EDGE_TRUE_VALUE;
2788 find_edge (entry_bb, l3_bb)->flags = EDGE_FALSE_VALUE;
2792 e = find_edge (cont_bb, l3_bb);
2793 ne = make_edge (l2_bb, l3_bb, EDGE_FALSE_VALUE);
2795 for (phi = phi_nodes (l3_bb); phi; phi = PHI_CHAIN (phi))
2796 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, ne),
2797 PHI_ARG_DEF_FROM_EDGE (phi, e));
2800 find_edge (cont_bb, l1_bb)->flags = EDGE_TRUE_VALUE;
2801 make_edge (cont_bb, l2_bb, EDGE_FALSE_VALUE);
2802 make_edge (l2_bb, l0_bb, EDGE_TRUE_VALUE);
2804 set_immediate_dominator (CDI_DOMINATORS, l2_bb,
2805 recompute_dominator (CDI_DOMINATORS, l2_bb));
2806 set_immediate_dominator (CDI_DOMINATORS, l3_bb,
2807 recompute_dominator (CDI_DOMINATORS, l3_bb));
2808 set_immediate_dominator (CDI_DOMINATORS, l0_bb,
2809 recompute_dominator (CDI_DOMINATORS, l0_bb));
2810 set_immediate_dominator (CDI_DOMINATORS, l1_bb,
2811 recompute_dominator (CDI_DOMINATORS, l1_bb));
2816 /* A subroutine of expand_omp_for. Generate code for a parallel
2817 loop with static schedule and no specified chunk size. Given
2820 for (V = N1; V cond N2; V += STEP) BODY;
2822 where COND is "<" or ">", we generate pseudocode
2828 n = (adj + N2 - N1) / STEP;
2830 q += (q * nthreads != n);
2832 e0 = min(s0 + q, n);
2834 if (s0 >= e0) goto L2; else goto L0;
2840 if (V cond e) goto L1;
2845 expand_omp_for_static_nochunk (struct omp_region *region,
2846 struct omp_for_data *fd)
2848 tree n, q, s0, e0, e, t, nthreads, threadid;
2849 tree type, vmain, vback;
2850 basic_block entry_bb, exit_bb, seq_start_bb, body_bb, cont_bb;
2852 block_stmt_iterator si;
2854 type = TREE_TYPE (fd->v);
2856 entry_bb = region->entry;
2857 cont_bb = region->cont;
2858 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2);
2859 gcc_assert (BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest);
2860 seq_start_bb = split_edge (FALLTHRU_EDGE (entry_bb));
2861 body_bb = single_succ (seq_start_bb);
2862 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
2863 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
2864 fin_bb = FALLTHRU_EDGE (cont_bb)->dest;
2865 exit_bb = region->exit;
2867 /* Iteration space partitioning goes in ENTRY_BB. */
2868 si = bsi_last (entry_bb);
2869 gcc_assert (TREE_CODE (bsi_stmt (si)) == OMP_FOR);
2871 t = build_call_expr (built_in_decls[BUILT_IN_OMP_GET_NUM_THREADS], 0);
2872 t = fold_convert (type, t);
2873 nthreads = force_gimple_operand_bsi (&si, t, true, NULL_TREE,
2874 true, BSI_SAME_STMT);
2876 t = build_call_expr (built_in_decls[BUILT_IN_OMP_GET_THREAD_NUM], 0);
2877 t = fold_convert (type, t);
2878 threadid = force_gimple_operand_bsi (&si, t, true, NULL_TREE,
2879 true, BSI_SAME_STMT);
2881 fd->n1 = force_gimple_operand_bsi (&si,
2882 fold_convert (type, fd->n1),
2884 true, BSI_SAME_STMT);
2886 fd->n2 = force_gimple_operand_bsi (&si,
2887 fold_convert (type, fd->n2),
2889 true, BSI_SAME_STMT);
2891 fd->step = force_gimple_operand_bsi (&si,
2892 fold_convert (type, fd->step),
2894 true, BSI_SAME_STMT);
2896 t = build_int_cst (type, (fd->cond_code == LT_EXPR ? -1 : 1));
2897 t = fold_build2 (PLUS_EXPR, type, fd->step, t);
2898 t = fold_build2 (PLUS_EXPR, type, t, fd->n2);
2899 t = fold_build2 (MINUS_EXPR, type, t, fd->n1);
2900 t = fold_build2 (TRUNC_DIV_EXPR, type, t, fd->step);
2901 t = fold_convert (type, t);
2902 n = force_gimple_operand_bsi (&si, t, true, NULL_TREE, true, BSI_SAME_STMT);
2904 t = fold_build2 (TRUNC_DIV_EXPR, type, n, nthreads);
2905 q = force_gimple_operand_bsi (&si, t, true, NULL_TREE, true, BSI_SAME_STMT);
2907 t = fold_build2 (MULT_EXPR, type, q, nthreads);
2908 t = fold_build2 (NE_EXPR, type, t, n);
2909 t = fold_build2 (PLUS_EXPR, type, q, t);
2910 q = force_gimple_operand_bsi (&si, t, true, NULL_TREE, true, BSI_SAME_STMT);
2912 t = build2 (MULT_EXPR, type, q, threadid);
2913 s0 = force_gimple_operand_bsi (&si, t, true, NULL_TREE, true, BSI_SAME_STMT);
2915 t = fold_build2 (PLUS_EXPR, type, s0, q);
2916 t = fold_build2 (MIN_EXPR, type, t, n);
2917 e0 = force_gimple_operand_bsi (&si, t, true, NULL_TREE, true, BSI_SAME_STMT);
2919 t = fold_convert (type, s0);
2920 t = fold_build2 (MULT_EXPR, type, t, fd->step);
2921 t = fold_build2 (PLUS_EXPR, type, t, fd->n1);
2922 t = force_gimple_operand_bsi (&si, t, false, NULL_TREE,
2923 true, BSI_SAME_STMT);
2924 t = build_gimple_modify_stmt (fd->v, t);
2925 bsi_insert_before (&si, t, BSI_SAME_STMT);
2926 if (gimple_in_ssa_p (cfun))
2927 SSA_NAME_DEF_STMT (fd->v) = t;
2929 t = build2 (GE_EXPR, boolean_type_node, s0, e0);
2930 t = build3 (COND_EXPR, void_type_node, t, NULL_TREE, NULL_TREE);
2931 bsi_insert_before (&si, t, BSI_SAME_STMT);
2933 /* Remove the OMP_FOR statement. */
2934 bsi_remove (&si, true);
2936 /* Setup code for sequential iteration goes in SEQ_START_BB. */
2937 si = bsi_start (seq_start_bb);
2939 t = fold_convert (type, e0);
2940 t = fold_build2 (MULT_EXPR, type, t, fd->step);
2941 t = fold_build2 (PLUS_EXPR, type, t, fd->n1);
2942 e = force_gimple_operand_bsi (&si, t, true, NULL_TREE,
2943 false, BSI_CONTINUE_LINKING);
2945 /* The code controlling the sequential loop replaces the OMP_CONTINUE. */
2946 si = bsi_last (cont_bb);
2948 gcc_assert (TREE_CODE (t) == OMP_CONTINUE);
2949 vmain = TREE_OPERAND (t, 1);
2950 vback = TREE_OPERAND (t, 0);
2952 t = fold_build2 (PLUS_EXPR, type, vmain, fd->step);
2953 t = force_gimple_operand_bsi (&si, t, false, NULL_TREE,
2954 true, BSI_SAME_STMT);
2955 t = build_gimple_modify_stmt (vback, t);
2956 bsi_insert_before (&si, t, BSI_SAME_STMT);
2957 if (gimple_in_ssa_p (cfun))
2958 SSA_NAME_DEF_STMT (vback) = t;
2960 t = build2 (fd->cond_code, boolean_type_node, vback, e);
2961 t = build3 (COND_EXPR, void_type_node, t, NULL_TREE, NULL_TREE);
2962 bsi_insert_before (&si, t, BSI_SAME_STMT);
2964 /* Remove the OMP_CONTINUE statement. */
2965 bsi_remove (&si, true);
2967 /* Replace the OMP_RETURN with a barrier, or nothing. */
2968 si = bsi_last (exit_bb);
2969 if (!OMP_RETURN_NOWAIT (bsi_stmt (si)))
2970 force_gimple_operand_bsi (&si, build_omp_barrier (), false, NULL_TREE,
2971 false, BSI_SAME_STMT);
2972 bsi_remove (&si, true);
2974 /* Connect all the blocks. */
2975 find_edge (entry_bb, seq_start_bb)->flags = EDGE_FALSE_VALUE;
2976 find_edge (entry_bb, fin_bb)->flags = EDGE_TRUE_VALUE;
2978 find_edge (cont_bb, body_bb)->flags = EDGE_TRUE_VALUE;
2979 find_edge (cont_bb, fin_bb)->flags = EDGE_FALSE_VALUE;
2981 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, entry_bb);
2982 set_immediate_dominator (CDI_DOMINATORS, body_bb,
2983 recompute_dominator (CDI_DOMINATORS, body_bb));
2984 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
2985 recompute_dominator (CDI_DOMINATORS, fin_bb));
2989 /* A subroutine of expand_omp_for. Generate code for a parallel
2990 loop with static schedule and a specified chunk size. Given
2993 for (V = N1; V cond N2; V += STEP) BODY;
2995 where COND is "<" or ">", we generate pseudocode
3001 n = (adj + N2 - N1) / STEP;
3003 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
3004 here so that V is defined
3005 if the loop is not entered
3007 s0 = (trip * nthreads + threadid) * CHUNK;
3008 e0 = min(s0 + CHUNK, n);
3009 if (s0 < n) goto L1; else goto L4;
3016 if (V cond e) goto L2; else goto L3;
3024 expand_omp_for_static_chunk (struct omp_region *region, struct omp_for_data *fd)
3026 tree n, s0, e0, e, t, phi, nphi, args;
3027 tree trip_var, trip_init, trip_main, trip_back, nthreads, threadid;
3028 tree type, cont, v_main, v_back, v_extra;
3029 basic_block entry_bb, exit_bb, body_bb, seq_start_bb, iter_part_bb;
3030 basic_block trip_update_bb, cont_bb, fin_bb;
3031 block_stmt_iterator si;
3034 type = TREE_TYPE (fd->v);
3036 entry_bb = region->entry;
3037 se = split_block (entry_bb, last_stmt (entry_bb));
3039 iter_part_bb = se->dest;
3040 cont_bb = region->cont;
3041 gcc_assert (EDGE_COUNT (iter_part_bb->succs) == 2);
3042 gcc_assert (BRANCH_EDGE (iter_part_bb)->dest
3043 == FALLTHRU_EDGE (cont_bb)->dest);
3044 seq_start_bb = split_edge (FALLTHRU_EDGE (iter_part_bb));
3045 body_bb = single_succ (seq_start_bb);
3046 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb);
3047 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2);
3048 fin_bb = FALLTHRU_EDGE (cont_bb)->dest;
3049 trip_update_bb = split_edge (FALLTHRU_EDGE (cont_bb));
3050 exit_bb = region->exit;
3052 /* Trip and adjustment setup goes in ENTRY_BB. */
3053 si = bsi_last (entry_bb);
3054 gcc_assert (TREE_CODE (bsi_stmt (si)) == OMP_FOR);
3056 t = build_call_expr (built_in_decls[BUILT_IN_OMP_GET_NUM_THREADS], 0);
3057 t = fold_convert (type, t);
3058 nthreads = force_gimple_operand_bsi (&si, t, true, NULL_TREE,
3059 true, BSI_SAME_STMT);
3061 t = build_call_expr (built_in_decls[BUILT_IN_OMP_GET_THREAD_NUM], 0);
3062 t = fold_convert (type, t);
3063 threadid = force_gimple_operand_bsi (&si, t, true, NULL_TREE,
3064 true, BSI_SAME_STMT);
3066 fd->n1 = force_gimple_operand_bsi (&si, fold_convert (type, fd->n1),
3068 true, BSI_SAME_STMT);
3069 fd->n2 = force_gimple_operand_bsi (&si, fold_convert (type, fd->n2),
3071 true, BSI_SAME_STMT);
3072 fd->step = force_gimple_operand_bsi (&si, fold_convert (type, fd->step),
3074 true, BSI_SAME_STMT);
3076 = force_gimple_operand_bsi (&si, fold_convert (type,
3079 true, BSI_SAME_STMT);
3081 t = build_int_cst (type, (fd->cond_code == LT_EXPR ? -1 : 1));
3082 t = fold_build2 (PLUS_EXPR, type, fd->step, t);
3083 t = fold_build2 (PLUS_EXPR, type, t, fd->n2);
3084 t = fold_build2 (MINUS_EXPR, type, t, fd->n1);
3085 t = fold_build2 (TRUNC_DIV_EXPR, type, t, fd->step);
3086 t = fold_convert (type, t);
3087 n = force_gimple_operand_bsi (&si, t, true, NULL_TREE,
3088 true, BSI_SAME_STMT);
3090 trip_var = create_tmp_var (type, ".trip");
3091 if (gimple_in_ssa_p (cfun))
3093 add_referenced_var (trip_var);
3094 trip_init = make_ssa_name (trip_var, NULL_TREE);
3095 trip_main = make_ssa_name (trip_var, NULL_TREE);
3096 trip_back = make_ssa_name (trip_var, NULL_TREE);
3100 trip_init = trip_var;
3101 trip_main = trip_var;
3102 trip_back = trip_var;
3105 t = build_gimple_modify_stmt (trip_init, build_int_cst (type, 0));
3106 bsi_insert_before (&si, t, BSI_SAME_STMT);
3107 if (gimple_in_ssa_p (cfun))
3108 SSA_NAME_DEF_STMT (trip_init) = t;
3110 t = fold_build2 (MULT_EXPR, type, threadid, fd->chunk_size);
3111 t = fold_build2 (MULT_EXPR, type, t, fd->step);
3112 t = fold_build2 (PLUS_EXPR, type, t, fd->n1);
3113 v_extra = force_gimple_operand_bsi (&si, t, true, NULL_TREE,
3114 true, BSI_SAME_STMT);
3116 /* Remove the OMP_FOR. */
3117 bsi_remove (&si, true);
3119 /* Iteration space partitioning goes in ITER_PART_BB. */
3120 si = bsi_last (iter_part_bb);
3122 t = fold_build2 (MULT_EXPR, type, trip_main, nthreads);
3123 t = fold_build2 (PLUS_EXPR, type, t, threadid);
3124 t = fold_build2 (MULT_EXPR, type, t, fd->chunk_size);
3125 s0 = force_gimple_operand_bsi (&si, t, true, NULL_TREE,
3126 false, BSI_CONTINUE_LINKING);
3128 t = fold_build2 (PLUS_EXPR, type, s0, fd->chunk_size);
3129 t = fold_build2 (MIN_EXPR, type, t, n);
3130 e0 = force_gimple_operand_bsi (&si, t, true, NULL_TREE,
3131 false, BSI_CONTINUE_LINKING);
3133 t = build2 (LT_EXPR, boolean_type_node, s0, n);
3134 t = build3 (COND_EXPR, void_type_node, t, NULL_TREE, NULL_TREE);
3135 bsi_insert_after (&si, t, BSI_CONTINUE_LINKING);
3137 /* Setup code for sequential iteration goes in SEQ_START_BB. */
3138 si = bsi_start (seq_start_bb);
3140 t = fold_convert (type, s0);
3141 t = fold_build2 (MULT_EXPR, type, t, fd->step);
3142 t = fold_build2 (PLUS_EXPR, type, t, fd->n1);
3143 t = force_gimple_operand_bsi (&si, t, false, NULL_TREE,
3144 false, BSI_CONTINUE_LINKING);
3145 t = build_gimple_modify_stmt (fd->v, t);
3146 bsi_insert_after (&si, t, BSI_CONTINUE_LINKING);
3147 if (gimple_in_ssa_p (cfun))
3148 SSA_NAME_DEF_STMT (fd->v) = t;
3150 t = fold_convert (type, e0);
3151 t = fold_build2 (MULT_EXPR, type, t, fd->step);
3152 t = fold_build2 (PLUS_EXPR, type, t, fd->n1);
3153 e = force_gimple_operand_bsi (&si, t, true, NULL_TREE,
3154 false, BSI_CONTINUE_LINKING);
3156 /* The code controlling the sequential loop goes in CONT_BB,
3157 replacing the OMP_CONTINUE. */
3158 si = bsi_last (cont_bb);
3159 cont = bsi_stmt (si);
3160 gcc_assert (TREE_CODE (cont) == OMP_CONTINUE);
3161 v_main = TREE_OPERAND (cont, 1);
3162 v_back = TREE_OPERAND (cont, 0);
3164 t = build2 (PLUS_EXPR, type, v_main, fd->step);
3165 t = build_gimple_modify_stmt (v_back, t);
3166 bsi_insert_before (&si, t, BSI_SAME_STMT);
3167 if (gimple_in_ssa_p (cfun))
3168 SSA_NAME_DEF_STMT (v_back) = t;
3170 t = build2 (fd->cond_code, boolean_type_node, v_back, e);
3171 t = build3 (COND_EXPR, void_type_node, t, NULL_TREE, NULL_TREE);
3172 bsi_insert_before (&si, t, BSI_SAME_STMT);
3174 /* Remove OMP_CONTINUE. */
3175 bsi_remove (&si, true);
3177 /* Trip update code goes into TRIP_UPDATE_BB. */
3178 si = bsi_start (trip_update_bb);
3180 t = build_int_cst (type, 1);
3181 t = build2 (PLUS_EXPR, type, trip_main, t);
3182 t = build_gimple_modify_stmt (trip_back, t);
3183 bsi_insert_after (&si, t, BSI_CONTINUE_LINKING);
3184 if (gimple_in_ssa_p (cfun))
3185 SSA_NAME_DEF_STMT (trip_back) = t;
3187 /* Replace the OMP_RETURN with a barrier, or nothing. */
3188 si = bsi_last (exit_bb);
3189 if (!OMP_RETURN_NOWAIT (bsi_stmt (si)))
3190 force_gimple_operand_bsi (&si, build_omp_barrier (), false, NULL_TREE,
3191 false, BSI_SAME_STMT);
3192 bsi_remove (&si, true);
3194 /* Connect the new blocks. */
3195 find_edge (iter_part_bb, seq_start_bb)->flags = EDGE_TRUE_VALUE;
3196 find_edge (iter_part_bb, fin_bb)->flags = EDGE_FALSE_VALUE;
3198 find_edge (cont_bb, body_bb)->flags = EDGE_TRUE_VALUE;
3199 find_edge (cont_bb, trip_update_bb)->flags = EDGE_FALSE_VALUE;
3201 redirect_edge_and_branch (single_succ_edge (trip_update_bb), iter_part_bb);
3203 if (gimple_in_ssa_p (cfun))
3205 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
3206 remove arguments of the phi nodes in fin_bb. We need to create
3207 appropriate phi nodes in iter_part_bb instead. */
3208 se = single_pred_edge (fin_bb);
3209 re = single_succ_edge (trip_update_bb);
3210 ene = single_succ_edge (entry_bb);
3212 args = PENDING_STMT (re);
3213 PENDING_STMT (re) = NULL_TREE;
3214 for (phi = phi_nodes (fin_bb);
3216 phi = PHI_CHAIN (phi), args = TREE_CHAIN (args))
3218 t = PHI_RESULT (phi);
3219 gcc_assert (t == TREE_PURPOSE (args));
3220 nphi = create_phi_node (t, iter_part_bb);
3221 SSA_NAME_DEF_STMT (t) = nphi;
3223 t = PHI_ARG_DEF_FROM_EDGE (phi, se);
3224 /* A special case -- fd->v is not yet computed in iter_part_bb, we
3225 need to use v_extra instead. */
3228 add_phi_arg (nphi, t, ene);
3229 add_phi_arg (nphi, TREE_VALUE (args), re);
3231 gcc_assert (!phi && !args);
3232 while ((phi = phi_nodes (fin_bb)) != NULL_TREE)
3233 remove_phi_node (phi, NULL_TREE, false);
3235 /* Make phi node for trip. */
3236 phi = create_phi_node (trip_main, iter_part_bb);
3237 SSA_NAME_DEF_STMT (trip_main) = phi;
3238 add_phi_arg (phi, trip_back, single_succ_edge (trip_update_bb));
3239 add_phi_arg (phi, trip_init, single_succ_edge (entry_bb));
3242 set_immediate_dominator (CDI_DOMINATORS, trip_update_bb, cont_bb);
3243 set_immediate_dominator (CDI_DOMINATORS, iter_part_bb,
3244 recompute_dominator (CDI_DOMINATORS, iter_part_bb));
3245 set_immediate_dominator (CDI_DOMINATORS, fin_bb,
3246 recompute_dominator (CDI_DOMINATORS, fin_bb));
3247 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb,
3248 recompute_dominator (CDI_DOMINATORS, seq_start_bb));
3249 set_immediate_dominator (CDI_DOMINATORS, body_bb,
3250 recompute_dominator (CDI_DOMINATORS, body_bb));
3254 /* Expand the OpenMP loop defined by REGION. */
3257 expand_omp_for (struct omp_region *region)
3259 struct omp_for_data fd;
3261 extract_omp_for_data (last_stmt (region->entry), &fd);
3262 region->sched_kind = fd.sched_kind;
3264 if (fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC
3266 && region->cont != NULL)
3268 if (fd.chunk_size == NULL)
3269 expand_omp_for_static_nochunk (region, &fd);
3271 expand_omp_for_static_chunk (region, &fd);
3275 int fn_index = fd.sched_kind + fd.have_ordered * 4;
3276 int start_ix = BUILT_IN_GOMP_LOOP_STATIC_START + fn_index;
3277 int next_ix = BUILT_IN_GOMP_LOOP_STATIC_NEXT + fn_index;
3278 expand_omp_for_generic (region, &fd, start_ix, next_ix);
3281 update_ssa (TODO_update_ssa_only_virtuals);
3285 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
3287 v = GOMP_sections_start (n);
3304 v = GOMP_sections_next ();
3309 If this is a combined parallel sections, replace the call to
3310 GOMP_sections_start with call to GOMP_sections_next. */
3313 expand_omp_sections (struct omp_region *region)
3315 tree label_vec, l1, l2, t, u, sections_stmt, vin, vmain, vnext, cont;
3316 unsigned i, casei, len;
3317 basic_block entry_bb, l0_bb, l1_bb, l2_bb, default_bb;
3318 block_stmt_iterator si;
3319 struct omp_region *inner;
3320 bool exit_reachable = region->cont != NULL;
3322 gcc_assert (exit_reachable == (region->exit != NULL));
3323 entry_bb = region->entry;
3324 l0_bb = single_succ (entry_bb);
3325 l1_bb = region->cont;
3326 l2_bb = region->exit;
3329 gcc_assert (single_pred (l2_bb) == l0_bb);
3330 default_bb = create_empty_bb (l1_bb->prev_bb);
3331 l1 = tree_block_label (l1_bb);
3332 l2 = tree_block_label (l2_bb);
3336 default_bb = create_empty_bb (l0_bb);
3338 l2 = tree_block_label (default_bb);
3341 /* We will build a switch() with enough cases for all the
3342 OMP_SECTION regions, a '0' case to handle the end of more work
3343 and a default case to abort if something goes wrong. */
3344 len = EDGE_COUNT (l0_bb->succs);
3345 label_vec = make_tree_vec (len + 1);
3347 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
3348 OMP_SECTIONS statement. */
3349 si = bsi_last (entry_bb);
3350 sections_stmt = bsi_stmt (si);
3351 gcc_assert (TREE_CODE (sections_stmt) == OMP_SECTIONS);
3352 vin = OMP_SECTIONS_CONTROL (sections_stmt);
3353 if (!is_combined_parallel (region))
3355 /* If we are not inside a combined parallel+sections region,
3356 call GOMP_sections_start. */
3357 t = build_int_cst (unsigned_type_node,
3358 exit_reachable ? len - 1 : len);
3359 u = built_in_decls[BUILT_IN_GOMP_SECTIONS_START];
3360 t = build_call_expr (u, 1, t);
3364 /* Otherwise, call GOMP_sections_next. */
3365 u = built_in_decls[BUILT_IN_GOMP_SECTIONS_NEXT];
3366 t = build_call_expr (u, 0);
3368 t = build_gimple_modify_stmt (vin, t);
3369 bsi_insert_after (&si, t, BSI_SAME_STMT);
3370 if (gimple_in_ssa_p (cfun))
3371 SSA_NAME_DEF_STMT (vin) = t;
3372 bsi_remove (&si, true);
3374 /* The switch() statement replacing OMP_SECTIONS_SWITCH goes in L0_BB. */
3375 si = bsi_last (l0_bb);
3376 gcc_assert (TREE_CODE (bsi_stmt (si)) == OMP_SECTIONS_SWITCH);
3379 cont = last_stmt (l1_bb);
3380 gcc_assert (TREE_CODE (cont) == OMP_CONTINUE);
3381 vmain = TREE_OPERAND (cont, 1);
3382 vnext = TREE_OPERAND (cont, 0);
3390 t = build3 (SWITCH_EXPR, void_type_node, vmain, NULL, label_vec);
3391 bsi_insert_after (&si, t, BSI_SAME_STMT);
3392 bsi_remove (&si, true);
3397 t = build3 (CASE_LABEL_EXPR, void_type_node,
3398 build_int_cst (unsigned_type_node, 0), NULL, l2);
3399 TREE_VEC_ELT (label_vec, 0) = t;
3403 /* Convert each OMP_SECTION into a CASE_LABEL_EXPR. */
3404 for (inner = region->inner, casei = 1;
3406 inner = inner->next, i++, casei++)
3408 basic_block s_entry_bb, s_exit_bb;
3410 s_entry_bb = inner->entry;
3411 s_exit_bb = inner->exit;
3413 t = tree_block_label (s_entry_bb);
3414 u = build_int_cst (unsigned_type_node, casei);
3415 u = build3 (CASE_LABEL_EXPR, void_type_node, u, NULL, t);
3416 TREE_VEC_ELT (label_vec, i) = u;
3418 si = bsi_last (s_entry_bb);
3419 gcc_assert (TREE_CODE (bsi_stmt (si)) == OMP_SECTION);
3420 gcc_assert (i < len || OMP_SECTION_LAST (bsi_stmt (si)));
3421 bsi_remove (&si, true);
3422 single_succ_edge (s_entry_bb)->flags = EDGE_FALLTHRU;
3424 if (s_exit_bb == NULL)
3427 si = bsi_last (s_exit_bb);
3428 gcc_assert (TREE_CODE (bsi_stmt (si)) == OMP_RETURN);
3429 bsi_remove (&si, true);
3431 single_succ_edge (s_exit_bb)->flags = EDGE_FALLTHRU;
3434 /* Error handling code goes in DEFAULT_BB. */
3435 t = tree_block_label (default_bb);
3436 u = build3 (CASE_LABEL_EXPR, void_type_node, NULL, NULL, t);
3437 TREE_VEC_ELT (label_vec, len) = u;
3438 make_edge (l0_bb, default_bb, 0);
3440 si = bsi_start (default_bb);
3441 t = build_call_expr (built_in_decls[BUILT_IN_TRAP], 0);
3442 bsi_insert_after (&si, t, BSI_CONTINUE_LINKING);
3446 /* Code to get the next section goes in L1_BB. */
3447 si = bsi_last (l1_bb);
3448 gcc_assert (TREE_CODE (bsi_stmt (si)) == OMP_CONTINUE);
3450 t = build_call_expr (built_in_decls[BUILT_IN_GOMP_SECTIONS_NEXT], 0);
3451 t = build_gimple_modify_stmt (vnext, t);
3452 bsi_insert_after (&si, t, BSI_SAME_STMT);
3453 if (gimple_in_ssa_p (cfun))
3454 SSA_NAME_DEF_STMT (vnext) = t;
3455 bsi_remove (&si, true);
3457 single_succ_edge (l1_bb)->flags = EDGE_FALLTHRU;
3459 /* Cleanup function replaces OMP_RETURN in EXIT_BB. */
3460 si = bsi_last (l2_bb);
3461 if (OMP_RETURN_NOWAIT (bsi_stmt (si)))
3462 t = built_in_decls[BUILT_IN_GOMP_SECTIONS_END_NOWAIT];
3464 t = built_in_decls[BUILT_IN_GOMP_SECTIONS_END];
3465 t = build_call_expr (t, 0);
3466 bsi_insert_after (&si, t, BSI_SAME_STMT);
3467 bsi_remove (&si, true);
3470 set_immediate_dominator (CDI_DOMINATORS, default_bb, l0_bb);
3474 /* Expand code for an OpenMP single directive. We've already expanded
3475 much of the code, here we simply place the GOMP_barrier call. */
3478 expand_omp_single (struct omp_region *region)
3480 basic_block entry_bb, exit_bb;
3481 block_stmt_iterator si;
3482 bool need_barrier = false;
3484 entry_bb = region->entry;
3485 exit_bb = region->exit;
3487 si = bsi_last (entry_bb);
3488 /* The terminal barrier at the end of a GOMP_single_copy sequence cannot
3489 be removed. We need to ensure that the thread that entered the single
3490 does not exit before the data is copied out by the other threads. */
3491 if (find_omp_clause (OMP_SINGLE_CLAUSES (bsi_stmt (si)),
3492 OMP_CLAUSE_COPYPRIVATE))
3493 need_barrier = true;
3494 gcc_assert (TREE_CODE (bsi_stmt (si)) == OMP_SINGLE);
3495 bsi_remove (&si, true);
3496 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
3498 si = bsi_last (exit_bb);
3499 if (!OMP_RETURN_NOWAIT (bsi_stmt (si)) || need_barrier)
3500 force_gimple_operand_bsi (&si, build_omp_barrier (), false, NULL_TREE,
3501 false, BSI_SAME_STMT);
3502 bsi_remove (&si, true);
3503 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
3507 /* Generic expansion for OpenMP synchronization directives: master,
3508 ordered and critical. All we need to do here is remove the entry
3509 and exit markers for REGION. */
3512 expand_omp_synch (struct omp_region *region)
3514 basic_block entry_bb, exit_bb;
3515 block_stmt_iterator si;
3517 entry_bb = region->entry;
3518 exit_bb = region->exit;
3520 si = bsi_last (entry_bb);
3521 gcc_assert (TREE_CODE (bsi_stmt (si)) == OMP_SINGLE
3522 || TREE_CODE (bsi_stmt (si)) == OMP_MASTER
3523 || TREE_CODE (bsi_stmt (si)) == OMP_ORDERED
3524 || TREE_CODE (bsi_stmt (si)) == OMP_CRITICAL);
3525 bsi_remove (&si, true);
3526 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU;
3530 si = bsi_last (exit_bb);
3531 gcc_assert (TREE_CODE (bsi_stmt (si)) == OMP_RETURN);
3532 bsi_remove (&si, true);
3533 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU;
3537 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
3538 operation as a __sync_fetch_and_op builtin. INDEX is log2 of the
3539 size of the data type, and thus usable to find the index of the builtin
3540 decl. Returns false if the expression is not of the proper form. */
3543 expand_omp_atomic_fetch_op (basic_block load_bb,
3544 tree addr, tree loaded_val,
3545 tree stored_val, int index)
3547 enum built_in_function base;
3548 tree decl, itype, call;
3549 enum insn_code *optab;
3551 basic_block store_bb = single_succ (load_bb);
3552 block_stmt_iterator bsi;
3555 /* We expect to find the following sequences:
3558 OMP_ATOMIC_LOAD (tmp, mem)
3561 val = tmp OP something; (or: something OP tmp)
3564 ???FIXME: Allow a more flexible sequence.
3565 Perhaps use data flow to pick the statements.
3569 bsi = bsi_after_labels (store_bb);
3570 stmt = bsi_stmt (bsi);
3571 if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
3574 if (TREE_CODE (bsi_stmt (bsi)) != OMP_ATOMIC_STORE)
3577 if (!operand_equal_p (GIMPLE_STMT_OPERAND (stmt, 0), stored_val, 0))
3580 rhs = GIMPLE_STMT_OPERAND (stmt, 1);
3582 /* Check for one of the supported fetch-op operations. */
3583 switch (TREE_CODE (rhs))
3586 case POINTER_PLUS_EXPR:
3587 base = BUILT_IN_FETCH_AND_ADD_N;
3588 optab = sync_add_optab;
3591 base = BUILT_IN_FETCH_AND_SUB_N;
3592 optab = sync_add_optab;
3595 base = BUILT_IN_FETCH_AND_AND_N;
3596 optab = sync_and_optab;
3599 base = BUILT_IN_FETCH_AND_OR_N;
3600 optab = sync_ior_optab;
3603 base = BUILT_IN_FETCH_AND_XOR_N;
3604 optab = sync_xor_optab;
3609 /* Make sure the expression is of the proper form. */
3610 if (operand_equal_p (TREE_OPERAND (rhs, 0), loaded_val, 0))
3611 rhs = TREE_OPERAND (rhs, 1);
3612 else if (commutative_tree_code (TREE_CODE (rhs))
3613 && operand_equal_p (TREE_OPERAND (rhs, 1), loaded_val, 0))
3614 rhs = TREE_OPERAND (rhs, 0);
3618 decl = built_in_decls[base + index + 1];
3619 itype = TREE_TYPE (TREE_TYPE (decl));
3621 if (optab[TYPE_MODE (itype)] == CODE_FOR_nothing)
3624 bsi = bsi_last (load_bb);
3625 gcc_assert (TREE_CODE (bsi_stmt (bsi)) == OMP_ATOMIC_LOAD);
3626 call = build_call_expr (decl, 2, addr, fold_convert (itype, rhs));
3627 force_gimple_operand_bsi (&bsi, call, true, NULL_TREE, true, BSI_SAME_STMT);
3628 bsi_remove (&bsi, true);
3630 bsi = bsi_last (store_bb);
3631 gcc_assert (TREE_CODE (bsi_stmt (bsi)) == OMP_ATOMIC_STORE);
3632 bsi_remove (&bsi, true);
3633 bsi = bsi_last (store_bb);
3634 bsi_remove (&bsi, true);
3636 if (gimple_in_ssa_p (cfun))
3637 update_ssa (TODO_update_ssa_no_phi);
3642 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
3646 newval = rhs; // with oldval replacing *addr in rhs
3647 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
3648 if (oldval != newval)
3651 INDEX is log2 of the size of the data type, and thus usable to find the
3652 index of the builtin decl. */
3655 expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb,
3656 tree addr, tree loaded_val, tree stored_val,
3659 tree loadedi, storedi, initial, new_stored, new_storedi, old_vali;
3660 tree type, itype, cmpxchg, iaddr;
3661 block_stmt_iterator bsi;
3662 basic_block loop_header = single_succ (load_bb);
3666 cmpxchg = built_in_decls[BUILT_IN_VAL_COMPARE_AND_SWAP_N + index + 1];
3667 type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
3668 itype = TREE_TYPE (TREE_TYPE (cmpxchg));
3670 if (sync_compare_and_swap[TYPE_MODE (itype)] == CODE_FOR_nothing)
3673 /* Load the initial value, replacing the OMP_ATOMIC_LOAD. */
3674 bsi = bsi_last (load_bb);
3675 gcc_assert (TREE_CODE (bsi_stmt (bsi)) == OMP_ATOMIC_LOAD);
3676 initial = force_gimple_operand_bsi (&bsi, build_fold_indirect_ref (addr),
3677 true, NULL_TREE, true, BSI_SAME_STMT);
3678 /* Move the value to the LOADED_VAL temporary. */
3679 if (gimple_in_ssa_p (cfun))
3681 gcc_assert (phi_nodes (loop_header) == NULL_TREE);
3682 phi = create_phi_node (loaded_val, loop_header);
3683 SSA_NAME_DEF_STMT (loaded_val) = phi;
3684 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, single_succ_edge (load_bb)),
3688 bsi_insert_before (&bsi,
3689 build_gimple_modify_stmt (loaded_val, initial),
3691 bsi_remove (&bsi, true);
3693 bsi = bsi_last (store_bb);
3694 gcc_assert (TREE_CODE (bsi_stmt (bsi)) == OMP_ATOMIC_STORE);
3696 /* For floating-point values, we'll need to view-convert them to integers
3697 so that we can perform the atomic compare and swap. Simplify the
3698 following code by always setting up the "i"ntegral variables. */
3699 if (INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type))
3701 loadedi = loaded_val;
3702 storedi = stored_val;
3707 loadedi = force_gimple_operand_bsi (&bsi,
3708 build1 (VIEW_CONVERT_EXPR, itype,
3710 NULL_TREE, true, BSI_SAME_STMT);
3712 force_gimple_operand_bsi (&bsi,
3713 build1 (VIEW_CONVERT_EXPR, itype,
3714 stored_val), true, NULL_TREE, true,
3716 iaddr = fold_convert (build_pointer_type (itype), addr);
3719 /* Build the compare&swap statement. */
3720 new_storedi = build_call_expr (cmpxchg, 3, iaddr, loadedi, storedi);
3721 new_storedi = force_gimple_operand_bsi (&bsi,
3722 fold_convert (itype, new_storedi),
3724 true, BSI_SAME_STMT);
3725 if (storedi == stored_val)
3726 new_stored = new_storedi;
3728 new_stored = force_gimple_operand_bsi (&bsi,
3729 build1 (VIEW_CONVERT_EXPR, type,
3731 NULL_TREE, true, BSI_SAME_STMT);
3733 if (gimple_in_ssa_p (cfun))
3737 old_vali = create_tmp_var (itype, NULL);
3738 x = build_gimple_modify_stmt (old_vali, loadedi);
3739 bsi_insert_before (&bsi, x, BSI_SAME_STMT);
3741 x = build_gimple_modify_stmt (loaded_val, new_stored);
3742 bsi_insert_before (&bsi, x, BSI_SAME_STMT);
3745 /* Note that we always perform the comparison as an integer, even for
3746 floating point. This allows the atomic operation to properly
3747 succeed even with NaNs and -0.0. */
3748 x = build3 (COND_EXPR, void_type_node,
3749 build2 (NE_EXPR, boolean_type_node,
3750 new_storedi, old_vali), NULL_TREE, NULL_TREE);
3751 bsi_insert_before (&bsi, x, BSI_SAME_STMT);
3754 e = single_succ_edge (store_bb);
3755 e->flags &= ~EDGE_FALLTHRU;
3756 e->flags |= EDGE_FALSE_VALUE;
3758 e = make_edge (store_bb, loop_header, EDGE_TRUE_VALUE);
3760 /* Copy the new value to loaded_val (we already did that before the condition
3761 if we are not in SSA). */
3762 if (gimple_in_ssa_p (cfun))
3764 phi = phi_nodes (loop_header);
3765 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e), new_stored);
3768 /* Remove OMP_ATOMIC_STORE. */
3769 bsi_remove (&bsi, true);
3771 if (gimple_in_ssa_p (cfun))
3772 update_ssa (TODO_update_ssa_no_phi);
3777 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
3779 GOMP_atomic_start ();
3783 The result is not globally atomic, but works so long as all parallel
3784 references are within #pragma omp atomic directives. According to
3785 responses received from omp@openmp.org, appears to be within spec.
3786 Which makes sense, since that's how several other compilers handle
3787 this situation as well.
3788 LOADED_VAL and ADDR are the operands of OMP_ATOMIC_LOAD we're expanding.
3789 STORED_VAL is the operand of the matching OMP_ATOMIC_STORE.
3792 OMP_ATOMIC_LOAD (loaded_val, addr) with
3796 OMP_ATOMIC_ATORE (stored_val) with
3801 expand_omp_atomic_mutex (basic_block load_bb, basic_block store_bb,
3802 tree addr, tree loaded_val, tree stored_val)
3804 block_stmt_iterator bsi;
3807 bsi = bsi_last (load_bb);
3808 gcc_assert (TREE_CODE (bsi_stmt (bsi)) == OMP_ATOMIC_LOAD);
3810 t = built_in_decls[BUILT_IN_GOMP_ATOMIC_START];
3811 t = build_function_call_expr (t, 0);
3812 force_gimple_operand_bsi (&bsi, t, true, NULL_TREE, true, BSI_SAME_STMT);
3814 t = build_gimple_modify_stmt (loaded_val, build_fold_indirect_ref (addr));
3815 if (gimple_in_ssa_p (cfun))
3816 SSA_NAME_DEF_STMT (loaded_val) = t;
3817 bsi_insert_before (&bsi, t, BSI_SAME_STMT);
3818 bsi_remove (&bsi, true);
3820 bsi = bsi_last (store_bb);
3821 gcc_assert (TREE_CODE (bsi_stmt (bsi)) == OMP_ATOMIC_STORE);
3823 t = build_gimple_modify_stmt (build_fold_indirect_ref (unshare_expr (addr)),
3825 bsi_insert_before (&bsi, t, BSI_SAME_STMT);
3827 t = built_in_decls[BUILT_IN_GOMP_ATOMIC_END];
3828 t = build_function_call_expr (t, 0);
3829 force_gimple_operand_bsi (&bsi, t, true, NULL_TREE, true, BSI_SAME_STMT);
3830 bsi_remove (&bsi, true);
3832 if (gimple_in_ssa_p (cfun))
3833 update_ssa (TODO_update_ssa_no_phi);
3837 /* Expand an OMP_ATOMIC statement. We try to expand
3838 using expand_omp_atomic_fetch_op. If it failed, we try to
3839 call expand_omp_atomic_pipeline, and if it fails too, the
3840 ultimate fallback is wrapping the operation in a mutex
3841 (expand_omp_atomic_mutex). REGION is the atomic region built
3842 by build_omp_regions_1(). */
3845 expand_omp_atomic (struct omp_region *region)
3847 basic_block load_bb = region->entry, store_bb = region->exit;
3848 tree load = last_stmt (load_bb), store = last_stmt (store_bb);
3849 tree loaded_val = TREE_OPERAND (load, 0);
3850 tree addr = TREE_OPERAND (load, 1);
3851 tree stored_val = TREE_OPERAND (store, 0);
3852 tree type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr)));
3853 HOST_WIDE_INT index;
3855 /* Make sure the type is one of the supported sizes. */
3856 index = tree_low_cst (TYPE_SIZE_UNIT (type), 1);
3857 index = exact_log2 (index);
3858 if (index >= 0 && index <= 4)
3860 unsigned int align = TYPE_ALIGN_UNIT (type);
3862 /* __sync builtins require strict data alignment. */
3863 if (exact_log2 (align) >= index)
3865 /* When possible, use specialized atomic update functions. */
3866 if ((INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type))
3867 && store_bb == single_succ (load_bb))
3869 if (expand_omp_atomic_fetch_op (load_bb, addr,
3870 loaded_val, stored_val, index))
3874 /* If we don't have specialized __sync builtins, try and implement
3875 as a compare and swap loop. */
3876 if (expand_omp_atomic_pipeline (load_bb, store_bb, addr,
3877 loaded_val, stored_val, index))
3882 /* The ultimate fallback is wrapping the operation in a mutex. */
3883 expand_omp_atomic_mutex (load_bb, store_bb, addr, loaded_val, stored_val);
3887 /* Expand the parallel region tree rooted at REGION. Expansion
3888 proceeds in depth-first order. Innermost regions are expanded
3889 first. This way, parallel regions that require a new function to
3890 be created (e.g., OMP_PARALLEL) can be expanded without having any
3891 internal dependencies in their body. */
3894 expand_omp (struct omp_region *region)
3899 expand_omp (region->inner);
3901 switch (region->type)
3904 expand_omp_parallel (region);
3908 expand_omp_for (region);
3912 expand_omp_sections (region);
3916 /* Individual omp sections are handled together with their
3917 parent OMP_SECTIONS region. */
3921 expand_omp_single (region);
3927 expand_omp_synch (region);
3930 case OMP_ATOMIC_LOAD:
3931 expand_omp_atomic (region);
3939 region = region->next;
3944 /* Helper for build_omp_regions. Scan the dominator tree starting at
3945 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
3946 true, the function ends once a single tree is built (otherwise, whole
3947 forest of OMP constructs may be built). */
3950 build_omp_regions_1 (basic_block bb, struct omp_region *parent,
3953 block_stmt_iterator si;
3958 if (!bsi_end_p (si) && OMP_DIRECTIVE_P (bsi_stmt (si)))
3960 struct omp_region *region;
3961 enum tree_code code;
3963 stmt = bsi_stmt (si);
3964 code = TREE_CODE (stmt);
3965 if (code == OMP_RETURN)
3967 /* STMT is the return point out of region PARENT. Mark it
3968 as the exit point and make PARENT the immediately
3969 enclosing region. */
3970 gcc_assert (parent);
3973 parent = parent->outer;
3975 /* If REGION is a parallel region, determine whether it is
3976 a combined parallel+workshare region. */
3977 if (region->type == OMP_PARALLEL)
3978 determine_parallel_type (region);
3980 else if (code == OMP_ATOMIC_STORE)
3982 /* OMP_ATOMIC_STORE is analoguous to OMP_RETURN, but matches with
3984 gcc_assert (parent);
3985 gcc_assert (parent->type == OMP_ATOMIC_LOAD);
3988 parent = parent->outer;
3991 else if (code == OMP_CONTINUE)
3993 gcc_assert (parent);
3996 else if (code == OMP_SECTIONS_SWITCH)
3998 /* OMP_SECTIONS_SWITCH is part of OMP_SECTIONS, and we do nothing for
4003 /* Otherwise, this directive becomes the parent for a new
4005 region = new_omp_region (bb, code, parent);
4010 if (single_tree && !parent)
4013 for (son = first_dom_son (CDI_DOMINATORS, bb);
4015 son = next_dom_son (CDI_DOMINATORS, son))
4016 build_omp_regions_1 (son, parent, single_tree);
4019 /* Builds the tree of OMP regions rooted at ROOT, storing it to
4023 build_omp_regions_root (basic_block root)
4025 gcc_assert (root_omp_region == NULL);
4026 build_omp_regions_1 (root, NULL, true);
4027 gcc_assert (root_omp_region != NULL);
4030 /* Expands omp construct (and its subconstructs) starting in HEAD. */
4033 omp_expand_local (basic_block head)
4035 build_omp_regions_root (head);
4036 if (dump_file && (dump_flags & TDF_DETAILS))
4038 fprintf (dump_file, "\nOMP region tree\n\n");
4039 dump_omp_region (dump_file, root_omp_region, 0);
4040 fprintf (dump_file, "\n");
4043 remove_exit_barriers (root_omp_region);
4044 expand_omp (root_omp_region);
4046 free_omp_regions ();
4049 /* Scan the CFG and build a tree of OMP regions. Return the root of
4050 the OMP region tree. */
4053 build_omp_regions (void)
4055 gcc_assert (root_omp_region == NULL);
4056 calculate_dominance_info (CDI_DOMINATORS);
4057 build_omp_regions_1 (ENTRY_BLOCK_PTR, NULL, false);
4061 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
4064 execute_expand_omp (void)
4066 build_omp_regions ();
4068 if (!root_omp_region)
4073 fprintf (dump_file, "\nOMP region tree\n\n");
4074 dump_omp_region (dump_file, root_omp_region, 0);
4075 fprintf (dump_file, "\n");
4078 remove_exit_barriers (root_omp_region);
4080 expand_omp (root_omp_region);
4082 cleanup_tree_cfg ();
4084 free_omp_regions ();
4089 /* OMP expansion in SSA form. For testing purposes only. */
4092 gate_expand_omp_ssa (void)
4094 return flag_openmp_ssa && flag_openmp != 0 && errorcount == 0;
4097 struct tree_opt_pass pass_expand_omp_ssa =
4099 "ompexpssa", /* name */
4100 gate_expand_omp_ssa, /* gate */
4101 execute_expand_omp, /* execute */
4104 0, /* static_pass_number */
4106 PROP_gimple_any, /* properties_required */
4107 PROP_gimple_lomp, /* properties_provided */
4108 0, /* properties_destroyed */
4109 0, /* todo_flags_start */
4110 TODO_dump_func, /* todo_flags_finish */
4114 /* OMP expansion -- the default pass, run before creation of SSA form. */
4117 gate_expand_omp (void)
4119 return ((!flag_openmp_ssa || !optimize)
4120 && flag_openmp != 0 && errorcount == 0);
4123 struct tree_opt_pass pass_expand_omp =
4125 "ompexp", /* name */
4126 gate_expand_omp, /* gate */
4127 execute_expand_omp, /* execute */
4130 0, /* static_pass_number */
4132 PROP_gimple_any, /* properties_required */
4133 PROP_gimple_lomp, /* properties_provided */
4134 0, /* properties_destroyed */
4135 0, /* todo_flags_start */
4136 TODO_dump_func, /* todo_flags_finish */
4140 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
4142 /* Lower the OpenMP sections directive in *STMT_P. */
4145 lower_omp_sections (tree *stmt_p, omp_context *ctx)
4147 tree new_stmt, stmt, body, bind, block, ilist, olist, new_body, control;
4149 tree_stmt_iterator tsi;
4154 push_gimplify_context ();
4158 lower_rec_input_clauses (OMP_SECTIONS_CLAUSES (stmt), &ilist, &dlist, ctx);
4160 tsi = tsi_start (OMP_SECTIONS_BODY (stmt));
4161 for (len = 0; !tsi_end_p (tsi); len++, tsi_next (&tsi))
4164 tsi = tsi_start (OMP_SECTIONS_BODY (stmt));
4165 body = alloc_stmt_list ();
4166 for (i = 0; i < len; i++, tsi_next (&tsi))
4169 tree sec_start, sec_end;
4171 sec_start = tsi_stmt (tsi);
4172 sctx = maybe_lookup_ctx (sec_start);
4175 append_to_statement_list (sec_start, &body);
4177 lower_omp (&OMP_SECTION_BODY (sec_start), sctx);
4178 append_to_statement_list (OMP_SECTION_BODY (sec_start), &body);
4179 OMP_SECTION_BODY (sec_start) = NULL;
4183 tree l = alloc_stmt_list ();
4184 lower_lastprivate_clauses (OMP_SECTIONS_CLAUSES (stmt), NULL,
4186 append_to_statement_list (l, &body);
4187 OMP_SECTION_LAST (sec_start) = 1;
4190 sec_end = make_node (OMP_RETURN);
4191 append_to_statement_list (sec_end, &body);
4194 block = make_node (BLOCK);
4195 bind = build3 (BIND_EXPR, void_type_node, NULL, body, block);
4198 lower_reduction_clauses (OMP_SECTIONS_CLAUSES (stmt), &olist, ctx);
4200 pop_gimplify_context (NULL_TREE);
4201 record_vars_into (ctx->block_vars, ctx->cb.dst_fn);
4203 new_stmt = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
4204 TREE_SIDE_EFFECTS (new_stmt) = 1;
4206 new_body = alloc_stmt_list ();
4207 append_to_statement_list (ilist, &new_body);
4208 append_to_statement_list (stmt, &new_body);
4209 append_to_statement_list (make_node (OMP_SECTIONS_SWITCH), &new_body);
4210 append_to_statement_list (bind, &new_body);
4212 control = create_tmp_var (unsigned_type_node, ".section");
4213 t = build2 (OMP_CONTINUE, void_type_node, control, control);
4214 OMP_SECTIONS_CONTROL (stmt) = control;
4215 append_to_statement_list (t, &new_body);
4217 append_to_statement_list (olist, &new_body);
4218 append_to_statement_list (dlist, &new_body);
4220 maybe_catch_exception (&new_body);
4222 t = make_node (OMP_RETURN);
4223 OMP_RETURN_NOWAIT (t) = !!find_omp_clause (OMP_SECTIONS_CLAUSES (stmt),
4225 append_to_statement_list (t, &new_body);
4227 BIND_EXPR_BODY (new_stmt) = new_body;
4228 OMP_SECTIONS_BODY (stmt) = NULL;
4234 /* A subroutine of lower_omp_single. Expand the simple form of
4235 an OMP_SINGLE, without a copyprivate clause:
4237 if (GOMP_single_start ())
4239 [ GOMP_barrier (); ] -> unless 'nowait' is present.
4241 FIXME. It may be better to delay expanding the logic of this until
4242 pass_expand_omp. The expanded logic may make the job more difficult
4243 to a synchronization analysis pass. */
4246 lower_omp_single_simple (tree single_stmt, tree *pre_p)
4250 t = build_call_expr (built_in_decls[BUILT_IN_GOMP_SINGLE_START], 0);
4251 t = build3 (COND_EXPR, void_type_node, t,
4252 OMP_SINGLE_BODY (single_stmt), NULL);
4253 gimplify_and_add (t, pre_p);
4257 /* A subroutine of lower_omp_single. Expand the simple form of
4258 an OMP_SINGLE, with a copyprivate clause:
4260 #pragma omp single copyprivate (a, b, c)
4262 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
4265 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
4271 GOMP_single_copy_end (©out);
4282 FIXME. It may be better to delay expanding the logic of this until
4283 pass_expand_omp. The expanded logic may make the job more difficult
4284 to a synchronization analysis pass. */
4287 lower_omp_single_copy (tree single_stmt, tree *pre_p, omp_context *ctx)
4289 tree ptr_type, t, l0, l1, l2, copyin_seq;
4291 ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o");
4293 ptr_type = build_pointer_type (ctx->record_type);
4294 ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i");
4296 l0 = create_artificial_label ();
4297 l1 = create_artificial_label ();
4298 l2 = create_artificial_label ();
4300 t = build_call_expr (built_in_decls[BUILT_IN_GOMP_SINGLE_COPY_START], 0);
4301 t = fold_convert (ptr_type, t);
4302 t = build_gimple_modify_stmt (ctx->receiver_decl, t);
4303 gimplify_and_add (t, pre_p);
4305 t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl,
4306 build_int_cst (ptr_type, 0));
4307 t = build3 (COND_EXPR, void_type_node, t,
4308 build_and_jump (&l0), build_and_jump (&l1));
4309 gimplify_and_add (t, pre_p);
4311 t = build1 (LABEL_EXPR, void_type_node, l0);
4312 gimplify_and_add (t, pre_p);
4314 append_to_statement_list (OMP_SINGLE_BODY (single_stmt), pre_p);
4317 lower_copyprivate_clauses (OMP_SINGLE_CLAUSES (single_stmt), pre_p,
4320 t = build_fold_addr_expr (ctx->sender_decl);
4321 t = build_call_expr (built_in_decls[BUILT_IN_GOMP_SINGLE_COPY_END], 1, t);
4322 gimplify_and_add (t, pre_p);
4324 t = build_and_jump (&l2);
4325 gimplify_and_add (t, pre_p);
4327 t = build1 (LABEL_EXPR, void_type_node, l1);
4328 gimplify_and_add (t, pre_p);
4330 append_to_statement_list (copyin_seq, pre_p);
4332 t = build1 (LABEL_EXPR, void_type_node, l2);
4333 gimplify_and_add (t, pre_p);
4337 /* Expand code for an OpenMP single directive. */
4340 lower_omp_single (tree *stmt_p, omp_context *ctx)
4342 tree t, bind, block, single_stmt = *stmt_p, dlist;
4344 push_gimplify_context ();
4346 block = make_node (BLOCK);
4347 *stmt_p = bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, block);
4348 TREE_SIDE_EFFECTS (bind) = 1;
4350 lower_rec_input_clauses (OMP_SINGLE_CLAUSES (single_stmt),
4351 &BIND_EXPR_BODY (bind), &dlist, ctx);
4352 lower_omp (&OMP_SINGLE_BODY (single_stmt), ctx);
4354 append_to_statement_list (single_stmt, &BIND_EXPR_BODY (bind));
4356 if (ctx->record_type)
4357 lower_omp_single_copy (single_stmt, &BIND_EXPR_BODY (bind), ctx);
4359 lower_omp_single_simple (single_stmt, &BIND_EXPR_BODY (bind));
4361 OMP_SINGLE_BODY (single_stmt) = NULL;
4363 append_to_statement_list (dlist, &BIND_EXPR_BODY (bind));
4365 maybe_catch_exception (&BIND_EXPR_BODY (bind));
4367 t = make_node (OMP_RETURN);
4368 OMP_RETURN_NOWAIT (t) = !!find_omp_clause (OMP_SINGLE_CLAUSES (single_stmt),
4370 append_to_statement_list (t, &BIND_EXPR_BODY (bind));
4372 pop_gimplify_context (bind);
4374 BIND_EXPR_VARS (bind) = chainon (BIND_EXPR_VARS (bind), ctx->block_vars);
4375 BLOCK_VARS (block) = BIND_EXPR_VARS (bind);
4379 /* Expand code for an OpenMP master directive. */
4382 lower_omp_master (tree *stmt_p, omp_context *ctx)
4384 tree bind, block, stmt = *stmt_p, lab = NULL, x;
4386 push_gimplify_context ();
4388 block = make_node (BLOCK);
4389 *stmt_p = bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, block);
4390 TREE_SIDE_EFFECTS (bind) = 1;
4392 append_to_statement_list (stmt, &BIND_EXPR_BODY (bind));
4394 x = build_call_expr (built_in_decls[BUILT_IN_OMP_GET_THREAD_NUM], 0);
4395 x = build2 (EQ_EXPR, boolean_type_node, x, integer_zero_node);
4396 x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab));
4397 gimplify_and_add (x, &BIND_EXPR_BODY (bind));
4399 lower_omp (&OMP_MASTER_BODY (stmt), ctx);
4400 maybe_catch_exception (&OMP_MASTER_BODY (stmt));
4401 append_to_statement_list (OMP_MASTER_BODY (stmt), &BIND_EXPR_BODY (bind));
4402 OMP_MASTER_BODY (stmt) = NULL;
4404 x = build1 (LABEL_EXPR, void_type_node, lab);
4405 gimplify_and_add (x, &BIND_EXPR_BODY (bind));
4407 x = make_node (OMP_RETURN);
4408 OMP_RETURN_NOWAIT (x) = 1;
4409 append_to_statement_list (x, &BIND_EXPR_BODY (bind));
4411 pop_gimplify_context (bind);
4413 BIND_EXPR_VARS (bind) = chainon (BIND_EXPR_VARS (bind), ctx->block_vars);
4414 BLOCK_VARS (block) = BIND_EXPR_VARS (bind);
4418 /* Expand code for an OpenMP ordered directive. */
4421 lower_omp_ordered (tree *stmt_p, omp_context *ctx)
4423 tree bind, block, stmt = *stmt_p, x;
4425 push_gimplify_context ();
4427 block = make_node (BLOCK);
4428 *stmt_p = bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, block);
4429 TREE_SIDE_EFFECTS (bind) = 1;
4431 append_to_statement_list (stmt, &BIND_EXPR_BODY (bind));
4433 x = build_call_expr (built_in_decls[BUILT_IN_GOMP_ORDERED_START], 0);
4434 gimplify_and_add (x, &BIND_EXPR_BODY (bind));
4436 lower_omp (&OMP_ORDERED_BODY (stmt), ctx);
4437 maybe_catch_exception (&OMP_ORDERED_BODY (stmt));
4438 append_to_statement_list (OMP_ORDERED_BODY (stmt), &BIND_EXPR_BODY (bind));
4439 OMP_ORDERED_BODY (stmt) = NULL;
4441 x = build_call_expr (built_in_decls[BUILT_IN_GOMP_ORDERED_END], 0);
4442 gimplify_and_add (x, &BIND_EXPR_BODY (bind));
4444 x = make_node (OMP_RETURN);
4445 OMP_RETURN_NOWAIT (x) = 1;
4446 append_to_statement_list (x, &BIND_EXPR_BODY (bind));
4448 pop_gimplify_context (bind);
4450 BIND_EXPR_VARS (bind) = chainon (BIND_EXPR_VARS (bind), ctx->block_vars);
4451 BLOCK_VARS (block) = BIND_EXPR_VARS (bind);
4455 /* Gimplify an OMP_CRITICAL statement. This is a relatively simple
4456 substitution of a couple of function calls. But in the NAMED case,
4457 requires that languages coordinate a symbol name. It is therefore
4458 best put here in common code. */
4460 static GTY((param1_is (tree), param2_is (tree)))
4461 splay_tree critical_name_mutexes;
4464 lower_omp_critical (tree *stmt_p, omp_context *ctx)
4466 tree bind, block, stmt = *stmt_p;
4467 tree t, lock, unlock, name;
4469 name = OMP_CRITICAL_NAME (stmt);
4475 if (!critical_name_mutexes)
4476 critical_name_mutexes
4477 = splay_tree_new_ggc (splay_tree_compare_pointers);
4479 n = splay_tree_lookup (critical_name_mutexes, (splay_tree_key) name);
4484 decl = create_tmp_var_raw (ptr_type_node, NULL);
4486 new_str = ACONCAT ((".gomp_critical_user_",
4487 IDENTIFIER_POINTER (name), NULL));
4488 DECL_NAME (decl) = get_identifier (new_str);
4489 TREE_PUBLIC (decl) = 1;
4490 TREE_STATIC (decl) = 1;
4491 DECL_COMMON (decl) = 1;
4492 DECL_ARTIFICIAL (decl) = 1;
4493 DECL_IGNORED_P (decl) = 1;
4494 varpool_finalize_decl (decl);
4496 splay_tree_insert (critical_name_mutexes, (splay_tree_key) name,
4497 (splay_tree_value) decl);
4500 decl = (tree) n->value;
4502 lock = built_in_decls[BUILT_IN_GOMP_CRITICAL_NAME_START];
4503 lock = build_call_expr (lock, 1, build_fold_addr_expr (decl));
4505 unlock = built_in_decls[BUILT_IN_GOMP_CRITICAL_NAME_END];
4506 unlock = build_call_expr (unlock, 1, build_fold_addr_expr (decl));
4510 lock = built_in_decls[BUILT_IN_GOMP_CRITICAL_START];
4511 lock = build_call_expr (lock, 0);
4513 unlock = built_in_decls[BUILT_IN_GOMP_CRITICAL_END];
4514 unlock = build_call_expr (unlock, 0);
4517 push_gimplify_context ();
4519 block = make_node (BLOCK);
4520 *stmt_p = bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, block);
4521 TREE_SIDE_EFFECTS (bind) = 1;
4523 append_to_statement_list (stmt, &BIND_EXPR_BODY (bind));
4525 gimplify_and_add (lock, &BIND_EXPR_BODY (bind));
4527 lower_omp (&OMP_CRITICAL_BODY (stmt), ctx);
4528 maybe_catch_exception (&OMP_CRITICAL_BODY (stmt));
4529 append_to_statement_list (OMP_CRITICAL_BODY (stmt), &BIND_EXPR_BODY (bind));
4530 OMP_CRITICAL_BODY (stmt) = NULL;
4532 gimplify_and_add (unlock, &BIND_EXPR_BODY (bind));
4534 t = make_node (OMP_RETURN);
4535 OMP_RETURN_NOWAIT (t) = 1;
4536 append_to_statement_list (t, &BIND_EXPR_BODY (bind));
4538 pop_gimplify_context (bind);
4539 BIND_EXPR_VARS (bind) = chainon (BIND_EXPR_VARS (bind), ctx->block_vars);
4540 BLOCK_VARS (block) = BIND_EXPR_VARS (bind);
4544 /* A subroutine of lower_omp_for. Generate code to emit the predicate
4545 for a lastprivate clause. Given a loop control predicate of (V
4546 cond N2), we gate the clause on (!(V cond N2)). The lowered form
4547 is appended to *DLIST, iterator initialization is appended to
4551 lower_omp_for_lastprivate (struct omp_for_data *fd, tree *body_p,
4552 tree *dlist, struct omp_context *ctx)
4554 tree clauses, cond, stmts, vinit, t;
4555 enum tree_code cond_code;
4557 cond_code = fd->cond_code;
4558 cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR;
4560 /* When possible, use a strict equality expression. This can let VRP
4561 type optimizations deduce the value and remove a copy. */
4562 if (host_integerp (fd->step, 0))
4564 HOST_WIDE_INT step = TREE_INT_CST_LOW (fd->step);
4565 if (step == 1 || step == -1)
4566 cond_code = EQ_EXPR;
4569 cond = build2 (cond_code, boolean_type_node, fd->v, fd->n2);
4571 clauses = OMP_FOR_CLAUSES (fd->for_stmt);
4573 lower_lastprivate_clauses (clauses, cond, &stmts, ctx);
4576 append_to_statement_list (stmts, dlist);
4578 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
4580 if (cond_code == EQ_EXPR
4581 && host_integerp (fd->n2, 0)
4582 && ! integer_zerop (fd->n2))
4583 vinit = build_int_cst (TREE_TYPE (fd->v), 0);
4585 /* Initialize the iterator variable, so that threads that don't execute
4586 any iterations don't execute the lastprivate clauses by accident. */
4587 t = build_gimple_modify_stmt (fd->v, vinit);
4588 gimplify_and_add (t, body_p);
4593 /* Lower code for an OpenMP loop directive. */
4596 lower_omp_for (tree *stmt_p, omp_context *ctx)
4598 tree t, stmt, ilist, dlist, new_stmt, *body_p, *rhs_p;
4599 struct omp_for_data fd;
4603 push_gimplify_context ();
4605 lower_omp (&OMP_FOR_PRE_BODY (stmt), ctx);
4606 lower_omp (&OMP_FOR_BODY (stmt), ctx);
4608 /* Move declaration of temporaries in the loop body before we make
4610 if (TREE_CODE (OMP_FOR_BODY (stmt)) == BIND_EXPR)
4611 record_vars_into (BIND_EXPR_VARS (OMP_FOR_BODY (stmt)), ctx->cb.dst_fn);
4613 new_stmt = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
4614 TREE_SIDE_EFFECTS (new_stmt) = 1;
4615 body_p = &BIND_EXPR_BODY (new_stmt);
4617 /* The pre-body and input clauses go before the lowered OMP_FOR. */
4620 append_to_statement_list (OMP_FOR_PRE_BODY (stmt), body_p);
4621 lower_rec_input_clauses (OMP_FOR_CLAUSES (stmt), body_p, &dlist, ctx);
4623 /* Lower the header expressions. At this point, we can assume that
4624 the header is of the form:
4626 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
4628 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
4629 using the .omp_data_s mapping, if needed. */
4630 rhs_p = &GIMPLE_STMT_OPERAND (OMP_FOR_INIT (stmt), 1);
4631 if (!is_gimple_min_invariant (*rhs_p))
4632 *rhs_p = get_formal_tmp_var (*rhs_p, body_p);
4634 rhs_p = &TREE_OPERAND (OMP_FOR_COND (stmt), 1);
4635 if (!is_gimple_min_invariant (*rhs_p))
4636 *rhs_p = get_formal_tmp_var (*rhs_p, body_p);
4638 rhs_p = &TREE_OPERAND (GIMPLE_STMT_OPERAND (OMP_FOR_INCR (stmt), 1), 1);
4639 if (!is_gimple_min_invariant (*rhs_p))
4640 *rhs_p = get_formal_tmp_var (*rhs_p, body_p);
4642 /* Once lowered, extract the bounds and clauses. */
4643 extract_omp_for_data (stmt, &fd);
4645 lower_omp_for_lastprivate (&fd, body_p, &dlist, ctx);
4647 append_to_statement_list (stmt, body_p);
4649 append_to_statement_list (OMP_FOR_BODY (stmt), body_p);
4651 t = build2 (OMP_CONTINUE, void_type_node, fd.v, fd.v);
4652 append_to_statement_list (t, body_p);
4654 /* After the loop, add exit clauses. */
4655 lower_reduction_clauses (OMP_FOR_CLAUSES (stmt), body_p, ctx);
4656 append_to_statement_list (dlist, body_p);
4658 maybe_catch_exception (body_p);
4660 /* Region exit marker goes at the end of the loop body. */
4661 t = make_node (OMP_RETURN);
4662 OMP_RETURN_NOWAIT (t) = fd.have_nowait;
4663 append_to_statement_list (t, body_p);
4665 pop_gimplify_context (NULL_TREE);
4666 record_vars_into (ctx->block_vars, ctx->cb.dst_fn);
4668 OMP_FOR_BODY (stmt) = NULL_TREE;
4669 OMP_FOR_PRE_BODY (stmt) = NULL_TREE;
4673 /* Callback for walk_stmts. Check if *TP only contains OMP_FOR
4677 check_combined_parallel (tree *tp, int *walk_subtrees, void *data)
4679 struct walk_stmt_info *wi = data;
4680 int *info = wi->info;
4683 switch (TREE_CODE (*tp))
4687 *info = *info == 0 ? 1 : -1;
4696 /* Lower the OpenMP parallel directive in *STMT_P. CTX holds context
4697 information for the directive. */
4700 lower_omp_parallel (tree *stmt_p, omp_context *ctx)
4702 tree clauses, par_bind, par_body, new_body, bind;
4703 tree olist, ilist, par_olist, par_ilist;
4704 tree stmt, child_fn, t;
4708 clauses = OMP_PARALLEL_CLAUSES (stmt);
4709 par_bind = OMP_PARALLEL_BODY (stmt);
4710 par_body = BIND_EXPR_BODY (par_bind);
4711 child_fn = ctx->cb.dst_fn;
4712 if (!OMP_PARALLEL_COMBINED (stmt))
4714 struct walk_stmt_info wi;
4717 memset (&wi, 0, sizeof (wi));
4718 wi.callback = check_combined_parallel;
4721 walk_stmts (&wi, &par_bind);
4723 OMP_PARALLEL_COMBINED (stmt) = 1;
4726 push_gimplify_context ();
4728 par_olist = NULL_TREE;
4729 par_ilist = NULL_TREE;
4730 lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx);
4731 lower_omp (&par_body, ctx);
4732 lower_reduction_clauses (clauses, &par_olist, ctx);
4734 /* Declare all the variables created by mapping and the variables
4735 declared in the scope of the parallel body. */
4736 record_vars_into (ctx->block_vars, child_fn);
4737 record_vars_into (BIND_EXPR_VARS (par_bind), child_fn);
4739 if (ctx->record_type)
4741 ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_data_o");
4742 OMP_PARALLEL_DATA_ARG (stmt) = ctx->sender_decl;
4747 lower_send_clauses (clauses, &ilist, &olist, ctx);
4748 lower_send_shared_vars (&ilist, &olist, ctx);
4750 /* Once all the expansions are done, sequence all the different
4751 fragments inside OMP_PARALLEL_BODY. */
4752 bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL);
4753 append_to_statement_list (ilist, &BIND_EXPR_BODY (bind));
4755 new_body = alloc_stmt_list ();
4757 if (ctx->record_type)
4759 t = build_fold_addr_expr (ctx->sender_decl);
4760 /* fixup_child_record_type might have changed receiver_decl's type. */
4761 t = fold_convert (TREE_TYPE (ctx->receiver_decl), t);
4762 t = build_gimple_modify_stmt (ctx->receiver_decl, t);
4763 append_to_statement_list (t, &new_body);
4766 append_to_statement_list (par_ilist, &new_body);
4767 append_to_statement_list (par_body, &new_body);
4768 append_to_statement_list (par_olist, &new_body);
4769 maybe_catch_exception (&new_body);
4770 t = make_node (OMP_RETURN);
4771 append_to_statement_list (t, &new_body);
4772 OMP_PARALLEL_BODY (stmt) = new_body;
4774 append_to_statement_list (stmt, &BIND_EXPR_BODY (bind));
4775 append_to_statement_list (olist, &BIND_EXPR_BODY (bind));
4779 pop_gimplify_context (NULL_TREE);
4783 /* Pass *TP back through the gimplifier within the context determined by WI.
4784 This handles replacement of DECL_VALUE_EXPR, as well as adjusting the
4785 flags on ADDR_EXPR. */
4788 lower_regimplify (tree *tp, struct walk_stmt_info *wi)
4790 enum gimplify_status gs;
4794 gs = gimplify_expr (tp, &pre, NULL, is_gimple_lvalue, fb_lvalue);
4795 else if (wi->val_only)
4796 gs = gimplify_expr (tp, &pre, NULL, is_gimple_val, fb_rvalue);
4798 gs = gimplify_expr (tp, &pre, NULL, is_gimple_formal_tmp_var, fb_rvalue);
4799 gcc_assert (gs == GS_ALL_DONE);
4802 tsi_link_before (&wi->tsi, pre, TSI_SAME_STMT);
4805 /* Copy EXP into a temporary. Insert the initialization statement before TSI. */
4808 init_tmp_var (tree exp, tree_stmt_iterator *tsi)
4812 t = create_tmp_var (TREE_TYPE (exp), NULL);
4813 DECL_GIMPLE_REG_P (t) = 1;
4814 stmt = build_gimple_modify_stmt (t, exp);
4815 SET_EXPR_LOCUS (stmt, EXPR_LOCUS (tsi_stmt (*tsi)));
4816 tsi_link_before (tsi, stmt, TSI_SAME_STMT);
4821 /* Similarly, but copy from the temporary and insert the statement
4822 after the iterator. */
4825 save_tmp_var (tree exp, tree_stmt_iterator *tsi)
4829 t = create_tmp_var (TREE_TYPE (exp), NULL);
4830 DECL_GIMPLE_REG_P (t) = 1;
4831 stmt = build_gimple_modify_stmt (exp, t);
4832 SET_EXPR_LOCUS (stmt, EXPR_LOCUS (tsi_stmt (*tsi)));
4833 tsi_link_after (tsi, stmt, TSI_SAME_STMT);
4838 /* Callback for walk_stmts. Lower the OpenMP directive pointed by TP. */
4841 lower_omp_1 (tree *tp, int *walk_subtrees, void *data)
4843 struct walk_stmt_info *wi = data;
4844 omp_context *ctx = wi->info;
4847 /* If we have issued syntax errors, avoid doing any heavy lifting.
4848 Just replace the OpenMP directives with a NOP to avoid
4849 confusing RTL expansion. */
4850 if (errorcount && OMP_DIRECTIVE_P (*tp))
4852 *tp = build_empty_stmt ();
4857 switch (TREE_CODE (*tp))
4860 ctx = maybe_lookup_ctx (t);
4861 lower_omp_parallel (tp, ctx);
4865 ctx = maybe_lookup_ctx (t);
4867 lower_omp_for (tp, ctx);
4871 ctx = maybe_lookup_ctx (t);
4873 lower_omp_sections (tp, ctx);
4877 ctx = maybe_lookup_ctx (t);
4879 lower_omp_single (tp, ctx);
4883 ctx = maybe_lookup_ctx (t);
4885 lower_omp_master (tp, ctx);
4889 ctx = maybe_lookup_ctx (t);
4891 lower_omp_ordered (tp, ctx);
4895 ctx = maybe_lookup_ctx (t);
4897 lower_omp_critical (tp, ctx);
4901 if (ctx && DECL_HAS_VALUE_EXPR_P (t))
4903 lower_regimplify (&t, wi);
4907 t = save_tmp_var (t, &wi->tsi);
4909 t = init_tmp_var (t, &wi->tsi);
4917 lower_regimplify (tp, wi);
4921 case ARRAY_RANGE_REF:
4925 case VIEW_CONVERT_EXPR:
4927 lower_regimplify (tp, wi);
4934 wi->val_only = true;
4935 lower_regimplify (&TREE_OPERAND (t, 0), wi);
4940 if (!TYPE_P (t) && !DECL_P (t))
4949 lower_omp (tree *stmt_p, omp_context *ctx)
4951 struct walk_stmt_info wi;
4953 memset (&wi, 0, sizeof (wi));
4954 wi.callback = lower_omp_1;
4957 wi.want_locations = true;
4959 walk_stmts (&wi, stmt_p);
4962 /* Main entry point. */
4965 execute_lower_omp (void)
4967 all_contexts = splay_tree_new (splay_tree_compare_pointers, 0,
4968 delete_omp_context);
4970 scan_omp (&DECL_SAVED_TREE (current_function_decl), NULL);
4971 gcc_assert (parallel_nesting_level == 0);
4973 if (all_contexts->root)
4974 lower_omp (&DECL_SAVED_TREE (current_function_decl), NULL);
4978 splay_tree_delete (all_contexts);
4979 all_contexts = NULL;
4985 gate_lower_omp (void)
4987 return flag_openmp != 0;
4990 struct tree_opt_pass pass_lower_omp =
4992 "omplower", /* name */
4993 gate_lower_omp, /* gate */
4994 execute_lower_omp, /* execute */
4997 0, /* static_pass_number */
4999 PROP_gimple_any, /* properties_required */
5000 PROP_gimple_lomp, /* properties_provided */
5001 0, /* properties_destroyed */
5002 0, /* todo_flags_start */
5003 TODO_dump_func, /* todo_flags_finish */
5007 /* The following is a utility to diagnose OpenMP structured block violations.
5008 It is not part of the "omplower" pass, as that's invoked too late. It
5009 should be invoked by the respective front ends after gimplification. */
5011 static splay_tree all_labels;
5013 /* Check for mismatched contexts and generate an error if needed. Return
5014 true if an error is detected. */
5017 diagnose_sb_0 (tree *stmt_p, tree branch_ctx, tree label_ctx)
5021 if ((label_ctx ? TREE_VALUE (label_ctx) : NULL) == branch_ctx)
5024 /* Try to avoid confusing the user by producing and error message
5025 with correct "exit" or "enter" verbage. We prefer "exit"
5026 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
5027 if (branch_ctx == NULL)
5033 if (TREE_VALUE (label_ctx) == branch_ctx)
5038 label_ctx = TREE_CHAIN (label_ctx);
5043 error ("invalid exit from OpenMP structured block");
5045 error ("invalid entry to OpenMP structured block");
5047 *stmt_p = build_empty_stmt ();
5051 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
5052 where in the tree each label is found. */
5055 diagnose_sb_1 (tree *tp, int *walk_subtrees, void *data)
5057 struct walk_stmt_info *wi = data;
5058 tree context = (tree) wi->info;
5063 switch (TREE_CODE (t))
5068 walk_tree (&OMP_CLAUSES (t), diagnose_sb_1, wi, NULL);
5074 /* The minimal context here is just a tree of statements. */
5075 inner_context = tree_cons (NULL, t, context);
5076 wi->info = inner_context;
5077 walk_stmts (wi, &OMP_BODY (t));
5082 walk_tree (&OMP_FOR_CLAUSES (t), diagnose_sb_1, wi, NULL);
5083 inner_context = tree_cons (NULL, t, context);
5084 wi->info = inner_context;
5085 walk_tree (&OMP_FOR_INIT (t), diagnose_sb_1, wi, NULL);
5086 walk_tree (&OMP_FOR_COND (t), diagnose_sb_1, wi, NULL);
5087 walk_tree (&OMP_FOR_INCR (t), diagnose_sb_1, wi, NULL);
5088 walk_stmts (wi, &OMP_FOR_PRE_BODY (t));
5089 walk_stmts (wi, &OMP_FOR_BODY (t));
5094 splay_tree_insert (all_labels, (splay_tree_key) LABEL_EXPR_LABEL (t),
5095 (splay_tree_value) context);
5105 /* Pass 2: Check each branch and see if its context differs from that of
5106 the destination label's context. */
5109 diagnose_sb_2 (tree *tp, int *walk_subtrees, void *data)
5111 struct walk_stmt_info *wi = data;
5112 tree context = (tree) wi->info;
5117 switch (TREE_CODE (t))
5122 walk_tree (&OMP_CLAUSES (t), diagnose_sb_2, wi, NULL);
5129 walk_stmts (wi, &OMP_BODY (t));
5134 walk_tree (&OMP_FOR_CLAUSES (t), diagnose_sb_2, wi, NULL);
5136 walk_tree (&OMP_FOR_INIT (t), diagnose_sb_2, wi, NULL);
5137 walk_tree (&OMP_FOR_COND (t), diagnose_sb_2, wi, NULL);
5138 walk_tree (&OMP_FOR_INCR (t), diagnose_sb_2, wi, NULL);
5139 walk_stmts (wi, &OMP_FOR_PRE_BODY (t));
5140 walk_stmts (wi, &OMP_FOR_BODY (t));
5146 tree lab = GOTO_DESTINATION (t);
5147 if (TREE_CODE (lab) != LABEL_DECL)
5150 n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
5151 diagnose_sb_0 (tp, context, n ? (tree) n->value : NULL_TREE);
5157 tree vec = SWITCH_LABELS (t);
5158 int i, len = TREE_VEC_LENGTH (vec);
5159 for (i = 0; i < len; ++i)
5161 tree lab = CASE_LABEL (TREE_VEC_ELT (vec, i));
5162 n = splay_tree_lookup (all_labels, (splay_tree_key) lab);
5163 if (diagnose_sb_0 (tp, context, (tree) n->value))
5170 diagnose_sb_0 (tp, context, NULL_TREE);
5181 diagnose_omp_structured_block_errors (tree fndecl)
5183 tree save_current = current_function_decl;
5184 struct walk_stmt_info wi;
5186 current_function_decl = fndecl;
5188 all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0);
5190 memset (&wi, 0, sizeof (wi));
5191 wi.callback = diagnose_sb_1;
5192 walk_stmts (&wi, &DECL_SAVED_TREE (fndecl));
5194 memset (&wi, 0, sizeof (wi));
5195 wi.callback = diagnose_sb_2;
5196 wi.want_locations = true;
5197 wi.want_return_expr = true;
5198 walk_stmts (&wi, &DECL_SAVED_TREE (fndecl));
5200 splay_tree_delete (all_labels);
5203 current_function_decl = save_current;
5206 #include "gt-omp-low.h"