#include "system.h"
#include "coretypes.h"
#include "tm.h"
-#include "errors.h"
#include "ggc.h"
#include "tree.h"
#include "target.h"
static tree vect_init_vector (tree, tree);
static void vect_finish_stmt_generation
(tree stmt, tree vec_stmt, block_stmt_iterator *bsi);
+static bool vect_is_simple_cond (tree, loop_vec_info);
+static void update_vuses_to_preheader (tree, struct loop*);
/* Utility function dealing with loop peeling (not peeling itself). */
static void vect_generate_tmps_on_preheader
static tree vect_build_loop_niters (loop_vec_info);
static void vect_update_ivs_after_vectorizer (loop_vec_info, tree, edge);
static tree vect_gen_niters_for_prolog_loop (loop_vec_info, tree);
-static void vect_update_inits_of_dr (struct data_reference *, tree niters);
+static void vect_update_init_of_dr (struct data_reference *, tree niters);
static void vect_update_inits_of_drs (loop_vec_info, tree);
static void vect_do_peeling_for_alignment (loop_vec_info, struct loops *);
static void vect_do_peeling_for_loop_bound
create_iv (init, step, NULL_TREE, loop, &incr_bsi, insert_after,
&indx_before_incr, &indx_after_incr);
incr = bsi_stmt (incr_bsi);
- get_stmt_operands (incr);
- set_stmt_info (stmt_ann (incr), new_stmt_vec_info (incr, loop_vinfo));
+ set_stmt_info ((tree_ann_t)stmt_ann (incr), new_stmt_vec_info (incr, loop_vinfo));
return indx_before_incr;
}
/* Function vect_align_data_ref.
- Handle mislignment of a memory accesses.
+ Handle misalignment of a memory accesses.
FORNOW: Can't handle misaligned accesses.
Make sure that the dataref is aligned. */
tree vect_ptr_type;
tree vect_ptr;
tree tag;
- v_may_def_optype v_may_defs = STMT_V_MAY_DEF_OPS (stmt);
- v_must_def_optype v_must_defs = STMT_V_MUST_DEF_OPS (stmt);
- vuse_optype vuses = STMT_VUSE_OPS (stmt);
- int nvuses, nv_may_defs, nv_must_defs;
- int i;
tree new_temp;
tree vec_stmt;
tree new_stmt_list = NULL_TREE;
add_referenced_tmp_var (vect_ptr);
- /** (2) Handle aliasing information of the new vector-pointer: **/
+ /** (2) Add aliasing information to the new vector-pointer:
+ (The points-to info (SSA_NAME_PTR_INFO) may be defined later.) **/
tag = STMT_VINFO_MEMTAG (stmt_info);
gcc_assert (tag);
- get_var_ann (vect_ptr)->type_mem_tag = tag;
- get_var_ann (vect_ptr)->subvars = STMT_VINFO_SUBVARS (stmt_info);
-
- /* Mark for renaming all aliased variables
- (i.e, the may-aliases of the type-mem-tag). */
- nvuses = NUM_VUSES (vuses);
- nv_may_defs = NUM_V_MAY_DEFS (v_may_defs);
- nv_must_defs = NUM_V_MUST_DEFS (v_must_defs);
- for (i = 0; i < nvuses; i++)
- {
- tree use = VUSE_OP (vuses, i);
- if (TREE_CODE (use) == SSA_NAME)
- bitmap_set_bit (vars_to_rename, var_ann (SSA_NAME_VAR (use))->uid);
- }
- for (i = 0; i < nv_may_defs; i++)
- {
- tree def = V_MAY_DEF_RESULT (v_may_defs, i);
- if (TREE_CODE (def) == SSA_NAME)
- bitmap_set_bit (vars_to_rename, var_ann (SSA_NAME_VAR (def))->uid);
- }
- for (i = 0; i < nv_must_defs; i++)
- {
- tree def = V_MUST_DEF_RESULT (v_must_defs, i);
- if (TREE_CODE (def) == SSA_NAME)
- bitmap_set_bit (vars_to_rename, var_ann (SSA_NAME_VAR (def))->uid);
- }
+ /* If tag is a variable (and NOT_A_TAG) than a new type alias
+ tag must be created with tag added to its may alias list. */
+ if (var_ann (tag)->mem_tag_kind == NOT_A_TAG)
+ new_type_alias (vect_ptr, tag);
+ else
+ var_ann (vect_ptr)->type_mem_tag = tag;
+ var_ann (vect_ptr)->subvars = STMT_VINFO_SUBVARS (stmt_info);
/** (3) Calculate the initial address the vector-pointer, and set
the vector-pointer to point to it before the loop: **/
/** (4) Handle the updating of the vector-pointer inside the loop: **/
if (only_init) /* No update in loop is required. */
- return vect_ptr_init;
+ {
+ /* Copy the points-to information if it exists. */
+ if (STMT_VINFO_PTR_INFO (stmt_info))
+ duplicate_ssa_name_ptr_info (vect_ptr_init,
+ STMT_VINFO_PTR_INFO (stmt_info));
+ return vect_ptr_init;
+ }
idx = vect_create_index_for_vector_ref (loop_vinfo);
bsi_insert_before (bsi, vec_stmt, BSI_SAME_STMT);
data_ref_ptr = TREE_OPERAND (vec_stmt, 0);
+ /* Copy the points-to information if it exists. */
+ if (STMT_VINFO_PTR_INFO (stmt_info))
+ duplicate_ssa_name_ptr_info (data_ref_ptr, STMT_VINFO_PTR_INFO (stmt_info));
return data_ref_ptr;
}
stmt_vec_info def_stmt_info = NULL;
stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
- int nunits = GET_MODE_NUNITS (TYPE_MODE (vectype));
+ int nunits = TYPE_VECTOR_SUBPARTS (vectype);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- basic_block bb;
tree vec_inv;
+ tree vec_cst;
tree t = NULL_TREE;
tree def;
int i;
+ enum vect_def_type dt;
+ bool is_simple_use;
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
{
print_generic_expr (vect_dump, op, TDF_SLIM);
}
- /** ===> Case 1: operand is a constant. **/
-
- if (TREE_CODE (op) == INTEGER_CST || TREE_CODE (op) == REAL_CST)
+ is_simple_use = vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt);
+ gcc_assert (is_simple_use);
+ if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
{
- /* Create 'vect_cst_ = {cst,cst,...,cst}' */
-
- tree vec_cst;
-
- /* Build a tree with vector elements. */
- if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
- fprintf (vect_dump, "Create vector_cst. nunits = %d", nunits);
-
- for (i = nunits - 1; i >= 0; --i)
+ if (def)
{
- t = tree_cons (NULL_TREE, op, t);
+ fprintf (vect_dump, "def = ");
+ print_generic_expr (vect_dump, def, TDF_SLIM);
+ }
+ if (def_stmt)
+ {
+ fprintf (vect_dump, " def_stmt = ");
+ print_generic_expr (vect_dump, def_stmt, TDF_SLIM);
}
- vec_cst = build_vector (vectype, t);
- return vect_init_vector (stmt, vec_cst);
- }
-
- gcc_assert (TREE_CODE (op) == SSA_NAME);
-
- /** ===> Case 2: operand is an SSA_NAME - find the stmt that defines it. **/
-
- def_stmt = SSA_NAME_DEF_STMT (op);
- def_stmt_info = vinfo_for_stmt (def_stmt);
-
- if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
- {
- fprintf (vect_dump, "vect_get_vec_def_for_operand: def_stmt: ");
- print_generic_expr (vect_dump, def_stmt, TDF_SLIM);
}
+ /* FORNOW */
+ gcc_assert (dt != vect_reduction_def);
- /** ==> Case 2.1: operand is defined inside the loop. **/
-
- if (def_stmt_info)
+ switch (dt)
{
- /* Get the def from the vectorized stmt. */
-
- vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
- gcc_assert (vec_stmt);
- vec_oprnd = TREE_OPERAND (vec_stmt, 0);
- return vec_oprnd;
- }
-
+ /* Case 1: operand is a constant. */
+ case vect_constant_def:
+ {
+ /* Create 'vect_cst_ = {cst,cst,...,cst}' */
+ if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
+ fprintf (vect_dump, "Create vector_cst. nunits = %d", nunits);
+
+ for (i = nunits - 1; i >= 0; --i)
+ {
+ t = tree_cons (NULL_TREE, op, t);
+ }
+ vec_cst = build_vector (vectype, t);
+ return vect_init_vector (stmt, vec_cst);
+ }
+
+ /* Case 2: operand is defined outside the loop - loop invariant. */
+ case vect_invariant_def:
+ {
+ /* Create 'vec_inv = {inv,inv,..,inv}' */
+ if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
+ fprintf (vect_dump, "Create vector_inv.");
+
+ for (i = nunits - 1; i >= 0; --i)
+ {
+ t = tree_cons (NULL_TREE, def, t);
+ }
+
+ vec_inv = build_constructor (vectype, t);
+ return vect_init_vector (stmt, vec_inv);
+ }
+
+ /* Case 3: operand is defined inside the loop. */
+ case vect_loop_def:
+ {
+ /* Get the def from the vectorized stmt. */
+ def_stmt_info = vinfo_for_stmt (def_stmt);
+ vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
+ gcc_assert (vec_stmt);
+ vec_oprnd = TREE_OPERAND (vec_stmt, 0);
+ return vec_oprnd;
+ }
+
+ /* Case 4: operand is defined by loop-header phi - induction. */
+ case vect_induction_def:
+ {
+ if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
+ fprintf (vect_dump, "induction - unsupported.");
+ internal_error ("no support for induction"); /* FORNOW */
+ }
- /** ==> Case 2.2: operand is defined by the loop-header phi-node -
- it is a reduction/induction. **/
-
- bb = bb_for_stmt (def_stmt);
- if (TREE_CODE (def_stmt) == PHI_NODE && flow_bb_inside_loop_p (loop, bb))
- {
- if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
- fprintf (vect_dump, "reduction/induction - unsupported.");
- internal_error ("no support for reduction/induction"); /* FORNOW */
- }
-
-
- /** ==> Case 2.3: operand is defined outside the loop -
- it is a loop invariant. */
-
- switch (TREE_CODE (def_stmt))
- {
- case PHI_NODE:
- def = PHI_RESULT (def_stmt);
- break;
- case MODIFY_EXPR:
- def = TREE_OPERAND (def_stmt, 0);
- break;
- case NOP_EXPR:
- def = TREE_OPERAND (def_stmt, 0);
- gcc_assert (IS_EMPTY_STMT (def_stmt));
- def = op;
- break;
default:
- if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
- {
- fprintf (vect_dump, "unsupported defining stmt: ");
- print_generic_expr (vect_dump, def_stmt, TDF_SLIM);
- }
- internal_error ("unsupported defining stmt");
- }
-
- /* Build a tree with vector elements.
- Create 'vec_inv = {inv,inv,..,inv}' */
-
- if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
- fprintf (vect_dump, "Create vector_inv.");
-
- for (i = nunits - 1; i >= 0; --i)
- {
- t = tree_cons (NULL_TREE, def, t);
+ gcc_unreachable ();
}
-
- vec_inv = build_constructor (vectype, t);
- return vect_init_vector (stmt, vec_inv);
}
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
tree new_temp;
+ tree def, def_stmt;
+ enum vect_def_type dt;
/* Is vectorizable assignment? */
+ if (!STMT_VINFO_RELEVANT_P (stmt_info))
+ return false;
+
+ gcc_assert (STMT_VINFO_DEF_TYPE (stmt_info) == vect_loop_def);
if (TREE_CODE (stmt) != MODIFY_EXPR)
return false;
return false;
op = TREE_OPERAND (stmt, 1);
- if (!vect_is_simple_use (op, loop_vinfo, NULL))
+ if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt))
{
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
fprintf (vect_dump, "use not simple.");
}
+/* Function vect_min_worthwhile_factor.
+
+ For a loop where we could vectorize the operation indicated by CODE,
+ return the minimum vectorization factor that makes it worthwhile
+ to use generic vectors. */
+static int
+vect_min_worthwhile_factor (enum tree_code code)
+{
+ switch (code)
+ {
+ case PLUS_EXPR:
+ case MINUS_EXPR:
+ case NEGATE_EXPR:
+ return 4;
+
+ case BIT_AND_EXPR:
+ case BIT_IOR_EXPR:
+ case BIT_XOR_EXPR:
+ case BIT_NOT_EXPR:
+ return 2;
+
+ default:
+ return INT_MAX;
+ }
+}
+
+
/* Function vectorizable_operation.
Check if STMT performs a binary or unary operation that can be vectorized.
int op_type;
tree op;
optab optab;
+ tree def, def_stmt;
+ enum vect_def_type dt;
/* Is STMT a vectorizable binary/unary operation? */
+ if (!STMT_VINFO_RELEVANT_P (stmt_info))
+ return false;
+
+ gcc_assert (STMT_VINFO_DEF_TYPE (stmt_info) == vect_loop_def);
+
+ if (STMT_VINFO_LIVE_P (stmt_info))
+ {
+ /* FORNOW: not yet supported. */
+ if (vect_print_dump_info (REPORT_DETAILS, LOOP_LOC (loop_vinfo)))
+ fprintf (vect_dump, "value used after loop.");
+ return false;
+ }
+
if (TREE_CODE (stmt) != MODIFY_EXPR)
return false;
for (i = 0; i < op_type; i++)
{
op = TREE_OPERAND (operation, i);
- if (!vect_is_simple_use (op, loop_vinfo, NULL))
+ if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt))
{
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
fprintf (vect_dump, "use not simple.");
{
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
fprintf (vect_dump, "op not supported by target.");
+ if (GET_MODE_SIZE (vec_mode) != UNITS_PER_WORD
+ || LOOP_VINFO_VECT_FACTOR (loop_vinfo)
+ < vect_min_worthwhile_factor (code))
+ return false;
+ if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
+ fprintf (vect_dump, "proceeding using word mode.");
+ }
+
+ /* Worthwhile without SIMD support? */
+ if (!VECTOR_MODE_P (TYPE_MODE (vectype))
+ && LOOP_VINFO_VECT_FACTOR (loop_vinfo)
+ < vect_min_worthwhile_factor (code))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
+ fprintf (vect_dump, "not worthwhile without SIMD support.");
return false;
}
enum machine_mode vec_mode;
tree dummy;
enum dr_alignment_support alignment_support_cheme;
+ ssa_op_iter iter;
+ tree def, def_stmt;
+ enum vect_def_type dt;
/* Is vectorizable store? */
return false;
op = TREE_OPERAND (stmt, 1);
- if (!vect_is_simple_use (op, loop_vinfo, NULL))
+ if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt))
{
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
fprintf (vect_dump, "use not simple.");
alignment_support_cheme = vect_supportable_dr_alignment (dr);
gcc_assert (alignment_support_cheme);
- gcc_assert (alignment_support_cheme = dr_aligned); /* FORNOW */
+ gcc_assert (alignment_support_cheme == dr_aligned); /* FORNOW */
/* Handle use - get the vectorized def from the defining stmt. */
vec_oprnd1 = vect_get_vec_def_for_operand (op, stmt);
*vec_stmt = build2 (MODIFY_EXPR, vectype, data_ref, vec_oprnd1);
vect_finish_stmt_generation (stmt, *vec_stmt, bsi);
+ /* Copy the V_MAY_DEFS representing the aliasing of the original array
+ element's definition to the vector's definition then update the
+ defining statement. The original is being deleted so the same
+ SSA_NAMEs can be used. */
+ copy_virtual_operands (*vec_stmt, stmt);
+
+ FOR_EACH_SSA_TREE_OPERAND (def, stmt, iter, SSA_OP_VMAYDEF)
+ {
+ SSA_NAME_DEF_STMT (def) = *vec_stmt;
+
+ /* If this virtual def has a use outside the loop and a loop peel is
+ performed then the def may be renamed by the peel. Mark it for
+ renaming so the later use will also be renamed. */
+ mark_sym_for_renaming (SSA_NAME_VAR (def));
+ }
+
return true;
}
enum dr_alignment_support alignment_support_cheme;
/* Is vectorizable load? */
+ if (!STMT_VINFO_RELEVANT_P (stmt_info))
+ return false;
+
+ gcc_assert (STMT_VINFO_DEF_TYPE (stmt_info) == vect_loop_def);
+
+ if (STMT_VINFO_LIVE_P (stmt_info))
+ {
+ /* FORNOW: not yet supported. */
+ if (vect_print_dump_info (REPORT_DETAILS, LOOP_LOC (loop_vinfo)))
+ fprintf (vect_dump, "value used after loop.");
+ return false;
+ }
if (TREE_CODE (stmt) != MODIFY_EXPR)
return false;
new_temp = make_ssa_name (vec_dest, new_stmt);
TREE_OPERAND (new_stmt, 0) = new_temp;
vect_finish_stmt_generation (stmt, new_stmt, bsi);
+ copy_virtual_operands (new_stmt, stmt);
}
else if (alignment_support_cheme == dr_unaligned_software_pipeline)
{
new_bb = bsi_insert_on_edge_immediate (pe, new_stmt);
gcc_assert (!new_bb);
msq_init = TREE_OPERAND (new_stmt, 0);
+ copy_virtual_operands (new_stmt, stmt);
+ update_vuses_to_preheader (new_stmt, loop);
/* <2> Create lsq = *(floor(p2')) in the loop */
offset = build_int_cst (integer_type_node,
- GET_MODE_NUNITS (TYPE_MODE (vectype)));
+ TYPE_VECTOR_SUBPARTS (vectype));
offset = int_const_binop (MINUS_EXPR, offset, integer_one_node, 1);
vec_dest = vect_create_destination_var (scalar_dest, vectype);
dataref_ptr = vect_create_data_ref_ptr (stmt, bsi, offset, &dummy, false);
TREE_OPERAND (new_stmt, 0) = new_temp;
vect_finish_stmt_generation (stmt, new_stmt, bsi);
lsq = TREE_OPERAND (new_stmt, 0);
+ copy_virtual_operands (new_stmt, stmt);
/* <3> */
gcc_assert (!new_bb);
magic = TREE_OPERAND (new_stmt, 0);
- /* Since we have just created a CALL_EXPR, we may need to
- rename call-clobbered variables. */
- mark_call_clobbered_vars_to_rename ();
+ /* The result of the CALL_EXPR to this builtin is determined from
+ the value of the parameter and no global variables are touched
+ which makes the builtin a "const" function. Requiring the
+ builtin to have the "const" attribute makes it unnecessary
+ to call mark_call_clobbered_vars_to_rename. */
+ gcc_assert (TREE_READONLY (builtin_decl));
}
else
{
}
+/* Function vectorizable_live_operation.
+
+ STMT computes a value that is used outside the loop. Check if
+ it can be supported. */
+
+bool
+vectorizable_live_operation (tree stmt,
+ block_stmt_iterator *bsi ATTRIBUTE_UNUSED,
+ tree *vec_stmt ATTRIBUTE_UNUSED)
+{
+ tree operation;
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ int i;
+ enum tree_code code;
+ int op_type;
+ tree op;
+ tree def, def_stmt;
+ enum vect_def_type dt;
+
+ if (!STMT_VINFO_LIVE_P (stmt_info))
+ return false;
+
+ if (TREE_CODE (stmt) != MODIFY_EXPR)
+ return false;
+
+ if (TREE_CODE (TREE_OPERAND (stmt, 0)) != SSA_NAME)
+ return false;
+
+ operation = TREE_OPERAND (stmt, 1);
+ code = TREE_CODE (operation);
+
+ op_type = TREE_CODE_LENGTH (code);
+
+ /* FORNOW: support only if all uses are invariant. This means
+ that the scalar operations can remain in place, unvectorized.
+ The original last scalar value that they compute will be used. */
+
+ for (i = 0; i < op_type; i++)
+ {
+ op = TREE_OPERAND (operation, i);
+ if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
+ fprintf (vect_dump, "use not simple.");
+ return false;
+ }
+
+ if (dt != vect_invariant_def && dt != vect_constant_def)
+ return false;
+ }
+
+ /* No transformation is required for the cases we currently support. */
+ return true;
+}
+
+
+/* Function vect_is_simple_cond.
+
+ Input:
+ LOOP - the loop that is being vectorized.
+ COND - Condition that is checked for simple use.
+
+ Returns whether a COND can be vectorized. Checks whether
+ condition operands are supportable using vec_is_simple_use. */
+
+static bool
+vect_is_simple_cond (tree cond, loop_vec_info loop_vinfo)
+{
+ tree lhs, rhs;
+ tree def;
+ enum vect_def_type dt;
+
+ if (!COMPARISON_CLASS_P (cond))
+ return false;
+
+ lhs = TREE_OPERAND (cond, 0);
+ rhs = TREE_OPERAND (cond, 1);
+
+ if (TREE_CODE (lhs) == SSA_NAME)
+ {
+ tree lhs_def_stmt = SSA_NAME_DEF_STMT (lhs);
+ if (!vect_is_simple_use (lhs, loop_vinfo, &lhs_def_stmt, &def, &dt))
+ return false;
+ }
+ else if (TREE_CODE (lhs) != INTEGER_CST && TREE_CODE (lhs) != REAL_CST)
+ return false;
+
+ if (TREE_CODE (rhs) == SSA_NAME)
+ {
+ tree rhs_def_stmt = SSA_NAME_DEF_STMT (rhs);
+ if (!vect_is_simple_use (rhs, loop_vinfo, &rhs_def_stmt, &def, &dt))
+ return false;
+ }
+ else if (TREE_CODE (rhs) != INTEGER_CST && TREE_CODE (rhs) != REAL_CST)
+ return false;
+
+ return true;
+}
+
+/* vectorizable_condition.
+
+ Check if STMT is conditional modify expression that can be vectorized.
+ If VEC_STMT is also passed, vectorize the STMT: create a vectorized
+ stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
+ at BSI.
+
+ Return FALSE if not a vectorizable STMT, TRUE otherwise. */
+
+bool
+vectorizable_condition (tree stmt, block_stmt_iterator *bsi, tree *vec_stmt)
+{
+ tree scalar_dest = NULL_TREE;
+ tree vec_dest = NULL_TREE;
+ tree op = NULL_TREE;
+ tree cond_expr, then_clause, else_clause;
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+ tree vec_cond_lhs, vec_cond_rhs, vec_then_clause, vec_else_clause;
+ tree vec_compare, vec_cond_expr;
+ tree new_temp;
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ enum machine_mode vec_mode;
+ tree def;
+ enum vect_def_type dt;
+
+ if (!STMT_VINFO_RELEVANT_P (stmt_info))
+ return false;
+
+ gcc_assert (STMT_VINFO_DEF_TYPE (stmt_info) == vect_loop_def);
+
+ if (STMT_VINFO_LIVE_P (stmt_info))
+ {
+ /* FORNOW: not yet supported. */
+ if (vect_print_dump_info (REPORT_DETAILS, LOOP_LOC (loop_vinfo)))
+ fprintf (vect_dump, "value used after loop.");
+ return false;
+ }
+
+ if (TREE_CODE (stmt) != MODIFY_EXPR)
+ return false;
+
+ op = TREE_OPERAND (stmt, 1);
+
+ if (TREE_CODE (op) != COND_EXPR)
+ return false;
+
+ cond_expr = TREE_OPERAND (op, 0);
+ then_clause = TREE_OPERAND (op, 1);
+ else_clause = TREE_OPERAND (op, 2);
+
+ if (!vect_is_simple_cond (cond_expr, loop_vinfo))
+ return false;
+
+ if (TREE_CODE (then_clause) == SSA_NAME)
+ {
+ tree then_def_stmt = SSA_NAME_DEF_STMT (then_clause);
+ if (!vect_is_simple_use (then_clause, loop_vinfo,
+ &then_def_stmt, &def, &dt))
+ return false;
+ }
+ else if (TREE_CODE (then_clause) != INTEGER_CST
+ && TREE_CODE (then_clause) != REAL_CST)
+ return false;
+
+ if (TREE_CODE (else_clause) == SSA_NAME)
+ {
+ tree else_def_stmt = SSA_NAME_DEF_STMT (else_clause);
+ if (!vect_is_simple_use (else_clause, loop_vinfo,
+ &else_def_stmt, &def, &dt))
+ return false;
+ }
+ else if (TREE_CODE (else_clause) != INTEGER_CST
+ && TREE_CODE (else_clause) != REAL_CST)
+ return false;
+
+
+ vec_mode = TYPE_MODE (vectype);
+
+ if (!vec_stmt)
+ {
+ STMT_VINFO_TYPE (stmt_info) = condition_vec_info_type;
+ return expand_vec_cond_expr_p (op, vec_mode);
+ }
+
+ /* Transform */
+
+ /* Handle def. */
+ scalar_dest = TREE_OPERAND (stmt, 0);
+ vec_dest = vect_create_destination_var (scalar_dest, vectype);
+
+ /* Handle cond expr. */
+ vec_cond_lhs =
+ vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), stmt);
+ vec_cond_rhs =
+ vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), stmt);
+ vec_then_clause = vect_get_vec_def_for_operand (then_clause, stmt);
+ vec_else_clause = vect_get_vec_def_for_operand (else_clause, stmt);
+
+ /* Arguments are ready. create the new vector stmt. */
+ vec_compare = build2 (TREE_CODE (cond_expr), vectype,
+ vec_cond_lhs, vec_cond_rhs);
+ vec_cond_expr = build (VEC_COND_EXPR, vectype,
+ vec_compare, vec_then_clause, vec_else_clause);
+
+ *vec_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, vec_cond_expr);
+ new_temp = make_ssa_name (vec_dest, *vec_stmt);
+ TREE_OPERAND (*vec_stmt, 0) = new_temp;
+ vect_finish_stmt_generation (stmt, *vec_stmt, bsi);
+
+ return true;
+}
+
/* Function vect_transform_stmt.
Create a vectorized stmt to replace STMT, and insert it at BSI. */
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
bool done;
- switch (STMT_VINFO_TYPE (stmt_info))
+ if (STMT_VINFO_RELEVANT_P (stmt_info))
{
- case op_vec_info_type:
- done = vectorizable_operation (stmt, bsi, &vec_stmt);
- gcc_assert (done);
- break;
-
- case assignment_vec_info_type:
- done = vectorizable_assignment (stmt, bsi, &vec_stmt);
- gcc_assert (done);
- break;
-
- case load_vec_info_type:
- done = vectorizable_load (stmt, bsi, &vec_stmt);
- gcc_assert (done);
- break;
+ switch (STMT_VINFO_TYPE (stmt_info))
+ {
+ case op_vec_info_type:
+ done = vectorizable_operation (stmt, bsi, &vec_stmt);
+ gcc_assert (done);
+ break;
+
+ case assignment_vec_info_type:
+ done = vectorizable_assignment (stmt, bsi, &vec_stmt);
+ gcc_assert (done);
+ break;
+
+ case load_vec_info_type:
+ done = vectorizable_load (stmt, bsi, &vec_stmt);
+ gcc_assert (done);
+ break;
+
+ case store_vec_info_type:
+ done = vectorizable_store (stmt, bsi, &vec_stmt);
+ gcc_assert (done);
+ is_store = true;
+ break;
+
+ case condition_vec_info_type:
+ done = vectorizable_condition (stmt, bsi, &vec_stmt);
+ gcc_assert (done);
+ break;
+
+ default:
+ if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
+ fprintf (vect_dump, "stmt not supported.");
+ gcc_unreachable ();
+ }
+
+ STMT_VINFO_VEC_STMT (stmt_info) = vec_stmt;
+ }
- case store_vec_info_type:
- done = vectorizable_store (stmt, bsi, &vec_stmt);
+ if (STMT_VINFO_LIVE_P (stmt_info))
+ {
+ done = vectorizable_live_operation (stmt, bsi, &vec_stmt);
gcc_assert (done);
- is_store = true;
- break;
- default:
- if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
- fprintf (vect_dump, "stmt not supported.");
- gcc_unreachable ();
}
- STMT_VINFO_VEC_STMT (stmt_info) = vec_stmt;
-
- return is_store;
+ return is_store;
}
}
+/* Function update_vuses_to_preheader.
+
+ Input:
+ STMT - a statement with potential VUSEs.
+ LOOP - the loop whose preheader will contain STMT.
+
+ It's possible to vectorize a loop even though an SSA_NAME from a VUSE
+ appears to be defined in a V_MAY_DEF in another statement in a loop.
+ One such case is when the VUSE is at the dereference of a __restricted__
+ pointer in a load and the V_MAY_DEF is at the dereference of a different
+ __restricted__ pointer in a store. Vectorization may result in
+ copy_virtual_uses being called to copy the problematic VUSE to a new
+ statement that is being inserted in the loop preheader. This procedure
+ is called to change the SSA_NAME in the new statement's VUSE from the
+ SSA_NAME updated in the loop to the related SSA_NAME available on the
+ path entering the loop.
+
+ When this function is called, we have the following situation:
+
+ # vuse <name1>
+ S1: vload
+ do {
+ # name1 = phi < name0 , name2>
+
+ # vuse <name1>
+ S2: vload
+
+ # name2 = vdef <name1>
+ S3: vstore
+
+ }while...
+
+ Stmt S1 was created in the loop preheader block as part of misaligned-load
+ handling. This function fixes the name of the vuse of S1 from 'name1' to
+ 'name0'. */
+
+static void
+update_vuses_to_preheader (tree stmt, struct loop *loop)
+{
+ basic_block header_bb = loop->header;
+ edge preheader_e = loop_preheader_edge (loop);
+ ssa_op_iter iter;
+ use_operand_p use_p;
+
+ FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_VUSE)
+ {
+ tree ssa_name = USE_FROM_PTR (use_p);
+ tree def_stmt = SSA_NAME_DEF_STMT (ssa_name);
+ tree name_var = SSA_NAME_VAR (ssa_name);
+ basic_block bb = bb_for_stmt (def_stmt);
+
+ /* For a use before any definitions, def_stmt is a NOP_EXPR. */
+ if (!IS_EMPTY_STMT (def_stmt)
+ && flow_bb_inside_loop_p (loop, bb))
+ {
+ /* If the block containing the statement defining the SSA_NAME
+ is in the loop then it's necessary to find the definition
+ outside the loop using the PHI nodes of the header. */
+ tree phi;
+ bool updated = false;
+
+ for (phi = phi_nodes (header_bb); phi; phi = TREE_CHAIN (phi))
+ {
+ if (SSA_NAME_VAR (PHI_RESULT (phi)) == name_var)
+ {
+ SET_USE (use_p, PHI_ARG_DEF (phi, preheader_e->dest_idx));
+ updated = true;
+ break;
+ }
+ }
+ gcc_assert (updated);
+ }
+ }
+}
+
+
/* Function vect_update_ivs_after_vectorizer.
"Advance" the induction variables of LOOP to the value they should take
tree var, stmt, ni, ni_name;
block_stmt_iterator last_bsi;
+ if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
+ {
+ fprintf (vect_dump, "vect_update_ivs_after_vectorizer: phi: ");
+ print_generic_expr (vect_dump, phi, TDF_SLIM);
+ }
+
/* Skip virtual phi's. */
if (!is_gimple_reg (SSA_NAME_VAR (PHI_RESULT (phi))))
{
bsi_insert_before (&last_bsi, stmt, BSI_SAME_STMT);
/* Fix phi expressions in the successor bb. */
- gcc_assert (PHI_ARG_DEF_FROM_EDGE (phi1, update_e) ==
- PHI_ARG_DEF_FROM_EDGE (phi, single_succ_edge (loop->latch)));
SET_PHI_ARG_DEF (phi1, update_e->dest_idx, ni_name);
}
}
#endif
if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
- fprintf (vect_dump, "=== vect_transtorm_for_unknown_loop_bound ===");
+ fprintf (vect_dump, "=== vect_do_peeling_for_loop_bound ===");
/* Generate the following variables on the preheader of original loop:
Set the number of iterations for the loop represented by LOOP_VINFO
to the minimum between LOOP_NITERS (the original iteration count of the loop)
- and the misalignment of DR - the first data reference recorded in
+ and the misalignment of DR - the data reference recorded in
LOOP_VINFO_UNALIGNED_DR (LOOP_VINFO). As a result, after the execution of
this loop, the data reference DR will refer to an aligned location.
The following computation is generated:
- compute address misalignment in bytes:
- addr_mis = addr & (vectype_size - 1)
+ If the misalignment of DR is known at compile time:
+ addr_mis = int mis = DR_MISALIGNMENT (dr);
+ Else, compute address misalignment in bytes:
+ addr_mis = addr & (vectype_size - 1)
prolog_niters = min ( LOOP_NITERS , (VF - addr_mis/elem_size)&(VF-1) )
stmt_vec_info stmt_info = vinfo_for_stmt (dr_stmt);
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
int vectype_align = TYPE_ALIGN (vectype) / BITS_PER_UNIT;
- tree elem_misalign;
- tree byte_misalign;
- tree new_stmts = NULL_TREE;
- tree start_addr =
- vect_create_addr_base_for_vector_ref (dr_stmt, &new_stmts, NULL_TREE);
- tree ptr_type = TREE_TYPE (start_addr);
- tree size = TYPE_SIZE (ptr_type);
- tree type = lang_hooks.types.type_for_size (tree_low_cst (size, 1), 1);
- tree vectype_size_minus_1 = build_int_cst (type, vectype_align - 1);
tree vf_minus_1 = build_int_cst (unsigned_type_node, vf - 1);
tree niters_type = TREE_TYPE (loop_niters);
- tree elem_size_log =
- build_int_cst (unsigned_type_node, exact_log2 (vectype_align/vf));
- tree vf_tree = build_int_cst (unsigned_type_node, vf);
pe = loop_preheader_edge (loop);
- new_bb = bsi_insert_on_edge_immediate (pe, new_stmts);
- gcc_assert (!new_bb);
- /* Create: byte_misalign = addr & (vectype_size - 1) */
- byte_misalign = build2 (BIT_AND_EXPR, type, start_addr, vectype_size_minus_1);
+ if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo) > 0)
+ {
+ int byte_misalign = LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo);
+ int element_size = vectype_align/vf;
+ int elem_misalign = byte_misalign / element_size;
- /* Create: elem_misalign = byte_misalign / element_size */
- elem_misalign =
- build2 (RSHIFT_EXPR, unsigned_type_node, byte_misalign, elem_size_log);
+ if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
+ fprintf (vect_dump, "known alignment = %d.", byte_misalign);
+ iters = build_int_cst (niters_type, (vf - elem_misalign)&(vf-1));
+ }
+ else
+ {
+ tree new_stmts = NULL_TREE;
+ tree start_addr =
+ vect_create_addr_base_for_vector_ref (dr_stmt, &new_stmts, NULL_TREE);
+ tree ptr_type = TREE_TYPE (start_addr);
+ tree size = TYPE_SIZE (ptr_type);
+ tree type = lang_hooks.types.type_for_size (tree_low_cst (size, 1), 1);
+ tree vectype_size_minus_1 = build_int_cst (type, vectype_align - 1);
+ tree elem_size_log =
+ build_int_cst (unsigned_type_node, exact_log2 (vectype_align/vf));
+ tree vf_tree = build_int_cst (unsigned_type_node, vf);
+ tree byte_misalign;
+ tree elem_misalign;
+
+ new_bb = bsi_insert_on_edge_immediate (pe, new_stmts);
+ gcc_assert (!new_bb);
- /* Create: (niters_type) (VF - elem_misalign)&(VF - 1) */
- iters = build2 (MINUS_EXPR, unsigned_type_node, vf_tree, elem_misalign);
- iters = build2 (BIT_AND_EXPR, unsigned_type_node, iters, vf_minus_1);
- iters = fold_convert (niters_type, iters);
+ /* Create: byte_misalign = addr & (vectype_size - 1) */
+ byte_misalign =
+ build2 (BIT_AND_EXPR, type, start_addr, vectype_size_minus_1);
+ /* Create: elem_misalign = byte_misalign / element_size */
+ elem_misalign =
+ build2 (RSHIFT_EXPR, unsigned_type_node, byte_misalign, elem_size_log);
+
+ /* Create: (niters_type) (VF - elem_misalign)&(VF - 1) */
+ iters = build2 (MINUS_EXPR, unsigned_type_node, vf_tree, elem_misalign);
+ iters = build2 (BIT_AND_EXPR, unsigned_type_node, iters, vf_minus_1);
+ iters = fold_convert (niters_type, iters);
+ }
+
/* Create: prolog_loop_niters = min (iters, loop_niters) */
/* If the loop bound is known at compile time we already verified that it is
greater than vf; since the misalignment ('iters') is at most vf, there's
if (TREE_CODE (loop_niters) != INTEGER_CST)
iters = build2 (MIN_EXPR, niters_type, iters, loop_niters);
+ if (vect_print_dump_info (REPORT_DETAILS, UNKNOWN_LOC))
+ {
+ fprintf (vect_dump, "niters for prolog loop: ");
+ print_generic_expr (vect_dump, iters, TDF_SLIM);
+ }
+
var = create_tmp_var (niters_type, "prolog_loop_niters");
add_referenced_tmp_var (var);
iters_name = force_gimple_operand (iters, &stmt, false, var);
/* Insert stmt on loop preheader edge. */
- pe = loop_preheader_edge (loop);
if (stmt)
{
basic_block new_bb = bsi_insert_on_edge_immediate (pe, stmt);
}
-/* Function vect_update_inits_of_dr
+/* Function vect_update_init_of_dr
NITERS iterations were peeled from LOOP. DR represents a data reference
in LOOP. This function updates the information recorded in DR to
executed. Specifically, it updates the OFFSET field of stmt_info. */
static void
-vect_update_inits_of_dr (struct data_reference *dr, tree niters)
+vect_update_init_of_dr (struct data_reference *dr, tree niters)
{
stmt_vec_info stmt_info = vinfo_for_stmt (DR_STMT (dr));
tree offset = STMT_VINFO_VECT_INIT_OFFSET (stmt_info);
for (i = 0; i < VARRAY_ACTIVE_SIZE (loop_write_datarefs); i++)
{
struct data_reference *dr = VARRAY_GENERIC_PTR (loop_write_datarefs, i);
- vect_update_inits_of_dr (dr, niters);
+ vect_update_init_of_dr (dr, niters);
}
for (i = 0; i < VARRAY_ACTIVE_SIZE (loop_read_datarefs); i++)
{
struct data_reference *dr = VARRAY_GENERIC_PTR (loop_read_datarefs, i);
- vect_update_inits_of_dr (dr, niters);
+ vect_update_init_of_dr (dr, niters);
}
}
/* Update number of times loop executes. */
n_iters = LOOP_VINFO_NITERS (loop_vinfo);
- LOOP_VINFO_NITERS (loop_vinfo) =
- build2 (MINUS_EXPR, TREE_TYPE (n_iters), n_iters, niters_of_prolog_loop);
+ LOOP_VINFO_NITERS (loop_vinfo) = fold (build2 (MINUS_EXPR,
+ TREE_TYPE (n_iters), n_iters, niters_of_prolog_loop));
/* Update the init conditions of the access functions of all data refs. */
vect_update_inits_of_drs (loop_vinfo, niters_of_prolog_loop);
/* Peel the loop if there are data refs with unknown alignment.
Only one data ref with unknown store is allowed. */
- if (LOOP_DO_PEELING_FOR_ALIGNMENT (loop_vinfo))
+ if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo))
vect_do_peeling_for_alignment (loop_vinfo, loops);
/* If the loop has a symbolic number of iterations 'n' (i.e. it's not a
/* FORNOW: Verify that all stmts operate on the same number of
units and no inner unrolling is necessary. */
gcc_assert
- (GET_MODE_NUNITS (TYPE_MODE (STMT_VINFO_VECTYPE (stmt_info)))
+ (TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info))
== vectorization_factor);
#endif
/* -------- vectorize statement ------------ */
is_store = vect_transform_stmt (stmt, &si);
if (is_store)
{
- /* free the attached stmt_vec_info and remove the stmt. */
+ /* Free the attached stmt_vec_info and remove the stmt. */
stmt_ann_t ann = stmt_ann (stmt);
free (stmt_info);
- set_stmt_info (ann, NULL);
+ set_stmt_info ((tree_ann_t)ann, NULL);
bsi_remove (&si);
continue;
}
slpeel_make_loop_iterate_ntimes (loop, ratio);
+ /* The memory tags and pointers in vectorized statements need to
+ have their SSA forms updated. FIXME, why can't this be delayed
+ until all the loops have been transformed? */
+ update_ssa (TODO_update_ssa);
+
if (vect_print_dump_info (REPORT_VECTORIZED_LOOPS, LOOP_LOC (loop_vinfo)))
fprintf (vect_dump, "LOOP VECTORIZED.");
}
OSDN Git Service
