/* Transformation Utilities for Loop Vectorization.
- Copyright (C) 2003, 2004, 2005, 2006, 2007 Free Software Foundation, Inc.
+ Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc.
Contributed by Dorit Naishlos <dorit@il.ibm.com>
This file is part of GCC.
int vec_inside_cost = 0;
int vec_outside_cost = 0;
int scalar_single_iter_cost = 0;
+ int scalar_outside_cost = 0;
+ bool runtime_test = false;
int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
int nbbs = loop->num_nodes;
- int byte_misalign;
+ int byte_misalign = LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo);
int peel_guard_costs = 0;
int innerloop_iters = 0, factor;
VEC (slp_instance, heap) *slp_instances;
/* Cost model disabled. */
if (!flag_vect_cost_model)
{
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
fprintf (vect_dump, "cost model disabled.");
return 0;
}
+ /* If the number of iterations is unknown, or the
+ peeling-for-misalignment amount is unknown, we will have to generate
+ a runtime test to test the loop count against the threshold. */
+ if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
+ || (byte_misalign < 0))
+ runtime_test = true;
+
/* Requires loop versioning tests to handle misalignment. */
if (VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo)))
/* FIXME: Make cost depend on complexity of individual check. */
vec_outside_cost +=
VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo));
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
fprintf (vect_dump, "cost model: Adding cost of checks for loop "
"versioning to treat misalignment.\n");
}
/* FIXME: Make cost depend on complexity of individual check. */
vec_outside_cost +=
VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo));
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
fprintf (vect_dump, "cost model: Adding cost of checks for loop "
"versioning aliasing.\n");
}
factor = 1;
for (si = bsi_start (bb); !bsi_end_p (si); bsi_next (&si))
- {
- tree stmt = bsi_stmt (si);
- stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
- if (!STMT_VINFO_RELEVANT_P (stmt_info)
- && !STMT_VINFO_LIVE_P (stmt_info))
- continue;
- scalar_single_iter_cost += cost_for_stmt (stmt) * factor;
- vec_inside_cost += STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info) * factor;
+ {
+ tree stmt = bsi_stmt (si);
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ /* Skip stmts that are not vectorized inside the loop. */
+ if (!STMT_VINFO_RELEVANT_P (stmt_info)
+ && (!STMT_VINFO_LIVE_P (stmt_info)
+ || STMT_VINFO_DEF_TYPE (stmt_info) != vect_reduction_def))
+ continue;
+ scalar_single_iter_cost += cost_for_stmt (stmt) * factor;
+ vec_inside_cost += STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info) * factor;
/* FIXME: for stmts in the inner-loop in outer-loop vectorization,
some of the "outside" costs are generated inside the outer-loop. */
- vec_outside_cost += STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info);
- }
+ vec_outside_cost += STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info);
+ }
}
/* Add additional cost for the peeled instructions in prologue and epilogue
loop.
- FORNOW: If we dont know the value of peel_iters for prologue or epilogue
+ FORNOW: If we don't know the value of peel_iters for prologue or epilogue
at compile-time - we assume it's vf/2 (the worst would be vf-1).
TODO: Build an expression that represents peel_iters for prologue and
epilogue to be used in a run-time test. */
- byte_misalign = LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo);
-
if (byte_misalign < 0)
{
peel_iters_prologue = vf/2;
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
fprintf (vect_dump, "cost model: "
"prologue peel iters set to vf/2.");
/* If peeling for alignment is unknown, loop bound of main loop becomes
unknown. */
peel_iters_epilogue = vf/2;
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
fprintf (vect_dump, "cost model: "
"epilogue peel iters set to vf/2 because "
"peeling for alignment is unknown .");
if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
{
peel_iters_epilogue = vf/2;
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
fprintf (vect_dump, "cost model: "
"epilogue peel iters set to vf/2 because "
"loop iterations are unknown .");
+ (peel_iters_epilogue * scalar_single_iter_cost)
+ peel_guard_costs;
- /* Allow targets add additional (outside-of-loop) costs. FORNOW, the only
- information we provide for the target is whether testing against the
- threshold involves a runtime test. */
- if (targetm.vectorize.builtin_vectorization_cost)
- {
- bool runtime_test = false;
-
- /* If the number of iterations is unknown, or the
- peeling-for-misalignment amount is unknown, we eill have to generate
- a runtime test to test the loop count against the threshold. */
- if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
- || (byte_misalign < 0))
- runtime_test = true;
- vec_outside_cost +=
- targetm.vectorize.builtin_vectorization_cost (runtime_test);
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "cost model : Adding target out-of-loop cost = %d",
- targetm.vectorize.builtin_vectorization_cost (runtime_test));
+ /* FORNOW: The scalar outside cost is incremented in one of the
+ following ways:
+
+ 1. The vectorizer checks for alignment and aliasing and generates
+ a condition that allows dynamic vectorization. A cost model
+ check is ANDED with the versioning condition. Hence scalar code
+ path now has the added cost of the versioning check.
+
+ if (cost > th & versioning_check)
+ jmp to vector code
+
+ Hence run-time scalar is incremented by not-taken branch cost.
+
+ 2. The vectorizer then checks if a prologue is required. If the
+ cost model check was not done before during versioning, it has to
+ be done before the prologue check.
+
+ if (cost <= th)
+ prologue = scalar_iters
+ if (prologue == 0)
+ jmp to vector code
+ else
+ execute prologue
+ if (prologue == num_iters)
+ go to exit
+
+ Hence the run-time scalar cost is incremented by a taken branch,
+ plus a not-taken branch, plus a taken branch cost.
+
+ 3. The vectorizer then checks if an epilogue is required. If the
+ cost model check was not done before during prologue check, it
+ has to be done with the epilogue check.
+
+ if (prologue == 0)
+ jmp to vector code
+ else
+ execute prologue
+ if (prologue == num_iters)
+ go to exit
+ vector code:
+ if ((cost <= th) | (scalar_iters-prologue-epilogue == 0))
+ jmp to epilogue
+
+ Hence the run-time scalar cost should be incremented by 2 taken
+ branches.
+
+ TODO: The back end may reorder the BBS's differently and reverse
+ conditions/branch directions. Change the estimates below to
+ something more reasonable. */
+
+ if (runtime_test)
+ {
+ /* Cost model check occurs at versioning. */
+ if (VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo))
+ || VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo)))
+ scalar_outside_cost += TARG_COND_NOT_TAKEN_BRANCH_COST;
+ else
+ {
+ /* Cost model occurs at prologue generation. */
+ if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
+ scalar_outside_cost += 2 * TARG_COND_TAKEN_BRANCH_COST
+ + TARG_COND_NOT_TAKEN_BRANCH_COST;
+ /* Cost model check occurs at epilogue generation. */
+ else
+ scalar_outside_cost += 2 * TARG_COND_TAKEN_BRANCH_COST;
+ }
}
/* Add SLP costs. */
/* Calculate number of iterations required to make the vector version
profitable, relative to the loop bodies only. The following condition
- must hold true: ((SIC*VF)-VIC)*niters > VOC*VF, where
+ must hold true:
+ SIC * niters + SOC > VIC * ((niters-PL_ITERS-EP_ITERS)/VF) + VOC
+ where
SIC = scalar iteration cost, VIC = vector iteration cost,
- VOC = vector outside cost and VF = vectorization factor. */
+ VOC = vector outside cost, VF = vectorization factor,
+ PL_ITERS = prologue iterations, EP_ITERS= epilogue iterations
+ SOC = scalar outside cost for run time cost model check. */
if ((scalar_single_iter_cost * vf) > vec_inside_cost)
{
min_profitable_iters = 1;
else
{
- min_profitable_iters = (vec_outside_cost * vf
- - vec_inside_cost * peel_iters_prologue
+ min_profitable_iters = ((vec_outside_cost - scalar_outside_cost) * vf
+ - vec_inside_cost * peel_iters_prologue
- vec_inside_cost * peel_iters_epilogue)
/ ((scalar_single_iter_cost * vf)
- vec_inside_cost);
if ((scalar_single_iter_cost * vf * min_profitable_iters)
<= ((vec_inside_cost * min_profitable_iters)
- + (vec_outside_cost * vf)))
+ + ((vec_outside_cost - scalar_outside_cost) * vf)))
min_profitable_iters++;
}
}
/* vector version will never be profitable. */
else
{
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
fprintf (vect_dump, "cost model: vector iteration cost = %d "
"is divisible by scalar iteration cost = %d by a factor "
"greater than or equal to the vectorization factor = %d .",
return -1;
}
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
{
fprintf (vect_dump, "Cost model analysis: \n");
fprintf (vect_dump, " Vector inside of loop cost: %d\n",
vec_inside_cost);
fprintf (vect_dump, " Vector outside of loop cost: %d\n",
vec_outside_cost);
- fprintf (vect_dump, " Scalar cost: %d\n", scalar_single_iter_cost);
+ fprintf (vect_dump, " Scalar iteration cost: %d\n",
+ scalar_single_iter_cost);
+ fprintf (vect_dump, " Scalar outside cost: %d\n", scalar_outside_cost);
fprintf (vect_dump, " prologue iterations: %d\n",
peel_iters_prologue);
fprintf (vect_dump, " epilogue iterations: %d\n",
then skip the vectorized loop. */
min_profitable_iters--;
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
fprintf (vect_dump, " Profitability threshold = %d\n",
min_profitable_iters);
vectype = get_vectype_for_scalar_type (TREE_TYPE (reduction_op));
if (!vectype)
{
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
{
fprintf (vect_dump, "unsupported data-type ");
print_generic_expr (vect_dump, TREE_TYPE (reduction_op), TDF_SLIM);
int element_bitsize = tree_low_cst (bitsize, 1);
int nelements = vec_size_in_bits / element_bitsize;
- optab = optab_for_tree_code (code, vectype);
+ optab = optab_for_tree_code (code, vectype, optab_default);
/* We have a whole vector shift available. */
if (VECTOR_MODE_P (mode)
STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info) = outer_cost;
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
fprintf (vect_dump, "vect_model_reduction_cost: inside_cost = %d, "
"outside_cost = %d .", STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info),
STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info));
/* prologue cost for vec_init and vec_step. */
STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info) = 2 * TARG_SCALAR_TO_VEC_COST;
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
fprintf (vect_dump, "vect_model_induction_cost: inside_cost = %d, "
"outside_cost = %d .", STMT_VINFO_INSIDE_OF_LOOP_COST (stmt_info),
STMT_VINFO_OUTSIDE_OF_LOOP_COST (stmt_info));
outside_cost += TARG_SCALAR_TO_VEC_COST;
}
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
fprintf (vect_dump, "vect_model_simple_cost: inside_cost = %d, "
"outside_cost = %d .", inside_cost, outside_cost);
inside_cost = ncopies * exact_log2(group_size) * group_size
* TARG_VEC_STMT_COST;
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
fprintf (vect_dump, "vect_model_store_cost: strided group_size = %d .",
group_size);
/* Costs of the stores. */
inside_cost += ncopies * TARG_VEC_STORE_COST;
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
fprintf (vect_dump, "vect_model_store_cost: inside_cost = %d, "
"outside_cost = %d .", inside_cost, outside_cost);
inside_cost = ncopies * exact_log2(group_size) * group_size
* TARG_VEC_STMT_COST;
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
fprintf (vect_dump, "vect_model_load_cost: strided group_size = %d .",
group_size);
{
inside_cost += ncopies * TARG_VEC_LOAD_COST;
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
fprintf (vect_dump, "vect_model_load_cost: aligned.");
break;
/* Here, we assign an additional cost for the unaligned load. */
inside_cost += ncopies * TARG_VEC_UNALIGNED_LOAD_COST;
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
fprintf (vect_dump, "vect_model_load_cost: unaligned supported by "
"hardware.");
}
case dr_explicit_realign_optimized:
{
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
fprintf (vect_dump, "vect_model_load_cost: unaligned software "
"pipelined.");
gcc_unreachable ();
}
- if (vect_print_dump_info (REPORT_DETAILS))
+ if (vect_print_dump_info (REPORT_COST))
fprintf (vect_dump, "vect_model_load_cost: inside_cost = %d, "
"outside_cost = %d .", inside_cost, outside_cost);
else
set_symbol_mem_tag (vect_ptr, tag);
- var_ann (vect_ptr)->subvars = DR_SUBVARS (dr);
-
/** Note: If the dataref is in an inner-loop nested in LOOP, and we are
vectorizing LOOP (i.e. outer-loop vectorization), we need to create two
def-use update cycles for the pointer: One relative to the outer-loop
new_temp = vect_create_addr_base_for_vector_ref (stmt, &new_stmt_list,
offset, loop);
pe = loop_preheader_edge (loop);
- new_bb = bsi_insert_on_edge_immediate (pe, new_stmt_list);
- gcc_assert (!new_bb);
+ if (new_stmt_list)
+ {
+ new_bb = bsi_insert_on_edge_immediate (pe, new_stmt_list);
+ gcc_assert (!new_bb);
+ }
+
*initial_address = new_temp;
/* Create: p = (vectype *) initial_base */
created vectors. It is greater than 1 if unrolling is performed.
For example, we have two scalar operands, s1 and s2 (e.g., group of
- strided accesses of size two), while NUINTS is four (i.e., four scalars
+ strided accesses of size two), while NUNITS is four (i.e., four scalars
of this type can be packed in a vector). The output vector will contain
two copies of each scalar operand: {s1, s2, s1, s2}. (NUMBER_OF_COPIES
will be 2).
If GROUP_SIZE > NUNITS, the scalars will be split into several vectors
containing the operands.
- For example, NUINTS is four as before, and the group size is 8
+ For example, NUNITS is four as before, and the group size is 8
(s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
{s5, s6, s7, s8}. */
/* Make sure bsi points to the stmt that is being vectorized. */
gcc_assert (stmt == bsi_stmt (*bsi));
-#ifdef USE_MAPPED_LOCATION
SET_EXPR_LOCATION (vec_stmt, EXPR_LOCATION (stmt));
-#else
- SET_EXPR_LOCUS (vec_stmt, EXPR_LOCUS (stmt));
-#endif
}
have_whole_vector_shift = false;
else
{
- optab optab = optab_for_tree_code (code, vectype);
+ optab optab = optab_for_tree_code (code, vectype, optab_default);
if (optab_handler (optab, mode)->insn_code == CODE_FOR_nothing)
have_whole_vector_shift = false;
}
vec_size_in_bits = tree_low_cst (TYPE_SIZE (vectype), 1);
rhs = build3 (BIT_FIELD_REF, scalar_type, vec_temp, bitsize,
bitsize_zero_node);
- BIT_FIELD_REF_UNSIGNED (rhs) = TYPE_UNSIGNED (scalar_type);
epilog_stmt = build_gimple_modify_stmt (new_scalar_dest, rhs);
new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_temp;
tree rhs = build3 (BIT_FIELD_REF, scalar_type, vec_temp, bitsize,
bitpos);
- BIT_FIELD_REF_UNSIGNED (rhs) = TYPE_UNSIGNED (scalar_type);
epilog_stmt = build_gimple_modify_stmt (new_scalar_dest, rhs);
new_name = make_ssa_name (new_scalar_dest, epilog_stmt);
GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_name;
bitpos = bitsize_zero_node;
rhs = build3 (BIT_FIELD_REF, scalar_type, new_temp, bitsize, bitpos);
- BIT_FIELD_REF_UNSIGNED (rhs) = TYPE_UNSIGNED (scalar_type);
epilog_stmt = build_gimple_modify_stmt (new_scalar_dest, rhs);
new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_temp;
/* 4. Supportable by target? */
/* 4.1. check support for the operation in the loop */
- optab = optab_for_tree_code (code, vectype);
+ optab = optab_for_tree_code (code, vectype, optab_default);
if (!optab)
{
if (vect_print_dump_info (REPORT_DETAILS))
if (!reduction_code_for_scalar_code (orig_code, &epilog_reduc_code))
return false;
- reduc_optab = optab_for_tree_code (epilog_reduc_code, vectype);
+ reduc_optab = optab_for_tree_code (epilog_reduc_code, vectype, optab_default);
if (!reduc_optab)
{
if (vect_print_dump_info (REPORT_DETAILS))
*vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
}
+ if (vec_oprnds0)
+ VEC_free (tree, heap, vec_oprnds0);
+
return true;
}
VEC(tree,heap) *vec_oprnds = NULL;
tree vop;
+ /* FORNOW: SLP with multiple types is not supported. The SLP analysis
+ verifies this, so we can safely override NCOPIES with 1 here. */
+ if (slp_node)
+ ncopies = 1;
+
gcc_assert (ncopies >= 1);
if (ncopies > 1)
return false; /* FORNOW */
VEC(tree,heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL;
tree vop0, vop1;
unsigned int k;
+ bool shift_p = false;
bool scalar_shift_arg = false;
/* FORNOW: SLP with multiple types is not supported. The SLP analysis verifies
if (code == POINTER_PLUS_EXPR)
code = PLUS_EXPR;
- optab = optab_for_tree_code (code, vectype);
-
/* Support only unary or binary operations. */
op_type = TREE_OPERAND_LENGTH (operation);
if (op_type != unary_op && op_type != binary_op)
}
}
+ /* If this is a shift/rotate, determine whether the shift amount is a vector,
+ or scalar. If the shift/rotate amount is a vector, use the vector/vector
+ shift optabs. */
+ if (code == LSHIFT_EXPR || code == RSHIFT_EXPR || code == LROTATE_EXPR
+ || code == RROTATE_EXPR)
+ {
+ shift_p = true;
+
+ /* vector shifted by vector */
+ if (dt[1] == vect_loop_def)
+ {
+ optab = optab_for_tree_code (code, vectype, optab_vector);
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "vector/vector shift/rotate found.");
+ }
+
+ /* See if the machine has a vector shifted by scalar insn and if not
+ then see if it has a vector shifted by vector insn */
+ else if (dt[1] == vect_constant_def || dt[1] == vect_invariant_def)
+ {
+ optab = optab_for_tree_code (code, vectype, optab_scalar);
+ if (optab
+ && (optab_handler (optab, TYPE_MODE (vectype))->insn_code
+ != CODE_FOR_nothing))
+ {
+ scalar_shift_arg = true;
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "vector/scalar shift/rotate found.");
+ }
+ else
+ {
+ optab = optab_for_tree_code (code, vectype, optab_vector);
+ if (vect_print_dump_info (REPORT_DETAILS)
+ && optab
+ && (optab_handler (optab, TYPE_MODE (vectype))->insn_code
+ != CODE_FOR_nothing))
+ fprintf (vect_dump, "vector/vector shift/rotate found.");
+ }
+ }
+
+ else
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "operand mode requires invariant argument.");
+ return false;
+ }
+ }
+ else
+ optab = optab_for_tree_code (code, vectype, optab_default);
+
/* Supportable by target? */
if (!optab)
{
return false;
}
- if (code == LSHIFT_EXPR || code == RSHIFT_EXPR)
- {
- /* FORNOW: not yet supported. */
- if (!VECTOR_MODE_P (vec_mode))
- return false;
-
- /* Invariant argument is needed for a vector shift
- by a scalar shift operand. */
- optab_op2_mode = insn_data[icode].operand[2].mode;
- if (!VECTOR_MODE_P (optab_op2_mode))
- {
- if (dt[1] != vect_constant_def && dt[1] != vect_invariant_def)
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "operand mode requires invariant"
- " argument.");
- return false;
- }
-
- scalar_shift_arg = true;
- }
- }
-
if (!vec_stmt) /* transformation not required. */
{
STMT_VINFO_TYPE (stmt_info) = op_vec_info_type;
/* Handle uses. */
if (j == 0)
{
- if (op_type == binary_op
- && (code == LSHIFT_EXPR || code == RSHIFT_EXPR))
+ if (op_type == binary_op && scalar_shift_arg)
{
/* Vector shl and shr insn patterns can be defined with scalar
operand 2 (shift operand). In this case, use constant or loop
/* Arguments are ready. Create the new vector stmt. We are creating
two vector defs because the widened result does not fit in one vector.
- The vectorized stmt can be expressed as a call to a taregt builtin,
+ The vectorized stmt can be expressed as a call to a target builtin,
or a using a tree-code. */
/* Generate first half of the widened result: */
new_stmt = vect_gen_widened_results_half (code1, vectype_out, decl1,
/* Check that the operation is supported. */
interleave_high_optab = optab_for_tree_code (VEC_INTERLEAVE_HIGH_EXPR,
- vectype);
+ vectype, optab_default);
interleave_low_optab = optab_for_tree_code (VEC_INTERLEAVE_LOW_EXPR,
- vectype);
+ vectype, optab_default);
if (!interleave_high_optab || !interleave_low_optab)
{
if (vect_print_dump_info (REPORT_DETAILS))
tree scalar_dest, tmp;
int i;
unsigned int j;
- VEC(tree,heap) *first, *second;
scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
- first = VEC_alloc (tree, heap, length/2);
- second = VEC_alloc (tree, heap, length/2);
/* Check that the operation is supported. */
if (!vect_strided_store_supported (vectype))
}
}
+ VEC_free (tree, heap, dr_chain);
+ VEC_free (tree, heap, oprnds);
+ if (result_chain)
+ VEC_free (tree, heap, result_chain);
+
return true;
}
new_stmt = build_gimple_modify_stmt (vec_dest, data_ref);
new_temp = make_ssa_name (vec_dest, new_stmt);
GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
+ mark_symbols_for_renaming (new_stmt);
new_bb = bsi_insert_on_edge_immediate (pe, new_stmt);
gcc_assert (!new_bb);
msq_init = GIMPLE_STMT_OPERAND (new_stmt, 0);
mode = (int) TYPE_MODE (vectype);
- perm_even_optab = optab_for_tree_code (VEC_EXTRACT_EVEN_EXPR, vectype);
+ perm_even_optab = optab_for_tree_code (VEC_EXTRACT_EVEN_EXPR, vectype,
+ optab_default);
if (!perm_even_optab)
{
if (vect_print_dump_info (REPORT_DETAILS))
return false;
}
- perm_odd_optab = optab_for_tree_code (VEC_EXTRACT_ODD_EXPR, vectype);
+ perm_odd_optab = optab_for_tree_code (VEC_EXTRACT_ODD_EXPR, vectype,
+ optab_default);
if (!perm_odd_optab)
{
if (vect_print_dump_info (REPORT_DETAILS))
break;
/* Skip the gaps. Loads created for the gaps will be removed by dead
- code elimination pass later.
+ code elimination pass later. No need to check for the first stmt in
+ the group, since it always exists.
DR_GROUP_GAP is the number of steps in elements from the previous
access (if there is no gap DR_GROUP_GAP is 1). We skip loads that
correspond to the gaps.
*/
- if (gap_count < DR_GROUP_GAP (vinfo_for_stmt (next_stmt)))
+ if (next_stmt != first_stmt
+ && gap_count < DR_GROUP_GAP (vinfo_for_stmt (next_stmt)))
{
gap_count++;
continue;
break;
}
}
+
+ VEC_free (tree, heap, result_chain);
return true;
}
nested within an outer-loop that is being vectorized. */
if (nested_in_vect_loop_p (loop, stmt)
- && (TREE_INT_CST_LOW (DR_STEP (dr)) % UNITS_PER_SIMD_WORD != 0))
+ && (TREE_INT_CST_LOW (DR_STEP (dr))
+ % GET_MODE_SIZE (TYPE_MODE (vectype)) != 0))
{
gcc_assert (alignment_support_scheme != dr_explicit_realign_optimized);
compute_in_loop = true;
bitpos = bitsize_zero_node;
vec_inv = build3 (BIT_FIELD_REF, scalar_type, new_temp,
bitsize, bitpos);
- BIT_FIELD_REF_UNSIGNED (vec_inv) =
- TYPE_UNSIGNED (scalar_type);
vec_dest =
vect_create_destination_var (scalar_dest, NULL_TREE);
new_stmt = build_gimple_modify_stmt (vec_dest, vec_inv);
if (!vect_transform_strided_load (stmt, dr_chain, group_size, bsi))
return false;
*vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
+ VEC_free (tree, heap, dr_chain);
dr_chain = VEC_alloc (tree, heap, group_size);
}
else
}
}
+ if (dr_chain)
+ VEC_free (tree, heap, dr_chain);
+
return true;
}
vec_then_clause = vect_get_vec_def_for_operand (then_clause, stmt, NULL);
vec_else_clause = vect_get_vec_def_for_operand (else_clause, stmt, NULL);
- /* Arguments are ready. create the new vector stmt. */
+ /* Arguments are ready. Create the new vector stmt. */
vec_compare = build2 (TREE_CODE (cond_expr), vectype,
vec_cond_lhs, vec_cond_rhs);
vec_cond_expr = build3 (VEC_COND_EXPR, vectype,
}
}
+/* Return the more conservative threshold between the
+ min_profitable_iters returned by the cost model and the user
+ specified threshold, if provided. */
+
+static unsigned int
+conservative_cost_threshold (loop_vec_info loop_vinfo,
+ int min_profitable_iters)
+{
+ unsigned int th;
+ int min_scalar_loop_bound;
+
+ min_scalar_loop_bound = ((PARAM_VALUE (PARAM_MIN_VECT_LOOP_BOUND)
+ * LOOP_VINFO_VECT_FACTOR (loop_vinfo)) - 1);
+
+ /* Use the cost model only if it is more conservative than user specified
+ threshold. */
+ th = (unsigned) min_scalar_loop_bound;
+ if (min_profitable_iters
+ && (!min_scalar_loop_bound
+ || min_profitable_iters > min_scalar_loop_bound))
+ th = (unsigned) min_profitable_iters;
+
+ if (th && vect_print_dump_info (REPORT_COST))
+ fprintf (vect_dump, "Vectorization may not be profitable.");
+
+ return th;
+}
/* Function vect_do_peeling_for_loop_bound
edge update_e;
basic_block preheader;
int loop_num;
- unsigned int th;
- int min_scalar_loop_bound;
+ bool check_profitability = false;
+ unsigned int th = 0;
int min_profitable_iters;
if (vect_print_dump_info (REPORT_DETAILS))
loop_num = loop->num;
- /* Analyze cost to set threshhold for vectorized loop. */
- min_profitable_iters = LOOP_VINFO_COST_MODEL_MIN_ITERS (loop_vinfo);
- min_scalar_loop_bound = ((PARAM_VALUE (PARAM_MIN_VECT_LOOP_BOUND)
- * LOOP_VINFO_VECT_FACTOR (loop_vinfo)) - 1);
+ /* If cost model check not done during versioning and
+ peeling for alignment. */
+ if (!VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo))
+ && !VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo))
+ && !LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo))
+ {
+ check_profitability = true;
- /* Use the cost model only if it is more conservative than user specified
- threshold. */
+ /* Get profitability threshold for vectorized loop. */
+ min_profitable_iters = LOOP_VINFO_COST_MODEL_MIN_ITERS (loop_vinfo);
- th = (unsigned) min_scalar_loop_bound;
- if (min_profitable_iters
- && (!min_scalar_loop_bound
- || min_profitable_iters > min_scalar_loop_bound))
- th = (unsigned) min_profitable_iters;
-
- if (((LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo) < 0)
- || !LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
- && vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "vectorization may not be profitable.");
+ th = conservative_cost_threshold (loop_vinfo,
+ min_profitable_iters);
+ }
new_loop = slpeel_tree_peel_loop_to_edge (loop, single_exit (loop),
ratio_mult_vf_name, ni_name, false,
- th);
+ th, check_profitability);
gcc_assert (new_loop);
gcc_assert (loop_num == loop->num);
#ifdef ENABLE_CHECKING
tree niters_of_prolog_loop, ni_name;
tree n_iters;
struct loop *new_loop;
+ bool check_profitability = false;
+ unsigned int th = 0;
+ int min_profitable_iters;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_do_peeling_for_alignment ===");
ni_name = vect_build_loop_niters (loop_vinfo);
niters_of_prolog_loop = vect_gen_niters_for_prolog_loop (loop_vinfo, ni_name);
+
+ /* If cost model check not done during versioning. */
+ if (!VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo))
+ && !VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo)))
+ {
+ check_profitability = true;
+
+ /* Get profitability threshold for vectorized loop. */
+ min_profitable_iters = LOOP_VINFO_COST_MODEL_MIN_ITERS (loop_vinfo);
+
+ th = conservative_cost_threshold (loop_vinfo,
+ min_profitable_iters);
+ }
+
/* Peel the prolog loop and iterate it niters_of_prolog_loop. */
- new_loop =
- slpeel_tree_peel_loop_to_edge (loop, loop_preheader_edge (loop),
- niters_of_prolog_loop, ni_name, true, 0);
+ new_loop =
+ slpeel_tree_peel_loop_to_edge (loop, loop_preheader_edge (loop),
+ niters_of_prolog_loop, ni_name, true,
+ th, check_profitability);
+
gcc_assert (new_loop);
#ifdef ENABLE_CHECKING
slpeel_verify_cfg_after_peeling (new_loop, loop);
checked at runtime.
Input:
+ COND_EXPR - input conditional expression. New conditions will be chained
+ with logical AND operation.
LOOP_VINFO - two fields of the loop information are used.
LOOP_VINFO_PTR_MASK is the mask used to check the alignment.
LOOP_VINFO_MAY_MISALIGN_STMTS contains the refs to be checked.
test can be done as a&(n-1) == 0. For example, for 16
byte vectors the test is a&0xf == 0. */
-static tree
+static void
vect_create_cond_for_align_checks (loop_vec_info loop_vinfo,
+ tree *cond_expr,
tree *cond_expr_stmt_list)
{
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
tree or_tmp_name = NULL_TREE;
tree and_tmp, and_tmp_name, and_stmt;
tree ptrsize_zero;
+ tree part_cond_expr;
/* Check that mask is one less than a power of 2, i.e., mask is
all zeros followed by all ones. */
/* Make and_tmp the left operand of the conditional test against zero.
if and_tmp has a nonzero bit then some address is unaligned. */
ptrsize_zero = build_int_cst (int_ptrsize_type, 0);
- return build2 (EQ_EXPR, boolean_type_node,
- and_tmp_name, ptrsize_zero);
+ part_cond_expr = fold_build2 (EQ_EXPR, boolean_type_node,
+ and_tmp_name, ptrsize_zero);
+ if (*cond_expr)
+ *cond_expr = fold_build2 (TRUTH_AND_EXPR, boolean_type_node,
+ *cond_expr, part_cond_expr);
+ else
+ *cond_expr = part_cond_expr;
}
/* Function vect_vfa_segment_size.
Input:
COND_EXPR - input conditional expression. New conditions will be chained
- with logical and operation.
+ with logical AND operation.
LOOP_VINFO - field LOOP_VINFO_MAY_ALIAS_STMTS contains the list of ddrs
to be checked.
data references that may or may not be aligned. An additional
sequence of runtime tests is generated for each pairs of DDRs whose
independence was not proven. The vectorized version of loop is
- executed only if both alias and alignment tests are passed. */
+ executed only if both alias and alignment tests are passed.
+
+ The test generated to check which version of loop is executed
+ is modified to also check for profitability as indicated by the
+ cost model initially. */
static void
vect_loop_versioning (loop_vec_info loop_vinfo)
tree orig_phi, new_phi, arg;
unsigned prob = 4 * REG_BR_PROB_BASE / 5;
tree gimplify_stmt_list;
+ tree scalar_loop_iters = LOOP_VINFO_NITERS (loop_vinfo);
+ int min_profitable_iters = 0;
+ unsigned int th;
- if (!VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo))
- && !VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo)))
- return;
+ /* Get profitability threshold for vectorized loop. */
+ min_profitable_iters = LOOP_VINFO_COST_MODEL_MIN_ITERS (loop_vinfo);
+
+ th = conservative_cost_threshold (loop_vinfo,
+ min_profitable_iters);
+
+ cond_expr =
+ build2 (GT_EXPR, boolean_type_node, scalar_loop_iters,
+ build_int_cst (TREE_TYPE (scalar_loop_iters), th));
+
+ cond_expr = force_gimple_operand (cond_expr, &cond_expr_stmt_list,
+ false, NULL_TREE);
if (VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo)))
- cond_expr =
- vect_create_cond_for_align_checks (loop_vinfo, &cond_expr_stmt_list);
+ vect_create_cond_for_align_checks (loop_vinfo, &cond_expr,
+ &cond_expr_stmt_list);
if (VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo)))
- vect_create_cond_for_alias_checks (loop_vinfo, &cond_expr, &cond_expr_stmt_list);
+ vect_create_cond_for_alias_checks (loop_vinfo, &cond_expr,
+ &cond_expr_stmt_list);
cond_expr =
fold_build2 (NE_EXPR, boolean_type_node, cond_expr, integer_zero_node);
static void
vect_remove_stores (tree first_stmt)
{
- stmt_ann_t ann;
tree next = first_stmt;
tree tmp;
- stmt_vec_info next_stmt_info;
block_stmt_iterator next_si;
while (next)
/* Free the attached stmt_vec_info and remove the stmt. */
next_si = bsi_for_stmt (next);
bsi_remove (&next_si, true);
- next_stmt_info = vinfo_for_stmt (next);
- ann = stmt_ann (next);
- tmp = DR_GROUP_NEXT_DR (next_stmt_info);
- free (next_stmt_info);
- set_stmt_info (ann, NULL);
+ tmp = DR_GROUP_NEXT_DR (vinfo_for_stmt (next));
+ free_stmt_vec_info (next);
next = tmp;
}
}
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
int nbbs = loop->num_nodes;
- block_stmt_iterator si, next_si;
+ block_stmt_iterator si;
int i;
tree ratio = NULL;
int vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vec_transform_loop ===");
- vect_loop_versioning (loop_vinfo);
+
+ if (VEC_length (tree, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo))
+ || VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo)))
+ vect_loop_versioning (loop_vinfo);
/* CHECKME: we wouldn't need this if we called update_ssa once
for all loops. */
is_store = vect_transform_stmt (stmt, &si, &strided_store, NULL);
if (is_store)
{
- stmt_ann_t ann;
if (STMT_VINFO_STRIDED_ACCESS (stmt_info))
{
/* Interleaving. If IS_STORE is TRUE, the vectorization of the
interleaving chain was completed - free all the stores in
the chain. */
- tree next = DR_GROUP_FIRST_DR (stmt_info);
- tree tmp;
- stmt_vec_info next_stmt_info;
-
- while (next)
- {
- next_si = bsi_for_stmt (next);
- next_stmt_info = vinfo_for_stmt (next);
- /* Free the attached stmt_vec_info and remove the stmt. */
- ann = stmt_ann (next);
- tmp = DR_GROUP_NEXT_DR (next_stmt_info);
- free (next_stmt_info);
- set_stmt_info (ann, NULL);
- bsi_remove (&next_si, true);
- next = tmp;
- }
+ vect_remove_stores (DR_GROUP_FIRST_DR (stmt_info));
bsi_remove (&si, true);
continue;
}
else
{
/* Free the attached stmt_vec_info and remove the stmt. */
- ann = stmt_ann (stmt);
- free (stmt_info);
- set_stmt_info (ann, NULL);
+ free_stmt_vec_info (stmt);
bsi_remove (&si, true);
continue;
}
OSDN Git Service
