Check if DRA and DRB are a part of interleaving. In case they are, insert
DRA and DRB in an interleaving chain. */
-static void
+static bool
vect_check_interleaving (struct data_reference *dra,
struct data_reference *drb)
{
|| !vect_equal_offsets (DR_OFFSET (dra), DR_OFFSET (drb))
|| !tree_int_cst_compare (DR_INIT (dra), DR_INIT (drb))
|| DR_IS_READ (dra) != DR_IS_READ (drb))
- return;
+ return false;
/* Check:
1. data-refs are of the same type
2. their steps are equal
- 3. the step is greater than the difference between data-refs' inits */
+ 3. the step (if greater than zero) is greater than the difference between
+ data-refs' inits. */
type_size_a = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dra))));
type_size_b = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (drb))));
|| tree_int_cst_compare (DR_STEP (dra), DR_STEP (drb))
|| !types_compatible_p (TREE_TYPE (DR_REF (dra)),
TREE_TYPE (DR_REF (drb))))
- return;
+ return false;
init_a = TREE_INT_CST_LOW (DR_INIT (dra));
init_b = TREE_INT_CST_LOW (DR_INIT (drb));
and DRB is accessed before DRA. */
diff_mod_size = (init_a - init_b) % type_size_a;
- if ((init_a - init_b) > step)
- return;
+ if (step && (init_a - init_b) > step)
+ return false;
if (diff_mod_size == 0)
{
fprintf (vect_dump, " and ");
print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM);
}
- return;
+ return true;
}
}
else
interleaving, and DRA is accessed before DRB. */
diff_mod_size = (init_b - init_a) % type_size_a;
- if ((init_b - init_a) > step)
- return;
+ if (step && (init_b - init_a) > step)
+ return false;
if (diff_mod_size == 0)
{
fprintf (vect_dump, " and ");
print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM);
}
- return;
+ return true;
}
}
+
+ return false;
}
/* Check if data references pointed by DR_I and DR_J are same or
return true;
}
+
/* Function vect_analyze_data_ref_dependence.
Return TRUE if there (might) exist a dependence between a memory-reference
loop_vec_info loop_vinfo)
{
unsigned int i;
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- int vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ struct loop *loop = NULL;
+ int vectorization_factor = 0;
struct data_reference *dra = DDR_A (ddr);
struct data_reference *drb = DDR_B (ddr);
stmt_vec_info stmtinfo_a = vinfo_for_stmt (DR_STMT (dra));
return false;
}
- if ((DR_IS_READ (dra) && DR_IS_READ (drb)) || dra == drb)
+ if (loop_vinfo)
+ {
+ loop = LOOP_VINFO_LOOP (loop_vinfo);
+ vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ }
+
+ if ((DR_IS_READ (dra) && DR_IS_READ (drb) && loop_vinfo) || dra == drb)
return false;
if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know)
{
+ if (loop_vinfo)
+ {
+ if (vect_print_dump_info (REPORT_DR_DETAILS))
+ {
+ fprintf (vect_dump, "versioning for alias required: "
+ "can't determine dependence between ");
+ print_generic_expr (vect_dump, DR_REF (dra), TDF_SLIM);
+ fprintf (vect_dump, " and ");
+ print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM);
+ }
+
+ /* Add to list of ddrs that need to be tested at run-time. */
+ return !vect_mark_for_runtime_alias_test (ddr, loop_vinfo);
+ }
+
+ /* When vectorizing a basic block unknown depnedence can still mean
+ strided access. */
+ if (vect_check_interleaving (dra, drb))
+ return false;
+
if (vect_print_dump_info (REPORT_DR_DETAILS))
{
- fprintf (vect_dump,
- "versioning for alias required: can't determine dependence between ");
+ fprintf (vect_dump, "can't determine dependence between ");
print_generic_expr (vect_dump, DR_REF (dra), TDF_SLIM);
fprintf (vect_dump, " and ");
print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM);
}
- /* Add to list of ddrs that need to be tested at run-time. */
- return !vect_mark_for_runtime_alias_test (ddr, loop_vinfo);
+
+ return true;
+ }
+
+ /* Versioning for alias is not yet supported for basic block SLP, and
+ dependence distance is unapplicable, hence, in case of known data
+ dependence, basic block vectorization is impossible for now. */
+ if (!loop_vinfo)
+ {
+ if (dra != drb && vect_check_interleaving (dra, drb))
+ return false;
+
+ if (vect_print_dump_info (REPORT_DR_DETAILS))
+ {
+ fprintf (vect_dump, "determined dependence between ");
+ print_generic_expr (vect_dump, DR_REF (dra), TDF_SLIM);
+ fprintf (vect_dump, " and ");
+ print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM);
+ }
+
+ return true;
}
+ /* Loop-based vectorization and known data dependence. */
if (DDR_NUM_DIST_VECTS (ddr) == 0)
{
if (vect_print_dump_info (REPORT_DR_DETAILS))
continue;
}
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
{
- fprintf (vect_dump,
- "not vectorized, possible dependence "
- "between data-refs ");
+ fprintf (vect_dump, "not vectorized, possible dependence "
+ "between data-refs ");
print_generic_expr (vect_dump, DR_REF (dra), TDF_SLIM);
fprintf (vect_dump, " and ");
print_generic_expr (vect_dump, DR_REF (drb), TDF_SLIM);
exist any data dependences between them. */
bool
-vect_analyze_data_ref_dependences (loop_vec_info loop_vinfo)
+vect_analyze_data_ref_dependences (loop_vec_info loop_vinfo,
+ bb_vec_info bb_vinfo)
{
unsigned int i;
- VEC (ddr_p, heap) * ddrs = LOOP_VINFO_DDRS (loop_vinfo);
+ VEC (ddr_p, heap) *ddrs = NULL;
struct data_dependence_relation *ddr;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_analyze_dependences ===");
+ if (loop_vinfo)
+ ddrs = LOOP_VINFO_DDRS (loop_vinfo);
+ else
+ ddrs = BB_VINFO_DDRS (bb_vinfo);
+
for (i = 0; VEC_iterate (ddr_p, ddrs, i, ddr); i++)
if (vect_analyze_data_ref_dependence (ddr, loop_vinfo))
return false;
gimple stmt = DR_STMT (dr);
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ struct loop *loop = NULL;
tree ref = DR_REF (dr);
tree vectype;
tree base, base_addr;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "vect_compute_data_ref_alignment:");
+ if (loop_vinfo)
+ loop = LOOP_VINFO_LOOP (loop_vinfo);
+
/* Initialize misalignment to unknown. */
SET_DR_MISALIGNMENT (dr, -1);
stays the same throughout the execution of the inner-loop, which is why
we have to check that the stride of the dataref in the inner-loop evenly
divides by the vector size. */
- if (nested_in_vect_loop_p (loop, stmt))
+ if (loop && nested_in_vect_loop_p (loop, stmt))
{
tree step = DR_STEP (dr);
HOST_WIDE_INT dr_step = TREE_INT_CST_LOW (step);
Return FALSE if a data reference is found that cannot be vectorized. */
static bool
-vect_compute_data_refs_alignment (loop_vec_info loop_vinfo)
+vect_compute_data_refs_alignment (loop_vec_info loop_vinfo,
+ bb_vec_info bb_vinfo)
{
- VEC (data_reference_p, heap) *datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
+ VEC (data_reference_p, heap) *datarefs;
struct data_reference *dr;
unsigned int i;
+ if (loop_vinfo)
+ datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
+ else
+ datarefs = BB_VINFO_DATAREFS (bb_vinfo);
+
for (i = 0; VEC_iterate (data_reference_p, datarefs, i, dr); i++)
if (!vect_compute_data_ref_alignment (dr))
return false;
Return TRUE if all data references in the loop can be
handled with respect to alignment. */
-static bool
-vect_verify_datarefs_alignment (loop_vec_info loop_vinfo)
+bool
+vect_verify_datarefs_alignment (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo)
{
- VEC (data_reference_p, heap) *datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
+ VEC (data_reference_p, heap) *datarefs;
struct data_reference *dr;
enum dr_alignment_support supportable_dr_alignment;
unsigned int i;
+ if (loop_vinfo)
+ datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
+ else
+ datarefs = BB_VINFO_DATAREFS (bb_vinfo);
+
for (i = 0; VEC_iterate (data_reference_p, datarefs, i, dr); i++)
{
gimple stmt = DR_STMT (dr);
supportable_dr_alignment = vect_supportable_dr_alignment (dr);
if (!supportable_dr_alignment)
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
{
if (DR_IS_READ (dr))
fprintf (vect_dump,
/* While cost model enhancements are expected in the future, the high level
view of the code at this time is as follows:
- A) If there is a misaligned write then see if peeling to align this write
- can make all data references satisfy vect_supportable_dr_alignment.
- If so, update data structures as needed and return true. Note that
- at this time vect_supportable_dr_alignment is known to return false
- for a misaligned write.
+ A) If there is an unsupported misaligned access then see if peeling
+ to align this access can make all data references satisfy
+ vect_supportable_dr_alignment. If so, update data structures
+ as needed and return true.
B) If peeling wasn't possible and there is a data reference with an
unknown misalignment that does not satisfy vect_supportable_dr_alignment
in code size).
The scheme we use FORNOW: peel to force the alignment of the first
- misaligned store in the loop.
- Rationale: misaligned stores are not yet supported.
+ unsupported misaligned access in the loop.
TODO: Use a cost model. */
{
stmt = DR_STMT (dr);
stmt_info = vinfo_for_stmt (stmt);
+ supportable_dr_alignment = vect_supportable_dr_alignment (dr);
/* For interleaving, only the alignment of the first access
matters. */
&& DR_GROUP_FIRST_DR (stmt_info) != stmt)
continue;
- if (!DR_IS_READ (dr) && !aligned_access_p (dr))
+ if (!supportable_dr_alignment)
{
do_peeling = vector_alignment_reachable_p (dr);
if (do_peeling)
}
}
- vect_versioning_for_alias_required =
- (VEC_length (ddr_p, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo)) > 0);
+ vect_versioning_for_alias_required
+ = LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo);
/* Temporarily, if versioning for alias is required, we disable peeling
until we support peeling and versioning. Often peeling for alignment
will require peeling for loop-bound, which in turn requires that we
know how to adjust the loop ivs after the loop. */
if (vect_versioning_for_alias_required
- || !vect_can_advance_ivs_p (loop_vinfo)
+ || !vect_can_advance_ivs_p (loop_vinfo)
|| !slpeel_can_duplicate_loop_p (loop, single_exit (loop)))
do_peeling = false;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "Peeling for alignment will be applied.");
- stat = vect_verify_datarefs_alignment (loop_vinfo);
+ stat = vect_verify_datarefs_alignment (loop_vinfo, NULL);
gcc_assert (stat);
return stat;
}
}
/* Versioning requires at least one misaligned data reference. */
- if (VEC_length (gimple, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo)) == 0)
+ if (!LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo))
do_versioning = false;
else if (!do_versioning)
VEC_truncate (gimple, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo), 0);
/* Peeling and versioning can't be done together at this time. */
gcc_assert (! (do_peeling && do_versioning));
- stat = vect_verify_datarefs_alignment (loop_vinfo);
+ stat = vect_verify_datarefs_alignment (loop_vinfo, NULL);
gcc_assert (stat);
return stat;
}
/* This point is reached if neither peeling nor versioning is being done. */
gcc_assert (! (do_peeling || do_versioning));
- stat = vect_verify_datarefs_alignment (loop_vinfo);
+ stat = vect_verify_datarefs_alignment (loop_vinfo, NULL);
return stat;
}
Return FALSE if a data reference is found that cannot be vectorized. */
bool
-vect_analyze_data_refs_alignment (loop_vec_info loop_vinfo)
+vect_analyze_data_refs_alignment (loop_vec_info loop_vinfo,
+ bb_vec_info bb_vinfo)
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_analyze_data_refs_alignment ===");
- if (!vect_compute_data_refs_alignment (loop_vinfo))
+ if (!vect_compute_data_refs_alignment (loop_vinfo, bb_vinfo))
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
fprintf (vect_dump,
"not vectorized: can't calculate alignment for data ref.");
return false;
gimple stmt = DR_STMT (dr);
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
HOST_WIDE_INT dr_step = TREE_INT_CST_LOW (step);
HOST_WIDE_INT stride;
bool slp_impossible = false;
DR_GROUP_SIZE (vinfo_for_stmt (stmt)) = stride;
if (vect_print_dump_info (REPORT_DR_DETAILS))
{
- fprintf (vect_dump, "Detected single element interleaving %d ",
- DR_GROUP_SIZE (vinfo_for_stmt (stmt)));
+ fprintf (vect_dump, "Detected single element interleaving ");
print_generic_expr (vect_dump, DR_REF (dr), TDF_SLIM);
fprintf (vect_dump, " step ");
print_generic_expr (vect_dump, step, TDF_SLIM);
tree next_step;
tree prev_init = DR_INIT (data_ref);
gimple prev = stmt;
- HOST_WIDE_INT diff, count_in_bytes;
+ HOST_WIDE_INT diff, count_in_bytes, gaps = 0;
while (next)
{
fprintf (vect_dump, "interleaved store with gaps");
return false;
}
+
+ gaps += diff - 1;
}
/* Store the gap from the previous member of the group. If there is no
the type to get COUNT_IN_BYTES. */
count_in_bytes = type_size * count;
- /* Check that the size of the interleaving is not greater than STEP. */
- if (dr_step < count_in_bytes)
+ /* Check that the size of the interleaving (including gaps) is not
+ greater than STEP. */
+ if (dr_step && dr_step < count_in_bytes + gaps * type_size)
{
if (vect_print_dump_info (REPORT_DETAILS))
{
/* Check that the size of the interleaving is equal to STEP for stores,
i.e., that there are no gaps. */
- if (dr_step != count_in_bytes)
+ if (dr_step && dr_step != count_in_bytes)
{
if (DR_IS_READ (dr))
{
}
/* Check that STEP is a multiple of type size. */
- if ((dr_step % type_size) != 0)
+ if (dr_step && (dr_step % type_size) != 0)
{
if (vect_print_dump_info (REPORT_DETAILS))
{
if (slp_impossible)
return false;
}
+
+ if (stride == 0)
+ stride = count;
+
DR_GROUP_SIZE (vinfo_for_stmt (stmt)) = stride;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "Detected interleaving of size %d", (int)stride);
/* SLP: create an SLP data structure for every interleaving group of
stores for further analysis in vect_analyse_slp. */
if (!DR_IS_READ (dr) && !slp_impossible)
- VEC_safe_push (gimple, heap, LOOP_VINFO_STRIDED_STORES (loop_vinfo), stmt);
+ {
+ if (loop_vinfo)
+ VEC_safe_push (gimple, heap, LOOP_VINFO_STRIDED_STORES (loop_vinfo),
+ stmt);
+ if (bb_vinfo)
+ VEC_safe_push (gimple, heap, BB_VINFO_STRIDED_STORES (bb_vinfo),
+ stmt);
+ }
}
return true;
gimple stmt = DR_STMT (dr);
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ struct loop *loop = NULL;
HOST_WIDE_INT dr_step = TREE_INT_CST_LOW (step);
- if (!step)
+ if (loop_vinfo)
+ loop = LOOP_VINFO_LOOP (loop_vinfo);
+
+ if (loop_vinfo && !step)
{
if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "bad data-ref access");
+ fprintf (vect_dump, "bad data-ref access in loop");
return false;
}
- /* Don't allow invariant accesses. */
- if (dr_step == 0)
+ /* Don't allow invariant accesses in loops. */
+ if (loop_vinfo && dr_step == 0)
return false;
- if (nested_in_vect_loop_p (loop, stmt))
+ if (loop && nested_in_vect_loop_p (loop, stmt))
{
/* Interleaved accesses are not yet supported within outer-loop
vectorization for references in the inner-loop. */
return true;
}
- if (nested_in_vect_loop_p (loop, stmt))
+ if (loop && nested_in_vect_loop_p (loop, stmt))
{
if (vect_print_dump_info (REPORT_ALIGNMENT))
fprintf (vect_dump, "strided access in outer loop.");
FORNOW: handle only arrays and pointer accesses. */
bool
-vect_analyze_data_ref_accesses (loop_vec_info loop_vinfo)
+vect_analyze_data_ref_accesses (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo)
{
unsigned int i;
- VEC (data_reference_p, heap) *datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
+ VEC (data_reference_p, heap) *datarefs;
struct data_reference *dr;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_analyze_data_ref_accesses ===");
+ if (loop_vinfo)
+ datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
+ else
+ datarefs = BB_VINFO_DATAREFS (bb_vinfo);
+
for (i = 0; VEC_iterate (data_reference_p, datarefs, i, dr); i++)
if (!vect_analyze_data_ref_access (dr))
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
fprintf (vect_dump, "not vectorized: complicated access pattern.");
return false;
}
/* Function vect_analyze_data_refs.
- Find all the data references in the loop.
+ Find all the data references in the loop or basic block.
The general structure of the analysis of data refs in the vectorizer is as
follows:
- 1- vect_analyze_data_refs(loop): call compute_data_dependences_for_loop to
- find and analyze all data-refs in the loop and their dependences.
+ 1- vect_analyze_data_refs(loop/bb): call
+ compute_data_dependences_for_loop/bb to find and analyze all data-refs
+ in the loop/bb and their dependences.
2- vect_analyze_dependences(): apply dependence testing using ddrs.
3- vect_analyze_drs_alignment(): check that ref_stmt.alignment is ok.
4- vect_analyze_drs_access(): check that ref_stmt.step is ok.
*/
bool
-vect_analyze_data_refs (loop_vec_info loop_vinfo)
+vect_analyze_data_refs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo)
{
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ struct loop *loop = NULL;
+ basic_block bb = NULL;
unsigned int i;
VEC (data_reference_p, heap) *datarefs;
struct data_reference *dr;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_analyze_data_refs ===\n");
-
- compute_data_dependences_for_loop (loop, true,
- &LOOP_VINFO_DATAREFS (loop_vinfo),
- &LOOP_VINFO_DDRS (loop_vinfo));
+
+ if (loop_vinfo)
+ {
+ loop = LOOP_VINFO_LOOP (loop_vinfo);
+ compute_data_dependences_for_loop (loop, true,
+ &LOOP_VINFO_DATAREFS (loop_vinfo),
+ &LOOP_VINFO_DDRS (loop_vinfo));
+ datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
+ }
+ else
+ {
+ bb = BB_VINFO_BB (bb_vinfo);
+ compute_data_dependences_for_bb (bb, true,
+ &BB_VINFO_DATAREFS (bb_vinfo),
+ &BB_VINFO_DDRS (bb_vinfo));
+ datarefs = BB_VINFO_DATAREFS (bb_vinfo);
+ }
/* Go through the data-refs, check that the analysis succeeded. Update pointer
from stmt_vec_info struct to DR and vectype. */
- datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
for (i = 0; VEC_iterate (data_reference_p, datarefs, i, dr); i++)
{
if (!dr || !DR_REF (dr))
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
fprintf (vect_dump, "not vectorized: unhandled data-ref ");
return false;
}
if (!DR_BASE_ADDRESS (dr) || !DR_OFFSET (dr) || !DR_INIT (dr)
|| !DR_STEP (dr))
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
{
fprintf (vect_dump, "not vectorized: data ref analysis failed ");
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
if (TREE_CODE (DR_BASE_ADDRESS (dr)) == INTEGER_CST)
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
fprintf (vect_dump, "not vectorized: base addr of dr is a "
"constant");
return false;
inner-most enclosing loop). We do that by building a reference to the
first location accessed by the inner-loop, and analyze it relative to
the outer-loop. */
- if (nested_in_vect_loop_p (loop, stmt))
+ if (loop && nested_in_vect_loop_p (loop, stmt))
{
tree outer_step, outer_base, outer_init;
HOST_WIDE_INT pbitsize, pbitpos;
if (STMT_VINFO_DATA_REF (stmt_info))
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
{
fprintf (vect_dump,
"not vectorized: more than one data ref in stmt: ");
}
return false;
}
+
STMT_VINFO_DATA_REF (stmt_info) = dr;
/* Set vectype for STMT. */
get_vectype_for_scalar_type (scalar_type);
if (!STMT_VINFO_VECTYPE (stmt_info))
{
- if (vect_print_dump_info (REPORT_UNVECTORIZED_LOOPS))
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
{
fprintf (vect_dump,
"not vectorized: no vectype for stmt: ");
{
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
- struct loop *containing_loop = (gimple_bb (stmt))->loop_father;
tree data_ref_base = unshare_expr (DR_BASE_ADDRESS (dr));
tree base_name;
tree data_ref_base_var;
gimple_seq seq = NULL;
tree base_offset = unshare_expr (DR_OFFSET (dr));
tree init = unshare_expr (DR_INIT (dr));
- tree vect_ptr_type, addr_expr2;
+ tree vect_ptr_type;
tree step = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dr)));
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- gcc_assert (loop);
- if (loop != containing_loop)
+ if (loop_vinfo && loop && loop != (gimple_bb (stmt))->loop_father)
{
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ struct loop *outer_loop = LOOP_VINFO_LOOP (loop_vinfo);
- gcc_assert (nested_in_vect_loop_p (loop, stmt));
+ gcc_assert (nested_in_vect_loop_p (outer_loop, stmt));
data_ref_base = unshare_expr (STMT_VINFO_DR_BASE_ADDRESS (stmt_info));
base_offset = unshare_expr (STMT_VINFO_DR_OFFSET (stmt_info));
init = unshare_expr (STMT_VINFO_DR_INIT (stmt_info));
}
- /* Create data_ref_base */
- base_name = build_fold_indirect_ref (data_ref_base);
+ if (loop_vinfo)
+ base_name = build_fold_indirect_ref (data_ref_base);
+ else
+ {
+ base_offset = ssize_int (0);
+ init = ssize_int (0);
+ base_name = build_fold_indirect_ref (unshare_expr (DR_REF (dr)));
+ }
+
data_ref_base_var = create_tmp_var (TREE_TYPE (data_ref_base), "batmp");
add_referenced_var (data_ref_base_var);
data_ref_base = force_gimple_operand (data_ref_base, &seq, true,
}
/* base + base_offset */
- addr_base = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (data_ref_base),
- data_ref_base, base_offset);
-
+ if (loop_vinfo)
+ addr_base = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (data_ref_base),
+ data_ref_base, base_offset);
+ else
+ {
+ if (TREE_CODE (DR_REF (dr)) == INDIRECT_REF)
+ addr_base = unshare_expr (TREE_OPERAND (DR_REF (dr), 0));
+ else
+ addr_base = build1 (ADDR_EXPR,
+ build_pointer_type (TREE_TYPE (DR_REF (dr))),
+ unshare_expr (DR_REF (dr)));
+ }
+
vect_ptr_type = build_pointer_type (STMT_VINFO_VECTYPE (stmt_info));
- /* addr_expr = addr_base */
+ vec_stmt = fold_convert (vect_ptr_type, addr_base);
addr_expr = vect_get_new_vect_var (vect_ptr_type, vect_pointer_var,
get_name (base_name));
add_referenced_var (addr_expr);
- vec_stmt = fold_convert (vect_ptr_type, addr_base);
- addr_expr2 = vect_get_new_vect_var (vect_ptr_type, vect_pointer_var,
- get_name (base_name));
- add_referenced_var (addr_expr2);
- vec_stmt = force_gimple_operand (vec_stmt, &seq, false, addr_expr2);
+ vec_stmt = force_gimple_operand (vec_stmt, &seq, false, addr_expr);
gimple_seq_add_seq (new_stmt_list, seq);
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "created ");
print_generic_expr (vect_dump, vec_stmt, TDF_SLIM);
}
+
return vec_stmt;
}
tree base_name;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- bool nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt);
- struct loop *containing_loop = (gimple_bb (stmt))->loop_father;
+ struct loop *loop = NULL;
+ bool nested_in_vect_loop = false;
+ struct loop *containing_loop = NULL;
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
tree vect_ptr_type;
tree vect_ptr;
tree new_temp;
gimple vec_stmt;
gimple_seq new_stmt_list = NULL;
- edge pe;
+ edge pe = NULL;
basic_block new_bb;
tree vect_ptr_init;
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
tree indx_before_incr, indx_after_incr;
gimple incr;
tree step;
-
+ bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
+ gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
+
+ if (loop_vinfo)
+ {
+ loop = LOOP_VINFO_LOOP (loop_vinfo);
+ nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt);
+ containing_loop = (gimple_bb (stmt))->loop_father;
+ pe = loop_preheader_edge (loop);
+ }
+ else
+ {
+ gcc_assert (bb_vinfo);
+ only_init = true;
+ *ptr_incr = NULL;
+ }
+
/* Check the step (evolution) of the load in LOOP, and record
whether it's invariant. */
if (nested_in_vect_loop)
tree data_ref_base = base_name;
fprintf (vect_dump, "create vector-pointer variable to type: ");
print_generic_expr (vect_dump, vectype, TDF_SLIM);
- if (TREE_CODE (data_ref_base) == VAR_DECL)
- fprintf (vect_dump, " vectorizing a one dimensional array ref: ");
- else if (TREE_CODE (data_ref_base) == ARRAY_REF)
- fprintf (vect_dump, " vectorizing a multidimensional array ref: ");
+ if (TREE_CODE (data_ref_base) == VAR_DECL
+ || TREE_CODE (data_ref_base) == ARRAY_REF)
+ fprintf (vect_dump, " vectorizing an array ref: ");
else if (TREE_CODE (data_ref_base) == COMPONENT_REF)
fprintf (vect_dump, " vectorizing a record based array ref: ");
else if (TREE_CODE (data_ref_base) == SSA_NAME)
new_temp = vect_create_addr_base_for_vector_ref (stmt, &new_stmt_list,
offset, loop);
- pe = loop_preheader_edge (loop);
if (new_stmt_list)
{
- new_bb = gsi_insert_seq_on_edge_immediate (pe, new_stmt_list);
- gcc_assert (!new_bb);
+ if (pe)
+ {
+ new_bb = gsi_insert_seq_on_edge_immediate (pe, new_stmt_list);
+ gcc_assert (!new_bb);
+ }
+ else
+ gsi_insert_seq_before (&gsi, new_stmt_list, GSI_SAME_STMT);
}
*initial_address = new_temp;
fold_convert (vect_ptr_type, new_temp));
vect_ptr_init = make_ssa_name (vect_ptr, vec_stmt);
gimple_assign_set_lhs (vec_stmt, vect_ptr_init);
- new_bb = gsi_insert_on_edge_immediate (pe, vec_stmt);
- gcc_assert (!new_bb);
-
+ if (pe)
+ {
+ new_bb = gsi_insert_on_edge_immediate (pe, vec_stmt);
+ gcc_assert (!new_bb);
+ }
+ else
+ gsi_insert_before (&gsi, vec_stmt, GSI_SAME_STMT);
/** (4) Handle the updating of the vector-pointer inside the loop.
This is needed when ONLY_INIT is false, and also when AT_LOOP
is the inner-loop nested in LOOP (during outer-loop vectorization).
**/
- if (only_init && at_loop == loop) /* No update in loop is required. */
+ /* No update in loop is required. */
+ if (only_init && (!loop_vinfo || at_loop == loop))
{
/* Copy the points-to information if it exists. */
if (DR_PTR_INFO (dr))
vect_ptr, loop, &incr_gsi, insert_after,
&indx_before_incr, &indx_after_incr);
incr = gsi_stmt (incr_gsi);
- set_vinfo_for_stmt (incr, new_stmt_vec_info (incr, loop_vinfo));
+ set_vinfo_for_stmt (incr, new_stmt_vec_info (incr, loop_vinfo, NULL));
/* Copy the points-to information if it exists. */
if (DR_PTR_INFO (dr))
duplicate_ssa_name_ptr_info (indx_before_incr, DR_PTR_INFO (dr));
duplicate_ssa_name_ptr_info (indx_after_incr, DR_PTR_INFO (dr));
}
- merge_alias_info (vect_ptr_init, indx_before_incr);
- merge_alias_info (vect_ptr_init, indx_after_incr);
if (ptr_incr)
*ptr_incr = incr;
containing_loop, &incr_gsi, insert_after, &indx_before_incr,
&indx_after_incr);
incr = gsi_stmt (incr_gsi);
- set_vinfo_for_stmt (incr, new_stmt_vec_info (incr, loop_vinfo));
+ set_vinfo_for_stmt (incr, new_stmt_vec_info (incr, loop_vinfo, NULL));
/* Copy the points-to information if it exists. */
if (DR_PTR_INFO (dr))
duplicate_ssa_name_ptr_info (indx_before_incr, DR_PTR_INFO (dr));
duplicate_ssa_name_ptr_info (indx_after_incr, DR_PTR_INFO (dr));
}
- merge_alias_info (vect_ptr_init, indx_before_incr);
- merge_alias_info (vect_ptr_init, indx_after_incr);
if (ptr_incr)
*ptr_incr = incr;
/* Copy the points-to information if it exists. */
if (DR_PTR_INFO (dr))
duplicate_ssa_name_ptr_info (new_dataref_ptr, DR_PTR_INFO (dr));
- merge_alias_info (new_dataref_ptr, dataref_ptr);
if (!ptr_incr)
return new_dataref_ptr;
gimple stmt = DR_STMT (dr);
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
- enum machine_mode mode = (int) TYPE_MODE (vectype);
- struct loop *vect_loop = LOOP_VINFO_LOOP (STMT_VINFO_LOOP_VINFO (stmt_info));
- bool nested_in_vect_loop = nested_in_vect_loop_p (vect_loop, stmt);
+ enum machine_mode mode = TYPE_MODE (vectype);
bool invariant_in_outerloop = false;
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ struct loop *vect_loop = NULL;
+ bool nested_in_vect_loop = false;
if (aligned_access_p (dr))
return dr_aligned;
+ if (!loop_vinfo)
+ /* FORNOW: Misaligned accesses are supported only in loops. */
+ return dr_unaligned_unsupported;
+
+ vect_loop = LOOP_VINFO_LOOP (loop_vinfo);
+ nested_in_vect_loop = nested_in_vect_loop_p (vect_loop, stmt);
+
if (nested_in_vect_loop)
{
tree outerloop_step = STMT_VINFO_DR_STEP (stmt_info);
/* Can't software pipeline the loads, but can at least do them. */
return dr_unaligned_supported;
}
+ else
+ {
+ if (movmisalign_optab->handlers[mode].insn_code != CODE_FOR_nothing)
+ return dr_unaligned_supported;
+ }
/* Unsupported. */
return dr_unaligned_unsupported;
OSDN Git Service
