/* Data References Analysis and Manipulation Utilities for Vectorization.
- Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010
+ Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011
Free Software Foundation, Inc.
Contributed by Dorit Naishlos <dorit@il.ibm.com>
and Ira Rosen <irar@il.ibm.com>
#include "tree-scalar-evolution.h"
#include "tree-vectorizer.h"
#include "diagnostic-core.h"
-#include "toplev.h"
/* Need to include rtl.h, expr.h, etc. for optabs. */
#include "expr.h"
|| (TREE_CODE (base) == VAR_DECL
&& DECL_ALIGN (base) >= TYPE_ALIGN (vectype)));
+ /* If this is a backward running DR then first access in the larger
+ vectype actually is N-1 elements before the address in the DR.
+ Adjust misalign accordingly. */
+ if (tree_int_cst_compare (DR_STEP (dr), size_zero_node) < 0)
+ {
+ tree offset = ssize_int (TYPE_VECTOR_SUBPARTS (vectype) - 1);
+ /* DR_STEP(dr) is the same as -TYPE_SIZE of the scalar type,
+ otherwise we wouldn't be here. */
+ offset = fold_build2 (MULT_EXPR, ssizetype, offset, DR_STEP (dr));
+ /* PLUS because DR_STEP was negative. */
+ misalign = size_binop (PLUS_EXPR, misalign, offset);
+ }
+
/* Modulo alignment. */
misalign = size_binop (FLOOR_MOD_EXPR, misalign, alignment);
if (known_alignment_for_access_p (dr)
&& known_alignment_for_access_p (dr_peel))
{
+ bool negative = tree_int_cst_compare (DR_STEP (dr), size_zero_node) < 0;
int misal = DR_MISALIGNMENT (dr);
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
- misal += npeel * dr_size;
- misal %= GET_MODE_SIZE (TYPE_MODE (vectype));
+ misal += negative ? -npeel * dr_size : npeel * dr_size;
+ misal &= (TYPE_ALIGN (vectype) / BITS_PER_UNIT) - 1;
SET_DR_MISALIGNMENT (dr, misal);
return;
}
if (ba)
is_packed = contains_packed_reference (ba);
+ if (compare_tree_int (TYPE_SIZE (type), TYPE_ALIGN (type)) > 0)
+ is_packed = true;
+
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "Unknown misalignment, is_packed = %d",is_packed);
if (targetm.vectorize.vector_alignment_reachable (type, is_packed))
stmt = DR_STMT (dr);
stmt_info = vinfo_for_stmt (stmt);
+ if (!STMT_VINFO_RELEVANT (stmt_info))
+ continue;
+
/* For interleaving, only the alignment of the first access
matters. */
if (STMT_VINFO_STRIDED_ACCESS (stmt_info)
if (known_alignment_for_access_p (dr))
{
unsigned int npeel_tmp;
+ bool negative = tree_int_cst_compare (DR_STEP (dr),
+ size_zero_node) < 0;
/* Save info about DR in the hash table. */
if (!LOOP_VINFO_PEELING_HTAB (loop_vinfo))
nelements = TYPE_VECTOR_SUBPARTS (vectype);
mis = DR_MISALIGNMENT (dr) / GET_MODE_SIZE (TYPE_MODE (
TREE_TYPE (DR_REF (dr))));
- npeel_tmp = (nelements - mis) % vf;
+ npeel_tmp = (negative
+ ? (mis - nelements) : (nelements - mis))
+ & (nelements - 1);
/* For multiple types, it is possible that the bigger type access
will have more than one peeling option. E.g., a loop with two
if (known_alignment_for_access_p (dr0))
{
+ bool negative = tree_int_cst_compare (DR_STEP (dr0),
+ size_zero_node) < 0;
if (!npeel)
{
/* Since it's known at compile time, compute the number of
count. */
mis = DR_MISALIGNMENT (dr0);
mis /= GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (DR_REF (dr0))));
- npeel = nelements - mis;
+ npeel = ((negative ? mis - nelements : nelements - mis)
+ & (nelements - 1));
}
/* For interleaved data access every iteration accesses all the
if (DDR_NUM_DIST_VECTS (ddr) == 0)
return;
+ /* Data-dependence analysis reports a distance vector of zero
+ for data-references that overlap only in the first iteration
+ but have different sign step (see PR45764).
+ So as a sanity check require equal DR_STEP. */
+ if (!operand_equal_p (DR_STEP (dra), DR_STEP (drb), 0))
+ return;
+
loop_depth = index_in_loop_nest (loop->num, DDR_LOOP_NEST (ddr));
FOR_EACH_VEC_ELT (lambda_vector, DDR_DIST_VECTS (ddr), i, dist_v)
{
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
HOST_WIDE_INT dr_step = TREE_INT_CST_LOW (step);
- HOST_WIDE_INT stride;
+ HOST_WIDE_INT stride, last_accessed_element = 1;
bool slp_impossible = false;
+ struct loop *loop = NULL;
+
+ if (loop_vinfo)
+ loop = LOOP_VINFO_LOOP (loop_vinfo);
/* For interleaving, STRIDE is STEP counted in elements, i.e., the size of the
interleaving group (including gaps). */
fprintf (vect_dump, " step ");
print_generic_expr (vect_dump, step, TDF_SLIM);
}
+
+ if (loop_vinfo)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "Data access with gaps requires scalar "
+ "epilogue loop");
+ if (loop->inner)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "Peeling for outer loop is not"
+ " supported");
+ return false;
+ }
+
+ LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true;
+ }
+
return true;
}
next = DR_GROUP_NEXT_DR (vinfo_for_stmt (next));
continue;
}
+
prev = next;
/* Check that all the accesses have the same STEP. */
gaps += diff - 1;
}
+ last_accessed_element += diff;
+
/* Store the gap from the previous member of the group. If there is no
gap in the access, DR_GROUP_GAP is always 1. */
DR_GROUP_GAP (vinfo_for_stmt (next)) = diff;
VEC_safe_push (gimple, heap, BB_VINFO_STRIDED_STORES (bb_vinfo),
stmt);
}
+
+ /* There is a gap in the end of the group. */
+ if (stride - last_accessed_element > 0 && loop_vinfo)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "Data access with gaps requires scalar "
+ "epilogue loop");
+ if (loop->inner)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "Peeling for outer loop is not supported");
+ return false;
+ }
+
+ LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) = true;
+ }
}
return true;
{
loop = LOOP_VINFO_LOOP (loop_vinfo);
res = compute_data_dependences_for_loop
- (loop, true, &LOOP_VINFO_DATAREFS (loop_vinfo),
+ (loop, true,
+ &LOOP_VINFO_LOOP_NEST (loop_vinfo),
+ &LOOP_VINFO_DATAREFS (loop_vinfo),
&LOOP_VINFO_DDRS (loop_vinfo));
if (!res)
print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
- if (bb_vinfo)
- {
- /* Mark the statement as not vectorizable. */
- STMT_VINFO_VECTORIZABLE (stmt_info) = false;
- continue;
- }
- else
- return false;
+ return false;
}
if (TREE_CODE (DR_BASE_ADDRESS (dr)) == INTEGER_CST)
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
fprintf (vect_dump, "not vectorized: base addr of dr is a "
"constant");
- if (bb_vinfo)
+ return false;
+ }
+
+ if (TREE_THIS_VOLATILE (DR_REF (dr)))
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
{
- /* Mark the statement as not vectorizable. */
- STMT_VINFO_VECTORIZABLE (stmt_info) = false;
- continue;
+ fprintf (vect_dump, "not vectorized: volatile type ");
+ print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
}
- else
- return false;
+ return false;
}
base = unshare_expr (DR_BASE_ADDRESS (dr));
offset = unshare_expr (DR_OFFSET (dr));
init = unshare_expr (DR_INIT (dr));
- if (stmt_could_throw_p (stmt))
+ if (stmt_can_throw_internal (stmt))
{
if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
{
return false;
}
+ if (is_gimple_call (stmt))
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
+ {
+ fprintf (vect_dump, "not vectorized: dr in a call ");
+ print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
+ }
+ return false;
+ }
+
/* Update DR field in stmt_vec_info struct. */
/* If the dataref is in an inner-loop of the loop that is considered for
if (DR_PTR_INFO (dr)
&& TREE_CODE (vec_stmt) == SSA_NAME)
- duplicate_ssa_name_ptr_info (vec_stmt, DR_PTR_INFO (dr));
+ {
+ duplicate_ssa_name_ptr_info (vec_stmt, DR_PTR_INFO (dr));
+ if (offset)
+ {
+ SSA_NAME_PTR_INFO (vec_stmt)->align = 1;
+ SSA_NAME_PTR_INFO (vec_stmt)->misalign = 0;
+ }
+ }
if (vect_print_dump_info (REPORT_DETAILS))
{
/* Copy the points-to information if it exists. */
if (DR_PTR_INFO (dr))
- duplicate_ssa_name_ptr_info (new_dataref_ptr, DR_PTR_INFO (dr));
+ {
+ duplicate_ssa_name_ptr_info (new_dataref_ptr, DR_PTR_INFO (dr));
+ SSA_NAME_PTR_INFO (new_dataref_ptr)->align = 1;
+ SSA_NAME_PTR_INFO (new_dataref_ptr)->misalign = 0;
+ }
if (!ptr_incr)
return new_dataref_ptr;
OSDN Git Service
