stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
struct loop *loop = NULL;
- HOST_WIDE_INT dr_step = TREE_INT_CST_LOW (step);
+ HOST_WIDE_INT dr_step;
if (loop_vinfo)
loop = LOOP_VINFO_LOOP (loop_vinfo);
}
/* Allow invariant loads in loops. */
+ dr_step = TREE_INT_CST_LOW (step);
if (loop_vinfo && dr_step == 0)
{
GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)) = NULL;
return false;
}
+ if (is_gimple_call (stmt))
+ {
+ if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS))
+ {
+ fprintf (vect_dump, "not vectorized: dr in a call ");
+ print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM);
+ }
+
+ if (bb_vinfo)
+ {
+ STMT_VINFO_VECTORIZABLE (stmt_info) = false;
+ stop_bb_analysis = true;
+ continue;
+ }
+
+ if (gather)
+ free_data_ref (dr);
+ return false;
+ }
+
/* Update DR field in stmt_vec_info struct. */
/* If the dataref is in an inner-loop of the loop that is considered for
ddr = VEC_index (ddr_p, ddrs, k);
gcc_assert (DDR_A (ddr) == olddr && DDR_B (ddr) == olddr);
newddr = initialize_data_dependence_relation (dr, dr, nest);
- compute_self_dependence (newddr);
VEC_replace (ddr_p, ddrs, k, newddr);
free_dependence_relation (ddr);
VEC_replace (data_reference_p, datarefs, i, dr);
/* Function vect_strided_store_supported.
- Returns TRUE is INTERLEAVE_HIGH and INTERLEAVE_LOW operations are supported,
- and FALSE otherwise. */
+ Returns TRUE if interleave high and interleave low permutations
+ are supported, and FALSE otherwise. */
bool
vect_strided_store_supported (tree vectype, unsigned HOST_WIDE_INT count)
{
- optab ih_optab, il_optab;
- enum machine_mode mode;
-
- mode = TYPE_MODE (vectype);
+ enum machine_mode mode = TYPE_MODE (vectype);
/* vect_permute_store_chain requires the group size to be a power of two. */
if (exact_log2 (count) == -1)
return false;
}
- /* Check that the operation is supported. */
- ih_optab = optab_for_tree_code (VEC_INTERLEAVE_HIGH_EXPR,
- vectype, optab_default);
- il_optab = optab_for_tree_code (VEC_INTERLEAVE_LOW_EXPR,
- vectype, optab_default);
- if (il_optab && ih_optab
- && optab_handler (ih_optab, mode) != CODE_FOR_nothing
- && optab_handler (il_optab, mode) != CODE_FOR_nothing)
- return true;
-
- if (can_vec_perm_for_code_p (VEC_INTERLEAVE_HIGH_EXPR, mode, NULL)
- && can_vec_perm_for_code_p (VEC_INTERLEAVE_LOW_EXPR, mode, NULL))
- return true;
+ /* Check that the permutation is supported. */
+ if (VECTOR_MODE_P (mode))
+ {
+ unsigned int i, nelt = GET_MODE_NUNITS (mode);
+ unsigned char *sel = XALLOCAVEC (unsigned char, nelt);
+ for (i = 0; i < nelt / 2; i++)
+ {
+ sel[i * 2] = i;
+ sel[i * 2 + 1] = i + nelt;
+ }
+ if (can_vec_perm_p (mode, false, sel))
+ {
+ for (i = 0; i < nelt; i++)
+ sel[i] += nelt / 2;
+ if (can_vec_perm_p (mode, false, sel))
+ return true;
+ }
+ }
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "interleave op not supported by target.");
tree perm_dest, vect1, vect2, high, low;
gimple perm_stmt;
tree vectype = STMT_VINFO_VECTYPE (vinfo_for_stmt (stmt));
- int i;
- unsigned int j;
- enum tree_code high_code, low_code;
-
- gcc_assert (vect_strided_store_supported (vectype, length));
+ tree perm_mask_low, perm_mask_high;
+ unsigned int i, n;
+ unsigned int j, nelt = TYPE_VECTOR_SUBPARTS (vectype);
+ unsigned char *sel = XALLOCAVEC (unsigned char, nelt);
*result_chain = VEC_copy (tree, heap, dr_chain);
- for (i = 0; i < exact_log2 (length); i++)
+ for (i = 0, n = nelt / 2; i < n; i++)
+ {
+ sel[i * 2] = i;
+ sel[i * 2 + 1] = i + nelt;
+ }
+ perm_mask_high = vect_gen_perm_mask (vectype, sel);
+ gcc_assert (perm_mask_high != NULL);
+
+ for (i = 0; i < nelt; i++)
+ sel[i] += nelt / 2;
+ perm_mask_low = vect_gen_perm_mask (vectype, sel);
+ gcc_assert (perm_mask_low != NULL);
+
+ for (i = 0, n = exact_log2 (length); i < n; i++)
{
for (j = 0; j < length/2; j++)
{
vect2 = VEC_index (tree, dr_chain, j+length/2);
/* Create interleaving stmt:
- in the case of big endian:
- high = interleave_high (vect1, vect2)
- and in the case of little endian:
- high = interleave_low (vect1, vect2). */
+ high = VEC_PERM_EXPR <vect1, vect2, {0, nelt, 1, nelt+1, ...}> */
perm_dest = create_tmp_var (vectype, "vect_inter_high");
DECL_GIMPLE_REG_P (perm_dest) = 1;
add_referenced_var (perm_dest);
- if (BYTES_BIG_ENDIAN)
- {
- high_code = VEC_INTERLEAVE_HIGH_EXPR;
- low_code = VEC_INTERLEAVE_LOW_EXPR;
- }
- else
- {
- low_code = VEC_INTERLEAVE_HIGH_EXPR;
- high_code = VEC_INTERLEAVE_LOW_EXPR;
- }
- perm_stmt = gimple_build_assign_with_ops (high_code, perm_dest,
- vect1, vect2);
- high = make_ssa_name (perm_dest, perm_stmt);
- gimple_assign_set_lhs (perm_stmt, high);
+ high = make_ssa_name (perm_dest, NULL);
+ perm_stmt
+ = gimple_build_assign_with_ops3 (VEC_PERM_EXPR, high,
+ vect1, vect2, perm_mask_high);
vect_finish_stmt_generation (stmt, perm_stmt, gsi);
VEC_replace (tree, *result_chain, 2*j, high);
/* Create interleaving stmt:
- in the case of big endian:
- low = interleave_low (vect1, vect2)
- and in the case of little endian:
- low = interleave_high (vect1, vect2). */
+ low = VEC_PERM_EXPR <vect1, vect2, {nelt/2, nelt*3/2, nelt/2+1,
+ nelt*3/2+1, ...}> */
perm_dest = create_tmp_var (vectype, "vect_inter_low");
DECL_GIMPLE_REG_P (perm_dest) = 1;
add_referenced_var (perm_dest);
- perm_stmt = gimple_build_assign_with_ops (low_code, perm_dest,
- vect1, vect2);
- low = make_ssa_name (perm_dest, perm_stmt);
- gimple_assign_set_lhs (perm_stmt, low);
+ low = make_ssa_name (perm_dest, NULL);
+ perm_stmt
+ = gimple_build_assign_with_ops3 (VEC_PERM_EXPR, low,
+ vect1, vect2, perm_mask_low);
vect_finish_stmt_generation (stmt, perm_stmt, gsi);
VEC_replace (tree, *result_chain, 2*j+1, low);
}
/* Function vect_strided_load_supported.
- Returns TRUE is EXTRACT_EVEN and EXTRACT_ODD operations are supported,
+ Returns TRUE if even and odd permutations are supported,
and FALSE otherwise. */
bool
vect_strided_load_supported (tree vectype, unsigned HOST_WIDE_INT count)
{
- optab ee_optab, eo_optab;
- enum machine_mode mode;
-
- mode = TYPE_MODE (vectype);
+ enum machine_mode mode = TYPE_MODE (vectype);
/* vect_permute_load_chain requires the group size to be a power of two. */
if (exact_log2 (count) == -1)
return false;
}
- ee_optab = optab_for_tree_code (VEC_EXTRACT_EVEN_EXPR,
- vectype, optab_default);
- eo_optab = optab_for_tree_code (VEC_EXTRACT_ODD_EXPR,
- vectype, optab_default);
- if (ee_optab && eo_optab
- && optab_handler (ee_optab, mode) != CODE_FOR_nothing
- && optab_handler (eo_optab, mode) != CODE_FOR_nothing)
- return true;
+ /* Check that the permutation is supported. */
+ if (VECTOR_MODE_P (mode))
+ {
+ unsigned int i, nelt = GET_MODE_NUNITS (mode);
+ unsigned char *sel = XALLOCAVEC (unsigned char, nelt);
- if (can_vec_perm_for_code_p (VEC_EXTRACT_EVEN_EXPR, mode, NULL)
- && can_vec_perm_for_code_p (VEC_EXTRACT_ODD_EXPR, mode, NULL))
- return true;
+ for (i = 0; i < nelt; i++)
+ sel[i] = i * 2;
+ if (can_vec_perm_p (mode, false, sel))
+ {
+ for (i = 0; i < nelt; i++)
+ sel[i] = i * 2 + 1;
+ if (can_vec_perm_p (mode, false, sel))
+ return true;
+ }
+ }
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "extract even/odd not supported by target");
VEC(tree,heap) **result_chain)
{
tree perm_dest, data_ref, first_vect, second_vect;
+ tree perm_mask_even, perm_mask_odd;
gimple perm_stmt;
tree vectype = STMT_VINFO_VECTYPE (vinfo_for_stmt (stmt));
- int i;
- unsigned int j;
-
- gcc_assert (vect_strided_load_supported (vectype, length));
+ unsigned int i, j, log_length = exact_log2 (length);
+ unsigned nelt = TYPE_VECTOR_SUBPARTS (vectype);
+ unsigned char *sel = XALLOCAVEC (unsigned char, nelt);
*result_chain = VEC_copy (tree, heap, dr_chain);
- for (i = 0; i < exact_log2 (length); i++)
+
+ for (i = 0; i < nelt; ++i)
+ sel[i] = i * 2;
+ perm_mask_even = vect_gen_perm_mask (vectype, sel);
+ gcc_assert (perm_mask_even != NULL);
+
+ for (i = 0; i < nelt; ++i)
+ sel[i] = i * 2 + 1;
+ perm_mask_odd = vect_gen_perm_mask (vectype, sel);
+ gcc_assert (perm_mask_odd != NULL);
+
+ for (i = 0; i < log_length; i++)
{
- for (j = 0; j < length; j +=2)
+ for (j = 0; j < length; j += 2)
{
first_vect = VEC_index (tree, dr_chain, j);
second_vect = VEC_index (tree, dr_chain, j+1);
DECL_GIMPLE_REG_P (perm_dest) = 1;
add_referenced_var (perm_dest);
- perm_stmt = gimple_build_assign_with_ops (VEC_EXTRACT_EVEN_EXPR,
- perm_dest, first_vect,
- second_vect);
+ perm_stmt = gimple_build_assign_with_ops3 (VEC_PERM_EXPR, perm_dest,
+ first_vect, second_vect,
+ perm_mask_even);
data_ref = make_ssa_name (perm_dest, perm_stmt);
gimple_assign_set_lhs (perm_stmt, data_ref);
DECL_GIMPLE_REG_P (perm_dest) = 1;
add_referenced_var (perm_dest);
- perm_stmt = gimple_build_assign_with_ops (VEC_EXTRACT_ODD_EXPR,
- perm_dest, first_vect,
- second_vect);
+ perm_stmt = gimple_build_assign_with_ops3 (VEC_PERM_EXPR, perm_dest,
+ first_vect, second_vect,
+ perm_mask_odd);
+
data_ref = make_ssa_name (perm_dest, perm_stmt);
gimple_assign_set_lhs (perm_stmt, data_ref);
vect_finish_stmt_generation (stmt, perm_stmt, gsi);
OSDN Git Service
