X-Git-Url: http://git.sourceforge.jp/view?p=pf3gnuchains%2Fgcc-fork.git;a=blobdiff_plain;f=gcc%2Ftree-vect-stmts.c;h=6d15bda937e810b8d7b183c7e604b80d36fe31bd;hp=c95fe7d273e4ecd807a56101fbe4e505ed28603e;hb=6c07e6d85d043ec173479ec0096fabb8832a4299;hpb=182cf5a9a415f31df0f9a10e46faed1221484a35 diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index c95fe7d273e..6d15bda937e 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -37,6 +37,7 @@ along with GCC; see the file COPYING3. If not see #include "expr.h" #include "recog.h" #include "optabs.h" +#include "diagnostic-core.h" #include "toplev.h" #include "tree-vectorizer.h" #include "langhooks.h" @@ -545,6 +546,18 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) } +/* Get cost by calling cost target builtin. */ + +static inline +int vect_get_stmt_cost (enum vect_cost_for_stmt type_of_cost) +{ + tree dummy_type = NULL; + int dummy = 0; + + return targetm.vectorize.builtin_vectorization_cost (type_of_cost, + dummy_type, dummy); +} + int cost_for_stmt (gimple stmt) { @@ -553,9 +566,9 @@ cost_for_stmt (gimple stmt) switch (STMT_VINFO_TYPE (stmt_info)) { case load_vec_info_type: - return targetm.vectorize.builtin_vectorization_cost (scalar_load); + return vect_get_stmt_cost (scalar_load); case store_vec_info_type: - return targetm.vectorize.builtin_vectorization_cost (scalar_store); + return vect_get_stmt_cost (scalar_store); case op_vec_info_type: case condition_vec_info_type: case assignment_vec_info_type: @@ -565,7 +578,7 @@ cost_for_stmt (gimple stmt) case type_demotion_vec_info_type: case type_conversion_vec_info_type: case call_vec_info_type: - return targetm.vectorize.builtin_vectorization_cost (scalar_stmt); + return vect_get_stmt_cost (scalar_stmt); case undef_vec_info_type: default: gcc_unreachable (); @@ -589,15 +602,13 @@ vect_model_simple_cost (stmt_vec_info stmt_info, int ncopies, if (PURE_SLP_STMT (stmt_info)) return; - inside_cost = ncopies - * targetm.vectorize.builtin_vectorization_cost (vector_stmt); + inside_cost = ncopies * vect_get_stmt_cost (vector_stmt); /* FORNOW: Assuming maximum 2 args per stmts. */ for (i = 0; i < 2; i++) { if (dt[i] == vect_constant_def || dt[i] == vect_external_def) - outside_cost - += targetm.vectorize.builtin_vectorization_cost (vector_stmt); + outside_cost += vect_get_stmt_cost (vector_stmt); } if (vect_print_dump_info (REPORT_COST)) @@ -638,22 +649,39 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies, enum vect_def_type dt, slp_tree slp_node) { int group_size; - int inside_cost = 0, outside_cost = 0; + unsigned int inside_cost = 0, outside_cost = 0; + struct data_reference *first_dr; + gimple first_stmt; /* The SLP costs were already calculated during SLP tree build. */ if (PURE_SLP_STMT (stmt_info)) return; if (dt == vect_constant_def || dt == vect_external_def) - outside_cost - = targetm.vectorize.builtin_vectorization_cost (scalar_to_vec); + outside_cost = vect_get_stmt_cost (scalar_to_vec); /* Strided access? */ - if (DR_GROUP_FIRST_DR (stmt_info) && !slp_node) - group_size = vect_cost_strided_group_size (stmt_info); + if (DR_GROUP_FIRST_DR (stmt_info)) + { + if (slp_node) + { + first_stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (slp_node), 0); + group_size = 1; + } + else + { + first_stmt = DR_GROUP_FIRST_DR (stmt_info); + group_size = vect_cost_strided_group_size (stmt_info); + } + + first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)); + } /* Not a strided access. */ else - group_size = 1; + { + group_size = 1; + first_dr = STMT_VINFO_DATA_REF (stmt_info); + } /* Is this an access in a group of stores, which provide strided access? If so, add in the cost of the permutes. */ @@ -661,7 +689,7 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies, { /* Uses a high and low interleave operation for each needed permute. */ inside_cost = ncopies * exact_log2(group_size) * group_size - * targetm.vectorize.builtin_vectorization_cost (vector_stmt); + * vect_get_stmt_cost (vector_stmt); if (vect_print_dump_info (REPORT_COST)) fprintf (vect_dump, "vect_model_store_cost: strided group_size = %d .", @@ -670,8 +698,7 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies, } /* Costs of the stores. */ - inside_cost += ncopies - * targetm.vectorize.builtin_vectorization_cost (vector_store); + vect_get_store_cost (first_dr, ncopies, &inside_cost); if (vect_print_dump_info (REPORT_COST)) fprintf (vect_dump, "vect_model_store_cost: inside_cost = %d, " @@ -683,6 +710,49 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies, } +/* Calculate cost of DR's memory access. */ +void +vect_get_store_cost (struct data_reference *dr, int ncopies, + unsigned int *inside_cost) +{ + int alignment_support_scheme = vect_supportable_dr_alignment (dr, false); + + switch (alignment_support_scheme) + { + case dr_aligned: + { + *inside_cost += ncopies * vect_get_stmt_cost (vector_store); + + if (vect_print_dump_info (REPORT_COST)) + fprintf (vect_dump, "vect_model_store_cost: aligned."); + + break; + } + + case dr_unaligned_supported: + { + gimple stmt = DR_STMT (dr); + stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + tree vectype = STMT_VINFO_VECTYPE (stmt_info); + + /* Here, we assign an additional cost for the unaligned store. */ + *inside_cost += ncopies + * targetm.vectorize.builtin_vectorization_cost (unaligned_store, + vectype, DR_MISALIGNMENT (dr)); + + if (vect_print_dump_info (REPORT_COST)) + fprintf (vect_dump, "vect_model_store_cost: unaligned supported by " + "hardware."); + + break; + } + + default: + gcc_unreachable (); + } +} + + /* Function vect_model_load_cost Models cost for loads. In the case of strided accesses, the last access @@ -695,10 +765,9 @@ vect_model_load_cost (stmt_vec_info stmt_info, int ncopies, slp_tree slp_node) { int group_size; - int alignment_support_cheme; gimple first_stmt; struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr; - int inside_cost = 0, outside_cost = 0; + unsigned int inside_cost = 0, outside_cost = 0; /* The SLP costs were already calculated during SLP tree build. */ if (PURE_SLP_STMT (stmt_info)) @@ -718,29 +787,47 @@ vect_model_load_cost (stmt_vec_info stmt_info, int ncopies, slp_tree slp_node) first_dr = dr; } - alignment_support_cheme = vect_supportable_dr_alignment (first_dr); - /* Is this an access in a group of loads providing strided access? If so, add in the cost of the permutes. */ if (group_size > 1) { /* Uses an even and odd extract operations for each needed permute. */ inside_cost = ncopies * exact_log2(group_size) * group_size - * targetm.vectorize.builtin_vectorization_cost (vector_stmt); + * vect_get_stmt_cost (vector_stmt); if (vect_print_dump_info (REPORT_COST)) fprintf (vect_dump, "vect_model_load_cost: strided group_size = %d .", group_size); - } /* The loads themselves. */ - switch (alignment_support_cheme) + vect_get_load_cost (first_dr, ncopies, + ((!DR_GROUP_FIRST_DR (stmt_info)) || group_size > 1 || slp_node), + &inside_cost, &outside_cost); + + if (vect_print_dump_info (REPORT_COST)) + fprintf (vect_dump, "vect_model_load_cost: inside_cost = %d, " + "outside_cost = %d .", inside_cost, outside_cost); + + /* Set the costs either in STMT_INFO or SLP_NODE (if exists). */ + stmt_vinfo_set_inside_of_loop_cost (stmt_info, slp_node, inside_cost); + stmt_vinfo_set_outside_of_loop_cost (stmt_info, slp_node, outside_cost); +} + + +/* Calculate cost of DR's memory access. */ +void +vect_get_load_cost (struct data_reference *dr, int ncopies, + bool add_realign_cost, unsigned int *inside_cost, + unsigned int *outside_cost) +{ + int alignment_support_scheme = vect_supportable_dr_alignment (dr, false); + + switch (alignment_support_scheme) { case dr_aligned: { - inside_cost += ncopies - * targetm.vectorize.builtin_vectorization_cost (vector_load); + *inside_cost += ncopies * vect_get_stmt_cost (vector_load); if (vect_print_dump_info (REPORT_COST)) fprintf (vect_dump, "vect_model_load_cost: aligned."); @@ -749,10 +836,14 @@ vect_model_load_cost (stmt_vec_info stmt_info, int ncopies, slp_tree slp_node) } case dr_unaligned_supported: { - /* Here, we assign an additional cost for the unaligned load. */ - inside_cost += ncopies - * targetm.vectorize.builtin_vectorization_cost (unaligned_load); + gimple stmt = DR_STMT (dr); + stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + tree vectype = STMT_VINFO_VECTYPE (stmt_info); + /* Here, we assign an additional cost for the unaligned load. */ + *inside_cost += ncopies + * targetm.vectorize.builtin_vectorization_cost (unaligned_load, + vectype, DR_MISALIGNMENT (dr)); if (vect_print_dump_info (REPORT_COST)) fprintf (vect_dump, "vect_model_load_cost: unaligned supported by " "hardware."); @@ -761,16 +852,14 @@ vect_model_load_cost (stmt_vec_info stmt_info, int ncopies, slp_tree slp_node) } case dr_explicit_realign: { - inside_cost += ncopies * (2 - * targetm.vectorize.builtin_vectorization_cost (vector_load) - + targetm.vectorize.builtin_vectorization_cost (vector_stmt)); + *inside_cost += ncopies * (2 * vect_get_stmt_cost (vector_load) + + vect_get_stmt_cost (vector_stmt)); /* FIXME: If the misalignment remains fixed across the iterations of the containing loop, the following cost should be added to the outside costs. */ if (targetm.vectorize.builtin_mask_for_load) - inside_cost - += targetm.vectorize.builtin_vectorization_cost (vector_stmt); + *inside_cost += vect_get_stmt_cost (vector_stmt); break; } @@ -787,32 +876,21 @@ vect_model_load_cost (stmt_vec_info stmt_info, int ncopies, slp_tree slp_node) access in the group. Inside the loop, there is a load op and a realignment op. */ - if ((!DR_GROUP_FIRST_DR (stmt_info)) || group_size > 1 || slp_node) + if (add_realign_cost) { - outside_cost = 2 - * targetm.vectorize.builtin_vectorization_cost (vector_stmt); + *outside_cost = 2 * vect_get_stmt_cost (vector_stmt); if (targetm.vectorize.builtin_mask_for_load) - outside_cost - += targetm.vectorize.builtin_vectorization_cost (vector_stmt); + *outside_cost += vect_get_stmt_cost (vector_stmt); } - inside_cost += ncopies - * (targetm.vectorize.builtin_vectorization_cost (vector_load) - + targetm.vectorize.builtin_vectorization_cost (vector_stmt)); + *inside_cost += ncopies * (vect_get_stmt_cost (vector_load) + + vect_get_stmt_cost (vector_stmt)); break; } default: gcc_unreachable (); } - - if (vect_print_dump_info (REPORT_COST)) - fprintf (vect_dump, "vect_model_load_cost: inside_cost = %d, " - "outside_cost = %d .", inside_cost, outside_cost); - - /* Set the costs either in STMT_INFO or SLP_NODE (if exists). */ - stmt_vinfo_set_inside_of_loop_cost (stmt_info, slp_node, inside_cost); - stmt_vinfo_set_outside_of_loop_cost (stmt_info, slp_node, outside_cost); } @@ -1725,7 +1803,7 @@ vectorizable_conversion (gimple stmt, gimple_stmt_iterator *gsi, builtin_decl = targetm.vectorize.builtin_conversion (code, vectype_out, vectype_in); - for (i = 0; VEC_iterate (tree, vec_oprnds0, i, vop0); i++) + FOR_EACH_VEC_ELT (tree, vec_oprnds0, i, vop0) { /* Arguments are ready. create the new vector stmt. */ new_stmt = gimple_build_call (builtin_decl, 1, vop0); @@ -1926,7 +2004,7 @@ vectorizable_assignment (gimple stmt, gimple_stmt_iterator *gsi, vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds, NULL); /* Arguments are ready. create the new vector stmt. */ - for (i = 0; VEC_iterate (tree, vec_oprnds, i, vop); i++) + FOR_EACH_VEC_ELT (tree, vec_oprnds, i, vop) { if (CONVERT_EXPR_CODE_P (code)) vop = build1 (VIEW_CONVERT_EXPR, vectype, vop); @@ -2104,8 +2182,7 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, { optab = optab_for_tree_code (code, vectype, optab_scalar); if (optab - && (optab_handler (optab, TYPE_MODE (vectype))->insn_code - != CODE_FOR_nothing)) + && optab_handler (optab, TYPE_MODE (vectype)) != CODE_FOR_nothing) { scalar_shift_arg = true; if (vect_print_dump_info (REPORT_DETAILS)) @@ -2115,7 +2192,7 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, { optab = optab_for_tree_code (code, vectype, optab_vector); if (optab - && (optab_handler (optab, TYPE_MODE (vectype))->insn_code + && (optab_handler (optab, TYPE_MODE (vectype)) != CODE_FOR_nothing)) { if (vect_print_dump_info (REPORT_DETAILS)) @@ -2149,7 +2226,7 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, return false; } vec_mode = TYPE_MODE (vectype); - icode = (int) optab_handler (optab, vec_mode)->insn_code; + icode = (int) optab_handler (optab, vec_mode); if (icode == CODE_FOR_nothing) { if (vect_print_dump_info (REPORT_DETAILS)) @@ -2305,7 +2382,7 @@ vectorizable_operation (gimple stmt, gimple_stmt_iterator *gsi, vect_get_vec_defs_for_stmt_copy (dt, &vec_oprnds0, &vec_oprnds1); /* Arguments are ready. Create the new vector stmt. */ - for (i = 0; VEC_iterate (tree, vec_oprnds0, i, vop0); i++) + FOR_EACH_VEC_ELT (tree, vec_oprnds0, i, vop0) { vop1 = ((op_type == binary_op) ? VEC_index (tree, vec_oprnds1, i) : NULL); @@ -2645,7 +2722,7 @@ vect_create_vectorized_promotion_stmts (VEC (tree, heap) **vec_oprnds0, vec_dest = VEC_pop (tree, vec_dsts); vec_tmp = VEC_alloc (tree, heap, VEC_length (tree, *vec_oprnds0) * 2); - for (i = 0; VEC_iterate (tree, *vec_oprnds0, i, vop0); i++) + FOR_EACH_VEC_ELT (tree, *vec_oprnds0, i, vop0) { if (op_type == binary_op) vop1 = VEC_index (tree, *vec_oprnds1, i); @@ -3051,7 +3128,7 @@ vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, vec_mode = TYPE_MODE (vectype); /* FORNOW. In some cases can vectorize even if data-type not supported (e.g. - array initialization with 0). */ - if (optab_handler (mov_optab, (int)vec_mode)->insn_code == CODE_FOR_nothing) + if (optab_handler (mov_optab, vec_mode) == CODE_FOR_nothing) return false; if (!STMT_VINFO_DATA_REF (stmt_info)) @@ -3142,7 +3219,7 @@ vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, dr_chain = VEC_alloc (tree, heap, group_size); oprnds = VEC_alloc (tree, heap, group_size); - alignment_support_scheme = vect_supportable_dr_alignment (first_dr); + alignment_support_scheme = vect_supportable_dr_alignment (first_dr, false); gcc_assert (alignment_support_scheme); /* In case the vectorization factor (VF) is bigger than the number @@ -3270,6 +3347,8 @@ vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, next_stmt = first_stmt; for (i = 0; i < vec_num; i++) { + struct ptr_info_def *pi; + if (i > 0) /* Bump the vector pointer. */ dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, gsi, stmt, @@ -3282,20 +3361,28 @@ vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, vect_permute_store_chain(). */ vec_oprnd = VEC_index (tree, result_chain, i); + data_ref = build2 (MEM_REF, TREE_TYPE (vec_oprnd), dataref_ptr, + build_int_cst (reference_alias_ptr_type + (DR_REF (first_dr)), 0)); + pi = get_ptr_info (dataref_ptr); + pi->align = TYPE_ALIGN_UNIT (vectype); if (aligned_access_p (first_dr)) - data_ref = build_simple_mem_ref (dataref_ptr); - else - { - int mis = DR_MISALIGNMENT (first_dr); - tree tmis = (mis == -1 ? size_zero_node : size_int (mis)); - tmis = size_binop (MULT_EXPR, tmis, size_int (BITS_PER_UNIT)); - data_ref = build2 (MISALIGNED_INDIRECT_REF, vectype, dataref_ptr, tmis); - } - - /* If accesses through a pointer to vectype do not alias the original - memory reference we have a problem. This should never happen. */ - gcc_assert (alias_sets_conflict_p (get_alias_set (data_ref), - get_alias_set (gimple_assign_lhs (stmt)))); + pi->misalign = 0; + else if (DR_MISALIGNMENT (first_dr) == -1) + { + TREE_TYPE (data_ref) + = build_aligned_type (TREE_TYPE (data_ref), + TYPE_ALIGN (TREE_TYPE (vectype))); + pi->align = TYPE_ALIGN_UNIT (TREE_TYPE (vectype)); + pi->misalign = 0; + } + else + { + TREE_TYPE (data_ref) + = build_aligned_type (TREE_TYPE (data_ref), + TYPE_ALIGN (TREE_TYPE (vectype))); + pi->misalign = DR_MISALIGNMENT (first_dr); + } /* Arguments are ready. Create the new vector stmt. */ new_stmt = gimple_build_assign (data_ref, vec_oprnd); @@ -3349,7 +3436,7 @@ vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info), *first_dr; tree vectype = STMT_VINFO_VECTYPE (stmt_info); tree new_temp; - int mode; + enum machine_mode mode; gimple new_stmt = NULL; tree dummy; enum dr_alignment_support alignment_support_scheme; @@ -3430,11 +3517,11 @@ vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, return false; scalar_type = TREE_TYPE (DR_REF (dr)); - mode = (int) TYPE_MODE (vectype); + mode = TYPE_MODE (vectype); /* FORNOW. In some cases can vectorize even if data-type not supported (e.g. - data copies). */ - if (optab_handler (mov_optab, mode)->insn_code == CODE_FOR_nothing) + if (optab_handler (mov_optab, mode) == CODE_FOR_nothing) { if (vect_print_dump_info (REPORT_DETAILS)) fprintf (vect_dump, "Aligned load, but unsupported type."); @@ -3507,7 +3594,7 @@ vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, group_size = vec_num = 1; } - alignment_support_scheme = vect_supportable_dr_alignment (first_dr); + alignment_support_scheme = vect_supportable_dr_alignment (first_dr, false); gcc_assert (alignment_support_scheme); /* In case the vectorization factor (VF) is bigger than the number @@ -3660,17 +3747,35 @@ vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, switch (alignment_support_scheme) { case dr_aligned: - gcc_assert (aligned_access_p (first_dr)); - data_ref = build_simple_mem_ref (dataref_ptr); - break; case dr_unaligned_supported: { - int mis = DR_MISALIGNMENT (first_dr); - tree tmis = (mis == -1 ? size_zero_node : size_int (mis)); - - tmis = size_binop (MULT_EXPR, tmis, size_int(BITS_PER_UNIT)); - data_ref = - build2 (MISALIGNED_INDIRECT_REF, vectype, dataref_ptr, tmis); + struct ptr_info_def *pi; + data_ref + = build2 (MEM_REF, vectype, dataref_ptr, + build_int_cst (reference_alias_ptr_type + (DR_REF (first_dr)), 0)); + pi = get_ptr_info (dataref_ptr); + pi->align = TYPE_ALIGN_UNIT (vectype); + if (alignment_support_scheme == dr_aligned) + { + gcc_assert (aligned_access_p (first_dr)); + pi->misalign = 0; + } + else if (DR_MISALIGNMENT (first_dr) == -1) + { + TREE_TYPE (data_ref) + = build_aligned_type (TREE_TYPE (data_ref), + TYPE_ALIGN (TREE_TYPE (vectype))); + pi->align = TYPE_ALIGN_UNIT (TREE_TYPE (vectype)); + pi->misalign = 0; + } + else + { + TREE_TYPE (data_ref) + = build_aligned_type (TREE_TYPE (data_ref), + TYPE_ALIGN (TREE_TYPE (vectype))); + pi->misalign = DR_MISALIGNMENT (first_dr); + } break; } case dr_explicit_realign: @@ -3684,7 +3789,18 @@ vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, dr_explicit_realign, dataref_ptr, NULL); - data_ref = build1 (ALIGN_INDIRECT_REF, vectype, dataref_ptr); + new_stmt = gimple_build_assign_with_ops + (BIT_AND_EXPR, NULL_TREE, dataref_ptr, + build_int_cst + (TREE_TYPE (dataref_ptr), + -(HOST_WIDE_INT)TYPE_ALIGN_UNIT (vectype))); + ptr = make_ssa_name (SSA_NAME_VAR (dataref_ptr), new_stmt); + gimple_assign_set_lhs (new_stmt, ptr); + vect_finish_stmt_generation (stmt, new_stmt, gsi); + data_ref + = build2 (MEM_REF, vectype, ptr, + build_int_cst (reference_alias_ptr_type + (DR_REF (first_dr)), 0)); vec_dest = vect_create_destination_var (scalar_dest, vectype); new_stmt = gimple_build_assign (vec_dest, data_ref); new_temp = make_ssa_name (vec_dest, new_stmt); @@ -3697,19 +3813,37 @@ vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, bump = size_binop (MULT_EXPR, vs_minus_1, TYPE_SIZE_UNIT (scalar_type)); ptr = bump_vector_ptr (dataref_ptr, NULL, gsi, stmt, bump); - data_ref = build1 (ALIGN_INDIRECT_REF, vectype, ptr); + new_stmt = gimple_build_assign_with_ops + (BIT_AND_EXPR, NULL_TREE, ptr, + build_int_cst + (TREE_TYPE (ptr), + -(HOST_WIDE_INT)TYPE_ALIGN_UNIT (vectype))); + ptr = make_ssa_name (SSA_NAME_VAR (dataref_ptr), new_stmt); + gimple_assign_set_lhs (new_stmt, ptr); + vect_finish_stmt_generation (stmt, new_stmt, gsi); + data_ref + = build2 (MEM_REF, vectype, ptr, + build_int_cst (reference_alias_ptr_type + (DR_REF (first_dr)), 0)); break; } case dr_explicit_realign_optimized: - data_ref = build1 (ALIGN_INDIRECT_REF, vectype, dataref_ptr); + new_stmt = gimple_build_assign_with_ops + (BIT_AND_EXPR, NULL_TREE, dataref_ptr, + build_int_cst + (TREE_TYPE (dataref_ptr), + -(HOST_WIDE_INT)TYPE_ALIGN_UNIT (vectype))); + new_temp = make_ssa_name (SSA_NAME_VAR (dataref_ptr), new_stmt); + gimple_assign_set_lhs (new_stmt, new_temp); + vect_finish_stmt_generation (stmt, new_stmt, gsi); + data_ref + = build2 (MEM_REF, vectype, new_temp, + build_int_cst (reference_alias_ptr_type + (DR_REF (first_dr)), 0)); break; default: gcc_unreachable (); } - /* If accesses through a pointer to vectype do not alias the original - memory reference we have a problem. This should never happen. */ - gcc_assert (alias_sets_conflict_p (get_alias_set (data_ref), - get_alias_set (gimple_assign_rhs1 (stmt)))); vec_dest = vect_create_destination_var (scalar_dest, vectype); new_stmt = gimple_build_assign (vec_dest, data_ref); new_temp = make_ssa_name (vec_dest, new_stmt); @@ -3904,16 +4038,18 @@ vectorizable_condition (gimple stmt, gimple_stmt_iterator *gsi, loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); enum machine_mode vec_mode; tree def; - enum vect_def_type dt; + enum vect_def_type dt, dts[4]; int nunits = TYPE_VECTOR_SUBPARTS (vectype); int ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits; enum tree_code code; + stmt_vec_info prev_stmt_info = NULL; + int j; /* FORNOW: unsupported in basic block SLP. */ gcc_assert (loop_vinfo); gcc_assert (ncopies >= 1); - if (ncopies > 1) + if (reduc_index && ncopies > 1) return false; /* FORNOW */ if (!STMT_VINFO_RELEVANT_P (stmt_info)) @@ -4000,29 +4136,68 @@ vectorizable_condition (gimple stmt, gimple_stmt_iterator *gsi, vec_dest = vect_create_destination_var (scalar_dest, vectype); /* Handle cond expr. */ - vec_cond_lhs = - vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), stmt, NULL); - vec_cond_rhs = - vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), stmt, NULL); - if (reduc_index == 1) - vec_then_clause = reduc_def; - else - vec_then_clause = vect_get_vec_def_for_operand (then_clause, stmt, NULL); - if (reduc_index == 2) - vec_else_clause = reduc_def; - else - vec_else_clause = vect_get_vec_def_for_operand (else_clause, stmt, NULL); + for (j = 0; j < ncopies; j++) + { + gimple new_stmt; + if (j == 0) + { + gimple gtemp; + vec_cond_lhs = + vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 0), + stmt, NULL); + vect_is_simple_use (TREE_OPERAND (cond_expr, 0), loop_vinfo, + NULL, >emp, &def, &dts[0]); + vec_cond_rhs = + vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr, 1), + stmt, NULL); + vect_is_simple_use (TREE_OPERAND (cond_expr, 1), loop_vinfo, + NULL, >emp, &def, &dts[1]); + if (reduc_index == 1) + vec_then_clause = reduc_def; + else + { + vec_then_clause = vect_get_vec_def_for_operand (then_clause, + stmt, NULL); + vect_is_simple_use (then_clause, loop_vinfo, + NULL, >emp, &def, &dts[2]); + } + if (reduc_index == 2) + vec_else_clause = reduc_def; + else + { + vec_else_clause = vect_get_vec_def_for_operand (else_clause, + stmt, NULL); + vect_is_simple_use (else_clause, loop_vinfo, + NULL, >emp, &def, &dts[3]); + } + } + else + { + vec_cond_lhs = vect_get_vec_def_for_stmt_copy (dts[0], vec_cond_lhs); + vec_cond_rhs = vect_get_vec_def_for_stmt_copy (dts[1], vec_cond_rhs); + vec_then_clause = vect_get_vec_def_for_stmt_copy (dts[2], + vec_then_clause); + vec_else_clause = vect_get_vec_def_for_stmt_copy (dts[3], + vec_else_clause); + } - /* Arguments are ready. Create the new vector stmt. */ - vec_compare = build2 (TREE_CODE (cond_expr), vectype, - vec_cond_lhs, vec_cond_rhs); - vec_cond_expr = build3 (VEC_COND_EXPR, vectype, - vec_compare, vec_then_clause, vec_else_clause); + /* Arguments are ready. Create the new vector stmt. */ + vec_compare = build2 (TREE_CODE (cond_expr), vectype, + vec_cond_lhs, vec_cond_rhs); + vec_cond_expr = build3 (VEC_COND_EXPR, vectype, + vec_compare, vec_then_clause, vec_else_clause); - *vec_stmt = gimple_build_assign (vec_dest, vec_cond_expr); - new_temp = make_ssa_name (vec_dest, *vec_stmt); - gimple_assign_set_lhs (*vec_stmt, new_temp); - vect_finish_stmt_generation (stmt, *vec_stmt, gsi); + new_stmt = gimple_build_assign (vec_dest, vec_cond_expr); + new_temp = make_ssa_name (vec_dest, new_stmt); + gimple_assign_set_lhs (new_stmt, new_temp); + vect_finish_stmt_generation (stmt, new_stmt, gsi); + if (j == 0) + STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt; + else + STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt; + + prev_stmt_info = vinfo_for_stmt (new_stmt); + } return true; } @@ -4886,9 +5061,8 @@ supportable_widening_operation (enum tree_code code, gimple stmt, return false; vec_mode = TYPE_MODE (vectype); - if ((icode1 = optab_handler (optab1, vec_mode)->insn_code) == CODE_FOR_nothing - || (icode2 = optab_handler (optab2, vec_mode)->insn_code) - == CODE_FOR_nothing) + if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing + || (icode2 = optab_handler (optab2, vec_mode)) == CODE_FOR_nothing) return false; /* Check if it's a multi-step conversion that can be done using intermediate @@ -4920,16 +5094,16 @@ supportable_widening_operation (enum tree_code code, gimple stmt, optab4 = optab_for_tree_code (c2, intermediate_type, optab_default); if (!optab3 || !optab4 - || (icode1 = optab1->handlers[(int) prev_mode].insn_code) - == CODE_FOR_nothing + || ((icode1 = optab_handler (optab1, prev_mode)) + == CODE_FOR_nothing) || insn_data[icode1].operand[0].mode != intermediate_mode - || (icode2 = optab2->handlers[(int) prev_mode].insn_code) - == CODE_FOR_nothing + || ((icode2 = optab_handler (optab2, prev_mode)) + == CODE_FOR_nothing) || insn_data[icode2].operand[0].mode != intermediate_mode - || (icode1 = optab3->handlers[(int) intermediate_mode].insn_code) - == CODE_FOR_nothing - || (icode2 = optab4->handlers[(int) intermediate_mode].insn_code) - == CODE_FOR_nothing) + || ((icode1 = optab_handler (optab3, intermediate_mode)) + == CODE_FOR_nothing) + || ((icode2 = optab_handler (optab4, intermediate_mode)) + == CODE_FOR_nothing)) return false; VEC_quick_push (tree, *interm_types, intermediate_type); @@ -5016,8 +5190,7 @@ supportable_narrowing_operation (enum tree_code code, return false; vec_mode = TYPE_MODE (vectype); - if ((icode1 = optab_handler (optab1, vec_mode)->insn_code) - == CODE_FOR_nothing) + if ((icode1 = optab_handler (optab1, vec_mode)) == CODE_FOR_nothing) return false; /* Check if it's a multi-step conversion that can be done using intermediate @@ -5040,12 +5213,11 @@ supportable_narrowing_operation (enum tree_code code, interm_optab = optab_for_tree_code (c1, intermediate_type, optab_default); if (!interm_optab - || (icode1 = optab1->handlers[(int) prev_mode].insn_code) - == CODE_FOR_nothing + || ((icode1 = optab_handler (optab1, prev_mode)) + == CODE_FOR_nothing) || insn_data[icode1].operand[0].mode != intermediate_mode - || (icode1 - = interm_optab->handlers[(int) intermediate_mode].insn_code) - == CODE_FOR_nothing) + || ((icode1 = optab_handler (interm_optab, intermediate_mode)) + == CODE_FOR_nothing)) return false; VEC_quick_push (tree, *interm_types, intermediate_type);