#include "cfgloop.h"
#include "expr.h"
#include "optabs.h"
+#include "params.h"
#include "recog.h"
#include "tree-data-ref.h"
#include "tree-chrec.h"
static bool vect_transform_stmt (tree, block_stmt_iterator *, bool *);
static tree vect_create_destination_var (tree, tree);
static tree vect_create_data_ref_ptr
- (tree, block_stmt_iterator *, tree, tree *, tree *, bool);
+ (tree, block_stmt_iterator *, tree, tree *, tree *, bool, tree);
static tree vect_create_addr_base_for_vector_ref (tree, tree *, tree);
static tree vect_setup_realignment (tree, block_stmt_iterator *, tree *);
static tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
static tree vect_get_vec_def_for_operand (tree, tree, tree *);
-static tree vect_init_vector (tree, tree);
+static tree vect_init_vector (tree, tree, tree);
static void vect_finish_stmt_generation
(tree stmt, tree vec_stmt, block_stmt_iterator *bsi);
static bool vect_is_simple_cond (tree, loop_vec_info);
static tree vect_gen_niters_for_prolog_loop (loop_vec_info, tree);
static void vect_update_init_of_dr (struct data_reference *, tree niters);
static void vect_update_inits_of_drs (loop_vec_info, tree);
-static void vect_do_peeling_for_alignment (loop_vec_info, struct loops *);
-static void vect_do_peeling_for_loop_bound
- (loop_vec_info, tree *, struct loops *);
static int vect_min_worthwhile_factor (enum tree_code);
else
new_vect_var = create_tmp_var (type, prefix);
+ /* Mark vector typed variable as a gimple register variable. */
+ if (TREE_CODE (type) == VECTOR_TYPE)
+ DECL_GIMPLE_REG_P (new_vect_var) = true;
+
return new_vect_var;
}
struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
tree data_ref_base = unshare_expr (DR_BASE_ADDRESS (dr));
tree base_name = build_fold_indirect_ref (data_ref_base);
- tree ref = DR_REF (dr);
- tree scalar_type = TREE_TYPE (ref);
- tree scalar_ptr_type = build_pointer_type (scalar_type);
tree vec_stmt;
- tree new_temp;
tree addr_base, addr_expr;
tree dest, new_stmt;
tree base_offset = unshare_expr (DR_OFFSET (dr));
tree init = unshare_expr (DR_INIT (dr));
+ tree vect_ptr_type, addr_expr2;
/* Create base_offset */
base_offset = size_binop (PLUS_EXPR, base_offset, init);
addr_base = fold_build2 (PLUS_EXPR, TREE_TYPE (data_ref_base), data_ref_base,
base_offset);
+ vect_ptr_type = build_pointer_type (STMT_VINFO_VECTYPE (stmt_info));
+
/* addr_expr = addr_base */
- addr_expr = vect_get_new_vect_var (scalar_ptr_type, vect_pointer_var,
+ addr_expr = vect_get_new_vect_var (vect_ptr_type, vect_pointer_var,
get_name (base_name));
add_referenced_var (addr_expr);
- vec_stmt = build2 (MODIFY_EXPR, void_type_node, addr_expr, addr_base);
- new_temp = make_ssa_name (addr_expr, vec_stmt);
- TREE_OPERAND (vec_stmt, 0) = new_temp;
- append_to_statement_list_force (vec_stmt, new_stmt_list);
+ vec_stmt = fold_convert (vect_ptr_type, addr_base);
+ addr_expr2 = vect_get_new_vect_var (vect_ptr_type, vect_pointer_var,
+ get_name (base_name));
+ add_referenced_var (addr_expr2);
+ vec_stmt = force_gimple_operand (vec_stmt, &new_stmt, false, addr_expr2);
+ append_to_statement_list_force (new_stmt, new_stmt_list);
if (vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "created ");
print_generic_expr (vect_dump, vec_stmt, TDF_SLIM);
}
- return new_temp;
+ return vec_stmt;
}
Input:
1. STMT: a stmt that references memory. Expected to be of the form
- MODIFY_EXPR <name, data-ref> or MODIFY_EXPR <data-ref, name>.
+ GIMPLE_MODIFY_STMT <name, data-ref> or
+ GIMPLE_MODIFY_STMT <data-ref, name>.
2. BSI: block_stmt_iterator where new stmts can be added.
3. OFFSET (optional): an offset to be added to the initial address accessed
by the data-ref in STMT.
4. ONLY_INIT: indicate if vp is to be updated in the loop, or remain
pointing to the initial address.
+ 5. TYPE: if not NULL indicates the required type of the data-ref
Output:
1. Declare a new ptr to vector_type, and have it point to the base of the
vect_create_data_ref_ptr (tree stmt,
block_stmt_iterator *bsi ATTRIBUTE_UNUSED,
tree offset, tree *initial_address, tree *ptr_incr,
- bool only_init)
+ bool only_init, tree type)
{
tree base_name;
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
}
/** (1) Create the new vector-pointer variable: **/
-
- vect_ptr_type = build_pointer_type (vectype);
+ if (type)
+ vect_ptr_type = build_pointer_type (type);
+ else
+ vect_ptr_type = build_pointer_type (vectype);
vect_ptr = vect_get_new_vect_var (vect_ptr_type, vect_pointer_var,
get_name (base_name));
add_referenced_var (vect_ptr);
-
-
+
/** (2) Add aliasing information to the new vector-pointer:
(The points-to info (DR_PTR_INFO) may be defined later.) **/
if (!MTAG_P (tag))
new_type_alias (vect_ptr, tag, DR_REF (dr));
else
- var_ann (vect_ptr)->symbol_mem_tag = tag;
+ set_symbol_mem_tag (vect_ptr, tag);
var_ann (vect_ptr)->subvars = DR_SUBVARS (dr);
/* Create: p = (vectype *) initial_base */
vec_stmt = fold_convert (vect_ptr_type, new_temp);
- vec_stmt = build2 (MODIFY_EXPR, void_type_node, vect_ptr, vec_stmt);
+ vec_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vect_ptr, vec_stmt);
vect_ptr_init = make_ssa_name (vect_ptr, vec_stmt);
- TREE_OPERAND (vec_stmt, 0) = vect_ptr_init;
+ GIMPLE_STMT_OPERAND (vec_stmt, 0) = vect_ptr_init;
new_bb = bsi_insert_on_edge_immediate (pe, vec_stmt);
gcc_assert (!new_bb);
/* Function bump_vector_ptr
Increment a pointer (to a vector type) by vector-size. Connect the new
- increment stmt to the exising def-use update-chain of the pointer.
+ increment stmt to the existing def-use update-chain of the pointer.
The pointer def-use update-chain before this function:
DATAREF_PTR = phi (p_0, p_2)
use_operand_p use_p;
tree new_dataref_ptr;
- incr_stmt = build2 (MODIFY_EXPR, void_type_node, ptr_var,
+ incr_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, ptr_var,
build2 (PLUS_EXPR, vptr_type, dataref_ptr, update));
new_dataref_ptr = make_ssa_name (ptr_var, incr_stmt);
- TREE_OPERAND (incr_stmt, 0) = new_dataref_ptr;
+ GIMPLE_STMT_OPERAND (incr_stmt, 0) = new_dataref_ptr;
vect_finish_stmt_generation (stmt, incr_stmt, bsi);
/* Update the vector-pointer's cross-iteration increment. */
new_name = get_name (scalar_dest);
if (!new_name)
new_name = "var_";
- vec_dest = vect_get_new_vect_var (type, vect_simple_var, new_name);
+ vec_dest = vect_get_new_vect_var (type, kind, new_name);
add_referenced_var (vec_dest);
return vec_dest;
used in the vectorization of STMT. */
static tree
-vect_init_vector (tree stmt, tree vector_var)
+vect_init_vector (tree stmt, tree vector_var, tree vector_type)
{
stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
tree new_var;
tree init_stmt;
- tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
tree vec_oprnd;
edge pe;
tree new_temp;
basic_block new_bb;
- new_var = vect_get_new_vect_var (vectype, vect_simple_var, "cst_");
+ new_var = vect_get_new_vect_var (vector_type, vect_simple_var, "cst_");
add_referenced_var (new_var);
- init_stmt = build2 (MODIFY_EXPR, vectype, new_var, vector_var);
+ init_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, new_var, vector_var);
new_temp = make_ssa_name (new_var, init_stmt);
- TREE_OPERAND (init_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (init_stmt, 0) = new_temp;
pe = loop_preheader_edge (loop);
new_bb = bsi_insert_on_edge_immediate (pe, init_stmt);
print_generic_expr (vect_dump, init_stmt, TDF_SLIM);
}
- vec_oprnd = TREE_OPERAND (init_stmt, 0);
+ vec_oprnd = GIMPLE_STMT_OPERAND (init_stmt, 0);
return vec_oprnd;
}
+/* Function get_initial_def_for_induction
+
+ Input:
+ STMT - a stmt that performs an induction operation in the loop.
+ IV_PHI - the initial value of the induction variable
+
+ Output:
+ Return a vector variable, initialized with the first VF values of
+ the induction variable. E.g., for an iv with IV_PHI='X' and
+ evolution S, for a vector of 4 units, we want to return:
+ [X, X + S, X + 2*S, X + 3*S]. */
+
+static tree
+get_initial_def_for_induction (tree stmt, tree iv_phi)
+{
+ stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+ struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ tree scalar_type = TREE_TYPE (iv_phi);
+ tree vectype = get_vectype_for_scalar_type (scalar_type);
+ int nunits = GET_MODE_NUNITS (TYPE_MODE (vectype));
+ edge pe = loop_preheader_edge (loop);
+ basic_block new_bb;
+ block_stmt_iterator bsi;
+ tree vec, vec_init, vec_step, t;
+ tree access_fn;
+ tree new_var;
+ tree new_name;
+ tree init_stmt;
+ tree induction_phi, induc_def, new_stmt, vec_def, vec_dest;
+ tree init_expr, step_expr;
+ int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ int i;
+ bool ok;
+ int ncopies = vf / nunits;
+ tree expr;
+ stmt_vec_info phi_info = vinfo_for_stmt (iv_phi);
+
+ gcc_assert (phi_info);
+
+ if (STMT_VINFO_VEC_STMT (phi_info))
+ {
+ induction_phi = STMT_VINFO_VEC_STMT (phi_info);
+ gcc_assert (TREE_CODE (induction_phi) == PHI_NODE);
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "induction already vectorized:");
+ print_generic_expr (vect_dump, iv_phi, TDF_SLIM);
+ fprintf (vect_dump, "\n");
+ print_generic_expr (vect_dump, induction_phi, TDF_SLIM);
+ }
+
+ return PHI_RESULT (induction_phi);
+ }
+
+ gcc_assert (ncopies >= 1);
+
+ access_fn = analyze_scalar_evolution (loop, PHI_RESULT (iv_phi));
+ gcc_assert (access_fn);
+ ok = vect_is_simple_iv_evolution (loop->num, access_fn, &init_expr, &step_expr);
+ gcc_assert (ok);
+
+ /* Create the vector that holds the initial_value of the induction. */
+ new_name = init_expr;
+ t = NULL_TREE;
+ t = tree_cons (NULL_TREE, init_expr, t);
+ for (i = 1; i < nunits; i++)
+ {
+ /* Create: new_name = new_name + step_expr */
+ new_var = vect_get_new_vect_var (scalar_type, vect_scalar_var, "var_");
+ add_referenced_var (new_var);
+ init_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, new_var,
+ fold_build2 (PLUS_EXPR, scalar_type, new_name, step_expr));
+ new_name = make_ssa_name (new_var, init_stmt);
+ GIMPLE_STMT_OPERAND (init_stmt, 0) = new_name;
+
+ new_bb = bsi_insert_on_edge_immediate (pe, init_stmt);
+ gcc_assert (!new_bb);
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "created new init_stmt: ");
+ print_generic_expr (vect_dump, init_stmt, TDF_SLIM);
+ }
+ t = tree_cons (NULL_TREE, new_name, t);
+ }
+ vec = build_constructor_from_list (vectype, nreverse (t));
+ vec_init = vect_init_vector (stmt, vec, vectype);
+
+
+ /* Create the vector that holds the step of the induction. */
+ expr = build_int_cst (scalar_type, vf);
+ new_name = fold_build2 (MULT_EXPR, scalar_type, expr, step_expr);
+ t = NULL_TREE;
+ for (i = 0; i < nunits; i++)
+ t = tree_cons (NULL_TREE, unshare_expr (new_name), t);
+ vec = build_constructor_from_list (vectype, t);
+ vec_step = vect_init_vector (stmt, vec, vectype);
+
+
+ /* Create the following def-use cycle:
+ loop prolog:
+ vec_init = [X, X+S, X+2*S, X+3*S]
+ vec_step = [VF*S, VF*S, VF*S, VF*S]
+ loop:
+ vec_iv = PHI <vec_init, vec_loop>
+ ...
+ STMT
+ ...
+ vec_loop = vec_iv + vec_step; */
+
+ /* Create the induction-phi that defines the induction-operand. */
+ vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_");
+ add_referenced_var (vec_dest);
+ induction_phi = create_phi_node (vec_dest, loop->header);
+ set_stmt_info (get_stmt_ann (induction_phi),
+ new_stmt_vec_info (induction_phi, loop_vinfo));
+ induc_def = PHI_RESULT (induction_phi);
+
+ /* Create the iv update inside the loop */
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, NULL_TREE,
+ build2 (PLUS_EXPR, vectype, induc_def, vec_step));
+ vec_def = make_ssa_name (vec_dest, new_stmt);
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = vec_def;
+ bsi = bsi_for_stmt (stmt);
+ vect_finish_stmt_generation (stmt, new_stmt, &bsi);
+
+ /* Set the arguments of the phi node: */
+ add_phi_arg (induction_phi, vec_init, loop_preheader_edge (loop));
+ add_phi_arg (induction_phi, vec_def, loop_latch_edge (loop));
+
+
+ /* In case the vectorization factor (VF) is bigger than the number
+ of elements that we can fit in a vectype (nunits), we have to generate
+ more than one vector stmt - i.e - we need to "unroll" the
+ vector stmt by a factor VF/nunits. For more details see documentation
+ in vectorizable_operation. */
+
+ if (ncopies > 1)
+ {
+ stmt_vec_info prev_stmt_vinfo;
+
+ /* Create the vector that holds the step of the induction. */
+ expr = build_int_cst (scalar_type, nunits);
+ new_name = fold_build2 (MULT_EXPR, scalar_type, expr, step_expr);
+ t = NULL_TREE;
+ for (i = 0; i < nunits; i++)
+ t = tree_cons (NULL_TREE, unshare_expr (new_name), t);
+ vec = build_constructor_from_list (vectype, t);
+ vec_step = vect_init_vector (stmt, vec, vectype);
+
+ vec_def = induc_def;
+ prev_stmt_vinfo = vinfo_for_stmt (induction_phi);
+ for (i = 1; i < ncopies; i++)
+ {
+ /* vec_i = vec_prev + vec_{step*nunits} */
+
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, NULL_TREE,
+ build2 (PLUS_EXPR, vectype, vec_def, vec_step));
+ vec_def = make_ssa_name (vec_dest, new_stmt);
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = vec_def;
+ bsi = bsi_for_stmt (stmt);
+ vect_finish_stmt_generation (stmt, new_stmt, &bsi);
+
+ STMT_VINFO_RELATED_STMT (prev_stmt_vinfo) = new_stmt;
+ prev_stmt_vinfo = vinfo_for_stmt (new_stmt);
+ }
+ }
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "transform induction: created def-use cycle:");
+ print_generic_expr (vect_dump, induction_phi, TDF_SLIM);
+ fprintf (vect_dump, "\n");
+ print_generic_expr (vect_dump, SSA_NAME_DEF_STMT (vec_def), TDF_SLIM);
+ }
+
+ STMT_VINFO_VEC_STMT (phi_info) = induction_phi;
+ return induc_def;
+}
+
+
/* Function vect_get_vec_def_for_operand.
OP is an operand in STMT. This function returns a (vector) def that will be
int i;
enum vect_def_type dt;
bool is_simple_use;
+ tree vector_type;
if (vect_print_dump_info (REPORT_DETAILS))
{
{
t = tree_cons (NULL_TREE, op, t);
}
- vec_cst = build_vector (vectype, t);
- return vect_init_vector (stmt, vec_cst);
+ vector_type = get_vectype_for_scalar_type (TREE_TYPE (op));
+ vec_cst = build_vector (vector_type, t);
+
+ return vect_init_vector (stmt, vec_cst, vector_type);
}
/* Case 2: operand is defined outside the loop - loop invariant. */
}
/* FIXME: use build_constructor directly. */
- vec_inv = build_constructor_from_list (vectype, t);
- return vect_init_vector (stmt, vec_inv);
+ vector_type = get_vectype_for_scalar_type (TREE_TYPE (def));
+ vec_inv = build_constructor_from_list (vector_type, t);
+ return vect_init_vector (stmt, vec_inv, vector_type);
}
/* Case 3: operand is defined inside the loop. */
def_stmt_info = vinfo_for_stmt (def_stmt);
vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
gcc_assert (vec_stmt);
- vec_oprnd = TREE_OPERAND (vec_stmt, 0);
+ vec_oprnd = GIMPLE_STMT_OPERAND (vec_stmt, 0);
return vec_oprnd;
}
/* Case 5: operand is defined by loop-header phi - induction. */
case vect_induction_def:
{
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "induction - unsupported.");
- internal_error ("no support for induction"); /* FORNOW */
+ gcc_assert (TREE_CODE (def_stmt) == PHI_NODE);
+
+ /* Get the def before the loop */
+ return get_initial_def_for_induction (stmt, def_stmt);
}
default:
stmts operating on wider types we need to create 'VF/nunits' "copies" of the
vector stmt (each computing a vector of 'nunits' results, and together
computing 'VF' results in each iteration). This function is called when
- vectorizing such a stmt (e.g. vectorizing S2 in the illusration below, in
- which VF=16 and nuniti=4, so the number of copies required is 4):
+ vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in
+ which VF=16 and nunits=4, so the number of copies required is 4):
scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT
tree vec_stmt_for_operand;
stmt_vec_info def_stmt_info;
- if (dt == vect_invariant_def || dt == vect_constant_def)
- {
- /* Do nothing; can reuse same def. */ ;
- return vec_oprnd;
- }
+ /* Do nothing; can reuse same def. */
+ if (dt == vect_invariant_def || dt == vect_constant_def )
+ return vec_oprnd;
vec_stmt_for_operand = SSA_NAME_DEF_STMT (vec_oprnd);
def_stmt_info = vinfo_for_stmt (vec_stmt_for_operand);
gcc_assert (def_stmt_info);
vec_stmt_for_operand = STMT_VINFO_RELATED_STMT (def_stmt_info);
gcc_assert (vec_stmt_for_operand);
- vec_oprnd = TREE_OPERAND (vec_stmt_for_operand, 0);
+ vec_oprnd = GIMPLE_STMT_OPERAND (vec_stmt_for_operand, 0);
return vec_oprnd;
}
tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
int nunits = GET_MODE_NUNITS (TYPE_MODE (vectype));
int nelements;
- enum tree_code code = TREE_CODE (TREE_OPERAND (stmt, 1));
+ enum tree_code code = TREE_CODE (GIMPLE_STMT_OPERAND (stmt, 1));
tree type = TREE_TYPE (init_val);
tree def;
tree vec, t = NULL_TREE;
bool need_epilog_adjust;
int i;
+ tree vector_type;
gcc_assert (INTEGRAL_TYPE_P (type) || SCALAR_FLOAT_TYPE_P (type));
nelements += 1;
}
gcc_assert (nelements == nunits);
-
+
+ vector_type = get_vectype_for_scalar_type (TREE_TYPE (def));
if (TREE_CODE (init_val) == INTEGER_CST || TREE_CODE (init_val) == REAL_CST)
- vec = build_vector (vectype, t);
+ vec = build_vector (vector_type, t);
else
- vec = build_constructor_from_list (vectype, t);
+ vec = build_constructor_from_list (vector_type, t);
if (!need_epilog_adjust)
*scalar_def = NULL_TREE;
else
*scalar_def = init_val;
- return vect_init_vector (stmt, vec);
+ return vect_init_vector (stmt, vec, vector_type);
}
REDUCTION_PHI is the phi-node that carries the reduction computation.
This function:
- 1. Creates the reduction def-use cycle: sets the the arguments for
+ 1. Creates the reduction def-use cycle: sets the arguments for
REDUCTION_PHI:
The loop-entry argument is the vectorized initial-value of the reduction.
The loop-latch argument is VECT_DEF - the vector of partial sums.
tree epilog_stmt;
tree new_scalar_dest, exit_phi;
tree bitsize, bitpos, bytesize;
- enum tree_code code = TREE_CODE (TREE_OPERAND (stmt, 1));
+ enum tree_code code = TREE_CODE (GIMPLE_STMT_OPERAND (stmt, 1));
tree scalar_initial_def;
tree vec_initial_def;
tree orig_name;
tree reduction_op;
tree orig_stmt;
tree use_stmt;
- tree operation = TREE_OPERAND (stmt, 1);
+ tree operation = GIMPLE_STMT_OPERAND (stmt, 1);
int op_type;
- op_type = TREE_CODE_LENGTH (TREE_CODE (operation));
+ op_type = TREE_OPERAND_LENGTH (operation);
reduction_op = TREE_OPERAND (operation, op_type-1);
vectype = get_vectype_for_scalar_type (TREE_TYPE (reduction_op));
mode = TYPE_MODE (vectype);
gcc_assert (STMT_VINFO_IN_PATTERN_P (stmt_vinfo));
gcc_assert (STMT_VINFO_RELATED_STMT (stmt_vinfo) == stmt);
}
- code = TREE_CODE (TREE_OPERAND (orig_stmt, 1));
- scalar_dest = TREE_OPERAND (orig_stmt, 0);
+ code = TREE_CODE (GIMPLE_STMT_OPERAND (orig_stmt, 1));
+ scalar_dest = GIMPLE_STMT_OPERAND (orig_stmt, 0);
scalar_type = TREE_TYPE (scalar_dest);
new_scalar_dest = vect_create_destination_var (scalar_dest, NULL);
bitsize = TYPE_SIZE (scalar_type);
fprintf (vect_dump, "Reduce using direct vector reduction.");
vec_dest = vect_create_destination_var (scalar_dest, vectype);
- epilog_stmt = build2 (MODIFY_EXPR, vectype, vec_dest,
+ epilog_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest,
build1 (reduc_code, vectype, PHI_RESULT (new_phi)));
new_temp = make_ssa_name (vec_dest, epilog_stmt);
- TREE_OPERAND (epilog_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_temp;
bsi_insert_after (&exit_bsi, epilog_stmt, BSI_NEW_STMT);
extract_scalar_result = true;
{
tree bitpos = size_int (bit_offset);
- epilog_stmt = build2 (MODIFY_EXPR, vectype, vec_dest,
+ epilog_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
+ vec_dest,
build2 (shift_code, vectype,
new_temp, bitpos));
new_name = make_ssa_name (vec_dest, epilog_stmt);
- TREE_OPERAND (epilog_stmt, 0) = new_name;
+ GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_name;
bsi_insert_after (&exit_bsi, epilog_stmt, BSI_NEW_STMT);
- epilog_stmt = build2 (MODIFY_EXPR, vectype, vec_dest,
+ epilog_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
+ vec_dest,
build2 (code, vectype,
new_name, new_temp));
new_temp = make_ssa_name (vec_dest, epilog_stmt);
- TREE_OPERAND (epilog_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_temp;
bsi_insert_after (&exit_bsi, epilog_stmt, BSI_NEW_STMT);
}
rhs = build3 (BIT_FIELD_REF, scalar_type, vec_temp, bitsize,
bitsize_zero_node);
BIT_FIELD_REF_UNSIGNED (rhs) = TYPE_UNSIGNED (scalar_type);
- epilog_stmt = build2 (MODIFY_EXPR, scalar_type, new_scalar_dest, rhs);
+ epilog_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
+ new_scalar_dest, rhs);
new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
- TREE_OPERAND (epilog_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_temp;
bsi_insert_after (&exit_bsi, epilog_stmt, BSI_NEW_STMT);
for (bit_offset = element_bitsize;
bitpos);
BIT_FIELD_REF_UNSIGNED (rhs) = TYPE_UNSIGNED (scalar_type);
- epilog_stmt = build2 (MODIFY_EXPR, scalar_type, new_scalar_dest,
- rhs);
+ epilog_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
+ new_scalar_dest, rhs);
new_name = make_ssa_name (new_scalar_dest, epilog_stmt);
- TREE_OPERAND (epilog_stmt, 0) = new_name;
+ GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_name;
bsi_insert_after (&exit_bsi, epilog_stmt, BSI_NEW_STMT);
- epilog_stmt = build2 (MODIFY_EXPR, scalar_type, new_scalar_dest,
+ epilog_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
+ new_scalar_dest,
build2 (code, scalar_type, new_name, new_temp));
new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
- TREE_OPERAND (epilog_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_temp;
bsi_insert_after (&exit_bsi, epilog_stmt, BSI_NEW_STMT);
}
rhs = build3 (BIT_FIELD_REF, scalar_type, new_temp, bitsize, bitpos);
BIT_FIELD_REF_UNSIGNED (rhs) = TYPE_UNSIGNED (scalar_type);
- epilog_stmt = build2 (MODIFY_EXPR, scalar_type, new_scalar_dest, rhs);
+ epilog_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
+ new_scalar_dest, rhs);
new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
- TREE_OPERAND (epilog_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_temp;
bsi_insert_after (&exit_bsi, epilog_stmt, BSI_NEW_STMT);
}
if (scalar_initial_def)
{
- epilog_stmt = build2 (MODIFY_EXPR, scalar_type, new_scalar_dest,
+ epilog_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
+ new_scalar_dest,
build2 (code, scalar_type, new_temp, scalar_initial_def));
new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
- TREE_OPERAND (epilog_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_temp;
bsi_insert_after (&exit_bsi, epilog_stmt, BSI_NEW_STMT);
}
inside the loop body. The last operand is the reduction variable,
which is defined by the loop-header-phi. */
- gcc_assert (TREE_CODE (stmt) == MODIFY_EXPR);
+ gcc_assert (TREE_CODE (stmt) == GIMPLE_MODIFY_STMT);
- operation = TREE_OPERAND (stmt, 1);
+ operation = GIMPLE_STMT_OPERAND (stmt, 1);
code = TREE_CODE (operation);
- op_type = TREE_CODE_LENGTH (code);
+ op_type = TREE_OPERAND_LENGTH (operation);
if (op_type != binary_op && op_type != ternary_op)
return false;
- scalar_dest = TREE_OPERAND (stmt, 0);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
scalar_type = TREE_TYPE (scalar_dest);
/* All uses but the last are expected to be defined in the loop.
op = TREE_OPERAND (operation, i);
is_simple_use = vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt);
gcc_assert (is_simple_use);
- gcc_assert (dt == vect_loop_def || dt == vect_invariant_def ||
- dt == vect_constant_def);
+ if (dt != vect_loop_def
+ && dt != vect_invariant_def
+ && dt != vect_constant_def
+ && dt != vect_induction_def)
+ return false;
}
op = TREE_OPERAND (operation, i);
{
/* This is a reduction pattern: get the vectype from the type of the
reduction variable, and get the tree-code from orig_stmt. */
- orig_code = TREE_CODE (TREE_OPERAND (orig_stmt, 1));
+ orig_code = TREE_CODE (GIMPLE_STMT_OPERAND (orig_stmt, 1));
vectype = get_vectype_for_scalar_type (TREE_TYPE (def));
vec_mode = TYPE_MODE (vectype);
}
/* Get the vector def for the reduction variable from the vectorized
reduction operation generated in the previous iteration (j-1) */
- reduc_def = TREE_OPERAND (new_stmt ,0);
+ reduc_def = GIMPLE_STMT_OPERAND (new_stmt ,0);
}
/* Arguments are ready. create the new vector stmt. */
else
expr = build3 (code, vectype, loop_vec_def0, loop_vec_def1,
reduc_def);
- new_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest, expr);
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest, expr);
new_temp = make_ssa_name (vec_dest, new_stmt);
- TREE_OPERAND (new_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
vect_finish_stmt_generation (stmt, new_stmt, bsi);
if (j == 0)
return true;
}
+/* Checks if CALL can be vectorized in type VECTYPE. Returns
+ a function declaration if the target has a vectorized version
+ of the function, or NULL_TREE if the function cannot be vectorized. */
+
+tree
+vectorizable_function (tree call, tree vectype_out, tree vectype_in)
+{
+ tree fndecl = get_callee_fndecl (call);
+ enum built_in_function code;
+
+ /* We only handle functions that do not read or clobber memory -- i.e.
+ const or novops ones. */
+ if (!(call_expr_flags (call) & (ECF_CONST | ECF_NOVOPS)))
+ return NULL_TREE;
+
+ if (!fndecl
+ || TREE_CODE (fndecl) != FUNCTION_DECL
+ || !DECL_BUILT_IN (fndecl))
+ return NULL_TREE;
+
+ code = DECL_FUNCTION_CODE (fndecl);
+ return targetm.vectorize.builtin_vectorized_function (code, vectype_out,
+ vectype_in);
+}
+
+/* Function vectorizable_call.
+
+ Check if STMT performs a function call that can be vectorized.
+ If VEC_STMT is also passed, vectorize the STMT: create a vectorized
+ stmt to replace it, put it in VEC_STMT, and insert it at BSI.
+ Return FALSE if not a vectorizable STMT, TRUE otherwise. */
+
+bool
+vectorizable_call (tree stmt, block_stmt_iterator *bsi, tree *vec_stmt)
+{
+ tree vec_dest;
+ tree scalar_dest;
+ tree operation;
+ tree op, type;
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt), prev_stmt_info;
+ tree vectype_out, vectype_in;
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ tree fndecl, rhs, new_temp, def, def_stmt, rhs_type, lhs_type;
+ enum vect_def_type dt[2];
+ int ncopies, j, nargs;
+ call_expr_arg_iterator iter;
+
+ /* Is STMT a vectorizable call? */
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
+ return false;
+
+ if (TREE_CODE (GIMPLE_STMT_OPERAND (stmt, 0)) != SSA_NAME)
+ return false;
+
+ operation = GIMPLE_STMT_OPERAND (stmt, 1);
+ if (TREE_CODE (operation) != CALL_EXPR)
+ return false;
+
+ /* Process function arguments. */
+ rhs_type = NULL_TREE;
+ nargs = 0;
+ FOR_EACH_CALL_EXPR_ARG (op, iter, operation)
+ {
+ ++nargs;
+
+ /* Bail out if the function has more than two arguments, we
+ do not have interesting builtin functions to vectorize with
+ more than two arguments. */
+ if (nargs > 2)
+ return false;
+
+ /* We can only handle calls with arguments of the same type. */
+ if (rhs_type
+ && rhs_type != TREE_TYPE (op))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "argument types differ.");
+ return false;
+ }
+ rhs_type = TREE_TYPE (op);
+
+ if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt[nargs]))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "use not simple.");
+ return false;
+ }
+ }
+
+ /* No arguments is also not good. */
+ if (nargs == 0)
+ return false;
+
+ vectype_in = get_vectype_for_scalar_type (rhs_type);
+
+ lhs_type = TREE_TYPE (GIMPLE_STMT_OPERAND (stmt, 0));
+ vectype_out = get_vectype_for_scalar_type (lhs_type);
+
+ /* Only handle the case of vectors with the same number of elements.
+ FIXME: We need a way to handle for example the SSE2 cvtpd2dq
+ instruction which converts V2DFmode to V4SImode but only
+ using the lower half of the V4SImode result. */
+ if (TYPE_VECTOR_SUBPARTS (vectype_in) != TYPE_VECTOR_SUBPARTS (vectype_out))
+ return false;
+
+ /* For now, we only vectorize functions if a target specific builtin
+ is available. TODO -- in some cases, it might be profitable to
+ insert the calls for pieces of the vector, in order to be able
+ to vectorize other operations in the loop. */
+ fndecl = vectorizable_function (operation, vectype_out, vectype_in);
+ if (fndecl == NULL_TREE)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "function is not vectorizable.");
+
+ return false;
+ }
+
+ gcc_assert (ZERO_SSA_OPERANDS (stmt, SSA_OP_ALL_VIRTUALS));
+
+ if (!vec_stmt) /* transformation not required. */
+ {
+ STMT_VINFO_TYPE (stmt_info) = call_vec_info_type;
+ return true;
+ }
+
+ /** Transform. **/
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "transform operation.");
+
+ ncopies = (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
+ / TYPE_VECTOR_SUBPARTS (vectype_out));
+ gcc_assert (ncopies >= 1);
+
+ /* Handle def. */
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
+ vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
+
+ prev_stmt_info = NULL;
+ for (j = 0; j < ncopies; ++j)
+ {
+ tree new_stmt, vargs;
+ tree vec_oprnd[2];
+ int n;
+
+ /* Build argument list for the vectorized call. */
+ /* FIXME: Rewrite this so that it doesn't construct a temporary
+ list. */
+ vargs = NULL_TREE;
+ n = -1;
+ FOR_EACH_CALL_EXPR_ARG (op, iter, operation)
+ {
+ ++n;
+
+ if (j == 0)
+ vec_oprnd[n] = vect_get_vec_def_for_operand (op, stmt, NULL);
+ else
+ vec_oprnd[n] = vect_get_vec_def_for_stmt_copy (dt[n], vec_oprnd[n]);
+
+ vargs = tree_cons (NULL_TREE, vec_oprnd[n], vargs);
+ }
+ vargs = nreverse (vargs);
+
+ rhs = build_function_call_expr (fndecl, vargs);
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, NULL_TREE, vec_dest, rhs);
+ new_temp = make_ssa_name (vec_dest, new_stmt);
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
+
+ vect_finish_stmt_generation (stmt, new_stmt, bsi);
+
+ if (j == 0)
+ STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
+ else
+ STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
+ prev_stmt_info = vinfo_for_stmt (new_stmt);
+ }
+
+ /* The call in STMT might prevent it from being removed in dce. We however
+ cannot remove it here, due to the way the ssa name it defines is mapped
+ to the new definition. So just replace rhs of the statement with something
+ harmless. */
+ type = TREE_TYPE (scalar_dest);
+ GIMPLE_STMT_OPERAND (stmt, 1) = fold_convert (type, integer_zero_node);
+
+ return true;
+}
+
+
+/* Function vectorizable_conversion.
+
+Check if STMT performs a conversion operation, that can be vectorized.
+If VEC_STMT is also passed, vectorize the STMT: create a vectorized
+stmt to replace it, put it in VEC_STMT, and insert it at BSI.
+Return FALSE if not a vectorizable STMT, TRUE otherwise. */
+
+bool
+vectorizable_conversion (tree stmt, block_stmt_iterator * bsi,
+ tree * vec_stmt)
+{
+ tree vec_dest;
+ tree scalar_dest;
+ tree operation;
+ tree op0;
+ tree vec_oprnd0 = NULL_TREE;
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ enum tree_code code;
+ tree new_temp;
+ tree def, def_stmt;
+ enum vect_def_type dt0;
+ tree new_stmt;
+ int nunits_in;
+ int nunits_out;
+ int ncopies, j;
+ tree vectype_out, vectype_in;
+ tree rhs_type, lhs_type;
+ tree builtin_decl;
+ stmt_vec_info prev_stmt_info;
+
+ /* Is STMT a vectorizable conversion? */
+
+ if (!STMT_VINFO_RELEVANT_P (stmt_info))
+ return false;
+
+ gcc_assert (STMT_VINFO_DEF_TYPE (stmt_info) == vect_loop_def);
+
+ if (STMT_VINFO_LIVE_P (stmt_info))
+ {
+ /* FORNOW: not yet supported. */
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "value used after loop.");
+ return false;
+ }
+
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
+ return false;
+
+ if (TREE_CODE (GIMPLE_STMT_OPERAND (stmt, 0)) != SSA_NAME)
+ return false;
+
+ operation = GIMPLE_STMT_OPERAND (stmt, 1);
+ code = TREE_CODE (operation);
+ if (code != FIX_TRUNC_EXPR && code != FLOAT_EXPR)
+ return false;
+
+ /* Check types of lhs and rhs */
+ op0 = TREE_OPERAND (operation, 0);
+ rhs_type = TREE_TYPE (op0);
+ vectype_in = get_vectype_for_scalar_type (rhs_type);
+ nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
+
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
+ lhs_type = TREE_TYPE (scalar_dest);
+ vectype_out = get_vectype_for_scalar_type (lhs_type);
+ gcc_assert (STMT_VINFO_VECTYPE (stmt_info) == vectype_out);
+ nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
+
+ /* FORNOW: need to extend to support short<->float conversions as well. */
+ if (nunits_out != nunits_in)
+ return false;
+
+ /* Bail out if the types are both integral or non-integral */
+ if ((INTEGRAL_TYPE_P (rhs_type) && INTEGRAL_TYPE_P (lhs_type))
+ || (!INTEGRAL_TYPE_P (rhs_type) && !INTEGRAL_TYPE_P (lhs_type)))
+ return false;
+
+ /* Sanity check: make sure that at least one copy of the vectorized stmt
+ needs to be generated. */
+ ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
+ gcc_assert (ncopies >= 1);
+
+ if (!vect_is_simple_use (op0, loop_vinfo, &def_stmt, &def, &dt0))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "use not simple.");
+ return false;
+ }
+
+ /* Supportable by target? */
+ if (!targetm.vectorize.builtin_conversion (code, vectype_in))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "op not supported by target.");
+ return false;
+ }
+
+ if (!vec_stmt) /* transformation not required. */
+ {
+ STMT_VINFO_TYPE (stmt_info) = type_conversion_vec_info_type;
+ return true;
+ }
+
+ /** Transform. **/
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "transform conversion.");
+
+ /* Handle def. */
+ vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
+
+ prev_stmt_info = NULL;
+ for (j = 0; j < ncopies; j++)
+ {
+ tree sym;
+ ssa_op_iter iter;
+
+ if (j == 0)
+ vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
+ else
+ vec_oprnd0 = vect_get_vec_def_for_stmt_copy (dt0, vec_oprnd0);
+
+ builtin_decl =
+ targetm.vectorize.builtin_conversion (code, vectype_in);
+ new_stmt = build_call_expr (builtin_decl, 1, vec_oprnd0);
+
+ /* Arguments are ready. create the new vector stmt. */
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest,
+ new_stmt);
+ new_temp = make_ssa_name (vec_dest, new_stmt);
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
+ vect_finish_stmt_generation (stmt, new_stmt, bsi);
+ FOR_EACH_SSA_TREE_OPERAND (sym, new_stmt, iter, SSA_OP_ALL_VIRTUALS)
+ {
+ if (TREE_CODE (sym) == SSA_NAME)
+ sym = SSA_NAME_VAR (sym);
+ mark_sym_for_renaming (sym);
+ }
+
+ if (j == 0)
+ STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
+ else
+ STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
+ prev_stmt_info = vinfo_for_stmt (new_stmt);
+ }
+ return true;
+}
+
/* Function vectorizable_assignment.
gcc_assert (STMT_VINFO_DEF_TYPE (stmt_info) == vect_loop_def);
- if (TREE_CODE (stmt) != MODIFY_EXPR)
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
return false;
- scalar_dest = TREE_OPERAND (stmt, 0);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
if (TREE_CODE (scalar_dest) != SSA_NAME)
return false;
- op = TREE_OPERAND (stmt, 1);
+ op = GIMPLE_STMT_OPERAND (stmt, 1);
if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt))
{
if (vect_print_dump_info (REPORT_DETAILS))
vec_dest = vect_create_destination_var (scalar_dest, vectype);
/* Handle use. */
- op = TREE_OPERAND (stmt, 1);
+ op = GIMPLE_STMT_OPERAND (stmt, 1);
vec_oprnd = vect_get_vec_def_for_operand (op, stmt, NULL);
/* Arguments are ready. create the new vector stmt. */
- *vec_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, vec_oprnd);
+ *vec_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest, vec_oprnd);
new_temp = make_ssa_name (vec_dest, *vec_stmt);
- TREE_OPERAND (*vec_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (*vec_stmt, 0) = new_temp;
vect_finish_stmt_generation (stmt, *vec_stmt, bsi);
return true;
return false;
}
- if (TREE_CODE (stmt) != MODIFY_EXPR)
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
return false;
- if (TREE_CODE (TREE_OPERAND (stmt, 0)) != SSA_NAME)
+ if (TREE_CODE (GIMPLE_STMT_OPERAND (stmt, 0)) != SSA_NAME)
return false;
- scalar_dest = TREE_OPERAND (stmt, 0);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
vectype_out = get_vectype_for_scalar_type (TREE_TYPE (scalar_dest));
nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
if (nunits_out != nunits_in)
return false;
- operation = TREE_OPERAND (stmt, 1);
+ operation = GIMPLE_STMT_OPERAND (stmt, 1);
code = TREE_CODE (operation);
optab = optab_for_tree_code (code, vectype);
/* Support only unary or binary operations. */
- op_type = TREE_CODE_LENGTH (code);
+ op_type = TREE_OPERAND_LENGTH (operation);
if (op_type != unary_op && op_type != binary_op)
{
if (vect_print_dump_info (REPORT_DETAILS))
/* Arguments are ready. create the new vector stmt. */
if (op_type == binary_op)
- new_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest,
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest,
build2 (code, vectype, vec_oprnd0, vec_oprnd1));
else
- new_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest,
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest,
build1 (code, vectype, vec_oprnd0));
new_temp = make_ssa_name (vec_dest, new_stmt);
- TREE_OPERAND (new_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
vect_finish_stmt_generation (stmt, new_stmt, bsi);
if (j == 0)
return false;
}
- if (TREE_CODE (stmt) != MODIFY_EXPR)
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
return false;
- if (TREE_CODE (TREE_OPERAND (stmt, 0)) != SSA_NAME)
+ if (TREE_CODE (GIMPLE_STMT_OPERAND (stmt, 0)) != SSA_NAME)
return false;
- operation = TREE_OPERAND (stmt, 1);
+ operation = GIMPLE_STMT_OPERAND (stmt, 1);
code = TREE_CODE (operation);
if (code != NOP_EXPR && code != CONVERT_EXPR)
return false;
vectype_in = get_vectype_for_scalar_type (TREE_TYPE (op0));
nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
- scalar_dest = TREE_OPERAND (stmt, 0);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
scalar_type = TREE_TYPE (scalar_dest);
vectype_out = get_vectype_for_scalar_type (scalar_type);
nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
gcc_assert (ncopies >= 1);
+
+ if (! INTEGRAL_TYPE_P (scalar_type)
+ || !INTEGRAL_TYPE_P (TREE_TYPE (op0)))
+ return false;
/* Check the operands of the operation. */
if (!vect_is_simple_use (op0, loop_vinfo, &def_stmt, &def, &dt0))
/* Handle uses. */
if (j == 0)
{
- enum vect_def_type dt = vect_unknown_def_type; /* Dummy */
vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
- vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt, vec_oprnd0);
+ vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt0, vec_oprnd0);
}
else
{
/* Arguments are ready. Create the new vector stmt. */
expr = build2 (code, vectype_out, vec_oprnd0, vec_oprnd1);
- new_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest, expr);
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest, expr);
new_temp = make_ssa_name (vec_dest, new_stmt);
- TREE_OPERAND (new_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
vect_finish_stmt_generation (stmt, new_stmt, bsi);
if (j == 0)
tree vec_dest, block_stmt_iterator *bsi,
tree stmt)
{
- tree vec_params;
tree expr;
tree new_stmt;
tree new_temp;
if (code == CALL_EXPR)
{
/* Target specific support */
- vec_params = build_tree_list (NULL_TREE, vec_oprnd0);
- if (op_type == binary_op)
- vec_params = tree_cons (NULL_TREE, vec_oprnd1, vec_params);
- expr = build_function_call_expr (decl, vec_params);
+ if (op_type == binary_op)
+ expr = build_call_expr (decl, 2, vec_oprnd0, vec_oprnd1);
+ else
+ expr = build_call_expr (decl, 1, vec_oprnd0);
}
else
{
else
expr = build1 (code, vectype, vec_oprnd0);
}
- new_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest, expr);
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest, expr);
new_temp = make_ssa_name (vec_dest, new_stmt);
- TREE_OPERAND (new_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
vect_finish_stmt_generation (stmt, new_stmt, bsi);
if (code == CALL_EXPR)
return false;
}
- if (TREE_CODE (stmt) != MODIFY_EXPR)
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
return false;
- if (TREE_CODE (TREE_OPERAND (stmt, 0)) != SSA_NAME)
+ if (TREE_CODE (GIMPLE_STMT_OPERAND (stmt, 0)) != SSA_NAME)
return false;
- operation = TREE_OPERAND (stmt, 1);
+ operation = GIMPLE_STMT_OPERAND (stmt, 1);
code = TREE_CODE (operation);
if (code != NOP_EXPR && code != WIDEN_MULT_EXPR)
return false;
ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
gcc_assert (ncopies >= 1);
- scalar_dest = TREE_OPERAND (stmt, 0);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
vectype_out = get_vectype_for_scalar_type (TREE_TYPE (scalar_dest));
nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
if (nunits_out != nunits_in / 2) /* FORNOW */
return false;
+ if (! INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
+ || !INTEGRAL_TYPE_P (TREE_TYPE (op0)))
+ return false;
+
/* Check the operands of the operation. */
if (!vect_is_simple_use (op0, loop_vinfo, &def_stmt, &def, &dt0))
{
/* Function vect_permute_store_chain.
- Given a chain of interleaved strores in DR_CHAIN of LENGTH that must be
+ Given a chain of interleaved stores in DR_CHAIN of LENGTH that must be
a power of 2, generate interleave_high/low stmts to reorder the data
correctly for the stores. Return the final references for stores in
RESULT_CHAIN.
E.g., LENGTH is 4 and the scalar type is short, i.e., VF is 8.
- The input is 4 vectors each containg 8 elements. We assign a number to each
+ The input is 4 vectors each containing 8 elements. We assign a number to each
element, the input sequence is:
1st vec: 0 1 2 3 4 5 6 7
and of interleave_low: 2 6 3 7
- The permutaion is done in log LENGTH stages. In each stage interleave_high
+ The permutation is done in log LENGTH stages. In each stage interleave_high
and interleave_low stmts are created for each pair of vectors in DR_CHAIN,
where the first argument is taken from the first half of DR_CHAIN and the
second argument from it's second half.
unsigned int j;
VEC(tree,heap) *first, *second;
- scalar_dest = TREE_OPERAND (stmt, 0);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
first = VEC_alloc (tree, heap, length/2);
second = VEC_alloc (tree, heap, length/2);
vect1 = VEC_index (tree, dr_chain, j);
vect2 = VEC_index (tree, dr_chain, j+length/2);
- /* high = interleave_high (vect1, vect2); */
+ /* Create interleaving stmt:
+ in the case of big endian:
+ high = interleave_high (vect1, vect2)
+ and in the case of little endian:
+ high = interleave_low (vect1, vect2). */
perm_dest = create_tmp_var (vectype, "vect_inter_high");
+ DECL_GIMPLE_REG_P (perm_dest) = 1;
add_referenced_var (perm_dest);
- perm_stmt = build2 (MODIFY_EXPR, vectype, perm_dest,
- build2 (VEC_INTERLEAVE_HIGH_EXPR, vectype, vect1,
- vect2));
+ if (BYTES_BIG_ENDIAN)
+ perm_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, perm_dest,
+ build2 (VEC_INTERLEAVE_HIGH_EXPR, vectype,
+ vect1, vect2));
+ else
+ perm_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, perm_dest,
+ build2 (VEC_INTERLEAVE_LOW_EXPR, vectype,
+ vect1, vect2));
high = make_ssa_name (perm_dest, perm_stmt);
- TREE_OPERAND (perm_stmt, 0) = high;
+ GIMPLE_STMT_OPERAND (perm_stmt, 0) = high;
vect_finish_stmt_generation (stmt, perm_stmt, bsi);
VEC_replace (tree, *result_chain, 2*j, high);
- /* low = interleave_low (vect1, vect2); */
+ /* Create interleaving stmt:
+ in the case of big endian:
+ low = interleave_low (vect1, vect2)
+ and in the case of little endian:
+ low = interleave_high (vect1, vect2). */
perm_dest = create_tmp_var (vectype, "vect_inter_low");
+ DECL_GIMPLE_REG_P (perm_dest) = 1;
add_referenced_var (perm_dest);
- perm_stmt = build2 (MODIFY_EXPR, vectype, perm_dest,
- build2 (VEC_INTERLEAVE_LOW_EXPR, vectype, vect1,
- vect2));
+ if (BYTES_BIG_ENDIAN)
+ perm_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, perm_dest,
+ build2 (VEC_INTERLEAVE_LOW_EXPR, vectype,
+ vect1, vect2));
+ else
+ perm_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, perm_dest,
+ build2 (VEC_INTERLEAVE_HIGH_EXPR, vectype,
+ vect1, vect2));
low = make_ssa_name (perm_dest, perm_stmt);
- TREE_OPERAND (perm_stmt, 0) = low;
+ GIMPLE_STMT_OPERAND (perm_stmt, 0) = low;
vect_finish_stmt_generation (stmt, perm_stmt, bsi);
VEC_replace (tree, *result_chain, 2*j+1, low);
}
/* Is vectorizable store? */
- if (TREE_CODE (stmt) != MODIFY_EXPR)
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
return false;
- scalar_dest = TREE_OPERAND (stmt, 0);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
if (TREE_CODE (scalar_dest) != ARRAY_REF
&& TREE_CODE (scalar_dest) != INDIRECT_REF
&& !DR_GROUP_FIRST_DR (stmt_info))
return false;
- op = TREE_OPERAND (stmt, 1);
+ op = GIMPLE_STMT_OPERAND (stmt, 1);
if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt))
{
if (vect_print_dump_info (REPORT_DETAILS))
S3: &base + 1 = x1
S4: &base + 3 = x3
- We create vectorized storess starting from base address (the access of the
+ We create vectorized stores starting from base address (the access of the
first stmt in the chain (S2 in the above example), when the last store stmt
of the chain (S4) is reached:
And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
(the order of the data-refs in the output of vect_permute_store_chain
corresponds to the order of scalar stmts in the interleaving chain - see
- the documentaion of vect_permute_store_chain()).
+ the documentation of vect_permute_store_chain()).
In case of both multiple types and interleaving, above vector stores and
permutation stmts are created for every copy. The result vector stmts are
as an input to vect_permute_store_chain(), and OPRNDS as an input
to vect_get_vec_def_for_stmt_copy() for the next copy.
If the store is not strided, GROUP_SIZE is 1, and DR_CHAIN and
- OPRNDS are of size 1.
- */
+ OPRNDS are of size 1. */
next_stmt = first_stmt;
for (i = 0; i < group_size; i++)
{
is the exact number of stmts in the chain. Therefore, NEXT_STMT
can't be NULL_TREE. In case that there is no interleaving,
GROUP_SIZE is 1, and only one iteration of the loop will be
- executed.
- */
+ executed. */
gcc_assert (next_stmt);
- op = TREE_OPERAND (next_stmt, 1);
+ op = GIMPLE_STMT_OPERAND (next_stmt, 1);
vec_oprnd = vect_get_vec_def_for_operand (op, next_stmt, NULL);
VEC_quick_push(tree, dr_chain, vec_oprnd);
VEC_quick_push(tree, oprnds, vec_oprnd);
next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt));
}
dataref_ptr = vect_create_data_ref_ptr (first_stmt, bsi, NULL_TREE,
- &dummy, &ptr_incr, false);
+ &dummy, &ptr_incr, false,
+ TREE_TYPE (vec_oprnd));
}
else
{
and OPRNDS as an input to vect_get_vec_def_for_stmt_copy() for the
next copy.
If the store is not strided, GROUP_SIZE is 1, and DR_CHAIN and
- OPRNDS are of size 1.
- */
+ OPRNDS are of size 1. */
for (i = 0; i < group_size; i++)
{
vec_oprnd = vect_get_vec_def_for_stmt_copy (dt,
data_ref = build_fold_indirect_ref (dataref_ptr);
/* Arguments are ready. Create the new vector stmt. */
- new_stmt = build2 (MODIFY_EXPR, vectype, data_ref, vec_oprnd);
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, data_ref,
+ vec_oprnd);
vect_finish_stmt_generation (stmt, new_stmt, bsi);
- /* Set the V_MAY_DEFS for the vector pointer. If this virtual def has a
- use outside the loop and a loop peel is performed then the def may be
- renamed by the peel. Mark it for renaming so the later use will also
- be renamed. */
+ /* Set the VDEFs for the vector pointer. If this virtual def
+ has a use outside the loop and a loop peel is performed
+ then the def may be renamed by the peel. Mark it for
+ renaming so the later use will also be renamed. */
copy_virtual_operands (new_stmt, next_stmt);
if (j == 0)
{
- /* The original store is deleted so the same SSA_NAMEs can be used.
- */
- FOR_EACH_SSA_TREE_OPERAND (def, next_stmt, iter, SSA_OP_VMAYDEF)
+ /* The original store is deleted so the same SSA_NAMEs
+ can be used. */
+ FOR_EACH_SSA_TREE_OPERAND (def, next_stmt, iter, SSA_OP_VDEF)
{
SSA_NAME_DEF_STMT (def) = new_stmt;
mark_sym_for_renaming (SSA_NAME_VAR (def));
{
/* Create new names for all the definitions created by COPY and
add replacement mappings for each new name. */
- FOR_EACH_SSA_DEF_OPERAND (def_p, new_stmt, iter, SSA_OP_VMAYDEF)
+ FOR_EACH_SSA_DEF_OPERAND (def_p, new_stmt, iter, SSA_OP_VDEF)
{
create_new_def_for (DEF_FROM_PTR (def_p), new_stmt, def_p);
mark_sym_for_renaming (SSA_NAME_VAR (DEF_FROM_PTR (def_p)));
}
prev_stmt_info = vinfo_for_stmt (new_stmt);
- next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt));
+ next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt));
if (!next_stmt)
break;
/* Bump the vector pointer. */
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
edge pe = loop_preheader_edge (loop);
- tree scalar_dest = TREE_OPERAND (stmt, 0);
+ tree scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
tree vec_dest;
tree init_addr;
tree inc;
/* 1. Create msq_init = *(floor(p1)) in the loop preheader */
vec_dest = vect_create_destination_var (scalar_dest, vectype);
- ptr = vect_create_data_ref_ptr (stmt, bsi, NULL_TREE, &init_addr, &inc, true);
+ ptr = vect_create_data_ref_ptr (stmt, bsi, NULL_TREE, &init_addr, &inc, true,
+ NULL_TREE);
data_ref = build1 (ALIGN_INDIRECT_REF, vectype, ptr);
- new_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, data_ref);
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest, data_ref);
new_temp = make_ssa_name (vec_dest, new_stmt);
- TREE_OPERAND (new_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
new_bb = bsi_insert_on_edge_immediate (pe, new_stmt);
gcc_assert (!new_bb);
- msq_init = TREE_OPERAND (new_stmt, 0);
+ msq_init = GIMPLE_STMT_OPERAND (new_stmt, 0);
copy_virtual_operands (new_stmt, stmt);
update_vuses_to_preheader (new_stmt, loop);
if (targetm.vectorize.builtin_mask_for_load)
{
tree builtin_decl;
- tree params = build_tree_list (NULL_TREE, init_addr);
- vec_dest = vect_create_destination_var (scalar_dest,
- TREE_TYPE (new_stmt));
builtin_decl = targetm.vectorize.builtin_mask_for_load ();
- new_stmt = build_function_call_expr (builtin_decl, params);
- new_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest, new_stmt);
+ new_stmt = build_call_expr (builtin_decl, 1, init_addr);
+ vec_dest = vect_create_destination_var (scalar_dest,
+ TREE_TYPE (new_stmt));
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest,
+ new_stmt);
new_temp = make_ssa_name (vec_dest, new_stmt);
- TREE_OPERAND (new_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
new_bb = bsi_insert_on_edge_immediate (pe, new_stmt);
gcc_assert (!new_bb);
- *realignment_token = TREE_OPERAND (new_stmt, 0);
+ *realignment_token = GIMPLE_STMT_OPERAND (new_stmt, 0);
/* The result of the CALL_EXPR to this builtin is determined from
the value of the parameter and no global variables are touched
correctly. Return the final references for loads in RESULT_CHAIN.
E.g., LENGTH is 4 and the scalar type is short, i.e., VF is 8.
- The input is 4 vectors each containg 8 elements. We assign a number to each
+ The input is 4 vectors each containing 8 elements. We assign a number to each
element, the input sequence is:
1st vec: 0 1 2 3 4 5 6 7
and of extract_odd: 1 3 5 7
- The permutaion is done in log LENGTH stages. In each stage extract_even and
+ The permutation is done in log LENGTH stages. In each stage extract_even and
extract_odd stmts are created for each pair of vectors in DR_CHAIN in their
order. In our example,
/* data_ref = permute_even (first_data_ref, second_data_ref); */
perm_dest = create_tmp_var (vectype, "vect_perm_even");
+ DECL_GIMPLE_REG_P (perm_dest) = 1;
add_referenced_var (perm_dest);
- perm_stmt = build2 (MODIFY_EXPR, vectype, perm_dest,
+ perm_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, perm_dest,
build2 (VEC_EXTRACT_EVEN_EXPR, vectype,
first_vect, second_vect));
data_ref = make_ssa_name (perm_dest, perm_stmt);
- TREE_OPERAND (perm_stmt, 0) = data_ref;
+ GIMPLE_STMT_OPERAND (perm_stmt, 0) = data_ref;
vect_finish_stmt_generation (stmt, perm_stmt, bsi);
- mark_new_vars_to_rename (perm_stmt);
+ mark_symbols_for_renaming (perm_stmt);
VEC_replace (tree, *result_chain, j/2, data_ref);
/* data_ref = permute_odd (first_data_ref, second_data_ref); */
perm_dest = create_tmp_var (vectype, "vect_perm_odd");
+ DECL_GIMPLE_REG_P (perm_dest) = 1;
add_referenced_var (perm_dest);
- perm_stmt = build2 (MODIFY_EXPR, vectype, perm_dest,
+ perm_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, perm_dest,
build2 (VEC_EXTRACT_ODD_EXPR, vectype,
first_vect, second_vect));
data_ref = make_ssa_name (perm_dest, perm_stmt);
- TREE_OPERAND (perm_stmt, 0) = data_ref;
+ GIMPLE_STMT_OPERAND (perm_stmt, 0) = data_ref;
vect_finish_stmt_generation (stmt, perm_stmt, bsi);
- mark_new_vars_to_rename (perm_stmt);
+ mark_symbols_for_renaming (perm_stmt);
VEC_replace (tree, *result_chain, j/2+length/2, data_ref);
}
return false;
}
- if (TREE_CODE (stmt) != MODIFY_EXPR)
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
return false;
- scalar_dest = TREE_OPERAND (stmt, 0);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
if (TREE_CODE (scalar_dest) != SSA_NAME)
return false;
- op = TREE_OPERAND (stmt, 1);
+ op = GIMPLE_STMT_OPERAND (stmt, 1);
if (TREE_CODE (op) != ARRAY_REF
&& TREE_CODE (op) != INDIRECT_REF
&& !DR_GROUP_FIRST_DR (stmt_info))
And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
(the order of the data-refs in the output of vect_permute_load_chain
corresponds to the order of scalar stmts in the interleaving chain - see
- the documentaion of vect_permute_load_chain()).
+ the documentation of vect_permute_load_chain()).
The generation of permutation stmts and recording them in
STMT_VINFO_VEC_STMT is done in vect_transform_strided_load().
{
/* 1. Create the vector pointer update chain. */
if (j == 0)
- dataref_ptr = vect_create_data_ref_ptr (first_stmt, bsi, offset,
- &dummy, &ptr_incr, false);
+ dataref_ptr = vect_create_data_ref_ptr (first_stmt, bsi, offset, &dummy,
+ &ptr_incr, false, NULL_TREE);
else
dataref_ptr = bump_vector_ptr (dataref_ptr, ptr_incr, bsi, stmt);
gcc_unreachable ();
}
vec_dest = vect_create_destination_var (scalar_dest, vectype);
- new_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, data_ref);
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest,
+ data_ref);
new_temp = make_ssa_name (vec_dest, new_stmt);
- TREE_OPERAND (new_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
vect_finish_stmt_generation (stmt, new_stmt, bsi);
copy_virtual_operands (new_stmt, stmt);
- mark_new_vars_to_rename (new_stmt);
+ mark_symbols_for_renaming (new_stmt);
/* 3. Handle explicit realignment if necessary/supported. */
if (alignment_support_cheme == dr_unaligned_software_pipeline)
{
/* Create in loop:
<vec_dest = realign_load (msq, lsq, realignment_token)> */
- lsq = TREE_OPERAND (new_stmt, 0);
+ lsq = GIMPLE_STMT_OPERAND (new_stmt, 0);
if (!realignment_token)
realignment_token = dataref_ptr;
vec_dest = vect_create_destination_var (scalar_dest, vectype);
new_stmt =
build3 (REALIGN_LOAD_EXPR, vectype, msq, lsq, realignment_token);
- new_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, new_stmt);
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest,
+ new_stmt);
new_temp = make_ssa_name (vec_dest, new_stmt);
- TREE_OPERAND (new_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
vect_finish_stmt_generation (stmt, new_stmt, bsi);
if (i == group_size - 1 && j == ncopies - 1)
add_phi_arg (phi_stmt, lsq, loop_latch_edge (loop));
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
int i;
- enum tree_code code;
int op_type;
tree op;
tree def, def_stmt;
if (!STMT_VINFO_LIVE_P (stmt_info))
return false;
- if (TREE_CODE (stmt) != MODIFY_EXPR)
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
return false;
- if (TREE_CODE (TREE_OPERAND (stmt, 0)) != SSA_NAME)
+ if (TREE_CODE (GIMPLE_STMT_OPERAND (stmt, 0)) != SSA_NAME)
return false;
- operation = TREE_OPERAND (stmt, 1);
- code = TREE_CODE (operation);
-
- op_type = TREE_CODE_LENGTH (code);
+ operation = GIMPLE_STMT_OPERAND (stmt, 1);
+ op_type = TREE_OPERAND_LENGTH (operation);
/* FORNOW: support only if all uses are invariant. This means
that the scalar operations can remain in place, unvectorized.
return false;
}
- if (TREE_CODE (stmt) != MODIFY_EXPR)
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
return false;
- op = TREE_OPERAND (stmt, 1);
+ op = GIMPLE_STMT_OPERAND (stmt, 1);
if (TREE_CODE (op) != COND_EXPR)
return false;
/* Transform */
/* Handle def. */
- scalar_dest = TREE_OPERAND (stmt, 0);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
vec_dest = vect_create_destination_var (scalar_dest, vectype);
/* Handle cond expr. */
vec_cond_expr = build3 (VEC_COND_EXPR, vectype,
vec_compare, vec_then_clause, vec_else_clause);
- *vec_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, vec_cond_expr);
+ *vec_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest,
+ vec_cond_expr);
new_temp = make_ssa_name (vec_dest, *vec_stmt);
- TREE_OPERAND (*vec_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (*vec_stmt, 0) = new_temp;
vect_finish_stmt_generation (stmt, *vec_stmt, bsi);
return true;
gcc_assert (done);
break;
+ case type_conversion_vec_info_type:
+ done = vectorizable_conversion (stmt, bsi, &vec_stmt);
+ gcc_assert (done);
+ break;
+
case op_vec_info_type:
done = vectorizable_operation (stmt, bsi, &vec_stmt);
gcc_assert (done);
{
/* In case of interleaving, the whole chain is vectorized when the
last store in the chain is reached. Store stmts before the last
- one are skipped, and there vec_stmt_info shoudn't be freed
+ one are skipped, and there vec_stmt_info shouldn't be freed
meanwhile. */
*strided_store = true;
if (STMT_VINFO_VEC_STMT (stmt_info))
gcc_assert (done);
break;
+ case call_vec_info_type:
+ done = vectorizable_call (stmt, bsi, &vec_stmt);
+ break;
+
default:
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "stmt not supported.");
LOOP - the loop whose preheader will contain STMT.
It's possible to vectorize a loop even though an SSA_NAME from a VUSE
- appears to be defined in a V_MAY_DEF in another statement in a loop.
+ appears to be defined in a VDEF in another statement in a loop.
One such case is when the VUSE is at the dereference of a __restricted__
- pointer in a load and the V_MAY_DEF is at the dereference of a different
+ pointer in a load and the VDEF is at the dereference of a different
__restricted__ pointer in a store. Vectorization may result in
copy_virtual_uses being called to copy the problematic VUSE to a new
statement that is being inserted in the loop preheader. This procedure
loop->num));
ni = fold_build2 (PLUS_EXPR, TREE_TYPE (init_expr),
- fold_build2 (MULT_EXPR, TREE_TYPE (niters),
- niters, step_expr), init_expr);
+ fold_build2 (MULT_EXPR, TREE_TYPE (init_expr),
+ fold_convert (TREE_TYPE (init_expr),
+ niters),
+ step_expr),
+ init_expr);
var = create_tmp_var (TREE_TYPE (init_expr), "tmp");
add_referenced_var (var);
NITERS / VECTORIZATION_FACTOR times (this value is placed into RATIO). */
static void
-vect_do_peeling_for_loop_bound (loop_vec_info loop_vinfo, tree *ratio,
- struct loops *loops)
+vect_do_peeling_for_loop_bound (loop_vec_info loop_vinfo, tree *ratio)
{
tree ni_name, ratio_mult_vf_name;
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
edge update_e;
basic_block preheader;
int loop_num;
+ unsigned int th;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_do_peeling_for_loop_bound ===");
&ratio_mult_vf_name, ratio);
loop_num = loop->num;
- new_loop = slpeel_tree_peel_loop_to_edge (loop, loops, single_exit (loop),
- ratio_mult_vf_name, ni_name, false);
+ /* Threshold for vectorized loop. */
+ th = (PARAM_VALUE (PARAM_MIN_VECT_LOOP_BOUND)) *
+ LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ new_loop = slpeel_tree_peel_loop_to_edge (loop, single_exit (loop),
+ ratio_mult_vf_name, ni_name, false, th);
gcc_assert (new_loop);
gcc_assert (loop_num == loop->num);
#ifdef ENABLE_CHECKING
if (DR_GROUP_FIRST_DR (stmt_info))
{
- /* For interleaved access element size must be multipled by the size of
+ /* For interleaved access element size must be multiplied by the size of
the interleaved group. */
group_size = DR_GROUP_SIZE (vinfo_for_stmt (
DR_GROUP_FIRST_DR (stmt_info)));
peeling is recorded in LOOP_VINFO_UNALIGNED_DR. */
static void
-vect_do_peeling_for_alignment (loop_vec_info loop_vinfo, struct loops *loops)
+vect_do_peeling_for_alignment (loop_vec_info loop_vinfo)
{
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
tree niters_of_prolog_loop, ni_name;
/* Peel the prolog loop and iterate it niters_of_prolog_loop. */
new_loop =
- slpeel_tree_peel_loop_to_edge (loop, loops, loop_preheader_edge (loop),
- niters_of_prolog_loop, ni_name, true);
+ slpeel_tree_peel_loop_to_edge (loop, loop_preheader_edge (loop),
+ niters_of_prolog_loop, ni_name, true, 0);
gcc_assert (new_loop);
#ifdef ENABLE_CHECKING
slpeel_verify_cfg_after_peeling (new_loop, loop);
add_referenced_var (addr_tmp);
addr_tmp_name = make_ssa_name (addr_tmp, NULL_TREE);
addr_stmt = fold_convert (int_ptrsize_type, addr_base);
- addr_stmt = build2 (MODIFY_EXPR, void_type_node,
+ addr_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
addr_tmp_name, addr_stmt);
SSA_NAME_DEF_STMT (addr_tmp_name) = addr_stmt;
append_to_statement_list_force (addr_stmt, cond_expr_stmt_list);
or_tmp = create_tmp_var (int_ptrsize_type, tmp_name);
add_referenced_var (or_tmp);
new_or_tmp_name = make_ssa_name (or_tmp, NULL_TREE);
- or_stmt = build2 (MODIFY_EXPR, void_type_node, new_or_tmp_name,
+ or_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
+ new_or_tmp_name,
build2 (BIT_IOR_EXPR, int_ptrsize_type,
or_tmp_name,
addr_tmp_name));
add_referenced_var (and_tmp);
and_tmp_name = make_ssa_name (and_tmp, NULL_TREE);
- and_stmt = build2 (MODIFY_EXPR, void_type_node,
+ and_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
and_tmp_name,
build2 (BIT_AND_EXPR, int_ptrsize_type,
or_tmp_name, mask_cst));
stmts in the loop, and update the loop exit condition. */
void
-vect_transform_loop (loop_vec_info loop_vinfo,
- struct loops *loops ATTRIBUTE_UNUSED)
+vect_transform_loop (loop_vec_info loop_vinfo)
{
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
int nbbs = loop->num_nodes;
- block_stmt_iterator si;
+ block_stmt_iterator si, next_si;
int i;
tree ratio = NULL;
int vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
- bitmap_iterator bi;
- unsigned int j;
bool strided_store;
if (vect_print_dump_info (REPORT_DETAILS))
basic_block new_exit_bb;
edge new_exit_e, e;
tree orig_phi, new_phi, arg;
+ unsigned prob = 4 * REG_BR_PROB_BASE / 5;
cond_expr = vect_create_cond_for_align_checks (loop_vinfo,
&cond_expr_stmt_list);
initialize_original_copy_tables ();
- nloop = loop_version (loops, loop, cond_expr, &condition_bb, true);
+ nloop = loop_version (loop, cond_expr, &condition_bb,
+ prob, prob, REG_BR_PROB_BASE - prob, true);
free_original_copy_tables();
/** Loop versioning violates an assumption we try to maintain during
/* CHECKME: we wouldn't need this if we called update_ssa once
for all loops. */
- bitmap_zero (vect_vnames_to_rename);
+ bitmap_zero (vect_memsyms_to_rename);
/* Peel the loop if there are data refs with unknown alignment.
Only one data ref with unknown store is allowed. */
if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo))
- vect_do_peeling_for_alignment (loop_vinfo, loops);
+ vect_do_peeling_for_alignment (loop_vinfo);
/* If the loop has a symbolic number of iterations 'n' (i.e. it's not a
compile time constant), or it is a constant that doesn't divide by the
if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
|| (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
&& LOOP_VINFO_INT_NITERS (loop_vinfo) % vectorization_factor != 0))
- vect_do_peeling_for_loop_bound (loop_vinfo, &ratio, loops);
+ vect_do_peeling_for_loop_bound (loop_vinfo, &ratio);
else
ratio = build_int_cst (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)),
LOOP_VINFO_INT_NITERS (loop_vinfo) / vectorization_factor);
while (next)
{
+ next_si = bsi_for_stmt (next);
next_stmt_info = vinfo_for_stmt (next);
/* Free the attached stmt_vec_info and remove the stmt. */
ann = stmt_ann (next);
tmp = DR_GROUP_NEXT_DR (next_stmt_info);
free (next_stmt_info);
set_stmt_info (ann, NULL);
+ bsi_remove (&next_si, true);
next = tmp;
}
bsi_remove (&si, true);
continue;
}
}
- else
- {
- if (strided_store)
- {
- /* This is case of skipped interleaved store. We don't free
- its stmt_vec_info. */
- bsi_remove (&si, true);
- continue;
- }
- }
bsi_next (&si);
} /* stmts in BB */
} /* BBs in loop */
slpeel_make_loop_iterate_ntimes (loop, ratio);
- EXECUTE_IF_SET_IN_BITMAP (vect_vnames_to_rename, 0, j, bi)
- mark_sym_for_renaming (SSA_NAME_VAR (ssa_name (j)));
+ mark_set_for_renaming (vect_memsyms_to_rename);
/* The memory tags and pointers in vectorized statements need to
have their SSA forms updated. FIXME, why can't this be delayed
OSDN Git Service
