#include "cfgloop.h"
#include "expr.h"
#include "optabs.h"
+#include "params.h"
#include "recog.h"
#include "tree-data-ref.h"
#include "tree-chrec.h"
else
new_vect_var = create_tmp_var (type, prefix);
+ /* Mark vector typed variable as a gimple register variable. */
+ if (TREE_CODE (type) == VECTOR_TYPE)
+ DECL_GIMPLE_REG_P (new_vect_var) = true;
+
return new_vect_var;
}
Input:
1. STMT: a stmt that references memory. Expected to be of the form
- MODIFY_EXPR <name, data-ref> or MODIFY_EXPR <data-ref, name>.
+ GIMPLE_MODIFY_STMT <name, data-ref> or
+ GIMPLE_MODIFY_STMT <data-ref, name>.
2. BSI: block_stmt_iterator where new stmts can be added.
3. OFFSET (optional): an offset to be added to the initial address accessed
by the data-ref in STMT.
if (!MTAG_P (tag))
new_type_alias (vect_ptr, tag, DR_REF (dr));
else
- var_ann (vect_ptr)->symbol_mem_tag = tag;
+ set_symbol_mem_tag (vect_ptr, tag);
var_ann (vect_ptr)->subvars = DR_SUBVARS (dr);
/* Create: p = (vectype *) initial_base */
vec_stmt = fold_convert (vect_ptr_type, new_temp);
- vec_stmt = build2 (MODIFY_EXPR, void_type_node, vect_ptr, vec_stmt);
+ vec_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vect_ptr, vec_stmt);
vect_ptr_init = make_ssa_name (vect_ptr, vec_stmt);
- TREE_OPERAND (vec_stmt, 0) = vect_ptr_init;
+ GIMPLE_STMT_OPERAND (vec_stmt, 0) = vect_ptr_init;
new_bb = bsi_insert_on_edge_immediate (pe, vec_stmt);
gcc_assert (!new_bb);
use_operand_p use_p;
tree new_dataref_ptr;
- incr_stmt = build2 (MODIFY_EXPR, void_type_node, ptr_var,
+ incr_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, ptr_var,
build2 (PLUS_EXPR, vptr_type, dataref_ptr, update));
new_dataref_ptr = make_ssa_name (ptr_var, incr_stmt);
- TREE_OPERAND (incr_stmt, 0) = new_dataref_ptr;
+ GIMPLE_STMT_OPERAND (incr_stmt, 0) = new_dataref_ptr;
vect_finish_stmt_generation (stmt, incr_stmt, bsi);
/* Update the vector-pointer's cross-iteration increment. */
new_name = get_name (scalar_dest);
if (!new_name)
new_name = "var_";
- vec_dest = vect_get_new_vect_var (type, vect_simple_var, new_name);
+ vec_dest = vect_get_new_vect_var (type, kind, new_name);
add_referenced_var (vec_dest);
return vec_dest;
new_var = vect_get_new_vect_var (vector_type, vect_simple_var, "cst_");
add_referenced_var (new_var);
- init_stmt = build2 (MODIFY_EXPR, void_type_node, new_var, vector_var);
+ init_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, new_var, vector_var);
new_temp = make_ssa_name (new_var, init_stmt);
- TREE_OPERAND (init_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (init_stmt, 0) = new_temp;
pe = loop_preheader_edge (loop);
new_bb = bsi_insert_on_edge_immediate (pe, init_stmt);
print_generic_expr (vect_dump, init_stmt, TDF_SLIM);
}
- vec_oprnd = TREE_OPERAND (init_stmt, 0);
+ vec_oprnd = GIMPLE_STMT_OPERAND (init_stmt, 0);
return vec_oprnd;
}
+/* Function get_initial_def_for_induction
+
+ Input:
+ STMT - a stmt that performs an induction operation in the loop.
+ IV_PHI - the initial value of the induction variable
+
+ Output:
+ Return a vector variable, initialized with the first VF values of
+ the induction variable. E.g., for an iv with IV_PHI='X' and
+ evolution S, for a vector of 4 units, we want to return:
+ [X, X + S, X + 2*S, X + 3*S]. */
+
+static tree
+get_initial_def_for_induction (tree stmt, tree iv_phi)
+{
+ stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt);
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+ struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ tree scalar_type = TREE_TYPE (iv_phi);
+ tree vectype = get_vectype_for_scalar_type (scalar_type);
+ int nunits = GET_MODE_NUNITS (TYPE_MODE (vectype));
+ edge pe = loop_preheader_edge (loop);
+ basic_block new_bb;
+ block_stmt_iterator bsi;
+ tree vec, vec_init, vec_step, t;
+ tree access_fn;
+ tree new_var;
+ tree new_name;
+ tree init_stmt;
+ tree induction_phi, induc_def, new_stmt, vec_def, vec_dest;
+ tree init_expr, step_expr;
+ int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ int i;
+ bool ok;
+ int ncopies = vf / nunits;
+ tree expr;
+ stmt_vec_info phi_info = vinfo_for_stmt (iv_phi);
+
+ gcc_assert (phi_info);
+
+ if (STMT_VINFO_VEC_STMT (phi_info))
+ {
+ induction_phi = STMT_VINFO_VEC_STMT (phi_info);
+ gcc_assert (TREE_CODE (induction_phi) == PHI_NODE);
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "induction already vectorized:");
+ print_generic_expr (vect_dump, iv_phi, TDF_SLIM);
+ fprintf (vect_dump, "\n");
+ print_generic_expr (vect_dump, induction_phi, TDF_SLIM);
+ }
+
+ return PHI_RESULT (induction_phi);
+ }
+
+ gcc_assert (ncopies >= 1);
+
+ access_fn = analyze_scalar_evolution (loop, PHI_RESULT (iv_phi));
+ gcc_assert (access_fn);
+ ok = vect_is_simple_iv_evolution (loop->num, access_fn, &init_expr, &step_expr);
+ gcc_assert (ok);
+
+ /* Create the vector that holds the initial_value of the induction. */
+ new_name = init_expr;
+ t = NULL_TREE;
+ t = tree_cons (NULL_TREE, init_expr, t);
+ for (i = 1; i < nunits; i++)
+ {
+ /* Create: new_name = new_name + step_expr */
+ new_var = vect_get_new_vect_var (scalar_type, vect_scalar_var, "var_");
+ add_referenced_var (new_var);
+ init_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, new_var,
+ fold_build2 (PLUS_EXPR, scalar_type, new_name, step_expr));
+ new_name = make_ssa_name (new_var, init_stmt);
+ GIMPLE_STMT_OPERAND (init_stmt, 0) = new_name;
+
+ new_bb = bsi_insert_on_edge_immediate (pe, init_stmt);
+ gcc_assert (!new_bb);
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "created new init_stmt: ");
+ print_generic_expr (vect_dump, init_stmt, TDF_SLIM);
+ }
+ t = tree_cons (NULL_TREE, new_name, t);
+ }
+ vec = build_constructor_from_list (vectype, nreverse (t));
+ vec_init = vect_init_vector (stmt, vec, vectype);
+
+
+ /* Create the vector that holds the step of the induction. */
+ expr = build_int_cst (scalar_type, vf);
+ new_name = fold_build2 (MULT_EXPR, scalar_type, expr, step_expr);
+ t = NULL_TREE;
+ for (i = 0; i < nunits; i++)
+ t = tree_cons (NULL_TREE, unshare_expr (new_name), t);
+ vec = build_constructor_from_list (vectype, t);
+ vec_step = vect_init_vector (stmt, vec, vectype);
+
+
+ /* Create the following def-use cycle:
+ loop prolog:
+ vec_init = [X, X+S, X+2*S, X+3*S]
+ vec_step = [VF*S, VF*S, VF*S, VF*S]
+ loop:
+ vec_iv = PHI <vec_init, vec_loop>
+ ...
+ STMT
+ ...
+ vec_loop = vec_iv + vec_step; */
+
+ /* Create the induction-phi that defines the induction-operand. */
+ vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_");
+ add_referenced_var (vec_dest);
+ induction_phi = create_phi_node (vec_dest, loop->header);
+ set_stmt_info (get_stmt_ann (induction_phi),
+ new_stmt_vec_info (induction_phi, loop_vinfo));
+ induc_def = PHI_RESULT (induction_phi);
+
+ /* Create the iv update inside the loop */
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, NULL_TREE,
+ build2 (PLUS_EXPR, vectype, induc_def, vec_step));
+ vec_def = make_ssa_name (vec_dest, new_stmt);
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = vec_def;
+ bsi = bsi_for_stmt (stmt);
+ vect_finish_stmt_generation (stmt, new_stmt, &bsi);
+
+ /* Set the arguments of the phi node: */
+ add_phi_arg (induction_phi, vec_init, loop_preheader_edge (loop));
+ add_phi_arg (induction_phi, vec_def, loop_latch_edge (loop));
+
+
+ /* In case the vectorization factor (VF) is bigger than the number
+ of elements that we can fit in a vectype (nunits), we have to generate
+ more than one vector stmt - i.e - we need to "unroll" the
+ vector stmt by a factor VF/nunits. For more details see documentation
+ in vectorizable_operation. */
+
+ if (ncopies > 1)
+ {
+ stmt_vec_info prev_stmt_vinfo;
+
+ /* Create the vector that holds the step of the induction. */
+ expr = build_int_cst (scalar_type, nunits);
+ new_name = fold_build2 (MULT_EXPR, scalar_type, expr, step_expr);
+ t = NULL_TREE;
+ for (i = 0; i < nunits; i++)
+ t = tree_cons (NULL_TREE, unshare_expr (new_name), t);
+ vec = build_constructor_from_list (vectype, t);
+ vec_step = vect_init_vector (stmt, vec, vectype);
+
+ vec_def = induc_def;
+ prev_stmt_vinfo = vinfo_for_stmt (induction_phi);
+ for (i = 1; i < ncopies; i++)
+ {
+ /* vec_i = vec_prev + vec_{step*nunits} */
+
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, NULL_TREE,
+ build2 (PLUS_EXPR, vectype, vec_def, vec_step));
+ vec_def = make_ssa_name (vec_dest, new_stmt);
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = vec_def;
+ bsi = bsi_for_stmt (stmt);
+ vect_finish_stmt_generation (stmt, new_stmt, &bsi);
+
+ STMT_VINFO_RELATED_STMT (prev_stmt_vinfo) = new_stmt;
+ prev_stmt_vinfo = vinfo_for_stmt (new_stmt);
+ }
+ }
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "transform induction: created def-use cycle:");
+ print_generic_expr (vect_dump, induction_phi, TDF_SLIM);
+ fprintf (vect_dump, "\n");
+ print_generic_expr (vect_dump, SSA_NAME_DEF_STMT (vec_def), TDF_SLIM);
+ }
+
+ STMT_VINFO_VEC_STMT (phi_info) = induction_phi;
+ return induc_def;
+}
+
+
/* Function vect_get_vec_def_for_operand.
OP is an operand in STMT. This function returns a (vector) def that will be
def_stmt_info = vinfo_for_stmt (def_stmt);
vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info);
gcc_assert (vec_stmt);
- vec_oprnd = TREE_OPERAND (vec_stmt, 0);
+ vec_oprnd = GIMPLE_STMT_OPERAND (vec_stmt, 0);
return vec_oprnd;
}
/* Case 5: operand is defined by loop-header phi - induction. */
case vect_induction_def:
{
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "induction - unsupported.");
- internal_error ("no support for induction"); /* FORNOW */
+ gcc_assert (TREE_CODE (def_stmt) == PHI_NODE);
+
+ /* Get the def before the loop */
+ return get_initial_def_for_induction (stmt, def_stmt);
}
default:
vector stmt (each computing a vector of 'nunits' results, and together
computing 'VF' results in each iteration). This function is called when
vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in
- which VF=16 and nuniti=4, so the number of copies required is 4):
+ which VF=16 and nunits=4, so the number of copies required is 4):
scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT
tree vec_stmt_for_operand;
stmt_vec_info def_stmt_info;
- if (dt == vect_invariant_def || dt == vect_constant_def)
- {
- /* Do nothing; can reuse same def. */ ;
- return vec_oprnd;
- }
+ /* Do nothing; can reuse same def. */
+ if (dt == vect_invariant_def || dt == vect_constant_def )
+ return vec_oprnd;
vec_stmt_for_operand = SSA_NAME_DEF_STMT (vec_oprnd);
def_stmt_info = vinfo_for_stmt (vec_stmt_for_operand);
gcc_assert (def_stmt_info);
vec_stmt_for_operand = STMT_VINFO_RELATED_STMT (def_stmt_info);
gcc_assert (vec_stmt_for_operand);
- vec_oprnd = TREE_OPERAND (vec_stmt_for_operand, 0);
+ vec_oprnd = GIMPLE_STMT_OPERAND (vec_stmt_for_operand, 0);
return vec_oprnd;
}
tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
int nunits = GET_MODE_NUNITS (TYPE_MODE (vectype));
int nelements;
- enum tree_code code = TREE_CODE (TREE_OPERAND (stmt, 1));
+ enum tree_code code = TREE_CODE (GIMPLE_STMT_OPERAND (stmt, 1));
tree type = TREE_TYPE (init_val);
tree def;
tree vec, t = NULL_TREE;
REDUCTION_PHI is the phi-node that carries the reduction computation.
This function:
- 1. Creates the reduction def-use cycle: sets the the arguments for
+ 1. Creates the reduction def-use cycle: sets the arguments for
REDUCTION_PHI:
The loop-entry argument is the vectorized initial-value of the reduction.
The loop-latch argument is VECT_DEF - the vector of partial sums.
tree epilog_stmt;
tree new_scalar_dest, exit_phi;
tree bitsize, bitpos, bytesize;
- enum tree_code code = TREE_CODE (TREE_OPERAND (stmt, 1));
+ enum tree_code code = TREE_CODE (GIMPLE_STMT_OPERAND (stmt, 1));
tree scalar_initial_def;
tree vec_initial_def;
tree orig_name;
tree reduction_op;
tree orig_stmt;
tree use_stmt;
- tree operation = TREE_OPERAND (stmt, 1);
+ tree operation = GIMPLE_STMT_OPERAND (stmt, 1);
int op_type;
- op_type = TREE_CODE_LENGTH (TREE_CODE (operation));
+ op_type = TREE_OPERAND_LENGTH (operation);
reduction_op = TREE_OPERAND (operation, op_type-1);
vectype = get_vectype_for_scalar_type (TREE_TYPE (reduction_op));
mode = TYPE_MODE (vectype);
gcc_assert (STMT_VINFO_IN_PATTERN_P (stmt_vinfo));
gcc_assert (STMT_VINFO_RELATED_STMT (stmt_vinfo) == stmt);
}
- code = TREE_CODE (TREE_OPERAND (orig_stmt, 1));
- scalar_dest = TREE_OPERAND (orig_stmt, 0);
+ code = TREE_CODE (GIMPLE_STMT_OPERAND (orig_stmt, 1));
+ scalar_dest = GIMPLE_STMT_OPERAND (orig_stmt, 0);
scalar_type = TREE_TYPE (scalar_dest);
new_scalar_dest = vect_create_destination_var (scalar_dest, NULL);
bitsize = TYPE_SIZE (scalar_type);
fprintf (vect_dump, "Reduce using direct vector reduction.");
vec_dest = vect_create_destination_var (scalar_dest, vectype);
- epilog_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest,
+ epilog_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest,
build1 (reduc_code, vectype, PHI_RESULT (new_phi)));
new_temp = make_ssa_name (vec_dest, epilog_stmt);
- TREE_OPERAND (epilog_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_temp;
bsi_insert_after (&exit_bsi, epilog_stmt, BSI_NEW_STMT);
extract_scalar_result = true;
{
tree bitpos = size_int (bit_offset);
- epilog_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest,
+ epilog_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
+ vec_dest,
build2 (shift_code, vectype,
new_temp, bitpos));
new_name = make_ssa_name (vec_dest, epilog_stmt);
- TREE_OPERAND (epilog_stmt, 0) = new_name;
+ GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_name;
bsi_insert_after (&exit_bsi, epilog_stmt, BSI_NEW_STMT);
- epilog_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest,
+ epilog_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
+ vec_dest,
build2 (code, vectype,
new_name, new_temp));
new_temp = make_ssa_name (vec_dest, epilog_stmt);
- TREE_OPERAND (epilog_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_temp;
bsi_insert_after (&exit_bsi, epilog_stmt, BSI_NEW_STMT);
}
rhs = build3 (BIT_FIELD_REF, scalar_type, vec_temp, bitsize,
bitsize_zero_node);
BIT_FIELD_REF_UNSIGNED (rhs) = TYPE_UNSIGNED (scalar_type);
- epilog_stmt = build2 (MODIFY_EXPR, void_type_node, new_scalar_dest, rhs);
+ epilog_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
+ new_scalar_dest, rhs);
new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
- TREE_OPERAND (epilog_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_temp;
bsi_insert_after (&exit_bsi, epilog_stmt, BSI_NEW_STMT);
for (bit_offset = element_bitsize;
bitpos);
BIT_FIELD_REF_UNSIGNED (rhs) = TYPE_UNSIGNED (scalar_type);
- epilog_stmt = build2 (MODIFY_EXPR, void_type_node, new_scalar_dest,
- rhs);
+ epilog_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
+ new_scalar_dest, rhs);
new_name = make_ssa_name (new_scalar_dest, epilog_stmt);
- TREE_OPERAND (epilog_stmt, 0) = new_name;
+ GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_name;
bsi_insert_after (&exit_bsi, epilog_stmt, BSI_NEW_STMT);
- epilog_stmt = build2 (MODIFY_EXPR, void_type_node, new_scalar_dest,
+ epilog_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
+ new_scalar_dest,
build2 (code, scalar_type, new_name, new_temp));
new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
- TREE_OPERAND (epilog_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_temp;
bsi_insert_after (&exit_bsi, epilog_stmt, BSI_NEW_STMT);
}
rhs = build3 (BIT_FIELD_REF, scalar_type, new_temp, bitsize, bitpos);
BIT_FIELD_REF_UNSIGNED (rhs) = TYPE_UNSIGNED (scalar_type);
- epilog_stmt = build2 (MODIFY_EXPR, void_type_node, new_scalar_dest, rhs);
+ epilog_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
+ new_scalar_dest, rhs);
new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
- TREE_OPERAND (epilog_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_temp;
bsi_insert_after (&exit_bsi, epilog_stmt, BSI_NEW_STMT);
}
if (scalar_initial_def)
{
- epilog_stmt = build2 (MODIFY_EXPR, void_type_node, new_scalar_dest,
+ epilog_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
+ new_scalar_dest,
build2 (code, scalar_type, new_temp, scalar_initial_def));
new_temp = make_ssa_name (new_scalar_dest, epilog_stmt);
- TREE_OPERAND (epilog_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (epilog_stmt, 0) = new_temp;
bsi_insert_after (&exit_bsi, epilog_stmt, BSI_NEW_STMT);
}
inside the loop body. The last operand is the reduction variable,
which is defined by the loop-header-phi. */
- gcc_assert (TREE_CODE (stmt) == MODIFY_EXPR);
+ gcc_assert (TREE_CODE (stmt) == GIMPLE_MODIFY_STMT);
- operation = TREE_OPERAND (stmt, 1);
+ operation = GIMPLE_STMT_OPERAND (stmt, 1);
code = TREE_CODE (operation);
- op_type = TREE_CODE_LENGTH (code);
+ op_type = TREE_OPERAND_LENGTH (operation);
if (op_type != binary_op && op_type != ternary_op)
return false;
- scalar_dest = TREE_OPERAND (stmt, 0);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
scalar_type = TREE_TYPE (scalar_dest);
/* All uses but the last are expected to be defined in the loop.
op = TREE_OPERAND (operation, i);
is_simple_use = vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt);
gcc_assert (is_simple_use);
- gcc_assert (dt == vect_loop_def || dt == vect_invariant_def ||
- dt == vect_constant_def);
+ if (dt != vect_loop_def
+ && dt != vect_invariant_def
+ && dt != vect_constant_def
+ && dt != vect_induction_def)
+ return false;
}
op = TREE_OPERAND (operation, i);
{
/* This is a reduction pattern: get the vectype from the type of the
reduction variable, and get the tree-code from orig_stmt. */
- orig_code = TREE_CODE (TREE_OPERAND (orig_stmt, 1));
+ orig_code = TREE_CODE (GIMPLE_STMT_OPERAND (orig_stmt, 1));
vectype = get_vectype_for_scalar_type (TREE_TYPE (def));
vec_mode = TYPE_MODE (vectype);
}
/* Get the vector def for the reduction variable from the vectorized
reduction operation generated in the previous iteration (j-1) */
- reduc_def = TREE_OPERAND (new_stmt ,0);
+ reduc_def = GIMPLE_STMT_OPERAND (new_stmt ,0);
}
/* Arguments are ready. create the new vector stmt. */
else
expr = build3 (code, vectype, loop_vec_def0, loop_vec_def1,
reduc_def);
- new_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest, expr);
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest, expr);
new_temp = make_ssa_name (vec_dest, new_stmt);
- TREE_OPERAND (new_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
vect_finish_stmt_generation (stmt, new_stmt, bsi);
if (j == 0)
}
/* Checks if CALL can be vectorized in type VECTYPE. Returns
- true if the target has a vectorized version of the function,
- or false if the function cannot be vectorized. */
+ a function declaration if the target has a vectorized version
+ of the function, or NULL_TREE if the function cannot be vectorized. */
-bool
-vectorizable_function (tree call, tree vectype)
+tree
+vectorizable_function (tree call, tree vectype_out, tree vectype_in)
{
tree fndecl = get_callee_fndecl (call);
+ enum built_in_function code;
/* We only handle functions that do not read or clobber memory -- i.e.
const or novops ones. */
if (!(call_expr_flags (call) & (ECF_CONST | ECF_NOVOPS)))
- return false;
+ return NULL_TREE;
if (!fndecl
|| TREE_CODE (fndecl) != FUNCTION_DECL
|| !DECL_BUILT_IN (fndecl))
- return false;
+ return NULL_TREE;
- if (targetm.vectorize.builtin_vectorized_function (DECL_FUNCTION_CODE (fndecl), vectype))
- return true;
-
- return false;
-}
-
-/* Returns an expression that performs a call to vectorized version
- of FNDECL in type VECTYPE, with the arguments given by ARGS.
- If extra statements need to be generated, they are inserted
- before BSI. */
-
-static tree
-build_vectorized_function_call (tree fndecl,
- tree vectype, tree args)
-{
- tree vfndecl;
- enum built_in_function code = DECL_FUNCTION_CODE (fndecl);
-
- /* The target specific builtin should be available. */
- vfndecl = targetm.vectorize.builtin_vectorized_function (code, vectype);
- gcc_assert (vfndecl != NULL_TREE);
-
- return build_function_call_expr (vfndecl, args);
+ code = DECL_FUNCTION_CODE (fndecl);
+ return targetm.vectorize.builtin_vectorized_function (code, vectype_out,
+ vectype_in);
}
/* Function vectorizable_call.
tree vec_dest;
tree scalar_dest;
tree operation;
- tree op, args, type;
- tree vec_oprnd, vargs, *pvargs_end;
- stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
- tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+ tree op, type;
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt), prev_stmt_info;
+ tree vectype_out, vectype_in;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
- tree fndecl, rhs, new_temp, def, def_stmt;
- enum vect_def_type dt;
+ tree fndecl, rhs, new_temp, def, def_stmt, rhs_type, lhs_type;
+ enum vect_def_type dt[2];
+ int ncopies, j, nargs;
+ call_expr_arg_iterator iter;
/* Is STMT a vectorizable call? */
- if (TREE_CODE (stmt) != MODIFY_EXPR)
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
return false;
- if (TREE_CODE (TREE_OPERAND (stmt, 0)) != SSA_NAME)
+ if (TREE_CODE (GIMPLE_STMT_OPERAND (stmt, 0)) != SSA_NAME)
return false;
- operation = TREE_OPERAND (stmt, 1);
+ operation = GIMPLE_STMT_OPERAND (stmt, 1);
if (TREE_CODE (operation) != CALL_EXPR)
return false;
-
+
+ /* Process function arguments. */
+ rhs_type = NULL_TREE;
+ nargs = 0;
+ FOR_EACH_CALL_EXPR_ARG (op, iter, operation)
+ {
+ ++nargs;
+
+ /* Bail out if the function has more than two arguments, we
+ do not have interesting builtin functions to vectorize with
+ more than two arguments. */
+ if (nargs > 2)
+ return false;
+
+ /* We can only handle calls with arguments of the same type. */
+ if (rhs_type
+ && rhs_type != TREE_TYPE (op))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "argument types differ.");
+ return false;
+ }
+ rhs_type = TREE_TYPE (op);
+
+ if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt[nargs]))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "use not simple.");
+ return false;
+ }
+ }
+
+ /* No arguments is also not good. */
+ if (nargs == 0)
+ return false;
+
+ vectype_in = get_vectype_for_scalar_type (rhs_type);
+
+ lhs_type = TREE_TYPE (GIMPLE_STMT_OPERAND (stmt, 0));
+ vectype_out = get_vectype_for_scalar_type (lhs_type);
+
+ /* Only handle the case of vectors with the same number of elements.
+ FIXME: We need a way to handle for example the SSE2 cvtpd2dq
+ instruction which converts V2DFmode to V4SImode but only
+ using the lower half of the V4SImode result. */
+ if (TYPE_VECTOR_SUBPARTS (vectype_in) != TYPE_VECTOR_SUBPARTS (vectype_out))
+ return false;
+
/* For now, we only vectorize functions if a target specific builtin
is available. TODO -- in some cases, it might be profitable to
insert the calls for pieces of the vector, in order to be able
to vectorize other operations in the loop. */
- if (!vectorizable_function (operation, vectype))
+ fndecl = vectorizable_function (operation, vectype_out, vectype_in);
+ if (fndecl == NULL_TREE)
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "function is not vectorizable.");
return false;
}
- gcc_assert (!stmt_references_memory_p (stmt));
- for (args = TREE_OPERAND (operation, 1); args; args = TREE_CHAIN (args))
- {
- op = TREE_VALUE (args);
-
- if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt))
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "use not simple.");
- return false;
- }
- }
+ gcc_assert (ZERO_SSA_OPERANDS (stmt, SSA_OP_ALL_VIRTUALS));
if (!vec_stmt) /* transformation not required. */
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "transform operation.");
+ ncopies = (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
+ / TYPE_VECTOR_SUBPARTS (vectype_out));
+ gcc_assert (ncopies >= 1);
+
/* Handle def. */
- scalar_dest = TREE_OPERAND (stmt, 0);
- vec_dest = vect_create_destination_var (scalar_dest, vectype);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
+ vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
- /* Handle uses. */
- vargs = NULL_TREE;
- pvargs_end = &vargs;
- for (args = TREE_OPERAND (operation, 1); args; args = TREE_CHAIN (args))
+ prev_stmt_info = NULL;
+ for (j = 0; j < ncopies; ++j)
{
- op = TREE_VALUE (args);
- vec_oprnd = vect_get_vec_def_for_operand (op, stmt, NULL);
-
- *pvargs_end = tree_cons (NULL_TREE, vec_oprnd, NULL_TREE);
- pvargs_end = &TREE_CHAIN (*pvargs_end);
- }
+ tree new_stmt, vargs;
+ tree vec_oprnd[2];
+ int n;
+
+ /* Build argument list for the vectorized call. */
+ /* FIXME: Rewrite this so that it doesn't construct a temporary
+ list. */
+ vargs = NULL_TREE;
+ n = -1;
+ FOR_EACH_CALL_EXPR_ARG (op, iter, operation)
+ {
+ ++n;
- fndecl = get_callee_fndecl (operation);
- rhs = build_vectorized_function_call (fndecl, vectype, vargs);
- *vec_stmt = build2 (MODIFY_EXPR, vectype, vec_dest, rhs);
- new_temp = make_ssa_name (vec_dest, *vec_stmt);
- TREE_OPERAND (*vec_stmt, 0) = new_temp;
+ if (j == 0)
+ vec_oprnd[n] = vect_get_vec_def_for_operand (op, stmt, NULL);
+ else
+ vec_oprnd[n] = vect_get_vec_def_for_stmt_copy (dt[n], vec_oprnd[n]);
- vect_finish_stmt_generation (stmt, *vec_stmt, bsi);
+ vargs = tree_cons (NULL_TREE, vec_oprnd[n], vargs);
+ }
+ vargs = nreverse (vargs);
+
+ rhs = build_function_call_expr (fndecl, vargs);
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, NULL_TREE, vec_dest, rhs);
+ new_temp = make_ssa_name (vec_dest, new_stmt);
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
+
+ vect_finish_stmt_generation (stmt, new_stmt, bsi);
+
+ if (j == 0)
+ STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
+ else
+ STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
+ prev_stmt_info = vinfo_for_stmt (new_stmt);
+ }
/* The call in STMT might prevent it from being removed in dce. We however
cannot remove it here, due to the way the ssa name it defines is mapped
to the new definition. So just replace rhs of the statement with something
harmless. */
type = TREE_TYPE (scalar_dest);
- TREE_OPERAND (stmt, 1) = fold_convert (type, integer_zero_node);
+ GIMPLE_STMT_OPERAND (stmt, 1) = fold_convert (type, integer_zero_node);
+
+ return true;
+}
+
+
+/* Function vectorizable_conversion.
+
+Check if STMT performs a conversion operation, that can be vectorized.
+If VEC_STMT is also passed, vectorize the STMT: create a vectorized
+stmt to replace it, put it in VEC_STMT, and insert it at BSI.
+Return FALSE if not a vectorizable STMT, TRUE otherwise. */
+
+bool
+vectorizable_conversion (tree stmt, block_stmt_iterator * bsi,
+ tree * vec_stmt)
+{
+ tree vec_dest;
+ tree scalar_dest;
+ tree operation;
+ tree op0;
+ tree vec_oprnd0 = NULL_TREE;
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ enum tree_code code;
+ tree new_temp;
+ tree def, def_stmt;
+ enum vect_def_type dt0;
+ tree new_stmt;
+ int nunits_in;
+ int nunits_out;
+ int ncopies, j;
+ tree vectype_out, vectype_in;
+ tree rhs_type, lhs_type;
+ tree builtin_decl;
+ stmt_vec_info prev_stmt_info;
+
+ /* Is STMT a vectorizable conversion? */
+
+ if (!STMT_VINFO_RELEVANT_P (stmt_info))
+ return false;
+
+ gcc_assert (STMT_VINFO_DEF_TYPE (stmt_info) == vect_loop_def);
+
+ if (STMT_VINFO_LIVE_P (stmt_info))
+ {
+ /* FORNOW: not yet supported. */
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "value used after loop.");
+ return false;
+ }
+
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
+ return false;
+
+ if (TREE_CODE (GIMPLE_STMT_OPERAND (stmt, 0)) != SSA_NAME)
+ return false;
+
+ operation = GIMPLE_STMT_OPERAND (stmt, 1);
+ code = TREE_CODE (operation);
+ if (code != FIX_TRUNC_EXPR && code != FLOAT_EXPR)
+ return false;
+
+ /* Check types of lhs and rhs */
+ op0 = TREE_OPERAND (operation, 0);
+ rhs_type = TREE_TYPE (op0);
+ vectype_in = get_vectype_for_scalar_type (rhs_type);
+ nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
+
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
+ lhs_type = TREE_TYPE (scalar_dest);
+ vectype_out = get_vectype_for_scalar_type (lhs_type);
+ gcc_assert (STMT_VINFO_VECTYPE (stmt_info) == vectype_out);
+ nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
+
+ /* FORNOW: need to extend to support short<->float conversions as well. */
+ if (nunits_out != nunits_in)
+ return false;
+
+ /* Bail out if the types are both integral or non-integral */
+ if ((INTEGRAL_TYPE_P (rhs_type) && INTEGRAL_TYPE_P (lhs_type))
+ || (!INTEGRAL_TYPE_P (rhs_type) && !INTEGRAL_TYPE_P (lhs_type)))
+ return false;
+
+ /* Sanity check: make sure that at least one copy of the vectorized stmt
+ needs to be generated. */
+ ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
+ gcc_assert (ncopies >= 1);
+
+ if (!vect_is_simple_use (op0, loop_vinfo, &def_stmt, &def, &dt0))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "use not simple.");
+ return false;
+ }
+ /* Supportable by target? */
+ if (!targetm.vectorize.builtin_conversion (code, vectype_in))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "op not supported by target.");
+ return false;
+ }
+
+ if (!vec_stmt) /* transformation not required. */
+ {
+ STMT_VINFO_TYPE (stmt_info) = type_conversion_vec_info_type;
+ return true;
+ }
+
+ /** Transform. **/
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "transform conversion.");
+
+ /* Handle def. */
+ vec_dest = vect_create_destination_var (scalar_dest, vectype_out);
+
+ prev_stmt_info = NULL;
+ for (j = 0; j < ncopies; j++)
+ {
+ tree sym;
+ ssa_op_iter iter;
+
+ if (j == 0)
+ vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
+ else
+ vec_oprnd0 = vect_get_vec_def_for_stmt_copy (dt0, vec_oprnd0);
+
+ builtin_decl =
+ targetm.vectorize.builtin_conversion (code, vectype_in);
+ new_stmt = build_call_expr (builtin_decl, 1, vec_oprnd0);
+
+ /* Arguments are ready. create the new vector stmt. */
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest,
+ new_stmt);
+ new_temp = make_ssa_name (vec_dest, new_stmt);
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
+ vect_finish_stmt_generation (stmt, new_stmt, bsi);
+ FOR_EACH_SSA_TREE_OPERAND (sym, new_stmt, iter, SSA_OP_ALL_VIRTUALS)
+ {
+ if (TREE_CODE (sym) == SSA_NAME)
+ sym = SSA_NAME_VAR (sym);
+ mark_sym_for_renaming (sym);
+ }
+
+ if (j == 0)
+ STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_stmt;
+ else
+ STMT_VINFO_RELATED_STMT (prev_stmt_info) = new_stmt;
+ prev_stmt_info = vinfo_for_stmt (new_stmt);
+ }
return true;
}
gcc_assert (STMT_VINFO_DEF_TYPE (stmt_info) == vect_loop_def);
- if (TREE_CODE (stmt) != MODIFY_EXPR)
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
return false;
- scalar_dest = TREE_OPERAND (stmt, 0);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
if (TREE_CODE (scalar_dest) != SSA_NAME)
return false;
- op = TREE_OPERAND (stmt, 1);
+ op = GIMPLE_STMT_OPERAND (stmt, 1);
if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt))
{
if (vect_print_dump_info (REPORT_DETAILS))
vec_dest = vect_create_destination_var (scalar_dest, vectype);
/* Handle use. */
- op = TREE_OPERAND (stmt, 1);
+ op = GIMPLE_STMT_OPERAND (stmt, 1);
vec_oprnd = vect_get_vec_def_for_operand (op, stmt, NULL);
/* Arguments are ready. create the new vector stmt. */
- *vec_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest, vec_oprnd);
+ *vec_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest, vec_oprnd);
new_temp = make_ssa_name (vec_dest, *vec_stmt);
- TREE_OPERAND (*vec_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (*vec_stmt, 0) = new_temp;
vect_finish_stmt_generation (stmt, *vec_stmt, bsi);
return true;
return false;
}
- if (TREE_CODE (stmt) != MODIFY_EXPR)
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
return false;
- if (TREE_CODE (TREE_OPERAND (stmt, 0)) != SSA_NAME)
+ if (TREE_CODE (GIMPLE_STMT_OPERAND (stmt, 0)) != SSA_NAME)
return false;
- scalar_dest = TREE_OPERAND (stmt, 0);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
vectype_out = get_vectype_for_scalar_type (TREE_TYPE (scalar_dest));
nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
if (nunits_out != nunits_in)
return false;
- operation = TREE_OPERAND (stmt, 1);
+ operation = GIMPLE_STMT_OPERAND (stmt, 1);
code = TREE_CODE (operation);
optab = optab_for_tree_code (code, vectype);
/* Support only unary or binary operations. */
- op_type = TREE_CODE_LENGTH (code);
+ op_type = TREE_OPERAND_LENGTH (operation);
if (op_type != unary_op && op_type != binary_op)
{
if (vect_print_dump_info (REPORT_DETAILS))
/* Arguments are ready. create the new vector stmt. */
if (op_type == binary_op)
- new_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest,
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest,
build2 (code, vectype, vec_oprnd0, vec_oprnd1));
else
- new_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest,
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest,
build1 (code, vectype, vec_oprnd0));
new_temp = make_ssa_name (vec_dest, new_stmt);
- TREE_OPERAND (new_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
vect_finish_stmt_generation (stmt, new_stmt, bsi);
if (j == 0)
return false;
}
- if (TREE_CODE (stmt) != MODIFY_EXPR)
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
return false;
- if (TREE_CODE (TREE_OPERAND (stmt, 0)) != SSA_NAME)
+ if (TREE_CODE (GIMPLE_STMT_OPERAND (stmt, 0)) != SSA_NAME)
return false;
- operation = TREE_OPERAND (stmt, 1);
+ operation = GIMPLE_STMT_OPERAND (stmt, 1);
code = TREE_CODE (operation);
if (code != NOP_EXPR && code != CONVERT_EXPR)
return false;
vectype_in = get_vectype_for_scalar_type (TREE_TYPE (op0));
nunits_in = TYPE_VECTOR_SUBPARTS (vectype_in);
- scalar_dest = TREE_OPERAND (stmt, 0);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
scalar_type = TREE_TYPE (scalar_dest);
vectype_out = get_vectype_for_scalar_type (scalar_type);
nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_out;
gcc_assert (ncopies >= 1);
+
+ if (! INTEGRAL_TYPE_P (scalar_type)
+ || !INTEGRAL_TYPE_P (TREE_TYPE (op0)))
+ return false;
/* Check the operands of the operation. */
if (!vect_is_simple_use (op0, loop_vinfo, &def_stmt, &def, &dt0))
/* Handle uses. */
if (j == 0)
{
- enum vect_def_type dt = vect_unknown_def_type; /* Dummy */
vec_oprnd0 = vect_get_vec_def_for_operand (op0, stmt, NULL);
- vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt, vec_oprnd0);
+ vec_oprnd1 = vect_get_vec_def_for_stmt_copy (dt0, vec_oprnd0);
}
else
{
/* Arguments are ready. Create the new vector stmt. */
expr = build2 (code, vectype_out, vec_oprnd0, vec_oprnd1);
- new_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest, expr);
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest, expr);
new_temp = make_ssa_name (vec_dest, new_stmt);
- TREE_OPERAND (new_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
vect_finish_stmt_generation (stmt, new_stmt, bsi);
if (j == 0)
tree vec_dest, block_stmt_iterator *bsi,
tree stmt)
{
- tree vec_params;
tree expr;
tree new_stmt;
tree new_temp;
if (code == CALL_EXPR)
{
/* Target specific support */
- vec_params = build_tree_list (NULL_TREE, vec_oprnd0);
- if (op_type == binary_op)
- vec_params = tree_cons (NULL_TREE, vec_oprnd1, vec_params);
- expr = build_function_call_expr (decl, vec_params);
+ if (op_type == binary_op)
+ expr = build_call_expr (decl, 2, vec_oprnd0, vec_oprnd1);
+ else
+ expr = build_call_expr (decl, 1, vec_oprnd0);
}
else
{
else
expr = build1 (code, vectype, vec_oprnd0);
}
- new_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest, expr);
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest, expr);
new_temp = make_ssa_name (vec_dest, new_stmt);
- TREE_OPERAND (new_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
vect_finish_stmt_generation (stmt, new_stmt, bsi);
if (code == CALL_EXPR)
return false;
}
- if (TREE_CODE (stmt) != MODIFY_EXPR)
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
return false;
- if (TREE_CODE (TREE_OPERAND (stmt, 0)) != SSA_NAME)
+ if (TREE_CODE (GIMPLE_STMT_OPERAND (stmt, 0)) != SSA_NAME)
return false;
- operation = TREE_OPERAND (stmt, 1);
+ operation = GIMPLE_STMT_OPERAND (stmt, 1);
code = TREE_CODE (operation);
if (code != NOP_EXPR && code != WIDEN_MULT_EXPR)
return false;
ncopies = LOOP_VINFO_VECT_FACTOR (loop_vinfo) / nunits_in;
gcc_assert (ncopies >= 1);
- scalar_dest = TREE_OPERAND (stmt, 0);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
vectype_out = get_vectype_for_scalar_type (TREE_TYPE (scalar_dest));
nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
if (nunits_out != nunits_in / 2) /* FORNOW */
return false;
+ if (! INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest))
+ || !INTEGRAL_TYPE_P (TREE_TYPE (op0)))
+ return false;
+
/* Check the operands of the operation. */
if (!vect_is_simple_use (op0, loop_vinfo, &def_stmt, &def, &dt0))
{
unsigned int j;
VEC(tree,heap) *first, *second;
- scalar_dest = TREE_OPERAND (stmt, 0);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
first = VEC_alloc (tree, heap, length/2);
second = VEC_alloc (tree, heap, length/2);
vect1 = VEC_index (tree, dr_chain, j);
vect2 = VEC_index (tree, dr_chain, j+length/2);
- /* high = interleave_high (vect1, vect2); */
+ /* Create interleaving stmt:
+ in the case of big endian:
+ high = interleave_high (vect1, vect2)
+ and in the case of little endian:
+ high = interleave_low (vect1, vect2). */
perm_dest = create_tmp_var (vectype, "vect_inter_high");
+ DECL_GIMPLE_REG_P (perm_dest) = 1;
add_referenced_var (perm_dest);
- perm_stmt = build2 (MODIFY_EXPR, void_type_node, perm_dest,
- build2 (VEC_INTERLEAVE_HIGH_EXPR, vectype, vect1,
- vect2));
+ if (BYTES_BIG_ENDIAN)
+ perm_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, perm_dest,
+ build2 (VEC_INTERLEAVE_HIGH_EXPR, vectype,
+ vect1, vect2));
+ else
+ perm_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, perm_dest,
+ build2 (VEC_INTERLEAVE_LOW_EXPR, vectype,
+ vect1, vect2));
high = make_ssa_name (perm_dest, perm_stmt);
- TREE_OPERAND (perm_stmt, 0) = high;
+ GIMPLE_STMT_OPERAND (perm_stmt, 0) = high;
vect_finish_stmt_generation (stmt, perm_stmt, bsi);
VEC_replace (tree, *result_chain, 2*j, high);
- /* low = interleave_low (vect1, vect2); */
+ /* Create interleaving stmt:
+ in the case of big endian:
+ low = interleave_low (vect1, vect2)
+ and in the case of little endian:
+ low = interleave_high (vect1, vect2). */
perm_dest = create_tmp_var (vectype, "vect_inter_low");
+ DECL_GIMPLE_REG_P (perm_dest) = 1;
add_referenced_var (perm_dest);
- perm_stmt = build2 (MODIFY_EXPR, void_type_node, perm_dest,
- build2 (VEC_INTERLEAVE_LOW_EXPR, vectype, vect1,
- vect2));
+ if (BYTES_BIG_ENDIAN)
+ perm_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, perm_dest,
+ build2 (VEC_INTERLEAVE_LOW_EXPR, vectype,
+ vect1, vect2));
+ else
+ perm_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, perm_dest,
+ build2 (VEC_INTERLEAVE_HIGH_EXPR, vectype,
+ vect1, vect2));
low = make_ssa_name (perm_dest, perm_stmt);
- TREE_OPERAND (perm_stmt, 0) = low;
+ GIMPLE_STMT_OPERAND (perm_stmt, 0) = low;
vect_finish_stmt_generation (stmt, perm_stmt, bsi);
VEC_replace (tree, *result_chain, 2*j+1, low);
}
/* Is vectorizable store? */
- if (TREE_CODE (stmt) != MODIFY_EXPR)
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
return false;
- scalar_dest = TREE_OPERAND (stmt, 0);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
if (TREE_CODE (scalar_dest) != ARRAY_REF
&& TREE_CODE (scalar_dest) != INDIRECT_REF
&& !DR_GROUP_FIRST_DR (stmt_info))
return false;
- op = TREE_OPERAND (stmt, 1);
+ op = GIMPLE_STMT_OPERAND (stmt, 1);
if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &def, &dt))
{
if (vect_print_dump_info (REPORT_DETAILS))
S3: &base + 1 = x1
S4: &base + 3 = x3
- We create vectorized storess starting from base address (the access of the
+ We create vectorized stores starting from base address (the access of the
first stmt in the chain (S2 in the above example), when the last store stmt
of the chain (S4) is reached:
as an input to vect_permute_store_chain(), and OPRNDS as an input
to vect_get_vec_def_for_stmt_copy() for the next copy.
If the store is not strided, GROUP_SIZE is 1, and DR_CHAIN and
- OPRNDS are of size 1.
- */
+ OPRNDS are of size 1. */
next_stmt = first_stmt;
for (i = 0; i < group_size; i++)
{
is the exact number of stmts in the chain. Therefore, NEXT_STMT
can't be NULL_TREE. In case that there is no interleaving,
GROUP_SIZE is 1, and only one iteration of the loop will be
- executed.
- */
+ executed. */
gcc_assert (next_stmt);
- op = TREE_OPERAND (next_stmt, 1);
+ op = GIMPLE_STMT_OPERAND (next_stmt, 1);
vec_oprnd = vect_get_vec_def_for_operand (op, next_stmt, NULL);
VEC_quick_push(tree, dr_chain, vec_oprnd);
VEC_quick_push(tree, oprnds, vec_oprnd);
and OPRNDS as an input to vect_get_vec_def_for_stmt_copy() for the
next copy.
If the store is not strided, GROUP_SIZE is 1, and DR_CHAIN and
- OPRNDS are of size 1.
- */
+ OPRNDS are of size 1. */
for (i = 0; i < group_size; i++)
{
vec_oprnd = vect_get_vec_def_for_stmt_copy (dt,
data_ref = build_fold_indirect_ref (dataref_ptr);
/* Arguments are ready. Create the new vector stmt. */
- new_stmt = build2 (MODIFY_EXPR, void_type_node, data_ref,
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, data_ref,
vec_oprnd);
vect_finish_stmt_generation (stmt, new_stmt, bsi);
- /* Set the V_MAY_DEFS for the vector pointer. If this virtual def has a
- use outside the loop and a loop peel is performed then the def may be
- renamed by the peel. Mark it for renaming so the later use will also
- be renamed. */
+ /* Set the VDEFs for the vector pointer. If this virtual def
+ has a use outside the loop and a loop peel is performed
+ then the def may be renamed by the peel. Mark it for
+ renaming so the later use will also be renamed. */
copy_virtual_operands (new_stmt, next_stmt);
if (j == 0)
{
- /* The original store is deleted so the same SSA_NAMEs can be used.
- */
- FOR_EACH_SSA_TREE_OPERAND (def, next_stmt, iter, SSA_OP_VMAYDEF)
+ /* The original store is deleted so the same SSA_NAMEs
+ can be used. */
+ FOR_EACH_SSA_TREE_OPERAND (def, next_stmt, iter, SSA_OP_VDEF)
{
SSA_NAME_DEF_STMT (def) = new_stmt;
mark_sym_for_renaming (SSA_NAME_VAR (def));
{
/* Create new names for all the definitions created by COPY and
add replacement mappings for each new name. */
- FOR_EACH_SSA_DEF_OPERAND (def_p, new_stmt, iter, SSA_OP_VMAYDEF)
+ FOR_EACH_SSA_DEF_OPERAND (def_p, new_stmt, iter, SSA_OP_VDEF)
{
create_new_def_for (DEF_FROM_PTR (def_p), new_stmt, def_p);
mark_sym_for_renaming (SSA_NAME_VAR (DEF_FROM_PTR (def_p)));
}
prev_stmt_info = vinfo_for_stmt (new_stmt);
- next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt));
+ next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt));
if (!next_stmt)
break;
/* Bump the vector pointer. */
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
edge pe = loop_preheader_edge (loop);
- tree scalar_dest = TREE_OPERAND (stmt, 0);
+ tree scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
tree vec_dest;
tree init_addr;
tree inc;
ptr = vect_create_data_ref_ptr (stmt, bsi, NULL_TREE, &init_addr, &inc, true,
NULL_TREE);
data_ref = build1 (ALIGN_INDIRECT_REF, vectype, ptr);
- new_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest, data_ref);
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest, data_ref);
new_temp = make_ssa_name (vec_dest, new_stmt);
- TREE_OPERAND (new_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
new_bb = bsi_insert_on_edge_immediate (pe, new_stmt);
gcc_assert (!new_bb);
- msq_init = TREE_OPERAND (new_stmt, 0);
+ msq_init = GIMPLE_STMT_OPERAND (new_stmt, 0);
copy_virtual_operands (new_stmt, stmt);
update_vuses_to_preheader (new_stmt, loop);
if (targetm.vectorize.builtin_mask_for_load)
{
tree builtin_decl;
- tree params = build_tree_list (NULL_TREE, init_addr);
builtin_decl = targetm.vectorize.builtin_mask_for_load ();
- new_stmt = build_function_call_expr (builtin_decl, params);
+ new_stmt = build_call_expr (builtin_decl, 1, init_addr);
vec_dest = vect_create_destination_var (scalar_dest,
TREE_TYPE (new_stmt));
- new_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest, new_stmt);
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest,
+ new_stmt);
new_temp = make_ssa_name (vec_dest, new_stmt);
- TREE_OPERAND (new_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
new_bb = bsi_insert_on_edge_immediate (pe, new_stmt);
gcc_assert (!new_bb);
- *realignment_token = TREE_OPERAND (new_stmt, 0);
+ *realignment_token = GIMPLE_STMT_OPERAND (new_stmt, 0);
/* The result of the CALL_EXPR to this builtin is determined from
the value of the parameter and no global variables are touched
/* data_ref = permute_even (first_data_ref, second_data_ref); */
perm_dest = create_tmp_var (vectype, "vect_perm_even");
+ DECL_GIMPLE_REG_P (perm_dest) = 1;
add_referenced_var (perm_dest);
- perm_stmt = build2 (MODIFY_EXPR, void_type_node, perm_dest,
+ perm_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, perm_dest,
build2 (VEC_EXTRACT_EVEN_EXPR, vectype,
first_vect, second_vect));
data_ref = make_ssa_name (perm_dest, perm_stmt);
- TREE_OPERAND (perm_stmt, 0) = data_ref;
+ GIMPLE_STMT_OPERAND (perm_stmt, 0) = data_ref;
vect_finish_stmt_generation (stmt, perm_stmt, bsi);
- mark_new_vars_to_rename (perm_stmt);
+ mark_symbols_for_renaming (perm_stmt);
VEC_replace (tree, *result_chain, j/2, data_ref);
/* data_ref = permute_odd (first_data_ref, second_data_ref); */
perm_dest = create_tmp_var (vectype, "vect_perm_odd");
+ DECL_GIMPLE_REG_P (perm_dest) = 1;
add_referenced_var (perm_dest);
- perm_stmt = build2 (MODIFY_EXPR, void_type_node, perm_dest,
+ perm_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, perm_dest,
build2 (VEC_EXTRACT_ODD_EXPR, vectype,
first_vect, second_vect));
data_ref = make_ssa_name (perm_dest, perm_stmt);
- TREE_OPERAND (perm_stmt, 0) = data_ref;
+ GIMPLE_STMT_OPERAND (perm_stmt, 0) = data_ref;
vect_finish_stmt_generation (stmt, perm_stmt, bsi);
- mark_new_vars_to_rename (perm_stmt);
+ mark_symbols_for_renaming (perm_stmt);
VEC_replace (tree, *result_chain, j/2+length/2, data_ref);
}
return false;
}
- if (TREE_CODE (stmt) != MODIFY_EXPR)
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
return false;
- scalar_dest = TREE_OPERAND (stmt, 0);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
if (TREE_CODE (scalar_dest) != SSA_NAME)
return false;
- op = TREE_OPERAND (stmt, 1);
+ op = GIMPLE_STMT_OPERAND (stmt, 1);
if (TREE_CODE (op) != ARRAY_REF
&& TREE_CODE (op) != INDIRECT_REF
&& !DR_GROUP_FIRST_DR (stmt_info))
gcc_unreachable ();
}
vec_dest = vect_create_destination_var (scalar_dest, vectype);
- new_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest, data_ref);
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest,
+ data_ref);
new_temp = make_ssa_name (vec_dest, new_stmt);
- TREE_OPERAND (new_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
vect_finish_stmt_generation (stmt, new_stmt, bsi);
copy_virtual_operands (new_stmt, stmt);
- mark_new_vars_to_rename (new_stmt);
+ mark_symbols_for_renaming (new_stmt);
/* 3. Handle explicit realignment if necessary/supported. */
if (alignment_support_cheme == dr_unaligned_software_pipeline)
{
/* Create in loop:
<vec_dest = realign_load (msq, lsq, realignment_token)> */
- lsq = TREE_OPERAND (new_stmt, 0);
+ lsq = GIMPLE_STMT_OPERAND (new_stmt, 0);
if (!realignment_token)
realignment_token = dataref_ptr;
vec_dest = vect_create_destination_var (scalar_dest, vectype);
new_stmt =
build3 (REALIGN_LOAD_EXPR, vectype, msq, lsq, realignment_token);
- new_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest, new_stmt);
+ new_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest,
+ new_stmt);
new_temp = make_ssa_name (vec_dest, new_stmt);
- TREE_OPERAND (new_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (new_stmt, 0) = new_temp;
vect_finish_stmt_generation (stmt, new_stmt, bsi);
if (i == group_size - 1 && j == ncopies - 1)
add_phi_arg (phi_stmt, lsq, loop_latch_edge (loop));
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
int i;
- enum tree_code code;
int op_type;
tree op;
tree def, def_stmt;
if (!STMT_VINFO_LIVE_P (stmt_info))
return false;
- if (TREE_CODE (stmt) != MODIFY_EXPR)
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
return false;
- if (TREE_CODE (TREE_OPERAND (stmt, 0)) != SSA_NAME)
+ if (TREE_CODE (GIMPLE_STMT_OPERAND (stmt, 0)) != SSA_NAME)
return false;
- operation = TREE_OPERAND (stmt, 1);
- code = TREE_CODE (operation);
-
- op_type = TREE_CODE_LENGTH (code);
+ operation = GIMPLE_STMT_OPERAND (stmt, 1);
+ op_type = TREE_OPERAND_LENGTH (operation);
/* FORNOW: support only if all uses are invariant. This means
that the scalar operations can remain in place, unvectorized.
return false;
}
- if (TREE_CODE (stmt) != MODIFY_EXPR)
+ if (TREE_CODE (stmt) != GIMPLE_MODIFY_STMT)
return false;
- op = TREE_OPERAND (stmt, 1);
+ op = GIMPLE_STMT_OPERAND (stmt, 1);
if (TREE_CODE (op) != COND_EXPR)
return false;
/* Transform */
/* Handle def. */
- scalar_dest = TREE_OPERAND (stmt, 0);
+ scalar_dest = GIMPLE_STMT_OPERAND (stmt, 0);
vec_dest = vect_create_destination_var (scalar_dest, vectype);
/* Handle cond expr. */
vec_cond_expr = build3 (VEC_COND_EXPR, vectype,
vec_compare, vec_then_clause, vec_else_clause);
- *vec_stmt = build2 (MODIFY_EXPR, void_type_node, vec_dest, vec_cond_expr);
+ *vec_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node, vec_dest,
+ vec_cond_expr);
new_temp = make_ssa_name (vec_dest, *vec_stmt);
- TREE_OPERAND (*vec_stmt, 0) = new_temp;
+ GIMPLE_STMT_OPERAND (*vec_stmt, 0) = new_temp;
vect_finish_stmt_generation (stmt, *vec_stmt, bsi);
return true;
gcc_assert (done);
break;
+ case type_conversion_vec_info_type:
+ done = vectorizable_conversion (stmt, bsi, &vec_stmt);
+ gcc_assert (done);
+ break;
+
case op_vec_info_type:
done = vectorizable_operation (stmt, bsi, &vec_stmt);
gcc_assert (done);
LOOP - the loop whose preheader will contain STMT.
It's possible to vectorize a loop even though an SSA_NAME from a VUSE
- appears to be defined in a V_MAY_DEF in another statement in a loop.
+ appears to be defined in a VDEF in another statement in a loop.
One such case is when the VUSE is at the dereference of a __restricted__
- pointer in a load and the V_MAY_DEF is at the dereference of a different
+ pointer in a load and the VDEF is at the dereference of a different
__restricted__ pointer in a store. Vectorization may result in
copy_virtual_uses being called to copy the problematic VUSE to a new
statement that is being inserted in the loop preheader. This procedure
edge update_e;
basic_block preheader;
int loop_num;
+ unsigned int th;
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "=== vect_do_peeling_for_loop_bound ===");
&ratio_mult_vf_name, ratio);
loop_num = loop->num;
+ /* Threshold for vectorized loop. */
+ th = (PARAM_VALUE (PARAM_MIN_VECT_LOOP_BOUND)) *
+ LOOP_VINFO_VECT_FACTOR (loop_vinfo);
new_loop = slpeel_tree_peel_loop_to_edge (loop, single_exit (loop),
- ratio_mult_vf_name, ni_name, false);
+ ratio_mult_vf_name, ni_name, false, th);
gcc_assert (new_loop);
gcc_assert (loop_num == loop->num);
#ifdef ENABLE_CHECKING
/* Peel the prolog loop and iterate it niters_of_prolog_loop. */
new_loop =
slpeel_tree_peel_loop_to_edge (loop, loop_preheader_edge (loop),
- niters_of_prolog_loop, ni_name, true);
+ niters_of_prolog_loop, ni_name, true, 0);
gcc_assert (new_loop);
#ifdef ENABLE_CHECKING
slpeel_verify_cfg_after_peeling (new_loop, loop);
add_referenced_var (addr_tmp);
addr_tmp_name = make_ssa_name (addr_tmp, NULL_TREE);
addr_stmt = fold_convert (int_ptrsize_type, addr_base);
- addr_stmt = build2 (MODIFY_EXPR, void_type_node,
+ addr_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
addr_tmp_name, addr_stmt);
SSA_NAME_DEF_STMT (addr_tmp_name) = addr_stmt;
append_to_statement_list_force (addr_stmt, cond_expr_stmt_list);
or_tmp = create_tmp_var (int_ptrsize_type, tmp_name);
add_referenced_var (or_tmp);
new_or_tmp_name = make_ssa_name (or_tmp, NULL_TREE);
- or_stmt = build2 (MODIFY_EXPR, void_type_node, new_or_tmp_name,
+ or_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
+ new_or_tmp_name,
build2 (BIT_IOR_EXPR, int_ptrsize_type,
or_tmp_name,
addr_tmp_name));
add_referenced_var (and_tmp);
and_tmp_name = make_ssa_name (and_tmp, NULL_TREE);
- and_stmt = build2 (MODIFY_EXPR, void_type_node,
+ and_stmt = build2 (GIMPLE_MODIFY_STMT, void_type_node,
and_tmp_name,
build2 (BIT_AND_EXPR, int_ptrsize_type,
or_tmp_name, mask_cst));
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
basic_block *bbs = LOOP_VINFO_BBS (loop_vinfo);
int nbbs = loop->num_nodes;
- block_stmt_iterator si;
+ block_stmt_iterator si, next_si;
int i;
tree ratio = NULL;
int vectorization_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
- bitmap_iterator bi;
- unsigned int j;
bool strided_store;
if (vect_print_dump_info (REPORT_DETAILS))
basic_block new_exit_bb;
edge new_exit_e, e;
tree orig_phi, new_phi, arg;
+ unsigned prob = 4 * REG_BR_PROB_BASE / 5;
cond_expr = vect_create_cond_for_align_checks (loop_vinfo,
&cond_expr_stmt_list);
initialize_original_copy_tables ();
- nloop = loop_version (loop, cond_expr, &condition_bb, true);
+ nloop = loop_version (loop, cond_expr, &condition_bb,
+ prob, prob, REG_BR_PROB_BASE - prob, true);
free_original_copy_tables();
/** Loop versioning violates an assumption we try to maintain during
/* CHECKME: we wouldn't need this if we called update_ssa once
for all loops. */
- bitmap_zero (vect_vnames_to_rename);
+ bitmap_zero (vect_memsyms_to_rename);
/* Peel the loop if there are data refs with unknown alignment.
Only one data ref with unknown store is allowed. */
while (next)
{
+ next_si = bsi_for_stmt (next);
next_stmt_info = vinfo_for_stmt (next);
/* Free the attached stmt_vec_info and remove the stmt. */
ann = stmt_ann (next);
tmp = DR_GROUP_NEXT_DR (next_stmt_info);
free (next_stmt_info);
set_stmt_info (ann, NULL);
+ bsi_remove (&next_si, true);
next = tmp;
}
bsi_remove (&si, true);
continue;
}
}
- else
- {
- if (strided_store)
- {
- /* This is case of skipped interleaved store. We don't free
- its stmt_vec_info. */
- bsi_remove (&si, true);
- continue;
- }
- }
bsi_next (&si);
} /* stmts in BB */
} /* BBs in loop */
slpeel_make_loop_iterate_ntimes (loop, ratio);
- EXECUTE_IF_SET_IN_BITMAP (vect_vnames_to_rename, 0, j, bi)
- mark_sym_for_renaming (SSA_NAME_VAR (ssa_name (j)));
+ mark_set_for_renaming (vect_memsyms_to_rename);
/* The memory tags and pointers in vectorized statements need to
have their SSA forms updated. FIXME, why can't this be delayed
OSDN Git Service
