static gimple vect_recog_pow_pattern (VEC (gimple, heap) **, tree *, tree *);
static gimple vect_recog_over_widening_pattern (VEC (gimple, heap) **, tree *,
tree *);
+static gimple vect_recog_widen_shift_pattern (VEC (gimple, heap) **,
+ tree *, tree *);
+static gimple vect_recog_vector_vector_shift_pattern (VEC (gimple, heap) **,
+ tree *, tree *);
+static gimple vect_recog_sdivmod_pow2_pattern (VEC (gimple, heap) **,
+ tree *, tree *);
static gimple vect_recog_mixed_size_cond_pattern (VEC (gimple, heap) **,
tree *, tree *);
static gimple vect_recog_bool_pattern (VEC (gimple, heap) **, tree *, tree *);
vect_recog_dot_prod_pattern,
vect_recog_pow_pattern,
vect_recog_over_widening_pattern,
+ vect_recog_widen_shift_pattern,
+ vect_recog_vector_vector_shift_pattern,
+ vect_recog_sdivmod_pow2_pattern,
vect_recog_mixed_size_cond_pattern,
vect_recog_bool_pattern};
+static inline void
+append_pattern_def_seq (stmt_vec_info stmt_info, gimple stmt)
+{
+ gimple_seq_add_stmt_without_update (&STMT_VINFO_PATTERN_DEF_SEQ (stmt_info),
+ stmt);
+}
+
+static inline void
+new_pattern_def_seq (stmt_vec_info stmt_info, gimple stmt)
+{
+ STMT_VINFO_PATTERN_DEF_SEQ (stmt_info) = NULL;
+ append_pattern_def_seq (stmt_info, stmt);
+}
/* Function widened_name_p
}
-/* Handle two cases of multiplication by a constant. The first one is when
- the constant, CONST_OPRND, fits the type (HALF_TYPE) of the second
- operand (OPRND). In that case, we can peform widen-mult from HALF_TYPE to
- TYPE.
+/* Handle widening operation by a constant. At the moment we support MULT_EXPR
+ and LSHIFT_EXPR.
+
+ For MULT_EXPR we check that CONST_OPRND fits HALF_TYPE, and for LSHIFT_EXPR
+ we check that CONST_OPRND is less or equal to the size of HALF_TYPE.
Otherwise, if the type of the result (TYPE) is at least 4 times bigger than
- HALF_TYPE, and CONST_OPRND fits an intermediate type (2 times smaller than
- TYPE), we can perform widen-mult from the intermediate type to TYPE and
- replace a_T = (TYPE) a_t; with a_it - (interm_type) a_t; */
+ HALF_TYPE, and there is an intermediate type (2 times smaller than TYPE)
+ that satisfies the above restrictions, we can perform a widening opeartion
+ from the intermediate type to TYPE and replace a_T = (TYPE) a_t;
+ with a_it = (interm_type) a_t; */
static bool
-vect_handle_widen_mult_by_const (gimple stmt, tree const_oprnd, tree *oprnd,
- VEC (gimple, heap) **stmts, tree type,
- tree *half_type, gimple def_stmt)
+vect_handle_widen_op_by_const (gimple stmt, enum tree_code code,
+ tree const_oprnd, tree *oprnd,
+ VEC (gimple, heap) **stmts, tree type,
+ tree *half_type, gimple def_stmt)
{
tree new_type, new_oprnd, tmp;
gimple new_stmt;
loop_vec_info loop_info = STMT_VINFO_LOOP_VINFO (vinfo_for_stmt (stmt));
struct loop *loop = LOOP_VINFO_LOOP (loop_info);
- if (int_fits_type_p (const_oprnd, *half_type))
+ if (code != MULT_EXPR && code != LSHIFT_EXPR)
+ return false;
+
+ if (((code == MULT_EXPR && int_fits_type_p (const_oprnd, *half_type))
+ || (code == LSHIFT_EXPR
+ && compare_tree_int (const_oprnd, TYPE_PRECISION (*half_type))
+ != 1))
+ && TYPE_PRECISION (type) == (TYPE_PRECISION (*half_type) * 2))
{
/* CONST_OPRND is a constant of HALF_TYPE. */
*oprnd = gimple_assign_rhs1 (def_stmt);
|| !vinfo_for_stmt (def_stmt))
return false;
- /* TYPE is 4 times bigger than HALF_TYPE, try widen-mult for
+ /* TYPE is 4 times bigger than HALF_TYPE, try widening operation for
a type 2 times bigger than HALF_TYPE. */
new_type = build_nonstandard_integer_type (TYPE_PRECISION (type) / 2,
TYPE_UNSIGNED (type));
- if (!int_fits_type_p (const_oprnd, new_type))
+ if ((code == MULT_EXPR && !int_fits_type_p (const_oprnd, new_type))
+ || (code == LSHIFT_EXPR
+ && compare_tree_int (const_oprnd, TYPE_PRECISION (new_type)) == 1))
return false;
- /* Use NEW_TYPE for widen_mult. */
+ /* Use NEW_TYPE for widening operation. */
if (STMT_VINFO_RELATED_STMT (vinfo_for_stmt (def_stmt)))
{
new_stmt = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (def_stmt));
enum tree_code dummy_code;
int dummy_int;
VEC (tree, heap) *dummy_vec;
- bool op0_ok, op1_ok;
+ bool op1_ok;
if (!is_gimple_assign (last_stmt))
return NULL;
return NULL;
/* Check argument 0. */
- op0_ok = widened_name_p (oprnd0, last_stmt, &half_type0, &def_stmt0, false);
+ if (!widened_name_p (oprnd0, last_stmt, &half_type0, &def_stmt0, false))
+ return NULL;
/* Check argument 1. */
op1_ok = widened_name_p (oprnd1, last_stmt, &half_type1, &def_stmt1, false);
- /* In case of multiplication by a constant one of the operands may not match
- the pattern, but not both. */
- if (!op0_ok && !op1_ok)
- return NULL;
-
- if (op0_ok && op1_ok)
+ if (op1_ok)
{
oprnd0 = gimple_assign_rhs1 (def_stmt0);
oprnd1 = gimple_assign_rhs1 (def_stmt1);
}
- else if (!op0_ok)
- {
- if (TREE_CODE (oprnd0) == INTEGER_CST
- && TREE_CODE (half_type1) == INTEGER_TYPE
- && vect_handle_widen_mult_by_const (last_stmt, oprnd0, &oprnd1,
- stmts, type,
- &half_type1, def_stmt1))
- half_type0 = half_type1;
- else
- return NULL;
- }
- else if (!op1_ok)
+ else
{
if (TREE_CODE (oprnd1) == INTEGER_CST
&& TREE_CODE (half_type0) == INTEGER_TYPE
- && vect_handle_widen_mult_by_const (last_stmt, oprnd1, &oprnd0,
- stmts, type,
- &half_type0, def_stmt0))
+ && vect_handle_widen_op_by_const (last_stmt, MULT_EXPR, oprnd1,
+ &oprnd0, stmts, type,
+ &half_type0, def_stmt0))
half_type1 = half_type0;
else
return NULL;
NEW_DEF_STMT - in case DEF has to be promoted, we create two pattern
statements for STMT: the first one is a type promotion and the second
one is the operation itself. We return the type promotion statement
- in NEW_DEF_STMT and further store it in STMT_VINFO_PATTERN_DEF_STMT of
+ in NEW_DEF_STMT and further store it in STMT_VINFO_PATTERN_DEF_SEQ of
the second pattern statement. */
static bool
a. Its type is not sufficient for the operation, we create a new stmt:
a type conversion for OPRND from HALF_TYPE to INTERM_TYPE. We store
this statement in NEW_DEF_STMT, and it is later put in
- STMT_VINFO_PATTERN_DEF_STMT of the pattern statement for STMT.
+ STMT_VINFO_PATTERN_DEF_SEQ of the pattern statement for STMT.
b. OPRND is good to use in the new statement. */
if (first)
{
|| TREE_TYPE (gimple_assign_lhs (new_stmt)) != interm_type)
return false;
+ VEC_safe_push (gimple, heap, *stmts, def_stmt);
oprnd = gimple_assign_lhs (new_stmt);
}
else
tree var = NULL_TREE, new_type = NULL_TREE, tmp, new_oprnd;
bool first;
struct loop *loop = (gimple_bb (stmt))->loop_father;
+ tree type = NULL;
first = true;
while (1)
statetments, except for the case when the last statement in the
sequence doesn't have a corresponding pattern statement. In such
case we associate the last pattern statement with the last statement
- in the sequence. Therefore, we only add an original statetement to
+ in the sequence. Therefore, we only add the original statement to
the list if we know that it is not the last. */
if (prev_stmt)
VEC_safe_push (gimple, heap, *stmts, prev_stmt);
= gimple_build_assign_with_ops (gimple_assign_rhs_code (stmt), var,
op0, op1);
STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt)) = pattern_stmt;
- STMT_VINFO_PATTERN_DEF_STMT (vinfo_for_stmt (stmt)) = new_def_stmt;
+ new_pattern_def_seq (vinfo_for_stmt (stmt), new_def_stmt);
if (vect_print_dump_info (REPORT_DETAILS))
{
print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
}
+ type = gimple_expr_type (stmt);
prev_stmt = stmt;
stmt = use_stmt;
{
use_lhs = gimple_assign_lhs (use_stmt);
use_type = TREE_TYPE (use_lhs);
- /* Support only type promotion or signedess change. */
+ /* Support only type promotion or signedess change. Check that USE_TYPE
+ is not bigger than the original type. */
if (!INTEGRAL_TYPE_P (use_type)
- || TYPE_PRECISION (new_type) > TYPE_PRECISION (use_type))
+ || TYPE_PRECISION (new_type) > TYPE_PRECISION (use_type)
+ || TYPE_PRECISION (type) < TYPE_PRECISION (use_type))
return NULL;
if (TYPE_UNSIGNED (new_type) != TYPE_UNSIGNED (use_type)
else
{
if (prev_stmt)
- STMT_VINFO_PATTERN_DEF_STMT (vinfo_for_stmt (use_stmt))
- = STMT_VINFO_PATTERN_DEF_STMT (vinfo_for_stmt (prev_stmt));
+ STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (use_stmt))
+ = STMT_VINFO_PATTERN_DEF_SEQ (vinfo_for_stmt (prev_stmt));
*type_in = vectype;
*type_out = NULL_TREE;
return pattern_stmt;
}
+/* Detect widening shift pattern:
+
+ type a_t;
+ TYPE a_T, res_T;
+
+ S1 a_t = ;
+ S2 a_T = (TYPE) a_t;
+ S3 res_T = a_T << CONST;
+
+ where type 'TYPE' is at least double the size of type 'type'.
+
+ Also detect unsigned cases:
+
+ unsigned type a_t;
+ unsigned TYPE u_res_T;
+ TYPE a_T, res_T;
+
+ S1 a_t = ;
+ S2 a_T = (TYPE) a_t;
+ S3 res_T = a_T << CONST;
+ S4 u_res_T = (unsigned TYPE) res_T;
+
+ And a case when 'TYPE' is 4 times bigger than 'type'. In that case we
+ create an additional pattern stmt for S2 to create a variable of an
+ intermediate type, and perform widen-shift on the intermediate type:
+
+ type a_t;
+ interm_type a_it;
+ TYPE a_T, res_T, res_T';
+
+ S1 a_t = ;
+ S2 a_T = (TYPE) a_t;
+ '--> a_it = (interm_type) a_t;
+ S3 res_T = a_T << CONST;
+ '--> res_T' = a_it <<* CONST;
+
+ Input/Output:
+
+ * STMTS: Contains a stmt from which the pattern search begins.
+ In case of unsigned widen-shift, the original stmt (S3) is replaced with S4
+ in STMTS. When an intermediate type is used and a pattern statement is
+ created for S2, we also put S2 here (before S3).
+
+ Output:
+
+ * TYPE_IN: The type of the input arguments to the pattern.
+
+ * TYPE_OUT: The type of the output of this pattern.
+
+ * Return value: A new stmt that will be used to replace the sequence of
+ stmts that constitute the pattern. In this case it will be:
+ WIDEN_LSHIFT_EXPR <a_t, CONST>. */
+
+static gimple
+vect_recog_widen_shift_pattern (VEC (gimple, heap) **stmts,
+ tree *type_in, tree *type_out)
+{
+ gimple last_stmt = VEC_pop (gimple, *stmts);
+ gimple def_stmt0;
+ tree oprnd0, oprnd1;
+ tree type, half_type0;
+ gimple pattern_stmt, orig_stmt = NULL;
+ tree vectype, vectype_out = NULL_TREE;
+ tree dummy;
+ tree var;
+ enum tree_code dummy_code;
+ int dummy_int;
+ VEC (tree, heap) * dummy_vec;
+ gimple use_stmt = NULL;
+ bool over_widen = false;
+
+ if (!is_gimple_assign (last_stmt) || !vinfo_for_stmt (last_stmt))
+ return NULL;
+
+ orig_stmt = last_stmt;
+ if (STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (last_stmt)))
+ {
+ /* This statement was also detected as over-widening operation (it can't
+ be any other pattern, because only over-widening detects shifts).
+ LAST_STMT is the final type demotion statement, but its related
+ statement is shift. We analyze the related statement to catch cases:
+
+ orig code:
+ type a_t;
+ itype res;
+ TYPE a_T, res_T;
+
+ S1 a_T = (TYPE) a_t;
+ S2 res_T = a_T << CONST;
+ S3 res = (itype)res_T;
+
+ (size of type * 2 <= size of itype
+ and size of itype * 2 <= size of TYPE)
+
+ code after over-widening pattern detection:
+
+ S1 a_T = (TYPE) a_t;
+ --> a_it = (itype) a_t;
+ S2 res_T = a_T << CONST;
+ S3 res = (itype)res_T; <--- LAST_STMT
+ --> res = a_it << CONST;
+
+ after widen_shift:
+
+ S1 a_T = (TYPE) a_t;
+ --> a_it = (itype) a_t; - redundant
+ S2 res_T = a_T << CONST;
+ S3 res = (itype)res_T;
+ --> res = a_t w<< CONST;
+
+ i.e., we replace the three statements with res = a_t w<< CONST. */
+ last_stmt = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (last_stmt));
+ over_widen = true;
+ }
+
+ if (gimple_assign_rhs_code (last_stmt) != LSHIFT_EXPR)
+ return NULL;
+
+ oprnd0 = gimple_assign_rhs1 (last_stmt);
+ oprnd1 = gimple_assign_rhs2 (last_stmt);
+ if (TREE_CODE (oprnd0) != SSA_NAME || TREE_CODE (oprnd1) != INTEGER_CST)
+ return NULL;
+
+ /* Check operand 0: it has to be defined by a type promotion. */
+ if (!widened_name_p (oprnd0, last_stmt, &half_type0, &def_stmt0, false))
+ return NULL;
+
+ /* Check operand 1: has to be positive. We check that it fits the type
+ in vect_handle_widen_op_by_const (). */
+ if (tree_int_cst_compare (oprnd1, size_zero_node) <= 0)
+ return NULL;
+
+ oprnd0 = gimple_assign_rhs1 (def_stmt0);
+ type = gimple_expr_type (last_stmt);
+
+ /* Check if this a widening operation. */
+ if (!vect_handle_widen_op_by_const (last_stmt, LSHIFT_EXPR, oprnd1,
+ &oprnd0, stmts,
+ type, &half_type0, def_stmt0))
+ return NULL;
+
+ /* Handle unsigned case. Look for
+ S4 u_res_T = (unsigned TYPE) res_T;
+ Use unsigned TYPE as the type for WIDEN_LSHIFT_EXPR. */
+ if (TYPE_UNSIGNED (type) != TYPE_UNSIGNED (half_type0))
+ {
+ tree lhs = gimple_assign_lhs (last_stmt), use_lhs;
+ imm_use_iterator imm_iter;
+ use_operand_p use_p;
+ int nuses = 0;
+ tree use_type;
+
+ if (over_widen)
+ {
+ /* In case of over-widening pattern, S4 should be ORIG_STMT itself.
+ We check here that TYPE is the correct type for the operation,
+ i.e., it's the type of the original result. */
+ tree orig_type = gimple_expr_type (orig_stmt);
+ if ((TYPE_UNSIGNED (type) != TYPE_UNSIGNED (orig_type))
+ || (TYPE_PRECISION (type) != TYPE_PRECISION (orig_type)))
+ return NULL;
+ }
+ else
+ {
+ FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
+ {
+ if (is_gimple_debug (USE_STMT (use_p)))
+ continue;
+ use_stmt = USE_STMT (use_p);
+ nuses++;
+ }
+
+ if (nuses != 1 || !is_gimple_assign (use_stmt)
+ || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (use_stmt)))
+ return NULL;
+
+ use_lhs = gimple_assign_lhs (use_stmt);
+ use_type = TREE_TYPE (use_lhs);
+
+ if (!INTEGRAL_TYPE_P (use_type)
+ || (TYPE_UNSIGNED (type) == TYPE_UNSIGNED (use_type))
+ || (TYPE_PRECISION (type) != TYPE_PRECISION (use_type)))
+ return NULL;
+
+ type = use_type;
+ }
+ }
+
+ /* Pattern detected. */
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "vect_recog_widen_shift_pattern: detected: ");
+
+ /* Check target support. */
+ vectype = get_vectype_for_scalar_type (half_type0);
+ vectype_out = get_vectype_for_scalar_type (type);
+
+ if (!vectype
+ || !vectype_out
+ || !supportable_widening_operation (WIDEN_LSHIFT_EXPR, last_stmt,
+ vectype_out, vectype,
+ &dummy, &dummy, &dummy_code,
+ &dummy_code, &dummy_int,
+ &dummy_vec))
+ return NULL;
+
+ *type_in = vectype;
+ *type_out = vectype_out;
+
+ /* Pattern supported. Create a stmt to be used to replace the pattern. */
+ var = vect_recog_temp_ssa_var (type, NULL);
+ pattern_stmt =
+ gimple_build_assign_with_ops (WIDEN_LSHIFT_EXPR, var, oprnd0, oprnd1);
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
+
+ if (use_stmt)
+ last_stmt = use_stmt;
+ else
+ last_stmt = orig_stmt;
+
+ VEC_safe_push (gimple, heap, *stmts, last_stmt);
+ return pattern_stmt;
+}
+
+/* Detect a vector by vector shift pattern that wouldn't be otherwise
+ vectorized:
+
+ type a_t;
+ TYPE b_T, res_T;
+
+ S1 a_t = ;
+ S2 b_T = ;
+ S3 res_T = b_T op a_t;
+
+ where type 'TYPE' is a type with different size than 'type',
+ and op is <<, >> or rotate.
+
+ Also detect cases:
+
+ type a_t;
+ TYPE b_T, c_T, res_T;
+
+ S0 c_T = ;
+ S1 a_t = (type) c_T;
+ S2 b_T = ;
+ S3 res_T = b_T op a_t;
+
+ Input/Output:
+
+ * STMTS: Contains a stmt from which the pattern search begins,
+ i.e. the shift/rotate stmt. The original stmt (S3) is replaced
+ with a shift/rotate which has same type on both operands, in the
+ second case just b_T op c_T, in the first case with added cast
+ from a_t to c_T in STMT_VINFO_PATTERN_DEF_SEQ.
+
+ Output:
+
+ * TYPE_IN: The type of the input arguments to the pattern.
+
+ * TYPE_OUT: The type of the output of this pattern.
+
+ * Return value: A new stmt that will be used to replace the shift/rotate
+ S3 stmt. */
+
+static gimple
+vect_recog_vector_vector_shift_pattern (VEC (gimple, heap) **stmts,
+ tree *type_in, tree *type_out)
+{
+ gimple last_stmt = VEC_pop (gimple, *stmts);
+ tree oprnd0, oprnd1, lhs, var;
+ gimple pattern_stmt, def_stmt;
+ enum tree_code rhs_code;
+ stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+ enum vect_def_type dt;
+ tree def;
+
+ if (!is_gimple_assign (last_stmt))
+ return NULL;
+
+ rhs_code = gimple_assign_rhs_code (last_stmt);
+ switch (rhs_code)
+ {
+ case LSHIFT_EXPR:
+ case RSHIFT_EXPR:
+ case LROTATE_EXPR:
+ case RROTATE_EXPR:
+ break;
+ default:
+ return NULL;
+ }
+
+ if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
+ return NULL;
+
+ lhs = gimple_assign_lhs (last_stmt);
+ oprnd0 = gimple_assign_rhs1 (last_stmt);
+ oprnd1 = gimple_assign_rhs2 (last_stmt);
+ if (TREE_CODE (oprnd0) != SSA_NAME
+ || TREE_CODE (oprnd1) != SSA_NAME
+ || TYPE_MODE (TREE_TYPE (oprnd0)) == TYPE_MODE (TREE_TYPE (oprnd1))
+ || TYPE_PRECISION (TREE_TYPE (oprnd1))
+ != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (oprnd1)))
+ || TYPE_PRECISION (TREE_TYPE (lhs))
+ != TYPE_PRECISION (TREE_TYPE (oprnd0)))
+ return NULL;
+
+ if (!vect_is_simple_use (oprnd1, loop_vinfo, NULL, &def_stmt, &def, &dt))
+ return NULL;
+
+ if (dt != vect_internal_def)
+ return NULL;
+
+ *type_in = get_vectype_for_scalar_type (TREE_TYPE (oprnd0));
+ *type_out = *type_in;
+ if (*type_in == NULL_TREE)
+ return NULL;
+
+ def = NULL_TREE;
+ if (gimple_assign_cast_p (def_stmt))
+ {
+ tree rhs1 = gimple_assign_rhs1 (def_stmt);
+ if (TYPE_MODE (TREE_TYPE (rhs1)) == TYPE_MODE (TREE_TYPE (oprnd0))
+ && TYPE_PRECISION (TREE_TYPE (rhs1))
+ == TYPE_PRECISION (TREE_TYPE (oprnd0)))
+ def = rhs1;
+ }
+
+ if (def == NULL_TREE)
+ {
+ def = vect_recog_temp_ssa_var (TREE_TYPE (oprnd0), NULL);
+ def_stmt = gimple_build_assign_with_ops (NOP_EXPR, def, oprnd1,
+ NULL_TREE);
+ new_pattern_def_seq (stmt_vinfo, def_stmt);
+ }
+
+ /* Pattern detected. */
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "vect_recog_vector_vector_shift_pattern: detected: ");
+
+ /* Pattern supported. Create a stmt to be used to replace the pattern. */
+ var = vect_recog_temp_ssa_var (TREE_TYPE (oprnd0), NULL);
+ pattern_stmt = gimple_build_assign_with_ops (rhs_code, var, oprnd0, def);
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
+
+ VEC_safe_push (gimple, heap, *stmts, last_stmt);
+ return pattern_stmt;
+}
+
+/* Detect a signed division by power of two constant that wouldn't be
+ otherwise vectorized:
+
+ type a_t, b_t;
+
+ S1 a_t = b_t / N;
+
+ where type 'type' is a signed integral type and N is a constant positive
+ power of two.
+
+ Similarly handle signed modulo by power of two constant:
+
+ S4 a_t = b_t % N;
+
+ Input/Output:
+
+ * STMTS: Contains a stmt from which the pattern search begins,
+ i.e. the division stmt. S1 is replaced by:
+ S3 y_t = b_t < 0 ? N - 1 : 0;
+ S2 x_t = b_t + y_t;
+ S1' a_t = x_t >> log2 (N);
+
+ S4 is replaced by (where *_T temporaries have unsigned type):
+ S9 y_T = b_t < 0 ? -1U : 0U;
+ S8 z_T = y_T >> (sizeof (type_t) * CHAR_BIT - log2 (N));
+ S7 z_t = (type) z_T;
+ S6 w_t = b_t + z_t;
+ S5 x_t = w_t & (N - 1);
+ S4' a_t = x_t - z_t;
+
+ Output:
+
+ * TYPE_IN: The type of the input arguments to the pattern.
+
+ * TYPE_OUT: The type of the output of this pattern.
+
+ * Return value: A new stmt that will be used to replace the division
+ S1 or modulo S4 stmt. */
+
+static gimple
+vect_recog_sdivmod_pow2_pattern (VEC (gimple, heap) **stmts,
+ tree *type_in, tree *type_out)
+{
+ gimple last_stmt = VEC_pop (gimple, *stmts);
+ tree oprnd0, oprnd1, vectype, itype, cond;
+ gimple pattern_stmt, def_stmt;
+ enum tree_code rhs_code;
+ stmt_vec_info stmt_vinfo = vinfo_for_stmt (last_stmt);
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
+ optab optab;
+
+ if (!is_gimple_assign (last_stmt))
+ return NULL;
+
+ rhs_code = gimple_assign_rhs_code (last_stmt);
+ switch (rhs_code)
+ {
+ case TRUNC_DIV_EXPR:
+ case TRUNC_MOD_EXPR:
+ break;
+ default:
+ return NULL;
+ }
+
+ if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo))
+ return NULL;
+
+ oprnd0 = gimple_assign_rhs1 (last_stmt);
+ oprnd1 = gimple_assign_rhs2 (last_stmt);
+ itype = TREE_TYPE (oprnd0);
+ if (TREE_CODE (oprnd0) != SSA_NAME
+ || TREE_CODE (oprnd1) != INTEGER_CST
+ || TREE_CODE (itype) != INTEGER_TYPE
+ || TYPE_UNSIGNED (itype)
+ || TYPE_PRECISION (itype) != GET_MODE_PRECISION (TYPE_MODE (itype))
+ || !integer_pow2p (oprnd1)
+ || tree_int_cst_sgn (oprnd1) != 1)
+ return NULL;
+
+ vectype = get_vectype_for_scalar_type (itype);
+ if (vectype == NULL_TREE)
+ return NULL;
+
+ /* If the target can handle vectorized division or modulo natively,
+ don't attempt to optimize this. */
+ optab = optab_for_tree_code (rhs_code, vectype, optab_default);
+ if (optab != NULL)
+ {
+ enum machine_mode vec_mode = TYPE_MODE (vectype);
+ int icode = (int) optab_handler (optab, vec_mode);
+ if (icode != CODE_FOR_nothing
+ || GET_MODE_SIZE (vec_mode) == UNITS_PER_WORD)
+ return NULL;
+ }
+
+ /* Pattern detected. */
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "vect_recog_sdivmod_pow2_pattern: detected: ");
+
+ cond = build2 (LT_EXPR, boolean_type_node, oprnd0, build_int_cst (itype, 0));
+ if (rhs_code == TRUNC_DIV_EXPR)
+ {
+ tree var = vect_recog_temp_ssa_var (itype, NULL);
+ def_stmt
+ = gimple_build_assign_with_ops3 (COND_EXPR, var, cond,
+ fold_build2 (MINUS_EXPR, itype,
+ oprnd1,
+ build_int_cst (itype,
+ 1)),
+ build_int_cst (itype, 0));
+ new_pattern_def_seq (stmt_vinfo, def_stmt);
+ var = vect_recog_temp_ssa_var (itype, NULL);
+ def_stmt
+ = gimple_build_assign_with_ops (PLUS_EXPR, var, oprnd0,
+ gimple_assign_lhs (def_stmt));
+ append_pattern_def_seq (stmt_vinfo, def_stmt);
+
+ pattern_stmt
+ = gimple_build_assign_with_ops (RSHIFT_EXPR,
+ vect_recog_temp_ssa_var (itype, NULL),
+ var,
+ build_int_cst (itype,
+ tree_log2 (oprnd1)));
+ }
+ else
+ {
+ tree signmask;
+ STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo) = NULL;
+ if (compare_tree_int (oprnd1, 2) == 0)
+ {
+ signmask = vect_recog_temp_ssa_var (itype, NULL);
+ def_stmt
+ = gimple_build_assign_with_ops3 (COND_EXPR, signmask, cond,
+ build_int_cst (itype, 1),
+ build_int_cst (itype, 0));
+ append_pattern_def_seq (stmt_vinfo, def_stmt);
+ }
+ else
+ {
+ tree utype
+ = build_nonstandard_integer_type (TYPE_PRECISION (itype), 1);
+ tree vecutype = get_vectype_for_scalar_type (utype);
+ tree shift
+ = build_int_cst (utype, GET_MODE_BITSIZE (TYPE_MODE (itype))
+ - tree_log2 (oprnd1));
+ tree var = vect_recog_temp_ssa_var (utype, NULL);
+ stmt_vec_info def_stmt_vinfo;
+
+ def_stmt
+ = gimple_build_assign_with_ops3 (COND_EXPR, var, cond,
+ build_int_cst (utype, -1),
+ build_int_cst (utype, 0));
+ def_stmt_vinfo = new_stmt_vec_info (def_stmt, loop_vinfo, NULL);
+ set_vinfo_for_stmt (def_stmt, def_stmt_vinfo);
+ STMT_VINFO_VECTYPE (def_stmt_vinfo) = vecutype;
+ append_pattern_def_seq (stmt_vinfo, def_stmt);
+ var = vect_recog_temp_ssa_var (utype, NULL);
+ def_stmt
+ = gimple_build_assign_with_ops (RSHIFT_EXPR, var,
+ gimple_assign_lhs (def_stmt),
+ shift);
+ def_stmt_vinfo = new_stmt_vec_info (def_stmt, loop_vinfo, NULL);
+ set_vinfo_for_stmt (def_stmt, def_stmt_vinfo);
+ STMT_VINFO_VECTYPE (def_stmt_vinfo) = vecutype;
+ append_pattern_def_seq (stmt_vinfo, def_stmt);
+ signmask = vect_recog_temp_ssa_var (itype, NULL);
+ def_stmt
+ = gimple_build_assign_with_ops (NOP_EXPR, signmask, var,
+ NULL_TREE);
+ append_pattern_def_seq (stmt_vinfo, def_stmt);
+ }
+ def_stmt
+ = gimple_build_assign_with_ops (PLUS_EXPR,
+ vect_recog_temp_ssa_var (itype, NULL),
+ oprnd0, signmask);
+ append_pattern_def_seq (stmt_vinfo, def_stmt);
+ def_stmt
+ = gimple_build_assign_with_ops (BIT_AND_EXPR,
+ vect_recog_temp_ssa_var (itype, NULL),
+ gimple_assign_lhs (def_stmt),
+ fold_build2 (MINUS_EXPR, itype,
+ oprnd1,
+ build_int_cst (itype,
+ 1)));
+ append_pattern_def_seq (stmt_vinfo, def_stmt);
+
+ pattern_stmt
+ = gimple_build_assign_with_ops (MINUS_EXPR,
+ vect_recog_temp_ssa_var (itype, NULL),
+ gimple_assign_lhs (def_stmt),
+ signmask);
+ }
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ print_gimple_stmt (vect_dump, pattern_stmt, 0, TDF_SLIM);
+
+ VEC_safe_push (gimple, heap, *stmts, last_stmt);
+
+ *type_in = vectype;
+ *type_out = vectype;
+ return pattern_stmt;
+}
/* Function vect_recog_mixed_size_cond_pattern
vect_recog_temp_ssa_var (type, NULL),
gimple_assign_lhs (def_stmt), NULL_TREE);
- STMT_VINFO_PATTERN_DEF_STMT (stmt_vinfo) = def_stmt;
+ new_pattern_def_seq (stmt_vinfo, def_stmt);
def_stmt_info = new_stmt_vec_info (def_stmt, loop_vinfo, NULL);
set_vinfo_for_stmt (def_stmt, def_stmt_info);
STMT_VINFO_VECTYPE (def_stmt_info) = vecitype;
{
enum machine_mode mode = TYPE_MODE (TREE_TYPE (rhs1));
tree itype
- = build_nonstandard_integer_type (GET_MODE_BITSIZE (mode), 0);
+ = build_nonstandard_integer_type (GET_MODE_BITSIZE (mode), 1);
vecitype = get_vectype_for_scalar_type (itype);
if (vecitype == NULL_TREE)
return false;
/* Helper function of adjust_bool_pattern. Add a cast to TYPE to a previous
stmt (SSA_NAME_DEF_STMT of VAR) by moving the COND_EXPR from RELATED_STMT
- to PATTERN_DEF_STMT and adding a cast as RELATED_STMT. */
+ to PATTERN_DEF_SEQ and adding a cast as RELATED_STMT. */
static tree
adjust_bool_pattern_cast (tree type, tree var)
stmt_vec_info stmt_vinfo = vinfo_for_stmt (SSA_NAME_DEF_STMT (var));
gimple cast_stmt, pattern_stmt;
- gcc_assert (!STMT_VINFO_PATTERN_DEF_STMT (stmt_vinfo));
+ gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo));
pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_vinfo);
- STMT_VINFO_PATTERN_DEF_STMT (stmt_vinfo) = pattern_stmt;
+ new_pattern_def_seq (stmt_vinfo, pattern_stmt);
cast_stmt
= gimple_build_assign_with_ops (NOP_EXPR,
vect_recog_temp_ssa_var (type, NULL),
VEC_quick_push (gimple, *stmts, stmt);
STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt))
= STMT_VINFO_RELATED_STMT (stmt_def_vinfo);
- gcc_assert (!STMT_VINFO_PATTERN_DEF_STMT (stmt_def_vinfo));
+ gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_def_vinfo));
STMT_VINFO_RELATED_STMT (stmt_def_vinfo) = NULL;
return irhs2;
}
VEC_quick_push (gimple, *stmts, stmt);
STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt))
= STMT_VINFO_RELATED_STMT (stmt_def_vinfo);
- gcc_assert (!STMT_VINFO_PATTERN_DEF_STMT (stmt_def_vinfo));
+ gcc_assert (!STMT_VINFO_PATTERN_DEF_SEQ (stmt_def_vinfo));
STMT_VINFO_RELATED_STMT (stmt_def_vinfo) = NULL;
return irhs1;
}
default:
gcc_assert (TREE_CODE_CLASS (rhs_code) == tcc_comparison);
if (TREE_CODE (TREE_TYPE (rhs1)) != INTEGER_TYPE
- || TYPE_UNSIGNED (TREE_TYPE (rhs1)))
+ || !TYPE_UNSIGNED (TREE_TYPE (rhs1)))
{
enum machine_mode mode = TYPE_MODE (TREE_TYPE (rhs1));
itype
- = build_nonstandard_integer_type (GET_MODE_BITSIZE (mode), 0);
+ = build_nonstandard_integer_type (GET_MODE_BITSIZE (mode), 1);
}
else
itype = TREE_TYPE (rhs1);
rhs_code = gimple_assign_rhs_code (last_stmt);
if (CONVERT_EXPR_CODE_P (rhs_code))
{
- if (TREE_CODE (TREE_TYPE (lhs)) != INTEGER_TYPE)
+ if (TREE_CODE (TREE_TYPE (lhs)) != INTEGER_TYPE
+ || TYPE_PRECISION (TREE_TYPE (lhs)) == 1)
return NULL;
vectype = get_vectype_for_scalar_type (TREE_TYPE (lhs));
if (vectype == NULL_TREE)
VEC_safe_push (gimple, heap, *stmts, last_stmt);
return pattern_stmt;
}
+ else if (rhs_code == SSA_NAME
+ && STMT_VINFO_DATA_REF (stmt_vinfo))
+ {
+ stmt_vec_info pattern_stmt_info;
+ vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
+ gcc_assert (vectype != NULL_TREE);
+ if (!VECTOR_MODE_P (TYPE_MODE (vectype)))
+ return NULL;
+ if (!check_bool_pattern (var, loop_vinfo))
+ return NULL;
+
+ rhs = adjust_bool_pattern (var, TREE_TYPE (vectype), NULL_TREE, stmts);
+ lhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (vectype), lhs);
+ if (!useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (rhs)))
+ {
+ tree rhs2 = vect_recog_temp_ssa_var (TREE_TYPE (lhs), NULL);
+ gimple cast_stmt
+ = gimple_build_assign_with_ops (NOP_EXPR, rhs2, rhs, NULL_TREE);
+ new_pattern_def_seq (stmt_vinfo, cast_stmt);
+ rhs = rhs2;
+ }
+ pattern_stmt
+ = gimple_build_assign_with_ops (SSA_NAME, lhs, rhs, NULL_TREE);
+ pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo, NULL);
+ set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info);
+ STMT_VINFO_DATA_REF (pattern_stmt_info)
+ = STMT_VINFO_DATA_REF (stmt_vinfo);
+ STMT_VINFO_DR_BASE_ADDRESS (pattern_stmt_info)
+ = STMT_VINFO_DR_BASE_ADDRESS (stmt_vinfo);
+ STMT_VINFO_DR_INIT (pattern_stmt_info) = STMT_VINFO_DR_INIT (stmt_vinfo);
+ STMT_VINFO_DR_OFFSET (pattern_stmt_info)
+ = STMT_VINFO_DR_OFFSET (stmt_vinfo);
+ STMT_VINFO_DR_STEP (pattern_stmt_info) = STMT_VINFO_DR_STEP (stmt_vinfo);
+ STMT_VINFO_DR_ALIGNED_TO (pattern_stmt_info)
+ = STMT_VINFO_DR_ALIGNED_TO (stmt_vinfo);
+ DR_STMT (STMT_VINFO_DATA_REF (stmt_vinfo)) = pattern_stmt;
+ *type_out = vectype;
+ *type_in = vectype;
+ VEC_safe_push (gimple, heap, *stmts, last_stmt);
+ return pattern_stmt;
+ }
else
return NULL;
}
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (orig_stmt_info);
gimple def_stmt;
- set_vinfo_for_stmt (pattern_stmt,
- new_stmt_vec_info (pattern_stmt, loop_vinfo, NULL));
- gimple_set_bb (pattern_stmt, gimple_bb (orig_stmt));
pattern_stmt_info = vinfo_for_stmt (pattern_stmt);
+ if (pattern_stmt_info == NULL)
+ {
+ pattern_stmt_info = new_stmt_vec_info (pattern_stmt, loop_vinfo, NULL);
+ set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info);
+ }
+ gimple_set_bb (pattern_stmt, gimple_bb (orig_stmt));
STMT_VINFO_RELATED_STMT (pattern_stmt_info) = orig_stmt;
STMT_VINFO_DEF_TYPE (pattern_stmt_info)
- = STMT_VINFO_DEF_TYPE (orig_stmt_info);
+ = STMT_VINFO_DEF_TYPE (orig_stmt_info);
STMT_VINFO_VECTYPE (pattern_stmt_info) = pattern_vectype;
STMT_VINFO_IN_PATTERN_P (orig_stmt_info) = true;
STMT_VINFO_RELATED_STMT (orig_stmt_info) = pattern_stmt;
- STMT_VINFO_PATTERN_DEF_STMT (pattern_stmt_info)
- = STMT_VINFO_PATTERN_DEF_STMT (orig_stmt_info);
- if (STMT_VINFO_PATTERN_DEF_STMT (pattern_stmt_info))
+ STMT_VINFO_PATTERN_DEF_SEQ (pattern_stmt_info)
+ = STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_info);
+ if (STMT_VINFO_PATTERN_DEF_SEQ (pattern_stmt_info))
{
- def_stmt = STMT_VINFO_PATTERN_DEF_STMT (pattern_stmt_info);
- def_stmt_info = vinfo_for_stmt (def_stmt);
- if (def_stmt_info == NULL)
+ gimple_stmt_iterator si;
+ for (si = gsi_start (STMT_VINFO_PATTERN_DEF_SEQ (pattern_stmt_info));
+ !gsi_end_p (si); gsi_next (&si))
{
- def_stmt_info = new_stmt_vec_info (def_stmt, loop_vinfo, NULL);
- set_vinfo_for_stmt (def_stmt, def_stmt_info);
+ def_stmt = gsi_stmt (si);
+ def_stmt_info = vinfo_for_stmt (def_stmt);
+ if (def_stmt_info == NULL)
+ {
+ def_stmt_info = new_stmt_vec_info (def_stmt, loop_vinfo, NULL);
+ set_vinfo_for_stmt (def_stmt, def_stmt_info);
+ }
+ gimple_set_bb (def_stmt, gimple_bb (orig_stmt));
+ STMT_VINFO_RELATED_STMT (def_stmt_info) = orig_stmt;
+ STMT_VINFO_DEF_TYPE (def_stmt_info)
+ = STMT_VINFO_DEF_TYPE (orig_stmt_info);
+ if (STMT_VINFO_VECTYPE (def_stmt_info) == NULL_TREE)
+ STMT_VINFO_VECTYPE (def_stmt_info) = pattern_vectype;
}
- gimple_set_bb (def_stmt, gimple_bb (orig_stmt));
- STMT_VINFO_RELATED_STMT (def_stmt_info) = orig_stmt;
- STMT_VINFO_DEF_TYPE (def_stmt_info)
- = STMT_VINFO_DEF_TYPE (orig_stmt_info);
- if (STMT_VINFO_VECTYPE (def_stmt_info) == NULL_TREE)
- STMT_VINFO_VECTYPE (def_stmt_info) = pattern_vectype;
}
}
OSDN Git Service
