+ nloop_uses = 0;
+ n_out_of_loop_uses = 0;
+ FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs)
+ {
+ gimple use_stmt = USE_STMT (use_p);
+ if (is_gimple_debug (use_stmt))
+ continue;
+
+ use_stmt = USE_STMT (use_p);
+
+ /* Check if we got back to the reduction phi. */
+ if (use_stmt == phi)
+ {
+ loop_use_stmt = use_stmt;
+ found = true;
+ break;
+ }
+
+ if (flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
+ {
+ if (vinfo_for_stmt (use_stmt)
+ && !STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (use_stmt)))
+ {
+ loop_use_stmt = use_stmt;
+ nloop_uses++;
+ }
+ }
+ else
+ n_out_of_loop_uses++;
+
+ /* There are can be either a single use in the loop or two uses in
+ phi nodes. */
+ if (nloop_uses > 1 || (n_out_of_loop_uses && nloop_uses))
+ return false;
+ }
+
+ if (found)
+ break;
+
+ /* We reached a statement with no loop uses. */
+ if (nloop_uses == 0)
+ return false;
+
+ /* This is a loop exit phi, and we haven't reached the reduction phi. */
+ if (gimple_code (loop_use_stmt) == GIMPLE_PHI)
+ return false;
+
+ if (!is_gimple_assign (loop_use_stmt)
+ || code != gimple_assign_rhs_code (loop_use_stmt)
+ || !flow_bb_inside_loop_p (loop, gimple_bb (loop_use_stmt)))
+ return false;
+
+ /* Insert USE_STMT into reduction chain. */
+ use_stmt_info = vinfo_for_stmt (loop_use_stmt);
+ if (current_stmt)
+ {
+ current_stmt_info = vinfo_for_stmt (current_stmt);
+ GROUP_NEXT_ELEMENT (current_stmt_info) = loop_use_stmt;
+ GROUP_FIRST_ELEMENT (use_stmt_info)
+ = GROUP_FIRST_ELEMENT (current_stmt_info);
+ }
+ else
+ GROUP_FIRST_ELEMENT (use_stmt_info) = loop_use_stmt;
+
+ lhs = gimple_assign_lhs (loop_use_stmt);
+ current_stmt = loop_use_stmt;
+ size++;
+ }
+
+ if (!found || loop_use_stmt != phi || size < 2)
+ return false;
+
+ /* Swap the operands, if needed, to make the reduction operand be the second
+ operand. */
+ lhs = PHI_RESULT (phi);
+ next_stmt = GROUP_FIRST_ELEMENT (vinfo_for_stmt (current_stmt));
+ while (next_stmt)
+ {
+ if (gimple_assign_rhs2 (next_stmt) == lhs)
+ {
+ tree op = gimple_assign_rhs1 (next_stmt);
+ gimple def_stmt = NULL;
+
+ if (TREE_CODE (op) == SSA_NAME)
+ def_stmt = SSA_NAME_DEF_STMT (op);
+
+ /* Check that the other def is either defined in the loop
+ ("vect_internal_def"), or it's an induction (defined by a
+ loop-header phi-node). */
+ if (def_stmt
+ && gimple_bb (def_stmt)
+ && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
+ && (is_gimple_assign (def_stmt)
+ || is_gimple_call (def_stmt)
+ || STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt))
+ == vect_induction_def
+ || (gimple_code (def_stmt) == GIMPLE_PHI
+ && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt))
+ == vect_internal_def
+ && !is_loop_header_bb_p (gimple_bb (def_stmt)))))
+ {
+ lhs = gimple_assign_lhs (next_stmt);
+ next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt));
+ continue;
+ }
+
+ return false;
+ }
+ else
+ {
+ tree op = gimple_assign_rhs2 (next_stmt);
+ gimple def_stmt = NULL;
+
+ if (TREE_CODE (op) == SSA_NAME)
+ def_stmt = SSA_NAME_DEF_STMT (op);
+
+ /* Check that the other def is either defined in the loop
+ ("vect_internal_def"), or it's an induction (defined by a
+ loop-header phi-node). */
+ if (def_stmt
+ && gimple_bb (def_stmt)
+ && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt))
+ && (is_gimple_assign (def_stmt)
+ || is_gimple_call (def_stmt)
+ || STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt))
+ == vect_induction_def
+ || (gimple_code (def_stmt) == GIMPLE_PHI
+ && STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def_stmt))
+ == vect_internal_def
+ && !is_loop_header_bb_p (gimple_bb (def_stmt)))))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ {
+ fprintf (vect_dump, "swapping oprnds: ");
+ print_gimple_stmt (vect_dump, next_stmt, 0, TDF_SLIM);
+ }
+
+ swap_tree_operands (next_stmt,
+ gimple_assign_rhs1_ptr (next_stmt),
+ gimple_assign_rhs2_ptr (next_stmt));
+ mark_symbols_for_renaming (next_stmt);
+ }
+ else
+ return false;
+ }
+
+ lhs = gimple_assign_lhs (next_stmt);
+ next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt));
+ }
+
+ /* Save the chain for further analysis in SLP detection. */
+ first = GROUP_FIRST_ELEMENT (vinfo_for_stmt (current_stmt));
+ VEC_safe_push (gimple, heap, LOOP_VINFO_REDUCTION_CHAINS (loop_info), first);
+ GROUP_SIZE (vinfo_for_stmt (first)) = size;
+
+ return true;
+}
+
+
+/* Function vect_is_simple_reduction_1
+
+ (1) Detect a cross-iteration def-use cycle that represents a simple
+ reduction computation. We look for the following pattern:
+
+ loop_header:
+ a1 = phi < a0, a2 >
+ a3 = ...
+ a2 = operation (a3, a1)
+
+ such that:
+ 1. operation is commutative and associative and it is safe to
+ change the order of the computation (if CHECK_REDUCTION is true)
+ 2. no uses for a2 in the loop (a2 is used out of the loop)
+ 3. no uses of a1 in the loop besides the reduction operation
+ 4. no uses of a1 outside the loop.
+
+ Conditions 1,4 are tested here.
+ Conditions 2,3 are tested in vect_mark_stmts_to_be_vectorized.
+
+ (2) Detect a cross-iteration def-use cycle in nested loops, i.e.,
+ nested cycles, if CHECK_REDUCTION is false.
+
+ (3) Detect cycles of phi nodes in outer-loop vectorization, i.e., double
+ reductions:
+
+ a1 = phi < a0, a2 >
+ inner loop (def of a3)
+ a2 = phi < a3 >
+
+ If MODIFY is true it tries also to rework the code in-place to enable
+ detection of more reduction patterns. For the time being we rewrite
+ "res -= RHS" into "rhs += -RHS" when it seems worthwhile.
+*/
+
+static gimple
+vect_is_simple_reduction_1 (loop_vec_info loop_info, gimple phi,
+ bool check_reduction, bool *double_reduc,
+ bool modify)
+{
+ struct loop *loop = (gimple_bb (phi))->loop_father;
+ struct loop *vect_loop = LOOP_VINFO_LOOP (loop_info);
+ edge latch_e = loop_latch_edge (loop);
+ tree loop_arg = PHI_ARG_DEF_FROM_EDGE (phi, latch_e);
+ gimple def_stmt, def1 = NULL, def2 = NULL;
+ enum tree_code orig_code, code;
+ tree op1, op2, op3 = NULL_TREE, op4 = NULL_TREE;
+ tree type;
+ int nloop_uses;
+ tree name;
+ imm_use_iterator imm_iter;
+ use_operand_p use_p;
+ bool phi_def;
+
+ *double_reduc = false;
+
+ /* If CHECK_REDUCTION is true, we assume inner-most loop vectorization,
+ otherwise, we assume outer loop vectorization. */
+ gcc_assert ((check_reduction && loop == vect_loop)
+ || (!check_reduction && flow_loop_nested_p (vect_loop, loop)));
+
+ name = PHI_RESULT (phi);
+ nloop_uses = 0;
+ FOR_EACH_IMM_USE_FAST (use_p, imm_iter, name)
+ {
+ gimple use_stmt = USE_STMT (use_p);