+
+/* Calculate the cost of the memory access represented by DR. */
+
+static void
+vect_get_data_access_cost (struct data_reference *dr,
+ unsigned int *inside_cost,
+ unsigned int *outside_cost)
+{
+ gimple stmt = DR_STMT (dr);
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ int nunits = TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info));
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ int ncopies = vf / nunits;
+ bool supportable_dr_alignment = vect_supportable_dr_alignment (dr, true);
+
+ if (!supportable_dr_alignment)
+ *inside_cost = VECT_MAX_COST;
+ else
+ {
+ if (DR_IS_READ (dr))
+ vect_get_load_cost (dr, ncopies, true, inside_cost, outside_cost);
+ else
+ vect_get_store_cost (dr, ncopies, inside_cost);
+ }
+
+ if (vect_print_dump_info (REPORT_COST))
+ fprintf (vect_dump, "vect_get_data_access_cost: inside_cost = %d, "
+ "outside_cost = %d.", *inside_cost, *outside_cost);
+}
+
+
+static hashval_t
+vect_peeling_hash (const void *elem)
+{
+ const struct _vect_peel_info *peel_info;
+
+ peel_info = (const struct _vect_peel_info *) elem;
+ return (hashval_t) peel_info->npeel;
+}
+
+
+static int
+vect_peeling_hash_eq (const void *elem1, const void *elem2)
+{
+ const struct _vect_peel_info *a, *b;
+
+ a = (const struct _vect_peel_info *) elem1;
+ b = (const struct _vect_peel_info *) elem2;
+ return (a->npeel == b->npeel);
+}
+
+
+/* Insert DR into peeling hash table with NPEEL as key. */
+
+static void
+vect_peeling_hash_insert (loop_vec_info loop_vinfo, struct data_reference *dr,
+ int npeel)
+{
+ struct _vect_peel_info elem, *slot;
+ void **new_slot;
+ bool supportable_dr_alignment = vect_supportable_dr_alignment (dr, true);
+
+ elem.npeel = npeel;
+ slot = (vect_peel_info) htab_find (LOOP_VINFO_PEELING_HTAB (loop_vinfo),
+ &elem);
+ if (slot)
+ slot->count++;
+ else
+ {
+ slot = XNEW (struct _vect_peel_info);
+ slot->npeel = npeel;
+ slot->dr = dr;
+ slot->count = 1;
+ new_slot = htab_find_slot (LOOP_VINFO_PEELING_HTAB (loop_vinfo), slot,
+ INSERT);
+ *new_slot = slot;
+ }
+
+ if (!supportable_dr_alignment && !flag_vect_cost_model)
+ slot->count += VECT_MAX_COST;
+}
+
+
+/* Traverse peeling hash table to find peeling option that aligns maximum
+ number of data accesses. */
+
+static int
+vect_peeling_hash_get_most_frequent (void **slot, void *data)
+{
+ vect_peel_info elem = (vect_peel_info) *slot;
+ vect_peel_extended_info max = (vect_peel_extended_info) data;
+
+ if (elem->count > max->peel_info.count
+ || (elem->count == max->peel_info.count
+ && max->peel_info.npeel > elem->npeel))
+ {
+ max->peel_info.npeel = elem->npeel;
+ max->peel_info.count = elem->count;
+ max->peel_info.dr = elem->dr;
+ }
+
+ return 1;
+}
+
+
+/* Traverse peeling hash table and calculate cost for each peeling option.
+ Find the one with the lowest cost. */
+
+static int
+vect_peeling_hash_get_lowest_cost (void **slot, void *data)
+{
+ vect_peel_info elem = (vect_peel_info) *slot;
+ vect_peel_extended_info min = (vect_peel_extended_info) data;
+ int save_misalignment, dummy;
+ unsigned int inside_cost = 0, outside_cost = 0, i;
+ gimple stmt = DR_STMT (elem->dr);
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info);
+ VEC (data_reference_p, heap) *datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
+ struct data_reference *dr;
+
+ FOR_EACH_VEC_ELT (data_reference_p, datarefs, i, dr)
+ {
+ stmt = DR_STMT (dr);
+ stmt_info = vinfo_for_stmt (stmt);
+ /* For interleaving, only the alignment of the first access
+ matters. */
+ if (STMT_VINFO_STRIDED_ACCESS (stmt_info)
+ && GROUP_FIRST_ELEMENT (stmt_info) != stmt)
+ continue;
+
+ save_misalignment = DR_MISALIGNMENT (dr);
+ vect_update_misalignment_for_peel (dr, elem->dr, elem->npeel);
+ vect_get_data_access_cost (dr, &inside_cost, &outside_cost);
+ SET_DR_MISALIGNMENT (dr, save_misalignment);
+ }
+
+ outside_cost += vect_get_known_peeling_cost (loop_vinfo, elem->npeel, &dummy,
+ vect_get_single_scalar_iteration_cost (loop_vinfo));
+
+ if (inside_cost < min->inside_cost
+ || (inside_cost == min->inside_cost && outside_cost < min->outside_cost))
+ {
+ min->inside_cost = inside_cost;
+ min->outside_cost = outside_cost;
+ min->peel_info.dr = elem->dr;
+ min->peel_info.npeel = elem->npeel;
+ }
+
+ return 1;
+}
+
+
+/* Choose best peeling option by traversing peeling hash table and either
+ choosing an option with the lowest cost (if cost model is enabled) or the
+ option that aligns as many accesses as possible. */
+
+static struct data_reference *
+vect_peeling_hash_choose_best_peeling (loop_vec_info loop_vinfo,
+ unsigned int *npeel)
+{
+ struct _vect_peel_extended_info res;
+
+ res.peel_info.dr = NULL;
+
+ if (flag_vect_cost_model)
+ {
+ res.inside_cost = INT_MAX;
+ res.outside_cost = INT_MAX;
+ htab_traverse (LOOP_VINFO_PEELING_HTAB (loop_vinfo),
+ vect_peeling_hash_get_lowest_cost, &res);
+ }
+ else
+ {
+ res.peel_info.count = 0;
+ htab_traverse (LOOP_VINFO_PEELING_HTAB (loop_vinfo),
+ vect_peeling_hash_get_most_frequent, &res);
+ }
+
+ *npeel = res.peel_info.npeel;
+ return res.peel_info.dr;
+}
+
+