{
/* State of the upper 128bits of any AVX registers at exit. */
enum upper_128bits_state state;
- /* If the upper 128bits of any AVX registers are referenced. */
- enum upper_128bits_state referenced;
- /* Number of vzerouppers in this block. */
- unsigned int count;
/* TRUE if block has been processed. */
bool processed;
- /* TRUE if block has been rescanned. */
- bool rescanned;
} *block_info;
#define BLOCK_INFO(B) ((block_info) (B)->aux)
rtx vzeroupper_insn = NULL_RTX;
rtx pat;
int avx256;
- enum upper_128bits_state referenced = BLOCK_INFO (bb)->referenced;
- int count = BLOCK_INFO (bb)->count;
if (dump_file)
fprintf (dump_file, " [bb %i] entry: upper 128bits: %d\n",
/* Delete pending vzeroupper insertion. */
if (vzeroupper_insn)
{
- count--;
delete_insn (vzeroupper_insn);
vzeroupper_insn = NULL_RTX;
}
}
- else if (state != used && referenced != unused)
+ else if (state != used)
{
/* No need to call note_stores if the upper 128bits of
AVX registers are never referenced. */
note_stores (pat, check_avx256_stores, &state);
- if (state == used)
- referenced = used;
}
continue;
}
/* Process vzeroupper intrinsic. */
- count++;
avx256 = INTVAL (XVECEXP (pat, 0, 0));
if (state == unused)
fprintf (dump_file, "Delete redundant vzeroupper:\n");
print_rtl_single (dump_file, insn);
}
- count--;
delete_insn (insn);
}
else
fprintf (dump_file, "Delete callee pass vzeroupper:\n");
print_rtl_single (dump_file, insn);
}
- count--;
delete_insn (insn);
}
else
BLOCK_INFO (bb)->state = state;
- if (BLOCK_INFO (bb)->referenced == unknown)
- {
- /* The upper 128bits of AVX registers are never referenced if
- REFERENCED isn't updated. */
- if (referenced == unknown)
- referenced = unused;
- BLOCK_INFO (bb)->referenced = referenced;
- BLOCK_INFO (bb)->count = count;
- }
-
if (dump_file)
fprintf (dump_file, " [bb %i] exit: upper 128bits: %d\n",
bb->index, state);
}
/* Helper function for move_or_delete_vzeroupper. Process vzeroupper
- in BLOCK and its predecessor blocks recursively. */
+ in BLOCK and check its predecessor blocks. Treat UNKNOWN state
+ as USED if UNKNOWN_IS_UNUSED is true. */
static void
-move_or_delete_vzeroupper_1 (basic_block block)
+move_or_delete_vzeroupper_1 (basic_block block, bool unknown_is_unused)
{
edge e;
edge_iterator ei;
- enum upper_128bits_state state;
+ enum upper_128bits_state state, old_state, new_state;
+ bool seen_unknown;
if (dump_file)
fprintf (dump_file, " Process [bb %i]: status: %d\n",
if (BLOCK_INFO (block)->processed)
return;
- BLOCK_INFO (block)->processed = true;
-
- state = unknown;
+ state = unused;
- /* Process all predecessor edges of this block. */
+ /* Check all predecessor edges of this block. */
+ seen_unknown = false;
FOR_EACH_EDGE (e, ei, block->preds)
{
if (e->src == block)
continue;
- move_or_delete_vzeroupper_1 (e->src);
switch (BLOCK_INFO (e->src)->state)
{
case unknown:
- if (state == unused)
- state = unknown;
+ if (!unknown_is_unused)
+ seen_unknown = true;
+ case unused:
break;
case used:
state = used;
- break;
- case unused:
- break;
+ goto done;
}
}
- /* If state of any predecessor edges is unknown, we need to rescan. */
- if (state == unknown)
- cfun->machine->rescan_vzeroupper_p = 1;
+ if (seen_unknown)
+ state = unknown;
- /* Process this block. */
+done:
+ old_state = BLOCK_INFO (block)->state;
move_or_delete_vzeroupper_2 (block, state);
-}
-
-/* Helper function for move_or_delete_vzeroupper. Rescan vzeroupper
- in BLOCK and its predecessor blocks recursively. */
-
-static void
-rescan_move_or_delete_vzeroupper (basic_block block)
-{
- edge e;
- edge_iterator ei;
- enum upper_128bits_state state;
-
- if (dump_file)
- fprintf (dump_file, " Rescan [bb %i]: status: %d\n",
- block->index, BLOCK_INFO (block)->rescanned);
-
- if (BLOCK_INFO (block)->rescanned)
- return;
-
- BLOCK_INFO (block)->rescanned = true;
-
- state = unused;
+ new_state = BLOCK_INFO (block)->state;
- /* Rescan all predecessor edges of this block. */
- FOR_EACH_EDGE (e, ei, block->preds)
- {
- if (e->src == block)
- continue;
- rescan_move_or_delete_vzeroupper (e->src);
- /* For rescan, UKKNOWN state is treated as UNUSED. */
- if (BLOCK_INFO (e->src)->state == used)
- state = used;
- }
+ if (state != unknown || new_state == used)
+ BLOCK_INFO (block)->processed = true;
- /* Rescan this block only if there are vzerouppers or the upper
- 128bits of AVX registers are referenced. */
- if (BLOCK_INFO (block)->count == 0
- && (state == used || BLOCK_INFO (block)->referenced != used))
- {
- if (state == used)
- BLOCK_INFO (block)->state = state;
- if (dump_file)
- fprintf (dump_file, " [bb %i] exit: upper 128bits: %d\n",
- block->index, BLOCK_INFO (block)->state);
- }
- else
- move_or_delete_vzeroupper_2 (block, state);
+ /* Need to rescan if the upper 128bits of AVX registers are changed
+ to USED at exit. */
+ if (new_state != old_state && new_state == used)
+ cfun->machine->rescan_vzeroupper_p = 1;
}
/* Go through the instruction stream looking for vzeroupper. Delete
edge e;
edge_iterator ei;
basic_block bb;
- unsigned int count = 0;
+ unsigned int count;
/* Set up block info for each basic block. */
alloc_aux_for_blocks (sizeof (struct block_info_def));
cfun->machine->caller_pass_avx256_p
? used : unused);
BLOCK_INFO (e->dest)->processed = true;
- BLOCK_INFO (e->dest)->rescanned = true;
}
/* Process all basic blocks. */
- if (dump_file)
- fprintf (dump_file, "Process all basic blocks\n");
-
- FOR_EACH_BB (bb)
- {
- move_or_delete_vzeroupper_1 (bb);
- count += BLOCK_INFO (bb)->count;
- }
-
- /* Rescan all basic blocks if needed. */
- if (count && cfun->machine->rescan_vzeroupper_p)
+ count = 0;
+ do
{
if (dump_file)
- fprintf (dump_file, "Rescan all basic blocks\n");
-
+ fprintf (dump_file, "Process all basic blocks: trip %d\n",
+ count);
+ cfun->machine->rescan_vzeroupper_p = 0;
FOR_EACH_BB (bb)
- rescan_move_or_delete_vzeroupper (bb);
+ move_or_delete_vzeroupper_1 (bb, false);
}
+ while (cfun->machine->rescan_vzeroupper_p && count++ < 20);
+
+ /* FIXME: Is 20 big enough? */
+ if (count >= 20)
+ gcc_unreachable ();
+
+ if (dump_file)
+ fprintf (dump_file, "Process all basic blocks\n");
+
+ FOR_EACH_BB (bb)
+ move_or_delete_vzeroupper_1 (bb, true);
free_aux_for_blocks ();
}
OSDN Git Service
