#include "cselib.h"
#include "debug.h"
#include "dwarf2out.h"
+#include "sched-int.h"
+
+typedef struct block_info_def
+{
+ /* TRUE if the upper 128bits of any AVX registers are live at exit. */
+ bool upper_128bits_set;
+ /* TRUE if block has been processed. */
+ bool done;
+} *block_info;
+
+#define BLOCK_INFO(B) ((block_info) (B)->aux)
+
+enum call_avx256_state
+{
+ /* Callee returns 256bit AVX register. */
+ callee_return_avx256 = -1,
+ /* Callee returns and passes 256bit AVX register. */
+ callee_return_pass_avx256,
+ /* Callee passes 256bit AVX register. */
+ callee_pass_avx256,
+ /* Callee doesn't return nor passe 256bit AVX register, or no
+ 256bit AVX register in function return. */
+ call_no_avx256,
+ /* vzeroupper intrinsic. */
+ vzeroupper_intrinsic
+};
+
+/* Check if a 256bit AVX register is referenced in stores. */
+
+static void
+check_avx256_stores (rtx dest, const_rtx set, void *data)
+{
+ if ((REG_P (dest)
+ && VALID_AVX256_REG_MODE (GET_MODE (dest)))
+ || (GET_CODE (set) == SET
+ && REG_P (SET_SRC (set))
+ && VALID_AVX256_REG_MODE (GET_MODE (SET_SRC (set)))))
+ {
+ bool *upper_128bits_set = (bool *) data;
+ *upper_128bits_set = true;
+ }
+}
+
+/* Helper function for move_or_delete_vzeroupper_1. Look for vzeroupper
+ in basic block BB. Delete it if upper 128bit AVX registers are
+ unused. If it isn't deleted, move it to just before a jump insn.
+
+ UPPER_128BITS_LIVE is TRUE if the upper 128bits of any AVX registers
+ are live at entry. */
+
+static void
+move_or_delete_vzeroupper_2 (basic_block bb, bool upper_128bits_set)
+{
+ rtx insn;
+ rtx vzeroupper_insn = NULL_RTX;
+ rtx pat;
+ int avx256;
+
+ if (dump_file)
+ fprintf (dump_file, " BB [%i] entry: upper 128bits: %d\n",
+ bb->index, upper_128bits_set);
+
+ insn = BB_HEAD (bb);
+ while (insn != BB_END (bb))
+ {
+ insn = NEXT_INSN (insn);
+
+ if (!NONDEBUG_INSN_P (insn))
+ continue;
+
+ /* Move vzeroupper before jump/call. */
+ if (JUMP_P (insn) || CALL_P (insn))
+ {
+ if (!vzeroupper_insn)
+ continue;
+
+ if (PREV_INSN (insn) != vzeroupper_insn)
+ {
+ if (dump_file)
+ {
+ fprintf (dump_file, "Move vzeroupper after:\n");
+ print_rtl_single (dump_file, PREV_INSN (insn));
+ fprintf (dump_file, "before:\n");
+ print_rtl_single (dump_file, insn);
+ }
+ reorder_insns_nobb (vzeroupper_insn, vzeroupper_insn,
+ PREV_INSN (insn));
+ }
+ vzeroupper_insn = NULL_RTX;
+ continue;
+ }
+
+ pat = PATTERN (insn);
+
+ /* Check insn for vzeroupper intrinsic. */
+ if (GET_CODE (pat) == UNSPEC_VOLATILE
+ && XINT (pat, 1) == UNSPECV_VZEROUPPER)
+ {
+ if (dump_file)
+ {
+ /* Found vzeroupper intrinsic. */
+ fprintf (dump_file, "Found vzeroupper:\n");
+ print_rtl_single (dump_file, insn);
+ }
+ }
+ else
+ {
+ /* Check insn for vzeroall intrinsic. */
+ if (GET_CODE (pat) == PARALLEL
+ && GET_CODE (XVECEXP (pat, 0, 0)) == UNSPEC_VOLATILE
+ && XINT (XVECEXP (pat, 0, 0), 1) == UNSPECV_VZEROALL)
+ {
+ upper_128bits_set = false;
+
+ /* Delete pending vzeroupper insertion. */
+ if (vzeroupper_insn)
+ {
+ delete_insn (vzeroupper_insn);
+ vzeroupper_insn = NULL_RTX;
+ }
+ }
+ else if (!upper_128bits_set)
+ note_stores (pat, check_avx256_stores, &upper_128bits_set);
+ continue;
+ }
+
+ /* Process vzeroupper intrinsic. */
+ avx256 = INTVAL (XVECEXP (pat, 0, 0));
+
+ if (!upper_128bits_set)
+ {
+ /* Since the upper 128bits are cleared, callee must not pass
+ 256bit AVX register. We only need to check if callee
+ returns 256bit AVX register. */
+ upper_128bits_set = (avx256 == callee_return_avx256);
+
+ /* Remove unnecessary vzeroupper since
+ upper 128bits are cleared. */
+ if (dump_file)
+ {
+ fprintf (dump_file, "Delete redundant vzeroupper:\n");
+ print_rtl_single (dump_file, insn);
+ }
+ delete_insn (insn);
+ }
+ else if (avx256 == callee_return_pass_avx256
+ || avx256 == callee_pass_avx256)
+ {
+ /* Callee passes 256bit AVX register. Check if callee
+ returns 256bit AVX register. */
+ upper_128bits_set = (avx256 == callee_return_pass_avx256);
+
+ /* Must remove vzeroupper since
+ callee passes in 256bit AVX register. */
+ if (dump_file)
+ {
+ fprintf (dump_file, "Delete callee pass vzeroupper:\n");
+ print_rtl_single (dump_file, insn);
+ }
+ delete_insn (insn);
+ }
+ else
+ {
+ upper_128bits_set = false;
+ vzeroupper_insn = insn;
+ }
+ }
+
+ BLOCK_INFO (bb)->upper_128bits_set = upper_128bits_set;
+
+ if (dump_file)
+ fprintf (dump_file, " BB [%i] exit: upper 128bits: %d\n",
+ bb->index, upper_128bits_set);
+}
+
+/* Helper function for move_or_delete_vzeroupper. Process vzeroupper
+ in BLOCK and its predecessor blocks recursively. */
+
+static void
+move_or_delete_vzeroupper_1 (basic_block block)
+{
+ edge e;
+ edge_iterator ei;
+ bool upper_128bits_set;
+
+ if (dump_file)
+ fprintf (dump_file, " Process BB [%i]: status: %d\n",
+ block->index, BLOCK_INFO (block)->done);
+
+ if (BLOCK_INFO (block)->done)
+ return;
+
+ BLOCK_INFO (block)->done = true;
+
+ upper_128bits_set = false;
+
+ /* Process all predecessor edges of this block. */
+ FOR_EACH_EDGE (e, ei, block->preds)
+ {
+ if (e->src == block)
+ continue;
+ move_or_delete_vzeroupper_1 (e->src);
+ if (BLOCK_INFO (e->src)->upper_128bits_set)
+ upper_128bits_set = true;
+ }
+
+ /* Process this block. */
+ move_or_delete_vzeroupper_2 (block, upper_128bits_set);
+}
+
+/* Go through the instruction stream looking for vzeroupper. Delete
+ it if upper 128bit AVX registers are unused. If it isn't deleted,
+ move it to just before a jump insn. */
+
+static void
+move_or_delete_vzeroupper (void)
+{
+ edge e;
+ edge_iterator ei;
+
+ /* Set up block info for each basic block. */
+ alloc_aux_for_blocks (sizeof (struct block_info_def));
+
+ /* Process successor blocks of all entry points. */
+ if (dump_file)
+ fprintf (dump_file, "Process all entry points\n");
+
+ FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR->succs)
+ {
+ move_or_delete_vzeroupper_2 (e->dest,
+ cfun->machine->caller_pass_avx256_p);
+ BLOCK_INFO (e->dest)->done = true;
+ }
+
+ /* Process predecessor blocks of all exit points. */
+ if (dump_file)
+ fprintf (dump_file, "Process all exit points\n");
+
+ FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR->preds)
+ move_or_delete_vzeroupper_1 (e->src);
+
+ free_aux_for_blocks ();
+}
static rtx legitimize_dllimport_symbol (rtx, bool);
struct processor_costs bdver1_cost = {
COSTS_N_INSNS (1), /* cost of an add instruction */
- COSTS_N_INSNS (2), /* cost of a lea instruction */
+ COSTS_N_INSNS (1), /* cost of a lea instruction */
COSTS_N_INSNS (1), /* variable shift costs */
COSTS_N_INSNS (1), /* constant shift costs */
- {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
+ {COSTS_N_INSNS (4), /* cost of starting multiply for QI */
COSTS_N_INSNS (4), /* HI */
- COSTS_N_INSNS (3), /* SI */
- COSTS_N_INSNS (4), /* DI */
- COSTS_N_INSNS (5)}, /* other */
+ COSTS_N_INSNS (4), /* SI */
+ COSTS_N_INSNS (6), /* DI */
+ COSTS_N_INSNS (6)}, /* other */
0, /* cost of multiply per each bit set */
{COSTS_N_INSNS (19), /* cost of a divide/mod for QI */
COSTS_N_INSNS (35), /* HI */
8, /* "large" insn */
9, /* MOVE_RATIO */
4, /* cost for loading QImode using movzbl */
- {3, 4, 3}, /* cost of loading integer registers
+ {5, 5, 4}, /* cost of loading integer registers
in QImode, HImode and SImode.
Relative to reg-reg move (2). */
- {3, 4, 3}, /* cost of storing integer registers */
- 4, /* cost of reg,reg fld/fst */
- {4, 4, 12}, /* cost of loading fp registers
+ {4, 4, 4}, /* cost of storing integer registers */
+ 2, /* cost of reg,reg fld/fst */
+ {5, 5, 12}, /* cost of loading fp registers
in SFmode, DFmode and XFmode */
- {6, 6, 8}, /* cost of storing fp registers
+ {4, 4, 8}, /* cost of storing fp registers
in SFmode, DFmode and XFmode */
2, /* cost of moving MMX register */
- {3, 3}, /* cost of loading MMX registers
+ {4, 4}, /* cost of loading MMX registers
in SImode and DImode */
{4, 4}, /* cost of storing MMX registers
in SImode and DImode */
2, /* cost of moving SSE register */
- {4, 4, 3}, /* cost of loading SSE registers
+ {4, 4, 4}, /* cost of loading SSE registers
in SImode, DImode and TImode */
- {4, 4, 5}, /* cost of storing SSE registers
+ {4, 4, 4}, /* cost of storing SSE registers
in SImode, DImode and TImode */
- 3, /* MMX or SSE register to integer */
+ 2, /* MMX or SSE register to integer */
/* On K8:
MOVD reg64, xmmreg Double FSTORE 4
MOVD reg32, xmmreg Double FSTORE 4
1/1 1/1
MOVD reg32, xmmreg Double FADD 3
1/1 1/1 */
- 64, /* size of l1 cache. */
- 1024, /* size of l2 cache. */
+ 16, /* size of l1 cache. */
+ 2048, /* size of l2 cache. */
64, /* size of prefetch block */
/* New AMD processors never drop prefetches; if they cannot be performed
immediately, they are queued. We set number of simultaneous prefetches
time). */
100, /* number of parallel prefetches */
2, /* Branch cost */
- COSTS_N_INSNS (4), /* cost of FADD and FSUB insns. */
- COSTS_N_INSNS (4), /* cost of FMUL instruction. */
- COSTS_N_INSNS (19), /* cost of FDIV instruction. */
+ COSTS_N_INSNS (6), /* cost of FADD and FSUB insns. */
+ COSTS_N_INSNS (6), /* cost of FMUL instruction. */
+ COSTS_N_INSNS (42), /* cost of FDIV instruction. */
COSTS_N_INSNS (2), /* cost of FABS instruction. */
COSTS_N_INSNS (2), /* cost of FCHS instruction. */
- COSTS_N_INSNS (35), /* cost of FSQRT instruction. */
+ COSTS_N_INSNS (52), /* cost of FSQRT instruction. */
/* BDVER1 has optimized REP instruction for medium sized blocks, but for
very small blocks it is better to use loop. For large blocks, libcall
{{libcall, {{8, loop}, {24, unrolled_loop},
{2048, rep_prefix_4_byte}, {-1, libcall}}},
{libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
- 4, /* scalar_stmt_cost. */
- 2, /* scalar load_cost. */
- 2, /* scalar_store_cost. */
+ 6, /* scalar_stmt_cost. */
+ 4, /* scalar load_cost. */
+ 4, /* scalar_store_cost. */
6, /* vec_stmt_cost. */
0, /* vec_to_scalar_cost. */
2, /* scalar_to_vec_cost. */
- 2, /* vec_align_load_cost. */
- 2, /* vec_unalign_load_cost. */
- 2, /* vec_store_cost. */
+ 4, /* vec_align_load_cost. */
+ 4, /* vec_unalign_load_cost. */
+ 4, /* vec_store_cost. */
2, /* cond_taken_branch_cost. */
1, /* cond_not_taken_branch_cost. */
};
#define m_PENT4 (1<<PROCESSOR_PENTIUM4)
#define m_NOCONA (1<<PROCESSOR_NOCONA)
#define m_CORE2 (1<<PROCESSOR_CORE2)
+#define m_COREI7_32 (1<<PROCESSOR_COREI7_32)
+#define m_COREI7_64 (1<<PROCESSOR_COREI7_64)
#define m_ATOM (1<<PROCESSOR_ATOM)
#define m_GEODE (1<<PROCESSOR_GEODE)
#define m_BDVER1 (1<<PROCESSOR_BDVER1)
#define m_AMD_MULTIPLE (m_K8 | m_ATHLON | m_AMDFAM10 | m_BDVER1)
-#define m_GENERIC32 (1<<PROCESSOR_GENERIC32)
-#define m_GENERIC64 (1<<PROCESSOR_GENERIC64)
+#define m_GENERIC32 (1<<PROCESSOR_GENERIC32 | m_COREI7_32)
+#define m_GENERIC64 (1<<PROCESSOR_GENERIC64 | m_COREI7_64)
/* Generic instruction choice should be common subset of supported CPUs
(PPro/PENT4/NOCONA/CORE2/Athlon/K8). */
/* X86_TUNE_PAD_RETURNS */
m_AMD_MULTIPLE | m_CORE2 | m_GENERIC,
+ /* X86_TUNE_PAD_SHORT_FUNCTION: Pad short funtion. */
+ m_ATOM,
+
/* X86_TUNE_EXT_80387_CONSTANTS */
m_K6_GEODE | m_ATHLON_K8 | m_ATOM | m_PENT4 | m_NOCONA | m_PPRO
| m_CORE2 | m_GENERIC,
/* X86_TUNE_OPT_AGU: Optimize for Address Generation Unit. This flag
will impact LEA instruction selection. */
m_ATOM,
+
+ /* X86_TUNE_VECTORIZE_DOUBLE: Enable double precision vector
+ instructions. */
+ ~m_ATOM,
};
/* Feature tests against the various architecture variations. */
HOST_WIDE_INT frame_pointer_offset;
HOST_WIDE_INT hard_frame_pointer_offset;
HOST_WIDE_INT stack_pointer_offset;
+ HOST_WIDE_INT hfp_save_offset;
HOST_WIDE_INT reg_save_offset;
HOST_WIDE_INT sse_reg_save_offset;
static int ix86_regparm;
/* -mstackrealign option */
-extern int ix86_force_align_arg_pointer;
static const char ix86_force_align_arg_pointer_string[]
= "force_align_arg_pointer";
static void ix86_add_new_builtins (int);
static rtx ix86_expand_vec_perm_builtin (tree);
static tree ix86_canonical_va_list_type (tree);
+static void predict_jump (int);
+static unsigned int split_stack_prologue_scratch_regno (void);
+static bool i386_asm_output_addr_const_extra (FILE *, rtx);
enum ix86_function_specific_strings
{
static int ix86_tune_defaulted;
static int ix86_arch_specified;
-/* Bit flags that specify the ISA we are compiling for. */
-int ix86_isa_flags = TARGET_64BIT_DEFAULT | TARGET_SUBTARGET_ISA_DEFAULT;
-
/* A mask of ix86_isa_flags that includes bit X if X
was set or cleared on the command line. */
static int ix86_isa_flags_explicit;
{&k8_cost, 16, 7, 16, 7, 16},
{&nocona_cost, 0, 0, 0, 0, 0},
{&core2_cost, 16, 10, 16, 10, 16},
+ /* Core i7 32-bit. */
+ {&generic32_cost, 16, 10, 16, 10, 16},
+ /* Core i7 64-bit. */
+ {&generic64_cost, 16, 10, 16, 10, 16},
{&generic32_cost, 16, 7, 16, 7, 16},
{&generic64_cost, 16, 10, 16, 10, 16},
{&amdfam10_cost, 32, 24, 32, 7, 32},
"prescott",
"nocona",
"core2",
+ "corei7",
"atom",
"geode",
"k6",
{ "-msseregparm", MASK_SSEREGPARM },
{ "-mstack-arg-probe", MASK_STACK_PROBE },
{ "-mtls-direct-seg-refs", MASK_TLS_DIRECT_SEG_REFS },
+ { "-mvect8-ret-in-mem", MASK_VECT8_RETURNS },
+ { "-m8bit-idiv", MASK_USE_8BIT_IDIV },
+ { "-mvzeroupper", MASK_VZEROUPPER },
};
const char *opts[ARRAY_SIZE (isa_opts) + ARRAY_SIZE (flag_opts) + 6][2];
return;
}
\f
-/* Sometimes certain combinations of command options do not make
- sense on a particular target machine. You can define a macro
- `OVERRIDE_OPTIONS' to take account of this. This macro, if
- defined, is executed once just after all the command options have
- been parsed.
-
- Don't use this macro to turn on various extra optimizations for
- `-O'. That is what `OPTIMIZATION_OPTIONS' is for. */
+/* Override various settings based on options. If MAIN_ARGS_P, the
+ options are from the command line, otherwise they are from
+ attributes. */
-void
-override_options (bool main_args_p)
+static void
+ix86_option_override_internal (bool main_args_p)
{
int i;
unsigned int ix86_arch_mask, ix86_tune_mask;
{"core2", PROCESSOR_CORE2, CPU_CORE2,
PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
| PTA_SSSE3 | PTA_CX16},
+ {"corei7", PROCESSOR_COREI7_64, CPU_GENERIC64,
+ PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
+ | PTA_SSSE3 | PTA_SSE4_1 | PTA_SSE4_2 | PTA_CX16},
{"atom", PROCESSOR_ATOM, CPU_ATOM,
PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
| PTA_SSSE3 | PTA_CX16 | PTA_MOVBE},
{
ix86_schedule = processor_alias_table[i].schedule;
ix86_tune = processor_alias_table[i].processor;
- if (TARGET_64BIT && !(processor_alias_table[i].flags & PTA_64BIT))
+ if (TARGET_64BIT)
{
- if (ix86_tune_defaulted)
+ if (!(processor_alias_table[i].flags & PTA_64BIT))
{
- ix86_tune_string = "x86-64";
- for (i = 0; i < pta_size; i++)
- if (! strcmp (ix86_tune_string,
- processor_alias_table[i].name))
- break;
- ix86_schedule = processor_alias_table[i].schedule;
- ix86_tune = processor_alias_table[i].processor;
+ if (ix86_tune_defaulted)
+ {
+ ix86_tune_string = "x86-64";
+ for (i = 0; i < pta_size; i++)
+ if (! strcmp (ix86_tune_string,
+ processor_alias_table[i].name))
+ break;
+ ix86_schedule = processor_alias_table[i].schedule;
+ ix86_tune = processor_alias_table[i].processor;
+ }
+ else
+ error ("CPU you selected does not support x86-64 "
+ "instruction set");
+ }
+ }
+ else
+ {
+ /* Adjust tuning when compiling for 32-bit ABI. */
+ switch (ix86_tune)
+ {
+ case PROCESSOR_GENERIC64:
+ ix86_tune = PROCESSOR_GENERIC32;
+ ix86_schedule = CPU_PENTIUMPRO;
+ break;
+
+ case PROCESSOR_COREI7_64:
+ ix86_tune = PROCESSOR_COREI7_32;
+ ix86_schedule = CPU_PENTIUMPRO;
+ break;
+
+ default:
+ break;
}
- else
- error ("CPU you selected does not support x86-64 "
- "instruction set");
}
- /* Intel CPUs have always interpreted SSE prefetch instructions as
+ /* Intel CPUs have always interpreted SSE prefetch instructions as
NOPs; so, we can enable SSE prefetch instructions even when
-mtune (rather than -march) points us to a processor that has them.
However, the VIA C3 gives a SIGILL, so we only do that for i686 and
#define USE_IX86_FRAME_POINTER 0
#endif
+#ifndef USE_X86_64_FRAME_POINTER
+#define USE_X86_64_FRAME_POINTER 0
+#endif
+
/* Set the default values for switches whose default depends on TARGET_64BIT
in case they weren't overwritten by command line options. */
if (TARGET_64BIT)
{
- if (flag_zee == 2)
+ if (optimize > 1 && !global_options_set.x_flag_zee)
flag_zee = 1;
- if (flag_omit_frame_pointer == 2)
- flag_omit_frame_pointer = 1;
+ if (optimize >= 1 && !global_options_set.x_flag_omit_frame_pointer)
+ flag_omit_frame_pointer = !USE_X86_64_FRAME_POINTER;
if (flag_asynchronous_unwind_tables == 2)
- flag_asynchronous_unwind_tables = 1;
+ flag_unwind_tables = flag_asynchronous_unwind_tables = 1;
if (flag_pcc_struct_return == 2)
flag_pcc_struct_return = 0;
}
else
{
- if (flag_zee == 2)
- flag_zee = 0;
- if (flag_omit_frame_pointer == 2)
+ if (optimize >= 1 && !global_options_set.x_flag_omit_frame_pointer)
flag_omit_frame_pointer = !(USE_IX86_FRAME_POINTER || optimize_size);
if (flag_asynchronous_unwind_tables == 2)
flag_asynchronous_unwind_tables = !USE_IX86_FRAME_POINTER;
ix86_preferred_stack_boundary = PREFERRED_STACK_BOUNDARY_DEFAULT;
if (ix86_preferred_stack_boundary_string)
{
+ int min = (TARGET_64BIT ? 4 : 2);
+ int max = (TARGET_SEH ? 4 : 12);
+
i = atoi (ix86_preferred_stack_boundary_string);
- if (i < (TARGET_64BIT ? 4 : 2) || i > 12)
- error ("%spreferred-stack-boundary=%d%s is not between %d and 12",
- prefix, i, suffix, TARGET_64BIT ? 4 : 2);
+ if (i < min || i > max)
+ {
+ if (min == max)
+ error ("%spreferred-stack-boundary%s is not supported "
+ "for this target", prefix, suffix);
+ else
+ error ("%spreferred-stack-boundary=%d%s is not between %d and %d",
+ prefix, i, suffix, min, max);
+ }
else
ix86_preferred_stack_boundary = (1 << i) * BITS_PER_UNIT;
}
if (!TARGET_SCHEDULE)
flag_schedule_insns_after_reload = flag_schedule_insns = 0;
- if (!PARAM_SET_P (PARAM_SIMULTANEOUS_PREFETCHES))
- set_param_value ("simultaneous-prefetches",
- ix86_cost->simultaneous_prefetches);
- if (!PARAM_SET_P (PARAM_L1_CACHE_LINE_SIZE))
- set_param_value ("l1-cache-line-size", ix86_cost->prefetch_block);
- if (!PARAM_SET_P (PARAM_L1_CACHE_SIZE))
- set_param_value ("l1-cache-size", ix86_cost->l1_cache_size);
- if (!PARAM_SET_P (PARAM_L2_CACHE_SIZE))
- set_param_value ("l2-cache-size", ix86_cost->l2_cache_size);
+ maybe_set_param_value (PARAM_SIMULTANEOUS_PREFETCHES,
+ ix86_cost->simultaneous_prefetches,
+ global_options.x_param_values,
+ global_options_set.x_param_values);
+ maybe_set_param_value (PARAM_L1_CACHE_LINE_SIZE, ix86_cost->prefetch_block,
+ global_options.x_param_values,
+ global_options_set.x_param_values);
+ maybe_set_param_value (PARAM_L1_CACHE_SIZE, ix86_cost->l1_cache_size,
+ global_options.x_param_values,
+ global_options_set.x_param_values);
+ maybe_set_param_value (PARAM_L2_CACHE_SIZE, ix86_cost->l2_cache_size,
+ global_options.x_param_values,
+ global_options_set.x_param_values);
/* Enable sw prefetching at -O3 for CPUS that prefetching is helpful. */
if (flag_prefetch_loop_arrays < 0
/* If using typedef char *va_list, signal that __builtin_va_start (&ap, 0)
can be optimized to ap = __builtin_next_arg (0). */
- if (!TARGET_64BIT)
+ if (!TARGET_64BIT && !flag_split_stack)
targetm.expand_builtin_va_start = NULL;
if (TARGET_64BIT)
ix86_gen_one_cmpl2 = gen_one_cmpldi2;
ix86_gen_monitor = gen_sse3_monitor64;
ix86_gen_andsp = gen_anddi3;
- ix86_gen_allocate_stack_worker = gen_allocate_stack_worker_64;
+ ix86_gen_allocate_stack_worker = gen_allocate_stack_worker_probe_di;
ix86_gen_adjust_stack_and_probe = gen_adjust_stack_and_probedi;
ix86_gen_probe_stack_range = gen_probe_stack_rangedi;
}
ix86_gen_one_cmpl2 = gen_one_cmplsi2;
ix86_gen_monitor = gen_sse3_monitor;
ix86_gen_andsp = gen_andsi3;
- ix86_gen_allocate_stack_worker = gen_allocate_stack_worker_32;
+ ix86_gen_allocate_stack_worker = gen_allocate_stack_worker_probe_si;
ix86_gen_adjust_stack_and_probe = gen_adjust_stack_and_probesi;
ix86_gen_probe_stack_range = gen_probe_stack_rangesi;
}
sorry ("-mfentry isn't supported for 32-bit in combination with -fpic");
flag_fentry = 0;
}
- if (flag_fentry < 0)
+ else if (TARGET_SEH)
+ {
+ if (flag_fentry == 0)
+ sorry ("-mno-fentry isn't compatible with SEH");
+ flag_fentry = 1;
+ }
+ else if (flag_fentry < 0)
{
#if defined(PROFILE_BEFORE_PROLOGUE)
flag_fentry = 1;
if (main_args_p)
target_option_default_node = target_option_current_node
= build_target_option_node ();
+
+ if (TARGET_AVX)
+ {
+ /* Enable vzeroupper pass by default for TARGET_AVX. */
+ if (!(target_flags_explicit & MASK_VZEROUPPER))
+ target_flags |= MASK_VZEROUPPER;
+ }
+ else
+ {
+ /* Disable vzeroupper pass if TARGET_AVX is disabled. */
+ target_flags &= ~MASK_VZEROUPPER;
+ }
+}
+
+/* Return TRUE if type TYPE and mode MODE use 256bit AVX modes. */
+
+static bool
+use_avx256_p (enum machine_mode mode, const_tree type)
+{
+ return (VALID_AVX256_REG_MODE (mode)
+ || (type
+ && TREE_CODE (type) == VECTOR_TYPE
+ && int_size_in_bytes (type) == 32));
+}
+
+/* Return TRUE if VAL is passed in register with 256bit AVX modes. */
+
+static bool
+function_pass_avx256_p (const_rtx val)
+{
+ if (!val)
+ return false;
+
+ if (REG_P (val) && VALID_AVX256_REG_MODE (GET_MODE (val)))
+ return true;
+
+ if (GET_CODE (val) == PARALLEL)
+ {
+ int i;
+ rtx r;
+
+ for (i = XVECLEN (val, 0) - 1; i >= 0; i--)
+ {
+ r = XVECEXP (val, 0, i);
+ if (GET_CODE (r) == EXPR_LIST
+ && XEXP (r, 0)
+ && REG_P (XEXP (r, 0))
+ && (GET_MODE (XEXP (r, 0)) == OImode
+ || VALID_AVX256_REG_MODE (GET_MODE (XEXP (r, 0)))))
+ return true;
+ }
+ }
+
+ return false;
+}
+
+/* Implement the TARGET_OPTION_OVERRIDE hook. */
+
+static void
+ix86_option_override (void)
+{
+ ix86_option_override_internal (true);
}
/* Update register usage after having seen the compiler flags. */
ptr->tune_defaulted = ix86_tune_defaulted;
ptr->arch_specified = ix86_arch_specified;
ptr->ix86_isa_flags_explicit = ix86_isa_flags_explicit;
- ptr->target_flags_explicit = target_flags_explicit;
+ ptr->ix86_target_flags_explicit = target_flags_explicit;
/* The fields are char but the variables are not; make sure the
values fit in the fields. */
ix86_tune_defaulted = ptr->tune_defaulted;
ix86_arch_specified = ptr->arch_specified;
ix86_isa_flags_explicit = ptr->ix86_isa_flags_explicit;
- target_flags_explicit = ptr->target_flags_explicit;
+ target_flags_explicit = ptr->ix86_target_flags_explicit;
/* Recreate the arch feature tests if the arch changed */
if (old_arch != ix86_arch)
struct cl_target_option *ptr)
{
char *target_string
- = ix86_target_string (ptr->ix86_isa_flags, ptr->target_flags,
+ = ix86_target_string (ptr->x_ix86_isa_flags, ptr->x_target_flags,
NULL, NULL, NULL, false);
fprintf (file, "%*sarch = %d (%s)\n",
if (! ix86_valid_target_attribute_inner_p (args, option_strings))
return NULL_TREE;
- /* If the changed options are different from the default, rerun override_options,
- and then save the options away. The string options are are attribute options,
- and will be undone when we copy the save structure. */
- if (ix86_isa_flags != def->ix86_isa_flags
- || target_flags != def->target_flags
+ /* If the changed options are different from the default, rerun
+ ix86_option_override_internal, and then save the options away.
+ The string options are are attribute options, and will be undone
+ when we copy the save structure. */
+ if (ix86_isa_flags != def->x_ix86_isa_flags
+ || target_flags != def->x_target_flags
|| option_strings[IX86_FUNCTION_SPECIFIC_ARCH]
|| option_strings[IX86_FUNCTION_SPECIFIC_TUNE]
|| option_strings[IX86_FUNCTION_SPECIFIC_FPMATH])
ix86_fpmath_string = "sse,387";
/* Do any overrides, such as arch=xxx, or tune=xxx support. */
- override_options (false);
+ ix86_option_override_internal (false);
/* Add any builtin functions with the new isa if any. */
ix86_add_new_builtins (ix86_isa_flags);
/* If the function changed the optimization levels as well as setting target
options, start with the optimizations specified. */
if (func_optimize && func_optimize != old_optimize)
- cl_optimization_restore (TREE_OPTIMIZATION (func_optimize));
+ cl_optimization_restore (&global_options,
+ TREE_OPTIMIZATION (func_optimize));
/* The target attributes may also change some optimization flags, so update
the optimization options if necessary. */
- cl_target_option_save (&cur_target);
+ cl_target_option_save (&cur_target, &global_options);
new_target = ix86_valid_target_attribute_tree (args);
new_optimize = build_optimization_node ();
DECL_FUNCTION_SPECIFIC_OPTIMIZATION (fndecl) = new_optimize;
}
- cl_target_option_restore (&cur_target);
+ cl_target_option_restore (&global_options, &cur_target);
if (old_optimize != new_optimize)
- cl_optimization_restore (TREE_OPTIMIZATION (old_optimize));
+ cl_optimization_restore (&global_options,
+ TREE_OPTIMIZATION (old_optimize));
return ret;
}
/* Callee's isa options should a subset of the caller's, i.e. a SSE4 function
can inline a SSE2 function but a SSE2 function can't inline a SSE4
function. */
- if ((caller_opts->ix86_isa_flags & callee_opts->ix86_isa_flags)
- != callee_opts->ix86_isa_flags)
+ if ((caller_opts->x_ix86_isa_flags & callee_opts->x_ix86_isa_flags)
+ != callee_opts->x_ix86_isa_flags)
ret = false;
/* See if we have the same non-isa options. */
- else if (caller_opts->target_flags != callee_opts->target_flags)
+ else if (caller_opts->x_target_flags != callee_opts->x_target_flags)
ret = false;
/* See if arch, tune, etc. are the same. */
else if (new_tree)
{
- cl_target_option_restore (TREE_TARGET_OPTION (new_tree));
+ cl_target_option_restore (&global_options,
+ TREE_TARGET_OPTION (new_tree));
target_reinit ();
}
struct cl_target_option *def
= TREE_TARGET_OPTION (target_option_current_node);
- cl_target_option_restore (def);
+ cl_target_option_restore (&global_options, def);
target_reinit ();
}
}
ASM_OUTPUT_SKIP (file, size ? size : 1);
}
\f
-void
-optimization_options (int level, int size ATTRIBUTE_UNUSED)
-{
- /* For -O2 and beyond, turn off -fschedule-insns by default. It tends to
- make the problem with not enough registers even worse. */
+static const struct default_options ix86_option_optimization_table[] =
+ {
+ /* Turn off -fschedule-insns by default. It tends to make the
+ problem with not enough registers even worse. */
#ifdef INSN_SCHEDULING
- if (level > 1)
- flag_schedule_insns = 0;
+ { OPT_LEVELS_ALL, OPT_fschedule_insns, NULL, 0 },
+#endif
+
+#ifdef SUBTARGET_OPTIMIZATION_OPTIONS
+ SUBTARGET_OPTIMIZATION_OPTIONS,
#endif
+ { OPT_LEVELS_NONE, 0, NULL, 0 }
+ };
+/* Implement TARGET_OPTION_INIT_STRUCT. */
+
+static void
+ix86_option_init_struct (struct gcc_options *opts)
+{
if (TARGET_MACHO)
/* The Darwin libraries never set errno, so we might as well
avoid calling them when that's the only reason we would. */
- flag_errno_math = 0;
-
- /* The default values of these switches depend on the TARGET_64BIT
- that is not known at this moment. Mark these values with 2 and
- let user the to override these. In case there is no command line option
- specifying them, we will set the defaults in override_options. */
- if (optimize >= 1)
- flag_omit_frame_pointer = 2;
-
- /* For -O2 and beyond, turn on -fzee for x86_64 target. */
- if (level > 1)
- flag_zee = 2;
-
- flag_pcc_struct_return = 2;
- flag_asynchronous_unwind_tables = 2;
- flag_vect_cost_model = 1;
-#ifdef SUBTARGET_OPTIMIZATION_OPTIONS
- SUBTARGET_OPTIMIZATION_OPTIONS;
-#endif
+ opts->x_flag_errno_math = 0;
+
+ opts->x_flag_pcc_struct_return = 2;
+ opts->x_flag_asynchronous_unwind_tables = 2;
+ opts->x_flag_vect_cost_model = 1;
}
/* Decide whether we must probe the stack before any space allocation
/* If we are generating position-independent code, we cannot sibcall
optimize any indirect call, or a direct call to a global function,
- as the PLT requires %ebx be live. */
- if (!TARGET_64BIT && flag_pic && (!decl || !targetm.binds_local_p (decl)))
+ as the PLT requires %ebx be live. (Darwin does not have a PLT.) */
+ if (!TARGET_MACHO
+ && !TARGET_64BIT
+ && flag_pic
+ && (!decl || !targetm.binds_local_p (decl)))
return false;
/* If we need to align the outgoing stack, then sibcalling would
return false;
}
else if (VOID_TYPE_P (TREE_TYPE (DECL_RESULT (cfun->decl))))
- ;
+ {
+ /* Disable sibcall if we need to generate vzeroupper after
+ callee returns. */
+ if (TARGET_VZEROUPPER
+ && cfun->machine->callee_return_avx256_p
+ && !cfun->machine->caller_return_avx256_p)
+ return false;
+ }
else if (!rtx_equal_p (a, b))
return false;
if (local_regparm == 3 && DECL_STATIC_CHAIN (decl))
local_regparm = 2;
+ /* In 32-bit mode save a register for the split stack. */
+ if (!TARGET_64BIT && local_regparm == 3 && flag_split_stack)
+ local_regparm = 2;
+
/* Each fixed register usage increases register pressure,
so less registers should be used for argument passing.
This functionality can be overriden by an explicit
fprintf (asm_out_file, ASM_LONG " %#x\n", filler_cc);
}
+#ifdef SUBTARGET_ASM_UNWIND_INIT
+ SUBTARGET_ASM_UNWIND_INIT (asm_out_file);
+#endif
+
ASM_OUTPUT_LABEL (asm_out_file, fname);
/* Output magic byte marker, if hot-patch attribute is set. */
init_cumulative_args (CUMULATIVE_ARGS *cum, /* Argument info to initialize */
tree fntype, /* tree ptr for function decl */
rtx libname, /* SYMBOL_REF of library name or 0 */
- tree fndecl)
+ tree fndecl,
+ int caller)
{
- struct cgraph_local_info *i = fndecl ? cgraph_local_info (fndecl) : NULL;
+ struct cgraph_local_info *i;
+ tree fnret_type;
+
memset (cum, 0, sizeof (*cum));
+ /* Initialize for the current callee. */
+ if (caller)
+ {
+ cfun->machine->callee_pass_avx256_p = false;
+ cfun->machine->callee_return_avx256_p = false;
+ }
+
if (fndecl)
- cum->call_abi = ix86_function_abi (fndecl);
+ {
+ i = cgraph_local_info (fndecl);
+ cum->call_abi = ix86_function_abi (fndecl);
+ fnret_type = TREE_TYPE (TREE_TYPE (fndecl));
+ }
else
- cum->call_abi = ix86_function_type_abi (fntype);
+ {
+ i = NULL;
+ cum->call_abi = ix86_function_type_abi (fntype);
+ if (fntype)
+ fnret_type = TREE_TYPE (fntype);
+ else
+ fnret_type = NULL;
+ }
+
+ if (TARGET_VZEROUPPER && fnret_type)
+ {
+ rtx fnret_value = ix86_function_value (fnret_type, fntype,
+ false);
+ if (function_pass_avx256_p (fnret_value))
+ {
+ /* The return value of this function uses 256bit AVX modes. */
+ cfun->machine->use_avx256_p = true;
+ if (caller)
+ cfun->machine->callee_return_avx256_p = true;
+ else
+ cfun->machine->caller_return_avx256_p = true;
+ }
+ }
+
+ cum->caller = caller;
+
/* Set up the number of registers to use for passing arguments. */
if (cum->call_abi == MS_ABI && !ACCUMULATE_OUTGOING_ARGS)
{
enum machine_mode mode = omode;
HOST_WIDE_INT bytes, words;
+ rtx arg;
if (mode == BLKmode)
bytes = int_size_in_bytes (type);
mode = type_natural_mode (type, cum);
if (TARGET_64BIT && (cum ? cum->call_abi : ix86_abi) == MS_ABI)
- return function_arg_ms_64 (cum, mode, omode, named, bytes);
+ arg = function_arg_ms_64 (cum, mode, omode, named, bytes);
else if (TARGET_64BIT)
- return function_arg_64 (cum, mode, omode, type, named);
+ arg = function_arg_64 (cum, mode, omode, type, named);
else
- return function_arg_32 (cum, mode, omode, type, bytes, words);
+ arg = function_arg_32 (cum, mode, omode, type, bytes, words);
+
+ if (TARGET_VZEROUPPER && function_pass_avx256_p (arg))
+ {
+ /* This argument uses 256bit AVX modes. */
+ cfun->machine->use_avx256_p = true;
+ if (cum->caller)
+ cfun->machine->callee_pass_avx256_p = true;
+ else
+ cfun->machine->caller_pass_avx256_p = true;
+ }
+
+ return arg;
}
/* A C expression that indicates when an argument must be passed by
return 0;
}
-/* Return true when TYPE should be 128bit aligned for 32bit argument passing
- ABI. */
+/* Return true when TYPE should be 128bit aligned for 32bit argument
+ passing ABI. XXX: This function is obsolete and is only used for
+ checking psABI compatibility with previous versions of GCC. */
+
static bool
-contains_aligned_value_p (const_tree type)
+ix86_compat_aligned_value_p (const_tree type)
{
enum machine_mode mode = TYPE_MODE (type);
if (((TARGET_SSE && SSE_REG_MODE_P (mode))
for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
{
if (TREE_CODE (field) == FIELD_DECL
- && contains_aligned_value_p (TREE_TYPE (field)))
+ && ix86_compat_aligned_value_p (TREE_TYPE (field)))
return true;
}
break;
case ARRAY_TYPE:
/* Just for use if some languages passes arrays by value. */
- if (contains_aligned_value_p (TREE_TYPE (type)))
+ if (ix86_compat_aligned_value_p (TREE_TYPE (type)))
return true;
break;
return false;
}
-/* Gives the alignment boundary, in bits, of an argument with the
- specified mode and type. */
+/* Return the alignment boundary for MODE and TYPE with alignment ALIGN.
+ XXX: This function is obsolete and is only used for checking psABI
+ compatibility with previous versions of GCC. */
-int
-ix86_function_arg_boundary (enum machine_mode mode, const_tree type)
+static int
+ix86_compat_function_arg_boundary (enum machine_mode mode,
+ const_tree type, int align)
{
- int align;
- if (type)
- {
- /* Since the main variant type is used for call, we convert it to
- the main variant type. */
- type = TYPE_MAIN_VARIANT (type);
- align = TYPE_ALIGN (type);
- }
- else
- align = GET_MODE_ALIGNMENT (mode);
- if (align < PARM_BOUNDARY)
- align = PARM_BOUNDARY;
/* In 32bit, only _Decimal128 and __float128 are aligned to their
natural boundaries. */
if (!TARGET_64BIT && mode != TDmode && mode != TFmode)
}
else
{
- if (!contains_aligned_value_p (type))
+ if (!ix86_compat_aligned_value_p (type))
align = PARM_BOUNDARY;
}
}
return align;
}
+/* Return true when TYPE should be 128bit aligned for 32bit argument
+ passing ABI. */
+
+static bool
+ix86_contains_aligned_value_p (const_tree type)
+{
+ enum machine_mode mode = TYPE_MODE (type);
+
+ if (mode == XFmode || mode == XCmode)
+ return false;
+
+ if (TYPE_ALIGN (type) < 128)
+ return false;
+
+ if (AGGREGATE_TYPE_P (type))
+ {
+ /* Walk the aggregates recursively. */
+ switch (TREE_CODE (type))
+ {
+ case RECORD_TYPE:
+ case UNION_TYPE:
+ case QUAL_UNION_TYPE:
+ {
+ tree field;
+
+ /* Walk all the structure fields. */
+ for (field = TYPE_FIELDS (type);
+ field;
+ field = DECL_CHAIN (field))
+ {
+ if (TREE_CODE (field) == FIELD_DECL
+ && ix86_contains_aligned_value_p (TREE_TYPE (field)))
+ return true;
+ }
+ break;
+ }
+
+ case ARRAY_TYPE:
+ /* Just for use if some languages passes arrays by value. */
+ if (ix86_contains_aligned_value_p (TREE_TYPE (type)))
+ return true;
+ break;
+
+ default:
+ gcc_unreachable ();
+ }
+ }
+ else
+ return TYPE_ALIGN (type) >= 128;
+
+ return false;
+}
+
+/* Gives the alignment boundary, in bits, of an argument with the
+ specified mode and type. */
+
+int
+ix86_function_arg_boundary (enum machine_mode mode, const_tree type)
+{
+ int align;
+ if (type)
+ {
+ /* Since the main variant type is used for call, we convert it to
+ the main variant type. */
+ type = TYPE_MAIN_VARIANT (type);
+ align = TYPE_ALIGN (type);
+ }
+ else
+ align = GET_MODE_ALIGNMENT (mode);
+ if (align < PARM_BOUNDARY)
+ align = PARM_BOUNDARY;
+ else
+ {
+ static bool warned;
+ int saved_align = align;
+
+ if (!TARGET_64BIT)
+ {
+ /* i386 ABI defines XFmode arguments to be 4 byte aligned. */
+ if (!type)
+ {
+ if (mode == XFmode || mode == XCmode)
+ align = PARM_BOUNDARY;
+ }
+ else if (!ix86_contains_aligned_value_p (type))
+ align = PARM_BOUNDARY;
+
+ if (align < 128)
+ align = PARM_BOUNDARY;
+ }
+
+ if (warn_psabi
+ && !warned
+ && align != ix86_compat_function_arg_boundary (mode, type,
+ saved_align))
+ {
+ warned = true;
+ inform (input_location,
+ "The ABI of passing parameter with %dbyte"
+ " alignment has changed in GCC 4.6",
+ align / BITS_PER_UNIT);
+ }
+ }
+
+ return align;
+}
+
/* Return true if N is a possible register number of function value. */
static bool
return false;
/* MMX/3dNow values are returned in MM0,
- except when it doesn't exits. */
+ except when it doesn't exits or the ABI prescribes otherwise. */
if (size == 8)
- return !TARGET_MMX;
+ return !TARGET_MMX || TARGET_VECT8_RETURNS;
/* SSE values are returned in XMM0, except when it doesn't exist. */
if (size == 16)
#endif
}
-/* Return false iff TYPE is returned in memory. This version is used
- on Solaris 2. It is similar to the generic ix86_return_in_memory,
- but differs notably in that when MMX is available, 8-byte vectors
- are returned in memory, rather than in MMX registers. */
-
-bool
-ix86_solaris_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
-{
- int size;
- enum machine_mode mode = type_natural_mode (type, NULL);
-
- if (TARGET_64BIT)
- return return_in_memory_64 (type, mode);
-
- if (mode == BLKmode)
- return 1;
-
- size = int_size_in_bytes (type);
-
- if (VECTOR_MODE_P (mode))
- {
- /* Return in memory only if MMX registers *are* available. This
- seems backwards, but it is consistent with the existing
- Solaris x86 ABI. */
- if (size == 8)
- return TARGET_MMX;
- if (size == 16)
- return !TARGET_SSE;
- }
- else if (mode == TImode)
- return !TARGET_SSE;
- else if (mode == XFmode)
- return 0;
-
- return size > 12;
-}
-
/* When returning SSE vector types, we have a choice of either
(1) being abi incompatible with a -march switch, or
(2) generating an error.
DECL_FIELD_CONTEXT (f_ovf) = record;
DECL_FIELD_CONTEXT (f_sav) = record;
- TREE_CHAIN (record) = type_decl;
+ TYPE_STUB_DECL (record) = type_decl;
TYPE_NAME (record) = type_decl;
TYPE_FIELDS (record) = f_gpr;
DECL_CHAIN (f_gpr) = f_fpr;
tree f_gpr, f_fpr, f_ovf, f_sav;
tree gpr, fpr, ovf, sav, t;
tree type;
+ rtx ovf_rtx;
+
+ if (flag_split_stack
+ && cfun->machine->split_stack_varargs_pointer == NULL_RTX)
+ {
+ unsigned int scratch_regno;
+
+ /* When we are splitting the stack, we can't refer to the stack
+ arguments using internal_arg_pointer, because they may be on
+ the old stack. The split stack prologue will arrange to
+ leave a pointer to the old stack arguments in a scratch
+ register, which we here copy to a pseudo-register. The split
+ stack prologue can't set the pseudo-register directly because
+ it (the prologue) runs before any registers have been saved. */
+
+ scratch_regno = split_stack_prologue_scratch_regno ();
+ if (scratch_regno != INVALID_REGNUM)
+ {
+ rtx reg, seq;
+
+ reg = gen_reg_rtx (Pmode);
+ cfun->machine->split_stack_varargs_pointer = reg;
+
+ start_sequence ();
+ emit_move_insn (reg, gen_rtx_REG (Pmode, scratch_regno));
+ seq = get_insns ();
+ end_sequence ();
+
+ push_topmost_sequence ();
+ emit_insn_after (seq, entry_of_function ());
+ pop_topmost_sequence ();
+ }
+ }
/* Only 64bit target needs something special. */
if (!TARGET_64BIT || is_va_list_char_pointer (TREE_TYPE (valist)))
{
- std_expand_builtin_va_start (valist, nextarg);
+ if (cfun->machine->split_stack_varargs_pointer == NULL_RTX)
+ std_expand_builtin_va_start (valist, nextarg);
+ else
+ {
+ rtx va_r, next;
+
+ va_r = expand_expr (valist, NULL_RTX, VOIDmode, EXPAND_WRITE);
+ next = expand_binop (ptr_mode, add_optab,
+ cfun->machine->split_stack_varargs_pointer,
+ crtl->args.arg_offset_rtx,
+ NULL_RTX, 0, OPTAB_LIB_WIDEN);
+ convert_move (va_r, next, 0);
+ }
return;
}
/* Find the overflow area. */
type = TREE_TYPE (ovf);
- t = make_tree (type, crtl->args.internal_arg_pointer);
+ if (cfun->machine->split_stack_varargs_pointer == NULL_RTX)
+ ovf_rtx = crtl->args.internal_arg_pointer;
+ else
+ ovf_rtx = cfun->machine->split_stack_varargs_pointer;
+ t = make_tree (type, ovf_rtx);
if (words != 0)
t = build2 (POINTER_PLUS_EXPR, type, t,
size_int (words * UNITS_PER_WORD));
tree dest_addr, dest;
int cur_size = GET_MODE_SIZE (mode);
+ gcc_assert (prev_size <= INTVAL (XEXP (slot, 1)));
+ prev_size = INTVAL (XEXP (slot, 1));
if (prev_size + cur_size > size)
{
cur_size = size - prev_size;
dest_addr = fold_convert (daddr_type, addr);
dest_addr = fold_build2 (POINTER_PLUS_EXPR, daddr_type, dest_addr,
- size_int (INTVAL (XEXP (slot, 1))));
+ size_int (prev_size));
if (cur_size == GET_MODE_SIZE (mode))
{
src = build_va_arg_indirect_ref (src_addr);
if (SUBTARGET_FRAME_POINTER_REQUIRED)
return true;
- /* In override_options, TARGET_OMIT_LEAF_FRAME_POINTER turns off
- the frame pointer by default. Turn it back on now if we've not
- got a leaf function. */
+ /* In ix86_option_override_internal, TARGET_OMIT_LEAF_FRAME_POINTER
+ turns off the frame pointer by default. Turn it back on now if
+ we've not got a leaf function. */
if (TARGET_OMIT_LEAF_FRAME_POINTER
&& (!current_function_is_leaf
|| ix86_current_function_calls_tls_descriptor))
rtx xops[2];
int regno;
- for (regno = 0; regno < 8; ++regno)
+ for (regno = AX_REG; regno <= SP_REG; regno++)
{
char name[32];
tree decl;
- if (! ((pic_labels_used >> regno) & 1))
+ if (!(pic_labels_used & (1 << regno)))
continue;
get_pc_thunk_name (name, regno);
/* Make sure unwind info is emitted for the thunk if needed. */
final_start_function (emit_barrier (), asm_out_file, 1);
+ /* Pad stack IP move with 4 instructions (two NOPs count
+ as one instruction). */
+ if (TARGET_PAD_SHORT_FUNCTION)
+ {
+ int i = 8;
+
+ while (i--)
+ fputs ("\tnop\n", asm_out_file);
+ }
+
xops[0] = gen_rtx_REG (Pmode, regno);
xops[1] = gen_rtx_MEM (Pmode, stack_pointer_rtx);
output_asm_insn ("mov%z0\t{%1, %0|%0, %1}", xops);
- output_asm_insn ("ret", xops);
+ fputs ("\tret\n", asm_out_file);
final_end_function ();
init_insn_lengths ();
free_after_compilation (cfun);
set_cfun (NULL);
current_function_decl = NULL;
}
+
+ if (flag_split_stack)
+ file_end_indicate_split_stack ();
}
/* Emit code for the SET_GOT patterns. */
return stack_realign_fp ? hard_frame_pointer_rtx : virtual_stack_vars_rtx;
}
+/* On the x86 -fsplit-stack and -fstack-protector both use the same
+ field in the TCB, so they can not be used together. */
+
+static bool
+ix86_supports_split_stack (bool report ATTRIBUTE_UNUSED)
+{
+ bool ret = true;
+
+#ifndef TARGET_THREAD_SPLIT_STACK_OFFSET
+ if (report)
+ error ("%<-fsplit-stack%> currently only supported on GNU/Linux");
+ ret = false;
+#else
+ if (!HAVE_GAS_CFI_PERSONALITY_DIRECTIVE)
+ {
+ if (report)
+ error ("%<-fsplit-stack%> requires "
+ "assembler support for CFI directives");
+ ret = false;
+ }
+#endif
+
+ return ret;
+}
+
+/* When using -fsplit-stack, the allocation routines set a field in
+ the TCB to the bottom of the stack plus this much space, measured
+ in bytes. */
+
+#define SPLIT_STACK_AVAILABLE 256
+
/* Fill structure ix86_frame about frame of currently computed function. */
static void
gcc_assert (preferred_alignment >= STACK_BOUNDARY / BITS_PER_UNIT);
gcc_assert (preferred_alignment <= stack_alignment_needed);
+ /* For SEH we have to limit the amount of code movement into the prologue.
+ At present we do this via a BLOCKAGE, at which point there's very little
+ scheduling that can be done, which means that there's very little point
+ in doing anything except PUSHs. */
+ if (TARGET_SEH)
+ cfun->machine->use_fast_prologue_epilogue = false;
+
/* During reload iteration the amount of registers saved can change.
Recompute the value as needed. Do not recompute when amount of registers
didn't change as reload does multiple calls to the function and does not
expect the decision to change within single iteration. */
- if (!optimize_function_for_size_p (cfun)
- && cfun->machine->use_fast_prologue_epilogue_nregs != frame->nregs)
+ else if (!optimize_function_for_size_p (cfun)
+ && cfun->machine->use_fast_prologue_epilogue_nregs != frame->nregs)
{
int count = frame->nregs;
struct cgraph_node *node = cgraph_node (current_function_decl);
cfun->machine->use_fast_prologue_epilogue_nregs = count;
+
/* The fast prologue uses move instead of push to save registers. This
is significantly longer, but also executes faster as modern hardware
can execute the moves in parallel, but can't do that for push/pop.
/* Skip saved base pointer. */
if (frame_pointer_needed)
offset += UNITS_PER_WORD;
+ frame->hfp_save_offset = offset;
+ /* The traditional frame pointer location is at the top of the frame. */
frame->hard_frame_pointer_offset = offset;
/* Register save area */
else
frame->red_zone_size = 0;
frame->stack_pointer_offset -= frame->red_zone_size;
+
+ /* The SEH frame pointer location is near the bottom of the frame.
+ This is enforced by the fact that the difference between the
+ stack pointer and the frame pointer is limited to 240 bytes in
+ the unwind data structure. */
+ if (TARGET_SEH)
+ {
+ HOST_WIDE_INT diff;
+
+ /* If we can leave the frame pointer where it is, do so. */
+ diff = frame->stack_pointer_offset - frame->hard_frame_pointer_offset;
+ if (diff > 240 || (diff & 15) != 0)
+ {
+ /* Ideally we'd determine what portion of the local stack frame
+ (within the constraint of the lowest 240) is most heavily used.
+ But without that complication, simply bias the frame pointer
+ by 128 bytes so as to maximize the amount of the local stack
+ frame that is addressable with 8-bit offsets. */
+ frame->hard_frame_pointer_offset = frame->stack_pointer_offset - 128;
+ }
+ }
}
/* This is semi-inlined memory_address_length, but simplified
rtx insn;
if (! TARGET_64BIT)
- insn = emit_insn (gen_pro_epilogue_adjust_stack_si_1 (dest, src, offset));
+ insn = gen_pro_epilogue_adjust_stack_si_add (dest, src, offset);
else if (x86_64_immediate_operand (offset, DImode))
- insn = emit_insn (gen_pro_epilogue_adjust_stack_di_1 (dest, src, offset));
+ insn = gen_pro_epilogue_adjust_stack_di_add (dest, src, offset);
else
{
rtx tmp;
insn = emit_insn (gen_rtx_SET (DImode, tmp, offset));
if (style < 0)
RTX_FRAME_RELATED_P (insn) = 1;
- insn = emit_insn (gen_pro_epilogue_adjust_stack_di_2 (dest, src, tmp,
- offset));
+
+ insn = gen_pro_epilogue_adjust_stack_di_add (dest, src, tmp);
}
+ insn = emit_insn (insn);
if (style >= 0)
ix86_add_queued_cfa_restore_notes (insn);
/* Check if profiling is active and we shall use profiling before
prologue variant. If so sorry. */
if (crtl->profile && flag_fentry != 0)
- sorry ("ms_hook_prologue attribute isn't compatible with -mfentry for 32-bit");
+ sorry ("ms_hook_prologue attribute isn't compatible "
+ "with -mfentry for 32-bit");
/* In ix86_asm_output_function_label we emitted:
8b ff movl.s %edi,%edi
insn = emit_insn (gen_push (hard_frame_pointer_rtx));
RTX_FRAME_RELATED_P (insn) = 1;
- insn = emit_move_insn (hard_frame_pointer_rtx, stack_pointer_rtx);
- RTX_FRAME_RELATED_P (insn) = 1;
+ if (m->fs.sp_offset == frame.hard_frame_pointer_offset)
+ {
+ insn = emit_move_insn (hard_frame_pointer_rtx, stack_pointer_rtx);
+ RTX_FRAME_RELATED_P (insn) = 1;
- if (m->fs.cfa_reg == stack_pointer_rtx)
- m->fs.cfa_reg = hard_frame_pointer_rtx;
- gcc_assert (m->fs.sp_offset == frame.hard_frame_pointer_offset);
- m->fs.fp_offset = m->fs.sp_offset;
- m->fs.fp_valid = true;
+ if (m->fs.cfa_reg == stack_pointer_rtx)
+ m->fs.cfa_reg = hard_frame_pointer_rtx;
+ m->fs.fp_offset = m->fs.sp_offset;
+ m->fs.fp_valid = true;
+ }
}
int_registers_saved = (frame.nregs == 0);
else
{
rtx eax = gen_rtx_REG (Pmode, AX_REG);
- bool eax_live;
+ rtx r10 = NULL;
+ rtx (*adjust_stack_insn)(rtx, rtx, rtx);
- if (cfun->machine->call_abi == MS_ABI)
- eax_live = false;
- else
- eax_live = ix86_eax_live_at_start_p ();
+ bool eax_live = false;
+ bool r10_live = false;
+
+ if (TARGET_64BIT)
+ r10_live = (DECL_STATIC_CHAIN (current_function_decl) != 0);
+ if (!TARGET_64BIT_MS_ABI)
+ eax_live = ix86_eax_live_at_start_p ();
if (eax_live)
{
emit_insn (gen_push (eax));
allocate -= UNITS_PER_WORD;
}
+ if (r10_live)
+ {
+ r10 = gen_rtx_REG (Pmode, R10_REG);
+ emit_insn (gen_push (r10));
+ allocate -= UNITS_PER_WORD;
+ }
emit_move_insn (eax, GEN_INT (allocate));
+ emit_insn (ix86_gen_allocate_stack_worker (eax, eax));
- insn = emit_insn (ix86_gen_allocate_stack_worker (eax, eax));
+ /* Use the fact that AX still contains ALLOCATE. */
+ adjust_stack_insn = (TARGET_64BIT
+ ? gen_pro_epilogue_adjust_stack_di_sub
+ : gen_pro_epilogue_adjust_stack_si_sub);
- if (m->fs.cfa_reg == stack_pointer_rtx)
+ insn = emit_insn (adjust_stack_insn (stack_pointer_rtx,
+ stack_pointer_rtx, eax));
+
+ /* Note that SEH directives need to continue tracking the stack
+ pointer even after the frame pointer has been set up. */
+ if (m->fs.cfa_reg == stack_pointer_rtx || TARGET_SEH)
{
- m->fs.cfa_offset += allocate;
- t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, GEN_INT (-allocate));
- t = gen_rtx_SET (VOIDmode, stack_pointer_rtx, t);
- add_reg_note (insn, REG_CFA_ADJUST_CFA, t);
+ if (m->fs.cfa_reg == stack_pointer_rtx)
+ m->fs.cfa_offset += allocate;
+
RTX_FRAME_RELATED_P (insn) = 1;
+ add_reg_note (insn, REG_FRAME_RELATED_EXPR,
+ gen_rtx_SET (VOIDmode, stack_pointer_rtx,
+ plus_constant (stack_pointer_rtx,
+ -allocate)));
}
m->fs.sp_offset += allocate;
- if (eax_live)
- {
+ if (r10_live && eax_live)
+ {
t = choose_baseaddr (m->fs.sp_offset - allocate);
+ emit_move_insn (r10, gen_frame_mem (Pmode, t));
+ t = choose_baseaddr (m->fs.sp_offset - allocate - UNITS_PER_WORD);
emit_move_insn (eax, gen_frame_mem (Pmode, t));
}
+ else if (eax_live || r10_live)
+ {
+ t = choose_baseaddr (m->fs.sp_offset - allocate);
+ emit_move_insn ((eax_live ? eax : r10), gen_frame_mem (Pmode, t));
+ }
}
gcc_assert (m->fs.sp_offset == frame.stack_pointer_offset);
+ /* If we havn't already set up the frame pointer, do so now. */
+ if (frame_pointer_needed && !m->fs.fp_valid)
+ {
+ insn = ix86_gen_add3 (hard_frame_pointer_rtx, stack_pointer_rtx,
+ GEN_INT (frame.stack_pointer_offset
+ - frame.hard_frame_pointer_offset));
+ insn = emit_insn (insn);
+ RTX_FRAME_RELATED_P (insn) = 1;
+ add_reg_note (insn, REG_CFA_ADJUST_CFA, NULL);
+
+ if (m->fs.cfa_reg == stack_pointer_rtx)
+ m->fs.cfa_reg = hard_frame_pointer_rtx;
+ m->fs.fp_offset = frame.hard_frame_pointer_offset;
+ m->fs.fp_valid = true;
+ }
+
if (!int_registers_saved)
ix86_emit_save_regs_using_mov (frame.reg_save_offset);
if (frame.nsseregs)
/* Emit cld instruction if stringops are used in the function. */
if (TARGET_CLD && ix86_current_function_needs_cld)
emit_insn (gen_cld ());
+
+ /* SEH requires that the prologue end within 256 bytes of the start of
+ the function. Prevent instruction schedules that would extend that. */
+ if (TARGET_SEH)
+ emit_insn (gen_blockage ());
}
/* Emit code to restore REG using a POP insn. */
if (crtl->calls_eh_return && style != 2)
frame.reg_save_offset -= 2 * UNITS_PER_WORD;
+ /* EH_RETURN requires the use of moves to function properly. */
+ if (crtl->calls_eh_return)
+ restore_regs_via_mov = true;
+ /* SEH requires the use of pops to identify the epilogue. */
+ else if (TARGET_SEH)
+ restore_regs_via_mov = false;
/* If we're only restoring one register and sp is not valid then
using a move instruction to restore the register since it's
less work than reloading sp and popping the register. */
- if (!m->fs.sp_valid && frame.nregs <= 1)
- restore_regs_via_mov = true;
- /* EH_RETURN requires the use of moves to function properly. */
- else if (crtl->calls_eh_return)
+ else if (!m->fs.sp_valid && frame.nregs <= 1)
restore_regs_via_mov = true;
else if (TARGET_EPILOGUE_USING_MOVE
&& cfun->machine->use_fast_prologue_epilogue
}
else
{
+ /* SEH requires that the function end with (1) a stack adjustment
+ if necessary, (2) a sequence of pops, and (3) a return or
+ jump instruction. Prevent insns from the function body from
+ being scheduled into this sequence. */
+ if (TARGET_SEH)
+ {
+ /* Prevent a catch region from being adjacent to the standard
+ epilogue sequence. Unfortuantely crtl->uses_eh_lsda nor
+ several other flags that would be interesting to test are
+ not yet set up. */
+ if (flag_non_call_exceptions)
+ emit_insn (gen_nops (const1_rtx));
+ else
+ emit_insn (gen_blockage ());
+ }
+
/* First step is to deallocate the stack frame so that we can
pop the registers. */
if (!m->fs.sp_valid)
{
/* If the stack pointer is valid and pointing at the frame
pointer store address, then we only need a pop. */
- if (m->fs.sp_valid && m->fs.sp_offset == frame.hard_frame_pointer_offset)
+ if (m->fs.sp_valid && m->fs.sp_offset == frame.hfp_save_offset)
ix86_emit_restore_reg_using_pop (hard_frame_pointer_rtx);
/* Leave results in shorter dependency chains on CPUs that are
able to grok it fast. */
return;
}
+ /* Emit vzeroupper if needed. */
+ if (TARGET_VZEROUPPER
+ && cfun->machine->use_avx256_p
+ && !cfun->machine->caller_return_avx256_p)
+ {
+ cfun->machine->use_vzeroupper_p = 1;
+ emit_insn (gen_avx_vzeroupper (GEN_INT (call_no_avx256)));
+ }
+
if (crtl->args.pops_args && crtl->args.size)
{
rtx popc = GEN_INT (crtl->args.pops_args);
#endif
}
-\f
-/* Extract the parts of an RTL expression that is a valid memory address
- for an instruction. Return 0 if the structure of the address is
- grossly off. Return -1 if the address contains ASHIFT, so it is not
- strictly valid, but still used for computing length of lea instruction. */
-int
-ix86_decompose_address (rtx addr, struct ix86_address *out)
-{
- rtx base = NULL_RTX, index = NULL_RTX, disp = NULL_RTX;
- rtx base_reg, index_reg;
- HOST_WIDE_INT scale = 1;
- rtx scale_rtx = NULL_RTX;
- rtx tmp;
- int retval = 1;
- enum ix86_address_seg seg = SEG_DEFAULT;
+/* Return a scratch register to use in the split stack prologue. The
+ split stack prologue is used for -fsplit-stack. It is the first
+ instructions in the function, even before the regular prologue.
+ The scratch register can be any caller-saved register which is not
+ used for parameters or for the static chain. */
- if (REG_P (addr) || GET_CODE (addr) == SUBREG)
- base = addr;
- else if (GET_CODE (addr) == PLUS)
+static unsigned int
+split_stack_prologue_scratch_regno (void)
+{
+ if (TARGET_64BIT)
+ return R11_REG;
+ else
{
- rtx addends[4], op;
- int n = 0, i;
+ bool is_fastcall;
+ int regparm;
- op = addr;
- do
+ is_fastcall = (lookup_attribute ("fastcall",
+ TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
+ != NULL);
+ regparm = ix86_function_regparm (TREE_TYPE (cfun->decl), cfun->decl);
+
+ if (is_fastcall)
{
- if (n >= 4)
- return 0;
- addends[n++] = XEXP (op, 1);
- op = XEXP (op, 0);
+ if (DECL_STATIC_CHAIN (cfun->decl))
+ {
+ sorry ("-fsplit-stack does not support fastcall with "
+ "nested function");
+ return INVALID_REGNUM;
+ }
+ return AX_REG;
}
- while (GET_CODE (op) == PLUS);
- if (n >= 4)
- return 0;
- addends[n] = op;
-
- for (i = n; i >= 0; --i)
+ else if (regparm < 3)
{
- op = addends[i];
- switch (GET_CODE (op))
+ if (!DECL_STATIC_CHAIN (cfun->decl))
+ return CX_REG;
+ else
{
- case MULT:
- if (index)
- return 0;
- index = XEXP (op, 0);
- scale_rtx = XEXP (op, 1);
- break;
-
- case ASHIFT:
+ if (regparm >= 2)
+ {
+ sorry ("-fsplit-stack does not support 2 register "
+ " parameters for a nested function");
+ return INVALID_REGNUM;
+ }
+ return DX_REG;
+ }
+ }
+ else
+ {
+ /* FIXME: We could make this work by pushing a register
+ around the addition and comparison. */
+ sorry ("-fsplit-stack does not support 3 register parameters");
+ return INVALID_REGNUM;
+ }
+ }
+}
+
+/* A SYMBOL_REF for the function which allocates new stackspace for
+ -fsplit-stack. */
+
+static GTY(()) rtx split_stack_fn;
+
+/* Handle -fsplit-stack. These are the first instructions in the
+ function, even before the regular prologue. */
+
+void
+ix86_expand_split_stack_prologue (void)
+{
+ struct ix86_frame frame;
+ HOST_WIDE_INT allocate;
+ int args_size;
+ rtx label, limit, current, jump_insn, allocate_rtx, call_insn, call_fusage;
+ rtx scratch_reg = NULL_RTX;
+ rtx varargs_label = NULL_RTX;
+
+ gcc_assert (flag_split_stack && reload_completed);
+
+ ix86_finalize_stack_realign_flags ();
+ ix86_compute_frame_layout (&frame);
+ allocate = frame.stack_pointer_offset - INCOMING_FRAME_SP_OFFSET;
+
+ /* This is the label we will branch to if we have enough stack
+ space. We expect the basic block reordering pass to reverse this
+ branch if optimizing, so that we branch in the unlikely case. */
+ label = gen_label_rtx ();
+
+ /* We need to compare the stack pointer minus the frame size with
+ the stack boundary in the TCB. The stack boundary always gives
+ us SPLIT_STACK_AVAILABLE bytes, so if we need less than that we
+ can compare directly. Otherwise we need to do an addition. */
+
+ limit = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx),
+ UNSPEC_STACK_CHECK);
+ limit = gen_rtx_CONST (Pmode, limit);
+ limit = gen_rtx_MEM (Pmode, limit);
+ if (allocate < SPLIT_STACK_AVAILABLE)
+ current = stack_pointer_rtx;
+ else
+ {
+ unsigned int scratch_regno;
+ rtx offset;
+
+ /* We need a scratch register to hold the stack pointer minus
+ the required frame size. Since this is the very start of the
+ function, the scratch register can be any caller-saved
+ register which is not used for parameters. */
+ offset = GEN_INT (- allocate);
+ scratch_regno = split_stack_prologue_scratch_regno ();
+ if (scratch_regno == INVALID_REGNUM)
+ return;
+ scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
+ if (!TARGET_64BIT || x86_64_immediate_operand (offset, Pmode))
+ {
+ /* We don't use ix86_gen_add3 in this case because it will
+ want to split to lea, but when not optimizing the insn
+ will not be split after this point. */
+ emit_insn (gen_rtx_SET (VOIDmode, scratch_reg,
+ gen_rtx_PLUS (Pmode, stack_pointer_rtx,
+ offset)));
+ }
+ else
+ {
+ emit_move_insn (scratch_reg, offset);
+ emit_insn (gen_adddi3 (scratch_reg, scratch_reg,
+ stack_pointer_rtx));
+ }
+ current = scratch_reg;
+ }
+
+ ix86_expand_branch (GEU, current, limit, label);
+ jump_insn = get_last_insn ();
+ JUMP_LABEL (jump_insn) = label;
+
+ /* Mark the jump as very likely to be taken. */
+ add_reg_note (jump_insn, REG_BR_PROB,
+ GEN_INT (REG_BR_PROB_BASE - REG_BR_PROB_BASE / 100));
+
+ /* Get more stack space. We pass in the desired stack space and the
+ size of the arguments to copy to the new stack. In 32-bit mode
+ we push the parameters; __morestack will return on a new stack
+ anyhow. In 64-bit mode we pass the parameters in r10 and
+ r11. */
+ allocate_rtx = GEN_INT (allocate);
+ args_size = crtl->args.size >= 0 ? crtl->args.size : 0;
+ call_fusage = NULL_RTX;
+ if (TARGET_64BIT)
+ {
+ rtx reg;
+
+ reg = gen_rtx_REG (Pmode, R10_REG);
+
+ /* If this function uses a static chain, it will be in %r10.
+ Preserve it across the call to __morestack. */
+ if (DECL_STATIC_CHAIN (cfun->decl))
+ {
+ rtx rax;
+
+ rax = gen_rtx_REG (Pmode, AX_REG);
+ emit_move_insn (rax, reg);
+ use_reg (&call_fusage, rax);
+ }
+
+ emit_move_insn (reg, allocate_rtx);
+ use_reg (&call_fusage, reg);
+ reg = gen_rtx_REG (Pmode, R11_REG);
+ emit_move_insn (reg, GEN_INT (args_size));
+ use_reg (&call_fusage, reg);
+ }
+ else
+ {
+ emit_insn (gen_push (GEN_INT (args_size)));
+ emit_insn (gen_push (allocate_rtx));
+ }
+ if (split_stack_fn == NULL_RTX)
+ split_stack_fn = gen_rtx_SYMBOL_REF (Pmode, "__morestack");
+ call_insn = ix86_expand_call (NULL_RTX, gen_rtx_MEM (QImode, split_stack_fn),
+ GEN_INT (UNITS_PER_WORD), constm1_rtx,
+ NULL_RTX, 0);
+ add_function_usage_to (call_insn, call_fusage);
+
+ /* In order to make call/return prediction work right, we now need
+ to execute a return instruction. See
+ libgcc/config/i386/morestack.S for the details on how this works.
+
+ For flow purposes gcc must not see this as a return
+ instruction--we need control flow to continue at the subsequent
+ label. Therefore, we use an unspec. */
+ gcc_assert (crtl->args.pops_args < 65536);
+ emit_insn (gen_split_stack_return (GEN_INT (crtl->args.pops_args)));
+
+ /* If we are in 64-bit mode and this function uses a static chain,
+ we saved %r10 in %rax before calling _morestack. */
+ if (TARGET_64BIT && DECL_STATIC_CHAIN (cfun->decl))
+ emit_move_insn (gen_rtx_REG (Pmode, R10_REG),
+ gen_rtx_REG (Pmode, AX_REG));
+
+ /* If this function calls va_start, we need to store a pointer to
+ the arguments on the old stack, because they may not have been
+ all copied to the new stack. At this point the old stack can be
+ found at the frame pointer value used by __morestack, because
+ __morestack has set that up before calling back to us. Here we
+ store that pointer in a scratch register, and in
+ ix86_expand_prologue we store the scratch register in a stack
+ slot. */
+ if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
+ {
+ unsigned int scratch_regno;
+ rtx frame_reg;
+ int words;
+
+ scratch_regno = split_stack_prologue_scratch_regno ();
+ scratch_reg = gen_rtx_REG (Pmode, scratch_regno);
+ frame_reg = gen_rtx_REG (Pmode, BP_REG);
+
+ /* 64-bit:
+ fp -> old fp value
+ return address within this function
+ return address of caller of this function
+ stack arguments
+ So we add three words to get to the stack arguments.
+
+ 32-bit:
+ fp -> old fp value
+ return address within this function
+ first argument to __morestack
+ second argument to __morestack
+ return address of caller of this function
+ stack arguments
+ So we add five words to get to the stack arguments.
+ */
+ words = TARGET_64BIT ? 3 : 5;
+ emit_insn (gen_rtx_SET (VOIDmode, scratch_reg,
+ gen_rtx_PLUS (Pmode, frame_reg,
+ GEN_INT (words * UNITS_PER_WORD))));
+
+ varargs_label = gen_label_rtx ();
+ emit_jump_insn (gen_jump (varargs_label));
+ JUMP_LABEL (get_last_insn ()) = varargs_label;
+
+ emit_barrier ();
+ }
+
+ emit_label (label);
+ LABEL_NUSES (label) = 1;
+
+ /* If this function calls va_start, we now have to set the scratch
+ register for the case where we do not call __morestack. In this
+ case we need to set it based on the stack pointer. */
+ if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
+ {
+ emit_insn (gen_rtx_SET (VOIDmode, scratch_reg,
+ gen_rtx_PLUS (Pmode, stack_pointer_rtx,
+ GEN_INT (UNITS_PER_WORD))));
+
+ emit_label (varargs_label);
+ LABEL_NUSES (varargs_label) = 1;
+ }
+}
+
+/* We may have to tell the dataflow pass that the split stack prologue
+ is initializing a scratch register. */
+
+static void
+ix86_live_on_entry (bitmap regs)
+{
+ if (cfun->machine->split_stack_varargs_pointer != NULL_RTX)
+ {
+ gcc_assert (flag_split_stack);
+ bitmap_set_bit (regs, split_stack_prologue_scratch_regno ());
+ }
+}
+\f
+/* Extract the parts of an RTL expression that is a valid memory address
+ for an instruction. Return 0 if the structure of the address is
+ grossly off. Return -1 if the address contains ASHIFT, so it is not
+ strictly valid, but still used for computing length of lea instruction. */
+
+int
+ix86_decompose_address (rtx addr, struct ix86_address *out)
+{
+ rtx base = NULL_RTX, index = NULL_RTX, disp = NULL_RTX;
+ rtx base_reg, index_reg;
+ HOST_WIDE_INT scale = 1;
+ rtx scale_rtx = NULL_RTX;
+ rtx tmp;
+ int retval = 1;
+ enum ix86_address_seg seg = SEG_DEFAULT;
+
+ if (REG_P (addr) || GET_CODE (addr) == SUBREG)
+ base = addr;
+ else if (GET_CODE (addr) == PLUS)
+ {
+ rtx addends[4], op;
+ int n = 0, i;
+
+ op = addr;
+ do
+ {
+ if (n >= 4)
+ return 0;
+ addends[n++] = XEXP (op, 1);
+ op = XEXP (op, 0);
+ }
+ while (GET_CODE (op) == PLUS);
+ if (n >= 4)
+ return 0;
+ addends[n] = op;
+
+ for (i = n; i >= 0; --i)
+ {
+ op = addends[i];
+ switch (GET_CODE (op))
+ {
+ case MULT:
+ if (index)
+ return 0;
+ index = XEXP (op, 0);
+ scale_rtx = XEXP (op, 1);
+ break;
+
+ case ASHIFT:
if (index)
return 0;
index = XEXP (op, 0);
if (TARGET_DLLIMPORT_DECL_ATTRIBUTES
&& SYMBOL_REF_DLLIMPORT_P (x))
return false;
+
+#if TARGET_MACHO
+ /* mdynamic-no-pic */
+ if (MACHO_DYNAMIC_NO_PIC_P)
+ return machopic_symbol_defined_p (x);
+#endif
break;
case CONST_DOUBLE:
case UNSPEC_DTPOFF:
break;
+ case UNSPEC_STACK_CHECK:
+ gcc_assert (flag_split_stack);
+ break;
+
default:
/* Invalid address unspec. */
return false;
/* Non-constant pic memory reference. */
return false;
}
- else if (! legitimate_pic_address_disp_p (disp))
+ else if ((!TARGET_MACHO || flag_pic)
+ && ! legitimate_pic_address_disp_p (disp))
/* Displacement is an invalid pic construct. */
return false;
+#if TARGET_MACHO
+ else if (MACHO_DYNAMIC_NO_PIC_P && !legitimate_constant_p (disp))
+ /* displacment must be referenced via non_lazy_pointer */
+ return false;
+#endif
/* This code used to verify that a symbolic pic displacement
includes the pic_offset_table_rtx register.
if (flag_pic && SYMBOLIC_CONST (x))
return legitimize_pic_address (x, 0);
+#if TARGET_MACHO
+ if (MACHO_DYNAMIC_NO_PIC_P && SYMBOLIC_CONST (x))
+ return machopic_indirect_data_reference (x, 0);
+#endif
+
/* Canonicalize shifts by 0, 1, 2, 3 into multiply */
if (GET_CODE (x) == ASHIFT
&& CONST_INT_P (XEXP (x, 1))
break;
case UNSPEC:
+ if (XINT (x, 1) == UNSPEC_STACK_CHECK)
+ {
+ bool f = i386_asm_output_addr_const_extra (file, x);
+ gcc_assert (f);
+ break;
+ }
+
gcc_assert (XVECLEN (x, 0) == 1);
output_pic_addr_const (file, XVECEXP (x, 0, 0), code);
switch (XINT (x, 1))
return REG_P (x) && REGNO (x) == PIC_OFFSET_TABLE_REGNUM;
}
+/* Helper function for ix86_delegitimize_address.
+ Attempt to delegitimize TLS local-exec accesses. */
+
+static rtx
+ix86_delegitimize_tls_address (rtx orig_x)
+{
+ rtx x = orig_x, unspec;
+ struct ix86_address addr;
+
+ if (!TARGET_TLS_DIRECT_SEG_REFS)
+ return orig_x;
+ if (MEM_P (x))
+ x = XEXP (x, 0);
+ if (GET_CODE (x) != PLUS || GET_MODE (x) != Pmode)
+ return orig_x;
+ if (ix86_decompose_address (x, &addr) == 0
+ || addr.seg != (TARGET_64BIT ? SEG_FS : SEG_GS)
+ || addr.disp == NULL_RTX
+ || GET_CODE (addr.disp) != CONST)
+ return orig_x;
+ unspec = XEXP (addr.disp, 0);
+ if (GET_CODE (unspec) == PLUS && CONST_INT_P (XEXP (unspec, 1)))
+ unspec = XEXP (unspec, 0);
+ if (GET_CODE (unspec) != UNSPEC || XINT (unspec, 1) != UNSPEC_NTPOFF)
+ return orig_x;
+ x = XVECEXP (unspec, 0, 0);
+ gcc_assert (GET_CODE (x) == SYMBOL_REF);
+ if (unspec != XEXP (addr.disp, 0))
+ x = gen_rtx_PLUS (Pmode, x, XEXP (XEXP (addr.disp, 0), 1));
+ if (addr.index)
+ {
+ rtx idx = addr.index;
+ if (addr.scale != 1)
+ idx = gen_rtx_MULT (Pmode, idx, GEN_INT (addr.scale));
+ x = gen_rtx_PLUS (Pmode, idx, x);
+ }
+ if (addr.base)
+ x = gen_rtx_PLUS (Pmode, addr.base, x);
+ if (MEM_P (orig_x))
+ x = replace_equiv_address_nv (orig_x, x);
+ return x;
+}
+
/* In the name of slightly smaller debug output, and to cater to
general assembler lossage, recognize PIC+GOTOFF and turn it back
into a direct symbol reference.
|| GET_CODE (XEXP (x, 0)) != UNSPEC
|| XINT (XEXP (x, 0), 1) != UNSPEC_GOTPCREL
|| !MEM_P (orig_x))
- return orig_x;
+ return ix86_delegitimize_tls_address (orig_x);
x = XVECEXP (XEXP (x, 0), 0, 0);
if (GET_MODE (orig_x) != Pmode)
return simplify_gen_subreg (GET_MODE (orig_x), x, Pmode, 0);
if (GET_CODE (x) != PLUS
|| GET_CODE (XEXP (x, 1)) != CONST)
- return orig_x;
+ return ix86_delegitimize_tls_address (orig_x);
if (ix86_pic_register_p (XEXP (x, 0)))
/* %ebx + GOT/GOTOFF */
result = XVECEXP (x, 0, 0);
if (! result)
- return orig_x;
+ return ix86_delegitimize_tls_address (orig_x);
if (const_addend)
result = gen_rtx_CONST (Pmode, gen_rtx_PLUS (Pmode, result, const_addend));
}
if (CONST_INT_P (x))
fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (x));
- else if (flag_pic)
+ else if (flag_pic || MACHOPIC_INDIRECT)
output_pic_addr_const (file, x, code);
else
output_addr_const (file, x);
break;
#endif
+ case UNSPEC_STACK_CHECK:
+ {
+ int offset;
+
+ gcc_assert (flag_split_stack);
+
+#ifdef TARGET_THREAD_SPLIT_STACK_OFFSET
+ offset = TARGET_THREAD_SPLIT_STACK_OFFSET;
+#else
+ gcc_unreachable ();
+#endif
+
+ fprintf (file, "%s:%d", TARGET_64BIT ? "%fs" : "%gs", offset);
+ }
+ break;
+
default:
return false;
}
return true;
}
\f
-/* Split one or more DImode RTL references into pairs of SImode
+/* Split one or more double-mode RTL references into pairs of half-mode
references. The RTL can be REG, offsettable MEM, integer constant, or
- CONST_DOUBLE. "operands" is a pointer to an array of DImode RTL to
+ CONST_DOUBLE. "operands" is a pointer to an array of double-mode RTLs to
split and "num" is its length. lo_half and hi_half are output arrays
that parallel "operands". */
void
-split_di (rtx operands[], int num, rtx lo_half[], rtx hi_half[])
+split_double_mode (enum machine_mode mode, rtx operands[],
+ int num, rtx lo_half[], rtx hi_half[])
{
+ enum machine_mode half_mode;
+ unsigned int byte;
+
+ switch (mode)
+ {
+ case TImode:
+ half_mode = DImode;
+ break;
+ case DImode:
+ half_mode = SImode;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ byte = GET_MODE_SIZE (half_mode);
+
while (num--)
{
rtx op = operands[num];
but we still have to handle it. */
if (MEM_P (op))
{
- lo_half[num] = adjust_address (op, SImode, 0);
- hi_half[num] = adjust_address (op, SImode, 4);
+ lo_half[num] = adjust_address (op, half_mode, 0);
+ hi_half[num] = adjust_address (op, half_mode, byte);
}
else
{
- lo_half[num] = simplify_gen_subreg (SImode, op,
+ lo_half[num] = simplify_gen_subreg (half_mode, op,
GET_MODE (op) == VOIDmode
- ? DImode : GET_MODE (op), 0);
- hi_half[num] = simplify_gen_subreg (SImode, op,
+ ? mode : GET_MODE (op), 0);
+ hi_half[num] = simplify_gen_subreg (half_mode, op,
GET_MODE (op) == VOIDmode
- ? DImode : GET_MODE (op), 4);
- }
- }
-}
-/* Split one or more TImode RTL references into pairs of DImode
- references. The RTL can be REG, offsettable MEM, integer constant, or
- CONST_DOUBLE. "operands" is a pointer to an array of DImode RTL to
- split and "num" is its length. lo_half and hi_half are output arrays
- that parallel "operands". */
-
-void
-split_ti (rtx operands[], int num, rtx lo_half[], rtx hi_half[])
-{
- while (num--)
- {
- rtx op = operands[num];
-
- /* simplify_subreg refuse to split volatile memory addresses, but we
- still have to handle it. */
- if (MEM_P (op))
- {
- lo_half[num] = adjust_address (op, DImode, 0);
- hi_half[num] = adjust_address (op, DImode, 8);
- }
- else
- {
- lo_half[num] = simplify_gen_subreg (DImode, op, TImode, 0);
- hi_half[num] = simplify_gen_subreg (DImode, op, TImode, 8);
+ ? mode : GET_MODE (op), byte);
}
}
}
}
}
- if (flag_pic && mode == Pmode && symbolic_operand (op1, Pmode))
+ if ((flag_pic || MACHOPIC_INDIRECT)
+ && mode == Pmode && symbolic_operand (op1, Pmode))
{
if (TARGET_MACHO && !TARGET_64BIT)
{
#if TARGET_MACHO
- if (MACHOPIC_PURE)
+ /* dynamic-no-pic */
+ if (MACHOPIC_INDIRECT)
{
rtx temp = ((reload_in_progress
|| ((op0 && REG_P (op0))
&& mode == Pmode))
? op0 : gen_reg_rtx (Pmode));
op1 = machopic_indirect_data_reference (op1, temp);
- op1 = machopic_legitimize_pic_address (op1, mode,
- temp == op1 ? 0 : temp);
+ if (MACHOPIC_PURE)
+ op1 = machopic_legitimize_pic_address (op1, mode,
+ temp == op1 ? 0 : temp);
+ }
+ if (op0 != op1 && GET_CODE (op0) != MEM)
+ {
+ rtx insn = gen_rtx_SET (VOIDmode, op0, op1);
+ emit_insn (insn);
+ return;
}
- else if (MACHOPIC_INDIRECT)
- op1 = machopic_indirect_data_reference (op1, 0);
- if (op0 == op1)
+ if (GET_CODE (op0) == MEM)
+ op1 = force_reg (Pmode, op1);
+ else
+ {
+ rtx temp = op0;
+ if (GET_CODE (temp) != REG)
+ temp = gen_reg_rtx (Pmode);
+ temp = legitimize_pic_address (op1, temp);
+ if (temp == op0)
return;
+ op1 = temp;
+ }
+ /* dynamic-no-pic */
#endif
}
else
gcc_assert (code == PLUS);
emit_insn (op);
}
+ else if (reload_completed
+ && code == PLUS
+ && !rtx_equal_p (dst, src1))
+ {
+ /* This is going to be an LEA; avoid splitting it later. */
+ emit_insn (op);
+ }
else
{
clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
/* Source 1 cannot be a non-matching memory. */
if (MEM_P (src1) && !rtx_equal_p (dst, src1))
- return false;
+ {
+ /* Support "andhi/andsi/anddi" as a zero-extending move. */
+ return (code == AND
+ && (mode == HImode
+ || mode == SImode
+ || (TARGET_64BIT && mode == DImode))
+ && CONST_INT_P (src2)
+ && (INTVAL (src2) == 0xff
+ || INTVAL (src2) == 0xffff));
+ }
return true;
}
emit_move_insn (operands[0], dst);
}
+/* Split 32bit/64bit divmod with 8bit unsigned divmod if dividend and
+ divisor are within the the range [0-255]. */
+
+void
+ix86_split_idivmod (enum machine_mode mode, rtx operands[],
+ bool signed_p)
+{
+ rtx end_label, qimode_label;
+ rtx insn, div, mod;
+ rtx scratch, tmp0, tmp1, tmp2;
+ rtx (*gen_divmod4_1) (rtx, rtx, rtx, rtx);
+ rtx (*gen_zero_extend) (rtx, rtx);
+ rtx (*gen_test_ccno_1) (rtx, rtx);
+
+ switch (mode)
+ {
+ case SImode:
+ gen_divmod4_1 = signed_p ? gen_divmodsi4_1 : gen_udivmodsi4_1;
+ gen_test_ccno_1 = gen_testsi_ccno_1;
+ gen_zero_extend = gen_zero_extendqisi2;
+ break;
+ case DImode:
+ gen_divmod4_1 = signed_p ? gen_divmoddi4_1 : gen_udivmoddi4_1;
+ gen_test_ccno_1 = gen_testdi_ccno_1;
+ gen_zero_extend = gen_zero_extendqidi2;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ end_label = gen_label_rtx ();
+ qimode_label = gen_label_rtx ();
+
+ scratch = gen_reg_rtx (mode);
+
+ /* Use 8bit unsigned divimod if dividend and divisor are within the
+ the range [0-255]. */
+ emit_move_insn (scratch, operands[2]);
+ scratch = expand_simple_binop (mode, IOR, scratch, operands[3],
+ scratch, 1, OPTAB_DIRECT);
+ emit_insn (gen_test_ccno_1 (scratch, GEN_INT (-0x100)));
+ tmp0 = gen_rtx_REG (CCNOmode, FLAGS_REG);
+ tmp0 = gen_rtx_EQ (VOIDmode, tmp0, const0_rtx);
+ tmp0 = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp0,
+ gen_rtx_LABEL_REF (VOIDmode, qimode_label),
+ pc_rtx);
+ insn = emit_jump_insn (gen_rtx_SET (VOIDmode, pc_rtx, tmp0));
+ predict_jump (REG_BR_PROB_BASE * 50 / 100);
+ JUMP_LABEL (insn) = qimode_label;
+
+ /* Generate original signed/unsigned divimod. */
+ div = gen_divmod4_1 (operands[0], operands[1],
+ operands[2], operands[3]);
+ emit_insn (div);
+
+ /* Branch to the end. */
+ emit_jump_insn (gen_jump (end_label));
+ emit_barrier ();
+
+ /* Generate 8bit unsigned divide. */
+ emit_label (qimode_label);
+ /* Don't use operands[0] for result of 8bit divide since not all
+ registers support QImode ZERO_EXTRACT. */
+ tmp0 = simplify_gen_subreg (HImode, scratch, mode, 0);
+ tmp1 = simplify_gen_subreg (HImode, operands[2], mode, 0);
+ tmp2 = simplify_gen_subreg (QImode, operands[3], mode, 0);
+ emit_insn (gen_udivmodhiqi3 (tmp0, tmp1, tmp2));
+
+ if (signed_p)
+ {
+ div = gen_rtx_DIV (SImode, operands[2], operands[3]);
+ mod = gen_rtx_MOD (SImode, operands[2], operands[3]);
+ }
+ else
+ {
+ div = gen_rtx_UDIV (SImode, operands[2], operands[3]);
+ mod = gen_rtx_UMOD (SImode, operands[2], operands[3]);
+ }
+
+ /* Extract remainder from AH. */
+ tmp1 = gen_rtx_ZERO_EXTRACT (mode, tmp0, GEN_INT (8), GEN_INT (8));
+ if (REG_P (operands[1]))
+ insn = emit_move_insn (operands[1], tmp1);
+ else
+ {
+ /* Need a new scratch register since the old one has result
+ of 8bit divide. */
+ scratch = gen_reg_rtx (mode);
+ emit_move_insn (scratch, tmp1);
+ insn = emit_move_insn (operands[1], scratch);
+ }
+ set_unique_reg_note (insn, REG_EQUAL, mod);
+
+ /* Zero extend quotient from AL. */
+ tmp1 = gen_lowpart (QImode, tmp0);
+ insn = emit_insn (gen_zero_extend (operands[0], tmp1));
+ set_unique_reg_note (insn, REG_EQUAL, div);
+
+ emit_label (end_label);
+}
+
#define LEA_SEARCH_THRESHOLD 12
/* Search backward for non-agu definition of register number REGNO1
else
{
int dist_define, dist_use;
+
+ /* Return false if REGNO0 isn't used in memory address. */
+ dist_use = distance_agu_use (regno0, insn);
+ if (dist_use <= 0)
+ return false;
+
dist_define = distance_non_agu_define (regno1, regno2, insn);
if (dist_define <= 0)
return true;
/* If this insn has both backward non-agu dependence and forward
agu dependence, the one with short distance take effect. */
- dist_use = distance_agu_use (regno0, insn);
- if (dist_use <= 0
- || (dist_define + IX86_LEA_PRIORITY) < dist_use)
+ if ((dist_define + IX86_LEA_PRIORITY) < dist_use)
return false;
return true;
rtvec v;
switch (mode)
{
- case SImode:
+ case V4SImode:
gcc_assert (vect);
v = gen_rtvec (4, value, value, value, value);
return gen_rtx_CONST_VECTOR (V4SImode, v);
- case DImode:
+ case V2DImode:
gcc_assert (vect);
v = gen_rtvec (2, value, value);
return gen_rtx_CONST_VECTOR (V2DImode, v);
- case SFmode:
+ case V8SFmode:
+ if (vect)
+ v = gen_rtvec (8, value, value, value, value,
+ value, value, value, value);
+ else
+ v = gen_rtvec (8, value, CONST0_RTX (SFmode),
+ CONST0_RTX (SFmode), CONST0_RTX (SFmode),
+ CONST0_RTX (SFmode), CONST0_RTX (SFmode),
+ CONST0_RTX (SFmode), CONST0_RTX (SFmode));
+ return gen_rtx_CONST_VECTOR (V8SFmode, v);
+
+ case V4SFmode:
if (vect)
v = gen_rtvec (4, value, value, value, value);
else
CONST0_RTX (SFmode), CONST0_RTX (SFmode));
return gen_rtx_CONST_VECTOR (V4SFmode, v);
- case DFmode:
+ case V4DFmode:
+ if (vect)
+ v = gen_rtvec (4, value, value, value, value);
+ else
+ v = gen_rtvec (4, value, CONST0_RTX (DFmode),
+ CONST0_RTX (DFmode), CONST0_RTX (DFmode));
+ return gen_rtx_CONST_VECTOR (V4DFmode, v);
+
+ case V2DFmode:
if (vect)
v = gen_rtvec (2, value, value);
else
/* Find the sign bit, sign extended to 2*HWI. */
switch (mode)
{
- case SImode:
- case SFmode:
- imode = SImode;
- vec_mode = (mode == SImode) ? V4SImode : V4SFmode;
- lo = 0x80000000, hi = lo < 0;
+ case V4SImode:
+ case V8SFmode:
+ case V4SFmode:
+ vec_mode = mode;
+ mode = GET_MODE_INNER (mode);
+ imode = SImode;
+ lo = 0x80000000, hi = lo < 0;
break;
- case DImode:
- case DFmode:
+ case V2DImode:
+ case V4DFmode:
+ case V2DFmode:
+ vec_mode = mode;
+ mode = GET_MODE_INNER (mode);
imode = DImode;
- vec_mode = (mode == DImode) ? V2DImode : V2DFmode;
if (HOST_BITS_PER_WIDE_INT >= 64)
lo = (HOST_WIDE_INT)1 << shift, hi = -1;
else
if (vec_mode == VOIDmode)
return force_reg (mode, mask);
- v = ix86_build_const_vector (mode, vect, mask);
+ v = ix86_build_const_vector (vec_mode, vect, mask);
return force_reg (vec_mode, v);
}
ix86_expand_fp_absneg_operator (enum rtx_code code, enum machine_mode mode,
rtx operands[])
{
- rtx mask, set, use, clob, dst, src;
+ rtx mask, set, dst, src;
bool use_sse = false;
bool vector_mode = VECTOR_MODE_P (mode);
- enum machine_mode elt_mode = mode;
+ enum machine_mode vmode = mode;
if (vector_mode)
- {
- elt_mode = GET_MODE_INNER (mode);
- use_sse = true;
- }
+ use_sse = true;
else if (mode == TFmode)
use_sse = true;
else if (TARGET_SSE_MATH)
- use_sse = SSE_FLOAT_MODE_P (mode);
+ {
+ use_sse = SSE_FLOAT_MODE_P (mode);
+ if (mode == SFmode)
+ vmode = V4SFmode;
+ else if (mode == DFmode)
+ vmode = V2DFmode;
+ }
/* NEG and ABS performed with SSE use bitwise mask operations.
Create the appropriate mask now. */
if (use_sse)
- mask = ix86_build_signbit_mask (elt_mode, vector_mode, code == ABS);
+ mask = ix86_build_signbit_mask (vmode, vector_mode, code == ABS);
else
mask = NULL_RTX;
dst = operands[0];
src = operands[1];
- if (vector_mode)
- {
- set = gen_rtx_fmt_ee (code == NEG ? XOR : AND, mode, src, mask);
- set = gen_rtx_SET (VOIDmode, dst, set);
- emit_insn (set);
- }
- else
+ set = gen_rtx_fmt_e (code, mode, src);
+ set = gen_rtx_SET (VOIDmode, dst, set);
+
+ if (mask)
{
- set = gen_rtx_fmt_e (code, mode, src);
- set = gen_rtx_SET (VOIDmode, dst, set);
- if (mask)
- {
- use = gen_rtx_USE (VOIDmode, mask);
+ rtx use, clob;
+ rtvec par;
+
+ use = gen_rtx_USE (VOIDmode, mask);
+ if (vector_mode)
+ par = gen_rtvec (2, set, use);
+ else
+ {
clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
- emit_insn (gen_rtx_PARALLEL (VOIDmode,
- gen_rtvec (3, set, use, clob)));
+ par = gen_rtvec (3, set, use, clob);
}
- else
- emit_insn (set);
+ emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
}
+ else
+ emit_insn (set);
}
/* Expand a copysign operation. Special case operand 0 being a constant. */
void
ix86_expand_copysign (rtx operands[])
{
- enum machine_mode mode;
+ enum machine_mode mode, vmode;
rtx dest, op0, op1, mask, nmask;
dest = operands[0];
mode = GET_MODE (dest);
+ if (mode == SFmode)
+ vmode = V4SFmode;
+ else if (mode == DFmode)
+ vmode = V2DFmode;
+ else
+ vmode = mode;
+
if (GET_CODE (op0) == CONST_DOUBLE)
{
rtx (*copysign_insn)(rtx, rtx, rtx, rtx);
if (mode == SFmode || mode == DFmode)
{
- enum machine_mode vmode;
-
- vmode = mode == SFmode ? V4SFmode : V2DFmode;
-
if (op0 == CONST0_RTX (mode))
op0 = CONST0_RTX (vmode);
else
{
- rtx v = ix86_build_const_vector (mode, false, op0);
+ rtx v = ix86_build_const_vector (vmode, false, op0);
op0 = force_reg (vmode, v);
}
else if (op0 != CONST0_RTX (mode))
op0 = force_reg (mode, op0);
- mask = ix86_build_signbit_mask (mode, 0, 0);
+ mask = ix86_build_signbit_mask (vmode, 0, 0);
if (mode == SFmode)
copysign_insn = gen_copysignsf3_const;
{
rtx (*copysign_insn)(rtx, rtx, rtx, rtx, rtx, rtx);
- nmask = ix86_build_signbit_mask (mode, 0, 1);
- mask = ix86_build_signbit_mask (mode, 0, 0);
+ nmask = ix86_build_signbit_mask (vmode, 0, 1);
+ mask = ix86_build_signbit_mask (vmode, 0, 0);
if (mode == SFmode)
copysign_insn = gen_copysignsf3_var;
void
ix86_expand_branch (enum rtx_code code, rtx op0, rtx op1, rtx label)
{
+ enum machine_mode mode = GET_MODE (op0);
rtx tmp;
- switch (GET_MODE (op0))
+ switch (mode)
{
case SFmode:
case DFmode:
tmp = op0, op0 = op1, op1 = tmp;
code = swap_condition (code);
}
- if (GET_MODE (op0) == DImode)
- {
- split_di (&op0, 1, lo+0, hi+0);
- split_di (&op1, 1, lo+1, hi+1);
- submode = SImode;
- }
- else
- {
- split_ti (&op0, 1, lo+0, hi+0);
- split_ti (&op1, 1, lo+1, hi+1);
- submode = DImode;
- }
+
+ split_double_mode (mode, &op0, 1, lo+0, hi+0);
+ split_double_mode (mode, &op1, 1, lo+1, hi+1);
+
+ submode = mode == DImode ? SImode : DImode;
/* When comparing for equality, we can use (hi0^hi1)|(lo0^lo1) to
avoid two branches. This costs one extra insn, so disable when
enum machine_mode mode =
GET_MODE (op0) != VOIDmode ? GET_MODE (op0) : GET_MODE (op1);
- /* Do not handle DImode compares that go through special path. */
+ /* Do not handle double-mode compares that go through special path. */
if (mode == (TARGET_64BIT ? TImode : DImode))
return false;
/* Subtract (-(INT MAX) - 1) from both operands to make
them signed. */
- mask = ix86_build_signbit_mask (GET_MODE_INNER (mode),
- true, false);
+ mask = ix86_build_signbit_mask (mode, true, false);
gen_sub3 = (mode == V4SImode
? gen_subv4si3 : gen_subv2di3);
t1 = gen_reg_rtx (mode);
}
-/* Split operands 0 and 1 into SImode parts. Similar to split_di, but
- works for floating pointer parameters and nonoffsetable memories.
+/* Split operands 0 and 1 into half-mode parts. Similar to split_double_mode,
+ but works for floating pointer parameters and nonoffsetable memories.
For pushes, it returns just stack offsets; the values will be saved
in the right order. Maximally three parts are generated. */
if (!TARGET_64BIT)
{
if (mode == DImode)
- split_di (&operand, 1, &parts[0], &parts[1]);
+ split_double_mode (mode, &operand, 1, &parts[0], &parts[1]);
else
{
int i;
else
{
if (mode == TImode)
- split_ti (&operand, 1, &parts[0], &parts[1]);
+ split_double_mode (mode, &operand, 1, &parts[0], &parts[1]);
if (mode == XFmode || mode == TFmode)
{
enum machine_mode upper_mode = mode==XFmode ? SImode : DImode;
/* The DFmode expanders may ask us to move double.
For 64bit target this is single move. By hiding the fact
here we simplify i386.md splitters. */
- if (GET_MODE_SIZE (GET_MODE (operands[0])) == 8 && TARGET_64BIT)
+ if (TARGET_64BIT && GET_MODE_SIZE (GET_MODE (operands[0])) == 8)
{
/* Optimize constant pool reference to immediates. This is used by
fp moves, that force all constants to memory to allow combining. */
static void
ix86_expand_ashl_const (rtx operand, int count, enum machine_mode mode)
{
- if (count == 1)
+ rtx (*insn)(rtx, rtx, rtx);
+
+ if (count == 1
+ || (count * ix86_cost->add <= ix86_cost->shift_const
+ && !optimize_insn_for_size_p ()))
{
- emit_insn ((mode == DImode
- ? gen_addsi3
- : gen_adddi3) (operand, operand, operand));
+ insn = mode == DImode ? gen_addsi3 : gen_adddi3;
+ while (count-- > 0)
+ emit_insn (insn (operand, operand, operand));
}
- else if (!optimize_insn_for_size_p ()
- && count * ix86_cost->add <= ix86_cost->shift_const)
+ else
{
- int i;
- for (i=0; i<count; i++)
- {
- emit_insn ((mode == DImode
- ? gen_addsi3
- : gen_adddi3) (operand, operand, operand));
- }
+ insn = mode == DImode ? gen_ashlsi3 : gen_ashldi3;
+ emit_insn (insn (operand, operand, GEN_INT (count)));
}
- else
- emit_insn ((mode == DImode
- ? gen_ashlsi3
- : gen_ashldi3) (operand, operand, GEN_INT (count)));
}
void
ix86_split_ashl (rtx *operands, rtx scratch, enum machine_mode mode)
{
+ rtx (*gen_ashl3)(rtx, rtx, rtx);
+ rtx (*gen_shld)(rtx, rtx, rtx);
+ int half_width = GET_MODE_BITSIZE (mode) >> 1;
+
rtx low[2], high[2];
int count;
- const int single_width = mode == DImode ? 32 : 64;
if (CONST_INT_P (operands[2]))
{
- (mode == DImode ? split_di : split_ti) (operands, 2, low, high);
- count = INTVAL (operands[2]) & (single_width * 2 - 1);
+ split_double_mode (mode, operands, 2, low, high);
+ count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
- if (count >= single_width)
+ if (count >= half_width)
{
emit_move_insn (high[0], low[1]);
emit_move_insn (low[0], const0_rtx);
- if (count > single_width)
- ix86_expand_ashl_const (high[0], count - single_width, mode);
+ if (count > half_width)
+ ix86_expand_ashl_const (high[0], count - half_width, mode);
}
else
{
+ gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld;
+
if (!rtx_equal_p (operands[0], operands[1]))
emit_move_insn (operands[0], operands[1]);
- emit_insn ((mode == DImode
- ? gen_x86_shld
- : gen_x86_64_shld) (high[0], low[0], GEN_INT (count)));
+
+ emit_insn (gen_shld (high[0], low[0], GEN_INT (count)));
ix86_expand_ashl_const (low[0], count, mode);
}
return;
}
- (mode == DImode ? split_di : split_ti) (operands, 1, low, high);
+ split_double_mode (mode, operands, 1, low, high);
+
+ gen_ashl3 = mode == DImode ? gen_ashlsi3 : gen_ashldi3;
if (operands[1] == const1_rtx)
{
ix86_expand_clear (low[0]);
ix86_expand_clear (high[0]);
- emit_insn (gen_testqi_ccz_1 (operands[2], GEN_INT (single_width)));
+ emit_insn (gen_testqi_ccz_1 (operands[2], GEN_INT (half_width)));
d = gen_lowpart (QImode, low[0]);
d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
pentium4 a bit; no one else seems to care much either way. */
else
{
+ enum machine_mode half_mode;
+ rtx (*gen_lshr3)(rtx, rtx, rtx);
+ rtx (*gen_and3)(rtx, rtx, rtx);
+ rtx (*gen_xor3)(rtx, rtx, rtx);
+ HOST_WIDE_INT bits;
rtx x;
+ if (mode == DImode)
+ {
+ half_mode = SImode;
+ gen_lshr3 = gen_lshrsi3;
+ gen_and3 = gen_andsi3;
+ gen_xor3 = gen_xorsi3;
+ bits = 5;
+ }
+ else
+ {
+ half_mode = DImode;
+ gen_lshr3 = gen_lshrdi3;
+ gen_and3 = gen_anddi3;
+ gen_xor3 = gen_xordi3;
+ bits = 6;
+ }
+
if (TARGET_PARTIAL_REG_STALL && !optimize_insn_for_size_p ())
- x = gen_rtx_ZERO_EXTEND (mode == DImode ? SImode : DImode, operands[2]);
+ x = gen_rtx_ZERO_EXTEND (half_mode, operands[2]);
else
- x = gen_lowpart (mode == DImode ? SImode : DImode, operands[2]);
+ x = gen_lowpart (half_mode, operands[2]);
emit_insn (gen_rtx_SET (VOIDmode, high[0], x));
- emit_insn ((mode == DImode
- ? gen_lshrsi3
- : gen_lshrdi3) (high[0], high[0],
- GEN_INT (mode == DImode ? 5 : 6)));
- emit_insn ((mode == DImode
- ? gen_andsi3
- : gen_anddi3) (high[0], high[0], const1_rtx));
+ emit_insn (gen_lshr3 (high[0], high[0], GEN_INT (bits)));
+ emit_insn (gen_and3 (high[0], high[0], const1_rtx));
emit_move_insn (low[0], high[0]);
- emit_insn ((mode == DImode
- ? gen_xorsi3
- : gen_xordi3) (low[0], low[0], const1_rtx));
+ emit_insn (gen_xor3 (low[0], low[0], const1_rtx));
}
- emit_insn ((mode == DImode
- ? gen_ashlsi3
- : gen_ashldi3) (low[0], low[0], operands[2]));
- emit_insn ((mode == DImode
- ? gen_ashlsi3
- : gen_ashldi3) (high[0], high[0], operands[2]));
+ emit_insn (gen_ashl3 (low[0], low[0], operands[2]));
+ emit_insn (gen_ashl3 (high[0], high[0], operands[2]));
return;
}
}
else
{
+ gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld;
+
if (!rtx_equal_p (operands[0], operands[1]))
emit_move_insn (operands[0], operands[1]);
- (mode == DImode ? split_di : split_ti) (operands, 1, low, high);
- emit_insn ((mode == DImode
- ? gen_x86_shld
- : gen_x86_64_shld) (high[0], low[0], operands[2]));
+ split_double_mode (mode, operands, 1, low, high);
+ emit_insn (gen_shld (high[0], low[0], operands[2]));
}
- emit_insn ((mode == DImode
- ? gen_ashlsi3
- : gen_ashldi3) (low[0], low[0], operands[2]));
+ emit_insn (gen_ashl3 (low[0], low[0], operands[2]));
if (TARGET_CMOVE && scratch)
{
+ rtx (*gen_x86_shift_adj_1)(rtx, rtx, rtx, rtx)
+ = mode == DImode ? gen_x86_shiftsi_adj_1 : gen_x86_shiftdi_adj_1;
+
ix86_expand_clear (scratch);
- emit_insn ((mode == DImode
- ? gen_x86_shiftsi_adj_1
- : gen_x86_shiftdi_adj_1) (high[0], low[0], operands[2],
- scratch));
+ emit_insn (gen_x86_shift_adj_1 (high[0], low[0], operands[2], scratch));
}
else
- emit_insn ((mode == DImode
- ? gen_x86_shiftsi_adj_2
- : gen_x86_shiftdi_adj_2) (high[0], low[0], operands[2]));
+ {
+ rtx (*gen_x86_shift_adj_2)(rtx, rtx, rtx)
+ = mode == DImode ? gen_x86_shiftsi_adj_2 : gen_x86_shiftdi_adj_2;
+
+ emit_insn (gen_x86_shift_adj_2 (high[0], low[0], operands[2]));
+ }
}
void
ix86_split_ashr (rtx *operands, rtx scratch, enum machine_mode mode)
{
+ rtx (*gen_ashr3)(rtx, rtx, rtx)
+ = mode == DImode ? gen_ashrsi3 : gen_ashrdi3;
+ rtx (*gen_shrd)(rtx, rtx, rtx);
+ int half_width = GET_MODE_BITSIZE (mode) >> 1;
+
rtx low[2], high[2];
int count;
- const int single_width = mode == DImode ? 32 : 64;
if (CONST_INT_P (operands[2]))
{
- (mode == DImode ? split_di : split_ti) (operands, 2, low, high);
- count = INTVAL (operands[2]) & (single_width * 2 - 1);
+ split_double_mode (mode, operands, 2, low, high);
+ count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
- if (count == single_width * 2 - 1)
+ if (count == GET_MODE_BITSIZE (mode) - 1)
{
emit_move_insn (high[0], high[1]);
- emit_insn ((mode == DImode
- ? gen_ashrsi3
- : gen_ashrdi3) (high[0], high[0],
- GEN_INT (single_width - 1)));
+ emit_insn (gen_ashr3 (high[0], high[0],
+ GEN_INT (half_width - 1)));
emit_move_insn (low[0], high[0]);
}
- else if (count >= single_width)
+ else if (count >= half_width)
{
emit_move_insn (low[0], high[1]);
emit_move_insn (high[0], low[0]);
- emit_insn ((mode == DImode
- ? gen_ashrsi3
- : gen_ashrdi3) (high[0], high[0],
- GEN_INT (single_width - 1)));
- if (count > single_width)
- emit_insn ((mode == DImode
- ? gen_ashrsi3
- : gen_ashrdi3) (low[0], low[0],
- GEN_INT (count - single_width)));
+ emit_insn (gen_ashr3 (high[0], high[0],
+ GEN_INT (half_width - 1)));
+
+ if (count > half_width)
+ emit_insn (gen_ashr3 (low[0], low[0],
+ GEN_INT (count - half_width)));
}
else
{
+ gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
+
if (!rtx_equal_p (operands[0], operands[1]))
emit_move_insn (operands[0], operands[1]);
- emit_insn ((mode == DImode
- ? gen_x86_shrd
- : gen_x86_64_shrd) (low[0], high[0], GEN_INT (count)));
- emit_insn ((mode == DImode
- ? gen_ashrsi3
- : gen_ashrdi3) (high[0], high[0], GEN_INT (count)));
+
+ emit_insn (gen_shrd (low[0], high[0], GEN_INT (count)));
+ emit_insn (gen_ashr3 (high[0], high[0], GEN_INT (count)));
}
}
else
{
- if (!rtx_equal_p (operands[0], operands[1]))
+ gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
+
+ if (!rtx_equal_p (operands[0], operands[1]))
emit_move_insn (operands[0], operands[1]);
- (mode == DImode ? split_di : split_ti) (operands, 1, low, high);
+ split_double_mode (mode, operands, 1, low, high);
- emit_insn ((mode == DImode
- ? gen_x86_shrd
- : gen_x86_64_shrd) (low[0], high[0], operands[2]));
- emit_insn ((mode == DImode
- ? gen_ashrsi3
- : gen_ashrdi3) (high[0], high[0], operands[2]));
+ emit_insn (gen_shrd (low[0], high[0], operands[2]));
+ emit_insn (gen_ashr3 (high[0], high[0], operands[2]));
if (TARGET_CMOVE && scratch)
{
+ rtx (*gen_x86_shift_adj_1)(rtx, rtx, rtx, rtx)
+ = mode == DImode ? gen_x86_shiftsi_adj_1 : gen_x86_shiftdi_adj_1;
+
emit_move_insn (scratch, high[0]);
- emit_insn ((mode == DImode
- ? gen_ashrsi3
- : gen_ashrdi3) (scratch, scratch,
- GEN_INT (single_width - 1)));
- emit_insn ((mode == DImode
- ? gen_x86_shiftsi_adj_1
- : gen_x86_shiftdi_adj_1) (low[0], high[0], operands[2],
- scratch));
+ emit_insn (gen_ashr3 (scratch, scratch,
+ GEN_INT (half_width - 1)));
+ emit_insn (gen_x86_shift_adj_1 (low[0], high[0], operands[2],
+ scratch));
}
else
- emit_insn ((mode == DImode
- ? gen_x86_shiftsi_adj_3
- : gen_x86_shiftdi_adj_3) (low[0], high[0], operands[2]));
+ {
+ rtx (*gen_x86_shift_adj_3)(rtx, rtx, rtx)
+ = mode == DImode ? gen_x86_shiftsi_adj_3 : gen_x86_shiftdi_adj_3;
+
+ emit_insn (gen_x86_shift_adj_3 (low[0], high[0], operands[2]));
+ }
}
}
void
ix86_split_lshr (rtx *operands, rtx scratch, enum machine_mode mode)
{
+ rtx (*gen_lshr3)(rtx, rtx, rtx)
+ = mode == DImode ? gen_lshrsi3 : gen_lshrdi3;
+ rtx (*gen_shrd)(rtx, rtx, rtx);
+ int half_width = GET_MODE_BITSIZE (mode) >> 1;
+
rtx low[2], high[2];
int count;
- const int single_width = mode == DImode ? 32 : 64;
if (CONST_INT_P (operands[2]))
{
- (mode == DImode ? split_di : split_ti) (operands, 2, low, high);
- count = INTVAL (operands[2]) & (single_width * 2 - 1);
+ split_double_mode (mode, operands, 2, low, high);
+ count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1);
- if (count >= single_width)
+ if (count >= half_width)
{
emit_move_insn (low[0], high[1]);
ix86_expand_clear (high[0]);
- if (count > single_width)
- emit_insn ((mode == DImode
- ? gen_lshrsi3
- : gen_lshrdi3) (low[0], low[0],
- GEN_INT (count - single_width)));
+ if (count > half_width)
+ emit_insn (gen_lshr3 (low[0], low[0],
+ GEN_INT (count - half_width)));
}
else
{
+ gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
+
if (!rtx_equal_p (operands[0], operands[1]))
emit_move_insn (operands[0], operands[1]);
- emit_insn ((mode == DImode
- ? gen_x86_shrd
- : gen_x86_64_shrd) (low[0], high[0], GEN_INT (count)));
- emit_insn ((mode == DImode
- ? gen_lshrsi3
- : gen_lshrdi3) (high[0], high[0], GEN_INT (count)));
+
+ emit_insn (gen_shrd (low[0], high[0], GEN_INT (count)));
+ emit_insn (gen_lshr3 (high[0], high[0], GEN_INT (count)));
}
}
else
{
+ gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd;
+
if (!rtx_equal_p (operands[0], operands[1]))
emit_move_insn (operands[0], operands[1]);
- (mode == DImode ? split_di : split_ti) (operands, 1, low, high);
+ split_double_mode (mode, operands, 1, low, high);
- emit_insn ((mode == DImode
- ? gen_x86_shrd
- : gen_x86_64_shrd) (low[0], high[0], operands[2]));
- emit_insn ((mode == DImode
- ? gen_lshrsi3
- : gen_lshrdi3) (high[0], high[0], operands[2]));
+ emit_insn (gen_shrd (low[0], high[0], operands[2]));
+ emit_insn (gen_lshr3 (high[0], high[0], operands[2]));
- /* Heh. By reversing the arguments, we can reuse this pattern. */
if (TARGET_CMOVE && scratch)
{
+ rtx (*gen_x86_shift_adj_1)(rtx, rtx, rtx, rtx)
+ = mode == DImode ? gen_x86_shiftsi_adj_1 : gen_x86_shiftdi_adj_1;
+
ix86_expand_clear (scratch);
- emit_insn ((mode == DImode
- ? gen_x86_shiftsi_adj_1
- : gen_x86_shiftdi_adj_1) (low[0], high[0], operands[2],
- scratch));
+ emit_insn (gen_x86_shift_adj_1 (low[0], high[0], operands[2],
+ scratch));
}
else
- emit_insn ((mode == DImode
- ? gen_x86_shiftsi_adj_2
- : gen_x86_shiftdi_adj_2) (low[0], high[0], operands[2]));
+ {
+ rtx (*gen_x86_shift_adj_2)(rtx, rtx, rtx)
+ = mode == DImode ? gen_x86_shiftsi_adj_2 : gen_x86_shiftdi_adj_2;
+
+ emit_insn (gen_x86_shift_adj_2 (low[0], high[0], operands[2]));
+ }
}
}
static void
ix86_adjust_counter (rtx countreg, HOST_WIDE_INT value)
{
- if (GET_MODE (countreg) == DImode)
- emit_insn (gen_adddi3 (countreg, countreg, GEN_INT (-value)));
- else
- emit_insn (gen_addsi3 (countreg, countreg, GEN_INT (-value)));
+ rtx (*gen_add)(rtx, rtx, rtx)
+ = GET_MODE (countreg) == DImode ? gen_adddi3 : gen_addsi3;
+
+ emit_insn (gen_add (countreg, countreg, GEN_INT (-value)));
}
/* Zero extend possibly SImode EXP to Pmode register. */
{
unsigned int i;
enum stringop_alg alg = libcall;
- for (i = 0; i < NAX_STRINGOP_ALGS; i++)
+ for (i = 0; i < MAX_STRINGOP_ALGS; i++)
{
/* We get here if the algorithms that were not libcall-based
were rep-prefix based and we are unable to use rep prefixes
int i;
bool any_alg_usable_p = true;
- for (i = 0; i < NAX_STRINGOP_ALGS; i++)
+ for (i = 0; i < MAX_STRINGOP_ALGS; i++)
{
enum stringop_alg candidate = algs->size[i].alg;
any_alg_usable_p = any_alg_usable_p && ALG_USABLE_P (candidate);
return tmp;
}
-void
+rtx
ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
rtx callarg2,
rtx pop, int sibcall)
+ 2, vec));
}
+ /* Add UNSPEC_CALL_NEEDS_VZEROUPPER decoration. */
+ if (TARGET_VZEROUPPER && cfun->machine->use_avx256_p)
+ {
+ rtx unspec;
+ int avx256;
+
+ cfun->machine->use_vzeroupper_p = 1;
+ if (cfun->machine->callee_pass_avx256_p)
+ {
+ if (cfun->machine->callee_return_avx256_p)
+ avx256 = callee_return_pass_avx256;
+ else
+ avx256 = callee_pass_avx256;
+ }
+ else if (cfun->machine->callee_return_avx256_p)
+ avx256 = callee_return_avx256;
+ else
+ avx256 = call_no_avx256;
+
+ unspec = gen_rtx_UNSPEC (VOIDmode,
+ gen_rtvec (1, GEN_INT (avx256)),
+ UNSPEC_CALL_NEEDS_VZEROUPPER);
+ call = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, call, unspec));
+ }
+
call = emit_call_insn (call);
if (use)
CALL_INSN_FUNCTION_USAGE (call) = use;
+
+ return call;
+}
+
+void
+ix86_split_call_vzeroupper (rtx insn, rtx vzeroupper)
+{
+ rtx call = XVECEXP (PATTERN (insn), 0, 0);
+ emit_insn (gen_avx_vzeroupper (vzeroupper));
+ emit_call_insn (call);
+}
+
+void
+ix86_split_call_pop_vzeroupper (rtx insn, rtx vzeroupper)
+{
+ rtx call = XVECEXP (PATTERN (insn), 0, 0);
+ rtx pop = XVECEXP (PATTERN (insn), 0, 1);
+ emit_insn (gen_avx_vzeroupper (vzeroupper));
+ emit_call_insn (gen_rtx_PARALLEL (VOIDmode,
+ gen_rtvec (2, call, pop)));
}
+/* Output the assembly for a call instruction. */
+
+const char *
+ix86_output_call_insn (rtx insn, rtx call_op, int addr_op)
+{
+ bool direct_p = constant_call_address_operand (call_op, Pmode);
+ bool seh_nop_p = false;
+
+ gcc_assert (addr_op == 0 || addr_op == 1);
+
+ if (SIBLING_CALL_P (insn))
+ {
+ if (direct_p)
+ return addr_op ? "jmp\t%P1" : "jmp\t%P0";
+ /* SEH epilogue detection requires the indirect branch case
+ to include REX.W. */
+ else if (TARGET_SEH)
+ return addr_op ? "rex.W jmp %A1" : "rex.W jmp %A0";
+ else
+ return addr_op ? "jmp\t%A1" : "jmp\t%A0";
+ }
+
+ /* SEH unwinding can require an extra nop to be emitted in several
+ circumstances. Determine if we have one of those. */
+ if (TARGET_SEH)
+ {
+ rtx i;
+
+ for (i = NEXT_INSN (insn); i ; i = NEXT_INSN (i))
+ {
+ /* If we get to another real insn, we don't need the nop. */
+ if (INSN_P (i))
+ break;
+
+ /* If we get to the epilogue note, prevent a catch region from
+ being adjacent to the standard epilogue sequence. If non-
+ call-exceptions, we'll have done this during epilogue emission. */
+ if (NOTE_P (i) && NOTE_KIND (i) == NOTE_INSN_EPILOGUE_BEG
+ && !flag_non_call_exceptions
+ && !can_throw_internal (insn))
+ {
+ seh_nop_p = true;
+ break;
+ }
+ }
+
+ /* If we didn't find a real insn following the call, prevent the
+ unwinder from looking into the next function. */
+ if (i == NULL)
+ seh_nop_p = true;
+ }
+
+ if (direct_p)
+ {
+ if (seh_nop_p)
+ return addr_op ? "call\t%P1\n\tnop" : "call\t%P0\n\tnop";
+ else
+ return addr_op ? "call\t%P1" : "call\t%P0";
+ }
+ else
+ {
+ if (seh_nop_p)
+ return addr_op ? "call\t%A1\n\tnop" : "call\t%A0\n\tnop";
+ else
+ return addr_op ? "call\t%A1" : "call\t%A0";
+ }
+}
\f
/* Clear stack slot assignments remembered from previous functions.
This is called from INIT_EXPANDERS once before RTL is emitted for each
case PROCESSOR_K6:
return 1;
+ case PROCESSOR_CORE2:
+ case PROCESSOR_COREI7_32:
+ case PROCESSOR_COREI7_64:
+ /* Generally, we want haifa-sched:max_issue() to look ahead as far
+ as many instructions can be executed on a cycle, i.e.,
+ issue_rate. I wonder why tuning for many CPUs does not do this. */
+ return ix86_issue_rate ();
+
default:
return 0;
}
}
\f
+
+/* Model decoder of Core 2/i7.
+ Below hooks for multipass scheduling (see haifa-sched.c:max_issue)
+ track the instruction fetch block boundaries and make sure that long
+ (9+ bytes) instructions are assigned to D0. */
+
+/* Maximum length of an insn that can be handled by
+ a secondary decoder unit. '8' for Core 2/i7. */
+static int core2i7_secondary_decoder_max_insn_size;
+
+/* Ifetch block size, i.e., number of bytes decoder reads per cycle.
+ '16' for Core 2/i7. */
+static int core2i7_ifetch_block_size;
+
+/* Maximum number of instructions decoder can handle per cycle.
+ '6' for Core 2/i7. */
+static int core2i7_ifetch_block_max_insns;
+
+typedef struct ix86_first_cycle_multipass_data_ *
+ ix86_first_cycle_multipass_data_t;
+typedef const struct ix86_first_cycle_multipass_data_ *
+ const_ix86_first_cycle_multipass_data_t;
+
+/* A variable to store target state across calls to max_issue within
+ one cycle. */
+static struct ix86_first_cycle_multipass_data_ _ix86_first_cycle_multipass_data,
+ *ix86_first_cycle_multipass_data = &_ix86_first_cycle_multipass_data;
+
+/* Initialize DATA. */
+static void
+core2i7_first_cycle_multipass_init (void *_data)
+{
+ ix86_first_cycle_multipass_data_t data
+ = (ix86_first_cycle_multipass_data_t) _data;
+
+ data->ifetch_block_len = 0;
+ data->ifetch_block_n_insns = 0;
+ data->ready_try_change = NULL;
+ data->ready_try_change_size = 0;
+}
+
+/* Advancing the cycle; reset ifetch block counts. */
+static void
+core2i7_dfa_post_advance_cycle (void)
+{
+ ix86_first_cycle_multipass_data_t data = ix86_first_cycle_multipass_data;
+
+ gcc_assert (data->ifetch_block_n_insns <= core2i7_ifetch_block_max_insns);
+
+ data->ifetch_block_len = 0;
+ data->ifetch_block_n_insns = 0;
+}
+
+static int min_insn_size (rtx);
+
+/* Filter out insns from ready_try that the core will not be able to issue
+ on current cycle due to decoder. */
+static void
+core2i7_first_cycle_multipass_filter_ready_try
+(const_ix86_first_cycle_multipass_data_t data,
+ char *ready_try, int n_ready, bool first_cycle_insn_p)
+{
+ while (n_ready--)
+ {
+ rtx insn;
+ int insn_size;
+
+ if (ready_try[n_ready])
+ continue;
+
+ insn = get_ready_element (n_ready);
+ insn_size = min_insn_size (insn);
+
+ if (/* If this is a too long an insn for a secondary decoder ... */
+ (!first_cycle_insn_p
+ && insn_size > core2i7_secondary_decoder_max_insn_size)
+ /* ... or it would not fit into the ifetch block ... */
+ || data->ifetch_block_len + insn_size > core2i7_ifetch_block_size
+ /* ... or the decoder is full already ... */
+ || data->ifetch_block_n_insns + 1 > core2i7_ifetch_block_max_insns)
+ /* ... mask the insn out. */
+ {
+ ready_try[n_ready] = 1;
+
+ if (data->ready_try_change)
+ SET_BIT (data->ready_try_change, n_ready);
+ }
+ }
+}
+
+/* Prepare for a new round of multipass lookahead scheduling. */
+static void
+core2i7_first_cycle_multipass_begin (void *_data, char *ready_try, int n_ready,
+ bool first_cycle_insn_p)
+{
+ ix86_first_cycle_multipass_data_t data
+ = (ix86_first_cycle_multipass_data_t) _data;
+ const_ix86_first_cycle_multipass_data_t prev_data
+ = ix86_first_cycle_multipass_data;
+
+ /* Restore the state from the end of the previous round. */
+ data->ifetch_block_len = prev_data->ifetch_block_len;
+ data->ifetch_block_n_insns = prev_data->ifetch_block_n_insns;
+
+ /* Filter instructions that cannot be issued on current cycle due to
+ decoder restrictions. */
+ core2i7_first_cycle_multipass_filter_ready_try (data, ready_try, n_ready,
+ first_cycle_insn_p);
+}
+
+/* INSN is being issued in current solution. Account for its impact on
+ the decoder model. */
+static void
+core2i7_first_cycle_multipass_issue (void *_data, char *ready_try, int n_ready,
+ rtx insn, const void *_prev_data)
+{
+ ix86_first_cycle_multipass_data_t data
+ = (ix86_first_cycle_multipass_data_t) _data;
+ const_ix86_first_cycle_multipass_data_t prev_data
+ = (const_ix86_first_cycle_multipass_data_t) _prev_data;
+
+ int insn_size = min_insn_size (insn);
+
+ data->ifetch_block_len = prev_data->ifetch_block_len + insn_size;
+ data->ifetch_block_n_insns = prev_data->ifetch_block_n_insns + 1;
+ gcc_assert (data->ifetch_block_len <= core2i7_ifetch_block_size
+ && data->ifetch_block_n_insns <= core2i7_ifetch_block_max_insns);
+
+ /* Allocate or resize the bitmap for storing INSN's effect on ready_try. */
+ if (!data->ready_try_change)
+ {
+ data->ready_try_change = sbitmap_alloc (n_ready);
+ data->ready_try_change_size = n_ready;
+ }
+ else if (data->ready_try_change_size < n_ready)
+ {
+ data->ready_try_change = sbitmap_resize (data->ready_try_change,
+ n_ready, 0);
+ data->ready_try_change_size = n_ready;
+ }
+ sbitmap_zero (data->ready_try_change);
+
+ /* Filter out insns from ready_try that the core will not be able to issue
+ on current cycle due to decoder. */
+ core2i7_first_cycle_multipass_filter_ready_try (data, ready_try, n_ready,
+ false);
+}
+
+/* Revert the effect on ready_try. */
+static void
+core2i7_first_cycle_multipass_backtrack (const void *_data,
+ char *ready_try,
+ int n_ready ATTRIBUTE_UNUSED)
+{
+ const_ix86_first_cycle_multipass_data_t data
+ = (const_ix86_first_cycle_multipass_data_t) _data;
+ unsigned int i = 0;
+ sbitmap_iterator sbi;
+
+ gcc_assert (sbitmap_last_set_bit (data->ready_try_change) < n_ready);
+ EXECUTE_IF_SET_IN_SBITMAP (data->ready_try_change, 0, i, sbi)
+ {
+ ready_try[i] = 0;
+ }
+}
+
+/* Save the result of multipass lookahead scheduling for the next round. */
+static void
+core2i7_first_cycle_multipass_end (const void *_data)
+{
+ const_ix86_first_cycle_multipass_data_t data
+ = (const_ix86_first_cycle_multipass_data_t) _data;
+ ix86_first_cycle_multipass_data_t next_data
+ = ix86_first_cycle_multipass_data;
+
+ if (data != NULL)
+ {
+ next_data->ifetch_block_len = data->ifetch_block_len;
+ next_data->ifetch_block_n_insns = data->ifetch_block_n_insns;
+ }
+}
+
+/* Deallocate target data. */
+static void
+core2i7_first_cycle_multipass_fini (void *_data)
+{
+ ix86_first_cycle_multipass_data_t data
+ = (ix86_first_cycle_multipass_data_t) _data;
+
+ if (data->ready_try_change)
+ {
+ sbitmap_free (data->ready_try_change);
+ data->ready_try_change = NULL;
+ data->ready_try_change_size = 0;
+ }
+}
+
+/* Prepare for scheduling pass. */
+static void
+ix86_sched_init_global (FILE *dump ATTRIBUTE_UNUSED,
+ int verbose ATTRIBUTE_UNUSED,
+ int max_uid ATTRIBUTE_UNUSED)
+{
+ /* Install scheduling hooks for current CPU. Some of these hooks are used
+ in time-critical parts of the scheduler, so we only set them up when
+ they are actually used. */
+ switch (ix86_tune)
+ {
+ case PROCESSOR_CORE2:
+ case PROCESSOR_COREI7_32:
+ case PROCESSOR_COREI7_64:
+ targetm.sched.dfa_post_advance_cycle
+ = core2i7_dfa_post_advance_cycle;
+ targetm.sched.first_cycle_multipass_init
+ = core2i7_first_cycle_multipass_init;
+ targetm.sched.first_cycle_multipass_begin
+ = core2i7_first_cycle_multipass_begin;
+ targetm.sched.first_cycle_multipass_issue
+ = core2i7_first_cycle_multipass_issue;
+ targetm.sched.first_cycle_multipass_backtrack
+ = core2i7_first_cycle_multipass_backtrack;
+ targetm.sched.first_cycle_multipass_end
+ = core2i7_first_cycle_multipass_end;
+ targetm.sched.first_cycle_multipass_fini
+ = core2i7_first_cycle_multipass_fini;
+
+ /* Set decoder parameters. */
+ core2i7_secondary_decoder_max_insn_size = 8;
+ core2i7_ifetch_block_size = 16;
+ core2i7_ifetch_block_max_insns = 6;
+ break;
+
+ default:
+ targetm.sched.dfa_post_advance_cycle = NULL;
+ targetm.sched.first_cycle_multipass_init = NULL;
+ targetm.sched.first_cycle_multipass_begin = NULL;
+ targetm.sched.first_cycle_multipass_issue = NULL;
+ targetm.sched.first_cycle_multipass_backtrack = NULL;
+ targetm.sched.first_cycle_multipass_end = NULL;
+ targetm.sched.first_cycle_multipass_fini = NULL;
+ break;
+ }
+}
+
+\f
/* Compute the alignment given to a constant that is being placed in memory.
EXP is the constant and ALIGN is the alignment that the object would
ordinarily have.
decl = NULL;
}
+ if (use_avx256_p (mode, type))
+ cfun->machine->use_avx256_p = true;
+
/* Don't do dynamic stack realignment for long long objects with
-mpreferred-stack-boundary=2. */
if (!TARGET_64BIT
{
tree type, decl;
- if (TARGET_64BIT || align != 64 || ix86_preferred_stack_boundary >= 64)
- return align;
-
if (exp && DECL_P (exp))
{
type = TREE_TYPE (exp);
decl = NULL;
}
+ if (use_avx256_p (mode, type))
+ cfun->machine->use_avx256_p = true;
+
+ if (TARGET_64BIT || align != 64 || ix86_preferred_stack_boundary >= 64)
+ return align;
+
/* Don't do dynamic stack realignment for long long objects with
-mpreferred-stack-boundary=2. */
if ((mode == DImode || (type && TYPE_MODE (type) == DImode))
/* Vectorizer support builtins. */
IX86_BUILTIN_CPYSGNPS,
IX86_BUILTIN_CPYSGNPD,
+ IX86_BUILTIN_CPYSGNPS256,
+ IX86_BUILTIN_CPYSGNPD256,
IX86_BUILTIN_CVTUDQ2PS,
IX86_BUILTIN_VFMADDSD,
IX86_BUILTIN_VFMADDPS,
IX86_BUILTIN_VFMADDPD,
- IX86_BUILTIN_VFMSUBSS,
- IX86_BUILTIN_VFMSUBSD,
- IX86_BUILTIN_VFMSUBPS,
- IX86_BUILTIN_VFMSUBPD,
- IX86_BUILTIN_VFMADDSUBPS,
- IX86_BUILTIN_VFMADDSUBPD,
- IX86_BUILTIN_VFMSUBADDPS,
- IX86_BUILTIN_VFMSUBADDPD,
- IX86_BUILTIN_VFNMADDSS,
- IX86_BUILTIN_VFNMADDSD,
- IX86_BUILTIN_VFNMADDPS,
- IX86_BUILTIN_VFNMADDPD,
- IX86_BUILTIN_VFNMSUBSS,
- IX86_BUILTIN_VFNMSUBSD,
- IX86_BUILTIN_VFNMSUBPS,
- IX86_BUILTIN_VFNMSUBPD,
IX86_BUILTIN_VFMADDPS256,
IX86_BUILTIN_VFMADDPD256,
- IX86_BUILTIN_VFMSUBPS256,
- IX86_BUILTIN_VFMSUBPD256,
+ IX86_BUILTIN_VFMADDSUBPS,
+ IX86_BUILTIN_VFMADDSUBPD,
IX86_BUILTIN_VFMADDSUBPS256,
IX86_BUILTIN_VFMADDSUBPD256,
- IX86_BUILTIN_VFMSUBADDPS256,
- IX86_BUILTIN_VFMSUBADDPD256,
- IX86_BUILTIN_VFNMADDPS256,
- IX86_BUILTIN_VFNMADDPD256,
- IX86_BUILTIN_VFNMSUBPS256,
- IX86_BUILTIN_VFNMSUBPD256,
IX86_BUILTIN_VPCMOV,
IX86_BUILTIN_VPCMOV_V2DI,
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskpd256, "__builtin_ia32_movmskpd256", IX86_BUILTIN_MOVMSKPD256, UNKNOWN, (int) INT_FTYPE_V4DF },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskps256, "__builtin_ia32_movmskps256", IX86_BUILTIN_MOVMSKPS256, UNKNOWN, (int) INT_FTYPE_V8SF },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv8sf3, "__builtin_ia32_copysignps256", IX86_BUILTIN_CPYSGNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv4df3, "__builtin_ia32_copysignpd256", IX86_BUILTIN_CPYSGNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
+
{ OPTION_MASK_ISA_ABM, CODE_FOR_clzhi2_abm, "__builtin_clzs", IX86_BUILTIN_CLZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 },
/* F16C */
static const struct builtin_description bdesc_multi_arg[] =
{
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfmaddv4sf4, "__builtin_ia32_vfmaddss", IX86_BUILTIN_VFMADDSS, UNKNOWN, (int)MULTI_ARG_3_SF },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfmaddv2df4, "__builtin_ia32_vfmaddsd", IX86_BUILTIN_VFMADDSD, UNKNOWN, (int)MULTI_ARG_3_DF },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmaddv4sf4, "__builtin_ia32_vfmaddps", IX86_BUILTIN_VFMADDPS, UNKNOWN, (int)MULTI_ARG_3_SF },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmaddv2df4, "__builtin_ia32_vfmaddpd", IX86_BUILTIN_VFMADDPD, UNKNOWN, (int)MULTI_ARG_3_DF },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfmsubv4sf4, "__builtin_ia32_vfmsubss", IX86_BUILTIN_VFMSUBSS, UNKNOWN, (int)MULTI_ARG_3_SF },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfmsubv2df4, "__builtin_ia32_vfmsubsd", IX86_BUILTIN_VFMSUBSD, UNKNOWN, (int)MULTI_ARG_3_DF },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmsubv4sf4, "__builtin_ia32_vfmsubps", IX86_BUILTIN_VFMSUBPS, UNKNOWN, (int)MULTI_ARG_3_SF },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmsubv2df4, "__builtin_ia32_vfmsubpd", IX86_BUILTIN_VFMSUBPD, UNKNOWN, (int)MULTI_ARG_3_DF },
-
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfnmaddv4sf4, "__builtin_ia32_vfnmaddss", IX86_BUILTIN_VFNMADDSS, UNKNOWN, (int)MULTI_ARG_3_SF },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfnmaddv2df4, "__builtin_ia32_vfnmaddsd", IX86_BUILTIN_VFNMADDSD, UNKNOWN, (int)MULTI_ARG_3_DF },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fnmaddv4sf4, "__builtin_ia32_vfnmaddps", IX86_BUILTIN_VFNMADDPS, UNKNOWN, (int)MULTI_ARG_3_SF },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fnmaddv2df4, "__builtin_ia32_vfnmaddpd", IX86_BUILTIN_VFNMADDPD, UNKNOWN, (int)MULTI_ARG_3_DF },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfnmsubv4sf4, "__builtin_ia32_vfnmsubss", IX86_BUILTIN_VFNMSUBSS, UNKNOWN, (int)MULTI_ARG_3_SF },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfnmsubv2df4, "__builtin_ia32_vfnmsubsd", IX86_BUILTIN_VFNMSUBSD, UNKNOWN, (int)MULTI_ARG_3_DF },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fnmsubv4sf4, "__builtin_ia32_vfnmsubps", IX86_BUILTIN_VFNMSUBPS, UNKNOWN, (int)MULTI_ARG_3_SF },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fnmsubv2df4, "__builtin_ia32_vfnmsubpd", IX86_BUILTIN_VFNMSUBPD, UNKNOWN, (int)MULTI_ARG_3_DF },
-
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmaddsubv4sf4, "__builtin_ia32_vfmaddsubps", IX86_BUILTIN_VFMADDSUBPS, UNKNOWN, (int)MULTI_ARG_3_SF },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmaddsubv2df4, "__builtin_ia32_vfmaddsubpd", IX86_BUILTIN_VFMADDSUBPD, UNKNOWN, (int)MULTI_ARG_3_DF },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmsubaddv4sf4, "__builtin_ia32_vfmsubaddps", IX86_BUILTIN_VFMSUBADDPS, UNKNOWN, (int)MULTI_ARG_3_SF },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmsubaddv2df4, "__builtin_ia32_vfmsubaddpd", IX86_BUILTIN_VFMSUBADDPD, UNKNOWN, (int)MULTI_ARG_3_DF },
-
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmaddv8sf4256, "__builtin_ia32_vfmaddps256", IX86_BUILTIN_VFMADDPS256, UNKNOWN, (int)MULTI_ARG_3_SF2 },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmaddv4df4256, "__builtin_ia32_vfmaddpd256", IX86_BUILTIN_VFMADDPD256, UNKNOWN, (int)MULTI_ARG_3_DF2 },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmsubv8sf4256, "__builtin_ia32_vfmsubps256", IX86_BUILTIN_VFMSUBPS256, UNKNOWN, (int)MULTI_ARG_3_SF2 },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmsubv4df4256, "__builtin_ia32_vfmsubpd256", IX86_BUILTIN_VFMSUBPD256, UNKNOWN, (int)MULTI_ARG_3_DF2 },
-
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fnmaddv8sf4256, "__builtin_ia32_vfnmaddps256", IX86_BUILTIN_VFNMADDPS256, UNKNOWN, (int)MULTI_ARG_3_SF2 },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fnmaddv4df4256, "__builtin_ia32_vfnmaddpd256", IX86_BUILTIN_VFNMADDPD256, UNKNOWN, (int)MULTI_ARG_3_DF2 },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fnmsubv8sf4256, "__builtin_ia32_vfnmsubps256", IX86_BUILTIN_VFNMSUBPS256, UNKNOWN, (int)MULTI_ARG_3_SF2 },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fnmsubv4df4256, "__builtin_ia32_vfnmsubpd256", IX86_BUILTIN_VFNMSUBPD256, UNKNOWN, (int)MULTI_ARG_3_DF2 },
-
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmaddsubv8sf4, "__builtin_ia32_vfmaddsubps256", IX86_BUILTIN_VFMADDSUBPS256, UNKNOWN, (int)MULTI_ARG_3_SF2 },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmaddsubv4df4, "__builtin_ia32_vfmaddsubpd256", IX86_BUILTIN_VFMADDSUBPD256, UNKNOWN, (int)MULTI_ARG_3_DF2 },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmsubaddv8sf4, "__builtin_ia32_vfmsubaddps256", IX86_BUILTIN_VFMSUBADDPS256, UNKNOWN, (int)MULTI_ARG_3_SF2 },
- { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmsubaddv4df4, "__builtin_ia32_vfmsubaddpd256", IX86_BUILTIN_VFMSUBADDPD256, UNKNOWN, (int)MULTI_ARG_3_DF2 },
+ { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfmadd_v4sf,
+ "__builtin_ia32_vfmaddss", IX86_BUILTIN_VFMADDSS,
+ UNKNOWN, (int)MULTI_ARG_3_SF },
+ { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfmadd_v2df,
+ "__builtin_ia32_vfmaddsd", IX86_BUILTIN_VFMADDSD,
+ UNKNOWN, (int)MULTI_ARG_3_DF },
+
+ { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmadd_v4sf,
+ "__builtin_ia32_vfmaddps", IX86_BUILTIN_VFMADDPS,
+ UNKNOWN, (int)MULTI_ARG_3_SF },
+ { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmadd_v2df,
+ "__builtin_ia32_vfmaddpd", IX86_BUILTIN_VFMADDPD,
+ UNKNOWN, (int)MULTI_ARG_3_DF },
+ { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmadd_v8sf,
+ "__builtin_ia32_vfmaddps256", IX86_BUILTIN_VFMADDPS256,
+ UNKNOWN, (int)MULTI_ARG_3_SF2 },
+ { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmadd_v4df,
+ "__builtin_ia32_vfmaddpd256", IX86_BUILTIN_VFMADDPD256,
+ UNKNOWN, (int)MULTI_ARG_3_DF2 },
+
+ { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fmaddsub_v4sf,
+ "__builtin_ia32_vfmaddsubps", IX86_BUILTIN_VFMADDSUBPS,
+ UNKNOWN, (int)MULTI_ARG_3_SF },
+ { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fmaddsub_v2df,
+ "__builtin_ia32_vfmaddsubpd", IX86_BUILTIN_VFMADDSUBPD,
+ UNKNOWN, (int)MULTI_ARG_3_DF },
+ { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fmaddsub_v8sf,
+ "__builtin_ia32_vfmaddsubps256", IX86_BUILTIN_VFMADDSUBPS256,
+ UNKNOWN, (int)MULTI_ARG_3_SF2 },
+ { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fmaddsub_v4df,
+ "__builtin_ia32_vfmaddsubpd256", IX86_BUILTIN_VFMADDSUBPD256,
+ UNKNOWN, (int)MULTI_ARG_3_DF2 },
{ OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov", IX86_BUILTIN_VPCMOV, UNKNOWN, (int)MULTI_ARG_3_DI },
{ OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov_v2di", IX86_BUILTIN_VPCMOV_V2DI, UNKNOWN, (int)MULTI_ARG_3_DI },
{ OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv2df2, "__builtin_ia32_vfrczsd", IX86_BUILTIN_VFRCZSD, UNKNOWN, (int)MULTI_ARG_2_DF },
{ OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4sf2, "__builtin_ia32_vfrczps", IX86_BUILTIN_VFRCZPS, UNKNOWN, (int)MULTI_ARG_1_SF },
{ OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv2df2, "__builtin_ia32_vfrczpd", IX86_BUILTIN_VFRCZPD, UNKNOWN, (int)MULTI_ARG_1_DF },
- { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv8sf2256, "__builtin_ia32_vfrczps256", IX86_BUILTIN_VFRCZPS256, UNKNOWN, (int)MULTI_ARG_1_SF2 },
- { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4df2256, "__builtin_ia32_vfrczpd256", IX86_BUILTIN_VFRCZPD256, UNKNOWN, (int)MULTI_ARG_1_DF2 },
+ { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv8sf2, "__builtin_ia32_vfrczps256", IX86_BUILTIN_VFRCZPS256, UNKNOWN, (int)MULTI_ARG_1_SF2 },
+ { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4df2, "__builtin_ia32_vfrczpd256", IX86_BUILTIN_VFRCZPD256, UNKNOWN, (int)MULTI_ARG_1_DF2 },
{ OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbw, "__builtin_ia32_vphaddbw", IX86_BUILTIN_VPHADDBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
{ OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbd, "__builtin_ia32_vphaddbd", IX86_BUILTIN_VPHADDBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
if (TARGET_64BIT)
ix86_init_builtins_va_builtins_abi ();
+
+#ifdef SUBTARGET_INIT_BUILTINS
+ SUBTARGET_INIT_BUILTINS;
+#endif
}
/* Return the ix86 builtin for CODE. */
switch ((enum ix86_builtin_func_type) d->flag)
{
case VOID_FTYPE_VOID:
+ if (icode == CODE_FOR_avx_vzeroupper)
+ target = GEN_INT (vzeroupper_intrinsic);
emit_insn (GEN_FCN (icode) (target));
return 0;
case VOID_FTYPE_UINT64:
switch (fn)
{
case BUILT_IN_SQRT:
- if (out_mode == DFmode && out_n == 2
- && in_mode == DFmode && in_n == 2)
- return ix86_builtins[IX86_BUILTIN_SQRTPD];
+ if (out_mode == DFmode && in_mode == DFmode)
+ {
+ if (out_n == 2 && in_n == 2)
+ return ix86_builtins[IX86_BUILTIN_SQRTPD];
+ else if (out_n == 4 && in_n == 4)
+ return ix86_builtins[IX86_BUILTIN_SQRTPD256];
+ }
break;
case BUILT_IN_SQRTF:
- if (out_mode == SFmode && out_n == 4
- && in_mode == SFmode && in_n == 4)
- return ix86_builtins[IX86_BUILTIN_SQRTPS_NR];
+ if (out_mode == SFmode && in_mode == SFmode)
+ {
+ if (out_n == 4 && in_n == 4)
+ return ix86_builtins[IX86_BUILTIN_SQRTPS_NR];
+ else if (out_n == 8 && in_n == 8)
+ return ix86_builtins[IX86_BUILTIN_SQRTPS_NR256];
+ }
break;
case BUILT_IN_LRINT:
break;
case BUILT_IN_LRINTF:
- if (out_mode == SImode && out_n == 4
- && in_mode == SFmode && in_n == 4)
- return ix86_builtins[IX86_BUILTIN_CVTPS2DQ];
+ if (out_mode == SImode && in_mode == SFmode)
+ {
+ if (out_n == 4 && in_n == 4)
+ return ix86_builtins[IX86_BUILTIN_CVTPS2DQ];
+ else if (out_n == 8 && in_n == 8)
+ return ix86_builtins[IX86_BUILTIN_CVTPS2DQ256];
+ }
break;
case BUILT_IN_COPYSIGN:
- if (out_mode == DFmode && out_n == 2
- && in_mode == DFmode && in_n == 2)
- return ix86_builtins[IX86_BUILTIN_CPYSGNPD];
+ if (out_mode == DFmode && in_mode == DFmode)
+ {
+ if (out_n == 2 && in_n == 2)
+ return ix86_builtins[IX86_BUILTIN_CPYSGNPD];
+ else if (out_n == 4 && in_n == 4)
+ return ix86_builtins[IX86_BUILTIN_CPYSGNPD256];
+ }
break;
case BUILT_IN_COPYSIGNF:
- if (out_mode == SFmode && out_n == 4
- && in_mode == SFmode && in_n == 4)
- return ix86_builtins[IX86_BUILTIN_CPYSGNPS];
+ if (out_mode == SFmode && in_mode == SFmode)
+ {
+ if (out_n == 4 && in_n == 4)
+ return ix86_builtins[IX86_BUILTIN_CPYSGNPS];
+ else if (out_n == 8 && in_n == 8)
+ return ix86_builtins[IX86_BUILTIN_CPYSGNPS256];
+ }
break;
- default:
- ;
- }
+ case BUILT_IN_FMA:
+ if (out_mode == DFmode && in_mode == DFmode)
+ {
+ if (out_n == 2 && in_n == 2)
+ return ix86_builtins[IX86_BUILTIN_VFMADDPD];
+ if (out_n == 4 && in_n == 4)
+ return ix86_builtins[IX86_BUILTIN_VFMADDPD256];
+ }
+ break;
- /* Dispatch to a handler for a vectorization library. */
- if (ix86_veclib_handler)
+ case BUILT_IN_FMAF:
+ if (out_mode == SFmode && in_mode == SFmode)
+ {
+ if (out_n == 4 && in_n == 4)
+ return ix86_builtins[IX86_BUILTIN_VFMADDPS];
+ if (out_n == 8 && in_n == 8)
+ return ix86_builtins[IX86_BUILTIN_VFMADDPS256];
+ }
+ break;
+
+ default:
+ break;
+ }
+
+ /* Dispatch to a handler for a vectorization library. */
+ if (ix86_veclib_handler)
return ix86_veclib_handler ((enum built_in_function) fn, type_out,
type_in);
case V8SFmode:
return (TYPE_UNSIGNED (src_type)
? NULL_TREE
- : ix86_builtins[IX86_BUILTIN_CVTDQ2PS]);
+ : ix86_builtins[IX86_BUILTIN_CVTDQ2PS256]);
default:
return NULL_TREE;
}
case IX86_BUILTIN_SQRTPS_NR:
return ix86_builtins[IX86_BUILTIN_RSQRTPS_NR];
+ case IX86_BUILTIN_SQRTPS_NR256:
+ return ix86_builtins[IX86_BUILTIN_RSQRTPS_NR256];
+
default:
return NULL_TREE;
}
case DImode:
{
rtx operands[2];
- split_di (&operand, 1, operands, operands + 1);
+ split_double_mode (mode, &operand, 1, operands, operands + 1);
emit_insn (
gen_rtx_SET (VOIDmode,
gen_rtx_MEM (SImode,
return TARGET_SSE_MATH ? sse_fpmath_classes : no_sse_fpmath_classes;
}
-/* Put float CONST_DOUBLE in the constant pool instead of fp regs.
+/* Implement TARGET_PREFERRED_RELOAD_CLASS.
+
+ Put float CONST_DOUBLE in the constant pool instead of fp regs.
QImode must go into class Q_REGS.
Narrow ALL_REGS to GENERAL_REGS. This supports allowing movsf and
movdf to do mem-to-mem moves through integer regs. */
-enum reg_class
-ix86_preferred_reload_class (rtx x, enum reg_class regclass)
+
+static reg_class_t
+ix86_preferred_reload_class (rtx x, reg_class_t regclass)
{
enum machine_mode mode = GET_MODE (x);
/* Discourage putting floating-point values in SSE registers unless
SSE math is being used, and likewise for the 387 registers. */
-enum reg_class
-ix86_preferred_output_reload_class (rtx x, enum reg_class regclass)
+static reg_class_t
+ix86_preferred_output_reload_class (rtx x, reg_class_t regclass)
{
enum machine_mode mode = GET_MODE (x);
sprintf (lazy_ptr_name, "L%d$lz", label);
- if (MACHOPIC_PURE)
+ if (MACHOPIC_ATT_STUB)
+ switch_to_section (darwin_sections[machopic_picsymbol_stub3_section]);
+ else if (MACHOPIC_PURE)
+ {
+ if (TARGET_DEEP_BRANCH_PREDICTION)
+ switch_to_section (darwin_sections[machopic_picsymbol_stub2_section]);
+ else
switch_to_section (darwin_sections[machopic_picsymbol_stub_section]);
+ }
else
switch_to_section (darwin_sections[machopic_symbol_stub_section]);
fprintf (file, "%s:\n", stub);
fprintf (file, "\t.indirect_symbol %s\n", symbol_name);
- if (MACHOPIC_PURE)
+ if (MACHOPIC_ATT_STUB)
+ {
+ fprintf (file, "\thlt ; hlt ; hlt ; hlt ; hlt\n");
+ }
+ else if (MACHOPIC_PURE)
{
- fprintf (file, "\tcall\tLPC$%d\nLPC$%d:\tpopl\t%%eax\n", label, label);
- fprintf (file, "\tmovl\t%s-LPC$%d(%%eax),%%edx\n", lazy_ptr_name, label);
- fprintf (file, "\tjmp\t*%%edx\n");
+ /* PIC stub. */
+ if (TARGET_DEEP_BRANCH_PREDICTION)
+ {
+ /* 25-byte PIC stub using "CALL get_pc_thunk". */
+ rtx tmp = gen_rtx_REG (SImode, 2 /* ECX */);
+ output_set_got (tmp, NULL_RTX); /* "CALL ___<cpu>.get_pc_thunk.cx". */
+ fprintf (file, "LPC$%d:\tmovl\t%s-LPC$%d(%%ecx),%%ecx\n", label, lazy_ptr_name, label);
+ }
+ else
+ {
+ /* 26-byte PIC stub using inline picbase: "CALL L42 ! L42: pop %eax". */
+ fprintf (file, "\tcall LPC$%d\nLPC$%d:\tpopl %%ecx\n", label, label);
+ fprintf (file, "\tmovl %s-LPC$%d(%%ecx),%%ecx\n", lazy_ptr_name, label);
+ }
+ fprintf (file, "\tjmp\t*%%ecx\n");
}
else
fprintf (file, "\tjmp\t*%s\n", lazy_ptr_name);
+ /* The AT&T-style ("self-modifying") stub is not lazily bound, thus
+ it needs no stub-binding-helper. */
+ if (MACHOPIC_ATT_STUB)
+ return;
+
fprintf (file, "%s:\n", binder_name);
if (MACHOPIC_PURE)
{
- fprintf (file, "\tlea\t%s-LPC$%d(%%eax),%%eax\n", lazy_ptr_name, label);
- fputs ("\tpushl\t%eax\n", file);
+ fprintf (file, "\tlea\t%s-%s(%%ecx),%%ecx\n", lazy_ptr_name, binder_name);
+ fprintf (file, "\tpushl\t%%ecx\n");
}
else
fprintf (file, "\tpushl\t$%s\n", lazy_ptr_name);
fputs ("\tjmp\tdyld_stub_binding_helper\n", file);
+ /* N.B. Keep the correspondence of these
+ 'symbol_ptr/symbol_ptr2/symbol_ptr3' sections consistent with the
+ old-pic/new-pic/non-pic stubs; altering this will break
+ compatibility with existing dylibs. */
+ if (MACHOPIC_PURE)
+ {
+ /* PIC stubs. */
+ if (TARGET_DEEP_BRANCH_PREDICTION)
+ /* 25-byte PIC stub using "CALL get_pc_thunk". */
+ switch_to_section (darwin_sections[machopic_lazy_symbol_ptr2_section]);
+ else
+ /* 26-byte PIC stub using inline picbase: "CALL L42 ! L42: pop %ebx". */
switch_to_section (darwin_sections[machopic_lazy_symbol_ptr_section]);
+ }
+ else
+ /* 16-byte -mdynamic-no-pic stub. */
+ switch_to_section(darwin_sections[machopic_lazy_symbol_ptr3_section]);
+
fprintf (file, "%s:\n", lazy_ptr_name);
fprintf (file, "\t.indirect_symbol %s\n", symbol_name);
fprintf (file, ASM_LONG "%s\n", binder_name);
}
}
-#ifdef ASM_OUTPUT_MAX_SKIP_PAD
/* We don't have exact information about the insn sizes, but we may assume
quite safely that we are informed about all 1 byte insns and memory
address sizes. This is enough to eliminate unnecessary padding in
return 2;
}
+#ifdef ASM_OUTPUT_MAX_SKIP_PAD
+
/* AMD K8 core mispredicts jumps when there are more than 3 jumps in 16 byte
window. */
}
}
+/* Count the minimum number of instructions in BB. Return 4 if the
+ number of instructions >= 4. */
+
+static int
+ix86_count_insn_bb (basic_block bb)
+{
+ rtx insn;
+ int insn_count = 0;
+
+ /* Count number of instructions in this block. Return 4 if the number
+ of instructions >= 4. */
+ FOR_BB_INSNS (bb, insn)
+ {
+ /* Only happen in exit blocks. */
+ if (JUMP_P (insn)
+ && GET_CODE (PATTERN (insn)) == RETURN)
+ break;
+
+ if (NONDEBUG_INSN_P (insn)
+ && GET_CODE (PATTERN (insn)) != USE
+ && GET_CODE (PATTERN (insn)) != CLOBBER)
+ {
+ insn_count++;
+ if (insn_count >= 4)
+ return insn_count;
+ }
+ }
+
+ return insn_count;
+}
+
+
+/* Count the minimum number of instructions in code path in BB.
+ Return 4 if the number of instructions >= 4. */
+
+static int
+ix86_count_insn (basic_block bb)
+{
+ edge e;
+ edge_iterator ei;
+ int min_prev_count;
+
+ /* Only bother counting instructions along paths with no
+ more than 2 basic blocks between entry and exit. Given
+ that BB has an edge to exit, determine if a predecessor
+ of BB has an edge from entry. If so, compute the number
+ of instructions in the predecessor block. If there
+ happen to be multiple such blocks, compute the minimum. */
+ min_prev_count = 4;
+ FOR_EACH_EDGE (e, ei, bb->preds)
+ {
+ edge prev_e;
+ edge_iterator prev_ei;
+
+ if (e->src == ENTRY_BLOCK_PTR)
+ {
+ min_prev_count = 0;
+ break;
+ }
+ FOR_EACH_EDGE (prev_e, prev_ei, e->src->preds)
+ {
+ if (prev_e->src == ENTRY_BLOCK_PTR)
+ {
+ int count = ix86_count_insn_bb (e->src);
+ if (count < min_prev_count)
+ min_prev_count = count;
+ break;
+ }
+ }
+ }
+
+ if (min_prev_count < 4)
+ min_prev_count += ix86_count_insn_bb (bb);
+
+ return min_prev_count;
+}
+
+/* Pad short funtion to 4 instructions. */
+
+static void
+ix86_pad_short_function (void)
+{
+ edge e;
+ edge_iterator ei;
+
+ FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR->preds)
+ {
+ rtx ret = BB_END (e->src);
+ if (JUMP_P (ret) && GET_CODE (PATTERN (ret)) == RETURN)
+ {
+ int insn_count = ix86_count_insn (e->src);
+
+ /* Pad short function. */
+ if (insn_count < 4)
+ {
+ rtx insn = ret;
+
+ /* Find epilogue. */
+ while (insn
+ && (!NOTE_P (insn)
+ || NOTE_KIND (insn) != NOTE_INSN_EPILOGUE_BEG))
+ insn = PREV_INSN (insn);
+
+ if (!insn)
+ insn = ret;
+
+ /* Two NOPs are counted as one instruction. */
+ insn_count = 2 * (4 - insn_count);
+ emit_insn_before (gen_nops (GEN_INT (insn_count)), insn);
+ }
+ }
+ }
+}
+
/* Implement machine specific optimizations. We implement padding of returns
for K8 CPUs and pass to avoid 4 jumps in the single 16 byte window. */
static void
{
if (optimize && optimize_function_for_speed_p (cfun))
{
- if (TARGET_PAD_RETURNS)
+ if (TARGET_PAD_SHORT_FUNCTION)
+ ix86_pad_short_function ();
+ else if (TARGET_PAD_RETURNS)
ix86_pad_returns ();
#ifdef ASM_OUTPUT_MAX_SKIP_PAD
if (TARGET_FOUR_JUMP_LIMIT)
ix86_avoid_jump_mispredicts ();
#endif
}
+
+ /* Run the vzeroupper optimization if needed. */
+ if (cfun->machine->use_vzeroupper_p)
+ move_or_delete_vzeroupper ();
}
/* Return nonzero when QImode register that must be represented via REX prefix
two = CONST_DOUBLE_FROM_REAL_VALUE (dconst2, SFmode);
if (VECTOR_MODE_P (mode))
- two = ix86_build_const_vector (SFmode, true, two);
+ two = ix86_build_const_vector (mode, true, two);
two = force_reg (mode, two);
if (VECTOR_MODE_P (mode))
{
- mthree = ix86_build_const_vector (SFmode, true, mthree);
- mhalf = ix86_build_const_vector (SFmode, true, mhalf);
+ mthree = ix86_build_const_vector (mode, true, mthree);
+ mhalf = ix86_build_const_vector (mode, true, mhalf);
}
/* sqrt(a) = -0.5 * a * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0)
rtx sgn = gen_reg_rtx (mode);
if (mask == NULL_RTX)
{
- mask = ix86_build_signbit_mask (mode, VECTOR_MODE_P (mode), false);
+ enum machine_mode vmode;
+
+ if (mode == SFmode)
+ vmode = V4SFmode;
+ else if (mode == DFmode)
+ vmode = V2DFmode;
+ else
+ vmode = mode;
+
+ mask = ix86_build_signbit_mask (vmode, VECTOR_MODE_P (mode), false);
if (!VECTOR_MODE_P (mode))
{
/* We need to generate a scalar mode mask in this case. */
static rtx
ix86_expand_sse_fabs (rtx op0, rtx *smask)
{
- enum machine_mode mode = GET_MODE (op0);
+ enum machine_mode vmode, mode = GET_MODE (op0);
rtx xa, mask;
xa = gen_reg_rtx (mode);
- mask = ix86_build_signbit_mask (mode, VECTOR_MODE_P (mode), true);
+ if (mode == SFmode)
+ vmode = V4SFmode;
+ else if (mode == DFmode)
+ vmode = V2DFmode;
+ else
+ vmode = mode;
+ mask = ix86_build_signbit_mask (vmode, VECTOR_MODE_P (mode), true);
if (!VECTOR_MODE_P (mode))
{
/* We need to generate a scalar mode mask in this case. */
static bool
expand_vec_perm_even_odd_1 (struct expand_vec_perm_d *d, unsigned odd)
{
- rtx t1, t2, t3, t4;
+ rtx t1, t2, t3;
switch (d->vmode)
{
case V8SFmode:
{
- static const unsigned char perm1[8] = { 0, 2, 1, 3, 5, 6, 5, 7 };
- static const unsigned char perme[8] = { 0, 1, 8, 9, 4, 5, 12, 13 };
- static const unsigned char permo[8] = { 2, 3, 10, 11, 6, 7, 14, 15 };
+ int mask = odd ? 0xdd : 0x88;
t1 = gen_reg_rtx (V8SFmode);
t2 = gen_reg_rtx (V8SFmode);
t3 = gen_reg_rtx (V8SFmode);
- t4 = gen_reg_rtx (V8SFmode);
/* Shuffle within the 128-bit lanes to produce:
- { 0 2 1 3 4 6 5 7 } and { 8 a 9 b c e d f }. */
- expand_vselect (t1, d->op0, perm1, 8);
- expand_vselect (t2, d->op1, perm1, 8);
+ { 0 2 8 a 4 6 c e } | { 1 3 9 b 5 7 d f }. */
+ emit_insn (gen_avx_shufps256 (t1, d->op0, d->op1,
+ GEN_INT (mask)));
/* Shuffle the lanes around to produce:
- { 0 2 1 3 8 a 9 b } and { 4 6 5 7 c e d f }. */
- emit_insn (gen_avx_vperm2f128v8sf3 (t3, t1, t2, GEN_INT (0x20)));
- emit_insn (gen_avx_vperm2f128v8sf3 (t4, t1, t2, GEN_INT (0x31)));
-
- /* Now a vpermil2p will produce the result required. */
- /* ??? The vpermil2p requires a vector constant. Another option
- is a unpck[lh]ps to merge the two vectors to produce
- { 0 4 2 6 8 c a e } or { 1 5 3 7 9 d b f }. Then use another
- vpermilps to get the elements into the final order. */
- d->op0 = t3;
- d->op1 = t4;
- memcpy (d->perm, odd ? permo: perme, 8);
- expand_vec_perm_vpermil (d);
+ { 4 6 c e 0 2 8 a } and { 5 7 d f 1 3 9 b }. */
+ emit_insn (gen_avx_vperm2f128v8sf3 (t2, t1, t1,
+ GEN_INT (0x3)));
+
+ /* Shuffle within the 128-bit lanes to produce:
+ { 0 2 4 6 4 6 0 2 } | { 1 3 5 7 5 7 1 3 }. */
+ emit_insn (gen_avx_shufps256 (t3, t1, t2, GEN_INT (0x44)));
+
+ /* Shuffle within the 128-bit lanes to produce:
+ { 8 a c e c e 8 a } | { 9 b d f d f 9 b }. */
+ emit_insn (gen_avx_shufps256 (t2, t1, t2, GEN_INT (0xee)));
+
+ /* Shuffle the lanes around to produce:
+ { 0 2 4 6 8 a c e } | { 1 3 5 7 9 b d f }. */
+ emit_insn (gen_avx_vperm2f128v8sf3 (d->target, t3, t2,
+ GEN_INT (0x20)));
}
break;
return 0;
}
+#undef TARGET_SCHED_DISPATCH
+#define TARGET_SCHED_DISPATCH has_dispatch
+#undef TARGET_SCHED_DISPATCH_DO
+#define TARGET_SCHED_DISPATCH_DO do_dispatch
+
+/* The size of the dispatch window is the total number of bytes of
+ object code allowed in a window. */
+#define DISPATCH_WINDOW_SIZE 16
+
+/* Number of dispatch windows considered for scheduling. */
+#define MAX_DISPATCH_WINDOWS 3
+
+/* Maximum number of instructions in a window. */
+#define MAX_INSN 4
+
+/* Maximum number of immediate operands in a window. */
+#define MAX_IMM 4
+
+/* Maximum number of immediate bits allowed in a window. */
+#define MAX_IMM_SIZE 128
+
+/* Maximum number of 32 bit immediates allowed in a window. */
+#define MAX_IMM_32 4
+
+/* Maximum number of 64 bit immediates allowed in a window. */
+#define MAX_IMM_64 2
+
+/* Maximum total of loads or prefetches allowed in a window. */
+#define MAX_LOAD 2
+
+/* Maximum total of stores allowed in a window. */
+#define MAX_STORE 1
+
+#undef BIG
+#define BIG 100
+
+
+/* Dispatch groups. Istructions that affect the mix in a dispatch window. */
+enum dispatch_group {
+ disp_no_group = 0,
+ disp_load,
+ disp_store,
+ disp_load_store,
+ disp_prefetch,
+ disp_imm,
+ disp_imm_32,
+ disp_imm_64,
+ disp_branch,
+ disp_cmp,
+ disp_jcc,
+ disp_last
+};
+
+/* Number of allowable groups in a dispatch window. It is an array
+ indexed by dispatch_group enum. 100 is used as a big number,
+ because the number of these kind of operations does not have any
+ effect in dispatch window, but we need them for other reasons in
+ the table. */
+static unsigned int num_allowable_groups[disp_last] = {
+ 0, 2, 1, 1, 2, 4, 4, 2, 1, BIG, BIG
+};
+
+char group_name[disp_last + 1][16] = {
+ "disp_no_group", "disp_load", "disp_store", "disp_load_store",
+ "disp_prefetch", "disp_imm", "disp_imm_32", "disp_imm_64",
+ "disp_branch", "disp_cmp", "disp_jcc", "disp_last"
+};
+
+/* Instruction path. */
+enum insn_path {
+ no_path = 0,
+ path_single, /* Single micro op. */
+ path_double, /* Double micro op. */
+ path_multi, /* Instructions with more than 2 micro op.. */
+ last_path
+};
+
+/* sched_insn_info defines a window to the instructions scheduled in
+ the basic block. It contains a pointer to the insn_info table and
+ the instruction scheduled.
+
+ Windows are allocated for each basic block and are linked
+ together. */
+typedef struct sched_insn_info_s {
+ rtx insn;
+ enum dispatch_group group;
+ enum insn_path path;
+ int byte_len;
+ int imm_bytes;
+} sched_insn_info;
+
+/* Linked list of dispatch windows. This is a two way list of
+ dispatch windows of a basic block. It contains information about
+ the number of uops in the window and the total number of
+ instructions and of bytes in the object code for this dispatch
+ window. */
+typedef struct dispatch_windows_s {
+ int num_insn; /* Number of insn in the window. */
+ int num_uops; /* Number of uops in the window. */
+ int window_size; /* Number of bytes in the window. */
+ int window_num; /* Window number between 0 or 1. */
+ int num_imm; /* Number of immediates in an insn. */
+ int num_imm_32; /* Number of 32 bit immediates in an insn. */
+ int num_imm_64; /* Number of 64 bit immediates in an insn. */
+ int imm_size; /* Total immediates in the window. */
+ int num_loads; /* Total memory loads in the window. */
+ int num_stores; /* Total memory stores in the window. */
+ int violation; /* Violation exists in window. */
+ sched_insn_info *window; /* Pointer to the window. */
+ struct dispatch_windows_s *next;
+ struct dispatch_windows_s *prev;
+} dispatch_windows;
+
+/* Immediate valuse used in an insn. */
+typedef struct imm_info_s
+ {
+ int imm;
+ int imm32;
+ int imm64;
+ } imm_info;
+
+static dispatch_windows *dispatch_window_list;
+static dispatch_windows *dispatch_window_list1;
+
+/* Get dispatch group of insn. */
+
+static enum dispatch_group
+get_mem_group (rtx insn)
+{
+ enum attr_memory memory;
+
+ if (INSN_CODE (insn) < 0)
+ return disp_no_group;
+ memory = get_attr_memory (insn);
+ if (memory == MEMORY_STORE)
+ return disp_store;
+
+ if (memory == MEMORY_LOAD)
+ return disp_load;
+
+ if (memory == MEMORY_BOTH)
+ return disp_load_store;
+
+ return disp_no_group;
+}
+
+/* Return true if insn is a compare instruction. */
+
+static bool
+is_cmp (rtx insn)
+{
+ enum attr_type type;
+
+ type = get_attr_type (insn);
+ return (type == TYPE_TEST
+ || type == TYPE_ICMP
+ || type == TYPE_FCMP
+ || GET_CODE (PATTERN (insn)) == COMPARE);
+}
+
+/* Return true if a dispatch violation encountered. */
+
+static bool
+dispatch_violation (void)
+{
+ if (dispatch_window_list->next)
+ return dispatch_window_list->next->violation;
+ return dispatch_window_list->violation;
+}
+
+/* Return true if insn is a branch instruction. */
+
+static bool
+is_branch (rtx insn)
+{
+ return (CALL_P (insn) || JUMP_P (insn));
+}
+
+/* Return true if insn is a prefetch instruction. */
+
+static bool
+is_prefetch (rtx insn)
+{
+ return NONJUMP_INSN_P (insn) && GET_CODE (PATTERN (insn)) == PREFETCH;
+}
+
+/* This function initializes a dispatch window and the list container holding a
+ pointer to the window. */
+
+static void
+init_window (int window_num)
+{
+ int i;
+ dispatch_windows *new_list;
+
+ if (window_num == 0)
+ new_list = dispatch_window_list;
+ else
+ new_list = dispatch_window_list1;
+
+ new_list->num_insn = 0;
+ new_list->num_uops = 0;
+ new_list->window_size = 0;
+ new_list->next = NULL;
+ new_list->prev = NULL;
+ new_list->window_num = window_num;
+ new_list->num_imm = 0;
+ new_list->num_imm_32 = 0;
+ new_list->num_imm_64 = 0;
+ new_list->imm_size = 0;
+ new_list->num_loads = 0;
+ new_list->num_stores = 0;
+ new_list->violation = false;
+
+ for (i = 0; i < MAX_INSN; i++)
+ {
+ new_list->window[i].insn = NULL;
+ new_list->window[i].group = disp_no_group;
+ new_list->window[i].path = no_path;
+ new_list->window[i].byte_len = 0;
+ new_list->window[i].imm_bytes = 0;
+ }
+ return;
+}
+
+/* This function allocates and initializes a dispatch window and the
+ list container holding a pointer to the window. */
+
+static dispatch_windows *
+allocate_window (void)
+{
+ dispatch_windows *new_list = XNEW (struct dispatch_windows_s);
+ new_list->window = XNEWVEC (struct sched_insn_info_s, MAX_INSN + 1);
+
+ return new_list;
+}
+
+/* This routine initializes the dispatch scheduling information. It
+ initiates building dispatch scheduler tables and constructs the
+ first dispatch window. */
+
+static void
+init_dispatch_sched (void)
+{
+ /* Allocate a dispatch list and a window. */
+ dispatch_window_list = allocate_window ();
+ dispatch_window_list1 = allocate_window ();
+ init_window (0);
+ init_window (1);
+}
+
+/* This function returns true if a branch is detected. End of a basic block
+ does not have to be a branch, but here we assume only branches end a
+ window. */
+
+static bool
+is_end_basic_block (enum dispatch_group group)
+{
+ return group == disp_branch;
+}
+
+/* This function is called when the end of a window processing is reached. */
+
+static void
+process_end_window (void)
+{
+ gcc_assert (dispatch_window_list->num_insn <= MAX_INSN);
+ if (dispatch_window_list->next)
+ {
+ gcc_assert (dispatch_window_list1->num_insn <= MAX_INSN);
+ gcc_assert (dispatch_window_list->window_size
+ + dispatch_window_list1->window_size <= 48);
+ init_window (1);
+ }
+ init_window (0);
+}
+
+/* Allocates a new dispatch window and adds it to WINDOW_LIST.
+ WINDOW_NUM is either 0 or 1. A maximum of two windows are generated
+ for 48 bytes of instructions. Note that these windows are not dispatch
+ windows that their sizes are DISPATCH_WINDOW_SIZE. */
+
+static dispatch_windows *
+allocate_next_window (int window_num)
+{
+ if (window_num == 0)
+ {
+ if (dispatch_window_list->next)
+ init_window (1);
+ init_window (0);
+ return dispatch_window_list;
+ }
+
+ dispatch_window_list->next = dispatch_window_list1;
+ dispatch_window_list1->prev = dispatch_window_list;
+
+ return dispatch_window_list1;
+}
+
+/* Increment the number of immediate operands of an instruction. */
+
+static int
+find_constant_1 (rtx *in_rtx, imm_info *imm_values)
+{
+ if (*in_rtx == 0)
+ return 0;
+
+ switch ( GET_CODE (*in_rtx))
+ {
+ case CONST:
+ case SYMBOL_REF:
+ case CONST_INT:
+ (imm_values->imm)++;
+ if (x86_64_immediate_operand (*in_rtx, SImode))
+ (imm_values->imm32)++;
+ else
+ (imm_values->imm64)++;
+ break;
+
+ case CONST_DOUBLE:
+ (imm_values->imm)++;
+ (imm_values->imm64)++;
+ break;
+
+ case CODE_LABEL:
+ if (LABEL_KIND (*in_rtx) == LABEL_NORMAL)
+ {
+ (imm_values->imm)++;
+ (imm_values->imm32)++;
+ }
+ break;
+
+ default:
+ break;
+ }
+
+ return 0;
+}
+
+/* Compute number of immediate operands of an instruction. */
+
+static void
+find_constant (rtx in_rtx, imm_info *imm_values)
+{
+ for_each_rtx (INSN_P (in_rtx) ? &PATTERN (in_rtx) : &in_rtx,
+ (rtx_function) find_constant_1, (void *) imm_values);
+}
+
+/* Return total size of immediate operands of an instruction along with number
+ of corresponding immediate-operands. It initializes its parameters to zero
+ befor calling FIND_CONSTANT.
+ INSN is the input instruction. IMM is the total of immediates.
+ IMM32 is the number of 32 bit immediates. IMM64 is the number of 64
+ bit immediates. */
+
+static int
+get_num_immediates (rtx insn, int *imm, int *imm32, int *imm64)
+{
+ imm_info imm_values = {0, 0, 0};
+
+ find_constant (insn, &imm_values);
+ *imm = imm_values.imm;
+ *imm32 = imm_values.imm32;
+ *imm64 = imm_values.imm64;
+ return imm_values.imm32 * 4 + imm_values.imm64 * 8;
+}
+
+/* This function indicates if an operand of an instruction is an
+ immediate. */
+
+static bool
+has_immediate (rtx insn)
+{
+ int num_imm_operand;
+ int num_imm32_operand;
+ int num_imm64_operand;
+
+ if (insn)
+ return get_num_immediates (insn, &num_imm_operand, &num_imm32_operand,
+ &num_imm64_operand);
+ return false;
+}
+
+/* Return single or double path for instructions. */
+
+static enum insn_path
+get_insn_path (rtx insn)
+{
+ enum attr_amdfam10_decode path = get_attr_amdfam10_decode (insn);
+
+ if ((int)path == 0)
+ return path_single;
+
+ if ((int)path == 1)
+ return path_double;
+
+ return path_multi;
+}
+
+/* Return insn dispatch group. */
+
+static enum dispatch_group
+get_insn_group (rtx insn)
+{
+ enum dispatch_group group = get_mem_group (insn);
+ if (group)
+ return group;
+
+ if (is_branch (insn))
+ return disp_branch;
+
+ if (is_cmp (insn))
+ return disp_cmp;
+
+ if (has_immediate (insn))
+ return disp_imm;
+
+ if (is_prefetch (insn))
+ return disp_prefetch;
+
+ return disp_no_group;
+}
+
+/* Count number of GROUP restricted instructions in a dispatch
+ window WINDOW_LIST. */
+
+static int
+count_num_restricted (rtx insn, dispatch_windows *window_list)
+{
+ enum dispatch_group group = get_insn_group (insn);
+ int imm_size;
+ int num_imm_operand;
+ int num_imm32_operand;
+ int num_imm64_operand;
+
+ if (group == disp_no_group)
+ return 0;
+
+ if (group == disp_imm)
+ {
+ imm_size = get_num_immediates (insn, &num_imm_operand, &num_imm32_operand,
+ &num_imm64_operand);
+ if (window_list->imm_size + imm_size > MAX_IMM_SIZE
+ || num_imm_operand + window_list->num_imm > MAX_IMM
+ || (num_imm32_operand > 0
+ && (window_list->num_imm_32 + num_imm32_operand > MAX_IMM_32
+ || window_list->num_imm_64 * 2 + num_imm32_operand > MAX_IMM_32))
+ || (num_imm64_operand > 0
+ && (window_list->num_imm_64 + num_imm64_operand > MAX_IMM_64
+ || window_list->num_imm_32 + num_imm64_operand * 2 > MAX_IMM_32))
+ || (window_list->imm_size + imm_size == MAX_IMM_SIZE
+ && num_imm64_operand > 0
+ && ((window_list->num_imm_64 > 0
+ && window_list->num_insn >= 2)
+ || window_list->num_insn >= 3)))
+ return BIG;
+
+ return 1;
+ }
+
+ if ((group == disp_load_store
+ && (window_list->num_loads >= MAX_LOAD
+ || window_list->num_stores >= MAX_STORE))
+ || ((group == disp_load
+ || group == disp_prefetch)
+ && window_list->num_loads >= MAX_LOAD)
+ || (group == disp_store
+ && window_list->num_stores >= MAX_STORE))
+ return BIG;
+
+ return 1;
+}
+
+/* This function returns true if insn satisfies dispatch rules on the
+ last window scheduled. */
+
+static bool
+fits_dispatch_window (rtx insn)
+{
+ dispatch_windows *window_list = dispatch_window_list;
+ dispatch_windows *window_list_next = dispatch_window_list->next;
+ unsigned int num_restrict;
+ enum dispatch_group group = get_insn_group (insn);
+ enum insn_path path = get_insn_path (insn);
+ int sum;
+
+ /* Make disp_cmp and disp_jcc get scheduled at the latest. These
+ instructions should be given the lowest priority in the
+ scheduling process in Haifa scheduler to make sure they will be
+ scheduled in the same dispatch window as the refrence to them. */
+ if (group == disp_jcc || group == disp_cmp)
+ return false;
+
+ /* Check nonrestricted. */
+ if (group == disp_no_group || group == disp_branch)
+ return true;
+
+ /* Get last dispatch window. */
+ if (window_list_next)
+ window_list = window_list_next;
+
+ if (window_list->window_num == 1)
+ {
+ sum = window_list->prev->window_size + window_list->window_size;
+
+ if (sum == 32
+ || (min_insn_size (insn) + sum) >= 48)
+ /* Window 1 is full. Go for next window. */
+ return true;
+ }
+
+ num_restrict = count_num_restricted (insn, window_list);
+
+ if (num_restrict > num_allowable_groups[group])
+ return false;
+
+ /* See if it fits in the first window. */
+ if (window_list->window_num == 0)
+ {
+ /* The first widow should have only single and double path
+ uops. */
+ if (path == path_double
+ && (window_list->num_uops + 2) > MAX_INSN)
+ return false;
+ else if (path != path_single)
+ return false;
+ }
+ return true;
+}
+
+/* Add an instruction INSN with NUM_UOPS micro-operations to the
+ dispatch window WINDOW_LIST. */
+
+static void
+add_insn_window (rtx insn, dispatch_windows *window_list, int num_uops)
+{
+ int byte_len = min_insn_size (insn);
+ int num_insn = window_list->num_insn;
+ int imm_size;
+ sched_insn_info *window = window_list->window;
+ enum dispatch_group group = get_insn_group (insn);
+ enum insn_path path = get_insn_path (insn);
+ int num_imm_operand;
+ int num_imm32_operand;
+ int num_imm64_operand;
+
+ if (!window_list->violation && group != disp_cmp
+ && !fits_dispatch_window (insn))
+ window_list->violation = true;
+
+ imm_size = get_num_immediates (insn, &num_imm_operand, &num_imm32_operand,
+ &num_imm64_operand);
+
+ /* Initialize window with new instruction. */
+ window[num_insn].insn = insn;
+ window[num_insn].byte_len = byte_len;
+ window[num_insn].group = group;
+ window[num_insn].path = path;
+ window[num_insn].imm_bytes = imm_size;
+
+ window_list->window_size += byte_len;
+ window_list->num_insn = num_insn + 1;
+ window_list->num_uops = window_list->num_uops + num_uops;
+ window_list->imm_size += imm_size;
+ window_list->num_imm += num_imm_operand;
+ window_list->num_imm_32 += num_imm32_operand;
+ window_list->num_imm_64 += num_imm64_operand;
+
+ if (group == disp_store)
+ window_list->num_stores += 1;
+ else if (group == disp_load
+ || group == disp_prefetch)
+ window_list->num_loads += 1;
+ else if (group == disp_load_store)
+ {
+ window_list->num_stores += 1;
+ window_list->num_loads += 1;
+ }
+}
+
+/* Adds a scheduled instruction, INSN, to the current dispatch window.
+ If the total bytes of instructions or the number of instructions in
+ the window exceed allowable, it allocates a new window. */
+
+static void
+add_to_dispatch_window (rtx insn)
+{
+ int byte_len;
+ dispatch_windows *window_list;
+ dispatch_windows *next_list;
+ dispatch_windows *window0_list;
+ enum insn_path path;
+ enum dispatch_group insn_group;
+ bool insn_fits;
+ int num_insn;
+ int num_uops;
+ int window_num;
+ int insn_num_uops;
+ int sum;
+
+ if (INSN_CODE (insn) < 0)
+ return;
+
+ byte_len = min_insn_size (insn);
+ window_list = dispatch_window_list;
+ next_list = window_list->next;
+ path = get_insn_path (insn);
+ insn_group = get_insn_group (insn);
+
+ /* Get the last dispatch window. */
+ if (next_list)
+ window_list = dispatch_window_list->next;
+
+ if (path == path_single)
+ insn_num_uops = 1;
+ else if (path == path_double)
+ insn_num_uops = 2;
+ else
+ insn_num_uops = (int) path;
+
+ /* If current window is full, get a new window.
+ Window number zero is full, if MAX_INSN uops are scheduled in it.
+ Window number one is full, if window zero's bytes plus window
+ one's bytes is 32, or if the bytes of the new instruction added
+ to the total makes it greater than 48, or it has already MAX_INSN
+ instructions in it. */
+ num_insn = window_list->num_insn;
+ num_uops = window_list->num_uops;
+ window_num = window_list->window_num;
+ insn_fits = fits_dispatch_window (insn);
+
+ if (num_insn >= MAX_INSN
+ || num_uops + insn_num_uops > MAX_INSN
+ || !(insn_fits))
+ {
+ window_num = ~window_num & 1;
+ window_list = allocate_next_window (window_num);
+ }
+
+ if (window_num == 0)
+ {
+ add_insn_window (insn, window_list, insn_num_uops);
+ if (window_list->num_insn >= MAX_INSN
+ && insn_group == disp_branch)
+ {
+ process_end_window ();
+ return;
+ }
+ }
+ else if (window_num == 1)
+ {
+ window0_list = window_list->prev;
+ sum = window0_list->window_size + window_list->window_size;
+ if (sum == 32
+ || (byte_len + sum) >= 48)
+ {
+ process_end_window ();
+ window_list = dispatch_window_list;
+ }
+
+ add_insn_window (insn, window_list, insn_num_uops);
+ }
+ else
+ gcc_unreachable ();
+
+ if (is_end_basic_block (insn_group))
+ {
+ /* End of basic block is reached do end-basic-block process. */
+ process_end_window ();
+ return;
+ }
+}
+
+/* Print the dispatch window, WINDOW_NUM, to FILE. */
+
+DEBUG_FUNCTION static void
+debug_dispatch_window_file (FILE *file, int window_num)
+{
+ dispatch_windows *list;
+ int i;
+
+ if (window_num == 0)
+ list = dispatch_window_list;
+ else
+ list = dispatch_window_list1;
+
+ fprintf (file, "Window #%d:\n", list->window_num);
+ fprintf (file, " num_insn = %d, num_uops = %d, window_size = %d\n",
+ list->num_insn, list->num_uops, list->window_size);
+ fprintf (file, " num_imm = %d, num_imm_32 = %d, num_imm_64 = %d, imm_size = %d\n",
+ list->num_imm, list->num_imm_32, list->num_imm_64, list->imm_size);
+
+ fprintf (file, " num_loads = %d, num_stores = %d\n", list->num_loads,
+ list->num_stores);
+ fprintf (file, " insn info:\n");
+
+ for (i = 0; i < MAX_INSN; i++)
+ {
+ if (!list->window[i].insn)
+ break;
+ fprintf (file, " group[%d] = %s, insn[%d] = %p, path[%d] = %d byte_len[%d] = %d, imm_bytes[%d] = %d\n",
+ i, group_name[list->window[i].group],
+ i, (void *)list->window[i].insn,
+ i, list->window[i].path,
+ i, list->window[i].byte_len,
+ i, list->window[i].imm_bytes);
+ }
+}
+
+/* Print to stdout a dispatch window. */
+
+DEBUG_FUNCTION void
+debug_dispatch_window (int window_num)
+{
+ debug_dispatch_window_file (stdout, window_num);
+}
+
+/* Print INSN dispatch information to FILE. */
+
+DEBUG_FUNCTION static void
+debug_insn_dispatch_info_file (FILE *file, rtx insn)
+{
+ int byte_len;
+ enum insn_path path;
+ enum dispatch_group group;
+ int imm_size;
+ int num_imm_operand;
+ int num_imm32_operand;
+ int num_imm64_operand;
+
+ if (INSN_CODE (insn) < 0)
+ return;
+
+ byte_len = min_insn_size (insn);
+ path = get_insn_path (insn);
+ group = get_insn_group (insn);
+ imm_size = get_num_immediates (insn, &num_imm_operand, &num_imm32_operand,
+ &num_imm64_operand);
+
+ fprintf (file, " insn info:\n");
+ fprintf (file, " group = %s, path = %d, byte_len = %d\n",
+ group_name[group], path, byte_len);
+ fprintf (file, " num_imm = %d, num_imm_32 = %d, num_imm_64 = %d, imm_size = %d\n",
+ num_imm_operand, num_imm32_operand, num_imm64_operand, imm_size);
+}
+
+/* Print to STDERR the status of the ready list with respect to
+ dispatch windows. */
+
+DEBUG_FUNCTION void
+debug_ready_dispatch (void)
+{
+ int i;
+ int no_ready = number_in_ready ();
+
+ fprintf (stdout, "Number of ready: %d\n", no_ready);
+
+ for (i = 0; i < no_ready; i++)
+ debug_insn_dispatch_info_file (stdout, get_ready_element (i));
+}
+
+/* This routine is the driver of the dispatch scheduler. */
+
+static void
+do_dispatch (rtx insn, int mode)
+{
+ if (mode == DISPATCH_INIT)
+ init_dispatch_sched ();
+ else if (mode == ADD_TO_DISPATCH_WINDOW)
+ add_to_dispatch_window (insn);
+}
+
+/* Return TRUE if Dispatch Scheduling is supported. */
+
+static bool
+has_dispatch (rtx insn, int action)
+{
+ if (ix86_tune == PROCESSOR_BDVER1 && flag_dispatch_scheduler)
+ switch (action)
+ {
+ default:
+ return false;
+
+ case IS_DISPATCH_ON:
+ return true;
+ break;
+
+ case IS_CMP:
+ return is_cmp (insn);
+
+ case DISPATCH_VIOLATION:
+ return dispatch_violation ();
+
+ case FITS_DISPATCH_WINDOW:
+ return fits_dispatch_window (insn);
+ }
+
+ return false;
+}
+
+/* ??? No autovectorization into MMX or 3DNOW until we can reliably
+ place emms and femms instructions. */
+
+static enum machine_mode
+ix86_preferred_simd_mode (enum machine_mode mode)
+{
+ /* Disable double precision vectorizer if needed. */
+ if (mode == DFmode && !TARGET_VECTORIZE_DOUBLE)
+ return word_mode;
+
+ if (!TARGET_AVX && !TARGET_SSE)
+ return word_mode;
+
+ switch (mode)
+ {
+ case SFmode:
+ return TARGET_AVX ? V8SFmode : V4SFmode;
+ case DFmode:
+ return TARGET_AVX ? V4DFmode : V2DFmode;
+ case DImode:
+ return V2DImode;
+ case SImode:
+ return V4SImode;
+ case HImode:
+ return V8HImode;
+ case QImode:
+ return V16QImode;
+
+ default:;
+ }
+
+ return word_mode;
+}
+
+/* If AVX is enabled then try vectorizing with both 256bit and 128bit
+ vectors. */
+
+static unsigned int
+ix86_autovectorize_vector_sizes (void)
+{
+ return TARGET_AVX ? 32 | 16 : 0;
+}
+
/* Initialize the GCC target structure. */
#undef TARGET_RETURN_IN_MEMORY
#define TARGET_RETURN_IN_MEMORY ix86_return_in_memory
#undef TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA
#define TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA i386_asm_output_addr_const_extra
+#undef TARGET_SCHED_INIT_GLOBAL
+#define TARGET_SCHED_INIT_GLOBAL ix86_sched_init_global
#undef TARGET_SCHED_ADJUST_COST
#define TARGET_SCHED_ADJUST_COST ix86_adjust_cost
#undef TARGET_SCHED_ISSUE_RATE
#undef TARGET_HANDLE_OPTION
#define TARGET_HANDLE_OPTION ix86_handle_option
+#undef TARGET_OPTION_OVERRIDE
+#define TARGET_OPTION_OVERRIDE ix86_option_override
+#undef TARGET_OPTION_OPTIMIZATION_TABLE
+#define TARGET_OPTION_OPTIMIZATION_TABLE ix86_option_optimization_table
+#undef TARGET_OPTION_INIT_STRUCT
+#define TARGET_OPTION_INIT_STRUCT ix86_option_init_struct
+
#undef TARGET_REGISTER_MOVE_COST
#define TARGET_REGISTER_MOVE_COST ix86_register_move_cost
#undef TARGET_MEMORY_MOVE_COST
#undef TARGET_STACK_PROTECT_FAIL
#define TARGET_STACK_PROTECT_FAIL ix86_stack_protect_fail
+#undef TARGET_SUPPORTS_SPLIT_STACK
+#define TARGET_SUPPORTS_SPLIT_STACK ix86_supports_split_stack
+
#undef TARGET_FUNCTION_VALUE
#define TARGET_FUNCTION_VALUE ix86_function_value
#undef TARGET_SECONDARY_RELOAD
#define TARGET_SECONDARY_RELOAD ix86_secondary_reload
+#undef TARGET_PREFERRED_RELOAD_CLASS
+#define TARGET_PREFERRED_RELOAD_CLASS ix86_preferred_reload_class
+#undef TARGET_PREFERRED_OUTPUT_RELOAD_CLASS
+#define TARGET_PREFERRED_OUTPUT_RELOAD_CLASS ix86_preferred_output_reload_class
#undef TARGET_CLASS_LIKELY_SPILLED_P
#define TARGET_CLASS_LIKELY_SPILLED_P ix86_class_likely_spilled_p
#undef TARGET_VECTORIZE_BUILTIN_VEC_PERM_OK
#define TARGET_VECTORIZE_BUILTIN_VEC_PERM_OK \
ix86_vectorize_builtin_vec_perm_ok
+#undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
+#define TARGET_VECTORIZE_PREFERRED_SIMD_MODE \
+ ix86_preferred_simd_mode
+#undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES
+#define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES \
+ ix86_autovectorize_vector_sizes
#undef TARGET_SET_CURRENT_FUNCTION
#define TARGET_SET_CURRENT_FUNCTION ix86_set_current_function
#undef TARGET_CAN_ELIMINATE
#define TARGET_CAN_ELIMINATE ix86_can_eliminate
+#undef TARGET_EXTRA_LIVE_ON_ENTRY
+#define TARGET_EXTRA_LIVE_ON_ENTRY ix86_live_on_entry
+
#undef TARGET_ASM_CODE_END
#define TARGET_ASM_CODE_END ix86_code_end
OSDN Git Service
