COSTS_N_BYTES (2), /* cost of FABS instruction. */
COSTS_N_BYTES (2), /* cost of FCHS instruction. */
COSTS_N_BYTES (2), /* cost of FSQRT instruction. */
- {{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
+ {{{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}}},
- {{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
+ {{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
+ {rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}}}},
+ {{{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}}},
+ {{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
+ {rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}}}},
1, /* scalar_stmt_cost. */
1, /* scalar load_cost. */
1, /* scalar_store_cost. */
COSTS_N_INSNS (22), /* cost of FABS instruction. */
COSTS_N_INSNS (24), /* cost of FCHS instruction. */
COSTS_N_INSNS (122), /* cost of FSQRT instruction. */
- {{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
+ {{{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
DUMMY_STRINGOP_ALGS},
- {{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
+ {{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
+ DUMMY_STRINGOP_ALGS}},
+ {{{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
DUMMY_STRINGOP_ALGS},
+ {{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
+ DUMMY_STRINGOP_ALGS}},
1, /* scalar_stmt_cost. */
1, /* scalar load_cost. */
1, /* scalar_store_cost. */
COSTS_N_INSNS (3), /* cost of FABS instruction. */
COSTS_N_INSNS (3), /* cost of FCHS instruction. */
COSTS_N_INSNS (83), /* cost of FSQRT instruction. */
- {{rep_prefix_4_byte, {{-1, rep_prefix_4_byte}}},
+ {{{rep_prefix_4_byte, {{-1, rep_prefix_4_byte}}},
DUMMY_STRINGOP_ALGS},
- {{rep_prefix_4_byte, {{-1, rep_prefix_4_byte}}},
+ {{rep_prefix_4_byte, {{-1, rep_prefix_4_byte}}},
+ DUMMY_STRINGOP_ALGS}},
+ {{{rep_prefix_4_byte, {{-1, rep_prefix_4_byte}}},
DUMMY_STRINGOP_ALGS},
+ {{rep_prefix_4_byte, {{-1, rep_prefix_4_byte}}},
+ DUMMY_STRINGOP_ALGS}},
1, /* scalar_stmt_cost. */
1, /* scalar load_cost. */
1, /* scalar_store_cost. */
COSTS_N_INSNS (1), /* cost of FABS instruction. */
COSTS_N_INSNS (1), /* cost of FCHS instruction. */
COSTS_N_INSNS (70), /* cost of FSQRT instruction. */
- {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
+ {{{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
DUMMY_STRINGOP_ALGS},
- {{libcall, {{-1, rep_prefix_4_byte}}},
+ {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
+ DUMMY_STRINGOP_ALGS}},
+ {{{libcall, {{-1, rep_prefix_4_byte}}},
DUMMY_STRINGOP_ALGS},
+ {{libcall, {{-1, rep_prefix_4_byte}}},
+ DUMMY_STRINGOP_ALGS}},
1, /* scalar_stmt_cost. */
1, /* scalar load_cost. */
1, /* scalar_store_cost. */
noticeable win, for bigger blocks either rep movsl or rep movsb is
way to go. Rep movsb has apparently more expensive startup time in CPU,
but after 4K the difference is down in the noise. */
- {{rep_prefix_4_byte, {{128, loop}, {1024, unrolled_loop},
+ {{{rep_prefix_4_byte, {{128, loop}, {1024, unrolled_loop},
{8192, rep_prefix_4_byte}, {-1, rep_prefix_1_byte}}},
DUMMY_STRINGOP_ALGS},
- {{rep_prefix_4_byte, {{1024, unrolled_loop},
- {8192, rep_prefix_4_byte}, {-1, libcall}}},
+ {{rep_prefix_4_byte, {{128, loop}, {1024, unrolled_loop},
+ {8192, rep_prefix_4_byte}, {-1, rep_prefix_1_byte}}},
+ DUMMY_STRINGOP_ALGS}},
+ {{{rep_prefix_4_byte, {{1024, unrolled_loop},
+ {8192, rep_prefix_4_byte}, {-1, libcall}}},
DUMMY_STRINGOP_ALGS},
+ {{rep_prefix_4_byte, {{1024, unrolled_loop},
+ {8192, rep_prefix_4_byte}, {-1, libcall}}},
+ DUMMY_STRINGOP_ALGS}},
1, /* scalar_stmt_cost. */
1, /* scalar load_cost. */
1, /* scalar_store_cost. */
COSTS_N_INSNS (1), /* cost of FABS instruction. */
COSTS_N_INSNS (1), /* cost of FCHS instruction. */
COSTS_N_INSNS (54), /* cost of FSQRT instruction. */
- {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
+ {{{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
DUMMY_STRINGOP_ALGS},
- {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
+ {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
+ DUMMY_STRINGOP_ALGS}},
+ {{{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
DUMMY_STRINGOP_ALGS},
+ {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
+ DUMMY_STRINGOP_ALGS}},
1, /* scalar_stmt_cost. */
1, /* scalar load_cost. */
1, /* scalar_store_cost. */
COSTS_N_INSNS (2), /* cost of FABS instruction. */
COSTS_N_INSNS (2), /* cost of FCHS instruction. */
COSTS_N_INSNS (56), /* cost of FSQRT instruction. */
- {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
+ {{{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
DUMMY_STRINGOP_ALGS},
- {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
+ {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
+ DUMMY_STRINGOP_ALGS}},
+ {{{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
DUMMY_STRINGOP_ALGS},
+ {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
+ DUMMY_STRINGOP_ALGS}},
1, /* scalar_stmt_cost. */
1, /* scalar load_cost. */
1, /* scalar_store_cost. */
/* For some reason, Athlon deals better with REP prefix (relative to loops)
compared to K8. Alignment becomes important after 8 bytes for memcpy and
128 bytes for memset. */
- {{libcall, {{2048, rep_prefix_4_byte}, {-1, libcall}}},
+ {{{libcall, {{2048, rep_prefix_4_byte}, {-1, libcall}}},
DUMMY_STRINGOP_ALGS},
- {{libcall, {{2048, rep_prefix_4_byte}, {-1, libcall}}},
+ {{libcall, {{2048, rep_prefix_4_byte}, {-1, libcall}}},
+ DUMMY_STRINGOP_ALGS}},
+ {{{libcall, {{2048, rep_prefix_4_byte}, {-1, libcall}}},
DUMMY_STRINGOP_ALGS},
+ {{libcall, {{2048, rep_prefix_4_byte}, {-1, libcall}}},
+ DUMMY_STRINGOP_ALGS}},
1, /* scalar_stmt_cost. */
1, /* scalar load_cost. */
1, /* scalar_store_cost. */
/* K8 has optimized REP instruction for medium sized blocks, but for very
small blocks it is better to use loop. For large blocks, libcall can
do nontemporary accesses and beat inline considerably. */
- {{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
+ {{{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
{libcall, {{16, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
- {{libcall, {{8, loop}, {24, unrolled_loop},
+ {{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
+ {libcall, {{16, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}},
+ {{{libcall, {{8, loop}, {24, unrolled_loop},
{2048, rep_prefix_4_byte}, {-1, libcall}}},
{libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
+ {{libcall, {{8, loop}, {24, unrolled_loop},
+ {2048, rep_prefix_4_byte}, {-1, libcall}}},
+ {libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}},
4, /* scalar_stmt_cost. */
2, /* scalar load_cost. */
2, /* scalar_store_cost. */
/* AMDFAM10 has optimized REP instruction for medium sized blocks, but for
very small blocks it is better to use loop. For large blocks, libcall can
do nontemporary accesses and beat inline considerably. */
- {{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
- {libcall, {{16, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
- {{libcall, {{8, loop}, {24, unrolled_loop},
+ {{{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
+ {libcall, {{16, loop}, {512, rep_prefix_8_byte}, {-1, libcall}}}},
+ {{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
+ {libcall, {{16, loop}, {512, rep_prefix_8_byte}, {-1, libcall}}}}},
+ {{{libcall, {{8, loop}, {24, unrolled_loop},
{2048, rep_prefix_4_byte}, {-1, libcall}}},
{libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
+ {{libcall, {{8, loop}, {24, unrolled_loop},
+ {2048, rep_prefix_4_byte}, {-1, libcall}}},
+ {libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}},
4, /* scalar_stmt_cost. */
2, /* scalar load_cost. */
2, /* scalar_store_cost. */
/* BDVER1 has optimized REP instruction for medium sized blocks, but for
very small blocks it is better to use loop. For large blocks, libcall
can do nontemporary accesses and beat inline considerably. */
- {{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
+ {{{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
{libcall, {{16, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
- {{libcall, {{8, loop}, {24, unrolled_loop},
+ {{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
+ {libcall, {{16, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}},
+ {{{libcall, {{8, loop}, {24, unrolled_loop},
{2048, rep_prefix_4_byte}, {-1, libcall}}},
{libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
+ {{libcall, {{8, loop}, {24, unrolled_loop},
+ {2048, rep_prefix_4_byte}, {-1, libcall}}},
+ {libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}},
6, /* scalar_stmt_cost. */
4, /* scalar load_cost. */
4, /* scalar_store_cost. */
/* BDVER2 has optimized REP instruction for medium sized blocks, but for
very small blocks it is better to use loop. For large blocks, libcall
can do nontemporary accesses and beat inline considerably. */
- {{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
+ {{{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
{libcall, {{16, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
- {{libcall, {{8, loop}, {24, unrolled_loop},
+ {{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
+ {libcall, {{16, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}},
+ {{{libcall, {{8, loop}, {24, unrolled_loop},
{2048, rep_prefix_4_byte}, {-1, libcall}}},
{libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
+ {{libcall, {{8, loop}, {24, unrolled_loop},
+ {2048, rep_prefix_4_byte}, {-1, libcall}}},
+ {libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}},
6, /* scalar_stmt_cost. */
4, /* scalar load_cost. */
4, /* scalar_store_cost. */
/* BTVER1 has optimized REP instruction for medium sized blocks, but for
very small blocks it is better to use loop. For large blocks, libcall can
do nontemporary accesses and beat inline considerably. */
- {{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
+ {{{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
{libcall, {{16, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
- {{libcall, {{8, loop}, {24, unrolled_loop},
+ {{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
+ {libcall, {{16, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}},
+ {{{libcall, {{8, loop}, {24, unrolled_loop},
{2048, rep_prefix_4_byte}, {-1, libcall}}},
{libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
+ {{libcall, {{8, loop}, {24, unrolled_loop},
+ {2048, rep_prefix_4_byte}, {-1, libcall}}},
+ {libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}},
4, /* scalar_stmt_cost. */
2, /* scalar load_cost. */
2, /* scalar_store_cost. */
COSTS_N_INSNS (2), /* cost of FABS instruction. */
COSTS_N_INSNS (2), /* cost of FCHS instruction. */
COSTS_N_INSNS (43), /* cost of FSQRT instruction. */
- {{libcall, {{12, loop_1_byte}, {-1, rep_prefix_4_byte}}},
+
+ {{{libcall, {{12, loop_1_byte}, {-1, rep_prefix_4_byte}}},
DUMMY_STRINGOP_ALGS},
- {{libcall, {{6, loop_1_byte}, {48, loop}, {20480, rep_prefix_4_byte},
+ {{libcall, {{12, loop_1_byte}, {-1, rep_prefix_4_byte}}},
+ DUMMY_STRINGOP_ALGS}},
+
+ {{{libcall, {{6, loop_1_byte}, {48, loop}, {20480, rep_prefix_4_byte},
{-1, libcall}}},
DUMMY_STRINGOP_ALGS},
+ {{libcall, {{6, loop_1_byte}, {48, loop}, {20480, rep_prefix_4_byte},
+ {-1, libcall}}},
+ DUMMY_STRINGOP_ALGS}},
1, /* scalar_stmt_cost. */
1, /* scalar load_cost. */
1, /* scalar_store_cost. */
COSTS_N_INSNS (3), /* cost of FABS instruction. */
COSTS_N_INSNS (3), /* cost of FCHS instruction. */
COSTS_N_INSNS (44), /* cost of FSQRT instruction. */
- {{libcall, {{12, loop_1_byte}, {-1, rep_prefix_4_byte}}},
+
+ {{{libcall, {{12, loop_1_byte}, {-1, rep_prefix_4_byte}}},
{libcall, {{32, loop}, {20000, rep_prefix_8_byte},
{100000, unrolled_loop}, {-1, libcall}}}},
- {{libcall, {{6, loop_1_byte}, {48, loop}, {20480, rep_prefix_4_byte},
+ {{libcall, {{12, loop_1_byte}, {-1, rep_prefix_4_byte}}},
+ {libcall, {{32, loop}, {20000, rep_prefix_8_byte},
+ {100000, unrolled_loop}, {-1, libcall}}}}},
+
+ {{{libcall, {{6, loop_1_byte}, {48, loop}, {20480, rep_prefix_4_byte},
{-1, libcall}}},
{libcall, {{24, loop}, {64, unrolled_loop},
{8192, rep_prefix_8_byte}, {-1, libcall}}}},
+ {{libcall, {{6, loop_1_byte}, {48, loop}, {20480, rep_prefix_4_byte},
+ {-1, libcall}}},
+ {libcall, {{24, loop}, {64, unrolled_loop},
+ {8192, rep_prefix_8_byte}, {-1, libcall}}}}},
1, /* scalar_stmt_cost. */
1, /* scalar load_cost. */
1, /* scalar_store_cost. */
COSTS_N_INSNS (1), /* cost of movzx */
8, /* "large" insn */
17, /* MOVE_RATIO */
- 2, /* cost for loading QImode using movzbl */
+ 4, /* cost for loading QImode using movzbl */
{4, 4, 4}, /* cost of loading integer registers
in QImode, HImode and SImode.
Relative to reg-reg move (2). */
COSTS_N_INSNS (8), /* cost of FABS instruction. */
COSTS_N_INSNS (8), /* cost of FCHS instruction. */
COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
- {{libcall, {{11, loop}, {-1, rep_prefix_4_byte}}},
- {libcall, {{32, loop}, {64, rep_prefix_4_byte},
- {8192, rep_prefix_8_byte}, {-1, libcall}}}},
- {{libcall, {{8, loop}, {15, unrolled_loop},
- {2048, rep_prefix_4_byte}, {-1, libcall}}},
- {libcall, {{24, loop}, {32, unrolled_loop},
- {8192, rep_prefix_8_byte}, {-1, libcall}}}},
+
+ /* stringop_algs for memcpy.
+ SSE loops works best on Atom, but fall back into non-SSE unrolled loop variant
+ if that fails. */
+ {{{libcall, {{4096, sse_loop}, {4096, unrolled_loop}, {-1, libcall}}}, /* Known alignment. */
+ {libcall, {{4096, sse_loop}, {4096, unrolled_loop}, {-1, libcall}}}},
+ {{libcall, {{2048, sse_loop}, {2048, unrolled_loop}, {-1, libcall}}}, /* Unknown alignment. */
+ {libcall, {{2048, sse_loop}, {2048, unrolled_loop},
+ {-1, libcall}}}}},
+
+ /* stringop_algs for memset. */
+ {{{libcall, {{4096, sse_loop}, {4096, unrolled_loop}, {-1, libcall}}}, /* Known alignment. */
+ {libcall, {{4096, sse_loop}, {4096, unrolled_loop}, {-1, libcall}}}},
+ {{libcall, {{1024, sse_loop}, {1024, unrolled_loop}, /* Unknown alignment. */
+ {-1, libcall}}},
+ {libcall, {{2048, sse_loop}, {2048, unrolled_loop},
+ {-1, libcall}}}}},
+ 1, /* scalar_stmt_cost. */
+ 1, /* scalar load_cost. */
+ 1, /* scalar_store_cost. */
+ 1, /* vec_stmt_cost. */
+ 1, /* vec_to_scalar_cost. */
+ 1, /* scalar_to_vec_cost. */
+ 1, /* vec_align_load_cost. */
+ 2, /* vec_unalign_load_cost. */
+ 1, /* vec_store_cost. */
+ 3, /* cond_taken_branch_cost. */
+ 1, /* cond_not_taken_branch_cost. */
+};
+
+/* Core should produce code tuned for core variants. */
+static const
+struct processor_costs core_cost = {
+ COSTS_N_INSNS (1), /* cost of an add instruction */
+ /* On all chips taken into consideration lea is 2 cycles and more. With
+ this cost however our current implementation of synth_mult results in
+ use of unnecessary temporary registers causing regression on several
+ SPECfp benchmarks. */
+ COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */
+ COSTS_N_INSNS (1), /* variable shift costs */
+ COSTS_N_INSNS (1), /* constant shift costs */
+ {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
+ COSTS_N_INSNS (4), /* HI */
+ COSTS_N_INSNS (3), /* SI */
+ COSTS_N_INSNS (4), /* DI */
+ COSTS_N_INSNS (2)}, /* other */
+ 0, /* cost of multiply per each bit set */
+ {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
+ COSTS_N_INSNS (26), /* HI */
+ COSTS_N_INSNS (42), /* SI */
+ COSTS_N_INSNS (74), /* DI */
+ COSTS_N_INSNS (74)}, /* other */
+ COSTS_N_INSNS (1), /* cost of movsx */
+ COSTS_N_INSNS (1), /* cost of movzx */
+ 8, /* "large" insn */
+ 17, /* MOVE_RATIO */
+ 4, /* cost for loading QImode using movzbl */
+ {4, 4, 4}, /* cost of loading integer registers
+ in QImode, HImode and SImode.
+ Relative to reg-reg move (2). */
+ {4, 4, 4}, /* cost of storing integer registers */
+ 4, /* cost of reg,reg fld/fst */
+ {12, 12, 12}, /* cost of loading fp registers
+ in SFmode, DFmode and XFmode */
+ {6, 6, 8}, /* cost of storing fp registers
+ in SFmode, DFmode and XFmode */
+ 2, /* cost of moving MMX register */
+ {8, 8}, /* cost of loading MMX registers
+ in SImode and DImode */
+ {8, 8}, /* cost of storing MMX registers
+ in SImode and DImode */
+ 2, /* cost of moving SSE register */
+ {8, 8, 8}, /* cost of loading SSE registers
+ in SImode, DImode and TImode */
+ {8, 8, 8}, /* cost of storing SSE registers
+ in SImode, DImode and TImode */
+ 5, /* MMX or SSE register to integer */
+ 32, /* size of l1 cache. */
+ 512, /* size of l2 cache. */
+ 64, /* size of prefetch block */
+ 6, /* number of parallel prefetches */
+ /* Benchmarks shows large regressions on K8 sixtrack benchmark when this
+ value is increased to perhaps more appropriate value of 5. */
+ 3, /* Branch cost */
+ COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
+ COSTS_N_INSNS (8), /* cost of FMUL instruction. */
+ COSTS_N_INSNS (20), /* cost of FDIV instruction. */
+ COSTS_N_INSNS (8), /* cost of FABS instruction. */
+ COSTS_N_INSNS (8), /* cost of FCHS instruction. */
+ COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
+
+ /* stringop_algs for memcpy. */
+ {{{libcall, {{16, loop}, {24, unrolled_loop}, {1024, rep_prefix_4_byte}, {-1, libcall}}}, /* Known alignment. */
+ {libcall, {{16, loop}, {24, unrolled_loop}, {1024, rep_prefix_8_byte}, {-1, libcall}}}},
+ {{libcall, {{16, loop}, {24, unrolled_loop}, {1024, rep_prefix_4_byte}, {-1, libcall}}}, /* Unknown alignment. */
+ {libcall, {{16, loop}, {24, unrolled_loop}, {1024, rep_prefix_8_byte}, {-1, libcall}}}}},
+
+ /* stringop_algs for memset. */
+ {{{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}}, /* Known alignment. */
+ {libcall, {{256, rep_prefix_8_byte}, {-1, libcall}}}},
+ {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}}, /* Unknown alignment. */
+ {libcall, {{256, rep_prefix_8_byte}, {-1, libcall}}}}},
1, /* scalar_stmt_cost. */
1, /* scalar load_cost. */
1, /* scalar_store_cost. */
1, /* cond_not_taken_branch_cost. */
};
-/* Generic64 should produce code tuned for Nocona and K8. */
+/* Generic64 should produce code tuned for Nocona, Core, K8, Amdfam10 and buldozer. */
static const
struct processor_costs generic64_cost = {
COSTS_N_INSNS (1), /* cost of an add instruction */
COSTS_N_INSNS (8), /* cost of FABS instruction. */
COSTS_N_INSNS (8), /* cost of FCHS instruction. */
COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
- {DUMMY_STRINGOP_ALGS,
- {libcall, {{32, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
- {DUMMY_STRINGOP_ALGS,
- {libcall, {{32, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
+
+ {{DUMMY_STRINGOP_ALGS,
+ {libcall, {{16, rep_prefix_4_byte}, {128, rep_prefix_8_byte}, {4096, rep_prefix_1_byte}, {-1, libcall}}}},
+ {DUMMY_STRINGOP_ALGS,
+ {libcall, {{128, rep_prefix_4_byte}, {4096, rep_prefix_1_byte}, {-1, libcall}}}}},
+
+ {{DUMMY_STRINGOP_ALGS,
+ {libcall, {{16, rep_prefix_4_byte}, {512, unrolled_loop}, {4096, rep_prefix_1_byte}, {-1, libcall}}}},
+ {DUMMY_STRINGOP_ALGS,
+ {libcall, {{16, rep_prefix_4_byte}, {512, unrolled_loop}, {4096, rep_prefix_1_byte}, {-1, libcall}}}}},
1, /* scalar_stmt_cost. */
1, /* scalar load_cost. */
1, /* scalar_store_cost. */
1, /* cond_not_taken_branch_cost. */
};
-/* Generic32 should produce code tuned for PPro, Pentium4, Nocona,
- Athlon and K8. */
+/* Generic32 should produce code tuned for PPro, Pentium4, Nocona, Core
+ Athlon, K8, amdfam10, buldozer. */
static const
struct processor_costs generic32_cost = {
COSTS_N_INSNS (1), /* cost of an add instruction */
COSTS_N_INSNS (8), /* cost of FABS instruction. */
COSTS_N_INSNS (8), /* cost of FCHS instruction. */
COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
- {{libcall, {{32, loop}, {8192, rep_prefix_4_byte}, {-1, libcall}}},
+ /* stringop_algs for memcpy. */
+ {{{libcall, {{32, loop}, {8192, rep_prefix_4_byte}, {-1, libcall}}},
DUMMY_STRINGOP_ALGS},
- {{libcall, {{32, loop}, {8192, rep_prefix_4_byte}, {-1, libcall}}},
+ {{libcall, {{32, loop}, {8192, rep_prefix_4_byte}, {-1, libcall}}},
+ DUMMY_STRINGOP_ALGS}},
+ /* stringop_algs for memset. */
+ {{{libcall, {{32, loop}, {8192, rep_prefix_4_byte}, {-1, libcall}}},
DUMMY_STRINGOP_ALGS},
+ {{libcall, {{32, loop}, {8192, rep_prefix_4_byte}, {-1, libcall}}},
+ DUMMY_STRINGOP_ALGS}},
1, /* scalar_stmt_cost. */
1, /* scalar load_cost. */
1, /* scalar_store_cost. */
static bool ix86_expand_vector_init_one_nonzero (bool, enum machine_mode,
rtx, rtx, int);
static void ix86_add_new_builtins (HOST_WIDE_INT);
-static rtx ix86_expand_vec_perm_builtin (tree);
static tree ix86_canonical_va_list_type (tree);
static void predict_jump (int);
static unsigned int split_stack_prologue_scratch_regno (void);
static unsigned int ix86_minimum_incoming_stack_boundary (bool);
static enum calling_abi ix86_function_abi (const_tree);
+static rtx promote_duplicated_reg (enum machine_mode, rtx);
+static rtx promote_duplicated_reg_to_size (rtx, int, int, int);
\f
#ifndef SUBTARGET32_DEFAULT_CPU
{&k8_cost, 16, 7, 16, 7, 16},
{&nocona_cost, 0, 0, 0, 0, 0},
/* Core 2 32-bit. */
- {&generic32_cost, 16, 10, 16, 10, 16},
+ {&core_cost, 16, 10, 16, 10, 16},
/* Core 2 64-bit. */
- {&generic64_cost, 16, 10, 16, 10, 16},
+ {&core_cost, 16, 10, 16, 10, 16},
/* Core i7 32-bit. */
- {&generic32_cost, 16, 10, 16, 10, 16},
+ {&core_cost, 16, 10, 16, 10, 16},
/* Core i7 64-bit. */
- {&generic64_cost, 16, 10, 16, 10, 16},
+ {&core_cost, 16, 10, 16, 10, 16},
{&generic32_cost, 16, 7, 16, 7, 16},
{&generic64_cost, 16, 10, 16, 10, 16},
{&amdfam10_cost, 32, 24, 32, 7, 32},
{&bdver1_cost, 32, 24, 32, 7, 32},
{&bdver2_cost, 32, 24, 32, 7, 32},
{&btver1_cost, 32, 24, 32, 7, 32},
- {&atom_cost, 16, 7, 16, 7, 16}
+ {&atom_cost, 16, 15, 16, 7, 16}
};
static const char *const cpu_names[TARGET_CPU_DEFAULT_max] =
PTA_64BIT /* flags are only used for -march switch. */ },
};
+ /* -mrecip options. */
+ static struct
+ {
+ const char *string; /* option name */
+ unsigned int mask; /* mask bits to set */
+ }
+ const recip_options[] =
+ {
+ { "all", RECIP_MASK_ALL },
+ { "none", RECIP_MASK_NONE },
+ { "div", RECIP_MASK_DIV },
+ { "sqrt", RECIP_MASK_SQRT },
+ { "vec-div", RECIP_MASK_VEC_DIV },
+ { "vec-sqrt", RECIP_MASK_VEC_SQRT },
+ };
+
int const pta_size = ARRAY_SIZE (processor_alias_table);
/* Set up prefix/suffix so the error messages refer to either the command
target_flags &= ~MASK_VZEROUPPER;
}
+ if (ix86_recip_name)
+ {
+ char *p = ASTRDUP (ix86_recip_name);
+ char *q;
+ unsigned int mask, i;
+ bool invert;
+
+ while ((q = strtok (p, ",")) != NULL)
+ {
+ p = NULL;
+ if (*q == '!')
+ {
+ invert = true;
+ q++;
+ }
+ else
+ invert = false;
+
+ if (!strcmp (q, "default"))
+ mask = RECIP_MASK_ALL;
+ else
+ {
+ for (i = 0; i < ARRAY_SIZE (recip_options); i++)
+ if (!strcmp (q, recip_options[i].string))
+ {
+ mask = recip_options[i].mask;
+ break;
+ }
+
+ if (i == ARRAY_SIZE (recip_options))
+ {
+ error ("unknown option for -mrecip=%s", q);
+ invert = false;
+ mask = RECIP_MASK_NONE;
+ }
+ }
+
+ recip_mask_explicit |= mask;
+ if (invert)
+ recip_mask &= ~mask;
+ else
+ recip_mask |= mask;
+ }
+ }
+
+ if (TARGET_RECIP)
+ recip_mask |= RECIP_MASK_ALL & ~recip_mask_explicit;
+ else if (target_flags_explicit & MASK_RECIP)
+ recip_mask &= ~(RECIP_MASK_ALL & ~recip_mask_explicit);
+
/* Save the initial options in case the user does function specific
options. */
if (main_args_p)
ptr->arch_specified = ix86_arch_specified;
ptr->x_ix86_isa_flags_explicit = ix86_isa_flags_explicit;
ptr->ix86_target_flags_explicit = target_flags_explicit;
+ ptr->x_recip_mask_explicit = recip_mask_explicit;
/* The fields are char but the variables are not; make sure the
values fit in the fields. */
ix86_arch_specified = ptr->arch_specified;
ix86_isa_flags_explicit = ptr->x_ix86_isa_flags_explicit;
target_flags_explicit = ptr->ix86_target_flags_explicit;
+ recip_mask_explicit = ptr->x_recip_mask_explicit;
/* Recreate the arch feature tests if the arch changed */
if (old_arch != ix86_arch)
return NULL_TREE;
}
+/* The transactional memory builtins are implicitly regparm or fastcall
+ depending on the ABI. Override the generic do-nothing attribute that
+ these builtins were declared with, and replace it with one of the two
+ attributes that we expect elsewhere. */
+
+static tree
+ix86_handle_tm_regparm_attribute (tree *node, tree name ATTRIBUTE_UNUSED,
+ tree args ATTRIBUTE_UNUSED,
+ int flags ATTRIBUTE_UNUSED,
+ bool *no_add_attrs)
+{
+ tree alt;
+
+ /* In no case do we want to add the placeholder attribute. */
+ *no_add_attrs = true;
+
+ /* The 64-bit ABI is unchanged for transactional memory. */
+ if (TARGET_64BIT)
+ return NULL_TREE;
+
+ /* ??? Is there a better way to validate 32-bit windows? We have
+ cfun->machine->call_abi, but that seems to be set only for 64-bit. */
+ if (CHECK_STACK_LIMIT > 0)
+ alt = tree_cons (get_identifier ("fastcall"), NULL, NULL);
+ else
+ {
+ alt = tree_cons (NULL, build_int_cst (NULL, 2), NULL);
+ alt = tree_cons (get_identifier ("regparm"), alt, NULL);
+ }
+ decl_attributes (node, alt, flags);
+
+ return NULL_TREE;
+}
+
/* This function determines from TYPE the calling-convention. */
unsigned int
else
{
tree copy
- = build_call_expr (implicit_built_in_decls[BUILT_IN_MEMCPY],
+ = build_call_expr (builtin_decl_implicit (BUILT_IN_MEMCPY),
3, dest_addr, src_addr,
size_int (cur_size));
gimplify_and_add (copy, pre_p);
if (SUBTARGET_FRAME_POINTER_REQUIRED)
return true;
+ /* For older 32-bit runtimes setjmp requires valid frame-pointer. */
+ if (TARGET_32BIT_MS_ABI && cfun->calls_setjmp)
+ return true;
+
/* In ix86_option_override_internal, TARGET_OMIT_LEAF_FRAME_POINTER
turns off the frame pointer by default. Turn it back on now if
we've not got a leaf function. */
static void
ix86_add_cfa_restore_note (rtx insn, rtx reg, HOST_WIDE_INT cfa_offset)
{
- if (cfa_offset <= cfun->machine->fs.red_zone_offset)
+ if (!crtl->shrink_wrapped
+ && cfa_offset <= cfun->machine->fs.red_zone_offset)
return;
if (insn)
/* After stack_realign_needed is finalized, we can't no longer
change it. */
gcc_assert (crtl->stack_realign_needed == stack_realign);
+ return;
}
- else
- {
- crtl->stack_realign_needed = stack_realign;
- crtl->stack_realign_finalized = true;
+
+ /* If the only reason for frame_pointer_needed is that we conservatively
+ assumed stack realignment might be needed, but in the end nothing that
+ needed the stack alignment had been spilled, clear frame_pointer_needed
+ and say we don't need stack realignment. */
+ if (stack_realign
+ && !crtl->need_drap
+ && frame_pointer_needed
+ && current_function_is_leaf
+ && flag_omit_frame_pointer
+ && current_function_sp_is_unchanging
+ && !ix86_current_function_calls_tls_descriptor
+ && !crtl->accesses_prior_frames
+ && !cfun->calls_alloca
+ && !crtl->calls_eh_return
+ && !(flag_stack_check && STACK_CHECK_MOVING_SP)
+ && !ix86_frame_pointer_required ()
+ && get_frame_size () == 0
+ && ix86_nsaved_sseregs () == 0
+ && ix86_varargs_gpr_size + ix86_varargs_fpr_size == 0)
+ {
+ HARD_REG_SET set_up_by_prologue, prologue_used;
+ basic_block bb;
+
+ CLEAR_HARD_REG_SET (prologue_used);
+ CLEAR_HARD_REG_SET (set_up_by_prologue);
+ add_to_hard_reg_set (&set_up_by_prologue, Pmode, STACK_POINTER_REGNUM);
+ add_to_hard_reg_set (&set_up_by_prologue, Pmode, ARG_POINTER_REGNUM);
+ add_to_hard_reg_set (&set_up_by_prologue, Pmode,
+ HARD_FRAME_POINTER_REGNUM);
+ FOR_EACH_BB (bb)
+ {
+ rtx insn;
+ FOR_BB_INSNS (bb, insn)
+ if (NONDEBUG_INSN_P (insn)
+ && requires_stack_frame_p (insn, prologue_used,
+ set_up_by_prologue))
+ {
+ crtl->stack_realign_needed = stack_realign;
+ crtl->stack_realign_finalized = true;
+ return;
+ }
+ }
+
+ frame_pointer_needed = false;
+ stack_realign = false;
+ crtl->max_used_stack_slot_alignment = incoming_stack_boundary;
+ crtl->stack_alignment_needed = incoming_stack_boundary;
+ crtl->stack_alignment_estimated = incoming_stack_boundary;
+ if (crtl->preferred_stack_boundary > incoming_stack_boundary)
+ crtl->preferred_stack_boundary = incoming_stack_boundary;
+ df_finish_pass (true);
+ df_scan_alloc (NULL);
+ df_scan_blocks ();
+ df_compute_regs_ever_live (true);
+ df_analyze ();
}
+
+ crtl->stack_realign_needed = stack_realign;
+ crtl->stack_realign_finalized = true;
}
/* Expand the prologue into a bunch of separate insns. */
}
}
+/* Emit vzeroupper if needed. */
+
+void
+ix86_maybe_emit_epilogue_vzeroupper (void)
+{
+ if (TARGET_VZEROUPPER
+ && !TREE_THIS_VOLATILE (cfun->decl)
+ && !cfun->machine->caller_return_avx256_p)
+ emit_insn (gen_avx_vzeroupper (GEN_INT (call_no_avx256)));
+}
+
/* Restore function stack, frame, and registers. */
void
GEN_INT (m->fs.sp_offset - UNITS_PER_WORD),
style, true);
}
+ else
+ ix86_add_queued_cfa_restore_notes (get_last_insn ());
/* Sibcall epilogues don't want a return instruction. */
if (style == 0)
}
/* Emit vzeroupper if needed. */
- if (TARGET_VZEROUPPER
- && !TREE_THIS_VOLATILE (cfun->decl)
- && !cfun->machine->caller_return_avx256_p)
- emit_insn (gen_avx_vzeroupper (GEN_INT (call_no_avx256)));
+ ix86_maybe_emit_epilogue_vzeroupper ();
if (crtl->args.pops_args && crtl->args.size)
{
pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
popc, -1, true);
- emit_jump_insn (gen_return_indirect_internal (ecx));
+ emit_jump_insn (gen_simple_return_indirect_internal (ecx));
}
else
- emit_jump_insn (gen_return_pop_internal (popc));
+ emit_jump_insn (gen_simple_return_pop_internal (popc));
}
else
- emit_jump_insn (gen_return_internal ());
+ emit_jump_insn (gen_simple_return_internal ());
/* Restore the state back to the state from the prologue,
so that it's correct for the next epilogue. */
it looks like we might want one, insert a NOP. */
{
rtx insn = get_last_insn ();
+ rtx deleted_debug_label = NULL_RTX;
while (insn
&& NOTE_P (insn)
&& NOTE_KIND (insn) != NOTE_INSN_DELETED_LABEL)
- insn = PREV_INSN (insn);
+ {
+ /* Don't insert a nop for NOTE_INSN_DELETED_DEBUG_LABEL
+ notes only, instead set their CODE_LABEL_NUMBER to -1,
+ otherwise there would be code generation differences
+ in between -g and -g0. */
+ if (NOTE_P (insn) && NOTE_KIND (insn) == NOTE_INSN_DELETED_DEBUG_LABEL)
+ deleted_debug_label = insn;
+ insn = PREV_INSN (insn);
+ }
if (insn
&& (LABEL_P (insn)
|| (NOTE_P (insn)
&& NOTE_KIND (insn) == NOTE_INSN_DELETED_LABEL)))
fputs ("\tnop\n", file);
+ else if (deleted_debug_label)
+ for (insn = deleted_debug_label; insn; insn = NEXT_INSN (insn))
+ if (NOTE_KIND (insn) == NOTE_INSN_DELETED_DEBUG_LABEL)
+ CODE_LABEL_NUMBER (insn) = -1;
}
#endif
code = GET_MODE_SIZE (GET_MODE (x));
/* Irritatingly, AMD extended registers use different naming convention
- from the normal registers. */
+ from the normal registers: "r%d[bwd]" */
if (REX_INT_REG_P (x))
{
gcc_assert (TARGET_64BIT);
+ putc ('r', file);
+ fprint_ul (file, REGNO (x) - FIRST_REX_INT_REG + 8);
switch (code)
{
case 0:
error ("extended registers have no high halves");
break;
case 1:
- fprintf (file, "r%ib", REGNO (x) - FIRST_REX_INT_REG + 8);
+ putc ('b', file);
break;
case 2:
- fprintf (file, "r%iw", REGNO (x) - FIRST_REX_INT_REG + 8);
+ putc ('w', file);
break;
case 4:
- fprintf (file, "r%id", REGNO (x) - FIRST_REX_INT_REG + 8);
+ putc ('d', file);
break;
case 8:
- fprintf (file, "r%i", REGNO (x) - FIRST_REX_INT_REG + 8);
+ /* no suffix */
break;
default:
error ("unsupported operand size for extended register");
Y -- print condition for XOP pcom* instruction.
+ -- print a branch hint as 'cs' or 'ds' prefix
; -- print a semicolon (after prefixes due to bug in older gas).
+ ~ -- print "i" if TARGET_AVX2, "f" otherwise.
@ -- print a segment register of thread base pointer load
*/
fputs ("gs", file);
return;
+ case '~':
+ putc (TARGET_AVX2 ? 'i' : 'f', file);
+ return;
+
default:
output_operand_lossage ("invalid operand code '%c'", code);
}
gcc_unreachable ();
}
- /* Check for explicit size override (codes 'b', 'w' and 'k') */
+ /* Check for explicit size override (codes 'b', 'w', 'k',
+ 'q' and 'x') */
if (code == 'b')
size = "BYTE";
else if (code == 'w')
size = "WORD";
else if (code == 'k')
size = "DWORD";
+ else if (code == 'q')
+ size = "QWORD";
+ else if (code == 'x')
+ size = "XMMWORD";
fputs (size, file);
fputs (" PTR ", file);
ix86_print_operand_punct_valid_p (unsigned char code)
{
return (code == '@' || code == '*' || code == '+'
- || code == '&' || code == ';');
+ || code == '&' || code == ';' || code == '~');
}
\f
/* Print a memory operand whose address is ADDR. */
struct ix86_address parts;
rtx base, index, disp;
int scale;
- int ok = ix86_decompose_address (addr, &parts);
+ int ok;
+ bool vsib = false;
+
+ if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_VSIBADDR)
+ {
+ ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts);
+ gcc_assert (parts.index == NULL_RTX);
+ parts.index = XVECEXP (addr, 0, 1);
+ parts.scale = INTVAL (XVECEXP (addr, 0, 2));
+ addr = XVECEXP (addr, 0, 0);
+ vsib = true;
+ }
+ else
+ ok = ix86_decompose_address (addr, &parts);
gcc_assert (ok);
if (index)
{
putc (',', file);
- print_reg (index, code, file);
- if (scale != 1)
+ print_reg (index, vsib ? 0 : code, file);
+ if (scale != 1 || vsib)
fprintf (file, ",%d", scale);
}
putc (')', file);
if (index)
{
putc ('+', file);
- print_reg (index, code, file);
- if (scale != 1)
+ print_reg (index, vsib ? 0 : code, file);
+ if (scale != 1 || vsib)
fprintf (file, "*%d", scale);
}
putc (']', file);
if (MEM_P (src1) && !rtx_equal_p (dst, src1))
src1 = force_reg (mode, src1);
+ /* Improve address combine. */
+ if (code == PLUS
+ && GET_MODE_CLASS (mode) == MODE_INT
+ && MEM_P (src2))
+ src2 = force_reg (mode, src2);
+
operands[1] = src1;
operands[2] = src2;
return dst;
FOR_EACH_EDGE (e, ei, bb->preds)
{
- int bb_dist = distance_non_agu_define_in_bb (regno1, regno2,
- insn, distance,
- BB_END (e->src),
- &found_in_bb);
+ int bb_dist
+ = distance_non_agu_define_in_bb (regno1, regno2,
+ insn, distance,
+ BB_END (e->src),
+ &found_in_bb);
if (found_in_bb)
{
if (shortest_dist < 0)
shortest_dist = bb_dist;
else if (bb_dist > 0)
shortest_dist = MIN (bb_dist, shortest_dist);
- }
- found = found || found_in_bb;
+ found = true;
+ }
}
distance = shortest_dist;
extract_insn_cached (insn);
if (!found)
- distance = -1;
- else
- distance = distance >> 1;
+ return -1;
- return distance;
+ return distance >> 1;
}
/* Return the distance in half-cycles between INSN and the next
found and false otherwise. */
static int
-distance_agu_use_in_bb(unsigned int regno,
- rtx insn, int distance, rtx start,
- bool *found, bool *redefined)
+distance_agu_use_in_bb (unsigned int regno,
+ rtx insn, int distance, rtx start,
+ bool *found, bool *redefined)
{
basic_block bb = start ? BLOCK_FOR_INSN (start) : NULL;
rtx next = start;
FOR_EACH_EDGE (e, ei, bb->succs)
{
- int bb_dist = distance_agu_use_in_bb (regno0, insn,
- distance, BB_HEAD (e->dest),
- &found_in_bb, &redefined_in_bb);
+ int bb_dist
+ = distance_agu_use_in_bb (regno0, insn,
+ distance, BB_HEAD (e->dest),
+ &found_in_bb, &redefined_in_bb);
if (found_in_bb)
{
if (shortest_dist < 0)
shortest_dist = bb_dist;
else if (bb_dist > 0)
shortest_dist = MIN (bb_dist, shortest_dist);
- }
- found = found || found_in_bb;
+ found = true;
+ }
}
distance = shortest_dist;
}
if (!found || redefined)
- distance = -1;
- else
- distance = distance >> 1;
+ return -1;
- return distance;
+ return distance >> 1;
}
/* Define this macro to tune LEA priority vs ADD, it take effect when
false otherwise. */
static bool
-ix86_ok_to_clobber_flags(rtx insn)
+ix86_ok_to_clobber_flags (rtx insn)
{
basic_block bb = BLOCK_FOR_INSN (insn);
df_ref *use;
return !ix86_lea_outperforms (insn, regno0, regno1, regno2, 1);
}
+/* Return true if we should emit lea instruction instead of mov
+ instruction. */
+
+bool
+ix86_use_lea_for_mov (rtx insn, rtx operands[])
+{
+ unsigned int regno0;
+ unsigned int regno1;
+
+ /* Check if we need to optimize. */
+ if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
+ return false;
+
+ /* Use lea for reg to reg moves only. */
+ if (!REG_P (operands[0]) || !REG_P (operands[1]))
+ return false;
+
+ regno0 = true_regnum (operands[0]);
+ regno1 = true_regnum (operands[1]);
+
+ return ix86_lea_outperforms (insn, regno0, regno1, -1, 0);
+}
+
/* Return true if we need to split lea into a sequence of
instructions to avoid AGU stalls. */
return !ix86_lea_outperforms (insn, regno0, regno1, regno2, split_cost);
}
+/* Emit x86 binary operand CODE in mode MODE, where the first operand
+ matches destination. RTX includes clobber of FLAGS_REG. */
+
+static void
+ix86_emit_binop (enum rtx_code code, enum machine_mode mode,
+ rtx dst, rtx src)
+{
+ rtx op, clob;
+
+ op = gen_rtx_SET (VOIDmode, dst, gen_rtx_fmt_ee (code, mode, dst, src));
+ clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
+
+ emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
+}
+
/* Split lea instructions into a sequence of instructions
which are executed on ALU to avoid AGU stalls.
It is assumed that it is allowed to clobber flags register
unsigned int regno1 = INVALID_REGNUM;
unsigned int regno2 = INVALID_REGNUM;
struct ix86_address parts;
- rtx tmp, clob;
- rtvec par;
+ rtx tmp;
int ok, adds;
ok = ix86_decompose_address (operands[1], &parts);
gcc_assert (regno2 != regno0);
for (adds = parts.scale; adds > 0; adds--)
- {
- tmp = gen_rtx_PLUS (mode, operands[0], parts.index);
- tmp = gen_rtx_SET (VOIDmode, operands[0], tmp);
- clob = gen_rtx_CLOBBER (VOIDmode,
- gen_rtx_REG (CCmode, FLAGS_REG));
- par = gen_rtvec (2, tmp, clob);
- emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
- }
+ ix86_emit_binop (PLUS, mode, operands[0], parts.index);
}
else
{
emit_insn (gen_rtx_SET (VOIDmode, operands[0], parts.index));
/* Use shift for scaling. */
- tmp = gen_rtx_ASHIFT (mode, operands[0],
- GEN_INT (exact_log2 (parts.scale)));
- tmp = gen_rtx_SET (VOIDmode, operands[0], tmp);
- clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
- par = gen_rtvec (2, tmp, clob);
- emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
+ ix86_emit_binop (ASHIFT, mode, operands[0],
+ GEN_INT (exact_log2 (parts.scale)));
if (parts.base)
- {
- tmp = gen_rtx_PLUS (mode, operands[0], parts.base);
- tmp = gen_rtx_SET (VOIDmode, operands[0], tmp);
- clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
- par = gen_rtvec (2, tmp, clob);
- emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
- }
+ ix86_emit_binop (PLUS, mode, operands[0], parts.base);
if (parts.disp && parts.disp != const0_rtx)
- {
- tmp = gen_rtx_PLUS (mode, operands[0], parts.disp);
- tmp = gen_rtx_SET (VOIDmode, operands[0], tmp);
- clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
- par = gen_rtvec (2, tmp, clob);
- emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
- }
+ ix86_emit_binop (PLUS, mode, operands[0], parts.disp);
}
}
else if (!parts.base && !parts.index)
else
{
if (!parts.base)
- {
- if (regno0 != regno2)
- emit_insn (gen_rtx_SET (VOIDmode, operands[0], parts.index));
- }
+ {
+ if (regno0 != regno2)
+ emit_insn (gen_rtx_SET (VOIDmode, operands[0], parts.index));
+ }
else if (!parts.index)
- {
- if (regno0 != regno1)
- emit_insn (gen_rtx_SET (VOIDmode, operands[0], parts.base));
- }
- else
- {
- if (regno0 == regno1)
- tmp = gen_rtx_PLUS (mode, operands[0], parts.index);
- else if (regno0 == regno2)
- tmp = gen_rtx_PLUS (mode, operands[0], parts.base);
- else
- {
+ {
+ if (regno0 != regno1)
emit_insn (gen_rtx_SET (VOIDmode, operands[0], parts.base));
- tmp = gen_rtx_PLUS (mode, operands[0], parts.index);
- }
+ }
+ else
+ {
+ if (regno0 == regno1)
+ tmp = parts.index;
+ else if (regno0 == regno2)
+ tmp = parts.base;
+ else
+ {
+ emit_insn (gen_rtx_SET (VOIDmode, operands[0], parts.base));
+ tmp = parts.index;
+ }
- tmp = gen_rtx_SET (VOIDmode, operands[0], tmp);
- clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
- par = gen_rtvec (2, tmp, clob);
- emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
- }
+ ix86_emit_binop (PLUS, mode, operands[0], tmp);
+ }
if (parts.disp && parts.disp != const0_rtx)
- {
- tmp = gen_rtx_PLUS (mode, operands[0], parts.disp);
- tmp = gen_rtx_SET (VOIDmode, operands[0], tmp);
- clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
- par = gen_rtvec (2, tmp, clob);
- emit_insn (gen_rtx_PARALLEL (VOIDmode, par));
- }
+ ix86_emit_binop (PLUS, mode, operands[0], parts.disp);
}
}
x = gen_rtx_REG (V4SImode, REGNO (value));
if (vecmode == V4SFmode)
- emit_insn (gen_sse2_cvttps2dq (x, value));
+ emit_insn (gen_fix_truncv4sfv4si2 (x, value));
else
emit_insn (gen_sse2_cvttpd2dq (x, value));
value = x;
emit_move_insn (target, fp_hi);
}
+/* floatunsv{4,8}siv{4,8}sf2 expander. Expand code to convert
+ a vector of unsigned ints VAL to vector of floats TARGET. */
+
+void
+ix86_expand_vector_convert_uns_vsivsf (rtx target, rtx val)
+{
+ rtx tmp[8];
+ REAL_VALUE_TYPE TWO16r;
+ enum machine_mode intmode = GET_MODE (val);
+ enum machine_mode fltmode = GET_MODE (target);
+ rtx (*cvt) (rtx, rtx);
+
+ if (intmode == V4SImode)
+ cvt = gen_floatv4siv4sf2;
+ else
+ cvt = gen_floatv8siv8sf2;
+ tmp[0] = ix86_build_const_vector (intmode, 1, GEN_INT (0xffff));
+ tmp[0] = force_reg (intmode, tmp[0]);
+ tmp[1] = expand_simple_binop (intmode, AND, val, tmp[0], NULL_RTX, 1,
+ OPTAB_DIRECT);
+ tmp[2] = expand_simple_binop (intmode, LSHIFTRT, val, GEN_INT (16),
+ NULL_RTX, 1, OPTAB_DIRECT);
+ tmp[3] = gen_reg_rtx (fltmode);
+ emit_insn (cvt (tmp[3], tmp[1]));
+ tmp[4] = gen_reg_rtx (fltmode);
+ emit_insn (cvt (tmp[4], tmp[2]));
+ real_ldexp (&TWO16r, &dconst1, 16);
+ tmp[5] = const_double_from_real_value (TWO16r, SFmode);
+ tmp[5] = force_reg (fltmode, ix86_build_const_vector (fltmode, 1, tmp[5]));
+ tmp[6] = expand_simple_binop (fltmode, MULT, tmp[4], tmp[5], NULL_RTX, 1,
+ OPTAB_DIRECT);
+ tmp[7] = expand_simple_binop (fltmode, PLUS, tmp[3], tmp[6], target, 1,
+ OPTAB_DIRECT);
+ if (tmp[7] != target)
+ emit_move_insn (target, tmp[7]);
+}
+
+/* Adjust a V*SFmode/V*DFmode value VAL so that *sfix_trunc* resp. fix_trunc*
+ pattern can be used on it instead of *ufix_trunc* resp. fixuns_trunc*.
+ This is done by doing just signed conversion if < 0x1p31, and otherwise by
+ subtracting 0x1p31 first and xoring in 0x80000000 from *XORP afterwards. */
+
+rtx
+ix86_expand_adjust_ufix_to_sfix_si (rtx val, rtx *xorp)
+{
+ REAL_VALUE_TYPE TWO31r;
+ rtx two31r, tmp[4];
+ enum machine_mode mode = GET_MODE (val);
+ enum machine_mode scalarmode = GET_MODE_INNER (mode);
+ enum machine_mode intmode = GET_MODE_SIZE (mode) == 32 ? V8SImode : V4SImode;
+ rtx (*cmp) (rtx, rtx, rtx, rtx);
+ int i;
+
+ for (i = 0; i < 3; i++)
+ tmp[i] = gen_reg_rtx (mode);
+ real_ldexp (&TWO31r, &dconst1, 31);
+ two31r = const_double_from_real_value (TWO31r, scalarmode);
+ two31r = ix86_build_const_vector (mode, 1, two31r);
+ two31r = force_reg (mode, two31r);
+ switch (mode)
+ {
+ case V8SFmode: cmp = gen_avx_maskcmpv8sf3; break;
+ case V4SFmode: cmp = gen_sse_maskcmpv4sf3; break;
+ case V4DFmode: cmp = gen_avx_maskcmpv4df3; break;
+ case V2DFmode: cmp = gen_sse2_maskcmpv2df3; break;
+ default: gcc_unreachable ();
+ }
+ tmp[3] = gen_rtx_LE (mode, two31r, val);
+ emit_insn (cmp (tmp[0], two31r, val, tmp[3]));
+ tmp[1] = expand_simple_binop (mode, AND, tmp[0], two31r, tmp[1],
+ 0, OPTAB_DIRECT);
+ if (intmode == V4SImode || TARGET_AVX2)
+ *xorp = expand_simple_binop (intmode, ASHIFT,
+ gen_lowpart (intmode, tmp[0]),
+ GEN_INT (31), NULL_RTX, 0,
+ OPTAB_DIRECT);
+ else
+ {
+ rtx two31 = GEN_INT ((unsigned HOST_WIDE_INT) 1 << 31);
+ two31 = ix86_build_const_vector (intmode, 1, two31);
+ *xorp = expand_simple_binop (intmode, AND,
+ gen_lowpart (intmode, tmp[0]),
+ two31, NULL_RTX, 0,
+ OPTAB_DIRECT);
+ }
+ return expand_simple_binop (mode, MINUS, val, tmp[1], tmp[2],
+ 0, OPTAB_DIRECT);
+}
+
/* A subroutine of ix86_build_signbit_mask. If VECT is true,
then replicate the value for all elements of the vector
register. */
switch (mode)
{
+ case V32QImode:
+ case V16QImode:
+ case V16HImode:
+ case V8HImode:
case V8SImode:
case V4SImode:
case V4DImode:
{
rtx tmp;
- /* AVX supports all the needed comparisons, no need to swap arguments
- nor help reload. */
- if (TARGET_AVX)
- return code;
-
switch (code)
{
case LTGT:
case UNEQ:
+ /* AVX supports all the needed comparisons. */
+ if (TARGET_AVX)
+ break;
/* We have no LTGT as an operator. We could implement it with
NE & ORDERED, but this requires an extra temporary. It's
not clear that it's worth it. */
case NE:
case UNORDERED:
case ORDERED:
+ /* AVX has 3 operand comparisons, no need to swap anything. */
+ if (TARGET_AVX)
+ break;
/* For commutative operators, try to canonicalize the destination
operand to be first in the comparison - this helps reload to
avoid extra moves. */
case GT:
case UNLE:
case UNLT:
- /* These are not supported directly. Swap the comparison operands
- to transform into something that is supported. */
+ /* These are not supported directly before AVX, and furthermore
+ ix86_expand_sse_fp_minmax only optimizes LT/UNGE. Swap the
+ comparison operands to transform into something that is
+ supported. */
tmp = *pop0;
*pop0 = *pop1;
*pop1 = tmp;
enum machine_mode mode = GET_MODE (dest);
rtx t2, t3, x;
- if (op_false == CONST0_RTX (mode))
+ if (vector_all_ones_operand (op_true, mode)
+ && rtx_equal_p (op_false, CONST0_RTX (mode)))
+ {
+ emit_insn (gen_rtx_SET (VOIDmode, dest, cmp));
+ }
+ else if (op_false == CONST0_RTX (mode))
{
op_true = force_reg (mode, op_true);
x = gen_rtx_AND (mode, cmp, op_true);
x = gen_rtx_AND (mode, x, op_false);
emit_insn (gen_rtx_SET (VOIDmode, dest, x));
}
+ else if (INTEGRAL_MODE_P (mode) && op_true == CONSTM1_RTX (mode))
+ {
+ op_false = force_reg (mode, op_false);
+ x = gen_rtx_IOR (mode, cmp, op_false);
+ emit_insn (gen_rtx_SET (VOIDmode, dest, x));
+ }
else if (TARGET_XOP)
{
- rtx pcmov = gen_rtx_SET (mode, dest,
- gen_rtx_IF_THEN_ELSE (mode, cmp,
- op_true,
- op_false));
- emit_insn (pcmov);
+ op_true = force_reg (mode, op_true);
+
+ if (!nonimmediate_operand (op_false, mode))
+ op_false = force_reg (mode, op_false);
+
+ emit_insn (gen_rtx_SET (mode, dest,
+ gen_rtx_IF_THEN_ELSE (mode, cmp,
+ op_true,
+ op_false)));
}
else
{
- op_true = force_reg (mode, op_true);
+ rtx (*gen) (rtx, rtx, rtx, rtx) = NULL;
+
+ if (!nonimmediate_operand (op_true, mode))
+ op_true = force_reg (mode, op_true);
+
op_false = force_reg (mode, op_false);
- t2 = gen_reg_rtx (mode);
- if (optimize)
- t3 = gen_reg_rtx (mode);
+ switch (mode)
+ {
+ case V4SFmode:
+ if (TARGET_SSE4_1)
+ gen = gen_sse4_1_blendvps;
+ break;
+ case V2DFmode:
+ if (TARGET_SSE4_1)
+ gen = gen_sse4_1_blendvpd;
+ break;
+ case V16QImode:
+ case V8HImode:
+ case V4SImode:
+ case V2DImode:
+ if (TARGET_SSE4_1)
+ {
+ gen = gen_sse4_1_pblendvb;
+ dest = gen_lowpart (V16QImode, dest);
+ op_false = gen_lowpart (V16QImode, op_false);
+ op_true = gen_lowpart (V16QImode, op_true);
+ cmp = gen_lowpart (V16QImode, cmp);
+ }
+ break;
+ case V8SFmode:
+ if (TARGET_AVX)
+ gen = gen_avx_blendvps256;
+ break;
+ case V4DFmode:
+ if (TARGET_AVX)
+ gen = gen_avx_blendvpd256;
+ break;
+ case V32QImode:
+ case V16HImode:
+ case V8SImode:
+ case V4DImode:
+ if (TARGET_AVX2)
+ {
+ gen = gen_avx2_pblendvb;
+ dest = gen_lowpart (V32QImode, dest);
+ op_false = gen_lowpart (V32QImode, op_false);
+ op_true = gen_lowpart (V32QImode, op_true);
+ cmp = gen_lowpart (V32QImode, cmp);
+ }
+ break;
+ default:
+ break;
+ }
+
+ if (gen != NULL)
+ emit_insn (gen (dest, op_false, op_true, cmp));
else
- t3 = dest;
+ {
+ op_true = force_reg (mode, op_true);
- x = gen_rtx_AND (mode, op_true, cmp);
- emit_insn (gen_rtx_SET (VOIDmode, t2, x));
+ t2 = gen_reg_rtx (mode);
+ if (optimize)
+ t3 = gen_reg_rtx (mode);
+ else
+ t3 = dest;
- x = gen_rtx_NOT (mode, cmp);
- x = gen_rtx_AND (mode, x, op_false);
- emit_insn (gen_rtx_SET (VOIDmode, t3, x));
+ x = gen_rtx_AND (mode, op_true, cmp);
+ emit_insn (gen_rtx_SET (VOIDmode, t2, x));
- x = gen_rtx_IOR (mode, t3, t2);
- emit_insn (gen_rtx_SET (VOIDmode, dest, x));
+ x = gen_rtx_NOT (mode, cmp);
+ x = gen_rtx_AND (mode, x, op_false);
+ emit_insn (gen_rtx_SET (VOIDmode, t3, x));
+
+ x = gen_rtx_IOR (mode, t3, t2);
+ emit_insn (gen_rtx_SET (VOIDmode, dest, x));
+ }
}
}
bool
ix86_expand_int_vcond (rtx operands[])
{
- enum machine_mode mode = GET_MODE (operands[0]);
+ enum machine_mode data_mode = GET_MODE (operands[0]);
+ enum machine_mode mode = GET_MODE (operands[4]);
enum rtx_code code = GET_CODE (operands[3]);
bool negate = false;
rtx x, cop0, cop1;
}
}
- x = ix86_expand_sse_cmp (operands[0], code, cop0, cop1,
- operands[1+negate], operands[2-negate]);
+ /* Allow the comparison to be done in one mode, but the movcc to
+ happen in another mode. */
+ if (data_mode == mode)
+ {
+ x = ix86_expand_sse_cmp (operands[0], code, cop0, cop1,
+ operands[1+negate], operands[2-negate]);
+ }
+ else
+ {
+ gcc_assert (GET_MODE_SIZE (data_mode) == GET_MODE_SIZE (mode));
+ x = ix86_expand_sse_cmp (gen_lowpart (mode, operands[0]),
+ code, cop0, cop1,
+ operands[1+negate], operands[2-negate]);
+ x = gen_lowpart (data_mode, x);
+ }
ix86_expand_sse_movcc (operands[0], x, operands[1+negate],
operands[2-negate]);
return true;
}
+/* Expand a variable vector permutation. */
+
+void
+ix86_expand_vec_perm (rtx operands[])
+{
+ rtx target = operands[0];
+ rtx op0 = operands[1];
+ rtx op1 = operands[2];
+ rtx mask = operands[3];
+ rtx t1, t2, t3, t4, vt, vt2, vec[32];
+ enum machine_mode mode = GET_MODE (op0);
+ enum machine_mode maskmode = GET_MODE (mask);
+ int w, e, i;
+ bool one_operand_shuffle = rtx_equal_p (op0, op1);
+
+ /* Number of elements in the vector. */
+ w = GET_MODE_NUNITS (mode);
+ e = GET_MODE_UNIT_SIZE (mode);
+ gcc_assert (w <= 32);
+
+ if (TARGET_AVX2)
+ {
+ if (mode == V4DImode || mode == V4DFmode || mode == V16HImode)
+ {
+ /* Unfortunately, the VPERMQ and VPERMPD instructions only support
+ an constant shuffle operand. With a tiny bit of effort we can
+ use VPERMD instead. A re-interpretation stall for V4DFmode is
+ unfortunate but there's no avoiding it.
+ Similarly for V16HImode we don't have instructions for variable
+ shuffling, while for V32QImode we can use after preparing suitable
+ masks vpshufb; vpshufb; vpermq; vpor. */
+
+ if (mode == V16HImode)
+ {
+ maskmode = mode = V32QImode;
+ w = 32;
+ e = 1;
+ }
+ else
+ {
+ maskmode = mode = V8SImode;
+ w = 8;
+ e = 4;
+ }
+ t1 = gen_reg_rtx (maskmode);
+
+ /* Replicate the low bits of the V4DImode mask into V8SImode:
+ mask = { A B C D }
+ t1 = { A A B B C C D D }. */
+ for (i = 0; i < w / 2; ++i)
+ vec[i*2 + 1] = vec[i*2] = GEN_INT (i * 2);
+ vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
+ vt = force_reg (maskmode, vt);
+ mask = gen_lowpart (maskmode, mask);
+ if (maskmode == V8SImode)
+ emit_insn (gen_avx2_permvarv8si (t1, vt, mask));
+ else
+ emit_insn (gen_avx2_pshufbv32qi3 (t1, mask, vt));
+
+ /* Multiply the shuffle indicies by two. */
+ t1 = expand_simple_binop (maskmode, PLUS, t1, t1, t1, 1,
+ OPTAB_DIRECT);
+
+ /* Add one to the odd shuffle indicies:
+ t1 = { A*2, A*2+1, B*2, B*2+1, ... }. */
+ for (i = 0; i < w / 2; ++i)
+ {
+ vec[i * 2] = const0_rtx;
+ vec[i * 2 + 1] = const1_rtx;
+ }
+ vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
+ vt = force_const_mem (maskmode, vt);
+ t1 = expand_simple_binop (maskmode, PLUS, t1, vt, t1, 1,
+ OPTAB_DIRECT);
+
+ /* Continue as if V8SImode (resp. V32QImode) was used initially. */
+ operands[3] = mask = t1;
+ target = gen_lowpart (mode, target);
+ op0 = gen_lowpart (mode, op0);
+ op1 = gen_lowpart (mode, op1);
+ }
+
+ switch (mode)
+ {
+ case V8SImode:
+ /* The VPERMD and VPERMPS instructions already properly ignore
+ the high bits of the shuffle elements. No need for us to
+ perform an AND ourselves. */
+ if (one_operand_shuffle)
+ emit_insn (gen_avx2_permvarv8si (target, mask, op0));
+ else
+ {
+ t1 = gen_reg_rtx (V8SImode);
+ t2 = gen_reg_rtx (V8SImode);
+ emit_insn (gen_avx2_permvarv8si (t1, mask, op0));
+ emit_insn (gen_avx2_permvarv8si (t2, mask, op1));
+ goto merge_two;
+ }
+ return;
+
+ case V8SFmode:
+ mask = gen_lowpart (V8SFmode, mask);
+ if (one_operand_shuffle)
+ emit_insn (gen_avx2_permvarv8sf (target, mask, op0));
+ else
+ {
+ t1 = gen_reg_rtx (V8SFmode);
+ t2 = gen_reg_rtx (V8SFmode);
+ emit_insn (gen_avx2_permvarv8sf (t1, mask, op0));
+ emit_insn (gen_avx2_permvarv8sf (t2, mask, op1));
+ goto merge_two;
+ }
+ return;
+
+ case V4SImode:
+ /* By combining the two 128-bit input vectors into one 256-bit
+ input vector, we can use VPERMD and VPERMPS for the full
+ two-operand shuffle. */
+ t1 = gen_reg_rtx (V8SImode);
+ t2 = gen_reg_rtx (V8SImode);
+ emit_insn (gen_avx_vec_concatv8si (t1, op0, op1));
+ emit_insn (gen_avx_vec_concatv8si (t2, mask, mask));
+ emit_insn (gen_avx2_permvarv8si (t1, t2, t1));
+ emit_insn (gen_avx_vextractf128v8si (target, t1, const0_rtx));
+ return;
+
+ case V4SFmode:
+ t1 = gen_reg_rtx (V8SFmode);
+ t2 = gen_reg_rtx (V8SFmode);
+ mask = gen_lowpart (V4SFmode, mask);
+ emit_insn (gen_avx_vec_concatv8sf (t1, op0, op1));
+ emit_insn (gen_avx_vec_concatv8sf (t2, mask, mask));
+ emit_insn (gen_avx2_permvarv8sf (t1, t2, t1));
+ emit_insn (gen_avx_vextractf128v8sf (target, t1, const0_rtx));
+ return;
+
+ case V32QImode:
+ t1 = gen_reg_rtx (V32QImode);
+ t2 = gen_reg_rtx (V32QImode);
+ t3 = gen_reg_rtx (V32QImode);
+ vt2 = GEN_INT (128);
+ for (i = 0; i < 32; i++)
+ vec[i] = vt2;
+ vt = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, vec));
+ vt = force_reg (V32QImode, vt);
+ for (i = 0; i < 32; i++)
+ vec[i] = i < 16 ? vt2 : const0_rtx;
+ vt2 = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, vec));
+ vt2 = force_reg (V32QImode, vt2);
+ /* From mask create two adjusted masks, which contain the same
+ bits as mask in the low 7 bits of each vector element.
+ The first mask will have the most significant bit clear
+ if it requests element from the same 128-bit lane
+ and MSB set if it requests element from the other 128-bit lane.
+ The second mask will have the opposite values of the MSB,
+ and additionally will have its 128-bit lanes swapped.
+ E.g. { 07 12 1e 09 ... | 17 19 05 1f ... } mask vector will have
+ t1 { 07 92 9e 09 ... | 17 19 85 1f ... } and
+ t3 { 97 99 05 9f ... | 87 12 1e 89 ... } where each ...
+ stands for other 12 bytes. */
+ /* The bit whether element is from the same lane or the other
+ lane is bit 4, so shift it up by 3 to the MSB position. */
+ emit_insn (gen_ashlv4di3 (gen_lowpart (V4DImode, t1),
+ gen_lowpart (V4DImode, mask),
+ GEN_INT (3)));
+ /* Clear MSB bits from the mask just in case it had them set. */
+ emit_insn (gen_avx2_andnotv32qi3 (t2, vt, mask));
+ /* After this t1 will have MSB set for elements from other lane. */
+ emit_insn (gen_xorv32qi3 (t1, t1, vt2));
+ /* Clear bits other than MSB. */
+ emit_insn (gen_andv32qi3 (t1, t1, vt));
+ /* Or in the lower bits from mask into t3. */
+ emit_insn (gen_iorv32qi3 (t3, t1, t2));
+ /* And invert MSB bits in t1, so MSB is set for elements from the same
+ lane. */
+ emit_insn (gen_xorv32qi3 (t1, t1, vt));
+ /* Swap 128-bit lanes in t3. */
+ emit_insn (gen_avx2_permv4di_1 (gen_lowpart (V4DImode, t3),
+ gen_lowpart (V4DImode, t3),
+ const2_rtx, GEN_INT (3),
+ const0_rtx, const1_rtx));
+ /* And or in the lower bits from mask into t1. */
+ emit_insn (gen_iorv32qi3 (t1, t1, t2));
+ if (one_operand_shuffle)
+ {
+ /* Each of these shuffles will put 0s in places where
+ element from the other 128-bit lane is needed, otherwise
+ will shuffle in the requested value. */
+ emit_insn (gen_avx2_pshufbv32qi3 (t3, op0, t3));
+ emit_insn (gen_avx2_pshufbv32qi3 (t1, op0, t1));
+ /* For t3 the 128-bit lanes are swapped again. */
+ emit_insn (gen_avx2_permv4di_1 (gen_lowpart (V4DImode, t3),
+ gen_lowpart (V4DImode, t3),
+ const2_rtx, GEN_INT (3),
+ const0_rtx, const1_rtx));
+ /* And oring both together leads to the result. */
+ emit_insn (gen_iorv32qi3 (target, t1, t3));
+ return;
+ }
+
+ t4 = gen_reg_rtx (V32QImode);
+ /* Similarly to the above one_operand_shuffle code,
+ just for repeated twice for each operand. merge_two:
+ code will merge the two results together. */
+ emit_insn (gen_avx2_pshufbv32qi3 (t4, op0, t3));
+ emit_insn (gen_avx2_pshufbv32qi3 (t3, op1, t3));
+ emit_insn (gen_avx2_pshufbv32qi3 (t2, op0, t1));
+ emit_insn (gen_avx2_pshufbv32qi3 (t1, op1, t1));
+ emit_insn (gen_avx2_permv4di_1 (gen_lowpart (V4DImode, t4),
+ gen_lowpart (V4DImode, t4),
+ const2_rtx, GEN_INT (3),
+ const0_rtx, const1_rtx));
+ emit_insn (gen_avx2_permv4di_1 (gen_lowpart (V4DImode, t3),
+ gen_lowpart (V4DImode, t3),
+ const2_rtx, GEN_INT (3),
+ const0_rtx, const1_rtx));
+ emit_insn (gen_iorv32qi3 (t4, t2, t4));
+ emit_insn (gen_iorv32qi3 (t3, t1, t3));
+ t1 = t4;
+ t2 = t3;
+ goto merge_two;
+
+ default:
+ gcc_assert (GET_MODE_SIZE (mode) <= 16);
+ break;
+ }
+ }
+
+ if (TARGET_XOP)
+ {
+ /* The XOP VPPERM insn supports three inputs. By ignoring the
+ one_operand_shuffle special case, we avoid creating another
+ set of constant vectors in memory. */
+ one_operand_shuffle = false;
+
+ /* mask = mask & {2*w-1, ...} */
+ vt = GEN_INT (2*w - 1);
+ }
+ else
+ {
+ /* mask = mask & {w-1, ...} */
+ vt = GEN_INT (w - 1);
+ }
+
+ for (i = 0; i < w; i++)
+ vec[i] = vt;
+ vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
+ mask = expand_simple_binop (maskmode, AND, mask, vt,
+ NULL_RTX, 0, OPTAB_DIRECT);
+
+ /* For non-QImode operations, convert the word permutation control
+ into a byte permutation control. */
+ if (mode != V16QImode)
+ {
+ mask = expand_simple_binop (maskmode, ASHIFT, mask,
+ GEN_INT (exact_log2 (e)),
+ NULL_RTX, 0, OPTAB_DIRECT);
+
+ /* Convert mask to vector of chars. */
+ mask = force_reg (V16QImode, gen_lowpart (V16QImode, mask));
+
+ /* Replicate each of the input bytes into byte positions:
+ (v2di) --> {0,0,0,0,0,0,0,0, 8,8,8,8,8,8,8,8}
+ (v4si) --> {0,0,0,0, 4,4,4,4, 8,8,8,8, 12,12,12,12}
+ (v8hi) --> {0,0, 2,2, 4,4, 6,6, ...}. */
+ for (i = 0; i < 16; ++i)
+ vec[i] = GEN_INT (i/e * e);
+ vt = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, vec));
+ vt = force_const_mem (V16QImode, vt);
+ if (TARGET_XOP)
+ emit_insn (gen_xop_pperm (mask, mask, mask, vt));
+ else
+ emit_insn (gen_ssse3_pshufbv16qi3 (mask, mask, vt));
+
+ /* Convert it into the byte positions by doing
+ mask = mask + {0,1,..,16/w, 0,1,..,16/w, ...} */
+ for (i = 0; i < 16; ++i)
+ vec[i] = GEN_INT (i % e);
+ vt = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, vec));
+ vt = force_const_mem (V16QImode, vt);
+ emit_insn (gen_addv16qi3 (mask, mask, vt));
+ }
+
+ /* The actual shuffle operations all operate on V16QImode. */
+ op0 = gen_lowpart (V16QImode, op0);
+ op1 = gen_lowpart (V16QImode, op1);
+ target = gen_lowpart (V16QImode, target);
+
+ if (TARGET_XOP)
+ {
+ emit_insn (gen_xop_pperm (target, op0, op1, mask));
+ }
+ else if (one_operand_shuffle)
+ {
+ emit_insn (gen_ssse3_pshufbv16qi3 (target, op0, mask));
+ }
+ else
+ {
+ rtx xops[6];
+ bool ok;
+
+ /* Shuffle the two input vectors independently. */
+ t1 = gen_reg_rtx (V16QImode);
+ t2 = gen_reg_rtx (V16QImode);
+ emit_insn (gen_ssse3_pshufbv16qi3 (t1, op0, mask));
+ emit_insn (gen_ssse3_pshufbv16qi3 (t2, op1, mask));
+
+ merge_two:
+ /* Then merge them together. The key is whether any given control
+ element contained a bit set that indicates the second word. */
+ mask = operands[3];
+ vt = GEN_INT (w);
+ if (maskmode == V2DImode && !TARGET_SSE4_1)
+ {
+ /* Without SSE4.1, we don't have V2DImode EQ. Perform one
+ more shuffle to convert the V2DI input mask into a V4SI
+ input mask. At which point the masking that expand_int_vcond
+ will work as desired. */
+ rtx t3 = gen_reg_rtx (V4SImode);
+ emit_insn (gen_sse2_pshufd_1 (t3, gen_lowpart (V4SImode, mask),
+ const0_rtx, const0_rtx,
+ const2_rtx, const2_rtx));
+ mask = t3;
+ maskmode = V4SImode;
+ e = w = 4;
+ }
+
+ for (i = 0; i < w; i++)
+ vec[i] = vt;
+ vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec));
+ vt = force_reg (maskmode, vt);
+ mask = expand_simple_binop (maskmode, AND, mask, vt,
+ NULL_RTX, 0, OPTAB_DIRECT);
+
+ xops[0] = gen_lowpart (mode, operands[0]);
+ xops[1] = gen_lowpart (mode, t2);
+ xops[2] = gen_lowpart (mode, t1);
+ xops[3] = gen_rtx_EQ (maskmode, mask, vt);
+ xops[4] = mask;
+ xops[5] = vt;
+ ok = ix86_expand_int_vcond (xops);
+ gcc_assert (ok);
+ }
+}
+
/* Unpack OP[1] into the next wider integer vector type. UNSIGNED_P is
true if we should do zero extension, else sign extension. HIGH_P is
true if we want the N/2 high elements, else the low elements. */
if (TARGET_SSE4_1)
{
rtx (*unpack)(rtx, rtx);
+ rtx (*extract)(rtx, rtx) = NULL;
+ enum machine_mode halfmode = BLKmode;
switch (imode)
{
+ case V32QImode:
+ if (unsigned_p)
+ unpack = gen_avx2_zero_extendv16qiv16hi2;
+ else
+ unpack = gen_avx2_sign_extendv16qiv16hi2;
+ halfmode = V16QImode;
+ extract
+ = high_p ? gen_vec_extract_hi_v32qi : gen_vec_extract_lo_v32qi;
+ break;
+ case V16HImode:
+ if (unsigned_p)
+ unpack = gen_avx2_zero_extendv8hiv8si2;
+ else
+ unpack = gen_avx2_sign_extendv8hiv8si2;
+ halfmode = V8HImode;
+ extract
+ = high_p ? gen_vec_extract_hi_v16hi : gen_vec_extract_lo_v16hi;
+ break;
+ case V8SImode:
+ if (unsigned_p)
+ unpack = gen_avx2_zero_extendv4siv4di2;
+ else
+ unpack = gen_avx2_sign_extendv4siv4di2;
+ halfmode = V4SImode;
+ extract
+ = high_p ? gen_vec_extract_hi_v8si : gen_vec_extract_lo_v8si;
+ break;
case V16QImode:
if (unsigned_p)
unpack = gen_sse4_1_zero_extendv8qiv8hi2;
gcc_unreachable ();
}
- if (high_p)
+ if (GET_MODE_SIZE (imode) == 32)
+ {
+ tmp = gen_reg_rtx (halfmode);
+ emit_insn (extract (tmp, operands[1]));
+ }
+ else if (high_p)
{
/* Shift higher 8 bytes to lower 8 bytes. */
tmp = gen_reg_rtx (imode);
return SImode;
}
-/* When SRCPTR is non-NULL, output simple loop to move memory
+/* Helper function for expand_set_or_movmem_via_loop.
+
+ When SRCPTR is non-NULL, output simple loop to move memory
pointer to SRCPTR to DESTPTR via chunks of MODE unrolled UNROLL times,
overall size is COUNT specified in bytes. When SRCPTR is NULL, output the
equivalent loop to set memory by VALUE (supposed to be in MODE).
The size is rounded down to whole number of chunk size moved at once.
- SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info. */
+ SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info.
+ If ITER isn't NULL, than it'll be used in the generated loop without
+ initialization (that allows to generate several consequent loops using the
+ same iterator).
+ If CHANGE_PTRS is specified, DESTPTR and SRCPTR would be increased by
+ iterator value at the end of the function (as if they iterate in the loop).
+ Otherwise, their vaules'll stay unchanged.
-static void
-expand_set_or_movmem_via_loop (rtx destmem, rtx srcmem,
- rtx destptr, rtx srcptr, rtx value,
- rtx count, enum machine_mode mode, int unroll,
- int expected_size)
+ If EXPECTED_SIZE isn't -1, than it's used to compute branch-probabilities on
+ the loop backedge. When expected size is unknown (it's -1), the probability
+ is set to 80%.
+
+ Return value is rtx of iterator, used in the loop - it could be reused in
+ consequent calls of this function. */
+static rtx
+expand_set_or_movmem_via_loop_with_iter (rtx destmem, rtx srcmem,
+ rtx destptr, rtx srcptr, rtx value,
+ rtx count, rtx iter,
+ enum machine_mode mode, int unroll,
+ int expected_size, bool change_ptrs)
{
- rtx out_label, top_label, iter, tmp;
+ rtx out_label, top_label, tmp;
enum machine_mode iter_mode = counter_mode (count);
rtx piece_size = GEN_INT (GET_MODE_SIZE (mode) * unroll);
rtx piece_size_mask = GEN_INT (~((GET_MODE_SIZE (mode) * unroll) - 1));
rtx x_addr;
rtx y_addr;
int i;
+ bool reuse_iter = (iter != NULL_RTX);
top_label = gen_label_rtx ();
out_label = gen_label_rtx ();
- iter = gen_reg_rtx (iter_mode);
-
size = expand_simple_binop (iter_mode, AND, count, piece_size_mask,
- NULL, 1, OPTAB_DIRECT);
- /* Those two should combine. */
- if (piece_size == const1_rtx)
+ NULL, 1, OPTAB_DIRECT);
+ if (!reuse_iter)
{
- emit_cmp_and_jump_insns (size, const0_rtx, EQ, NULL_RTX, iter_mode,
+ iter = gen_reg_rtx (iter_mode);
+ /* Those two should combine. */
+ if (piece_size == const1_rtx)
+ {
+ emit_cmp_and_jump_insns (size, const0_rtx, EQ, NULL_RTX, iter_mode,
+ true, out_label);
+ predict_jump (REG_BR_PROB_BASE * 10 / 100);
+ }
+ emit_move_insn (iter, const0_rtx);
+ }
+ else
+ {
+ emit_cmp_and_jump_insns (iter, size, GE, NULL_RTX, iter_mode,
true, out_label);
- predict_jump (REG_BR_PROB_BASE * 10 / 100);
}
- emit_move_insn (iter, const0_rtx);
emit_label (top_label);
tmp = convert_modes (Pmode, iter_mode, iter, true);
x_addr = gen_rtx_PLUS (Pmode, destptr, tmp);
- destmem = change_address (destmem, mode, x_addr);
+ destmem =
+ adjust_automodify_address_nv (copy_rtx (destmem), mode, x_addr, 0);
if (srcmem)
{
y_addr = gen_rtx_PLUS (Pmode, srcptr, copy_rtx (tmp));
- srcmem = change_address (srcmem, mode, y_addr);
+ srcmem =
+ adjust_automodify_address_nv (copy_rtx (srcmem), mode, y_addr, 0);
/* When unrolling for chips that reorder memory reads and writes,
we can save registers by using single temporary.
}
else
predict_jump (REG_BR_PROB_BASE * 80 / 100);
- iter = ix86_zero_extend_to_Pmode (iter);
- tmp = expand_simple_binop (Pmode, PLUS, destptr, iter, destptr,
- true, OPTAB_LIB_WIDEN);
- if (tmp != destptr)
- emit_move_insn (destptr, tmp);
- if (srcptr)
+ if (change_ptrs)
{
- tmp = expand_simple_binop (Pmode, PLUS, srcptr, iter, srcptr,
+ iter = ix86_zero_extend_to_Pmode (iter);
+ tmp = expand_simple_binop (Pmode, PLUS, destptr, iter, destptr,
true, OPTAB_LIB_WIDEN);
- if (tmp != srcptr)
- emit_move_insn (srcptr, tmp);
+ if (tmp != destptr)
+ emit_move_insn (destptr, tmp);
+ if (srcptr)
+ {
+ tmp = expand_simple_binop (Pmode, PLUS, srcptr, iter, srcptr,
+ true, OPTAB_LIB_WIDEN);
+ if (tmp != srcptr)
+ emit_move_insn (srcptr, tmp);
+ }
}
emit_label (out_label);
+ return iter;
+}
+
+/* When SRCPTR is non-NULL, output simple loop to move memory
+ pointer to SRCPTR to DESTPTR via chunks of MODE unrolled UNROLL times,
+ overall size is COUNT specified in bytes. When SRCPTR is NULL, output the
+ equivalent loop to set memory by VALUE (supposed to be in MODE).
+
+ The size is rounded down to whole number of chunk size moved at once.
+ SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info. */
+
+static void
+expand_set_or_movmem_via_loop (rtx destmem, rtx srcmem,
+ rtx destptr, rtx srcptr, rtx value,
+ rtx count, enum machine_mode mode, int unroll,
+ int expected_size)
+{
+ expand_set_or_movmem_via_loop_with_iter (destmem, srcmem,
+ destptr, srcptr, value,
+ count, NULL_RTX, mode, unroll,
+ expected_size, true);
}
/* Output "rep; mov" instruction.
emit_insn (gen_strmov (destptr, dest, srcptr, src));
}
-/* Output code to copy at most count & (max_size - 1) bytes from SRC to DEST. */
+/* Emit strset instuction. If RHS is constant, and vector mode will be used,
+ then move this constant to a vector register before emitting strset. */
+static void
+emit_strset (rtx destmem, rtx value,
+ rtx destptr, enum machine_mode mode, int offset)
+{
+ rtx dest = adjust_automodify_address_nv (destmem, mode, destptr, offset);
+ emit_insn (gen_strset (destptr, dest, value));
+}
+
+/* Output code to copy (COUNT % MAX_SIZE) bytes from SRCPTR to DESTPTR.
+ SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info. */
static void
expand_movmem_epilogue (rtx destmem, rtx srcmem,
rtx destptr, rtx srcptr, rtx count, int max_size)
HOST_WIDE_INT countval = INTVAL (count);
int offset = 0;
- if ((countval & 0x10) && max_size > 16)
+ int remainder_size = countval % max_size;
+ enum machine_mode move_mode = Pmode;
+
+ /* Firstly, try to move data with the widest possible mode.
+ Remaining part we'll move using Pmode and narrower modes. */
+ if (TARGET_SSE)
{
- if (TARGET_64BIT)
- {
- emit_strmov (destmem, srcmem, destptr, srcptr, DImode, offset);
- emit_strmov (destmem, srcmem, destptr, srcptr, DImode, offset + 8);
- }
- else
- gcc_unreachable ();
- offset += 16;
+ if (max_size >= GET_MODE_SIZE (V4SImode))
+ move_mode = V4SImode;
+ else if (max_size >= GET_MODE_SIZE (DImode))
+ move_mode = DImode;
}
- if ((countval & 0x08) && max_size > 8)
+
+ while (remainder_size >= GET_MODE_SIZE (move_mode))
{
- if (TARGET_64BIT)
- emit_strmov (destmem, srcmem, destptr, srcptr, DImode, offset);
- else
- {
- emit_strmov (destmem, srcmem, destptr, srcptr, SImode, offset);
- emit_strmov (destmem, srcmem, destptr, srcptr, SImode, offset + 4);
- }
- offset += 8;
+ emit_strmov (destmem, srcmem, destptr, srcptr, move_mode, offset);
+ offset += GET_MODE_SIZE (move_mode);
+ remainder_size -= GET_MODE_SIZE (move_mode);
}
- if ((countval & 0x04) && max_size > 4)
+
+ /* Move the remaining part of epilogue - its size might be
+ a size of the widest mode. */
+ move_mode = Pmode;
+ while (remainder_size >= GET_MODE_SIZE (move_mode))
+ {
+ emit_strmov (destmem, srcmem, destptr, srcptr, move_mode, offset);
+ offset += GET_MODE_SIZE (move_mode);
+ remainder_size -= GET_MODE_SIZE (move_mode);
+ }
+
+ if (remainder_size >= 4)
{
- emit_strmov (destmem, srcmem, destptr, srcptr, SImode, offset);
+ emit_strmov (destmem, srcmem, destptr, srcptr, SImode, offset);
offset += 4;
+ remainder_size -= 4;
}
- if ((countval & 0x02) && max_size > 2)
+ if (remainder_size >= 2)
{
- emit_strmov (destmem, srcmem, destptr, srcptr, HImode, offset);
+ emit_strmov (destmem, srcmem, destptr, srcptr, HImode, offset);
offset += 2;
+ remainder_size -= 2;
}
- if ((countval & 0x01) && max_size > 1)
+ if (remainder_size >= 1)
{
- emit_strmov (destmem, srcmem, destptr, srcptr, QImode, offset);
+ emit_strmov (destmem, srcmem, destptr, srcptr, QImode, offset);
offset += 1;
+ remainder_size -= 1;
}
+ gcc_assert (remainder_size == 0);
return;
}
- if (max_size > 8)
+ if (max_size > 16)
{
count = expand_simple_binop (GET_MODE (count), AND, count, GEN_INT (max_size - 1),
count, 1, OPTAB_DIRECT);
*/
if (TARGET_SINGLE_STRINGOP)
{
+ if (max_size > 8)
+ {
+ rtx label = ix86_expand_aligntest (count, 8, true);
+ if (TARGET_64BIT)
+ {
+ src = change_address (srcmem, DImode, srcptr);
+ dest = change_address (destmem, DImode, destptr);
+ emit_insn (gen_strmov (destptr, dest, srcptr, src));
+ }
+ else
+ {
+ src = change_address (srcmem, SImode, srcptr);
+ dest = change_address (destmem, SImode, destptr);
+ emit_insn (gen_strmov (destptr, dest, srcptr, src));
+ emit_insn (gen_strmov (destptr, dest, srcptr, src));
+ }
+ emit_label (label);
+ LABEL_NUSES (label) = 1;
+ }
if (max_size > 4)
{
rtx label = ix86_expand_aligntest (count, 4, true);
rtx offset = force_reg (Pmode, const0_rtx);
rtx tmp;
+ if (max_size > 8)
+ {
+ rtx label = ix86_expand_aligntest (count, 8, true);
+ if (TARGET_64BIT)
+ {
+ src = change_address (srcmem, DImode, srcptr);
+ dest = change_address (destmem, DImode, destptr);
+ emit_move_insn (dest, src);
+ tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (8), NULL,
+ true, OPTAB_LIB_WIDEN);
+ }
+ else
+ {
+ src = change_address (srcmem, SImode, srcptr);
+ dest = change_address (destmem, SImode, destptr);
+ emit_move_insn (dest, src);
+ tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (4), NULL,
+ true, OPTAB_LIB_WIDEN);
+ if (tmp != offset)
+ emit_move_insn (offset, tmp);
+ tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (4), NULL,
+ true, OPTAB_LIB_WIDEN);
+ emit_move_insn (dest, src);
+ }
+ if (tmp != offset)
+ emit_move_insn (offset, tmp);
+ emit_label (label);
+ LABEL_NUSES (label) = 1;
+ }
if (max_size > 4)
{
rtx label = ix86_expand_aligntest (count, 4, true);
1, max_size / 2);
}
-/* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
+/* Output code to set with VALUE at most (COUNT % MAX_SIZE) bytes starting from
+ DESTPTR.
+ DESTMEM provides MEMrtx to feed proper aliasing info.
+ PROMOTED_TO_GPR_VALUE is rtx representing a GPR containing broadcasted VALUE.
+ PROMOTED_TO_VECTOR_VALUE is rtx representing a vector register containing
+ broadcasted VALUE.
+ PROMOTED_TO_GPR_VALUE and PROMOTED_TO_VECTOR_VALUE could be NULL if the
+ promotion hasn't been generated before. */
static void
-expand_setmem_epilogue (rtx destmem, rtx destptr, rtx value, rtx count, int max_size)
+expand_setmem_epilogue (rtx destmem, rtx destptr, rtx promoted_to_vector_value,
+ rtx promoted_to_gpr_value, rtx value, rtx count,
+ int max_size)
{
- rtx dest;
-
if (CONST_INT_P (count))
{
HOST_WIDE_INT countval = INTVAL (count);
int offset = 0;
- if ((countval & 0x10) && max_size > 16)
- {
- if (TARGET_64BIT)
- {
- dest = adjust_automodify_address_nv (destmem, DImode, destptr, offset);
- emit_insn (gen_strset (destptr, dest, value));
- dest = adjust_automodify_address_nv (destmem, DImode, destptr, offset + 8);
- emit_insn (gen_strset (destptr, dest, value));
- }
- else
- gcc_unreachable ();
- offset += 16;
- }
- if ((countval & 0x08) && max_size > 8)
+ int remainder_size = countval % max_size;
+ enum machine_mode move_mode = Pmode;
+
+ /* Firstly, try to move data with the widest possible mode.
+ Remaining part we'll move using Pmode and narrower modes. */
+
+ if (promoted_to_vector_value)
{
- if (TARGET_64BIT)
+ if (promoted_to_vector_value)
{
- dest = adjust_automodify_address_nv (destmem, DImode, destptr, offset);
- emit_insn (gen_strset (destptr, dest, value));
+ if (max_size >= GET_MODE_SIZE (V4SImode))
+ move_mode = V4SImode;
+ else if (max_size >= GET_MODE_SIZE (DImode))
+ move_mode = DImode;
}
- else
+ while (remainder_size >= GET_MODE_SIZE (move_mode))
{
- dest = adjust_automodify_address_nv (destmem, SImode, destptr, offset);
- emit_insn (gen_strset (destptr, dest, value));
- dest = adjust_automodify_address_nv (destmem, SImode, destptr, offset + 4);
- emit_insn (gen_strset (destptr, dest, value));
+ if (GET_MODE (destmem) != move_mode)
+ destmem = adjust_automodify_address_nv (destmem, move_mode,
+ destptr, offset);
+ emit_strset (destmem,
+ promoted_to_vector_value,
+ destptr,
+ move_mode, offset);
+
+ offset += GET_MODE_SIZE (move_mode);
+ remainder_size -= GET_MODE_SIZE (move_mode);
}
- offset += 8;
}
- if ((countval & 0x04) && max_size > 4)
+
+ /* Move the remaining part of epilogue - its size might be
+ a size of the widest mode. */
+ while (remainder_size >= GET_MODE_SIZE (Pmode))
{
- dest = adjust_automodify_address_nv (destmem, SImode, destptr, offset);
- emit_insn (gen_strset (destptr, dest, gen_lowpart (SImode, value)));
+ if (!promoted_to_gpr_value)
+ promoted_to_gpr_value = promote_duplicated_reg (Pmode, value);
+ emit_strset (destmem, promoted_to_gpr_value, destptr, Pmode, offset);
+ offset += GET_MODE_SIZE (Pmode);
+ remainder_size -= GET_MODE_SIZE (Pmode);
+ }
+
+ if (!promoted_to_gpr_value && remainder_size > 1)
+ promoted_to_gpr_value = promote_duplicated_reg (remainder_size >= 4
+ ? SImode : HImode, value);
+ if (remainder_size >= 4)
+ {
+ emit_strset (destmem, gen_lowpart (SImode, promoted_to_gpr_value), destptr,
+ SImode, offset);
offset += 4;
+ remainder_size -= 4;
}
- if ((countval & 0x02) && max_size > 2)
+ if (remainder_size >= 2)
{
- dest = adjust_automodify_address_nv (destmem, HImode, destptr, offset);
- emit_insn (gen_strset (destptr, dest, gen_lowpart (HImode, value)));
- offset += 2;
+ emit_strset (destmem, gen_lowpart (HImode, promoted_to_gpr_value), destptr,
+ HImode, offset);
+ offset +=2;
+ remainder_size -= 2;
}
- if ((countval & 0x01) && max_size > 1)
+ if (remainder_size >= 1)
{
- dest = adjust_automodify_address_nv (destmem, QImode, destptr, offset);
- emit_insn (gen_strset (destptr, dest, gen_lowpart (QImode, value)));
+ emit_strset (destmem,
+ promoted_to_gpr_value ? gen_lowpart (QImode, promoted_to_gpr_value) : value,
+ destptr,
+ QImode, offset);
offset += 1;
+ remainder_size -= 1;
}
+ gcc_assert (remainder_size == 0);
return;
}
+
+ /* count isn't const. */
if (max_size > 32)
{
- expand_setmem_epilogue_via_loop (destmem, destptr, value, count, max_size);
+ expand_setmem_epilogue_via_loop (destmem, destptr, value, count,
+ max_size);
return;
}
+
+ if (!promoted_to_gpr_value)
+ promoted_to_gpr_value = promote_duplicated_reg_to_size (value,
+ GET_MODE_SIZE (Pmode),
+ GET_MODE_SIZE (Pmode),
+ GET_MODE_SIZE (Pmode));
+
if (max_size > 16)
{
rtx label = ix86_expand_aligntest (count, 16, true);
- if (TARGET_64BIT)
+ if (TARGET_SSE && promoted_to_vector_value)
+ {
+ destmem = change_address (destmem,
+ GET_MODE (promoted_to_vector_value),
+ destptr);
+ emit_insn (gen_strset (destptr, destmem, promoted_to_vector_value));
+ }
+ else if (TARGET_64BIT)
{
- dest = change_address (destmem, DImode, destptr);
- emit_insn (gen_strset (destptr, dest, value));
- emit_insn (gen_strset (destptr, dest, value));
+ destmem = change_address (destmem, DImode, destptr);
+ emit_insn (gen_strset (destptr, destmem, promoted_to_gpr_value));
+ emit_insn (gen_strset (destptr, destmem, promoted_to_gpr_value));
}
else
{
- dest = change_address (destmem, SImode, destptr);
- emit_insn (gen_strset (destptr, dest, value));
- emit_insn (gen_strset (destptr, dest, value));
- emit_insn (gen_strset (destptr, dest, value));
- emit_insn (gen_strset (destptr, dest, value));
+ destmem = change_address (destmem, SImode, destptr);
+ emit_insn (gen_strset (destptr, destmem, promoted_to_gpr_value));
+ emit_insn (gen_strset (destptr, destmem, promoted_to_gpr_value));
+ emit_insn (gen_strset (destptr, destmem, promoted_to_gpr_value));
+ emit_insn (gen_strset (destptr, destmem, promoted_to_gpr_value));
}
emit_label (label);
LABEL_NUSES (label) = 1;
rtx label = ix86_expand_aligntest (count, 8, true);
if (TARGET_64BIT)
{
- dest = change_address (destmem, DImode, destptr);
- emit_insn (gen_strset (destptr, dest, value));
+ destmem = change_address (destmem, DImode, destptr);
+ emit_insn (gen_strset (destptr, destmem, promoted_to_gpr_value));
+ }
+ /* FIXME: When this hunk it output, IRA classifies promoted_to_vector_value
+ as NO_REGS. */
+ else if (TARGET_SSE && promoted_to_vector_value && 0)
+ {
+ destmem = change_address (destmem, V2SImode, destptr);
+ emit_insn (gen_strset (destptr, destmem,
+ gen_lowpart (V2SImode, promoted_to_vector_value)));
}
else
{
- dest = change_address (destmem, SImode, destptr);
- emit_insn (gen_strset (destptr, dest, value));
- emit_insn (gen_strset (destptr, dest, value));
+ destmem = change_address (destmem, SImode, destptr);
+ emit_insn (gen_strset (destptr, destmem, promoted_to_gpr_value));
+ emit_insn (gen_strset (destptr, destmem, promoted_to_gpr_value));
}
emit_label (label);
LABEL_NUSES (label) = 1;
if (max_size > 4)
{
rtx label = ix86_expand_aligntest (count, 4, true);
- dest = change_address (destmem, SImode, destptr);
- emit_insn (gen_strset (destptr, dest, gen_lowpart (SImode, value)));
+ destmem = change_address (destmem, SImode, destptr);
+ emit_insn (gen_strset (destptr, destmem,
+ gen_lowpart (SImode, promoted_to_gpr_value)));
emit_label (label);
LABEL_NUSES (label) = 1;
}
if (max_size > 2)
{
rtx label = ix86_expand_aligntest (count, 2, true);
- dest = change_address (destmem, HImode, destptr);
- emit_insn (gen_strset (destptr, dest, gen_lowpart (HImode, value)));
+ destmem = change_address (destmem, HImode, destptr);
+ emit_insn (gen_strset (destptr, destmem,
+ gen_lowpart (HImode, promoted_to_gpr_value)));
emit_label (label);
LABEL_NUSES (label) = 1;
}
if (max_size > 1)
{
rtx label = ix86_expand_aligntest (count, 1, true);
- dest = change_address (destmem, QImode, destptr);
- emit_insn (gen_strset (destptr, dest, gen_lowpart (QImode, value)));
+ destmem = change_address (destmem, QImode, destptr);
+ emit_insn (gen_strset (destptr, destmem,
+ gen_lowpart (QImode, promoted_to_gpr_value)));
emit_label (label);
LABEL_NUSES (label) = 1;
}
if (align <= 1 && desired_alignment > 1)
{
rtx label = ix86_expand_aligntest (destptr, 1, false);
- srcmem = change_address (srcmem, QImode, srcptr);
- destmem = change_address (destmem, QImode, destptr);
+ srcmem = adjust_automodify_address_nv (srcmem, QImode, srcptr, 0);
+ destmem = adjust_automodify_address_nv (destmem, QImode, destptr, 0);
emit_insn (gen_strmov (destptr, destmem, srcptr, srcmem));
ix86_adjust_counter (count, 1);
emit_label (label);
if (align <= 2 && desired_alignment > 2)
{
rtx label = ix86_expand_aligntest (destptr, 2, false);
- srcmem = change_address (srcmem, HImode, srcptr);
- destmem = change_address (destmem, HImode, destptr);
+ srcmem = adjust_automodify_address_nv (srcmem, HImode, srcptr, 0);
+ destmem = adjust_automodify_address_nv (destmem, HImode, destptr, 0);
emit_insn (gen_strmov (destptr, destmem, srcptr, srcmem));
ix86_adjust_counter (count, 2);
emit_label (label);
if (align <= 4 && desired_alignment > 4)
{
rtx label = ix86_expand_aligntest (destptr, 4, false);
- srcmem = change_address (srcmem, SImode, srcptr);
- destmem = change_address (destmem, SImode, destptr);
+ srcmem = adjust_automodify_address_nv (srcmem, SImode, srcptr, 0);
+ destmem = adjust_automodify_address_nv (destmem, SImode, destptr, 0);
emit_insn (gen_strmov (destptr, destmem, srcptr, srcmem));
ix86_adjust_counter (count, 4);
emit_label (label);
LABEL_NUSES (label) = 1;
}
- gcc_assert (desired_alignment <= 8);
+ if (align <= 8 && desired_alignment > 8)
+ {
+ rtx label = ix86_expand_aligntest (destptr, 8, false);
+ if (TARGET_64BIT || TARGET_SSE)
+ {
+ srcmem = adjust_automodify_address_nv (srcmem, DImode, srcptr, 0);
+ destmem = adjust_automodify_address_nv (destmem, DImode, destptr, 0);
+ emit_insn (gen_strmov (destptr, destmem, srcptr, srcmem));
+ }
+ else
+ {
+ srcmem = adjust_automodify_address_nv (srcmem, SImode, srcptr, 0);
+ destmem = adjust_automodify_address_nv (destmem, SImode, destptr, 0);
+ emit_insn (gen_strmov (destptr, destmem, srcptr, srcmem));
+ emit_insn (gen_strmov (destptr, destmem, srcptr, srcmem));
+ }
+ ix86_adjust_counter (count, 8);
+ emit_label (label);
+ LABEL_NUSES (label) = 1;
+ }
+ gcc_assert (desired_alignment <= 16);
}
/* Copy enough from DST to SRC to align DST known to DESIRED_ALIGN.
off = 4;
emit_insn (gen_strmov (destreg, dst, srcreg, src));
}
+ if (align_bytes & 8)
+ {
+ if (TARGET_64BIT || TARGET_SSE)
+ {
+ dst = adjust_automodify_address_nv (dst, DImode, destreg, off);
+ src = adjust_automodify_address_nv (src, DImode, srcreg, off);
+ emit_insn (gen_strmov (destreg, dst, srcreg, src));
+ }
+ else
+ {
+ dst = adjust_automodify_address_nv (dst, SImode, destreg, off);
+ src = adjust_automodify_address_nv (src, SImode, srcreg, off);
+ emit_insn (gen_strmov (destreg, dst, srcreg, src));
+ emit_insn (gen_strmov (destreg, dst, srcreg, src));
+ }
+ if (MEM_ALIGN (dst) < 8 * BITS_PER_UNIT)
+ set_mem_align (dst, 8 * BITS_PER_UNIT);
+ if (src_align_bytes >= 0)
+ {
+ unsigned int src_align = 0;
+ if ((src_align_bytes & 7) == (align_bytes & 7))
+ src_align = 8;
+ else if ((src_align_bytes & 3) == (align_bytes & 3))
+ src_align = 4;
+ else if ((src_align_bytes & 1) == (align_bytes & 1))
+ src_align = 2;
+ if (MEM_ALIGN (src) < src_align * BITS_PER_UNIT)
+ set_mem_align (src, src_align * BITS_PER_UNIT);
+ }
+ off = 8;
+ }
dst = adjust_automodify_address_nv (dst, BLKmode, destreg, off);
src = adjust_automodify_address_nv (src, BLKmode, srcreg, off);
if (MEM_ALIGN (dst) < (unsigned int) desired_align * BITS_PER_UNIT)
if (src_align_bytes >= 0)
{
unsigned int src_align = 0;
- if ((src_align_bytes & 7) == (align_bytes & 7))
+ if ((src_align_bytes & 15) == (align_bytes & 15))
+ src_align = 16;
+ else if ((src_align_bytes & 7) == (align_bytes & 7))
src_align = 8;
else if ((src_align_bytes & 3) == (align_bytes & 3))
src_align = 4;
if (align <= 1 && desired_alignment > 1)
{
rtx label = ix86_expand_aligntest (destptr, 1, false);
- destmem = change_address (destmem, QImode, destptr);
+ destmem = adjust_automodify_address_nv (destmem, QImode, destptr, 0);
emit_insn (gen_strset (destptr, destmem, gen_lowpart (QImode, value)));
ix86_adjust_counter (count, 1);
emit_label (label);
if (align <= 2 && desired_alignment > 2)
{
rtx label = ix86_expand_aligntest (destptr, 2, false);
- destmem = change_address (destmem, HImode, destptr);
+ destmem = adjust_automodify_address_nv (destmem, HImode, destptr, 0);
emit_insn (gen_strset (destptr, destmem, gen_lowpart (HImode, value)));
ix86_adjust_counter (count, 2);
emit_label (label);
if (align <= 4 && desired_alignment > 4)
{
rtx label = ix86_expand_aligntest (destptr, 4, false);
- destmem = change_address (destmem, SImode, destptr);
+ destmem = adjust_automodify_address_nv (destmem, SImode, destptr, 0);
emit_insn (gen_strset (destptr, destmem, gen_lowpart (SImode, value)));
ix86_adjust_counter (count, 4);
emit_label (label);
LABEL_NUSES (label) = 1;
}
- gcc_assert (desired_alignment <= 8);
+ if (align <= 8 && desired_alignment > 8)
+ {
+ rtx label = ix86_expand_aligntest (destptr, 8, false);
+ destmem = adjust_automodify_address_nv (destmem, SImode, destptr, 0);
+ emit_insn (gen_strset (destptr, destmem, gen_lowpart (SImode, value)));
+ emit_insn (gen_strset (destptr, destmem, gen_lowpart (SImode, value)));
+ ix86_adjust_counter (count, 8);
+ emit_label (label);
+ LABEL_NUSES (label) = 1;
+ }
+ gcc_assert (desired_alignment <= 16);
}
/* Set enough from DST to align DST known to by aligned by ALIGN to
emit_insn (gen_strset (destreg, dst,
gen_lowpart (SImode, value)));
}
+ if (align_bytes & 8)
+ {
+ dst = adjust_automodify_address_nv (dst, SImode, destreg, off);
+ emit_insn (gen_strset (destreg, dst,
+ gen_lowpart (SImode, value)));
+ off = 4;
+ dst = adjust_automodify_address_nv (dst, SImode, destreg, off);
+ emit_insn (gen_strset (destreg, dst,
+ gen_lowpart (SImode, value)));
+ if (MEM_ALIGN (dst) < 8 * BITS_PER_UNIT)
+ set_mem_align (dst, 8 * BITS_PER_UNIT);
+ off = 4;
+ }
dst = adjust_automodify_address_nv (dst, BLKmode, destreg, off);
if (MEM_ALIGN (dst) < (unsigned int) desired_align * BITS_PER_UNIT)
set_mem_align (dst, desired_align * BITS_PER_UNIT);
/* Given COUNT and EXPECTED_SIZE, decide on codegen of string operation. */
static enum stringop_alg
decide_alg (HOST_WIDE_INT count, HOST_WIDE_INT expected_size, bool memset,
- int *dynamic_check)
+ int *dynamic_check, bool align_unknown)
{
const struct stringop_algs * algs;
bool optimize_for_speed;
consider such algorithms if the user has appropriated those
registers for their own purposes. */
bool rep_prefix_usable = !(fixed_regs[CX_REG] || fixed_regs[DI_REG]
- || (memset
+ || (memset
? fixed_regs[AX_REG] : fixed_regs[SI_REG]));
-#define ALG_USABLE_P(alg) (rep_prefix_usable \
- || (alg != rep_prefix_1_byte \
- && alg != rep_prefix_4_byte \
- && alg != rep_prefix_8_byte))
+#define ALG_USABLE_P(alg) ((rep_prefix_usable \
+ || (alg != rep_prefix_1_byte \
+ && alg != rep_prefix_4_byte \
+ && alg != rep_prefix_8_byte)) \
+ && (TARGET_SSE2 || alg != sse_loop))
const struct processor_costs *cost;
/* Even if the string operation call is cold, we still might spend a lot
of time processing large blocks. */
if (optimize_function_for_size_p (cfun)
|| (optimize_insn_for_size_p ()
- && expected_size != -1 && expected_size < 256))
+ && expected_size != -1 && expected_size < 256))
optimize_for_speed = false;
else
optimize_for_speed = true;
+ if (!optimize)
+ return (rep_prefix_usable ? rep_prefix_1_byte : libcall);
+
cost = optimize_for_speed ? ix86_cost : &ix86_size_cost;
*dynamic_check = -1;
if (memset)
- algs = &cost->memset[TARGET_64BIT != 0];
+ algs = &cost->memset[align_unknown][TARGET_64BIT != 0];
else
- algs = &cost->memcpy[TARGET_64BIT != 0];
+ algs = &cost->memcpy[align_unknown][TARGET_64BIT != 0];
if (ix86_stringop_alg != no_stringop && ALG_USABLE_P (ix86_stringop_alg))
return ix86_stringop_alg;
/* rep; movq or rep; movl is the smallest variant. */
else if (!optimize_for_speed)
{
- if (!count || (count & 3))
- return rep_prefix_usable ? rep_prefix_1_byte : loop_1_byte;
+ if (!count || (count & 3) || memset)
+ return rep_prefix_usable ? rep_prefix_1_byte : libcall;
else
- return rep_prefix_usable ? rep_prefix_4_byte : loop;
+ return rep_prefix_usable ? rep_prefix_4_byte : libcall;
}
/* Very tiny blocks are best handled via the loop, REP is expensive to setup.
*/
int max = -1;
enum stringop_alg alg;
int i;
- bool any_alg_usable_p = true;
+ bool only_libcall_fits = true;
for (i = 0; i < MAX_STRINGOP_ALGS; i++)
- {
- enum stringop_alg candidate = algs->size[i].alg;
- any_alg_usable_p = any_alg_usable_p && ALG_USABLE_P (candidate);
+ {
+ enum stringop_alg candidate = algs->size[i].alg;
- if (candidate != libcall && candidate
- && ALG_USABLE_P (candidate))
- max = algs->size[i].max;
- }
+ if (candidate != libcall && candidate
+ && ALG_USABLE_P (candidate))
+ {
+ max = algs->size[i].max;
+ only_libcall_fits = false;
+ }
+ }
/* If there aren't any usable algorithms, then recursing on
- smaller sizes isn't going to find anything. Just return the
- simple byte-at-a-time copy loop. */
- if (!any_alg_usable_p)
- {
- /* Pick something reasonable. */
- if (TARGET_INLINE_STRINGOPS_DYNAMICALLY)
- *dynamic_check = 128;
- return loop_1_byte;
- }
+ smaller sizes isn't going to find anything. Just return the
+ simple byte-at-a-time copy loop. */
+ if (only_libcall_fits)
+ {
+ /* Pick something reasonable. */
+ if (TARGET_INLINE_STRINGOPS_DYNAMICALLY)
+ *dynamic_check = 128;
+ return loop_1_byte;
+ }
if (max == -1)
max = 4096;
- alg = decide_alg (count, max / 2, memset, dynamic_check);
+ alg = decide_alg (count, max / 2, memset, dynamic_check, align_unknown);
gcc_assert (*dynamic_check == -1);
gcc_assert (alg != libcall);
if (TARGET_INLINE_STRINGOPS_DYNAMICALLY)
case no_stringop:
gcc_unreachable ();
case loop:
+ desired_align = GET_MODE_SIZE (Pmode);
+ break;
case unrolled_loop:
desired_align = GET_MODE_SIZE (Pmode);
break;
+ case sse_loop:
+ desired_align = 16;
+ break;
case rep_prefix_8_byte:
desired_align = 8;
break;
enum stringop_alg alg;
int dynamic_check;
bool need_zero_guard = false;
+ bool align_unknown;
+ unsigned int unroll_factor;
+ enum machine_mode move_mode;
+ rtx loop_iter = NULL_RTX;
+ int dst_offset, src_offset;
if (CONST_INT_P (align_exp))
align = INTVAL (align_exp);
/* Step 0: Decide on preferred algorithm, desired alignment and
size of chunks to be copied by main loop. */
-
- alg = decide_alg (count, expected_size, false, &dynamic_check);
+ dst_offset = get_mem_align_offset (dst, MOVE_MAX*BITS_PER_UNIT);
+ src_offset = get_mem_align_offset (src, MOVE_MAX*BITS_PER_UNIT);
+ align_unknown = (dst_offset < 0
+ || src_offset < 0
+ || src_offset != dst_offset);
+ alg = decide_alg (count, expected_size, false, &dynamic_check, align_unknown);
desired_align = decide_alignment (align, alg, expected_size);
+ if (align_unknown)
+ desired_align = align;
+ unroll_factor = 1;
+ move_mode = Pmode;
if (!TARGET_ALIGN_STRINGOPS)
align = desired_align;
gcc_unreachable ();
case loop:
need_zero_guard = true;
- size_needed = GET_MODE_SIZE (Pmode);
+ move_mode = Pmode;
+ unroll_factor = 1;
+ size_needed = GET_MODE_SIZE (move_mode) * unroll_factor;
break;
case unrolled_loop:
need_zero_guard = true;
- size_needed = GET_MODE_SIZE (Pmode) * (TARGET_64BIT ? 4 : 2);
+ move_mode = Pmode;
+ unroll_factor = 1;
+ /* Select maximal available 1,2 or 4 unroll factor.
+ In 32bit we can not afford to use 4 registers inside the loop. */
+ if (!count)
+ unroll_factor = TARGET_64BIT ? 4 : 2;
+ else
+ while (GET_MODE_SIZE (move_mode) * unroll_factor * 2 < count
+ && unroll_factor < (TARGET_64BIT ? 4 :2))
+ unroll_factor *= 2;
+ size_needed = GET_MODE_SIZE (move_mode) * unroll_factor;
+ break;
+ case sse_loop:
+ need_zero_guard = true;
+ /* Use SSE instructions, if possible. */
+ move_mode = V4SImode;
+ /* Select maximal available 1,2 or 4 unroll factor. */
+ if (!count)
+ unroll_factor = 4;
+ else
+ while (GET_MODE_SIZE (move_mode) * unroll_factor * 2 < count
+ && unroll_factor < 4)
+ unroll_factor *= 2;
+ size_needed = GET_MODE_SIZE (move_mode) * unroll_factor;
break;
case rep_prefix_8_byte:
size_needed = 8;
}
else
{
+ /* SSE and unrolled algs re-use iteration counter in the epilogue. */
+ if (alg == sse_loop || alg == unrolled_loop)
+ {
+ loop_iter = gen_reg_rtx (counter_mode (count_exp));
+ emit_move_insn (loop_iter, const0_rtx);
+ }
label = gen_label_rtx ();
emit_cmp_and_jump_insns (count_exp,
GEN_INT (epilogue_size_needed),
dst = change_address (dst, BLKmode, destreg);
expand_movmem_prologue (dst, src, destreg, srcreg, count_exp, align,
desired_align);
+ set_mem_align (src, desired_align*BITS_PER_UNIT);
+ set_mem_align (dst, desired_align*BITS_PER_UNIT);
}
else
{
expand_set_or_movmem_via_loop (dst, src, destreg, srcreg, NULL,
count_exp, Pmode, 1, expected_size);
break;
+ case sse_loop:
case unrolled_loop:
- /* Unroll only by factor of 2 in 32bit mode, since we don't have enough
- registers for 4 temporaries anyway. */
- expand_set_or_movmem_via_loop (dst, src, destreg, srcreg, NULL,
- count_exp, Pmode, TARGET_64BIT ? 4 : 2,
- expected_size);
+ /* In some cases we want to use the same iterator in several adjacent
+ loops, so here we save loop iterator rtx and don't update addresses. */
+ loop_iter = expand_set_or_movmem_via_loop_with_iter (dst, src, destreg,
+ srcreg, NULL,
+ count_exp, loop_iter,
+ move_mode,
+ unroll_factor,
+ expected_size, false);
break;
case rep_prefix_8_byte:
expand_movmem_via_rep_mov (dst, src, destreg, srcreg, count_exp,
LABEL_NUSES (label) = 1;
}
+ /* We haven't updated addresses, so we'll do it now.
+ Also, if the epilogue seems to be big, we'll generate a loop (not
+ unrolled) in it. We'll do it only if alignment is unknown, because in
+ this case in epilogue we have to perform memmove by bytes, which is very
+ slow. */
+ if (alg == sse_loop || alg == unrolled_loop)
+ {
+ rtx tmp;
+ int remainder_size = epilogue_size_needed;
+
+ /* We may not need the epilgoue loop at all when the count is known
+ and alignment is not adjusted. */
+ if (count && desired_align <= align)
+ remainder_size = count % epilogue_size_needed;
+ if (remainder_size > 31)
+ {
+ /* Reduce epilogue's size by creating not-unrolled loop. If we won't
+ do this, we can have very big epilogue - when alignment is statically
+ unknown we'll have the epilogue byte by byte which may be very slow. */
+ loop_iter = expand_set_or_movmem_via_loop_with_iter (dst, src, destreg,
+ srcreg, NULL, count_exp,
+ loop_iter, move_mode, 1,
+ expected_size, false);
+ src = change_address (src, BLKmode, srcreg);
+ dst = change_address (dst, BLKmode, destreg);
+ epilogue_size_needed = GET_MODE_SIZE (move_mode);
+ }
+ tmp = expand_simple_binop (Pmode, PLUS, destreg, loop_iter, destreg,
+ true, OPTAB_LIB_WIDEN);
+ if (tmp != destreg)
+ emit_move_insn (destreg, tmp);
+
+ tmp = expand_simple_binop (Pmode, PLUS, srcreg, loop_iter, srcreg,
+ true, OPTAB_LIB_WIDEN);
+ if (tmp != srcreg)
+ emit_move_insn (srcreg, tmp);
+ }
if (count_exp != const0_rtx && epilogue_size_needed > 1)
expand_movmem_epilogue (dst, src, destreg, srcreg, count_exp,
epilogue_size_needed);
+
if (jump_around_label)
emit_label (jump_around_label);
return true;
rtx tmp;
int nops = mode == DImode ? 3 : 2;
+ if (VECTOR_MODE_P (mode))
+ {
+ enum machine_mode inner = GET_MODE_INNER (mode);
+ rtx promoted_val, vec_reg;
+ if (CONST_INT_P (val))
+ return ix86_build_const_vector (mode, true, val);
+
+ promoted_val = promote_duplicated_reg (inner, val);
+ vec_reg = gen_reg_rtx (mode);
+ switch (mode)
+ {
+ case V2DImode:
+ emit_insn (gen_vec_dupv2di (vec_reg, promoted_val));
+ break;
+ case V4SImode:
+ emit_insn (gen_vec_dupv4si (vec_reg, promoted_val));
+ break;
+ default:
+ gcc_unreachable ();
+ break;
+ }
+
+ return vec_reg;
+ }
gcc_assert (mode == SImode || mode == DImode);
+ if (mode == DImode && !TARGET_64BIT)
+ {
+ rtx vec_reg = promote_duplicated_reg (V4SImode, val);
+ vec_reg = convert_to_mode (V2DImode, vec_reg, 1);
+ return vec_reg;
+ }
if (val == const0_rtx)
return copy_to_mode_reg (mode, const0_rtx);
if (CONST_INT_P (val))
static rtx
promote_duplicated_reg_to_size (rtx val, int size_needed, int desired_align, int align)
{
- rtx promoted_val;
+ rtx promoted_val = NULL_RTX;
- if (TARGET_64BIT
- && (size_needed > 4 || (desired_align > align && desired_align > 4)))
- promoted_val = promote_duplicated_reg (DImode, val);
+ if (size_needed > 8)
+ {
+ /* We want to promote to vector register, so we expect that at least SSE
+ is available. */
+ gcc_assert (TARGET_SSE);
+
+ /* In case of promotion to vector register, we expect that val is a
+ constant or already promoted to GPR value. */
+ gcc_assert (GET_MODE (val) == Pmode || CONSTANT_P (val));
+ if (TARGET_64BIT)
+ promoted_val = promote_duplicated_reg (V2DImode, val);
+ else
+ promoted_val = promote_duplicated_reg (V4SImode, val);
+ }
+ else if (size_needed > 4)
+ {
+ gcc_assert (TARGET_64BIT);
+ promoted_val = promote_duplicated_reg (DImode, val);
+ }
else if (size_needed > 2 || (desired_align > align && desired_align > 2))
promoted_val = promote_duplicated_reg (SImode, val);
else if (size_needed > 1 || (desired_align > align && desired_align > 1))
int size_needed = 0, epilogue_size_needed;
int desired_align = 0, align_bytes = 0;
enum stringop_alg alg;
- rtx promoted_val = NULL;
- bool force_loopy_epilogue = false;
+ rtx gpr_promoted_val = NULL;
+ rtx vec_promoted_val = NULL;
int dynamic_check;
bool need_zero_guard = false;
+ bool align_unknown;
+ unsigned int unroll_factor;
+ enum machine_mode move_mode;
+ rtx loop_iter = NULL_RTX;
+ bool early_jump = false;
if (CONST_INT_P (align_exp))
align = INTVAL (align_exp);
/* Step 0: Decide on preferred algorithm, desired alignment and
size of chunks to be copied by main loop. */
- alg = decide_alg (count, expected_size, true, &dynamic_check);
+ align_unknown = !(CONST_INT_P (align_exp) && INTVAL (align_exp) > 0);
+ alg = decide_alg (count, expected_size, true, &dynamic_check, align_unknown);
desired_align = decide_alignment (align, alg, expected_size);
+ unroll_factor = 1;
+ move_mode = Pmode;
if (!TARGET_ALIGN_STRINGOPS)
align = desired_align;
gcc_unreachable ();
case loop:
need_zero_guard = true;
- size_needed = GET_MODE_SIZE (Pmode);
+ move_mode = Pmode;
+ size_needed = GET_MODE_SIZE (move_mode) * unroll_factor;
break;
case unrolled_loop:
need_zero_guard = true;
- size_needed = GET_MODE_SIZE (Pmode) * 4;
+ move_mode = Pmode;
+ unroll_factor = 1;
+ /* Select maximal available 1,2 or 4 unroll factor. */
+ if (!count)
+ unroll_factor = 4;
+ else
+ while (GET_MODE_SIZE (move_mode) * unroll_factor * 2 < count
+ && unroll_factor < 4)
+ unroll_factor *= 2;
+ size_needed = GET_MODE_SIZE (move_mode) * unroll_factor;
+ break;
+ case sse_loop:
+ need_zero_guard = true;
+ move_mode = TARGET_64BIT ? V2DImode : V4SImode;
+ unroll_factor = 1;
+ /* Select maximal available 1,2 or 4 unroll factor. */
+ if (!count)
+ unroll_factor = 4;
+ else
+ while (GET_MODE_SIZE (move_mode) * unroll_factor * 2 < count
+ && unroll_factor < 4)
+ unroll_factor *= 2;
+ size_needed = GET_MODE_SIZE (move_mode) * unroll_factor;
break;
case rep_prefix_8_byte:
size_needed = 8;
main loop and epilogue (ie one load of the big constant in the
front of all code. */
if (CONST_INT_P (val_exp))
- promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
- desired_align, align);
+ gpr_promoted_val = promote_duplicated_reg_to_size (val_exp,
+ GET_MODE_SIZE (Pmode),
+ GET_MODE_SIZE (Pmode),
+ align);
/* Ensure that alignment prologue won't copy past end of block. */
if (size_needed > 1 || (desired_align > 1 && desired_align > align))
{
Make sure it is power of 2. */
epilogue_size_needed = smallest_pow2_greater_than (epilogue_size_needed);
- /* To improve performance of small blocks, we jump around the VAL
- promoting mode. This mean that if the promoted VAL is not constant,
- we might not use it in the epilogue and have to use byte
- loop variant. */
- if (epilogue_size_needed > 2 && !promoted_val)
- force_loopy_epilogue = true;
if (count)
{
if (count < (unsigned HOST_WIDE_INT)epilogue_size_needed)
}
else
{
+ /* SSE and unrolled_lopo algs re-use iteration counter in the epilogue. */
+ if (alg == sse_loop || alg == unrolled_loop)
+ {
+ loop_iter = gen_reg_rtx (counter_mode (count_exp));
+ emit_move_insn (loop_iter, const0_rtx);
+ }
label = gen_label_rtx ();
+ early_jump = true;
emit_cmp_and_jump_insns (count_exp,
GEN_INT (epilogue_size_needed),
LTU, 0, counter_mode (count_exp), 1, label);
/* Step 2: Alignment prologue. */
/* Do the expensive promotion once we branched off the small blocks. */
- if (!promoted_val)
- promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
- desired_align, align);
+ if (!gpr_promoted_val)
+ gpr_promoted_val = promote_duplicated_reg_to_size (val_exp,
+ GET_MODE_SIZE (Pmode),
+ GET_MODE_SIZE (Pmode),
+ align);
gcc_assert (desired_align >= 1 && align >= 1);
if (desired_align > align)
the pain to maintain it for the first move, so throw away
the info early. */
dst = change_address (dst, BLKmode, destreg);
- expand_setmem_prologue (dst, destreg, promoted_val, count_exp, align,
+ expand_setmem_prologue (dst, destreg, gpr_promoted_val, count_exp, align,
desired_align);
+ set_mem_align (dst, desired_align*BITS_PER_UNIT);
}
else
{
/* If we know how many bytes need to be stored before dst is
sufficiently aligned, maintain aliasing info accurately. */
- dst = expand_constant_setmem_prologue (dst, destreg, promoted_val,
+ dst = expand_constant_setmem_prologue (dst, destreg, gpr_promoted_val,
desired_align, align_bytes);
count_exp = plus_constant (count_exp, -align_bytes);
count -= align_bytes;
+ if (count < (unsigned HOST_WIDE_INT) size_needed)
+ goto epilogue;
}
if (need_zero_guard
&& (count < (unsigned HOST_WIDE_INT) size_needed
emit_label (label);
LABEL_NUSES (label) = 1;
label = NULL;
- promoted_val = val_exp;
+ gpr_promoted_val = val_exp;
epilogue_size_needed = 1;
}
else if (label == NULL_RTX)
case no_stringop:
gcc_unreachable ();
case loop_1_byte:
- expand_set_or_movmem_via_loop (dst, NULL, destreg, NULL, promoted_val,
+ expand_set_or_movmem_via_loop (dst, NULL, destreg, NULL, val_exp,
count_exp, QImode, 1, expected_size);
break;
case loop:
- expand_set_or_movmem_via_loop (dst, NULL, destreg, NULL, promoted_val,
+ expand_set_or_movmem_via_loop (dst, NULL, destreg, NULL, gpr_promoted_val,
count_exp, Pmode, 1, expected_size);
break;
case unrolled_loop:
- expand_set_or_movmem_via_loop (dst, NULL, destreg, NULL, promoted_val,
- count_exp, Pmode, 4, expected_size);
+ loop_iter = expand_set_or_movmem_via_loop_with_iter (dst, NULL, destreg,
+ NULL, gpr_promoted_val, count_exp,
+ loop_iter, move_mode, unroll_factor,
+ expected_size, false);
+ break;
+ case sse_loop:
+ vec_promoted_val =
+ promote_duplicated_reg_to_size (gpr_promoted_val,
+ GET_MODE_SIZE (move_mode),
+ GET_MODE_SIZE (move_mode), align);
+ loop_iter = expand_set_or_movmem_via_loop_with_iter (dst, NULL, destreg,
+ NULL, vec_promoted_val, count_exp,
+ loop_iter, move_mode, unroll_factor,
+ expected_size, false);
break;
case rep_prefix_8_byte:
- expand_setmem_via_rep_stos (dst, destreg, promoted_val, count_exp,
+ gcc_assert (TARGET_64BIT);
+ expand_setmem_via_rep_stos (dst, destreg, gpr_promoted_val, count_exp,
DImode, val_exp);
break;
case rep_prefix_4_byte:
- expand_setmem_via_rep_stos (dst, destreg, promoted_val, count_exp,
+ expand_setmem_via_rep_stos (dst, destreg, gpr_promoted_val, count_exp,
SImode, val_exp);
break;
case rep_prefix_1_byte:
- expand_setmem_via_rep_stos (dst, destreg, promoted_val, count_exp,
+ expand_setmem_via_rep_stos (dst, destreg, gpr_promoted_val, count_exp,
QImode, val_exp);
break;
}
}
emit_label (label);
LABEL_NUSES (label) = 1;
+ /* We can not rely on fact that promoved value is known. */
+ vec_promoted_val = 0;
+ if (early_jump)
+ gpr_promoted_val = 0;
}
epilogue:
- if (count_exp != const0_rtx && epilogue_size_needed > 1)
+ if (alg == unrolled_loop || alg == sse_loop)
{
- if (force_loopy_epilogue)
- expand_setmem_epilogue_via_loop (dst, destreg, val_exp, count_exp,
- epilogue_size_needed);
- else
- expand_setmem_epilogue (dst, destreg, promoted_val, count_exp,
- epilogue_size_needed);
+ rtx tmp;
+ int remainder_size = epilogue_size_needed;
+ if (count && desired_align <= align)
+ remainder_size = count % epilogue_size_needed;
+ /* We may not need the epilgoue loop at all when the count is known
+ and alignment is not adjusted. */
+ if (remainder_size > 31
+ && (alg == sse_loop ? vec_promoted_val : gpr_promoted_val))
+ {
+ /* Reduce epilogue's size by creating not-unrolled loop. If we won't
+ do this, we can have very big epilogue - when alignment is statically
+ unknown we'll have the epilogue byte by byte which may be very slow. */
+ loop_iter = expand_set_or_movmem_via_loop_with_iter (dst, NULL, destreg,
+ NULL, (alg == sse_loop ? vec_promoted_val : gpr_promoted_val), count_exp,
+ loop_iter, move_mode, 1,
+ expected_size, false);
+ dst = change_address (dst, BLKmode, destreg);
+ epilogue_size_needed = GET_MODE_SIZE (move_mode);
+ }
+ tmp = expand_simple_binop (Pmode, PLUS, destreg, loop_iter, destreg,
+ true, OPTAB_LIB_WIDEN);
+ if (tmp != destreg)
+ emit_move_insn (destreg, tmp);
}
+ if (count_exp == const0_rtx || epilogue_size_needed <= 1)
+ ;
+ else if (!gpr_promoted_val)
+ expand_setmem_epilogue_via_loop (dst, destreg, val_exp, count_exp,
+ epilogue_size_needed);
+ else
+ expand_setmem_epilogue (dst, destreg, vec_promoted_val, gpr_promoted_val,
+ val_exp, count_exp, epilogue_size_needed);
if (jump_around_label)
emit_label (jump_around_label);
return true;
IX86_BUILTIN_PMULDQ128,
IX86_BUILTIN_PMULLD128,
- IX86_BUILTIN_ROUNDPD,
- IX86_BUILTIN_ROUNDPS,
IX86_BUILTIN_ROUNDSD,
IX86_BUILTIN_ROUNDSS,
+ IX86_BUILTIN_ROUNDPD,
+ IX86_BUILTIN_ROUNDPS,
+
IX86_BUILTIN_FLOORPD,
IX86_BUILTIN_CEILPD,
IX86_BUILTIN_TRUNCPD,
IX86_BUILTIN_RINTPD,
IX86_BUILTIN_ROUNDPD_AZ,
+
+ IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX,
+ IX86_BUILTIN_CEILPD_VEC_PACK_SFIX,
+ IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX,
+
IX86_BUILTIN_FLOORPS,
IX86_BUILTIN_CEILPS,
IX86_BUILTIN_TRUNCPS,
IX86_BUILTIN_RINTPS,
IX86_BUILTIN_ROUNDPS_AZ,
+ IX86_BUILTIN_FLOORPS_SFIX,
+ IX86_BUILTIN_CEILPS_SFIX,
+ IX86_BUILTIN_ROUNDPS_AZ_SFIX,
+
IX86_BUILTIN_PTESTZ,
IX86_BUILTIN_PTESTC,
IX86_BUILTIN_PTESTNZC,
IX86_BUILTIN_VEC_SET_V16QI,
IX86_BUILTIN_VEC_PACK_SFIX,
+ IX86_BUILTIN_VEC_PACK_SFIX256,
/* SSE4.2. */
IX86_BUILTIN_CRC32QI,
IX86_BUILTIN_TRUNCPD256,
IX86_BUILTIN_RINTPD256,
IX86_BUILTIN_ROUNDPD_AZ256,
+
+ IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256,
+ IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256,
+ IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256,
+
IX86_BUILTIN_FLOORPS256,
IX86_BUILTIN_CEILPS256,
IX86_BUILTIN_TRUNCPS256,
IX86_BUILTIN_RINTPS256,
IX86_BUILTIN_ROUNDPS_AZ256,
+ IX86_BUILTIN_FLOORPS_SFIX256,
+ IX86_BUILTIN_CEILPS_SFIX256,
+ IX86_BUILTIN_ROUNDPS_AZ_SFIX256,
+
IX86_BUILTIN_UNPCKHPD256,
IX86_BUILTIN_UNPCKLPD256,
IX86_BUILTIN_UNPCKHPS256,
IX86_BUILTIN_GATHERDIV4SI,
IX86_BUILTIN_GATHERDIV8SI,
+ /* Alternate 4 element gather for the vectorizer where
+ all operands are 32-byte wide. */
+ IX86_BUILTIN_GATHERALTSIV4DF,
+ IX86_BUILTIN_GATHERALTDIV8SF,
+ IX86_BUILTIN_GATHERALTSIV4DI,
+ IX86_BUILTIN_GATHERALTDIV8SI,
+
/* TFmode support builtins. */
IX86_BUILTIN_INFQ,
IX86_BUILTIN_HUGE_VALQ,
IX86_BUILTIN_CPYSGNPS256,
IX86_BUILTIN_CPYSGNPD256,
- IX86_BUILTIN_CVTUDQ2PS,
-
- IX86_BUILTIN_VEC_PERM_V2DF,
- IX86_BUILTIN_VEC_PERM_V4SF,
- IX86_BUILTIN_VEC_PERM_V2DI,
- IX86_BUILTIN_VEC_PERM_V4SI,
- IX86_BUILTIN_VEC_PERM_V8HI,
- IX86_BUILTIN_VEC_PERM_V16QI,
- IX86_BUILTIN_VEC_PERM_V2DI_U,
- IX86_BUILTIN_VEC_PERM_V4SI_U,
- IX86_BUILTIN_VEC_PERM_V8HI_U,
- IX86_BUILTIN_VEC_PERM_V16QI_U,
- IX86_BUILTIN_VEC_PERM_V4DF,
- IX86_BUILTIN_VEC_PERM_V8SF,
-
/* FMA4 instructions. */
IX86_BUILTIN_VFMADDSS,
IX86_BUILTIN_VFMADDSD,
/* SSE2 */
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_shufpd, "__builtin_ia32_shufpd", IX86_BUILTIN_SHUFPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
- { OPTION_MASK_ISA_SSE2, CODE_FOR_nothing, "__builtin_ia32_vec_perm_v2df", IX86_BUILTIN_VEC_PERM_V2DF, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DI },
- { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin_ia32_vec_perm_v4sf", IX86_BUILTIN_VEC_PERM_V4SF, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SI },
- { OPTION_MASK_ISA_SSE2, CODE_FOR_nothing, "__builtin_ia32_vec_perm_v2di", IX86_BUILTIN_VEC_PERM_V2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_V2DI },
- { OPTION_MASK_ISA_SSE2, CODE_FOR_nothing, "__builtin_ia32_vec_perm_v4si", IX86_BUILTIN_VEC_PERM_V4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_V4SI },
- { OPTION_MASK_ISA_SSE2, CODE_FOR_nothing, "__builtin_ia32_vec_perm_v8hi", IX86_BUILTIN_VEC_PERM_V8HI, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_V8HI },
- { OPTION_MASK_ISA_SSE2, CODE_FOR_nothing, "__builtin_ia32_vec_perm_v16qi", IX86_BUILTIN_VEC_PERM_V16QI, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI },
- { OPTION_MASK_ISA_SSE2, CODE_FOR_nothing, "__builtin_ia32_vec_perm_v2di_u", IX86_BUILTIN_VEC_PERM_V2DI_U, UNKNOWN, (int) V2UDI_FTYPE_V2UDI_V2UDI_V2UDI },
- { OPTION_MASK_ISA_SSE2, CODE_FOR_nothing, "__builtin_ia32_vec_perm_v4si_u", IX86_BUILTIN_VEC_PERM_V4SI_U, UNKNOWN, (int) V4USI_FTYPE_V4USI_V4USI_V4USI },
- { OPTION_MASK_ISA_SSE2, CODE_FOR_nothing, "__builtin_ia32_vec_perm_v8hi_u", IX86_BUILTIN_VEC_PERM_V8HI_U, UNKNOWN, (int) V8UHI_FTYPE_V8UHI_V8UHI_V8UHI },
- { OPTION_MASK_ISA_SSE2, CODE_FOR_nothing, "__builtin_ia32_vec_perm_v16qi_u", IX86_BUILTIN_VEC_PERM_V16QI_U, UNKNOWN, (int) V16UQI_FTYPE_V16UQI_V16UQI_V16UQI },
- { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin_ia32_vec_perm_v4df", IX86_BUILTIN_VEC_PERM_V4DF, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DI },
- { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin_ia32_vec_perm_v8sf", IX86_BUILTIN_VEC_PERM_V8SF, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SI },
-
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movmskpd, "__builtin_ia32_movmskpd", IX86_BUILTIN_MOVMSKPD, UNKNOWN, (int) INT_FTYPE_V2DF },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmovmskb, "__builtin_ia32_pmovmskb128", IX86_BUILTIN_PMOVMSKB128, UNKNOWN, (int) INT_FTYPE_V16QI },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sqrtv2df2, "__builtin_ia32_sqrtpd", IX86_BUILTIN_SQRTPD, UNKNOWN, (int) V2DF_FTYPE_V2DF },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtdq2pd, "__builtin_ia32_cvtdq2pd", IX86_BUILTIN_CVTDQ2PD, UNKNOWN, (int) V2DF_FTYPE_V4SI },
- { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtdq2ps, "__builtin_ia32_cvtdq2ps", IX86_BUILTIN_CVTDQ2PS, UNKNOWN, (int) V4SF_FTYPE_V4SI },
- { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtudq2ps, "__builtin_ia32_cvtudq2ps", IX86_BUILTIN_CVTUDQ2PS, UNKNOWN, (int) V4SF_FTYPE_V4SI },
+ { OPTION_MASK_ISA_SSE2, CODE_FOR_floatv4siv4sf2, "__builtin_ia32_cvtdq2ps", IX86_BUILTIN_CVTDQ2PS, UNKNOWN, (int) V4SF_FTYPE_V4SI },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2dq, "__builtin_ia32_cvtpd2dq", IX86_BUILTIN_CVTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2pi, "__builtin_ia32_cvtpd2pi", IX86_BUILTIN_CVTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2dq, "__builtin_ia32_cvtps2dq", IX86_BUILTIN_CVTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2pd, "__builtin_ia32_cvtps2pd", IX86_BUILTIN_CVTPS2PD, UNKNOWN, (int) V2DF_FTYPE_V4SF },
- { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttps2dq, "__builtin_ia32_cvttps2dq", IX86_BUILTIN_CVTTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
+ { OPTION_MASK_ISA_SSE2, CODE_FOR_fix_truncv4sfv4si2, "__builtin_ia32_cvttps2dq", IX86_BUILTIN_CVTTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_addv2df3, "__builtin_ia32_addpd", IX86_BUILTIN_ADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
{ OPTION_MASK_ISA_SSE2, CODE_FOR_subv2df3, "__builtin_ia32_subpd", IX86_BUILTIN_SUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
{ OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_truncpd", IX86_BUILTIN_TRUNCPD, (enum rtx_code) ROUND_TRUNC, (int) V2DF_FTYPE_V2DF_ROUND },
{ OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_rintpd", IX86_BUILTIN_RINTPD, (enum rtx_code) ROUND_MXCSR, (int) V2DF_FTYPE_V2DF_ROUND },
+ { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__builtin_ia32_floorpd_vec_pack_sfix", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX, (enum rtx_code) ROUND_FLOOR, (int) V4SI_FTYPE_V2DF_V2DF_ROUND },
+ { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd_vec_pack_sfix, "__builtin_ia32_ceilpd_vec_pack_sfix", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX, (enum rtx_code) ROUND_CEIL, (int) V4SI_FTYPE_V2DF_V2DF_ROUND },
+
{ OPTION_MASK_ISA_ROUND, CODE_FOR_roundv2df2, "__builtin_ia32_roundpd_az", IX86_BUILTIN_ROUNDPD_AZ, UNKNOWN, (int) V2DF_FTYPE_V2DF },
+ { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv2df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
{ OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_floorps", IX86_BUILTIN_FLOORPS, (enum rtx_code) ROUND_FLOOR, (int) V4SF_FTYPE_V4SF_ROUND },
{ OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_ceilps", IX86_BUILTIN_CEILPS, (enum rtx_code) ROUND_CEIL, (int) V4SF_FTYPE_V4SF_ROUND },
{ OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_truncps", IX86_BUILTIN_TRUNCPS, (enum rtx_code) ROUND_TRUNC, (int) V4SF_FTYPE_V4SF_ROUND },
{ OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_rintps", IX86_BUILTIN_RINTPS, (enum rtx_code) ROUND_MXCSR, (int) V4SF_FTYPE_V4SF_ROUND },
+ { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps_sfix, "__builtin_ia32_floorps_sfix", IX86_BUILTIN_FLOORPS_SFIX, (enum rtx_code) ROUND_FLOOR, (int) V4SI_FTYPE_V4SF_ROUND },
+ { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps_sfix, "__builtin_ia32_ceilps_sfix", IX86_BUILTIN_CEILPS_SFIX, (enum rtx_code) ROUND_CEIL, (int) V4SI_FTYPE_V4SF_ROUND },
+
{ OPTION_MASK_ISA_ROUND, CODE_FOR_roundv4sf2, "__builtin_ia32_roundps_az", IX86_BUILTIN_ROUNDPS_AZ, UNKNOWN, (int) V4SF_FTYPE_V4SF },
+ { OPTION_MASK_ISA_ROUND, CODE_FOR_roundv4sf2_sfix, "__builtin_ia32_roundps_az_sfix", IX86_BUILTIN_ROUNDPS_AZ_SFIX, UNKNOWN, (int) V4SI_FTYPE_V4SF },
{ OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestz128", IX86_BUILTIN_PTESTZ, EQ, (int) INT_FTYPE_V2DI_V2DI_PTEST },
{ OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestc128", IX86_BUILTIN_PTESTC, LTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v4df, "__builtin_ia32_vextractf128_pd256", IX86_BUILTIN_EXTRACTF128PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF_INT },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8sf, "__builtin_ia32_vextractf128_ps256", IX86_BUILTIN_EXTRACTF128PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF_INT },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8si, "__builtin_ia32_vextractf128_si256", IX86_BUILTIN_EXTRACTF128SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI_INT },
- { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtdq2pd256, "__builtin_ia32_cvtdq2pd256", IX86_BUILTIN_CVTDQ2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SI },
- { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtdq2ps256, "__builtin_ia32_cvtdq2ps256", IX86_BUILTIN_CVTDQ2PS256, UNKNOWN, (int) V8SF_FTYPE_V8SI },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_floatv4siv4df2, "__builtin_ia32_cvtdq2pd256", IX86_BUILTIN_CVTDQ2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SI },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_floatv8siv8sf2, "__builtin_ia32_cvtdq2ps256", IX86_BUILTIN_CVTDQ2PS256, UNKNOWN, (int) V8SF_FTYPE_V8SI },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2ps256, "__builtin_ia32_cvtpd2ps256", IX86_BUILTIN_CVTPD2PS256, UNKNOWN, (int) V4SF_FTYPE_V4DF },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2dq256, "__builtin_ia32_cvtps2dq256", IX86_BUILTIN_CVTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2pd256, "__builtin_ia32_cvtps2pd256", IX86_BUILTIN_CVTPS2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SF },
- { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvttpd2dq256, "__builtin_ia32_cvttpd2dq256", IX86_BUILTIN_CVTTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv4dfv4si2, "__builtin_ia32_cvttpd2dq256", IX86_BUILTIN_CVTTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2dq256, "__builtin_ia32_cvtpd2dq256", IX86_BUILTIN_CVTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
- { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvttps2dq256, "__builtin_ia32_cvttps2dq256", IX86_BUILTIN_CVTTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_fix_truncv8sfv8si2, "__builtin_ia32_cvttps2dq256", IX86_BUILTIN_CVTTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v4df3, "__builtin_ia32_vperm2f128_pd256", IX86_BUILTIN_VPERM2F128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8sf3, "__builtin_ia32_vperm2f128_ps256", IX86_BUILTIN_VPERM2F128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8si3, "__builtin_ia32_vperm2f128_si256", IX86_BUILTIN_VPERM2F128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_rintpd256", IX86_BUILTIN_RINTPD256, (enum rtx_code) ROUND_MXCSR, (int) V4DF_FTYPE_V4DF_ROUND },
{ OPTION_MASK_ISA_AVX, CODE_FOR_roundv4df2, "__builtin_ia32_roundpd_az256", IX86_BUILTIN_ROUNDPD_AZ256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_roundv4df2_vec_pack_sfix, "__builtin_ia32_roundpd_az_vec_pack_sfix256", IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V4DF_V4DF },
+
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_floorpd_vec_pack_sfix256", IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V4DF_V4DF_ROUND },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd_vec_pack_sfix256, "__builtin_ia32_ceilpd_vec_pack_sfix256", IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V4DF_V4DF_ROUND },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_floorps256", IX86_BUILTIN_FLOORPS256, (enum rtx_code) ROUND_FLOOR, (int) V8SF_FTYPE_V8SF_ROUND },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_ceilps256", IX86_BUILTIN_CEILPS256, (enum rtx_code) ROUND_CEIL, (int) V8SF_FTYPE_V8SF_ROUND },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_truncps256", IX86_BUILTIN_TRUNCPS256, (enum rtx_code) ROUND_TRUNC, (int) V8SF_FTYPE_V8SF_ROUND },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_rintps256", IX86_BUILTIN_RINTPS256, (enum rtx_code) ROUND_MXCSR, (int) V8SF_FTYPE_V8SF_ROUND },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps_sfix256, "__builtin_ia32_floorps_sfix256", IX86_BUILTIN_FLOORPS_SFIX256, (enum rtx_code) ROUND_FLOOR, (int) V8SI_FTYPE_V8SF_ROUND },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps_sfix256, "__builtin_ia32_ceilps_sfix256", IX86_BUILTIN_CEILPS_SFIX256, (enum rtx_code) ROUND_CEIL, (int) V8SI_FTYPE_V8SF_ROUND },
+
{ OPTION_MASK_ISA_AVX, CODE_FOR_roundv8sf2, "__builtin_ia32_roundps_az256", IX86_BUILTIN_ROUNDPS_AZ256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_roundv8sf2_sfix, "__builtin_ia32_roundps_az_sfix256", IX86_BUILTIN_ROUNDPS_AZ_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhpd256, "__builtin_ia32_unpckhpd256", IX86_BUILTIN_UNPCKHPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
{ OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklpd256, "__builtin_ia32_unpcklpd256", IX86_BUILTIN_UNPCKLPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
{ OPTION_MASK_ISA_AVX, CODE_FOR_copysignv8sf3, "__builtin_ia32_copysignps256", IX86_BUILTIN_CPYSGNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
{ OPTION_MASK_ISA_AVX, CODE_FOR_copysignv4df3, "__builtin_ia32_copysignpd256", IX86_BUILTIN_CPYSGNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_vec_pack_sfix_v4df, "__builtin_ia32_vec_pack_sfix256 ", IX86_BUILTIN_VEC_PACK_SFIX256, UNKNOWN, (int) V8SI_FTYPE_V4DF_V4DF },
+
/* AVX2 */
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_mpsadbw, "__builtin_ia32_mpsadbw256", IX86_BUILTIN_MPSADBW256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI_INT },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_absv32qi2, "__builtin_ia32_pabsb256", IX86_BUILTIN_PABSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ssaddv16hi3, "__builtin_ia32_paddsw256", IX86_BUILTIN_PADDSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_usaddv32qi3, "__builtin_ia32_paddusb256", IX86_BUILTIN_PADDUSB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_usaddv16hi3, "__builtin_ia32_paddusw256", IX86_BUILTIN_PADDUSW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
- { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_palignrv4di, "__builtin_ia32_palignr256", IX86_BUILTIN_PALIGNR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT_CONVERT },
+ { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_palignrv2ti, "__builtin_ia32_palignr256", IX86_BUILTIN_PALIGNR256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT_CONVERT },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_andv4di3, "__builtin_ia32_andsi256", IX86_BUILTIN_AND256I, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_andnotv4di3, "__builtin_ia32_andnotsi256", IX86_BUILTIN_ANDNOT256I, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_uavgv32qi3, "__builtin_ia32_pavgb256", IX86_BUILTIN_PAVGB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv32qi3, "__builtin_ia32_psignb256", IX86_BUILTIN_PSIGNB256, UNKNOWN, (int) V32QI_FTYPE_V32QI_V32QI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv16hi3, "__builtin_ia32_psignw256", IX86_BUILTIN_PSIGNW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V16HI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_psignv8si3 , "__builtin_ia32_psignd256", IX86_BUILTIN_PSIGND256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
- { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshlqv4di3, "__builtin_ia32_pslldqi256", IX86_BUILTIN_PSLLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT },
- { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshlv16hi3, "__builtin_ia32_psllwi256", IX86_BUILTIN_PSLLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
- { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshlv16hi3, "__builtin_ia32_psllw256", IX86_BUILTIN_PSLLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
- { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshlv8si3, "__builtin_ia32_pslldi256", IX86_BUILTIN_PSLLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
- { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshlv8si3, "__builtin_ia32_pslld256", IX86_BUILTIN_PSLLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
- { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshlv4di3, "__builtin_ia32_psllqi256", IX86_BUILTIN_PSLLQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_COUNT },
- { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshlv4di3, "__builtin_ia32_psllq256", IX86_BUILTIN_PSLLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_COUNT },
+ { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlv2ti3, "__builtin_ia32_pslldqi256", IX86_BUILTIN_PSLLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_CONVERT },
+ { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv16hi3, "__builtin_ia32_psllwi256", IX86_BUILTIN_PSLLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
+ { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv16hi3, "__builtin_ia32_psllw256", IX86_BUILTIN_PSLLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
+ { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv8si3, "__builtin_ia32_pslldi256", IX86_BUILTIN_PSLLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
+ { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv8si3, "__builtin_ia32_pslld256", IX86_BUILTIN_PSLLD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
+ { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv4di3, "__builtin_ia32_psllqi256", IX86_BUILTIN_PSLLQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_COUNT },
+ { OPTION_MASK_ISA_AVX2, CODE_FOR_ashlv4di3, "__builtin_ia32_psllq256", IX86_BUILTIN_PSLLQ256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_COUNT },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv16hi3, "__builtin_ia32_psrawi256", IX86_BUILTIN_PSRAWI256, UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv16hi3, "__builtin_ia32_psraw256", IX86_BUILTIN_PSRAW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv8si3, "__builtin_ia32_psradi256", IX86_BUILTIN_PSRADI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_ashrv8si3, "__builtin_ia32_psrad256", IX86_BUILTIN_PSRAD256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_COUNT },
- { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrqv4di3, "__builtin_ia32_psrldqi256", IX86_BUILTIN_PSRLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT },
+ { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrv2ti3, "__builtin_ia32_psrldqi256", IX86_BUILTIN_PSRLDQI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_INT_CONVERT },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv16hi3, "__builtin_ia32_psrlwi256", IX86_BUILTIN_PSRLWI256 , UNKNOWN, (int) V16HI_FTYPE_V16HI_SI_COUNT },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv16hi3, "__builtin_ia32_psrlw256", IX86_BUILTIN_PSRLW256, UNKNOWN, (int) V16HI_FTYPE_V16HI_V8HI_COUNT },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_lshrv8si3, "__builtin_ia32_psrldi256", IX86_BUILTIN_PSRLDI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_SI_COUNT },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_permv2ti, "__builtin_ia32_permti256", IX86_BUILTIN_VPERMTI256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI_INT },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_extracti128, "__builtin_ia32_extract128i256", IX86_BUILTIN_VEXTRACT128I256, UNKNOWN, (int) V2DI_FTYPE_V4DI_INT },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_inserti128, "__builtin_ia32_insert128i256", IX86_BUILTIN_VINSERT128I256, UNKNOWN, (int) V4DI_FTYPE_V4DI_V2DI_INT },
- { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshlvv4di, "__builtin_ia32_psllv4di", IX86_BUILTIN_PSLLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
- { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshlvv2di, "__builtin_ia32_psllv2di", IX86_BUILTIN_PSLLVV2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
- { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshlvv8si, "__builtin_ia32_psllv8si", IX86_BUILTIN_PSLLVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
- { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshlvv4si, "__builtin_ia32_psllv4si", IX86_BUILTIN_PSLLVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
+ { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv4di, "__builtin_ia32_psllv4di", IX86_BUILTIN_PSLLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
+ { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv2di, "__builtin_ia32_psllv2di", IX86_BUILTIN_PSLLVV2DI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
+ { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv8si, "__builtin_ia32_psllv8si", IX86_BUILTIN_PSLLVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
+ { OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashlvv4si, "__builtin_ia32_psllv4si", IX86_BUILTIN_PSLLVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashrvv8si, "__builtin_ia32_psrav8si", IX86_BUILTIN_PSRAVV8SI, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_ashrvv4si, "__builtin_ia32_psrav4si", IX86_BUILTIN_PSRAVV4SI, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
{ OPTION_MASK_ISA_AVX2, CODE_FOR_avx2_lshrvv4di, "__builtin_ia32_psrlv4di", IX86_BUILTIN_PSRLVV4DI, UNKNOWN, (int) V4DI_FTYPE_V4DI_V4DI },
{ OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv4si3, "__builtin_ia32_vprotdi", IX86_BUILTIN_VPROTD_IMM, UNKNOWN, (int)MULTI_ARG_2_SI_IMM },
{ OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv8hi3, "__builtin_ia32_vprotwi", IX86_BUILTIN_VPROTW_IMM, UNKNOWN, (int)MULTI_ARG_2_HI_IMM },
{ OPTION_MASK_ISA_XOP, CODE_FOR_xop_rotlv16qi3, "__builtin_ia32_vprotbi", IX86_BUILTIN_VPROTB_IMM, UNKNOWN, (int)MULTI_ARG_2_QI_IMM },
- { OPTION_MASK_ISA_XOP, CODE_FOR_xop_ashlv2di3, "__builtin_ia32_vpshaq", IX86_BUILTIN_VPSHAQ, UNKNOWN, (int)MULTI_ARG_2_DI },
- { OPTION_MASK_ISA_XOP, CODE_FOR_xop_ashlv4si3, "__builtin_ia32_vpshad", IX86_BUILTIN_VPSHAD, UNKNOWN, (int)MULTI_ARG_2_SI },
- { OPTION_MASK_ISA_XOP, CODE_FOR_xop_ashlv8hi3, "__builtin_ia32_vpshaw", IX86_BUILTIN_VPSHAW, UNKNOWN, (int)MULTI_ARG_2_HI },
- { OPTION_MASK_ISA_XOP, CODE_FOR_xop_ashlv16qi3, "__builtin_ia32_vpshab", IX86_BUILTIN_VPSHAB, UNKNOWN, (int)MULTI_ARG_2_QI },
- { OPTION_MASK_ISA_XOP, CODE_FOR_xop_lshlv2di3, "__builtin_ia32_vpshlq", IX86_BUILTIN_VPSHLQ, UNKNOWN, (int)MULTI_ARG_2_DI },
- { OPTION_MASK_ISA_XOP, CODE_FOR_xop_lshlv4si3, "__builtin_ia32_vpshld", IX86_BUILTIN_VPSHLD, UNKNOWN, (int)MULTI_ARG_2_SI },
- { OPTION_MASK_ISA_XOP, CODE_FOR_xop_lshlv8hi3, "__builtin_ia32_vpshlw", IX86_BUILTIN_VPSHLW, UNKNOWN, (int)MULTI_ARG_2_HI },
- { OPTION_MASK_ISA_XOP, CODE_FOR_xop_lshlv16qi3, "__builtin_ia32_vpshlb", IX86_BUILTIN_VPSHLB, UNKNOWN, (int)MULTI_ARG_2_QI },
+ { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav2di3, "__builtin_ia32_vpshaq", IX86_BUILTIN_VPSHAQ, UNKNOWN, (int)MULTI_ARG_2_DI },
+ { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav4si3, "__builtin_ia32_vpshad", IX86_BUILTIN_VPSHAD, UNKNOWN, (int)MULTI_ARG_2_SI },
+ { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav8hi3, "__builtin_ia32_vpshaw", IX86_BUILTIN_VPSHAW, UNKNOWN, (int)MULTI_ARG_2_HI },
+ { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shav16qi3, "__builtin_ia32_vpshab", IX86_BUILTIN_VPSHAB, UNKNOWN, (int)MULTI_ARG_2_QI },
+ { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv2di3, "__builtin_ia32_vpshlq", IX86_BUILTIN_VPSHLQ, UNKNOWN, (int)MULTI_ARG_2_DI },
+ { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv4si3, "__builtin_ia32_vpshld", IX86_BUILTIN_VPSHLD, UNKNOWN, (int)MULTI_ARG_2_SI },
+ { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv8hi3, "__builtin_ia32_vpshlw", IX86_BUILTIN_VPSHLW, UNKNOWN, (int)MULTI_ARG_2_HI },
+ { OPTION_MASK_ISA_XOP, CODE_FOR_xop_shlv16qi3, "__builtin_ia32_vpshlb", IX86_BUILTIN_VPSHLB, UNKNOWN, (int)MULTI_ARG_2_QI },
{ OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv4sf2, "__builtin_ia32_vfrczss", IX86_BUILTIN_VFRCZSS, UNKNOWN, (int)MULTI_ARG_2_SF },
{ OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv2df2, "__builtin_ia32_vfrczsd", IX86_BUILTIN_VFRCZSD, UNKNOWN, (int)MULTI_ARG_2_DF },
{ OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v8sf3, "__builtin_ia32_vpermil2ps256", IX86_BUILTIN_VPERMIL2PS256, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I1 },
};
+\f
+/* TM vector builtins. */
+
+/* Reuse the existing x86-specific `struct builtin_description' cause
+ we're lazy. Add casts to make them fit. */
+static const struct builtin_description bdesc_tm[] =
+{
+ { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WM64", (enum ix86_builtins) BUILT_IN_TM_STORE_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
+ { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WaRM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
+ { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_WaWM64", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M64, UNKNOWN, VOID_FTYPE_PV2SI_V2SI },
+ { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
+ { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RaRM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
+ { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RaWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
+ { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_RfWM64", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M64, UNKNOWN, V2SI_FTYPE_PCV2SI },
+
+ { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WM128", (enum ix86_builtins) BUILT_IN_TM_STORE_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
+ { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WaRM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
+ { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_WaWM128", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M128, UNKNOWN, VOID_FTYPE_PV4SF_V4SF },
+ { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
+ { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RaRM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
+ { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RaWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
+ { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_RfWM128", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M128, UNKNOWN, V4SF_FTYPE_PCV4SF },
+
+ { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WM256", (enum ix86_builtins) BUILT_IN_TM_STORE_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WaRM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAR_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_WaWM256", (enum ix86_builtins) BUILT_IN_TM_STORE_WAW_M256, UNKNOWN, VOID_FTYPE_PV8SF_V8SF },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RaRM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAR_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RaWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RAW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_RfWM256", (enum ix86_builtins) BUILT_IN_TM_LOAD_RFW_M256, UNKNOWN, V8SF_FTYPE_PCV8SF },
+
+ { OPTION_MASK_ISA_MMX, CODE_FOR_nothing, "__builtin__ITM_LM64", (enum ix86_builtins) BUILT_IN_TM_LOG_M64, UNKNOWN, VOID_FTYPE_PCVOID },
+ { OPTION_MASK_ISA_SSE, CODE_FOR_nothing, "__builtin__ITM_LM128", (enum ix86_builtins) BUILT_IN_TM_LOG_M128, UNKNOWN, VOID_FTYPE_PCVOID },
+ { OPTION_MASK_ISA_AVX, CODE_FOR_nothing, "__builtin__ITM_LM256", (enum ix86_builtins) BUILT_IN_TM_LOG_M256, UNKNOWN, VOID_FTYPE_PCVOID },
+};
+
+/* TM callbacks. */
+
+/* Return the builtin decl needed to load a vector of TYPE. */
+
+static tree
+ix86_builtin_tm_load (tree type)
+{
+ if (TREE_CODE (type) == VECTOR_TYPE)
+ {
+ switch (tree_low_cst (TYPE_SIZE (type), 1))
+ {
+ case 64:
+ return builtin_decl_explicit (BUILT_IN_TM_LOAD_M64);
+ case 128:
+ return builtin_decl_explicit (BUILT_IN_TM_LOAD_M128);
+ case 256:
+ return builtin_decl_explicit (BUILT_IN_TM_LOAD_M256);
+ }
+ }
+ return NULL_TREE;
+}
+
+/* Return the builtin decl needed to store a vector of TYPE. */
+
+static tree
+ix86_builtin_tm_store (tree type)
+{
+ if (TREE_CODE (type) == VECTOR_TYPE)
+ {
+ switch (tree_low_cst (TYPE_SIZE (type), 1))
+ {
+ case 64:
+ return builtin_decl_explicit (BUILT_IN_TM_STORE_M64);
+ case 128:
+ return builtin_decl_explicit (BUILT_IN_TM_STORE_M128);
+ case 256:
+ return builtin_decl_explicit (BUILT_IN_TM_STORE_M256);
+ }
+ }
+ return NULL_TREE;
+}
+\f
+/* Initialize the transactional memory vector load/store builtins. */
+
+static void
+ix86_init_tm_builtins (void)
+{
+ enum ix86_builtin_func_type ftype;
+ const struct builtin_description *d;
+ size_t i;
+ tree decl;
+ tree attrs_load, attrs_type_load, attrs_store, attrs_type_store;
+ tree attrs_log, attrs_type_log;
+
+ if (!flag_tm)
+ return;
+
+ /* Use whatever attributes a normal TM load has. */
+ decl = builtin_decl_explicit (BUILT_IN_TM_LOAD_1);
+ attrs_load = DECL_ATTRIBUTES (decl);
+ attrs_type_load = TYPE_ATTRIBUTES (TREE_TYPE (decl));
+ /* Use whatever attributes a normal TM store has. */
+ decl = builtin_decl_explicit (BUILT_IN_TM_STORE_1);
+ attrs_store = DECL_ATTRIBUTES (decl);
+ attrs_type_store = TYPE_ATTRIBUTES (TREE_TYPE (decl));
+ /* Use whatever attributes a normal TM log has. */
+ decl = builtin_decl_explicit (BUILT_IN_TM_LOG);
+ attrs_log = DECL_ATTRIBUTES (decl);
+ attrs_type_log = TYPE_ATTRIBUTES (TREE_TYPE (decl));
+
+ for (i = 0, d = bdesc_tm;
+ i < ARRAY_SIZE (bdesc_tm);
+ i++, d++)
+ {
+ if ((d->mask & ix86_isa_flags) != 0
+ || (lang_hooks.builtin_function
+ == lang_hooks.builtin_function_ext_scope))
+ {
+ tree type, attrs, attrs_type;
+ enum built_in_function code = (enum built_in_function) d->code;
+
+ ftype = (enum ix86_builtin_func_type) d->flag;
+ type = ix86_get_builtin_func_type (ftype);
+
+ if (BUILTIN_TM_LOAD_P (code))
+ {
+ attrs = attrs_load;
+ attrs_type = attrs_type_load;
+ }
+ else if (BUILTIN_TM_STORE_P (code))
+ {
+ attrs = attrs_store;
+ attrs_type = attrs_type_store;
+ }
+ else
+ {
+ attrs = attrs_log;
+ attrs_type = attrs_type_log;
+ }
+ decl = add_builtin_function (d->name, type, code, BUILT_IN_NORMAL,
+ /* The builtin without the prefix for
+ calling it directly. */
+ d->name + strlen ("__builtin_"),
+ attrs);
+ /* add_builtin_function() will set the DECL_ATTRIBUTES, now
+ set the TYPE_ATTRIBUTES. */
+ decl_attributes (&TREE_TYPE (decl), attrs_type, ATTR_FLAG_BUILT_IN);
+
+ set_builtin_decl (code, decl, false);
+ }
+ }
+}
/* Set up all the MMX/SSE builtins, even builtins for instructions that are not
in the current target ISA to allow the user to compile particular modules
V4SI_FTYPE_V4SI_PCINT_V4DI_V4SI_INT,
IX86_BUILTIN_GATHERDIV8SI);
+ def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatheraltsiv4df ",
+ V4DF_FTYPE_V4DF_PCDOUBLE_V8SI_V4DF_INT,
+ IX86_BUILTIN_GATHERALTSIV4DF);
+
+ def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatheraltdiv4sf256 ",
+ V8SF_FTYPE_V8SF_PCFLOAT_V4DI_V8SF_INT,
+ IX86_BUILTIN_GATHERALTDIV8SF);
+
+ def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatheraltsiv4di ",
+ V4DI_FTYPE_V4DI_PCINT64_V8SI_V4DI_INT,
+ IX86_BUILTIN_GATHERALTSIV4DI);
+
+ def_builtin (OPTION_MASK_ISA_AVX2, "__builtin_ia32_gatheraltdiv4si256 ",
+ V8SI_FTYPE_V8SI_PCINT_V4DI_V8SI_INT,
+ IX86_BUILTIN_GATHERALTDIV8SI);
+
/* MMX access to the vec_init patterns. */
def_builtin_const (OPTION_MASK_ISA_MMX, "__builtin_ia32_vec_init_v2si",
V2SI_FTYPE_INT_INT, IX86_BUILTIN_VEC_INIT_V2SI);
TREE_READONLY (t) = 1;
ix86_builtins[(int) IX86_BUILTIN_COPYSIGNQ] = t;
+ ix86_init_tm_builtins ();
ix86_init_mmx_sse_builtins ();
if (TARGET_LP64)
return SUBREG_REG (target);
}
-/* Subroutine of ix86_expand_args_builtin to take care of round insns. */
+/* Subroutines of ix86_expand_args_builtin to take care of round insns. */
static rtx
ix86_expand_sse_round (const struct builtin_description *d, tree exp,
return target;
}
+static rtx
+ix86_expand_sse_round_vec_pack_sfix (const struct builtin_description *d,
+ tree exp, rtx target)
+{
+ rtx pat;
+ tree arg0 = CALL_EXPR_ARG (exp, 0);
+ tree arg1 = CALL_EXPR_ARG (exp, 1);
+ rtx op0 = expand_normal (arg0);
+ rtx op1 = expand_normal (arg1);
+ rtx op2;
+ enum machine_mode tmode = insn_data[d->icode].operand[0].mode;
+ enum machine_mode mode0 = insn_data[d->icode].operand[1].mode;
+ enum machine_mode mode1 = insn_data[d->icode].operand[2].mode;
+
+ if (optimize || target == 0
+ || GET_MODE (target) != tmode
+ || !insn_data[d->icode].operand[0].predicate (target, tmode))
+ target = gen_reg_rtx (tmode);
+
+ op0 = safe_vector_operand (op0, mode0);
+ op1 = safe_vector_operand (op1, mode1);
+
+ if ((optimize && !register_operand (op0, mode0))
+ || !insn_data[d->icode].operand[0].predicate (op0, mode0))
+ op0 = copy_to_mode_reg (mode0, op0);
+ if ((optimize && !register_operand (op1, mode1))
+ || !insn_data[d->icode].operand[1].predicate (op1, mode1))
+ op1 = copy_to_mode_reg (mode1, op1);
+
+ op2 = GEN_INT (d->comparison);
+
+ pat = GEN_FCN (d->icode) (target, op0, op1, op2);
+ if (! pat)
+ return 0;
+ emit_insn (pat);
+ return target;
+}
+
/* Subroutine of ix86_expand_builtin to take care of ptest insns. */
static rtx
case V4DF_FTYPE_V4DF_ROUND:
case V4SF_FTYPE_V4SF_ROUND:
case V8SF_FTYPE_V8SF_ROUND:
+ case V4SI_FTYPE_V4SF_ROUND:
+ case V8SI_FTYPE_V8SF_ROUND:
return ix86_expand_sse_round (d, exp, target);
+ case V4SI_FTYPE_V2DF_V2DF_ROUND:
+ case V8SI_FTYPE_V4DF_V4DF_ROUND:
+ return ix86_expand_sse_round_vec_pack_sfix (d, exp, target);
case INT_FTYPE_V8SF_V8SF_PTEST:
case INT_FTYPE_V4DI_V4DI_PTEST:
case INT_FTYPE_V4DF_V4DF_PTEST:
case V32QI_FTYPE_V32QI_V32QI:
case V16HI_FTYPE_V32QI_V32QI:
case V16HI_FTYPE_V16HI_V16HI:
+ case V8SI_FTYPE_V4DF_V4DF:
case V8SI_FTYPE_V8SI_V8SI:
case V8SI_FTYPE_V16HI_V16HI:
case V4DI_FTYPE_V4DI_V4DI:
rmode = V1TImode;
nargs_constant = 1;
break;
+ case V4DI_FTYPE_V4DI_INT_CONVERT:
+ nargs = 2;
+ rmode = V2TImode;
+ nargs_constant = 1;
+ break;
case V8HI_FTYPE_V8HI_INT:
case V8HI_FTYPE_V8SF_INT:
case V8HI_FTYPE_V4SF_INT:
error ("the last argument must be an 1-bit immediate");
return const0_rtx;
- case CODE_FOR_sse4_1_roundpd:
- case CODE_FOR_sse4_1_roundps:
case CODE_FOR_sse4_1_roundsd:
case CODE_FOR_sse4_1_roundss:
+
+ case CODE_FOR_sse4_1_roundpd:
+ case CODE_FOR_sse4_1_roundps:
+ case CODE_FOR_avx_roundpd256:
+ case CODE_FOR_avx_roundps256:
+
+ case CODE_FOR_sse4_1_roundpd_vec_pack_sfix:
+ case CODE_FOR_sse4_1_roundps_sfix:
+ case CODE_FOR_avx_roundpd_vec_pack_sfix256:
+ case CODE_FOR_avx_roundps_sfix256:
+
case CODE_FOR_sse4_1_blendps:
case CODE_FOR_avx_blendpd256:
case CODE_FOR_avx_vpermilv4df:
- case CODE_FOR_avx_roundpd256:
- case CODE_FOR_avx_roundps256:
error ("the last argument must be a 4-bit immediate");
return const0_rtx;
klass = store;
memory = 0;
break;
- break;
case UINT64_FTYPE_VOID:
case UNSIGNED_FTYPE_VOID:
nargs = 0;
case IX86_BUILTIN_VEC_SET_V16QI:
return ix86_expand_vec_set_builtin (exp);
- case IX86_BUILTIN_VEC_PERM_V2DF:
- case IX86_BUILTIN_VEC_PERM_V4SF:
- case IX86_BUILTIN_VEC_PERM_V2DI:
- case IX86_BUILTIN_VEC_PERM_V4SI:
- case IX86_BUILTIN_VEC_PERM_V8HI:
- case IX86_BUILTIN_VEC_PERM_V16QI:
- case IX86_BUILTIN_VEC_PERM_V2DI_U:
- case IX86_BUILTIN_VEC_PERM_V4SI_U:
- case IX86_BUILTIN_VEC_PERM_V8HI_U:
- case IX86_BUILTIN_VEC_PERM_V16QI_U:
- case IX86_BUILTIN_VEC_PERM_V4DF:
- case IX86_BUILTIN_VEC_PERM_V8SF:
- return ix86_expand_vec_perm_builtin (exp);
-
case IX86_BUILTIN_INFQ:
case IX86_BUILTIN_HUGE_VALQ:
{
icode = CODE_FOR_avx2_gatherdiv4sf;
goto gather_gen;
case IX86_BUILTIN_GATHERDIV8SF:
- icode = CODE_FOR_avx2_gatherdiv4sf256;
+ icode = CODE_FOR_avx2_gatherdiv8sf;
goto gather_gen;
case IX86_BUILTIN_GATHERSIV2DI:
icode = CODE_FOR_avx2_gathersiv2di;
icode = CODE_FOR_avx2_gatherdiv4si;
goto gather_gen;
case IX86_BUILTIN_GATHERDIV8SI:
- icode = CODE_FOR_avx2_gatherdiv4si256;
+ icode = CODE_FOR_avx2_gatherdiv8si;
+ goto gather_gen;
+ case IX86_BUILTIN_GATHERALTSIV4DF:
+ icode = CODE_FOR_avx2_gathersiv4df;
+ goto gather_gen;
+ case IX86_BUILTIN_GATHERALTDIV8SF:
+ icode = CODE_FOR_avx2_gatherdiv8sf;
+ goto gather_gen;
+ case IX86_BUILTIN_GATHERALTSIV4DI:
+ icode = CODE_FOR_avx2_gathersiv4df;
+ goto gather_gen;
+ case IX86_BUILTIN_GATHERALTDIV8SI:
+ icode = CODE_FOR_avx2_gatherdiv8si;
+ goto gather_gen;
gather_gen:
arg0 = CALL_EXPR_ARG (exp, 0);
op4 = expand_normal (arg4);
/* Note the arg order is different from the operand order. */
mode0 = insn_data[icode].operand[1].mode;
- mode1 = insn_data[icode].operand[2].mode;
mode2 = insn_data[icode].operand[3].mode;
mode3 = insn_data[icode].operand[4].mode;
mode4 = insn_data[icode].operand[5].mode;
- if (target == NULL_RTX)
- target = gen_reg_rtx (insn_data[icode].operand[0].mode);
+ if (target == NULL_RTX
+ || GET_MODE (target) != insn_data[icode].operand[0].mode)
+ subtarget = gen_reg_rtx (insn_data[icode].operand[0].mode);
+ else
+ subtarget = target;
+
+ if (fcode == IX86_BUILTIN_GATHERALTSIV4DF
+ || fcode == IX86_BUILTIN_GATHERALTSIV4DI)
+ {
+ rtx half = gen_reg_rtx (V4SImode);
+ if (!nonimmediate_operand (op2, V8SImode))
+ op2 = copy_to_mode_reg (V8SImode, op2);
+ emit_insn (gen_vec_extract_lo_v8si (half, op2));
+ op2 = half;
+ }
+ else if (fcode == IX86_BUILTIN_GATHERALTDIV8SF
+ || fcode == IX86_BUILTIN_GATHERALTDIV8SI)
+ {
+ rtx (*gen) (rtx, rtx);
+ rtx half = gen_reg_rtx (mode0);
+ if (mode0 == V4SFmode)
+ gen = gen_vec_extract_lo_v8sf;
+ else
+ gen = gen_vec_extract_lo_v8si;
+ if (!nonimmediate_operand (op0, GET_MODE (op0)))
+ op0 = copy_to_mode_reg (GET_MODE (op0), op0);
+ emit_insn (gen (half, op0));
+ op0 = half;
+ if (!nonimmediate_operand (op3, GET_MODE (op3)))
+ op3 = copy_to_mode_reg (GET_MODE (op3), op3);
+ emit_insn (gen (half, op3));
+ op3 = half;
+ }
/* Force memory operand only with base register here. But we
don't want to do it on memory operand for other builtin
if (GET_MODE (op1) != Pmode)
op1 = convert_to_mode (Pmode, op1, 1);
op1 = force_reg (Pmode, op1);
- op1 = gen_rtx_MEM (mode1, op1);
if (!insn_data[icode].operand[1].predicate (op0, mode0))
op0 = copy_to_mode_reg (mode0, op0);
- if (!insn_data[icode].operand[2].predicate (op1, mode1))
- op1 = copy_to_mode_reg (mode1, op1);
+ if (!insn_data[icode].operand[2].predicate (op1, Pmode))
+ op1 = copy_to_mode_reg (Pmode, op1);
if (!insn_data[icode].operand[3].predicate (op2, mode2))
op2 = copy_to_mode_reg (mode2, op2);
if (!insn_data[icode].operand[4].predicate (op3, mode3))
error ("last argument must be scale 1, 2, 4, 8");
return const0_rtx;
}
- pat = GEN_FCN (icode) (target, op0, op1, op2, op3, op4);
+
+ /* Optimize. If mask is known to have all high bits set,
+ replace op0 with pc_rtx to signal that the instruction
+ overwrites the whole destination and doesn't use its
+ previous contents. */
+ if (optimize)
+ {
+ if (TREE_CODE (arg3) == VECTOR_CST)
+ {
+ tree elt;
+ unsigned int negative = 0;
+ for (elt = TREE_VECTOR_CST_ELTS (arg3);
+ elt; elt = TREE_CHAIN (elt))
+ {
+ tree cst = TREE_VALUE (elt);
+ if (TREE_CODE (cst) == INTEGER_CST
+ && tree_int_cst_sign_bit (cst))
+ negative++;
+ else if (TREE_CODE (cst) == REAL_CST
+ && REAL_VALUE_NEGATIVE (TREE_REAL_CST (cst)))
+ negative++;
+ }
+ if (negative == TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg3)))
+ op0 = pc_rtx;
+ }
+ else if (TREE_CODE (arg3) == SSA_NAME)
+ {
+ /* Recognize also when mask is like:
+ __v2df src = _mm_setzero_pd ();
+ __v2df mask = _mm_cmpeq_pd (src, src);
+ or
+ __v8sf src = _mm256_setzero_ps ();
+ __v8sf mask = _mm256_cmp_ps (src, src, _CMP_EQ_OQ);
+ as that is a cheaper way to load all ones into
+ a register than having to load a constant from
+ memory. */
+ gimple def_stmt = SSA_NAME_DEF_STMT (arg3);
+ if (is_gimple_call (def_stmt))
+ {
+ tree fndecl = gimple_call_fndecl (def_stmt);
+ if (fndecl
+ && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD)
+ switch ((unsigned int) DECL_FUNCTION_CODE (fndecl))
+ {
+ case IX86_BUILTIN_CMPPD:
+ case IX86_BUILTIN_CMPPS:
+ case IX86_BUILTIN_CMPPD256:
+ case IX86_BUILTIN_CMPPS256:
+ if (!integer_zerop (gimple_call_arg (def_stmt, 2)))
+ break;
+ /* FALLTHRU */
+ case IX86_BUILTIN_CMPEQPD:
+ case IX86_BUILTIN_CMPEQPS:
+ if (initializer_zerop (gimple_call_arg (def_stmt, 0))
+ && initializer_zerop (gimple_call_arg (def_stmt,
+ 1)))
+ op0 = pc_rtx;
+ break;
+ default:
+ break;
+ }
+ }
+ }
+ }
+
+ pat = GEN_FCN (icode) (subtarget, op0, op1, op2, op3, op4);
if (! pat)
return const0_rtx;
emit_insn (pat);
+
+ if (fcode == IX86_BUILTIN_GATHERDIV8SF
+ || fcode == IX86_BUILTIN_GATHERDIV8SI)
+ {
+ enum machine_mode tmode = GET_MODE (subtarget) == V8SFmode
+ ? V4SFmode : V4SImode;
+ if (target == NULL_RTX)
+ target = gen_reg_rtx (tmode);
+ if (tmode == V4SFmode)
+ emit_insn (gen_vec_extract_lo_v8sf (target, subtarget));
+ else
+ emit_insn (gen_vec_extract_lo_v8si (target, subtarget));
+ }
+ else
+ target = subtarget;
+
return target;
default:
}
break;
+ case BUILT_IN_IFLOOR:
+ case BUILT_IN_LFLOOR:
+ case BUILT_IN_LLFLOOR:
+ /* The round insn does not trap on denormals. */
+ if (flag_trapping_math || !TARGET_ROUND)
+ break;
+
+ if (out_mode == SImode && in_mode == DFmode)
+ {
+ if (out_n == 4 && in_n == 2)
+ return ix86_builtins[IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX];
+ else if (out_n == 8 && in_n == 4)
+ return ix86_builtins[IX86_BUILTIN_FLOORPD_VEC_PACK_SFIX256];
+ }
+ break;
+
+ case BUILT_IN_IFLOORF:
+ case BUILT_IN_LFLOORF:
+ case BUILT_IN_LLFLOORF:
+ /* The round insn does not trap on denormals. */
+ if (flag_trapping_math || !TARGET_ROUND)
+ break;
+
+ if (out_mode == SImode && in_mode == SFmode)
+ {
+ if (out_n == 4 && in_n == 4)
+ return ix86_builtins[IX86_BUILTIN_FLOORPS_SFIX];
+ else if (out_n == 8 && in_n == 8)
+ return ix86_builtins[IX86_BUILTIN_FLOORPS_SFIX256];
+ }
+ break;
+
+ case BUILT_IN_ICEIL:
+ case BUILT_IN_LCEIL:
+ case BUILT_IN_LLCEIL:
+ /* The round insn does not trap on denormals. */
+ if (flag_trapping_math || !TARGET_ROUND)
+ break;
+
+ if (out_mode == SImode && in_mode == DFmode)
+ {
+ if (out_n == 4 && in_n == 2)
+ return ix86_builtins[IX86_BUILTIN_CEILPD_VEC_PACK_SFIX];
+ else if (out_n == 8 && in_n == 4)
+ return ix86_builtins[IX86_BUILTIN_CEILPD_VEC_PACK_SFIX256];
+ }
+ break;
+
+ case BUILT_IN_ICEILF:
+ case BUILT_IN_LCEILF:
+ case BUILT_IN_LLCEILF:
+ /* The round insn does not trap on denormals. */
+ if (flag_trapping_math || !TARGET_ROUND)
+ break;
+
+ if (out_mode == SImode && in_mode == SFmode)
+ {
+ if (out_n == 4 && in_n == 4)
+ return ix86_builtins[IX86_BUILTIN_CEILPS_SFIX];
+ else if (out_n == 8 && in_n == 8)
+ return ix86_builtins[IX86_BUILTIN_CEILPS_SFIX256];
+ }
+ break;
+
+ case BUILT_IN_IRINT:
case BUILT_IN_LRINT:
- if (out_mode == SImode && out_n == 4
- && in_mode == DFmode && in_n == 2)
- return ix86_builtins[IX86_BUILTIN_VEC_PACK_SFIX];
+ case BUILT_IN_LLRINT:
+ if (out_mode == SImode && in_mode == DFmode)
+ {
+ if (out_n == 4 && in_n == 2)
+ return ix86_builtins[IX86_BUILTIN_VEC_PACK_SFIX];
+ else if (out_n == 8 && in_n == 4)
+ return ix86_builtins[IX86_BUILTIN_VEC_PACK_SFIX256];
+ }
break;
+ case BUILT_IN_IRINTF:
case BUILT_IN_LRINTF:
+ case BUILT_IN_LLRINTF:
if (out_mode == SImode && in_mode == SFmode)
{
if (out_n == 4 && in_n == 4)
}
break;
+ case BUILT_IN_IROUND:
+ case BUILT_IN_LROUND:
+ case BUILT_IN_LLROUND:
+ /* The round insn does not trap on denormals. */
+ if (flag_trapping_math || !TARGET_ROUND)
+ break;
+
+ if (out_mode == SImode && in_mode == DFmode)
+ {
+ if (out_n == 4 && in_n == 2)
+ return ix86_builtins[IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX];
+ else if (out_n == 8 && in_n == 4)
+ return ix86_builtins[IX86_BUILTIN_ROUNDPD_AZ_VEC_PACK_SFIX256];
+ }
+ break;
+
+ case BUILT_IN_IROUNDF:
+ case BUILT_IN_LROUNDF:
+ case BUILT_IN_LLROUNDF:
+ /* The round insn does not trap on denormals. */
+ if (flag_trapping_math || !TARGET_ROUND)
+ break;
+
+ if (out_mode == SImode && in_mode == SFmode)
+ {
+ if (out_n == 4 && in_n == 4)
+ return ix86_builtins[IX86_BUILTIN_ROUNDPS_AZ_SFIX];
+ else if (out_n == 8 && in_n == 8)
+ return ix86_builtins[IX86_BUILTIN_ROUNDPS_AZ_SFIX256];
+ }
+ break;
+
case BUILT_IN_COPYSIGN:
if (out_mode == DFmode && in_mode == DFmode)
{
return NULL_TREE;
}
- bname = IDENTIFIER_POINTER (DECL_NAME (implicit_built_in_decls[fn]));
+ bname = IDENTIFIER_POINTER (DECL_NAME (builtin_decl_implicit (fn)));
if (fn == BUILT_IN_LOGF)
strcpy (name, "vmlsLn4");
name[4] &= ~0x20;
arity = 0;
- for (args = DECL_ARGUMENTS (implicit_built_in_decls[fn]); args;
+ for (args = DECL_ARGUMENTS (builtin_decl_implicit (fn));
+ args;
args = TREE_CHAIN (args))
arity++;
return NULL_TREE;
}
- bname = IDENTIFIER_POINTER (DECL_NAME (implicit_built_in_decls[fn]));
+ bname = IDENTIFIER_POINTER (DECL_NAME (builtin_decl_implicit (fn)));
sprintf (name + 7, "%s", bname+10);
arity = 0;
- for (args = DECL_ARGUMENTS (implicit_built_in_decls[fn]); args;
+ for (args = DECL_ARGUMENTS (builtin_decl_implicit (fn));
+ args;
args = TREE_CHAIN (args))
arity++;
return new_fndecl;
}
-
-/* Returns a decl of a function that implements conversion of an integer vector
- into a floating-point vector, or vice-versa. DEST_TYPE and SRC_TYPE
- are the types involved when converting according to CODE.
+/* Returns a decl of a function that implements gather load with
+ memory type MEM_VECTYPE and index type INDEX_VECTYPE and SCALE.
Return NULL_TREE if it is not available. */
static tree
-ix86_vectorize_builtin_conversion (unsigned int code,
- tree dest_type, tree src_type)
+ix86_vectorize_builtin_gather (const_tree mem_vectype,
+ const_tree index_type, int scale)
{
- if (! TARGET_SSE2)
+ bool si;
+ enum ix86_builtins code;
+
+ if (! TARGET_AVX2)
return NULL_TREE;
- switch (code)
- {
- case FLOAT_EXPR:
- switch (TYPE_MODE (src_type))
- {
- case V4SImode:
- switch (TYPE_MODE (dest_type))
- {
- case V4SFmode:
- return (TYPE_UNSIGNED (src_type)
- ? ix86_builtins[IX86_BUILTIN_CVTUDQ2PS]
- : ix86_builtins[IX86_BUILTIN_CVTDQ2PS]);
- case V4DFmode:
- return (TYPE_UNSIGNED (src_type)
- ? NULL_TREE
- : ix86_builtins[IX86_BUILTIN_CVTDQ2PD256]);
- default:
- return NULL_TREE;
- }
- break;
- case V8SImode:
- switch (TYPE_MODE (dest_type))
- {
- case V8SFmode:
- return (TYPE_UNSIGNED (src_type)
- ? NULL_TREE
- : ix86_builtins[IX86_BUILTIN_CVTDQ2PS256]);
- default:
- return NULL_TREE;
- }
- break;
- default:
- return NULL_TREE;
- }
+ if ((TREE_CODE (index_type) != INTEGER_TYPE
+ && !POINTER_TYPE_P (index_type))
+ || (TYPE_MODE (index_type) != SImode
+ && TYPE_MODE (index_type) != DImode))
+ return NULL_TREE;
- case FIX_TRUNC_EXPR:
- switch (TYPE_MODE (dest_type))
- {
- case V4SImode:
- switch (TYPE_MODE (src_type))
- {
- case V4SFmode:
- return (TYPE_UNSIGNED (dest_type)
- ? NULL_TREE
- : ix86_builtins[IX86_BUILTIN_CVTTPS2DQ]);
- case V4DFmode:
- return (TYPE_UNSIGNED (dest_type)
- ? NULL_TREE
- : ix86_builtins[IX86_BUILTIN_CVTTPD2DQ256]);
- default:
- return NULL_TREE;
- }
- break;
+ if (TYPE_PRECISION (index_type) > POINTER_SIZE)
+ return NULL_TREE;
- case V8SImode:
- switch (TYPE_MODE (src_type))
- {
- case V8SFmode:
- return (TYPE_UNSIGNED (dest_type)
- ? NULL_TREE
- : ix86_builtins[IX86_BUILTIN_CVTTPS2DQ256]);
- default:
- return NULL_TREE;
- }
- break;
+ /* v*gather* insn sign extends index to pointer mode. */
+ if (TYPE_PRECISION (index_type) < POINTER_SIZE
+ && TYPE_UNSIGNED (index_type))
+ return NULL_TREE;
- default:
- return NULL_TREE;
- }
+ if (scale <= 0
+ || scale > 8
+ || (scale & (scale - 1)) != 0)
+ return NULL_TREE;
+ si = TYPE_MODE (index_type) == SImode;
+ switch (TYPE_MODE (mem_vectype))
+ {
+ case V2DFmode:
+ code = si ? IX86_BUILTIN_GATHERSIV2DF : IX86_BUILTIN_GATHERDIV2DF;
+ break;
+ case V4DFmode:
+ code = si ? IX86_BUILTIN_GATHERALTSIV4DF : IX86_BUILTIN_GATHERDIV4DF;
+ break;
+ case V2DImode:
+ code = si ? IX86_BUILTIN_GATHERSIV2DI : IX86_BUILTIN_GATHERDIV2DI;
+ break;
+ case V4DImode:
+ code = si ? IX86_BUILTIN_GATHERALTSIV4DI : IX86_BUILTIN_GATHERDIV4DI;
+ break;
+ case V4SFmode:
+ code = si ? IX86_BUILTIN_GATHERSIV4SF : IX86_BUILTIN_GATHERDIV4SF;
+ break;
+ case V8SFmode:
+ code = si ? IX86_BUILTIN_GATHERSIV8SF : IX86_BUILTIN_GATHERALTDIV8SF;
+ break;
+ case V4SImode:
+ code = si ? IX86_BUILTIN_GATHERSIV4SI : IX86_BUILTIN_GATHERDIV4SI;
+ break;
+ case V8SImode:
+ code = si ? IX86_BUILTIN_GATHERSIV8SI : IX86_BUILTIN_GATHERALTDIV8SI;
+ break;
default:
return NULL_TREE;
}
- return NULL_TREE;
+ return ix86_builtins[code];
}
/* Returns a code for a target-specific builtin that implements
return mask + 1;
}
\f
-
/* Store OPERAND to the memory after reload is completed. This means
that we can't easily use assign_stack_local. */
rtx
}
}
- emit_insn (ix86_gen_add3 (delta_dst, delta_dst, delta_rtx));
+ ix86_emit_binop (PLUS, Pmode, delta_dst, delta_rtx);
}
/* Adjust the this parameter by a value stored in the vtable. */
REGNO (this_reg)),
vcall_mem));
else
- emit_insn (ix86_gen_add3 (this_reg, this_reg, vcall_mem));
+ ix86_emit_binop (PLUS, Pmode, this_reg, vcall_mem);
}
/* If necessary, drop THIS back to its stack slot. */
}
if (replace)
{
- emit_jump_insn_before (gen_return_internal_long (), ret);
+ emit_jump_insn_before (gen_simple_return_internal_long (), ret);
delete_insn (ret);
}
}
emit_label (donelab);
}
\f
-/* AVX does not support 32-byte integer vector operations,
- thus the longest vector we are faced with is V16QImode. */
-#define MAX_VECT_LEN 16
+/* AVX2 does support 32-byte integer vector operations,
+ thus the longest vector we are faced with is V32QImode. */
+#define MAX_VECT_LEN 32
struct expand_vec_perm_d
{
}
}
-/* Expand a vector reduction. FN is the binary pattern to reduce;
- DEST is the destination; IN is the input vector. */
+/* Generate code to copy vector bits i / 2 ... i - 1 from vector SRC
+ to bits 0 ... i / 2 - 1 of vector DEST, which has the same mode.
+ The upper bits of DEST are undefined, though they shouldn't cause
+ exceptions (some bits from src or all zeros are ok). */
-void
-ix86_expand_reduc (rtx (*fn) (rtx, rtx, rtx), rtx dest, rtx in)
+static void
+emit_reduc_half (rtx dest, rtx src, int i)
{
- rtx tmp1, tmp2, tmp3, tmp4, tmp5;
- enum machine_mode mode = GET_MODE (in);
- int i;
-
- tmp1 = gen_reg_rtx (mode);
- tmp2 = gen_reg_rtx (mode);
- tmp3 = gen_reg_rtx (mode);
-
- switch (mode)
+ rtx tem;
+ switch (GET_MODE (src))
{
case V4SFmode:
- emit_insn (gen_sse_movhlps (tmp1, in, in));
- emit_insn (fn (tmp2, tmp1, in));
- emit_insn (gen_sse_shufps_v4sf (tmp3, tmp2, tmp2,
- const1_rtx, const1_rtx,
- GEN_INT (1+4), GEN_INT (1+4)));
+ if (i == 128)
+ tem = gen_sse_movhlps (dest, src, src);
+ else
+ tem = gen_sse_shufps_v4sf (dest, src, src, const1_rtx, const1_rtx,
+ GEN_INT (1 + 4), GEN_INT (1 + 4));
+ break;
+ case V2DFmode:
+ tem = gen_vec_interleave_highv2df (dest, src, src);
+ break;
+ case V16QImode:
+ case V8HImode:
+ case V4SImode:
+ case V2DImode:
+ tem = gen_sse2_lshrv1ti3 (gen_lowpart (V1TImode, dest),
+ gen_lowpart (V1TImode, src),
+ GEN_INT (i / 2));
break;
case V8SFmode:
- tmp4 = gen_reg_rtx (mode);
- tmp5 = gen_reg_rtx (mode);
- emit_insn (gen_avx_vperm2f128v8sf3 (tmp4, in, in, const1_rtx));
- emit_insn (fn (tmp5, tmp4, in));
- emit_insn (gen_avx_shufps256 (tmp1, tmp5, tmp5, GEN_INT (2+12)));
- emit_insn (fn (tmp2, tmp1, tmp5));
- emit_insn (gen_avx_shufps256 (tmp3, tmp2, tmp2, const1_rtx));
+ if (i == 256)
+ tem = gen_avx_vperm2f128v8sf3 (dest, src, src, const1_rtx);
+ else
+ tem = gen_avx_shufps256 (dest, src, src,
+ GEN_INT (i == 128 ? 2 + (3 << 2) : 1));
break;
case V4DFmode:
- emit_insn (gen_avx_vperm2f128v4df3 (tmp1, in, in, const1_rtx));
- emit_insn (fn (tmp2, tmp1, in));
- emit_insn (gen_avx_shufpd256 (tmp3, tmp2, tmp2, const1_rtx));
+ if (i == 256)
+ tem = gen_avx_vperm2f128v4df3 (dest, src, src, const1_rtx);
+ else
+ tem = gen_avx_shufpd256 (dest, src, src, const1_rtx);
break;
case V32QImode:
case V16HImode:
case V8SImode:
case V4DImode:
- emit_insn (gen_avx2_permv2ti (gen_lowpart (V4DImode, tmp1),
- gen_lowpart (V4DImode, in),
- gen_lowpart (V4DImode, in),
- const1_rtx));
- tmp4 = in;
- tmp5 = tmp1;
- for (i = 64; i >= GET_MODE_BITSIZE (GET_MODE_INNER (mode)); i >>= 1)
- {
- if (i != 64)
- {
- tmp2 = gen_reg_rtx (mode);
- tmp3 = gen_reg_rtx (mode);
- }
- emit_insn (fn (tmp2, tmp4, tmp5));
- emit_insn (gen_avx2_lshrv2ti3 (gen_lowpart (V2TImode, tmp3),
- gen_lowpart (V2TImode, tmp2),
- GEN_INT (i)));
- tmp4 = tmp2;
- tmp5 = tmp3;
- }
+ if (i == 256)
+ tem = gen_avx2_permv2ti (gen_lowpart (V4DImode, dest),
+ gen_lowpart (V4DImode, src),
+ gen_lowpart (V4DImode, src),
+ const1_rtx);
+ else
+ tem = gen_avx2_lshrv2ti3 (gen_lowpart (V2TImode, dest),
+ gen_lowpart (V2TImode, src),
+ GEN_INT (i / 2));
break;
default:
gcc_unreachable ();
}
- emit_insn (fn (dest, tmp2, tmp3));
+ emit_insn (tem);
+}
+
+/* Expand a vector reduction. FN is the binary pattern to reduce;
+ DEST is the destination; IN is the input vector. */
+
+void
+ix86_expand_reduc (rtx (*fn) (rtx, rtx, rtx), rtx dest, rtx in)
+{
+ rtx half, dst, vec = in;
+ enum machine_mode mode = GET_MODE (in);
+ int i;
+
+ /* SSE4 has a special instruction for V8HImode UMIN reduction. */
+ if (TARGET_SSE4_1
+ && mode == V8HImode
+ && fn == gen_uminv8hi3)
+ {
+ emit_insn (gen_sse4_1_phminposuw (dest, in));
+ return;
+ }
+
+ for (i = GET_MODE_BITSIZE (mode);
+ i > GET_MODE_BITSIZE (GET_MODE_INNER (mode));
+ i >>= 1)
+ {
+ half = gen_reg_rtx (mode);
+ emit_reduc_half (half, vec, i);
+ if (i == GET_MODE_BITSIZE (GET_MODE_INNER (mode)) * 2)
+ dst = dest;
+ else
+ dst = gen_reg_rtx (mode);
+ emit_insn (fn (dst, half, vec));
+ vec = dst;
+ }
}
\f
/* Target hook for scalar_mode_supported_p. */
res = gen_reg_rtx (outmode);
half = CONST_DOUBLE_FROM_REAL_VALUE (dconsthalf, inmode);
-
+
/* round(a) = sgn(a) * floor(fabs(a) + 0.5) */
/* scratch = fxam(op1) */
/* a / b = a * ((rcp(b) + rcp(b)) - (b * rcp(b) * rcp (b))) */
+ b = force_reg (mode, b);
+
/* x0 = rcp(b) estimate */
emit_insn (gen_rtx_SET (VOIDmode, x0,
gen_rtx_UNSPEC (mode, gen_rtvec (1, b),
/* sqrt(a) = -0.5 * a * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0)
rsqrt(a) = -0.5 * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0) */
+ a = force_reg (mode, a);
+
/* x0 = rsqrt(a) estimate */
emit_insn (gen_rtx_SET (VOIDmode, x0,
gen_rtx_UNSPEC (mode, gen_rtvec (1, a),
for FP arguments. */
{ "sseregparm", 0, 0, false, true, true, ix86_handle_cconv_attribute,
true },
+ /* The transactional memory builtins are implicitly regparm or fastcall
+ depending on the ABI. Override the generic do-nothing attribute that
+ these builtins were declared with. */
+ { "*tm regparm", 0, 0, false, true, true, ix86_handle_tm_regparm_attribute,
+ true },
/* force_align_arg_pointer says this function realigns the stack at entry. */
{ (const char *)&ix86_force_align_arg_pointer_string, 0, 0,
false, true, true, ix86_handle_cconv_attribute, false },
}
-/* Implement targetm.vectorize.builtin_vec_perm. */
-
-static tree
-ix86_vectorize_builtin_vec_perm (tree vec_type, tree *mask_type)
-{
- tree itype = TREE_TYPE (vec_type);
- bool u = TYPE_UNSIGNED (itype);
- enum machine_mode vmode = TYPE_MODE (vec_type);
- enum ix86_builtins fcode;
- bool ok = TARGET_SSE2;
-
- switch (vmode)
- {
- case V4DFmode:
- ok = TARGET_AVX;
- fcode = IX86_BUILTIN_VEC_PERM_V4DF;
- goto get_di;
- case V2DFmode:
- fcode = IX86_BUILTIN_VEC_PERM_V2DF;
- get_di:
- itype = ix86_get_builtin_type (IX86_BT_DI);
- break;
-
- case V8SFmode:
- ok = TARGET_AVX;
- fcode = IX86_BUILTIN_VEC_PERM_V8SF;
- goto get_si;
- case V4SFmode:
- ok = TARGET_SSE;
- fcode = IX86_BUILTIN_VEC_PERM_V4SF;
- get_si:
- itype = ix86_get_builtin_type (IX86_BT_SI);
- break;
-
- case V2DImode:
- fcode = u ? IX86_BUILTIN_VEC_PERM_V2DI_U : IX86_BUILTIN_VEC_PERM_V2DI;
- break;
- case V4SImode:
- fcode = u ? IX86_BUILTIN_VEC_PERM_V4SI_U : IX86_BUILTIN_VEC_PERM_V4SI;
- break;
- case V8HImode:
- fcode = u ? IX86_BUILTIN_VEC_PERM_V8HI_U : IX86_BUILTIN_VEC_PERM_V8HI;
- break;
- case V16QImode:
- fcode = u ? IX86_BUILTIN_VEC_PERM_V16QI_U : IX86_BUILTIN_VEC_PERM_V16QI;
- break;
- default:
- ok = false;
- break;
- }
-
- if (!ok)
- return NULL_TREE;
-
- *mask_type = itype;
- return ix86_builtins[(int) fcode];
-}
-
/* Return a vector mode with twice as many elements as VMODE. */
/* ??? Consider moving this to a table generated by genmodes.c. */
}
/* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
- in terms of blendp[sd] / pblendw / pblendvb. */
+ in terms of blendp[sd] / pblendw / pblendvb / vpblendd. */
static bool
expand_vec_perm_blend (struct expand_vec_perm_d *d)
enum machine_mode vmode = d->vmode;
unsigned i, mask, nelt = d->nelt;
rtx target, op0, op1, x;
+ rtx rperm[32], vperm;
- if (!TARGET_SSE4_1 || d->op0 == d->op1)
+ if (d->op0 == d->op1)
return false;
- if (!(GET_MODE_SIZE (vmode) == 16 || vmode == V4DFmode || vmode == V8SFmode))
+ if (TARGET_AVX2 && GET_MODE_SIZE (vmode) == 32)
+ ;
+ else if (TARGET_AVX && (vmode == V4DFmode || vmode == V8SFmode))
+ ;
+ else if (TARGET_SSE4_1 && GET_MODE_SIZE (vmode) == 16)
+ ;
+ else
return false;
/* This is a blend, not a permute. Elements must stay in their
/* ??? Without SSE4.1, we could implement this with and/andn/or. This
decision should be extracted elsewhere, so that we only try that
sequence once all budget==3 options have been tried. */
-
- /* For bytes, see if bytes move in pairs so we can use pblendw with
- an immediate argument, rather than pblendvb with a vector argument. */
- if (vmode == V16QImode)
- {
- bool pblendw_ok = true;
- for (i = 0; i < 16 && pblendw_ok; i += 2)
- pblendw_ok = (d->perm[i] + 1 == d->perm[i + 1]);
-
- if (!pblendw_ok)
- {
- rtx rperm[16], vperm;
-
- for (i = 0; i < nelt; ++i)
- rperm[i] = (d->perm[i] < nelt ? const0_rtx : constm1_rtx);
-
- vperm = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, rperm));
- vperm = force_reg (V16QImode, vperm);
-
- emit_insn (gen_sse4_1_pblendvb (d->target, d->op0, d->op1, vperm));
- return true;
- }
- }
-
target = d->target;
op0 = d->op0;
op1 = d->op1;
case V2DFmode:
case V4SFmode:
case V8HImode:
+ case V8SImode:
for (i = 0; i < nelt; ++i)
mask |= (d->perm[i] >= nelt) << i;
break;
case V2DImode:
for (i = 0; i < 2; ++i)
mask |= (d->perm[i] >= 2 ? 15 : 0) << (i * 4);
+ vmode = V8HImode;
goto do_subreg;
case V4SImode:
for (i = 0; i < 4; ++i)
mask |= (d->perm[i] >= 4 ? 3 : 0) << (i * 2);
+ vmode = V8HImode;
goto do_subreg;
case V16QImode:
+ /* See if bytes move in pairs so we can use pblendw with
+ an immediate argument, rather than pblendvb with a vector
+ argument. */
+ for (i = 0; i < 16; i += 2)
+ if (d->perm[i] + 1 != d->perm[i + 1])
+ {
+ use_pblendvb:
+ for (i = 0; i < nelt; ++i)
+ rperm[i] = (d->perm[i] < nelt ? const0_rtx : constm1_rtx);
+
+ finish_pblendvb:
+ vperm = gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (nelt, rperm));
+ vperm = force_reg (vmode, vperm);
+
+ if (GET_MODE_SIZE (vmode) == 16)
+ emit_insn (gen_sse4_1_pblendvb (target, op0, op1, vperm));
+ else
+ emit_insn (gen_avx2_pblendvb (target, op0, op1, vperm));
+ return true;
+ }
+
for (i = 0; i < 8; ++i)
mask |= (d->perm[i * 2] >= 16) << i;
+ vmode = V8HImode;
+ /* FALLTHRU */
do_subreg:
- vmode = V8HImode;
target = gen_lowpart (vmode, target);
op0 = gen_lowpart (vmode, op0);
op1 = gen_lowpart (vmode, op1);
break;
+ case V32QImode:
+ /* See if bytes move in pairs. If not, vpblendvb must be used. */
+ for (i = 0; i < 32; i += 2)
+ if (d->perm[i] + 1 != d->perm[i + 1])
+ goto use_pblendvb;
+ /* See if bytes move in quadruplets. If yes, vpblendd
+ with immediate can be used. */
+ for (i = 0; i < 32; i += 4)
+ if (d->perm[i] + 2 != d->perm[i + 2])
+ break;
+ if (i < 32)
+ {
+ /* See if bytes move the same in both lanes. If yes,
+ vpblendw with immediate can be used. */
+ for (i = 0; i < 16; i += 2)
+ if (d->perm[i] + 16 != d->perm[i + 16])
+ goto use_pblendvb;
+
+ /* Use vpblendw. */
+ for (i = 0; i < 16; ++i)
+ mask |= (d->perm[i * 2] >= 32) << i;
+ vmode = V16HImode;
+ goto do_subreg;
+ }
+
+ /* Use vpblendd. */
+ for (i = 0; i < 8; ++i)
+ mask |= (d->perm[i * 4] >= 32) << i;
+ vmode = V8SImode;
+ goto do_subreg;
+
+ case V16HImode:
+ /* See if words move in pairs. If yes, vpblendd can be used. */
+ for (i = 0; i < 16; i += 2)
+ if (d->perm[i] + 1 != d->perm[i + 1])
+ break;
+ if (i < 16)
+ {
+ /* See if words move the same in both lanes. If not,
+ vpblendvb must be used. */
+ for (i = 0; i < 8; i++)
+ if (d->perm[i] + 8 != d->perm[i + 8])
+ {
+ /* Use vpblendvb. */
+ for (i = 0; i < 32; ++i)
+ rperm[i] = (d->perm[i / 2] < 16 ? const0_rtx : constm1_rtx);
+
+ vmode = V32QImode;
+ nelt = 32;
+ target = gen_lowpart (vmode, target);
+ op0 = gen_lowpart (vmode, op0);
+ op1 = gen_lowpart (vmode, op1);
+ goto finish_pblendvb;
+ }
+
+ /* Use vpblendw. */
+ for (i = 0; i < 16; ++i)
+ mask |= (d->perm[i] >= 16) << i;
+ break;
+ }
+
+ /* Use vpblendd. */
+ for (i = 0; i < 8; ++i)
+ mask |= (d->perm[i * 2] >= 16) << i;
+ vmode = V8SImode;
+ goto do_subreg;
+
+ case V4DImode:
+ /* Use vpblendd. */
+ for (i = 0; i < 4; ++i)
+ mask |= (d->perm[i] >= 4 ? 3 : 0) << (i * 2);
+ vmode = V8SImode;
+ goto do_subreg;
+
default:
gcc_unreachable ();
}
return true;
}
-/* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
- in terms of pshufb or vpperm. */
+/* Return true if permutation D can be performed as VMODE permutation
+ instead. */
static bool
-expand_vec_perm_pshufb (struct expand_vec_perm_d *d)
+valid_perm_using_mode_p (enum machine_mode vmode, struct expand_vec_perm_d *d)
{
- unsigned i, nelt, eltsz;
- rtx rperm[16], vperm, target, op0, op1;
+ unsigned int i, j, chunk;
- if (!(d->op0 == d->op1 ? TARGET_SSSE3 : TARGET_XOP))
- return false;
- if (GET_MODE_SIZE (d->vmode) != 16)
+ if (GET_MODE_CLASS (vmode) != MODE_VECTOR_INT
+ || GET_MODE_CLASS (d->vmode) != MODE_VECTOR_INT
+ || GET_MODE_SIZE (vmode) != GET_MODE_SIZE (d->vmode))
return false;
- if (d->testing_p)
+ if (GET_MODE_NUNITS (vmode) >= d->nelt)
return true;
+ chunk = d->nelt / GET_MODE_NUNITS (vmode);
+ for (i = 0; i < d->nelt; i += chunk)
+ if (d->perm[i] & (chunk - 1))
+ return false;
+ else
+ for (j = 1; j < chunk; ++j)
+ if (d->perm[i] + j != d->perm[i + j])
+ return false;
+
+ return true;
+}
+
+/* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D
+ in terms of pshufb, vpperm, vpermq, vpermd or vperm2i128. */
+
+static bool
+expand_vec_perm_pshufb (struct expand_vec_perm_d *d)
+{
+ unsigned i, nelt, eltsz, mask;
+ unsigned char perm[32];
+ enum machine_mode vmode = V16QImode;
+ rtx rperm[32], vperm, target, op0, op1;
+
nelt = d->nelt;
- eltsz = GET_MODE_SIZE (GET_MODE_INNER (d->vmode));
- for (i = 0; i < nelt; ++i)
+ if (d->op0 != d->op1)
{
- unsigned j, e = d->perm[i];
- for (j = 0; j < eltsz; ++j)
- rperm[i * eltsz + j] = GEN_INT (e * eltsz + j);
+ if (!TARGET_XOP || GET_MODE_SIZE (d->vmode) != 16)
+ {
+ if (TARGET_AVX2
+ && valid_perm_using_mode_p (V2TImode, d))
+ {
+ if (d->testing_p)
+ return true;
+
+ /* Use vperm2i128 insn. The pattern uses
+ V4DImode instead of V2TImode. */
+ target = gen_lowpart (V4DImode, d->target);
+ op0 = gen_lowpart (V4DImode, d->op0);
+ op1 = gen_lowpart (V4DImode, d->op1);
+ rperm[0]
+ = GEN_INT (((d->perm[0] & (nelt / 2)) ? 1 : 0)
+ || ((d->perm[nelt / 2] & (nelt / 2)) ? 2 : 0));
+ emit_insn (gen_avx2_permv2ti (target, op0, op1, rperm[0]));
+ return true;
+ }
+ return false;
+ }
}
+ else
+ {
+ if (GET_MODE_SIZE (d->vmode) == 16)
+ {
+ if (!TARGET_SSSE3)
+ return false;
+ }
+ else if (GET_MODE_SIZE (d->vmode) == 32)
+ {
+ if (!TARGET_AVX2)
+ return false;
- vperm = gen_rtx_CONST_VECTOR (V16QImode, gen_rtvec_v (16, rperm));
- vperm = force_reg (V16QImode, vperm);
+ /* V4DImode should be already handled through
+ expand_vselect by vpermq instruction. */
+ gcc_assert (d->vmode != V4DImode);
- target = gen_lowpart (V16QImode, d->target);
- op0 = gen_lowpart (V16QImode, d->op0);
+ vmode = V32QImode;
+ if (d->vmode == V8SImode
+ || d->vmode == V16HImode
+ || d->vmode == V32QImode)
+ {
+ /* First see if vpermq can be used for
+ V8SImode/V16HImode/V32QImode. */
+ if (valid_perm_using_mode_p (V4DImode, d))
+ {
+ for (i = 0; i < 4; i++)
+ perm[i] = (d->perm[i * nelt / 4] * 4 / nelt) & 3;
+ if (d->testing_p)
+ return true;
+ return expand_vselect (gen_lowpart (V4DImode, d->target),
+ gen_lowpart (V4DImode, d->op0),
+ perm, 4);
+ }
+
+ /* Next see if vpermd can be used. */
+ if (valid_perm_using_mode_p (V8SImode, d))
+ vmode = V8SImode;
+ }
+
+ if (vmode == V32QImode)
+ {
+ /* vpshufb only works intra lanes, it is not
+ possible to shuffle bytes in between the lanes. */
+ for (i = 0; i < nelt; ++i)
+ if ((d->perm[i] ^ i) & (nelt / 2))
+ return false;
+ }
+ }
+ else
+ return false;
+ }
+
+ if (d->testing_p)
+ return true;
+
+ if (vmode == V8SImode)
+ for (i = 0; i < 8; ++i)
+ rperm[i] = GEN_INT ((d->perm[i * nelt / 8] * 8 / nelt) & 7);
+ else
+ {
+ eltsz = GET_MODE_SIZE (GET_MODE_INNER (d->vmode));
+ if (d->op0 != d->op1)
+ mask = 2 * nelt - 1;
+ else if (vmode == V16QImode)
+ mask = nelt - 1;
+ else
+ mask = nelt / 2 - 1;
+
+ for (i = 0; i < nelt; ++i)
+ {
+ unsigned j, e = d->perm[i] & mask;
+ for (j = 0; j < eltsz; ++j)
+ rperm[i * eltsz + j] = GEN_INT (e * eltsz + j);
+ }
+ }
+
+ vperm = gen_rtx_CONST_VECTOR (vmode,
+ gen_rtvec_v (GET_MODE_NUNITS (vmode), rperm));
+ vperm = force_reg (vmode, vperm);
+
+ target = gen_lowpart (vmode, d->target);
+ op0 = gen_lowpart (vmode, d->op0);
if (d->op0 == d->op1)
- emit_insn (gen_ssse3_pshufbv16qi3 (target, op0, vperm));
+ {
+ if (vmode == V16QImode)
+ emit_insn (gen_ssse3_pshufbv16qi3 (target, op0, vperm));
+ else if (vmode == V32QImode)
+ emit_insn (gen_avx2_pshufbv32qi3 (target, op0, vperm));
+ else
+ emit_insn (gen_avx2_permvarv8si (target, vperm, op0));
+ }
else
{
- op1 = gen_lowpart (V16QImode, d->op1);
+ op1 = gen_lowpart (vmode, d->op1);
emit_insn (gen_xop_pperm (target, op0, op1, vperm));
}
if (d->op0 == d->op1)
{
int mask = nelt - 1;
+ bool identity_perm = true;
+ bool broadcast_perm = true;
for (i = 0; i < nelt; i++)
- perm2[i] = d->perm[i] & mask;
+ {
+ perm2[i] = d->perm[i] & mask;
+ if (perm2[i] != i)
+ identity_perm = false;
+ if (perm2[i])
+ broadcast_perm = false;
+ }
+
+ if (identity_perm)
+ {
+ if (!d->testing_p)
+ emit_move_insn (d->target, d->op0);
+ return true;
+ }
+ else if (broadcast_perm && TARGET_AVX2)
+ {
+ /* Use vpbroadcast{b,w,d}. */
+ rtx op = d->op0, (*gen) (rtx, rtx) = NULL;
+ switch (d->vmode)
+ {
+ case V32QImode:
+ op = gen_lowpart (V16QImode, op);
+ gen = gen_avx2_pbroadcastv32qi;
+ break;
+ case V16HImode:
+ op = gen_lowpart (V8HImode, op);
+ gen = gen_avx2_pbroadcastv16hi;
+ break;
+ case V8SImode:
+ op = gen_lowpart (V4SImode, op);
+ gen = gen_avx2_pbroadcastv8si;
+ break;
+ case V16QImode:
+ gen = gen_avx2_pbroadcastv16qi;
+ break;
+ case V8HImode:
+ gen = gen_avx2_pbroadcastv8hi;
+ break;
+ /* For other modes prefer other shuffles this function creates. */
+ default: break;
+ }
+ if (gen != NULL)
+ {
+ if (!d->testing_p)
+ emit_insn (gen (d->target, op));
+ return true;
+ }
+ }
if (expand_vselect (d->target, d->op0, perm2, nelt))
return true;
if (expand_vec_perm_vpermil (d))
return true;
- /* Try the SSSE3 pshufb or XOP vpperm variable permutation. */
+ /* Try the SSSE3 pshufb or XOP vpperm or AVX2 vperm2i128,
+ vpshufb, vpermd or vpermq variable permutation. */
if (expand_vec_perm_pshufb (d))
return true;
{
struct expand_vec_perm_d dremap, dfinal;
unsigned i, nelt = d->nelt, nelt2 = nelt / 2;
- unsigned contents, h1, h2, h3, h4;
+ unsigned HOST_WIDE_INT contents;
unsigned char remap[2 * MAX_VECT_LEN];
rtx seq;
- bool ok;
+ bool ok, same_halves = false;
- if (d->op0 == d->op1)
- return false;
-
- /* The 256-bit unpck[lh]p[sd] instructions only operate within the 128-bit
- lanes. We can use similar techniques with the vperm2f128 instruction,
- but it requires slightly different logic. */
- if (GET_MODE_SIZE (d->vmode) != 16)
+ if (GET_MODE_SIZE (d->vmode) == 16)
+ {
+ if (d->op0 == d->op1)
+ return false;
+ }
+ else if (GET_MODE_SIZE (d->vmode) == 32)
+ {
+ if (!TARGET_AVX)
+ return false;
+ /* For 32-byte modes allow even d->op0 == d->op1.
+ The lack of cross-lane shuffling in some instructions
+ might prevent a single insn shuffle. */
+ }
+ else
return false;
/* Examine from whence the elements come. */
contents = 0;
for (i = 0; i < nelt; ++i)
- contents |= 1u << d->perm[i];
-
- /* Split the two input vectors into 4 halves. */
- h1 = (1u << nelt2) - 1;
- h2 = h1 << nelt2;
- h3 = h2 << nelt2;
- h4 = h3 << nelt2;
+ contents |= ((unsigned HOST_WIDE_INT) 1) << d->perm[i];
memset (remap, 0xff, sizeof (remap));
dremap = *d;
- /* If the elements from the low halves use interleave low, and similarly
- for interleave high. If the elements are from mis-matched halves, we
- can use shufps for V4SF/V4SI or do a DImode shuffle. */
- if ((contents & (h1 | h3)) == contents)
+ if (GET_MODE_SIZE (d->vmode) == 16)
{
- for (i = 0; i < nelt2; ++i)
+ unsigned HOST_WIDE_INT h1, h2, h3, h4;
+
+ /* Split the two input vectors into 4 halves. */
+ h1 = (((unsigned HOST_WIDE_INT) 1) << nelt2) - 1;
+ h2 = h1 << nelt2;
+ h3 = h2 << nelt2;
+ h4 = h3 << nelt2;
+
+ /* If the elements from the low halves use interleave low, and similarly
+ for interleave high. If the elements are from mis-matched halves, we
+ can use shufps for V4SF/V4SI or do a DImode shuffle. */
+ if ((contents & (h1 | h3)) == contents)
{
- remap[i] = i * 2;
- remap[i + nelt] = i * 2 + 1;
- dremap.perm[i * 2] = i;
- dremap.perm[i * 2 + 1] = i + nelt;
+ /* punpckl* */
+ for (i = 0; i < nelt2; ++i)
+ {
+ remap[i] = i * 2;
+ remap[i + nelt] = i * 2 + 1;
+ dremap.perm[i * 2] = i;
+ dremap.perm[i * 2 + 1] = i + nelt;
+ }
+ if (!TARGET_SSE2 && d->vmode == V4SImode)
+ dremap.vmode = V4SFmode;
}
- }
- else if ((contents & (h2 | h4)) == contents)
- {
- for (i = 0; i < nelt2; ++i)
+ else if ((contents & (h2 | h4)) == contents)
{
- remap[i + nelt2] = i * 2;
- remap[i + nelt + nelt2] = i * 2 + 1;
- dremap.perm[i * 2] = i + nelt2;
- dremap.perm[i * 2 + 1] = i + nelt + nelt2;
+ /* punpckh* */
+ for (i = 0; i < nelt2; ++i)
+ {
+ remap[i + nelt2] = i * 2;
+ remap[i + nelt + nelt2] = i * 2 + 1;
+ dremap.perm[i * 2] = i + nelt2;
+ dremap.perm[i * 2 + 1] = i + nelt + nelt2;
+ }
+ if (!TARGET_SSE2 && d->vmode == V4SImode)
+ dremap.vmode = V4SFmode;
}
- }
- else if ((contents & (h1 | h4)) == contents)
- {
- for (i = 0; i < nelt2; ++i)
+ else if ((contents & (h1 | h4)) == contents)
{
- remap[i] = i;
- remap[i + nelt + nelt2] = i + nelt2;
- dremap.perm[i] = i;
- dremap.perm[i + nelt2] = i + nelt + nelt2;
+ /* shufps */
+ for (i = 0; i < nelt2; ++i)
+ {
+ remap[i] = i;
+ remap[i + nelt + nelt2] = i + nelt2;
+ dremap.perm[i] = i;
+ dremap.perm[i + nelt2] = i + nelt + nelt2;
+ }
+ if (nelt != 4)
+ {
+ /* shufpd */
+ dremap.vmode = V2DImode;
+ dremap.nelt = 2;
+ dremap.perm[0] = 0;
+ dremap.perm[1] = 3;
+ }
}
- if (nelt != 4)
+ else if ((contents & (h2 | h3)) == contents)
{
- dremap.vmode = V2DImode;
- dremap.nelt = 2;
- dremap.perm[0] = 0;
- dremap.perm[1] = 3;
+ /* shufps */
+ for (i = 0; i < nelt2; ++i)
+ {
+ remap[i + nelt2] = i;
+ remap[i + nelt] = i + nelt2;
+ dremap.perm[i] = i + nelt2;
+ dremap.perm[i + nelt2] = i + nelt;
+ }
+ if (nelt != 4)
+ {
+ /* shufpd */
+ dremap.vmode = V2DImode;
+ dremap.nelt = 2;
+ dremap.perm[0] = 1;
+ dremap.perm[1] = 2;
+ }
}
+ else
+ return false;
}
- else if ((contents & (h2 | h3)) == contents)
+ else
{
- for (i = 0; i < nelt2; ++i)
+ unsigned int nelt4 = nelt / 4, nzcnt = 0;
+ unsigned HOST_WIDE_INT q[8];
+ unsigned int nonzero_halves[4];
+
+ /* Split the two input vectors into 8 quarters. */
+ q[0] = (((unsigned HOST_WIDE_INT) 1) << nelt4) - 1;
+ for (i = 1; i < 8; ++i)
+ q[i] = q[0] << (nelt4 * i);
+ for (i = 0; i < 4; ++i)
+ if (((q[2 * i] | q[2 * i + 1]) & contents) != 0)
+ {
+ nonzero_halves[nzcnt] = i;
+ ++nzcnt;
+ }
+
+ if (nzcnt == 1)
+ {
+ gcc_assert (d->op0 == d->op1);
+ nonzero_halves[1] = nonzero_halves[0];
+ same_halves = true;
+ }
+ else if (d->op0 == d->op1)
{
- remap[i + nelt2] = i;
- remap[i + nelt] = i + nelt2;
- dremap.perm[i] = i + nelt2;
- dremap.perm[i + nelt2] = i + nelt;
+ gcc_assert (nonzero_halves[0] == 0);
+ gcc_assert (nonzero_halves[1] == 1);
}
- if (nelt != 4)
+
+ if (nzcnt <= 2)
{
- dremap.vmode = V2DImode;
- dremap.nelt = 2;
- dremap.perm[0] = 1;
- dremap.perm[1] = 2;
+ if (d->perm[0] / nelt2 == nonzero_halves[1])
+ {
+ /* Attempt to increase the likelyhood that dfinal
+ shuffle will be intra-lane. */
+ char tmph = nonzero_halves[0];
+ nonzero_halves[0] = nonzero_halves[1];
+ nonzero_halves[1] = tmph;
+ }
+
+ /* vperm2f128 or vperm2i128. */
+ for (i = 0; i < nelt2; ++i)
+ {
+ remap[i + nonzero_halves[1] * nelt2] = i + nelt2;
+ remap[i + nonzero_halves[0] * nelt2] = i;
+ dremap.perm[i + nelt2] = i + nonzero_halves[1] * nelt2;
+ dremap.perm[i] = i + nonzero_halves[0] * nelt2;
+ }
+
+ if (d->vmode != V8SFmode
+ && d->vmode != V4DFmode
+ && d->vmode != V8SImode)
+ {
+ dremap.vmode = V8SImode;
+ dremap.nelt = 8;
+ for (i = 0; i < 4; ++i)
+ {
+ dremap.perm[i] = i + nonzero_halves[0] * 4;
+ dremap.perm[i + 4] = i + nonzero_halves[1] * 4;
+ }
+ }
}
+ else if (d->op0 == d->op1)
+ return false;
+ else if (TARGET_AVX2
+ && (contents & (q[0] | q[2] | q[4] | q[6])) == contents)
+ {
+ /* vpunpckl* */
+ for (i = 0; i < nelt4; ++i)
+ {
+ remap[i] = i * 2;
+ remap[i + nelt] = i * 2 + 1;
+ remap[i + nelt2] = i * 2 + nelt2;
+ remap[i + nelt + nelt2] = i * 2 + nelt2 + 1;
+ dremap.perm[i * 2] = i;
+ dremap.perm[i * 2 + 1] = i + nelt;
+ dremap.perm[i * 2 + nelt2] = i + nelt2;
+ dremap.perm[i * 2 + nelt2 + 1] = i + nelt + nelt2;
+ }
+ }
+ else if (TARGET_AVX2
+ && (contents & (q[1] | q[3] | q[5] | q[7])) == contents)
+ {
+ /* vpunpckh* */
+ for (i = 0; i < nelt4; ++i)
+ {
+ remap[i + nelt4] = i * 2;
+ remap[i + nelt + nelt4] = i * 2 + 1;
+ remap[i + nelt2 + nelt4] = i * 2 + nelt2;
+ remap[i + nelt + nelt2 + nelt4] = i * 2 + nelt2 + 1;
+ dremap.perm[i * 2] = i + nelt4;
+ dremap.perm[i * 2 + 1] = i + nelt + nelt4;
+ dremap.perm[i * 2 + nelt2] = i + nelt2 + nelt4;
+ dremap.perm[i * 2 + nelt2 + 1] = i + nelt + nelt2 + nelt4;
+ }
+ }
+ else
+ return false;
}
- else
- return false;
/* Use the remapping array set up above to move the elements from their
swizzled locations into their final destinations. */
{
unsigned e = remap[d->perm[i]];
gcc_assert (e < nelt);
- dfinal.perm[i] = e;
+ /* If same_halves is true, both halves of the remapped vector are the
+ same. Avoid cross-lane accesses if possible. */
+ if (same_halves && i >= nelt2)
+ {
+ gcc_assert (e < nelt2);
+ dfinal.perm[i] = e + nelt2;
+ }
+ else
+ dfinal.perm[i] = e;
}
dfinal.op0 = gen_reg_rtx (dfinal.vmode);
dfinal.op1 = dfinal.op0;
if (!ok)
return false;
+ if (d->testing_p)
+ return true;
+
if (dremap.vmode != dfinal.vmode)
{
dremap.target = gen_lowpart (dremap.vmode, dremap.target);
return true;
}
+/* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
+ a single vector cross-lane permutation into vpermq followed
+ by any of the single insn permutations. */
+
+static bool
+expand_vec_perm_vpermq_perm_1 (struct expand_vec_perm_d *d)
+{
+ struct expand_vec_perm_d dremap, dfinal;
+ unsigned i, j, nelt = d->nelt, nelt2 = nelt / 2, nelt4 = nelt / 4;
+ unsigned contents[2];
+ bool ok;
+
+ if (!(TARGET_AVX2
+ && (d->vmode == V32QImode || d->vmode == V16HImode)
+ && d->op0 == d->op1))
+ return false;
+
+ contents[0] = 0;
+ contents[1] = 0;
+ for (i = 0; i < nelt2; ++i)
+ {
+ contents[0] |= 1u << (d->perm[i] / nelt4);
+ contents[1] |= 1u << (d->perm[i + nelt2] / nelt4);
+ }
+
+ for (i = 0; i < 2; ++i)
+ {
+ unsigned int cnt = 0;
+ for (j = 0; j < 4; ++j)
+ if ((contents[i] & (1u << j)) != 0 && ++cnt > 2)
+ return false;
+ }
+
+ if (d->testing_p)
+ return true;
+
+ dremap = *d;
+ dremap.vmode = V4DImode;
+ dremap.nelt = 4;
+ dremap.target = gen_reg_rtx (V4DImode);
+ dremap.op0 = gen_lowpart (V4DImode, d->op0);
+ dremap.op1 = dremap.op0;
+ for (i = 0; i < 2; ++i)
+ {
+ unsigned int cnt = 0;
+ for (j = 0; j < 4; ++j)
+ if ((contents[i] & (1u << j)) != 0)
+ dremap.perm[2 * i + cnt++] = j;
+ for (; cnt < 2; ++cnt)
+ dremap.perm[2 * i + cnt] = 0;
+ }
+
+ dfinal = *d;
+ dfinal.op0 = gen_lowpart (dfinal.vmode, dremap.target);
+ dfinal.op1 = dfinal.op0;
+ for (i = 0, j = 0; i < nelt; ++i)
+ {
+ if (i == nelt2)
+ j = 2;
+ dfinal.perm[i] = (d->perm[i] & (nelt4 - 1)) | (j ? nelt2 : 0);
+ if ((d->perm[i] / nelt4) == dremap.perm[j])
+ ;
+ else if ((d->perm[i] / nelt4) == dremap.perm[j + 1])
+ dfinal.perm[i] |= nelt4;
+ else
+ gcc_unreachable ();
+ }
+
+ ok = expand_vec_perm_1 (&dremap);
+ gcc_assert (ok);
+
+ ok = expand_vec_perm_1 (&dfinal);
+ gcc_assert (ok);
+
+ return true;
+}
+
+/* A subroutine of ix86_expand_vec_perm_builtin_1. Try to simplify
+ a two vector permutation using 2 intra-lane interleave insns
+ and cross-lane shuffle for 32-byte vectors. */
+
+static bool
+expand_vec_perm_interleave3 (struct expand_vec_perm_d *d)
+{
+ unsigned i, nelt;
+ rtx (*gen) (rtx, rtx, rtx);
+
+ if (d->op0 == d->op1)
+ return false;
+ if (TARGET_AVX2 && GET_MODE_SIZE (d->vmode) == 32)
+ ;
+ else if (TARGET_AVX && (d->vmode == V8SFmode || d->vmode == V4DFmode))
+ ;
+ else
+ return false;
+
+ nelt = d->nelt;
+ if (d->perm[0] != 0 && d->perm[0] != nelt / 2)
+ return false;
+ for (i = 0; i < nelt; i += 2)
+ if (d->perm[i] != d->perm[0] + i / 2
+ || d->perm[i + 1] != d->perm[0] + i / 2 + nelt)
+ return false;
+
+ if (d->testing_p)
+ return true;
+
+ switch (d->vmode)
+ {
+ case V32QImode:
+ if (d->perm[0])
+ gen = gen_vec_interleave_highv32qi;
+ else
+ gen = gen_vec_interleave_lowv32qi;
+ break;
+ case V16HImode:
+ if (d->perm[0])
+ gen = gen_vec_interleave_highv16hi;
+ else
+ gen = gen_vec_interleave_lowv16hi;
+ break;
+ case V8SImode:
+ if (d->perm[0])
+ gen = gen_vec_interleave_highv8si;
+ else
+ gen = gen_vec_interleave_lowv8si;
+ break;
+ case V4DImode:
+ if (d->perm[0])
+ gen = gen_vec_interleave_highv4di;
+ else
+ gen = gen_vec_interleave_lowv4di;
+ break;
+ case V8SFmode:
+ if (d->perm[0])
+ gen = gen_vec_interleave_highv8sf;
+ else
+ gen = gen_vec_interleave_lowv8sf;
+ break;
+ case V4DFmode:
+ if (d->perm[0])
+ gen = gen_vec_interleave_highv4df;
+ else
+ gen = gen_vec_interleave_lowv4df;
+ break;
+ default:
+ gcc_unreachable ();
+ }
+
+ emit_insn (gen (d->target, d->op0, d->op1));
+ return true;
+}
+
/* A subroutine of expand_vec_perm_even_odd_1. Implement the double-word
permutation with two pshufb insns and an ior. We should have already
failed all two instruction sequences. */
return true;
}
+/* Implement arbitrary permutation of one V32QImode and V16QImode operand
+ with two vpshufb insns, vpermq and vpor. We should have already failed
+ all two or three instruction sequences. */
+
+static bool
+expand_vec_perm_vpshufb2_vpermq (struct expand_vec_perm_d *d)
+{
+ rtx rperm[2][32], vperm, l, h, hp, op, m128;
+ unsigned int i, nelt, eltsz;
+
+ if (!TARGET_AVX2
+ || d->op0 != d->op1
+ || (d->vmode != V32QImode && d->vmode != V16HImode))
+ return false;
+
+ if (d->testing_p)
+ return true;
+
+ nelt = d->nelt;
+ eltsz = GET_MODE_SIZE (GET_MODE_INNER (d->vmode));
+
+ /* Generate two permutation masks. If the required element is within
+ the same lane, it is shuffled in. If the required element from the
+ other lane, force a zero by setting bit 7 in the permutation mask.
+ In the other mask the mask has non-negative elements if element
+ is requested from the other lane, but also moved to the other lane,
+ so that the result of vpshufb can have the two V2TImode halves
+ swapped. */
+ m128 = GEN_INT (-128);
+ for (i = 0; i < nelt; ++i)
+ {
+ unsigned j, e = d->perm[i] & (nelt / 2 - 1);
+ unsigned which = ((d->perm[i] ^ i) & (nelt / 2)) * eltsz;
+
+ for (j = 0; j < eltsz; ++j)
+ {
+ rperm[!!which][(i * eltsz + j) ^ which] = GEN_INT (e * eltsz + j);
+ rperm[!which][(i * eltsz + j) ^ (which ^ 16)] = m128;
+ }
+ }
+
+ vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[1]));
+ vperm = force_reg (V32QImode, vperm);
+
+ h = gen_reg_rtx (V32QImode);
+ op = gen_lowpart (V32QImode, d->op0);
+ emit_insn (gen_avx2_pshufbv32qi3 (h, op, vperm));
+
+ /* Swap the 128-byte lanes of h into hp. */
+ hp = gen_reg_rtx (V4DImode);
+ op = gen_lowpart (V4DImode, h);
+ emit_insn (gen_avx2_permv4di_1 (hp, op, const2_rtx, GEN_INT (3), const0_rtx,
+ const1_rtx));
+
+ vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[0]));
+ vperm = force_reg (V32QImode, vperm);
+
+ l = gen_reg_rtx (V32QImode);
+ op = gen_lowpart (V32QImode, d->op0);
+ emit_insn (gen_avx2_pshufbv32qi3 (l, op, vperm));
+
+ op = gen_lowpart (V32QImode, d->target);
+ emit_insn (gen_iorv32qi3 (op, l, gen_lowpart (V32QImode, hp)));
+
+ return true;
+}
+
+/* A subroutine of expand_vec_perm_even_odd_1. Implement extract-even
+ and extract-odd permutations of two V32QImode and V16QImode operand
+ with two vpshufb insns, vpor and vpermq. We should have already
+ failed all two or three instruction sequences. */
+
+static bool
+expand_vec_perm_vpshufb2_vpermq_even_odd (struct expand_vec_perm_d *d)
+{
+ rtx rperm[2][32], vperm, l, h, ior, op, m128;
+ unsigned int i, nelt, eltsz;
+
+ if (!TARGET_AVX2
+ || d->op0 == d->op1
+ || (d->vmode != V32QImode && d->vmode != V16HImode))
+ return false;
+
+ for (i = 0; i < d->nelt; ++i)
+ if ((d->perm[i] ^ (i * 2)) & (3 * d->nelt / 2))
+ return false;
+
+ if (d->testing_p)
+ return true;
+
+ nelt = d->nelt;
+ eltsz = GET_MODE_SIZE (GET_MODE_INNER (d->vmode));
+
+ /* Generate two permutation masks. In the first permutation mask
+ the first quarter will contain indexes for the first half
+ of the op0, the second quarter will contain bit 7 set, third quarter
+ will contain indexes for the second half of the op0 and the
+ last quarter bit 7 set. In the second permutation mask
+ the first quarter will contain bit 7 set, the second quarter
+ indexes for the first half of the op1, the third quarter bit 7 set
+ and last quarter indexes for the second half of the op1.
+ I.e. the first mask e.g. for V32QImode extract even will be:
+ 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128
+ (all values masked with 0xf except for -128) and second mask
+ for extract even will be
+ -128, ..., -128, 0, 2, ..., 0xe, -128, ..., -128, 0, 2, ..., 0xe. */
+ m128 = GEN_INT (-128);
+ for (i = 0; i < nelt; ++i)
+ {
+ unsigned j, e = d->perm[i] & (nelt / 2 - 1);
+ unsigned which = d->perm[i] >= nelt;
+ unsigned xorv = (i >= nelt / 4 && i < 3 * nelt / 4) ? 24 : 0;
+
+ for (j = 0; j < eltsz; ++j)
+ {
+ rperm[which][(i * eltsz + j) ^ xorv] = GEN_INT (e * eltsz + j);
+ rperm[1 - which][(i * eltsz + j) ^ xorv] = m128;
+ }
+ }
+
+ vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[0]));
+ vperm = force_reg (V32QImode, vperm);
+
+ l = gen_reg_rtx (V32QImode);
+ op = gen_lowpart (V32QImode, d->op0);
+ emit_insn (gen_avx2_pshufbv32qi3 (l, op, vperm));
+
+ vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[1]));
+ vperm = force_reg (V32QImode, vperm);
+
+ h = gen_reg_rtx (V32QImode);
+ op = gen_lowpart (V32QImode, d->op1);
+ emit_insn (gen_avx2_pshufbv32qi3 (h, op, vperm));
+
+ ior = gen_reg_rtx (V32QImode);
+ emit_insn (gen_iorv32qi3 (ior, l, h));
+
+ /* Permute the V4DImode quarters using { 0, 2, 1, 3 } permutation. */
+ op = gen_lowpart (V4DImode, d->target);
+ ior = gen_lowpart (V4DImode, ior);
+ emit_insn (gen_avx2_permv4di_1 (op, ior, const0_rtx, const2_rtx,
+ const1_rtx, GEN_INT (3)));
+
+ return true;
+}
+
/* A subroutine of ix86_expand_vec_perm_builtin_1. Implement extract-even
and extract-odd permutations. */
}
break;
+ case V16HImode:
+ case V32QImode:
+ return expand_vec_perm_vpshufb2_vpermq_even_odd (d);
+
+ case V4DImode:
+ if (!TARGET_AVX2)
+ {
+ struct expand_vec_perm_d d_copy = *d;
+ d_copy.vmode = V4DFmode;
+ d_copy.target = gen_lowpart (V4DFmode, d->target);
+ d_copy.op0 = gen_lowpart (V4DFmode, d->op0);
+ d_copy.op1 = gen_lowpart (V4DFmode, d->op1);
+ return expand_vec_perm_even_odd_1 (&d_copy, odd);
+ }
+
+ t1 = gen_reg_rtx (V4DImode);
+ t2 = gen_reg_rtx (V4DImode);
+
+ /* Shuffle the lanes around into { 0 1 4 5 } and { 2 3 6 7 }. */
+ emit_insn (gen_avx2_permv2ti (t1, d->op0, d->op1, GEN_INT (0x20)));
+ emit_insn (gen_avx2_permv2ti (t2, d->op0, d->op1, GEN_INT (0x31)));
+
+ /* Now an vpunpck[lh]qdq will produce the result required. */
+ if (odd)
+ t3 = gen_avx2_interleave_highv4di (d->target, t1, t2);
+ else
+ t3 = gen_avx2_interleave_lowv4di (d->target, t1, t2);
+ emit_insn (t3);
+ break;
+
+ case V8SImode:
+ if (!TARGET_AVX2)
+ {
+ struct expand_vec_perm_d d_copy = *d;
+ d_copy.vmode = V8SFmode;
+ d_copy.target = gen_lowpart (V8SFmode, d->target);
+ d_copy.op0 = gen_lowpart (V8SFmode, d->op0);
+ d_copy.op1 = gen_lowpart (V8SFmode, d->op1);
+ return expand_vec_perm_even_odd_1 (&d_copy, odd);
+ }
+
+ t1 = gen_reg_rtx (V8SImode);
+ t2 = gen_reg_rtx (V8SImode);
+
+ /* Shuffle the lanes around into
+ { 0 1 2 3 8 9 a b } and { 4 5 6 7 c d e f }. */
+ emit_insn (gen_avx2_permv2ti (gen_lowpart (V4DImode, t1),
+ gen_lowpart (V4DImode, d->op0),
+ gen_lowpart (V4DImode, d->op1),
+ GEN_INT (0x20)));
+ emit_insn (gen_avx2_permv2ti (gen_lowpart (V4DImode, t2),
+ gen_lowpart (V4DImode, d->op0),
+ gen_lowpart (V4DImode, d->op1),
+ GEN_INT (0x31)));
+
+ /* Swap the 2nd and 3rd position in each lane into
+ { 0 2 1 3 8 a 9 b } and { 4 6 5 7 c e d f }. */
+ emit_insn (gen_avx2_pshufdv3 (t1, t1,
+ GEN_INT (2 * 4 + 1 * 16 + 3 * 64)));
+ emit_insn (gen_avx2_pshufdv3 (t2, t2,
+ GEN_INT (2 * 4 + 1 * 16 + 3 * 64)));
+
+ /* Now an vpunpck[lh]qdq will produce
+ { 0 2 4 6 8 a c e } resp. { 1 3 5 7 9 b d f }. */
+ if (odd)
+ t3 = gen_avx2_interleave_highv4di (gen_lowpart (V4DImode, d->target),
+ gen_lowpart (V4DImode, t1),
+ gen_lowpart (V4DImode, t2));
+ else
+ t3 = gen_avx2_interleave_lowv4di (gen_lowpart (V4DImode, d->target),
+ gen_lowpart (V4DImode, t1),
+ gen_lowpart (V4DImode, t2));
+ emit_insn (t3);
+ break;
+
default:
gcc_unreachable ();
}
gcc_assert (ok);
return true;
+ case V32QImode:
+ case V16HImode:
+ case V8SImode:
+ case V4DImode:
+ /* For AVX2 broadcasts of the first element vpbroadcast* or
+ vpermq should be used by expand_vec_perm_1. */
+ gcc_assert (!TARGET_AVX2 || d->perm[0]);
+ return false;
+
default:
gcc_unreachable ();
}
return expand_vec_perm_broadcast_1 (d);
}
-/* The guts of ix86_expand_vec_perm_builtin, also used by the ok hook.
+/* Implement arbitrary permutation of two V32QImode and V16QImode operands
+ with 4 vpshufb insns, 2 vpermq and 3 vpor. We should have already failed
+ all the shorter instruction sequences. */
+
+static bool
+expand_vec_perm_vpshufb4_vpermq2 (struct expand_vec_perm_d *d)
+{
+ rtx rperm[4][32], vperm, l[2], h[2], op, m128;
+ unsigned int i, nelt, eltsz;
+ bool used[4];
+
+ if (!TARGET_AVX2
+ || d->op0 == d->op1
+ || (d->vmode != V32QImode && d->vmode != V16HImode))
+ return false;
+
+ if (d->testing_p)
+ return true;
+
+ nelt = d->nelt;
+ eltsz = GET_MODE_SIZE (GET_MODE_INNER (d->vmode));
+
+ /* Generate 4 permutation masks. If the required element is within
+ the same lane, it is shuffled in. If the required element from the
+ other lane, force a zero by setting bit 7 in the permutation mask.
+ In the other mask the mask has non-negative elements if element
+ is requested from the other lane, but also moved to the other lane,
+ so that the result of vpshufb can have the two V2TImode halves
+ swapped. */
+ m128 = GEN_INT (-128);
+ for (i = 0; i < 32; ++i)
+ {
+ rperm[0][i] = m128;
+ rperm[1][i] = m128;
+ rperm[2][i] = m128;
+ rperm[3][i] = m128;
+ }
+ used[0] = false;
+ used[1] = false;
+ used[2] = false;
+ used[3] = false;
+ for (i = 0; i < nelt; ++i)
+ {
+ unsigned j, e = d->perm[i] & (nelt / 2 - 1);
+ unsigned xlane = ((d->perm[i] ^ i) & (nelt / 2)) * eltsz;
+ unsigned int which = ((d->perm[i] & nelt) ? 2 : 0) + (xlane ? 1 : 0);
+
+ for (j = 0; j < eltsz; ++j)
+ rperm[which][(i * eltsz + j) ^ xlane] = GEN_INT (e * eltsz + j);
+ used[which] = true;
+ }
+
+ for (i = 0; i < 2; ++i)
+ {
+ if (!used[2 * i + 1])
+ {
+ h[i] = NULL_RTX;
+ continue;
+ }
+ vperm = gen_rtx_CONST_VECTOR (V32QImode,
+ gen_rtvec_v (32, rperm[2 * i + 1]));
+ vperm = force_reg (V32QImode, vperm);
+ h[i] = gen_reg_rtx (V32QImode);
+ op = gen_lowpart (V32QImode, i ? d->op1 : d->op0);
+ emit_insn (gen_avx2_pshufbv32qi3 (h[i], op, vperm));
+ }
+
+ /* Swap the 128-byte lanes of h[X]. */
+ for (i = 0; i < 2; ++i)
+ {
+ if (h[i] == NULL_RTX)
+ continue;
+ op = gen_reg_rtx (V4DImode);
+ emit_insn (gen_avx2_permv4di_1 (op, gen_lowpart (V4DImode, h[i]),
+ const2_rtx, GEN_INT (3), const0_rtx,
+ const1_rtx));
+ h[i] = gen_lowpart (V32QImode, op);
+ }
+
+ for (i = 0; i < 2; ++i)
+ {
+ if (!used[2 * i])
+ {
+ l[i] = NULL_RTX;
+ continue;
+ }
+ vperm = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, rperm[2 * i]));
+ vperm = force_reg (V32QImode, vperm);
+ l[i] = gen_reg_rtx (V32QImode);
+ op = gen_lowpart (V32QImode, i ? d->op1 : d->op0);
+ emit_insn (gen_avx2_pshufbv32qi3 (l[i], op, vperm));
+ }
+
+ for (i = 0; i < 2; ++i)
+ {
+ if (h[i] && l[i])
+ {
+ op = gen_reg_rtx (V32QImode);
+ emit_insn (gen_iorv32qi3 (op, l[i], h[i]));
+ l[i] = op;
+ }
+ else if (h[i])
+ l[i] = h[i];
+ }
+
+ gcc_assert (l[0] && l[1]);
+ op = gen_lowpart (V32QImode, d->target);
+ emit_insn (gen_iorv32qi3 (op, l[0], l[1]));
+ return true;
+}
+
+/* The guts of ix86_expand_vec_perm_const, also used by the ok hook.
With all of the interface bits taken care of, perform the expansion
in D and return true on success. */
static bool
-ix86_expand_vec_perm_builtin_1 (struct expand_vec_perm_d *d)
+ix86_expand_vec_perm_const_1 (struct expand_vec_perm_d *d)
{
/* Try a single instruction expansion. */
if (expand_vec_perm_1 (d))
if (expand_vec_perm_broadcast (d))
return true;
+ if (expand_vec_perm_vpermq_perm_1 (d))
+ return true;
+
/* Try sequences of three instructions. */
if (expand_vec_perm_pshufb2 (d))
return true;
+ if (expand_vec_perm_interleave3 (d))
+ return true;
+
+ /* Try sequences of four instructions. */
+
+ if (expand_vec_perm_vpshufb2_vpermq (d))
+ return true;
+
+ if (expand_vec_perm_vpshufb2_vpermq_even_odd (d))
+ return true;
+
/* ??? Look for narrow permutations whose element orderings would
allow the promotion to a wider mode. */
if (expand_vec_perm_even_odd (d))
return true;
- return false;
-}
-
-/* Extract the values from the vector CST into the permutation array in D.
- Return 0 on error, 1 if all values from the permutation come from the
- first vector, 2 if all values from the second vector, and 3 otherwise. */
-
-static int
-extract_vec_perm_cst (struct expand_vec_perm_d *d, tree cst)
-{
- tree list = TREE_VECTOR_CST_ELTS (cst);
- unsigned i, nelt = d->nelt;
- int ret = 0;
-
- for (i = 0; i < nelt; ++i, list = TREE_CHAIN (list))
- {
- unsigned HOST_WIDE_INT e;
-
- if (!host_integerp (TREE_VALUE (list), 1))
- return 0;
- e = tree_low_cst (TREE_VALUE (list), 1);
- if (e >= 2 * nelt)
- return 0;
-
- ret |= (e < nelt ? 1 : 2);
- d->perm[i] = e;
- }
- gcc_assert (list == NULL);
-
- /* For all elements from second vector, fold the elements to first. */
- if (ret == 2)
- for (i = 0; i < nelt; ++i)
- d->perm[i] -= nelt;
+ /* Even longer sequences. */
+ if (expand_vec_perm_vpshufb4_vpermq2 (d))
+ return true;
- return ret;
+ return false;
}
-static rtx
-ix86_expand_vec_perm_builtin (tree exp)
+bool
+ix86_expand_vec_perm_const (rtx operands[4])
{
struct expand_vec_perm_d d;
- tree arg0, arg1, arg2;
+ unsigned char perm[MAX_VECT_LEN];
+ int i, nelt, which;
+ rtx sel;
- arg0 = CALL_EXPR_ARG (exp, 0);
- arg1 = CALL_EXPR_ARG (exp, 1);
- arg2 = CALL_EXPR_ARG (exp, 2);
+ d.target = operands[0];
+ d.op0 = operands[1];
+ d.op1 = operands[2];
+ sel = operands[3];
- d.vmode = TYPE_MODE (TREE_TYPE (arg0));
- d.nelt = GET_MODE_NUNITS (d.vmode);
- d.testing_p = false;
+ d.vmode = GET_MODE (d.target);
gcc_assert (VECTOR_MODE_P (d.vmode));
+ d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
+ d.testing_p = false;
+
+ gcc_assert (GET_CODE (sel) == CONST_VECTOR);
+ gcc_assert (XVECLEN (sel, 0) == nelt);
+ gcc_checking_assert (sizeof (d.perm) == sizeof (perm));
- if (TREE_CODE (arg2) != VECTOR_CST)
+ for (i = which = 0; i < nelt; ++i)
{
- error_at (EXPR_LOCATION (exp),
- "vector permutation requires vector constant");
- goto exit_error;
+ rtx e = XVECEXP (sel, 0, i);
+ int ei = INTVAL (e) & (2 * nelt - 1);
+
+ which |= (ei < nelt ? 1 : 2);
+ d.perm[i] = ei;
+ perm[i] = ei;
}
- switch (extract_vec_perm_cst (&d, arg2))
+ switch (which)
{
default:
gcc_unreachable();
- case 0:
- error_at (EXPR_LOCATION (exp), "invalid vector permutation constant");
- goto exit_error;
-
case 3:
- if (!operand_equal_p (arg0, arg1, 0))
- {
- d.op0 = expand_expr (arg0, NULL_RTX, d.vmode, EXPAND_NORMAL);
- d.op0 = force_reg (d.vmode, d.op0);
- d.op1 = expand_expr (arg1, NULL_RTX, d.vmode, EXPAND_NORMAL);
- d.op1 = force_reg (d.vmode, d.op1);
- break;
- }
+ if (!rtx_equal_p (d.op0, d.op1))
+ break;
/* The elements of PERM do not suggest that only the first operand
is used, but both operands are identical. Allow easier matching
of the permutation by folding the permutation into the single
input vector. */
- {
- unsigned i, nelt = d.nelt;
- for (i = 0; i < nelt; ++i)
- if (d.perm[i] >= nelt)
- d.perm[i] -= nelt;
- }
+ for (i = 0; i < nelt; ++i)
+ if (d.perm[i] >= nelt)
+ d.perm[i] -= nelt;
/* FALLTHRU */
case 1:
- d.op0 = expand_expr (arg0, NULL_RTX, d.vmode, EXPAND_NORMAL);
- d.op0 = force_reg (d.vmode, d.op0);
d.op1 = d.op0;
break;
case 2:
- d.op0 = expand_expr (arg1, NULL_RTX, d.vmode, EXPAND_NORMAL);
- d.op0 = force_reg (d.vmode, d.op0);
- d.op1 = d.op0;
+ for (i = 0; i < nelt; ++i)
+ d.perm[i] -= nelt;
+ d.op0 = d.op1;
break;
}
- d.target = gen_reg_rtx (d.vmode);
- if (ix86_expand_vec_perm_builtin_1 (&d))
- return d.target;
+ if (ix86_expand_vec_perm_const_1 (&d))
+ return true;
- /* For compiler generated permutations, we should never got here, because
- the compiler should also be checking the ok hook. But since this is a
- builtin the user has access too, so don't abort. */
- switch (d.nelt)
+ /* If the mask says both arguments are needed, but they are the same,
+ the above tried to expand with d.op0 == d.op1. If that didn't work,
+ retry with d.op0 != d.op1 as that is what testing has been done with. */
+ if (which == 3 && d.op0 == d.op1)
{
- case 2:
- sorry ("vector permutation (%d %d)", d.perm[0], d.perm[1]);
- break;
- case 4:
- sorry ("vector permutation (%d %d %d %d)",
- d.perm[0], d.perm[1], d.perm[2], d.perm[3]);
- break;
- case 8:
- sorry ("vector permutation (%d %d %d %d %d %d %d %d)",
- d.perm[0], d.perm[1], d.perm[2], d.perm[3],
- d.perm[4], d.perm[5], d.perm[6], d.perm[7]);
- break;
- case 16:
- sorry ("vector permutation "
- "(%d %d %d %d %d %d %d %d %d %d %d %d %d %d %d %d)",
- d.perm[0], d.perm[1], d.perm[2], d.perm[3],
- d.perm[4], d.perm[5], d.perm[6], d.perm[7],
- d.perm[8], d.perm[9], d.perm[10], d.perm[11],
- d.perm[12], d.perm[13], d.perm[14], d.perm[15]);
- break;
- default:
- gcc_unreachable ();
+ rtx seq;
+ bool ok;
+
+ memcpy (d.perm, perm, sizeof (perm));
+ d.op1 = gen_reg_rtx (d.vmode);
+ start_sequence ();
+ ok = ix86_expand_vec_perm_const_1 (&d);
+ seq = get_insns ();
+ end_sequence ();
+ if (ok)
+ {
+ emit_move_insn (d.op1, d.op0);
+ emit_insn (seq);
+ return true;
+ }
}
- exit_error:
- return CONST0_RTX (d.vmode);
+
+ return false;
}
-/* Implement targetm.vectorize.builtin_vec_perm_ok. */
+/* Implement targetm.vectorize.vec_perm_const_ok. */
static bool
-ix86_vectorize_builtin_vec_perm_ok (tree vec_type, tree mask)
+ix86_vectorize_vec_perm_const_ok (enum machine_mode vmode,
+ const unsigned char *sel)
{
struct expand_vec_perm_d d;
- int vec_mask;
+ unsigned int i, nelt, which;
bool ret, one_vec;
- d.vmode = TYPE_MODE (vec_type);
- d.nelt = GET_MODE_NUNITS (d.vmode);
+ d.vmode = vmode;
+ d.nelt = nelt = GET_MODE_NUNITS (d.vmode);
d.testing_p = true;
/* Given sufficient ISA support we can just return true here
return true;
}
- vec_mask = extract_vec_perm_cst (&d, mask);
+ /* Extract the values from the vector CST into the permutation
+ array in D. */
+ memcpy (d.perm, sel, nelt);
+ for (i = which = 0; i < nelt; ++i)
+ {
+ unsigned char e = d.perm[i];
+ gcc_assert (e < 2 * nelt);
+ which |= (e < nelt ? 1 : 2);
+ }
+
+ /* For all elements from second vector, fold the elements to first. */
+ if (which == 2)
+ for (i = 0; i < nelt; ++i)
+ d.perm[i] -= nelt;
- /* This hook is cannot be called in response to something that the
- user does (unlike the builtin expander) so we shouldn't ever see
- an error generated from the extract. */
- gcc_assert (vec_mask > 0 && vec_mask <= 3);
- one_vec = (vec_mask != 3);
+ /* Check whether the mask can be applied to the vector type. */
+ one_vec = (which != 3);
/* Implementable with shufps or pshufd. */
if (one_vec && (d.vmode == V4SFmode || d.vmode == V4SImode))
d.op1 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 3);
start_sequence ();
- ret = ix86_expand_vec_perm_builtin_1 (&d);
+ ret = ix86_expand_vec_perm_const_1 (&d);
end_sequence ();
return ret;
switch (mode)
{
case QImode:
- return TARGET_AVX2 ? V32QImode : V16QImode;
+ return (TARGET_AVX && !TARGET_PREFER_AVX128) ? V32QImode : V16QImode;
case HImode:
- return TARGET_AVX2 ? V16HImode : V8HImode;
+ return (TARGET_AVX && !TARGET_PREFER_AVX128) ? V16HImode : V8HImode;
case SImode:
- return TARGET_AVX2 ? V8SImode : V4SImode;
+ return (TARGET_AVX && !TARGET_PREFER_AVX128) ? V8SImode : V4SImode;
case DImode:
- return TARGET_AVX2 ? V4DImode : V2DImode;
+ return (TARGET_AVX && !TARGET_PREFER_AVX128) ? V4DImode : V2DImode;
case SFmode:
if (TARGET_AVX && !TARGET_PREFER_AVX128)
#define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
ix86_builtin_vectorized_function
-#undef TARGET_VECTORIZE_BUILTIN_CONVERSION
-#define TARGET_VECTORIZE_BUILTIN_CONVERSION ix86_vectorize_builtin_conversion
+#undef TARGET_VECTORIZE_BUILTIN_TM_LOAD
+#define TARGET_VECTORIZE_BUILTIN_TM_LOAD ix86_builtin_tm_load
+
+#undef TARGET_VECTORIZE_BUILTIN_TM_STORE
+#define TARGET_VECTORIZE_BUILTIN_TM_STORE ix86_builtin_tm_store
+
+#undef TARGET_VECTORIZE_BUILTIN_GATHER
+#define TARGET_VECTORIZE_BUILTIN_GATHER ix86_vectorize_builtin_gather
#undef TARGET_BUILTIN_RECIPROCAL
#define TARGET_BUILTIN_RECIPROCAL ix86_builtin_reciprocal
#undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
#define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \
ix86_builtin_vectorization_cost
-#undef TARGET_VECTORIZE_BUILTIN_VEC_PERM
-#define TARGET_VECTORIZE_BUILTIN_VEC_PERM \
- ix86_vectorize_builtin_vec_perm
-#undef TARGET_VECTORIZE_BUILTIN_VEC_PERM_OK
-#define TARGET_VECTORIZE_BUILTIN_VEC_PERM_OK \
- ix86_vectorize_builtin_vec_perm_ok
+#undef TARGET_VECTORIZE_VEC_PERM_CONST_OK
+#define TARGET_VECTORIZE_VEC_PERM_CONST_OK \
+ ix86_vectorize_vec_perm_const_ok
#undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE
#define TARGET_VECTORIZE_PREFERRED_SIMD_MODE \
ix86_preferred_simd_mode
OSDN Git Service
