1 /* Subroutines used for code generation on IA-32.
2 Copyright (C) 1988, 1992, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001,
3 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009
4 Free Software Foundation, Inc.
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 3, or (at your option)
13 GCC is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
30 #include "hard-reg-set.h"
32 #include "insn-config.h"
33 #include "conditions.h"
35 #include "insn-codes.h"
36 #include "insn-attr.h"
45 #include "basic-block.h"
48 #include "target-def.h"
49 #include "langhooks.h"
54 #include "tm-constrs.h"
58 static int x86_builtin_vectorization_cost (bool);
59 static rtx legitimize_dllimport_symbol (rtx, bool);
61 #ifndef CHECK_STACK_LIMIT
62 #define CHECK_STACK_LIMIT (-1)
65 /* Return index of given mode in mult and division cost tables. */
66 #define MODE_INDEX(mode) \
67 ((mode) == QImode ? 0 \
68 : (mode) == HImode ? 1 \
69 : (mode) == SImode ? 2 \
70 : (mode) == DImode ? 3 \
73 /* Processor costs (relative to an add) */
74 /* We assume COSTS_N_INSNS is defined as (N)*4 and an addition is 2 bytes. */
75 #define COSTS_N_BYTES(N) ((N) * 2)
77 #define DUMMY_STRINGOP_ALGS {libcall, {{-1, libcall}}}
80 struct processor_costs ix86_size_cost = {/* costs for tuning for size */
81 COSTS_N_BYTES (2), /* cost of an add instruction */
82 COSTS_N_BYTES (3), /* cost of a lea instruction */
83 COSTS_N_BYTES (2), /* variable shift costs */
84 COSTS_N_BYTES (3), /* constant shift costs */
85 {COSTS_N_BYTES (3), /* cost of starting multiply for QI */
86 COSTS_N_BYTES (3), /* HI */
87 COSTS_N_BYTES (3), /* SI */
88 COSTS_N_BYTES (3), /* DI */
89 COSTS_N_BYTES (5)}, /* other */
90 0, /* cost of multiply per each bit set */
91 {COSTS_N_BYTES (3), /* cost of a divide/mod for QI */
92 COSTS_N_BYTES (3), /* HI */
93 COSTS_N_BYTES (3), /* SI */
94 COSTS_N_BYTES (3), /* DI */
95 COSTS_N_BYTES (5)}, /* other */
96 COSTS_N_BYTES (3), /* cost of movsx */
97 COSTS_N_BYTES (3), /* cost of movzx */
100 2, /* cost for loading QImode using movzbl */
101 {2, 2, 2}, /* cost of loading integer registers
102 in QImode, HImode and SImode.
103 Relative to reg-reg move (2). */
104 {2, 2, 2}, /* cost of storing integer registers */
105 2, /* cost of reg,reg fld/fst */
106 {2, 2, 2}, /* cost of loading fp registers
107 in SFmode, DFmode and XFmode */
108 {2, 2, 2}, /* cost of storing fp registers
109 in SFmode, DFmode and XFmode */
110 3, /* cost of moving MMX register */
111 {3, 3}, /* cost of loading MMX registers
112 in SImode and DImode */
113 {3, 3}, /* cost of storing MMX registers
114 in SImode and DImode */
115 3, /* cost of moving SSE register */
116 {3, 3, 3}, /* cost of loading SSE registers
117 in SImode, DImode and TImode */
118 {3, 3, 3}, /* cost of storing SSE registers
119 in SImode, DImode and TImode */
120 3, /* MMX or SSE register to integer */
121 0, /* size of l1 cache */
122 0, /* size of l2 cache */
123 0, /* size of prefetch block */
124 0, /* number of parallel prefetches */
126 COSTS_N_BYTES (2), /* cost of FADD and FSUB insns. */
127 COSTS_N_BYTES (2), /* cost of FMUL instruction. */
128 COSTS_N_BYTES (2), /* cost of FDIV instruction. */
129 COSTS_N_BYTES (2), /* cost of FABS instruction. */
130 COSTS_N_BYTES (2), /* cost of FCHS instruction. */
131 COSTS_N_BYTES (2), /* cost of FSQRT instruction. */
132 {{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
133 {rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}}},
134 {{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
135 {rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}}},
136 1, /* scalar_stmt_cost. */
137 1, /* scalar load_cost. */
138 1, /* scalar_store_cost. */
139 1, /* vec_stmt_cost. */
140 1, /* vec_to_scalar_cost. */
141 1, /* scalar_to_vec_cost. */
142 1, /* vec_align_load_cost. */
143 1, /* vec_unalign_load_cost. */
144 1, /* vec_store_cost. */
145 1, /* cond_taken_branch_cost. */
146 1, /* cond_not_taken_branch_cost. */
149 /* Processor costs (relative to an add) */
151 struct processor_costs i386_cost = { /* 386 specific costs */
152 COSTS_N_INSNS (1), /* cost of an add instruction */
153 COSTS_N_INSNS (1), /* cost of a lea instruction */
154 COSTS_N_INSNS (3), /* variable shift costs */
155 COSTS_N_INSNS (2), /* constant shift costs */
156 {COSTS_N_INSNS (6), /* cost of starting multiply for QI */
157 COSTS_N_INSNS (6), /* HI */
158 COSTS_N_INSNS (6), /* SI */
159 COSTS_N_INSNS (6), /* DI */
160 COSTS_N_INSNS (6)}, /* other */
161 COSTS_N_INSNS (1), /* cost of multiply per each bit set */
162 {COSTS_N_INSNS (23), /* cost of a divide/mod for QI */
163 COSTS_N_INSNS (23), /* HI */
164 COSTS_N_INSNS (23), /* SI */
165 COSTS_N_INSNS (23), /* DI */
166 COSTS_N_INSNS (23)}, /* other */
167 COSTS_N_INSNS (3), /* cost of movsx */
168 COSTS_N_INSNS (2), /* cost of movzx */
169 15, /* "large" insn */
171 4, /* cost for loading QImode using movzbl */
172 {2, 4, 2}, /* cost of loading integer registers
173 in QImode, HImode and SImode.
174 Relative to reg-reg move (2). */
175 {2, 4, 2}, /* cost of storing integer registers */
176 2, /* cost of reg,reg fld/fst */
177 {8, 8, 8}, /* cost of loading fp registers
178 in SFmode, DFmode and XFmode */
179 {8, 8, 8}, /* cost of storing fp registers
180 in SFmode, DFmode and XFmode */
181 2, /* cost of moving MMX register */
182 {4, 8}, /* cost of loading MMX registers
183 in SImode and DImode */
184 {4, 8}, /* cost of storing MMX registers
185 in SImode and DImode */
186 2, /* cost of moving SSE register */
187 {4, 8, 16}, /* cost of loading SSE registers
188 in SImode, DImode and TImode */
189 {4, 8, 16}, /* cost of storing SSE registers
190 in SImode, DImode and TImode */
191 3, /* MMX or SSE register to integer */
192 0, /* size of l1 cache */
193 0, /* size of l2 cache */
194 0, /* size of prefetch block */
195 0, /* number of parallel prefetches */
197 COSTS_N_INSNS (23), /* cost of FADD and FSUB insns. */
198 COSTS_N_INSNS (27), /* cost of FMUL instruction. */
199 COSTS_N_INSNS (88), /* cost of FDIV instruction. */
200 COSTS_N_INSNS (22), /* cost of FABS instruction. */
201 COSTS_N_INSNS (24), /* cost of FCHS instruction. */
202 COSTS_N_INSNS (122), /* cost of FSQRT instruction. */
203 {{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
204 DUMMY_STRINGOP_ALGS},
205 {{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
206 DUMMY_STRINGOP_ALGS},
207 1, /* scalar_stmt_cost. */
208 1, /* scalar load_cost. */
209 1, /* scalar_store_cost. */
210 1, /* vec_stmt_cost. */
211 1, /* vec_to_scalar_cost. */
212 1, /* scalar_to_vec_cost. */
213 1, /* vec_align_load_cost. */
214 2, /* vec_unalign_load_cost. */
215 1, /* vec_store_cost. */
216 3, /* cond_taken_branch_cost. */
217 1, /* cond_not_taken_branch_cost. */
221 struct processor_costs i486_cost = { /* 486 specific costs */
222 COSTS_N_INSNS (1), /* cost of an add instruction */
223 COSTS_N_INSNS (1), /* cost of a lea instruction */
224 COSTS_N_INSNS (3), /* variable shift costs */
225 COSTS_N_INSNS (2), /* constant shift costs */
226 {COSTS_N_INSNS (12), /* cost of starting multiply for QI */
227 COSTS_N_INSNS (12), /* HI */
228 COSTS_N_INSNS (12), /* SI */
229 COSTS_N_INSNS (12), /* DI */
230 COSTS_N_INSNS (12)}, /* other */
231 1, /* cost of multiply per each bit set */
232 {COSTS_N_INSNS (40), /* cost of a divide/mod for QI */
233 COSTS_N_INSNS (40), /* HI */
234 COSTS_N_INSNS (40), /* SI */
235 COSTS_N_INSNS (40), /* DI */
236 COSTS_N_INSNS (40)}, /* other */
237 COSTS_N_INSNS (3), /* cost of movsx */
238 COSTS_N_INSNS (2), /* cost of movzx */
239 15, /* "large" insn */
241 4, /* cost for loading QImode using movzbl */
242 {2, 4, 2}, /* cost of loading integer registers
243 in QImode, HImode and SImode.
244 Relative to reg-reg move (2). */
245 {2, 4, 2}, /* cost of storing integer registers */
246 2, /* cost of reg,reg fld/fst */
247 {8, 8, 8}, /* cost of loading fp registers
248 in SFmode, DFmode and XFmode */
249 {8, 8, 8}, /* cost of storing fp registers
250 in SFmode, DFmode and XFmode */
251 2, /* cost of moving MMX register */
252 {4, 8}, /* cost of loading MMX registers
253 in SImode and DImode */
254 {4, 8}, /* cost of storing MMX registers
255 in SImode and DImode */
256 2, /* cost of moving SSE register */
257 {4, 8, 16}, /* cost of loading SSE registers
258 in SImode, DImode and TImode */
259 {4, 8, 16}, /* cost of storing SSE registers
260 in SImode, DImode and TImode */
261 3, /* MMX or SSE register to integer */
262 4, /* size of l1 cache. 486 has 8kB cache
263 shared for code and data, so 4kB is
264 not really precise. */
265 4, /* size of l2 cache */
266 0, /* size of prefetch block */
267 0, /* number of parallel prefetches */
269 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
270 COSTS_N_INSNS (16), /* cost of FMUL instruction. */
271 COSTS_N_INSNS (73), /* cost of FDIV instruction. */
272 COSTS_N_INSNS (3), /* cost of FABS instruction. */
273 COSTS_N_INSNS (3), /* cost of FCHS instruction. */
274 COSTS_N_INSNS (83), /* cost of FSQRT instruction. */
275 {{rep_prefix_4_byte, {{-1, rep_prefix_4_byte}}},
276 DUMMY_STRINGOP_ALGS},
277 {{rep_prefix_4_byte, {{-1, rep_prefix_4_byte}}},
278 DUMMY_STRINGOP_ALGS},
279 1, /* scalar_stmt_cost. */
280 1, /* scalar load_cost. */
281 1, /* scalar_store_cost. */
282 1, /* vec_stmt_cost. */
283 1, /* vec_to_scalar_cost. */
284 1, /* scalar_to_vec_cost. */
285 1, /* vec_align_load_cost. */
286 2, /* vec_unalign_load_cost. */
287 1, /* vec_store_cost. */
288 3, /* cond_taken_branch_cost. */
289 1, /* cond_not_taken_branch_cost. */
293 struct processor_costs pentium_cost = {
294 COSTS_N_INSNS (1), /* cost of an add instruction */
295 COSTS_N_INSNS (1), /* cost of a lea instruction */
296 COSTS_N_INSNS (4), /* variable shift costs */
297 COSTS_N_INSNS (1), /* constant shift costs */
298 {COSTS_N_INSNS (11), /* cost of starting multiply for QI */
299 COSTS_N_INSNS (11), /* HI */
300 COSTS_N_INSNS (11), /* SI */
301 COSTS_N_INSNS (11), /* DI */
302 COSTS_N_INSNS (11)}, /* other */
303 0, /* cost of multiply per each bit set */
304 {COSTS_N_INSNS (25), /* cost of a divide/mod for QI */
305 COSTS_N_INSNS (25), /* HI */
306 COSTS_N_INSNS (25), /* SI */
307 COSTS_N_INSNS (25), /* DI */
308 COSTS_N_INSNS (25)}, /* other */
309 COSTS_N_INSNS (3), /* cost of movsx */
310 COSTS_N_INSNS (2), /* cost of movzx */
311 8, /* "large" insn */
313 6, /* cost for loading QImode using movzbl */
314 {2, 4, 2}, /* cost of loading integer registers
315 in QImode, HImode and SImode.
316 Relative to reg-reg move (2). */
317 {2, 4, 2}, /* cost of storing integer registers */
318 2, /* cost of reg,reg fld/fst */
319 {2, 2, 6}, /* cost of loading fp registers
320 in SFmode, DFmode and XFmode */
321 {4, 4, 6}, /* cost of storing fp registers
322 in SFmode, DFmode and XFmode */
323 8, /* cost of moving MMX register */
324 {8, 8}, /* cost of loading MMX registers
325 in SImode and DImode */
326 {8, 8}, /* cost of storing MMX registers
327 in SImode and DImode */
328 2, /* cost of moving SSE register */
329 {4, 8, 16}, /* cost of loading SSE registers
330 in SImode, DImode and TImode */
331 {4, 8, 16}, /* cost of storing SSE registers
332 in SImode, DImode and TImode */
333 3, /* MMX or SSE register to integer */
334 8, /* size of l1 cache. */
335 8, /* size of l2 cache */
336 0, /* size of prefetch block */
337 0, /* number of parallel prefetches */
339 COSTS_N_INSNS (3), /* cost of FADD and FSUB insns. */
340 COSTS_N_INSNS (3), /* cost of FMUL instruction. */
341 COSTS_N_INSNS (39), /* cost of FDIV instruction. */
342 COSTS_N_INSNS (1), /* cost of FABS instruction. */
343 COSTS_N_INSNS (1), /* cost of FCHS instruction. */
344 COSTS_N_INSNS (70), /* cost of FSQRT instruction. */
345 {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
346 DUMMY_STRINGOP_ALGS},
347 {{libcall, {{-1, rep_prefix_4_byte}}},
348 DUMMY_STRINGOP_ALGS},
349 1, /* scalar_stmt_cost. */
350 1, /* scalar load_cost. */
351 1, /* scalar_store_cost. */
352 1, /* vec_stmt_cost. */
353 1, /* vec_to_scalar_cost. */
354 1, /* scalar_to_vec_cost. */
355 1, /* vec_align_load_cost. */
356 2, /* vec_unalign_load_cost. */
357 1, /* vec_store_cost. */
358 3, /* cond_taken_branch_cost. */
359 1, /* cond_not_taken_branch_cost. */
363 struct processor_costs pentiumpro_cost = {
364 COSTS_N_INSNS (1), /* cost of an add instruction */
365 COSTS_N_INSNS (1), /* cost of a lea instruction */
366 COSTS_N_INSNS (1), /* variable shift costs */
367 COSTS_N_INSNS (1), /* constant shift costs */
368 {COSTS_N_INSNS (4), /* cost of starting multiply for QI */
369 COSTS_N_INSNS (4), /* HI */
370 COSTS_N_INSNS (4), /* SI */
371 COSTS_N_INSNS (4), /* DI */
372 COSTS_N_INSNS (4)}, /* other */
373 0, /* cost of multiply per each bit set */
374 {COSTS_N_INSNS (17), /* cost of a divide/mod for QI */
375 COSTS_N_INSNS (17), /* HI */
376 COSTS_N_INSNS (17), /* SI */
377 COSTS_N_INSNS (17), /* DI */
378 COSTS_N_INSNS (17)}, /* other */
379 COSTS_N_INSNS (1), /* cost of movsx */
380 COSTS_N_INSNS (1), /* cost of movzx */
381 8, /* "large" insn */
383 2, /* cost for loading QImode using movzbl */
384 {4, 4, 4}, /* cost of loading integer registers
385 in QImode, HImode and SImode.
386 Relative to reg-reg move (2). */
387 {2, 2, 2}, /* cost of storing integer registers */
388 2, /* cost of reg,reg fld/fst */
389 {2, 2, 6}, /* cost of loading fp registers
390 in SFmode, DFmode and XFmode */
391 {4, 4, 6}, /* cost of storing fp registers
392 in SFmode, DFmode and XFmode */
393 2, /* cost of moving MMX register */
394 {2, 2}, /* cost of loading MMX registers
395 in SImode and DImode */
396 {2, 2}, /* cost of storing MMX registers
397 in SImode and DImode */
398 2, /* cost of moving SSE register */
399 {2, 2, 8}, /* cost of loading SSE registers
400 in SImode, DImode and TImode */
401 {2, 2, 8}, /* cost of storing SSE registers
402 in SImode, DImode and TImode */
403 3, /* MMX or SSE register to integer */
404 8, /* size of l1 cache. */
405 256, /* size of l2 cache */
406 32, /* size of prefetch block */
407 6, /* number of parallel prefetches */
409 COSTS_N_INSNS (3), /* cost of FADD and FSUB insns. */
410 COSTS_N_INSNS (5), /* cost of FMUL instruction. */
411 COSTS_N_INSNS (56), /* cost of FDIV instruction. */
412 COSTS_N_INSNS (2), /* cost of FABS instruction. */
413 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
414 COSTS_N_INSNS (56), /* cost of FSQRT instruction. */
415 /* PentiumPro has optimized rep instructions for blocks aligned by 8 bytes (we ensure
416 the alignment). For small blocks inline loop is still a noticeable win, for bigger
417 blocks either rep movsl or rep movsb is way to go. Rep movsb has apparently
418 more expensive startup time in CPU, but after 4K the difference is down in the noise.
420 {{rep_prefix_4_byte, {{128, loop}, {1024, unrolled_loop},
421 {8192, rep_prefix_4_byte}, {-1, rep_prefix_1_byte}}},
422 DUMMY_STRINGOP_ALGS},
423 {{rep_prefix_4_byte, {{1024, unrolled_loop},
424 {8192, rep_prefix_4_byte}, {-1, libcall}}},
425 DUMMY_STRINGOP_ALGS},
426 1, /* scalar_stmt_cost. */
427 1, /* scalar load_cost. */
428 1, /* scalar_store_cost. */
429 1, /* vec_stmt_cost. */
430 1, /* vec_to_scalar_cost. */
431 1, /* scalar_to_vec_cost. */
432 1, /* vec_align_load_cost. */
433 2, /* vec_unalign_load_cost. */
434 1, /* vec_store_cost. */
435 3, /* cond_taken_branch_cost. */
436 1, /* cond_not_taken_branch_cost. */
440 struct processor_costs geode_cost = {
441 COSTS_N_INSNS (1), /* cost of an add instruction */
442 COSTS_N_INSNS (1), /* cost of a lea instruction */
443 COSTS_N_INSNS (2), /* variable shift costs */
444 COSTS_N_INSNS (1), /* constant shift costs */
445 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
446 COSTS_N_INSNS (4), /* HI */
447 COSTS_N_INSNS (7), /* SI */
448 COSTS_N_INSNS (7), /* DI */
449 COSTS_N_INSNS (7)}, /* other */
450 0, /* cost of multiply per each bit set */
451 {COSTS_N_INSNS (15), /* cost of a divide/mod for QI */
452 COSTS_N_INSNS (23), /* HI */
453 COSTS_N_INSNS (39), /* SI */
454 COSTS_N_INSNS (39), /* DI */
455 COSTS_N_INSNS (39)}, /* other */
456 COSTS_N_INSNS (1), /* cost of movsx */
457 COSTS_N_INSNS (1), /* cost of movzx */
458 8, /* "large" insn */
460 1, /* cost for loading QImode using movzbl */
461 {1, 1, 1}, /* cost of loading integer registers
462 in QImode, HImode and SImode.
463 Relative to reg-reg move (2). */
464 {1, 1, 1}, /* cost of storing integer registers */
465 1, /* cost of reg,reg fld/fst */
466 {1, 1, 1}, /* cost of loading fp registers
467 in SFmode, DFmode and XFmode */
468 {4, 6, 6}, /* cost of storing fp registers
469 in SFmode, DFmode and XFmode */
471 1, /* cost of moving MMX register */
472 {1, 1}, /* cost of loading MMX registers
473 in SImode and DImode */
474 {1, 1}, /* cost of storing MMX registers
475 in SImode and DImode */
476 1, /* cost of moving SSE register */
477 {1, 1, 1}, /* cost of loading SSE registers
478 in SImode, DImode and TImode */
479 {1, 1, 1}, /* cost of storing SSE registers
480 in SImode, DImode and TImode */
481 1, /* MMX or SSE register to integer */
482 64, /* size of l1 cache. */
483 128, /* size of l2 cache. */
484 32, /* size of prefetch block */
485 1, /* number of parallel prefetches */
487 COSTS_N_INSNS (6), /* cost of FADD and FSUB insns. */
488 COSTS_N_INSNS (11), /* cost of FMUL instruction. */
489 COSTS_N_INSNS (47), /* cost of FDIV instruction. */
490 COSTS_N_INSNS (1), /* cost of FABS instruction. */
491 COSTS_N_INSNS (1), /* cost of FCHS instruction. */
492 COSTS_N_INSNS (54), /* cost of FSQRT instruction. */
493 {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
494 DUMMY_STRINGOP_ALGS},
495 {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
496 DUMMY_STRINGOP_ALGS},
497 1, /* scalar_stmt_cost. */
498 1, /* scalar load_cost. */
499 1, /* scalar_store_cost. */
500 1, /* vec_stmt_cost. */
501 1, /* vec_to_scalar_cost. */
502 1, /* scalar_to_vec_cost. */
503 1, /* vec_align_load_cost. */
504 2, /* vec_unalign_load_cost. */
505 1, /* vec_store_cost. */
506 3, /* cond_taken_branch_cost. */
507 1, /* cond_not_taken_branch_cost. */
511 struct processor_costs k6_cost = {
512 COSTS_N_INSNS (1), /* cost of an add instruction */
513 COSTS_N_INSNS (2), /* cost of a lea instruction */
514 COSTS_N_INSNS (1), /* variable shift costs */
515 COSTS_N_INSNS (1), /* constant shift costs */
516 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
517 COSTS_N_INSNS (3), /* HI */
518 COSTS_N_INSNS (3), /* SI */
519 COSTS_N_INSNS (3), /* DI */
520 COSTS_N_INSNS (3)}, /* other */
521 0, /* cost of multiply per each bit set */
522 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
523 COSTS_N_INSNS (18), /* HI */
524 COSTS_N_INSNS (18), /* SI */
525 COSTS_N_INSNS (18), /* DI */
526 COSTS_N_INSNS (18)}, /* other */
527 COSTS_N_INSNS (2), /* cost of movsx */
528 COSTS_N_INSNS (2), /* cost of movzx */
529 8, /* "large" insn */
531 3, /* cost for loading QImode using movzbl */
532 {4, 5, 4}, /* cost of loading integer registers
533 in QImode, HImode and SImode.
534 Relative to reg-reg move (2). */
535 {2, 3, 2}, /* cost of storing integer registers */
536 4, /* cost of reg,reg fld/fst */
537 {6, 6, 6}, /* cost of loading fp registers
538 in SFmode, DFmode and XFmode */
539 {4, 4, 4}, /* cost of storing fp registers
540 in SFmode, DFmode and XFmode */
541 2, /* cost of moving MMX register */
542 {2, 2}, /* cost of loading MMX registers
543 in SImode and DImode */
544 {2, 2}, /* cost of storing MMX registers
545 in SImode and DImode */
546 2, /* cost of moving SSE register */
547 {2, 2, 8}, /* cost of loading SSE registers
548 in SImode, DImode and TImode */
549 {2, 2, 8}, /* cost of storing SSE registers
550 in SImode, DImode and TImode */
551 6, /* MMX or SSE register to integer */
552 32, /* size of l1 cache. */
553 32, /* size of l2 cache. Some models
554 have integrated l2 cache, but
555 optimizing for k6 is not important
556 enough to worry about that. */
557 32, /* size of prefetch block */
558 1, /* number of parallel prefetches */
560 COSTS_N_INSNS (2), /* cost of FADD and FSUB insns. */
561 COSTS_N_INSNS (2), /* cost of FMUL instruction. */
562 COSTS_N_INSNS (56), /* cost of FDIV instruction. */
563 COSTS_N_INSNS (2), /* cost of FABS instruction. */
564 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
565 COSTS_N_INSNS (56), /* cost of FSQRT instruction. */
566 {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
567 DUMMY_STRINGOP_ALGS},
568 {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
569 DUMMY_STRINGOP_ALGS},
570 1, /* scalar_stmt_cost. */
571 1, /* scalar load_cost. */
572 1, /* scalar_store_cost. */
573 1, /* vec_stmt_cost. */
574 1, /* vec_to_scalar_cost. */
575 1, /* scalar_to_vec_cost. */
576 1, /* vec_align_load_cost. */
577 2, /* vec_unalign_load_cost. */
578 1, /* vec_store_cost. */
579 3, /* cond_taken_branch_cost. */
580 1, /* cond_not_taken_branch_cost. */
584 struct processor_costs athlon_cost = {
585 COSTS_N_INSNS (1), /* cost of an add instruction */
586 COSTS_N_INSNS (2), /* cost of a lea instruction */
587 COSTS_N_INSNS (1), /* variable shift costs */
588 COSTS_N_INSNS (1), /* constant shift costs */
589 {COSTS_N_INSNS (5), /* cost of starting multiply for QI */
590 COSTS_N_INSNS (5), /* HI */
591 COSTS_N_INSNS (5), /* SI */
592 COSTS_N_INSNS (5), /* DI */
593 COSTS_N_INSNS (5)}, /* other */
594 0, /* cost of multiply per each bit set */
595 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
596 COSTS_N_INSNS (26), /* HI */
597 COSTS_N_INSNS (42), /* SI */
598 COSTS_N_INSNS (74), /* DI */
599 COSTS_N_INSNS (74)}, /* other */
600 COSTS_N_INSNS (1), /* cost of movsx */
601 COSTS_N_INSNS (1), /* cost of movzx */
602 8, /* "large" insn */
604 4, /* cost for loading QImode using movzbl */
605 {3, 4, 3}, /* cost of loading integer registers
606 in QImode, HImode and SImode.
607 Relative to reg-reg move (2). */
608 {3, 4, 3}, /* cost of storing integer registers */
609 4, /* cost of reg,reg fld/fst */
610 {4, 4, 12}, /* cost of loading fp registers
611 in SFmode, DFmode and XFmode */
612 {6, 6, 8}, /* cost of storing fp registers
613 in SFmode, DFmode and XFmode */
614 2, /* cost of moving MMX register */
615 {4, 4}, /* cost of loading MMX registers
616 in SImode and DImode */
617 {4, 4}, /* cost of storing MMX registers
618 in SImode and DImode */
619 2, /* cost of moving SSE register */
620 {4, 4, 6}, /* cost of loading SSE registers
621 in SImode, DImode and TImode */
622 {4, 4, 5}, /* cost of storing SSE registers
623 in SImode, DImode and TImode */
624 5, /* MMX or SSE register to integer */
625 64, /* size of l1 cache. */
626 256, /* size of l2 cache. */
627 64, /* size of prefetch block */
628 6, /* number of parallel prefetches */
630 COSTS_N_INSNS (4), /* cost of FADD and FSUB insns. */
631 COSTS_N_INSNS (4), /* cost of FMUL instruction. */
632 COSTS_N_INSNS (24), /* cost of FDIV instruction. */
633 COSTS_N_INSNS (2), /* cost of FABS instruction. */
634 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
635 COSTS_N_INSNS (35), /* cost of FSQRT instruction. */
636 /* For some reason, Athlon deals better with REP prefix (relative to loops)
637 compared to K8. Alignment becomes important after 8 bytes for memcpy and
638 128 bytes for memset. */
639 {{libcall, {{2048, rep_prefix_4_byte}, {-1, libcall}}},
640 DUMMY_STRINGOP_ALGS},
641 {{libcall, {{2048, rep_prefix_4_byte}, {-1, libcall}}},
642 DUMMY_STRINGOP_ALGS},
643 1, /* scalar_stmt_cost. */
644 1, /* scalar load_cost. */
645 1, /* scalar_store_cost. */
646 1, /* vec_stmt_cost. */
647 1, /* vec_to_scalar_cost. */
648 1, /* scalar_to_vec_cost. */
649 1, /* vec_align_load_cost. */
650 2, /* vec_unalign_load_cost. */
651 1, /* vec_store_cost. */
652 3, /* cond_taken_branch_cost. */
653 1, /* cond_not_taken_branch_cost. */
657 struct processor_costs k8_cost = {
658 COSTS_N_INSNS (1), /* cost of an add instruction */
659 COSTS_N_INSNS (2), /* cost of a lea instruction */
660 COSTS_N_INSNS (1), /* variable shift costs */
661 COSTS_N_INSNS (1), /* constant shift costs */
662 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
663 COSTS_N_INSNS (4), /* HI */
664 COSTS_N_INSNS (3), /* SI */
665 COSTS_N_INSNS (4), /* DI */
666 COSTS_N_INSNS (5)}, /* other */
667 0, /* cost of multiply per each bit set */
668 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
669 COSTS_N_INSNS (26), /* HI */
670 COSTS_N_INSNS (42), /* SI */
671 COSTS_N_INSNS (74), /* DI */
672 COSTS_N_INSNS (74)}, /* other */
673 COSTS_N_INSNS (1), /* cost of movsx */
674 COSTS_N_INSNS (1), /* cost of movzx */
675 8, /* "large" insn */
677 4, /* cost for loading QImode using movzbl */
678 {3, 4, 3}, /* cost of loading integer registers
679 in QImode, HImode and SImode.
680 Relative to reg-reg move (2). */
681 {3, 4, 3}, /* cost of storing integer registers */
682 4, /* cost of reg,reg fld/fst */
683 {4, 4, 12}, /* cost of loading fp registers
684 in SFmode, DFmode and XFmode */
685 {6, 6, 8}, /* cost of storing fp registers
686 in SFmode, DFmode and XFmode */
687 2, /* cost of moving MMX register */
688 {3, 3}, /* cost of loading MMX registers
689 in SImode and DImode */
690 {4, 4}, /* cost of storing MMX registers
691 in SImode and DImode */
692 2, /* cost of moving SSE register */
693 {4, 3, 6}, /* cost of loading SSE registers
694 in SImode, DImode and TImode */
695 {4, 4, 5}, /* cost of storing SSE registers
696 in SImode, DImode and TImode */
697 5, /* MMX or SSE register to integer */
698 64, /* size of l1 cache. */
699 512, /* size of l2 cache. */
700 64, /* size of prefetch block */
701 /* New AMD processors never drop prefetches; if they cannot be performed
702 immediately, they are queued. We set number of simultaneous prefetches
703 to a large constant to reflect this (it probably is not a good idea not
704 to limit number of prefetches at all, as their execution also takes some
706 100, /* number of parallel prefetches */
708 COSTS_N_INSNS (4), /* cost of FADD and FSUB insns. */
709 COSTS_N_INSNS (4), /* cost of FMUL instruction. */
710 COSTS_N_INSNS (19), /* cost of FDIV instruction. */
711 COSTS_N_INSNS (2), /* cost of FABS instruction. */
712 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
713 COSTS_N_INSNS (35), /* cost of FSQRT instruction. */
714 /* K8 has optimized REP instruction for medium sized blocks, but for very small
715 blocks it is better to use loop. For large blocks, libcall can do
716 nontemporary accesses and beat inline considerably. */
717 {{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
718 {libcall, {{16, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
719 {{libcall, {{8, loop}, {24, unrolled_loop},
720 {2048, rep_prefix_4_byte}, {-1, libcall}}},
721 {libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
722 4, /* scalar_stmt_cost. */
723 2, /* scalar load_cost. */
724 2, /* scalar_store_cost. */
725 5, /* vec_stmt_cost. */
726 0, /* vec_to_scalar_cost. */
727 2, /* scalar_to_vec_cost. */
728 2, /* vec_align_load_cost. */
729 3, /* vec_unalign_load_cost. */
730 3, /* vec_store_cost. */
731 3, /* cond_taken_branch_cost. */
732 2, /* cond_not_taken_branch_cost. */
735 struct processor_costs amdfam10_cost = {
736 COSTS_N_INSNS (1), /* cost of an add instruction */
737 COSTS_N_INSNS (2), /* cost of a lea instruction */
738 COSTS_N_INSNS (1), /* variable shift costs */
739 COSTS_N_INSNS (1), /* constant shift costs */
740 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
741 COSTS_N_INSNS (4), /* HI */
742 COSTS_N_INSNS (3), /* SI */
743 COSTS_N_INSNS (4), /* DI */
744 COSTS_N_INSNS (5)}, /* other */
745 0, /* cost of multiply per each bit set */
746 {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */
747 COSTS_N_INSNS (35), /* HI */
748 COSTS_N_INSNS (51), /* SI */
749 COSTS_N_INSNS (83), /* DI */
750 COSTS_N_INSNS (83)}, /* other */
751 COSTS_N_INSNS (1), /* cost of movsx */
752 COSTS_N_INSNS (1), /* cost of movzx */
753 8, /* "large" insn */
755 4, /* cost for loading QImode using movzbl */
756 {3, 4, 3}, /* cost of loading integer registers
757 in QImode, HImode and SImode.
758 Relative to reg-reg move (2). */
759 {3, 4, 3}, /* cost of storing integer registers */
760 4, /* cost of reg,reg fld/fst */
761 {4, 4, 12}, /* cost of loading fp registers
762 in SFmode, DFmode and XFmode */
763 {6, 6, 8}, /* cost of storing fp registers
764 in SFmode, DFmode and XFmode */
765 2, /* cost of moving MMX register */
766 {3, 3}, /* cost of loading MMX registers
767 in SImode and DImode */
768 {4, 4}, /* cost of storing MMX registers
769 in SImode and DImode */
770 2, /* cost of moving SSE register */
771 {4, 4, 3}, /* cost of loading SSE registers
772 in SImode, DImode and TImode */
773 {4, 4, 5}, /* cost of storing SSE registers
774 in SImode, DImode and TImode */
775 3, /* MMX or SSE register to integer */
777 MOVD reg64, xmmreg Double FSTORE 4
778 MOVD reg32, xmmreg Double FSTORE 4
780 MOVD reg64, xmmreg Double FADD 3
782 MOVD reg32, xmmreg Double FADD 3
784 64, /* size of l1 cache. */
785 512, /* size of l2 cache. */
786 64, /* size of prefetch block */
787 /* New AMD processors never drop prefetches; if they cannot be performed
788 immediately, they are queued. We set number of simultaneous prefetches
789 to a large constant to reflect this (it probably is not a good idea not
790 to limit number of prefetches at all, as their execution also takes some
792 100, /* number of parallel prefetches */
794 COSTS_N_INSNS (4), /* cost of FADD and FSUB insns. */
795 COSTS_N_INSNS (4), /* cost of FMUL instruction. */
796 COSTS_N_INSNS (19), /* cost of FDIV instruction. */
797 COSTS_N_INSNS (2), /* cost of FABS instruction. */
798 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
799 COSTS_N_INSNS (35), /* cost of FSQRT instruction. */
801 /* AMDFAM10 has optimized REP instruction for medium sized blocks, but for
802 very small blocks it is better to use loop. For large blocks, libcall can
803 do nontemporary accesses and beat inline considerably. */
804 {{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
805 {libcall, {{16, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
806 {{libcall, {{8, loop}, {24, unrolled_loop},
807 {2048, rep_prefix_4_byte}, {-1, libcall}}},
808 {libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
809 4, /* scalar_stmt_cost. */
810 2, /* scalar load_cost. */
811 2, /* scalar_store_cost. */
812 6, /* vec_stmt_cost. */
813 0, /* vec_to_scalar_cost. */
814 2, /* scalar_to_vec_cost. */
815 2, /* vec_align_load_cost. */
816 2, /* vec_unalign_load_cost. */
817 2, /* vec_store_cost. */
818 2, /* cond_taken_branch_cost. */
819 1, /* cond_not_taken_branch_cost. */
823 struct processor_costs pentium4_cost = {
824 COSTS_N_INSNS (1), /* cost of an add instruction */
825 COSTS_N_INSNS (3), /* cost of a lea instruction */
826 COSTS_N_INSNS (4), /* variable shift costs */
827 COSTS_N_INSNS (4), /* constant shift costs */
828 {COSTS_N_INSNS (15), /* cost of starting multiply for QI */
829 COSTS_N_INSNS (15), /* HI */
830 COSTS_N_INSNS (15), /* SI */
831 COSTS_N_INSNS (15), /* DI */
832 COSTS_N_INSNS (15)}, /* other */
833 0, /* cost of multiply per each bit set */
834 {COSTS_N_INSNS (56), /* cost of a divide/mod for QI */
835 COSTS_N_INSNS (56), /* HI */
836 COSTS_N_INSNS (56), /* SI */
837 COSTS_N_INSNS (56), /* DI */
838 COSTS_N_INSNS (56)}, /* other */
839 COSTS_N_INSNS (1), /* cost of movsx */
840 COSTS_N_INSNS (1), /* cost of movzx */
841 16, /* "large" insn */
843 2, /* cost for loading QImode using movzbl */
844 {4, 5, 4}, /* cost of loading integer registers
845 in QImode, HImode and SImode.
846 Relative to reg-reg move (2). */
847 {2, 3, 2}, /* cost of storing integer registers */
848 2, /* cost of reg,reg fld/fst */
849 {2, 2, 6}, /* cost of loading fp registers
850 in SFmode, DFmode and XFmode */
851 {4, 4, 6}, /* cost of storing fp registers
852 in SFmode, DFmode and XFmode */
853 2, /* cost of moving MMX register */
854 {2, 2}, /* cost of loading MMX registers
855 in SImode and DImode */
856 {2, 2}, /* cost of storing MMX registers
857 in SImode and DImode */
858 12, /* cost of moving SSE register */
859 {12, 12, 12}, /* cost of loading SSE registers
860 in SImode, DImode and TImode */
861 {2, 2, 8}, /* cost of storing SSE registers
862 in SImode, DImode and TImode */
863 10, /* MMX or SSE register to integer */
864 8, /* size of l1 cache. */
865 256, /* size of l2 cache. */
866 64, /* size of prefetch block */
867 6, /* number of parallel prefetches */
869 COSTS_N_INSNS (5), /* cost of FADD and FSUB insns. */
870 COSTS_N_INSNS (7), /* cost of FMUL instruction. */
871 COSTS_N_INSNS (43), /* cost of FDIV instruction. */
872 COSTS_N_INSNS (2), /* cost of FABS instruction. */
873 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
874 COSTS_N_INSNS (43), /* cost of FSQRT instruction. */
875 {{libcall, {{12, loop_1_byte}, {-1, rep_prefix_4_byte}}},
876 DUMMY_STRINGOP_ALGS},
877 {{libcall, {{6, loop_1_byte}, {48, loop}, {20480, rep_prefix_4_byte},
879 DUMMY_STRINGOP_ALGS},
880 1, /* scalar_stmt_cost. */
881 1, /* scalar load_cost. */
882 1, /* scalar_store_cost. */
883 1, /* vec_stmt_cost. */
884 1, /* vec_to_scalar_cost. */
885 1, /* scalar_to_vec_cost. */
886 1, /* vec_align_load_cost. */
887 2, /* vec_unalign_load_cost. */
888 1, /* vec_store_cost. */
889 3, /* cond_taken_branch_cost. */
890 1, /* cond_not_taken_branch_cost. */
894 struct processor_costs nocona_cost = {
895 COSTS_N_INSNS (1), /* cost of an add instruction */
896 COSTS_N_INSNS (1), /* cost of a lea instruction */
897 COSTS_N_INSNS (1), /* variable shift costs */
898 COSTS_N_INSNS (1), /* constant shift costs */
899 {COSTS_N_INSNS (10), /* cost of starting multiply for QI */
900 COSTS_N_INSNS (10), /* HI */
901 COSTS_N_INSNS (10), /* SI */
902 COSTS_N_INSNS (10), /* DI */
903 COSTS_N_INSNS (10)}, /* other */
904 0, /* cost of multiply per each bit set */
905 {COSTS_N_INSNS (66), /* cost of a divide/mod for QI */
906 COSTS_N_INSNS (66), /* HI */
907 COSTS_N_INSNS (66), /* SI */
908 COSTS_N_INSNS (66), /* DI */
909 COSTS_N_INSNS (66)}, /* other */
910 COSTS_N_INSNS (1), /* cost of movsx */
911 COSTS_N_INSNS (1), /* cost of movzx */
912 16, /* "large" insn */
914 4, /* cost for loading QImode using movzbl */
915 {4, 4, 4}, /* cost of loading integer registers
916 in QImode, HImode and SImode.
917 Relative to reg-reg move (2). */
918 {4, 4, 4}, /* cost of storing integer registers */
919 3, /* cost of reg,reg fld/fst */
920 {12, 12, 12}, /* cost of loading fp registers
921 in SFmode, DFmode and XFmode */
922 {4, 4, 4}, /* cost of storing fp registers
923 in SFmode, DFmode and XFmode */
924 6, /* cost of moving MMX register */
925 {12, 12}, /* cost of loading MMX registers
926 in SImode and DImode */
927 {12, 12}, /* cost of storing MMX registers
928 in SImode and DImode */
929 6, /* cost of moving SSE register */
930 {12, 12, 12}, /* cost of loading SSE registers
931 in SImode, DImode and TImode */
932 {12, 12, 12}, /* cost of storing SSE registers
933 in SImode, DImode and TImode */
934 8, /* MMX or SSE register to integer */
935 8, /* size of l1 cache. */
936 1024, /* size of l2 cache. */
937 128, /* size of prefetch block */
938 8, /* number of parallel prefetches */
940 COSTS_N_INSNS (6), /* cost of FADD and FSUB insns. */
941 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
942 COSTS_N_INSNS (40), /* cost of FDIV instruction. */
943 COSTS_N_INSNS (3), /* cost of FABS instruction. */
944 COSTS_N_INSNS (3), /* cost of FCHS instruction. */
945 COSTS_N_INSNS (44), /* cost of FSQRT instruction. */
946 {{libcall, {{12, loop_1_byte}, {-1, rep_prefix_4_byte}}},
947 {libcall, {{32, loop}, {20000, rep_prefix_8_byte},
948 {100000, unrolled_loop}, {-1, libcall}}}},
949 {{libcall, {{6, loop_1_byte}, {48, loop}, {20480, rep_prefix_4_byte},
951 {libcall, {{24, loop}, {64, unrolled_loop},
952 {8192, rep_prefix_8_byte}, {-1, libcall}}}},
953 1, /* scalar_stmt_cost. */
954 1, /* scalar load_cost. */
955 1, /* scalar_store_cost. */
956 1, /* vec_stmt_cost. */
957 1, /* vec_to_scalar_cost. */
958 1, /* scalar_to_vec_cost. */
959 1, /* vec_align_load_cost. */
960 2, /* vec_unalign_load_cost. */
961 1, /* vec_store_cost. */
962 3, /* cond_taken_branch_cost. */
963 1, /* cond_not_taken_branch_cost. */
967 struct processor_costs core2_cost = {
968 COSTS_N_INSNS (1), /* cost of an add instruction */
969 COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */
970 COSTS_N_INSNS (1), /* variable shift costs */
971 COSTS_N_INSNS (1), /* constant shift costs */
972 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
973 COSTS_N_INSNS (3), /* HI */
974 COSTS_N_INSNS (3), /* SI */
975 COSTS_N_INSNS (3), /* DI */
976 COSTS_N_INSNS (3)}, /* other */
977 0, /* cost of multiply per each bit set */
978 {COSTS_N_INSNS (22), /* cost of a divide/mod for QI */
979 COSTS_N_INSNS (22), /* HI */
980 COSTS_N_INSNS (22), /* SI */
981 COSTS_N_INSNS (22), /* DI */
982 COSTS_N_INSNS (22)}, /* other */
983 COSTS_N_INSNS (1), /* cost of movsx */
984 COSTS_N_INSNS (1), /* cost of movzx */
985 8, /* "large" insn */
987 2, /* cost for loading QImode using movzbl */
988 {6, 6, 6}, /* cost of loading integer registers
989 in QImode, HImode and SImode.
990 Relative to reg-reg move (2). */
991 {4, 4, 4}, /* cost of storing integer registers */
992 2, /* cost of reg,reg fld/fst */
993 {6, 6, 6}, /* cost of loading fp registers
994 in SFmode, DFmode and XFmode */
995 {4, 4, 4}, /* cost of storing fp registers
996 in SFmode, DFmode and XFmode */
997 2, /* cost of moving MMX register */
998 {6, 6}, /* cost of loading MMX registers
999 in SImode and DImode */
1000 {4, 4}, /* cost of storing MMX registers
1001 in SImode and DImode */
1002 2, /* cost of moving SSE register */
1003 {6, 6, 6}, /* cost of loading SSE registers
1004 in SImode, DImode and TImode */
1005 {4, 4, 4}, /* cost of storing SSE registers
1006 in SImode, DImode and TImode */
1007 2, /* MMX or SSE register to integer */
1008 32, /* size of l1 cache. */
1009 2048, /* size of l2 cache. */
1010 128, /* size of prefetch block */
1011 8, /* number of parallel prefetches */
1012 3, /* Branch cost */
1013 COSTS_N_INSNS (3), /* cost of FADD and FSUB insns. */
1014 COSTS_N_INSNS (5), /* cost of FMUL instruction. */
1015 COSTS_N_INSNS (32), /* cost of FDIV instruction. */
1016 COSTS_N_INSNS (1), /* cost of FABS instruction. */
1017 COSTS_N_INSNS (1), /* cost of FCHS instruction. */
1018 COSTS_N_INSNS (58), /* cost of FSQRT instruction. */
1019 {{libcall, {{11, loop}, {-1, rep_prefix_4_byte}}},
1020 {libcall, {{32, loop}, {64, rep_prefix_4_byte},
1021 {8192, rep_prefix_8_byte}, {-1, libcall}}}},
1022 {{libcall, {{8, loop}, {15, unrolled_loop},
1023 {2048, rep_prefix_4_byte}, {-1, libcall}}},
1024 {libcall, {{24, loop}, {32, unrolled_loop},
1025 {8192, rep_prefix_8_byte}, {-1, libcall}}}},
1026 1, /* scalar_stmt_cost. */
1027 1, /* scalar load_cost. */
1028 1, /* scalar_store_cost. */
1029 1, /* vec_stmt_cost. */
1030 1, /* vec_to_scalar_cost. */
1031 1, /* scalar_to_vec_cost. */
1032 1, /* vec_align_load_cost. */
1033 2, /* vec_unalign_load_cost. */
1034 1, /* vec_store_cost. */
1035 3, /* cond_taken_branch_cost. */
1036 1, /* cond_not_taken_branch_cost. */
1039 /* Generic64 should produce code tuned for Nocona and K8. */
1041 struct processor_costs generic64_cost = {
1042 COSTS_N_INSNS (1), /* cost of an add instruction */
1043 /* On all chips taken into consideration lea is 2 cycles and more. With
1044 this cost however our current implementation of synth_mult results in
1045 use of unnecessary temporary registers causing regression on several
1046 SPECfp benchmarks. */
1047 COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */
1048 COSTS_N_INSNS (1), /* variable shift costs */
1049 COSTS_N_INSNS (1), /* constant shift costs */
1050 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1051 COSTS_N_INSNS (4), /* HI */
1052 COSTS_N_INSNS (3), /* SI */
1053 COSTS_N_INSNS (4), /* DI */
1054 COSTS_N_INSNS (2)}, /* other */
1055 0, /* cost of multiply per each bit set */
1056 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
1057 COSTS_N_INSNS (26), /* HI */
1058 COSTS_N_INSNS (42), /* SI */
1059 COSTS_N_INSNS (74), /* DI */
1060 COSTS_N_INSNS (74)}, /* other */
1061 COSTS_N_INSNS (1), /* cost of movsx */
1062 COSTS_N_INSNS (1), /* cost of movzx */
1063 8, /* "large" insn */
1064 17, /* MOVE_RATIO */
1065 4, /* cost for loading QImode using movzbl */
1066 {4, 4, 4}, /* cost of loading integer registers
1067 in QImode, HImode and SImode.
1068 Relative to reg-reg move (2). */
1069 {4, 4, 4}, /* cost of storing integer registers */
1070 4, /* cost of reg,reg fld/fst */
1071 {12, 12, 12}, /* cost of loading fp registers
1072 in SFmode, DFmode and XFmode */
1073 {6, 6, 8}, /* cost of storing fp registers
1074 in SFmode, DFmode and XFmode */
1075 2, /* cost of moving MMX register */
1076 {8, 8}, /* cost of loading MMX registers
1077 in SImode and DImode */
1078 {8, 8}, /* cost of storing MMX registers
1079 in SImode and DImode */
1080 2, /* cost of moving SSE register */
1081 {8, 8, 8}, /* cost of loading SSE registers
1082 in SImode, DImode and TImode */
1083 {8, 8, 8}, /* cost of storing SSE registers
1084 in SImode, DImode and TImode */
1085 5, /* MMX or SSE register to integer */
1086 32, /* size of l1 cache. */
1087 512, /* size of l2 cache. */
1088 64, /* size of prefetch block */
1089 6, /* number of parallel prefetches */
1090 /* Benchmarks shows large regressions on K8 sixtrack benchmark when this value
1091 is increased to perhaps more appropriate value of 5. */
1092 3, /* Branch cost */
1093 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
1094 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
1095 COSTS_N_INSNS (20), /* cost of FDIV instruction. */
1096 COSTS_N_INSNS (8), /* cost of FABS instruction. */
1097 COSTS_N_INSNS (8), /* cost of FCHS instruction. */
1098 COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
1099 {DUMMY_STRINGOP_ALGS,
1100 {libcall, {{32, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
1101 {DUMMY_STRINGOP_ALGS,
1102 {libcall, {{32, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
1103 1, /* scalar_stmt_cost. */
1104 1, /* scalar load_cost. */
1105 1, /* scalar_store_cost. */
1106 1, /* vec_stmt_cost. */
1107 1, /* vec_to_scalar_cost. */
1108 1, /* scalar_to_vec_cost. */
1109 1, /* vec_align_load_cost. */
1110 2, /* vec_unalign_load_cost. */
1111 1, /* vec_store_cost. */
1112 3, /* cond_taken_branch_cost. */
1113 1, /* cond_not_taken_branch_cost. */
1116 /* Generic32 should produce code tuned for Athlon, PPro, Pentium4, Nocona and K8. */
1118 struct processor_costs generic32_cost = {
1119 COSTS_N_INSNS (1), /* cost of an add instruction */
1120 COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */
1121 COSTS_N_INSNS (1), /* variable shift costs */
1122 COSTS_N_INSNS (1), /* constant shift costs */
1123 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1124 COSTS_N_INSNS (4), /* HI */
1125 COSTS_N_INSNS (3), /* SI */
1126 COSTS_N_INSNS (4), /* DI */
1127 COSTS_N_INSNS (2)}, /* other */
1128 0, /* cost of multiply per each bit set */
1129 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
1130 COSTS_N_INSNS (26), /* HI */
1131 COSTS_N_INSNS (42), /* SI */
1132 COSTS_N_INSNS (74), /* DI */
1133 COSTS_N_INSNS (74)}, /* other */
1134 COSTS_N_INSNS (1), /* cost of movsx */
1135 COSTS_N_INSNS (1), /* cost of movzx */
1136 8, /* "large" insn */
1137 17, /* MOVE_RATIO */
1138 4, /* cost for loading QImode using movzbl */
1139 {4, 4, 4}, /* cost of loading integer registers
1140 in QImode, HImode and SImode.
1141 Relative to reg-reg move (2). */
1142 {4, 4, 4}, /* cost of storing integer registers */
1143 4, /* cost of reg,reg fld/fst */
1144 {12, 12, 12}, /* cost of loading fp registers
1145 in SFmode, DFmode and XFmode */
1146 {6, 6, 8}, /* cost of storing fp registers
1147 in SFmode, DFmode and XFmode */
1148 2, /* cost of moving MMX register */
1149 {8, 8}, /* cost of loading MMX registers
1150 in SImode and DImode */
1151 {8, 8}, /* cost of storing MMX registers
1152 in SImode and DImode */
1153 2, /* cost of moving SSE register */
1154 {8, 8, 8}, /* cost of loading SSE registers
1155 in SImode, DImode and TImode */
1156 {8, 8, 8}, /* cost of storing SSE registers
1157 in SImode, DImode and TImode */
1158 5, /* MMX or SSE register to integer */
1159 32, /* size of l1 cache. */
1160 256, /* size of l2 cache. */
1161 64, /* size of prefetch block */
1162 6, /* number of parallel prefetches */
1163 3, /* Branch cost */
1164 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
1165 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
1166 COSTS_N_INSNS (20), /* cost of FDIV instruction. */
1167 COSTS_N_INSNS (8), /* cost of FABS instruction. */
1168 COSTS_N_INSNS (8), /* cost of FCHS instruction. */
1169 COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
1170 {{libcall, {{32, loop}, {8192, rep_prefix_4_byte}, {-1, libcall}}},
1171 DUMMY_STRINGOP_ALGS},
1172 {{libcall, {{32, loop}, {8192, rep_prefix_4_byte}, {-1, libcall}}},
1173 DUMMY_STRINGOP_ALGS},
1174 1, /* scalar_stmt_cost. */
1175 1, /* scalar load_cost. */
1176 1, /* scalar_store_cost. */
1177 1, /* vec_stmt_cost. */
1178 1, /* vec_to_scalar_cost. */
1179 1, /* scalar_to_vec_cost. */
1180 1, /* vec_align_load_cost. */
1181 2, /* vec_unalign_load_cost. */
1182 1, /* vec_store_cost. */
1183 3, /* cond_taken_branch_cost. */
1184 1, /* cond_not_taken_branch_cost. */
1187 const struct processor_costs *ix86_cost = &pentium_cost;
1189 /* Processor feature/optimization bitmasks. */
1190 #define m_386 (1<<PROCESSOR_I386)
1191 #define m_486 (1<<PROCESSOR_I486)
1192 #define m_PENT (1<<PROCESSOR_PENTIUM)
1193 #define m_PPRO (1<<PROCESSOR_PENTIUMPRO)
1194 #define m_PENT4 (1<<PROCESSOR_PENTIUM4)
1195 #define m_NOCONA (1<<PROCESSOR_NOCONA)
1196 #define m_CORE2 (1<<PROCESSOR_CORE2)
1198 #define m_GEODE (1<<PROCESSOR_GEODE)
1199 #define m_K6 (1<<PROCESSOR_K6)
1200 #define m_K6_GEODE (m_K6 | m_GEODE)
1201 #define m_K8 (1<<PROCESSOR_K8)
1202 #define m_ATHLON (1<<PROCESSOR_ATHLON)
1203 #define m_ATHLON_K8 (m_K8 | m_ATHLON)
1204 #define m_AMDFAM10 (1<<PROCESSOR_AMDFAM10)
1205 #define m_AMD_MULTIPLE (m_K8 | m_ATHLON | m_AMDFAM10)
1207 #define m_GENERIC32 (1<<PROCESSOR_GENERIC32)
1208 #define m_GENERIC64 (1<<PROCESSOR_GENERIC64)
1210 /* Generic instruction choice should be common subset of supported CPUs
1211 (PPro/PENT4/NOCONA/CORE2/Athlon/K8). */
1212 #define m_GENERIC (m_GENERIC32 | m_GENERIC64)
1214 /* Feature tests against the various tunings. */
1215 unsigned char ix86_tune_features[X86_TUNE_LAST];
1217 /* Feature tests against the various tunings used to create ix86_tune_features
1218 based on the processor mask. */
1219 static unsigned int initial_ix86_tune_features[X86_TUNE_LAST] = {
1220 /* X86_TUNE_USE_LEAVE: Leave does not affect Nocona SPEC2000 results
1221 negatively, so enabling for Generic64 seems like good code size
1222 tradeoff. We can't enable it for 32bit generic because it does not
1223 work well with PPro base chips. */
1224 m_386 | m_K6_GEODE | m_AMD_MULTIPLE | m_CORE2 | m_GENERIC64,
1226 /* X86_TUNE_PUSH_MEMORY */
1227 m_386 | m_K6_GEODE | m_AMD_MULTIPLE | m_PENT4
1228 | m_NOCONA | m_CORE2 | m_GENERIC,
1230 /* X86_TUNE_ZERO_EXTEND_WITH_AND */
1233 /* X86_TUNE_UNROLL_STRLEN */
1234 m_486 | m_PENT | m_PPRO | m_AMD_MULTIPLE | m_K6 | m_CORE2 | m_GENERIC,
1236 /* X86_TUNE_DEEP_BRANCH_PREDICTION */
1237 m_PPRO | m_K6_GEODE | m_AMD_MULTIPLE | m_PENT4 | m_GENERIC,
1239 /* X86_TUNE_BRANCH_PREDICTION_HINTS: Branch hints were put in P4 based
1240 on simulation result. But after P4 was made, no performance benefit
1241 was observed with branch hints. It also increases the code size.
1242 As a result, icc never generates branch hints. */
1245 /* X86_TUNE_DOUBLE_WITH_ADD */
1248 /* X86_TUNE_USE_SAHF */
1249 m_PPRO | m_K6_GEODE | m_K8 | m_AMDFAM10 | m_PENT4
1250 | m_NOCONA | m_CORE2 | m_GENERIC,
1252 /* X86_TUNE_MOVX: Enable to zero extend integer registers to avoid
1253 partial dependencies. */
1254 m_AMD_MULTIPLE | m_PPRO | m_PENT4 | m_NOCONA
1255 | m_CORE2 | m_GENERIC | m_GEODE /* m_386 | m_K6 */,
1257 /* X86_TUNE_PARTIAL_REG_STALL: We probably ought to watch for partial
1258 register stalls on Generic32 compilation setting as well. However
1259 in current implementation the partial register stalls are not eliminated
1260 very well - they can be introduced via subregs synthesized by combine
1261 and can happen in caller/callee saving sequences. Because this option
1262 pays back little on PPro based chips and is in conflict with partial reg
1263 dependencies used by Athlon/P4 based chips, it is better to leave it off
1264 for generic32 for now. */
1267 /* X86_TUNE_PARTIAL_FLAG_REG_STALL */
1268 m_CORE2 | m_GENERIC,
1270 /* X86_TUNE_USE_HIMODE_FIOP */
1271 m_386 | m_486 | m_K6_GEODE,
1273 /* X86_TUNE_USE_SIMODE_FIOP */
1274 ~(m_PPRO | m_AMD_MULTIPLE | m_PENT | m_CORE2 | m_GENERIC),
1276 /* X86_TUNE_USE_MOV0 */
1279 /* X86_TUNE_USE_CLTD */
1280 ~(m_PENT | m_K6 | m_CORE2 | m_GENERIC),
1282 /* X86_TUNE_USE_XCHGB: Use xchgb %rh,%rl instead of rolw/rorw $8,rx. */
1285 /* X86_TUNE_SPLIT_LONG_MOVES */
1288 /* X86_TUNE_READ_MODIFY_WRITE */
1291 /* X86_TUNE_READ_MODIFY */
1294 /* X86_TUNE_PROMOTE_QIMODE */
1295 m_K6_GEODE | m_PENT | m_386 | m_486 | m_AMD_MULTIPLE | m_CORE2
1296 | m_GENERIC /* | m_PENT4 ? */,
1298 /* X86_TUNE_FAST_PREFIX */
1299 ~(m_PENT | m_486 | m_386),
1301 /* X86_TUNE_SINGLE_STRINGOP */
1302 m_386 | m_PENT4 | m_NOCONA,
1304 /* X86_TUNE_QIMODE_MATH */
1307 /* X86_TUNE_HIMODE_MATH: On PPro this flag is meant to avoid partial
1308 register stalls. Just like X86_TUNE_PARTIAL_REG_STALL this option
1309 might be considered for Generic32 if our scheme for avoiding partial
1310 stalls was more effective. */
1313 /* X86_TUNE_PROMOTE_QI_REGS */
1316 /* X86_TUNE_PROMOTE_HI_REGS */
1319 /* X86_TUNE_ADD_ESP_4: Enable if add/sub is preferred over 1/2 push/pop. */
1320 m_AMD_MULTIPLE | m_K6_GEODE | m_PENT4 | m_NOCONA | m_CORE2 | m_GENERIC,
1322 /* X86_TUNE_ADD_ESP_8 */
1323 m_AMD_MULTIPLE | m_PPRO | m_K6_GEODE | m_386
1324 | m_486 | m_PENT4 | m_NOCONA | m_CORE2 | m_GENERIC,
1326 /* X86_TUNE_SUB_ESP_4 */
1327 m_AMD_MULTIPLE | m_PPRO | m_PENT4 | m_NOCONA | m_CORE2 | m_GENERIC,
1329 /* X86_TUNE_SUB_ESP_8 */
1330 m_AMD_MULTIPLE | m_PPRO | m_386 | m_486
1331 | m_PENT4 | m_NOCONA | m_CORE2 | m_GENERIC,
1333 /* X86_TUNE_INTEGER_DFMODE_MOVES: Enable if integer moves are preferred
1334 for DFmode copies */
1335 ~(m_AMD_MULTIPLE | m_PENT4 | m_NOCONA | m_PPRO | m_CORE2
1336 | m_GENERIC | m_GEODE),
1338 /* X86_TUNE_PARTIAL_REG_DEPENDENCY */
1339 m_AMD_MULTIPLE | m_PENT4 | m_NOCONA | m_CORE2 | m_GENERIC,
1341 /* X86_TUNE_SSE_PARTIAL_REG_DEPENDENCY: In the Generic model we have a
1342 conflict here in between PPro/Pentium4 based chips that thread 128bit
1343 SSE registers as single units versus K8 based chips that divide SSE
1344 registers to two 64bit halves. This knob promotes all store destinations
1345 to be 128bit to allow register renaming on 128bit SSE units, but usually
1346 results in one extra microop on 64bit SSE units. Experimental results
1347 shows that disabling this option on P4 brings over 20% SPECfp regression,
1348 while enabling it on K8 brings roughly 2.4% regression that can be partly
1349 masked by careful scheduling of moves. */
1350 m_PENT4 | m_NOCONA | m_PPRO | m_CORE2 | m_GENERIC | m_AMDFAM10,
1352 /* X86_TUNE_SSE_UNALIGNED_MOVE_OPTIMAL */
1355 /* X86_TUNE_SSE_SPLIT_REGS: Set for machines where the type and dependencies
1356 are resolved on SSE register parts instead of whole registers, so we may
1357 maintain just lower part of scalar values in proper format leaving the
1358 upper part undefined. */
1361 /* X86_TUNE_SSE_TYPELESS_STORES */
1364 /* X86_TUNE_SSE_LOAD0_BY_PXOR */
1365 m_PPRO | m_PENT4 | m_NOCONA,
1367 /* X86_TUNE_MEMORY_MISMATCH_STALL */
1368 m_AMD_MULTIPLE | m_PENT4 | m_NOCONA | m_CORE2 | m_GENERIC,
1370 /* X86_TUNE_PROLOGUE_USING_MOVE */
1371 m_ATHLON_K8 | m_PPRO | m_CORE2 | m_GENERIC,
1373 /* X86_TUNE_EPILOGUE_USING_MOVE */
1374 m_ATHLON_K8 | m_PPRO | m_CORE2 | m_GENERIC,
1376 /* X86_TUNE_SHIFT1 */
1379 /* X86_TUNE_USE_FFREEP */
1382 /* X86_TUNE_INTER_UNIT_MOVES */
1383 ~(m_AMD_MULTIPLE | m_GENERIC),
1385 /* X86_TUNE_INTER_UNIT_CONVERSIONS */
1388 /* X86_TUNE_FOUR_JUMP_LIMIT: Some CPU cores are not able to predict more
1389 than 4 branch instructions in the 16 byte window. */
1390 m_PPRO | m_AMD_MULTIPLE | m_PENT4 | m_NOCONA | m_CORE2 | m_GENERIC,
1392 /* X86_TUNE_SCHEDULE */
1393 m_PPRO | m_AMD_MULTIPLE | m_K6_GEODE | m_PENT | m_CORE2 | m_GENERIC,
1395 /* X86_TUNE_USE_BT */
1396 m_AMD_MULTIPLE | m_CORE2 | m_GENERIC,
1398 /* X86_TUNE_USE_INCDEC */
1399 ~(m_PENT4 | m_NOCONA | m_GENERIC),
1401 /* X86_TUNE_PAD_RETURNS */
1402 m_AMD_MULTIPLE | m_CORE2 | m_GENERIC,
1404 /* X86_TUNE_EXT_80387_CONSTANTS */
1405 m_K6_GEODE | m_ATHLON_K8 | m_PENT4 | m_NOCONA | m_PPRO | m_CORE2 | m_GENERIC,
1407 /* X86_TUNE_SHORTEN_X87_SSE */
1410 /* X86_TUNE_AVOID_VECTOR_DECODE */
1413 /* X86_TUNE_PROMOTE_HIMODE_IMUL: Modern CPUs have same latency for HImode
1414 and SImode multiply, but 386 and 486 do HImode multiply faster. */
1417 /* X86_TUNE_SLOW_IMUL_IMM32_MEM: Imul of 32-bit constant and memory is
1418 vector path on AMD machines. */
1419 m_K8 | m_GENERIC64 | m_AMDFAM10,
1421 /* X86_TUNE_SLOW_IMUL_IMM8: Imul of 8-bit constant is vector path on AMD
1423 m_K8 | m_GENERIC64 | m_AMDFAM10,
1425 /* X86_TUNE_MOVE_M1_VIA_OR: On pentiums, it is faster to load -1 via OR
1429 /* X86_TUNE_NOT_UNPAIRABLE: NOT is not pairable on Pentium, while XOR is,
1430 but one byte longer. */
1433 /* X86_TUNE_NOT_VECTORMODE: On AMD K6, NOT is vector decoded with memory
1434 operand that cannot be represented using a modRM byte. The XOR
1435 replacement is long decoded, so this split helps here as well. */
1438 /* X86_TUNE_USE_VECTOR_FP_CONVERTS: Prefer vector packed SSE conversion
1440 m_AMDFAM10 | m_GENERIC,
1442 /* X86_TUNE_USE_VECTOR_CONVERTS: Prefer vector packed SSE conversion
1443 from integer to FP. */
1446 /* X86_TUNE_FUSE_CMP_AND_BRANCH: Fuse a compare or test instruction
1447 with a subsequent conditional jump instruction into a single
1448 compare-and-branch uop. */
1452 /* Feature tests against the various architecture variations. */
1453 unsigned char ix86_arch_features[X86_ARCH_LAST];
1455 /* Feature tests against the various architecture variations, used to create
1456 ix86_arch_features based on the processor mask. */
1457 static unsigned int initial_ix86_arch_features[X86_ARCH_LAST] = {
1458 /* X86_ARCH_CMOVE: Conditional move was added for pentiumpro. */
1459 ~(m_386 | m_486 | m_PENT | m_K6),
1461 /* X86_ARCH_CMPXCHG: Compare and exchange was added for 80486. */
1464 /* X86_ARCH_CMPXCHG8B: Compare and exchange 8 bytes was added for pentium. */
1467 /* X86_ARCH_XADD: Exchange and add was added for 80486. */
1470 /* X86_ARCH_BSWAP: Byteswap was added for 80486. */
1474 static const unsigned int x86_accumulate_outgoing_args
1475 = m_AMD_MULTIPLE | m_PENT4 | m_NOCONA | m_PPRO | m_CORE2 | m_GENERIC;
1477 static const unsigned int x86_arch_always_fancy_math_387
1478 = m_PENT | m_PPRO | m_AMD_MULTIPLE | m_PENT4
1479 | m_NOCONA | m_CORE2 | m_GENERIC;
1481 static enum stringop_alg stringop_alg = no_stringop;
1483 /* In case the average insn count for single function invocation is
1484 lower than this constant, emit fast (but longer) prologue and
1486 #define FAST_PROLOGUE_INSN_COUNT 20
1488 /* Names for 8 (low), 8 (high), and 16-bit registers, respectively. */
1489 static const char *const qi_reg_name[] = QI_REGISTER_NAMES;
1490 static const char *const qi_high_reg_name[] = QI_HIGH_REGISTER_NAMES;
1491 static const char *const hi_reg_name[] = HI_REGISTER_NAMES;
1493 /* Array of the smallest class containing reg number REGNO, indexed by
1494 REGNO. Used by REGNO_REG_CLASS in i386.h. */
1496 enum reg_class const regclass_map[FIRST_PSEUDO_REGISTER] =
1498 /* ax, dx, cx, bx */
1499 AREG, DREG, CREG, BREG,
1500 /* si, di, bp, sp */
1501 SIREG, DIREG, NON_Q_REGS, NON_Q_REGS,
1503 FP_TOP_REG, FP_SECOND_REG, FLOAT_REGS, FLOAT_REGS,
1504 FLOAT_REGS, FLOAT_REGS, FLOAT_REGS, FLOAT_REGS,
1507 /* flags, fpsr, fpcr, frame */
1508 NO_REGS, NO_REGS, NO_REGS, NON_Q_REGS,
1510 SSE_FIRST_REG, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
1513 MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS,
1516 NON_Q_REGS, NON_Q_REGS, NON_Q_REGS, NON_Q_REGS,
1517 NON_Q_REGS, NON_Q_REGS, NON_Q_REGS, NON_Q_REGS,
1518 /* SSE REX registers */
1519 SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
1523 /* The "default" register map used in 32bit mode. */
1525 int const dbx_register_map[FIRST_PSEUDO_REGISTER] =
1527 0, 2, 1, 3, 6, 7, 4, 5, /* general regs */
1528 12, 13, 14, 15, 16, 17, 18, 19, /* fp regs */
1529 -1, -1, -1, -1, -1, /* arg, flags, fpsr, fpcr, frame */
1530 21, 22, 23, 24, 25, 26, 27, 28, /* SSE */
1531 29, 30, 31, 32, 33, 34, 35, 36, /* MMX */
1532 -1, -1, -1, -1, -1, -1, -1, -1, /* extended integer registers */
1533 -1, -1, -1, -1, -1, -1, -1, -1, /* extended SSE registers */
1536 static int const x86_64_int_parameter_registers[6] =
1538 5 /*RDI*/, 4 /*RSI*/, 1 /*RDX*/, 2 /*RCX*/,
1539 FIRST_REX_INT_REG /*R8 */, FIRST_REX_INT_REG + 1 /*R9 */
1542 static int const x86_64_ms_abi_int_parameter_registers[4] =
1544 2 /*RCX*/, 1 /*RDX*/,
1545 FIRST_REX_INT_REG /*R8 */, FIRST_REX_INT_REG + 1 /*R9 */
1548 static int const x86_64_int_return_registers[4] =
1550 0 /*RAX*/, 1 /*RDX*/, 5 /*RDI*/, 4 /*RSI*/
1553 /* The "default" register map used in 64bit mode. */
1554 int const dbx64_register_map[FIRST_PSEUDO_REGISTER] =
1556 0, 1, 2, 3, 4, 5, 6, 7, /* general regs */
1557 33, 34, 35, 36, 37, 38, 39, 40, /* fp regs */
1558 -1, -1, -1, -1, -1, /* arg, flags, fpsr, fpcr, frame */
1559 17, 18, 19, 20, 21, 22, 23, 24, /* SSE */
1560 41, 42, 43, 44, 45, 46, 47, 48, /* MMX */
1561 8,9,10,11,12,13,14,15, /* extended integer registers */
1562 25, 26, 27, 28, 29, 30, 31, 32, /* extended SSE registers */
1565 /* Define the register numbers to be used in Dwarf debugging information.
1566 The SVR4 reference port C compiler uses the following register numbers
1567 in its Dwarf output code:
1568 0 for %eax (gcc regno = 0)
1569 1 for %ecx (gcc regno = 2)
1570 2 for %edx (gcc regno = 1)
1571 3 for %ebx (gcc regno = 3)
1572 4 for %esp (gcc regno = 7)
1573 5 for %ebp (gcc regno = 6)
1574 6 for %esi (gcc regno = 4)
1575 7 for %edi (gcc regno = 5)
1576 The following three DWARF register numbers are never generated by
1577 the SVR4 C compiler or by the GNU compilers, but SDB on x86/svr4
1578 believes these numbers have these meanings.
1579 8 for %eip (no gcc equivalent)
1580 9 for %eflags (gcc regno = 17)
1581 10 for %trapno (no gcc equivalent)
1582 It is not at all clear how we should number the FP stack registers
1583 for the x86 architecture. If the version of SDB on x86/svr4 were
1584 a bit less brain dead with respect to floating-point then we would
1585 have a precedent to follow with respect to DWARF register numbers
1586 for x86 FP registers, but the SDB on x86/svr4 is so completely
1587 broken with respect to FP registers that it is hardly worth thinking
1588 of it as something to strive for compatibility with.
1589 The version of x86/svr4 SDB I have at the moment does (partially)
1590 seem to believe that DWARF register number 11 is associated with
1591 the x86 register %st(0), but that's about all. Higher DWARF
1592 register numbers don't seem to be associated with anything in
1593 particular, and even for DWARF regno 11, SDB only seems to under-
1594 stand that it should say that a variable lives in %st(0) (when
1595 asked via an `=' command) if we said it was in DWARF regno 11,
1596 but SDB still prints garbage when asked for the value of the
1597 variable in question (via a `/' command).
1598 (Also note that the labels SDB prints for various FP stack regs
1599 when doing an `x' command are all wrong.)
1600 Note that these problems generally don't affect the native SVR4
1601 C compiler because it doesn't allow the use of -O with -g and
1602 because when it is *not* optimizing, it allocates a memory
1603 location for each floating-point variable, and the memory
1604 location is what gets described in the DWARF AT_location
1605 attribute for the variable in question.
1606 Regardless of the severe mental illness of the x86/svr4 SDB, we
1607 do something sensible here and we use the following DWARF
1608 register numbers. Note that these are all stack-top-relative
1610 11 for %st(0) (gcc regno = 8)
1611 12 for %st(1) (gcc regno = 9)
1612 13 for %st(2) (gcc regno = 10)
1613 14 for %st(3) (gcc regno = 11)
1614 15 for %st(4) (gcc regno = 12)
1615 16 for %st(5) (gcc regno = 13)
1616 17 for %st(6) (gcc regno = 14)
1617 18 for %st(7) (gcc regno = 15)
1619 int const svr4_dbx_register_map[FIRST_PSEUDO_REGISTER] =
1621 0, 2, 1, 3, 6, 7, 5, 4, /* general regs */
1622 11, 12, 13, 14, 15, 16, 17, 18, /* fp regs */
1623 -1, 9, -1, -1, -1, /* arg, flags, fpsr, fpcr, frame */
1624 21, 22, 23, 24, 25, 26, 27, 28, /* SSE registers */
1625 29, 30, 31, 32, 33, 34, 35, 36, /* MMX registers */
1626 -1, -1, -1, -1, -1, -1, -1, -1, /* extended integer registers */
1627 -1, -1, -1, -1, -1, -1, -1, -1, /* extended SSE registers */
1630 /* Test and compare insns in i386.md store the information needed to
1631 generate branch and scc insns here. */
1633 rtx ix86_compare_op0 = NULL_RTX;
1634 rtx ix86_compare_op1 = NULL_RTX;
1635 rtx ix86_compare_emitted = NULL_RTX;
1637 /* Define the structure for the machine field in struct function. */
1639 struct stack_local_entry GTY(())
1641 unsigned short mode;
1644 struct stack_local_entry *next;
1647 /* Structure describing stack frame layout.
1648 Stack grows downward:
1654 saved frame pointer if frame_pointer_needed
1655 <- HARD_FRAME_POINTER
1664 [va_arg registers] (
1665 > to_allocate <- FRAME_POINTER
1677 HOST_WIDE_INT frame;
1679 int outgoing_arguments_size;
1682 HOST_WIDE_INT to_allocate;
1683 /* The offsets relative to ARG_POINTER. */
1684 HOST_WIDE_INT frame_pointer_offset;
1685 HOST_WIDE_INT hard_frame_pointer_offset;
1686 HOST_WIDE_INT stack_pointer_offset;
1688 /* When save_regs_using_mov is set, emit prologue using
1689 move instead of push instructions. */
1690 bool save_regs_using_mov;
1693 /* Code model option. */
1694 enum cmodel ix86_cmodel;
1696 enum asm_dialect ix86_asm_dialect = ASM_ATT;
1698 enum tls_dialect ix86_tls_dialect = TLS_DIALECT_GNU;
1700 /* Which unit we are generating floating point math for. */
1701 enum fpmath_unit ix86_fpmath;
1703 /* Which cpu are we scheduling for. */
1704 enum attr_cpu ix86_schedule;
1706 /* Which cpu are we optimizing for. */
1707 enum processor_type ix86_tune;
1709 /* Which instruction set architecture to use. */
1710 enum processor_type ix86_arch;
1712 /* true if sse prefetch instruction is not NOOP. */
1713 int x86_prefetch_sse;
1715 /* ix86_regparm_string as a number */
1716 static int ix86_regparm;
1718 /* -mstackrealign option */
1719 extern int ix86_force_align_arg_pointer;
1720 static const char ix86_force_align_arg_pointer_string[]
1721 = "force_align_arg_pointer";
1723 static rtx (*ix86_gen_leave) (void);
1724 static rtx (*ix86_gen_pop1) (rtx);
1725 static rtx (*ix86_gen_add3) (rtx, rtx, rtx);
1726 static rtx (*ix86_gen_sub3) (rtx, rtx, rtx);
1727 static rtx (*ix86_gen_sub3_carry) (rtx, rtx, rtx, rtx);
1728 static rtx (*ix86_gen_one_cmpl2) (rtx, rtx);
1729 static rtx (*ix86_gen_monitor) (rtx, rtx, rtx);
1730 static rtx (*ix86_gen_andsp) (rtx, rtx, rtx);
1732 /* Preferred alignment for stack boundary in bits. */
1733 unsigned int ix86_preferred_stack_boundary;
1735 /* Alignment for incoming stack boundary in bits specified at
1737 static unsigned int ix86_user_incoming_stack_boundary;
1739 /* Default alignment for incoming stack boundary in bits. */
1740 static unsigned int ix86_default_incoming_stack_boundary;
1742 /* Alignment for incoming stack boundary in bits. */
1743 unsigned int ix86_incoming_stack_boundary;
1745 /* Values 1-5: see jump.c */
1746 int ix86_branch_cost;
1748 /* Calling abi specific va_list type nodes. */
1749 static GTY(()) tree sysv_va_list_type_node;
1750 static GTY(()) tree ms_va_list_type_node;
1752 /* Variables which are this size or smaller are put in the data/bss
1753 or ldata/lbss sections. */
1755 int ix86_section_threshold = 65536;
1757 /* Prefix built by ASM_GENERATE_INTERNAL_LABEL. */
1758 char internal_label_prefix[16];
1759 int internal_label_prefix_len;
1761 /* Fence to use after loop using movnt. */
1764 /* Register class used for passing given 64bit part of the argument.
1765 These represent classes as documented by the PS ABI, with the exception
1766 of SSESF, SSEDF classes, that are basically SSE class, just gcc will
1767 use SF or DFmode move instead of DImode to avoid reformatting penalties.
1769 Similarly we play games with INTEGERSI_CLASS to use cheaper SImode moves
1770 whenever possible (upper half does contain padding). */
1771 enum x86_64_reg_class
1774 X86_64_INTEGER_CLASS,
1775 X86_64_INTEGERSI_CLASS,
1783 X86_64_COMPLEX_X87_CLASS,
1786 static const char * const x86_64_reg_class_name[] =
1788 "no", "integer", "integerSI", "sse", "sseSF", "sseDF",
1789 "sseup", "x87", "x87up", "cplx87", "no"
1792 #define MAX_CLASSES 4
1794 /* Table of constants used by fldpi, fldln2, etc.... */
1795 static REAL_VALUE_TYPE ext_80387_constants_table [5];
1796 static bool ext_80387_constants_init = 0;
1799 static struct machine_function * ix86_init_machine_status (void);
1800 static rtx ix86_function_value (const_tree, const_tree, bool);
1801 static int ix86_function_regparm (const_tree, const_tree);
1802 static void ix86_compute_frame_layout (struct ix86_frame *);
1803 static bool ix86_expand_vector_init_one_nonzero (bool, enum machine_mode,
1805 static void ix86_add_new_builtins (int);
1807 enum ix86_function_specific_strings
1809 IX86_FUNCTION_SPECIFIC_ARCH,
1810 IX86_FUNCTION_SPECIFIC_TUNE,
1811 IX86_FUNCTION_SPECIFIC_FPMATH,
1812 IX86_FUNCTION_SPECIFIC_MAX
1815 static char *ix86_target_string (int, int, const char *, const char *,
1816 const char *, bool);
1817 static void ix86_debug_options (void) ATTRIBUTE_UNUSED;
1818 static void ix86_function_specific_save (struct cl_target_option *);
1819 static void ix86_function_specific_restore (struct cl_target_option *);
1820 static void ix86_function_specific_print (FILE *, int,
1821 struct cl_target_option *);
1822 static bool ix86_valid_target_attribute_p (tree, tree, tree, int);
1823 static bool ix86_valid_target_attribute_inner_p (tree, char *[]);
1824 static bool ix86_can_inline_p (tree, tree);
1825 static void ix86_set_current_function (tree);
1828 /* The svr4 ABI for the i386 says that records and unions are returned
1830 #ifndef DEFAULT_PCC_STRUCT_RETURN
1831 #define DEFAULT_PCC_STRUCT_RETURN 1
1834 /* Whether -mtune= or -march= were specified */
1835 static int ix86_tune_defaulted;
1836 static int ix86_arch_specified;
1838 /* Bit flags that specify the ISA we are compiling for. */
1839 int ix86_isa_flags = TARGET_64BIT_DEFAULT | TARGET_SUBTARGET_ISA_DEFAULT;
1841 /* A mask of ix86_isa_flags that includes bit X if X
1842 was set or cleared on the command line. */
1843 static int ix86_isa_flags_explicit;
1845 /* Define a set of ISAs which are available when a given ISA is
1846 enabled. MMX and SSE ISAs are handled separately. */
1848 #define OPTION_MASK_ISA_MMX_SET OPTION_MASK_ISA_MMX
1849 #define OPTION_MASK_ISA_3DNOW_SET \
1850 (OPTION_MASK_ISA_3DNOW | OPTION_MASK_ISA_MMX_SET)
1852 #define OPTION_MASK_ISA_SSE_SET OPTION_MASK_ISA_SSE
1853 #define OPTION_MASK_ISA_SSE2_SET \
1854 (OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_SSE_SET)
1855 #define OPTION_MASK_ISA_SSE3_SET \
1856 (OPTION_MASK_ISA_SSE3 | OPTION_MASK_ISA_SSE2_SET)
1857 #define OPTION_MASK_ISA_SSSE3_SET \
1858 (OPTION_MASK_ISA_SSSE3 | OPTION_MASK_ISA_SSE3_SET)
1859 #define OPTION_MASK_ISA_SSE4_1_SET \
1860 (OPTION_MASK_ISA_SSE4_1 | OPTION_MASK_ISA_SSSE3_SET)
1861 #define OPTION_MASK_ISA_SSE4_2_SET \
1862 (OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_SSE4_1_SET)
1863 #define OPTION_MASK_ISA_AVX_SET \
1864 (OPTION_MASK_ISA_AVX | OPTION_MASK_ISA_SSE4_2_SET)
1865 #define OPTION_MASK_ISA_FMA_SET \
1866 (OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_AVX_SET)
1868 /* SSE4 includes both SSE4.1 and SSE4.2. -msse4 should be the same
1870 #define OPTION_MASK_ISA_SSE4_SET OPTION_MASK_ISA_SSE4_2_SET
1872 #define OPTION_MASK_ISA_SSE4A_SET \
1873 (OPTION_MASK_ISA_SSE4A | OPTION_MASK_ISA_SSE3_SET)
1874 #define OPTION_MASK_ISA_SSE5_SET \
1875 (OPTION_MASK_ISA_SSE5 | OPTION_MASK_ISA_SSE4A_SET)
1877 /* AES and PCLMUL need SSE2 because they use xmm registers */
1878 #define OPTION_MASK_ISA_AES_SET \
1879 (OPTION_MASK_ISA_AES | OPTION_MASK_ISA_SSE2_SET)
1880 #define OPTION_MASK_ISA_PCLMUL_SET \
1881 (OPTION_MASK_ISA_PCLMUL | OPTION_MASK_ISA_SSE2_SET)
1883 #define OPTION_MASK_ISA_ABM_SET \
1884 (OPTION_MASK_ISA_ABM | OPTION_MASK_ISA_POPCNT)
1885 #define OPTION_MASK_ISA_POPCNT_SET OPTION_MASK_ISA_POPCNT
1886 #define OPTION_MASK_ISA_CX16_SET OPTION_MASK_ISA_CX16
1887 #define OPTION_MASK_ISA_SAHF_SET OPTION_MASK_ISA_SAHF
1889 /* Define a set of ISAs which aren't available when a given ISA is
1890 disabled. MMX and SSE ISAs are handled separately. */
1892 #define OPTION_MASK_ISA_MMX_UNSET \
1893 (OPTION_MASK_ISA_MMX | OPTION_MASK_ISA_3DNOW_UNSET)
1894 #define OPTION_MASK_ISA_3DNOW_UNSET \
1895 (OPTION_MASK_ISA_3DNOW | OPTION_MASK_ISA_3DNOW_A_UNSET)
1896 #define OPTION_MASK_ISA_3DNOW_A_UNSET OPTION_MASK_ISA_3DNOW_A
1898 #define OPTION_MASK_ISA_SSE_UNSET \
1899 (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_SSE2_UNSET)
1900 #define OPTION_MASK_ISA_SSE2_UNSET \
1901 (OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_SSE3_UNSET)
1902 #define OPTION_MASK_ISA_SSE3_UNSET \
1903 (OPTION_MASK_ISA_SSE3 \
1904 | OPTION_MASK_ISA_SSSE3_UNSET \
1905 | OPTION_MASK_ISA_SSE4A_UNSET )
1906 #define OPTION_MASK_ISA_SSSE3_UNSET \
1907 (OPTION_MASK_ISA_SSSE3 | OPTION_MASK_ISA_SSE4_1_UNSET)
1908 #define OPTION_MASK_ISA_SSE4_1_UNSET \
1909 (OPTION_MASK_ISA_SSE4_1 | OPTION_MASK_ISA_SSE4_2_UNSET)
1910 #define OPTION_MASK_ISA_SSE4_2_UNSET \
1911 (OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_AVX_UNSET )
1912 #define OPTION_MASK_ISA_AVX_UNSET \
1913 (OPTION_MASK_ISA_AVX | OPTION_MASK_ISA_FMA_UNSET)
1914 #define OPTION_MASK_ISA_FMA_UNSET OPTION_MASK_ISA_FMA
1916 /* SSE4 includes both SSE4.1 and SSE4.2. -mno-sse4 should the same
1918 #define OPTION_MASK_ISA_SSE4_UNSET OPTION_MASK_ISA_SSE4_1_UNSET
1920 #define OPTION_MASK_ISA_SSE4A_UNSET \
1921 (OPTION_MASK_ISA_SSE4A | OPTION_MASK_ISA_SSE5_UNSET)
1922 #define OPTION_MASK_ISA_SSE5_UNSET OPTION_MASK_ISA_SSE5
1923 #define OPTION_MASK_ISA_AES_UNSET OPTION_MASK_ISA_AES
1924 #define OPTION_MASK_ISA_PCLMUL_UNSET OPTION_MASK_ISA_PCLMUL
1925 #define OPTION_MASK_ISA_ABM_UNSET OPTION_MASK_ISA_ABM
1926 #define OPTION_MASK_ISA_POPCNT_UNSET OPTION_MASK_ISA_POPCNT
1927 #define OPTION_MASK_ISA_CX16_UNSET OPTION_MASK_ISA_CX16
1928 #define OPTION_MASK_ISA_SAHF_UNSET OPTION_MASK_ISA_SAHF
1930 /* Vectorization library interface and handlers. */
1931 tree (*ix86_veclib_handler)(enum built_in_function, tree, tree) = NULL;
1932 static tree ix86_veclibabi_svml (enum built_in_function, tree, tree);
1933 static tree ix86_veclibabi_acml (enum built_in_function, tree, tree);
1935 /* Processor target table, indexed by processor number */
1938 const struct processor_costs *cost; /* Processor costs */
1939 const int align_loop; /* Default alignments. */
1940 const int align_loop_max_skip;
1941 const int align_jump;
1942 const int align_jump_max_skip;
1943 const int align_func;
1946 static const struct ptt processor_target_table[PROCESSOR_max] =
1948 {&i386_cost, 4, 3, 4, 3, 4},
1949 {&i486_cost, 16, 15, 16, 15, 16},
1950 {&pentium_cost, 16, 7, 16, 7, 16},
1951 {&pentiumpro_cost, 16, 15, 16, 10, 16},
1952 {&geode_cost, 0, 0, 0, 0, 0},
1953 {&k6_cost, 32, 7, 32, 7, 32},
1954 {&athlon_cost, 16, 7, 16, 7, 16},
1955 {&pentium4_cost, 0, 0, 0, 0, 0},
1956 {&k8_cost, 16, 7, 16, 7, 16},
1957 {&nocona_cost, 0, 0, 0, 0, 0},
1958 {&core2_cost, 16, 10, 16, 10, 16},
1959 {&generic32_cost, 16, 7, 16, 7, 16},
1960 {&generic64_cost, 16, 10, 16, 10, 16},
1961 {&amdfam10_cost, 32, 24, 32, 7, 32}
1964 static const char *const cpu_names[TARGET_CPU_DEFAULT_max] =
1989 /* Implement TARGET_HANDLE_OPTION. */
1992 ix86_handle_option (size_t code, const char *arg ATTRIBUTE_UNUSED, int value)
1999 ix86_isa_flags |= OPTION_MASK_ISA_MMX_SET;
2000 ix86_isa_flags_explicit |= OPTION_MASK_ISA_MMX_SET;
2004 ix86_isa_flags &= ~OPTION_MASK_ISA_MMX_UNSET;
2005 ix86_isa_flags_explicit |= OPTION_MASK_ISA_MMX_UNSET;
2012 ix86_isa_flags |= OPTION_MASK_ISA_3DNOW_SET;
2013 ix86_isa_flags_explicit |= OPTION_MASK_ISA_3DNOW_SET;
2017 ix86_isa_flags &= ~OPTION_MASK_ISA_3DNOW_UNSET;
2018 ix86_isa_flags_explicit |= OPTION_MASK_ISA_3DNOW_UNSET;
2028 ix86_isa_flags |= OPTION_MASK_ISA_SSE_SET;
2029 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE_SET;
2033 ix86_isa_flags &= ~OPTION_MASK_ISA_SSE_UNSET;
2034 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE_UNSET;
2041 ix86_isa_flags |= OPTION_MASK_ISA_SSE2_SET;
2042 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE2_SET;
2046 ix86_isa_flags &= ~OPTION_MASK_ISA_SSE2_UNSET;
2047 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE2_UNSET;
2054 ix86_isa_flags |= OPTION_MASK_ISA_SSE3_SET;
2055 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE3_SET;
2059 ix86_isa_flags &= ~OPTION_MASK_ISA_SSE3_UNSET;
2060 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE3_UNSET;
2067 ix86_isa_flags |= OPTION_MASK_ISA_SSSE3_SET;
2068 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSSE3_SET;
2072 ix86_isa_flags &= ~OPTION_MASK_ISA_SSSE3_UNSET;
2073 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSSE3_UNSET;
2080 ix86_isa_flags |= OPTION_MASK_ISA_SSE4_1_SET;
2081 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE4_1_SET;
2085 ix86_isa_flags &= ~OPTION_MASK_ISA_SSE4_1_UNSET;
2086 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE4_1_UNSET;
2093 ix86_isa_flags |= OPTION_MASK_ISA_SSE4_2_SET;
2094 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE4_2_SET;
2098 ix86_isa_flags &= ~OPTION_MASK_ISA_SSE4_2_UNSET;
2099 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE4_2_UNSET;
2106 ix86_isa_flags |= OPTION_MASK_ISA_AVX_SET;
2107 ix86_isa_flags_explicit |= OPTION_MASK_ISA_AVX_SET;
2111 ix86_isa_flags &= ~OPTION_MASK_ISA_AVX_UNSET;
2112 ix86_isa_flags_explicit |= OPTION_MASK_ISA_AVX_UNSET;
2119 ix86_isa_flags |= OPTION_MASK_ISA_FMA_SET;
2120 ix86_isa_flags_explicit |= OPTION_MASK_ISA_FMA_SET;
2124 ix86_isa_flags &= ~OPTION_MASK_ISA_FMA_UNSET;
2125 ix86_isa_flags_explicit |= OPTION_MASK_ISA_FMA_UNSET;
2130 ix86_isa_flags |= OPTION_MASK_ISA_SSE4_SET;
2131 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE4_SET;
2135 ix86_isa_flags &= ~OPTION_MASK_ISA_SSE4_UNSET;
2136 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE4_UNSET;
2142 ix86_isa_flags |= OPTION_MASK_ISA_SSE4A_SET;
2143 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE4A_SET;
2147 ix86_isa_flags &= ~OPTION_MASK_ISA_SSE4A_UNSET;
2148 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE4A_UNSET;
2155 ix86_isa_flags |= OPTION_MASK_ISA_SSE5_SET;
2156 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE5_SET;
2160 ix86_isa_flags &= ~OPTION_MASK_ISA_SSE5_UNSET;
2161 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE5_UNSET;
2168 ix86_isa_flags |= OPTION_MASK_ISA_ABM_SET;
2169 ix86_isa_flags_explicit |= OPTION_MASK_ISA_ABM_SET;
2173 ix86_isa_flags &= ~OPTION_MASK_ISA_ABM_UNSET;
2174 ix86_isa_flags_explicit |= OPTION_MASK_ISA_ABM_UNSET;
2181 ix86_isa_flags |= OPTION_MASK_ISA_POPCNT_SET;
2182 ix86_isa_flags_explicit |= OPTION_MASK_ISA_POPCNT_SET;
2186 ix86_isa_flags &= ~OPTION_MASK_ISA_POPCNT_UNSET;
2187 ix86_isa_flags_explicit |= OPTION_MASK_ISA_POPCNT_UNSET;
2194 ix86_isa_flags |= OPTION_MASK_ISA_SAHF_SET;
2195 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SAHF_SET;
2199 ix86_isa_flags &= ~OPTION_MASK_ISA_SAHF_UNSET;
2200 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SAHF_UNSET;
2207 ix86_isa_flags |= OPTION_MASK_ISA_CX16_SET;
2208 ix86_isa_flags_explicit |= OPTION_MASK_ISA_CX16_SET;
2212 ix86_isa_flags &= ~OPTION_MASK_ISA_CX16_UNSET;
2213 ix86_isa_flags_explicit |= OPTION_MASK_ISA_CX16_UNSET;
2220 ix86_isa_flags |= OPTION_MASK_ISA_AES_SET;
2221 ix86_isa_flags_explicit |= OPTION_MASK_ISA_AES_SET;
2225 ix86_isa_flags &= ~OPTION_MASK_ISA_AES_UNSET;
2226 ix86_isa_flags_explicit |= OPTION_MASK_ISA_AES_UNSET;
2233 ix86_isa_flags |= OPTION_MASK_ISA_PCLMUL_SET;
2234 ix86_isa_flags_explicit |= OPTION_MASK_ISA_PCLMUL_SET;
2238 ix86_isa_flags &= ~OPTION_MASK_ISA_PCLMUL_UNSET;
2239 ix86_isa_flags_explicit |= OPTION_MASK_ISA_PCLMUL_UNSET;
2248 /* Return a string the documents the current -m options. The caller is
2249 responsible for freeing the string. */
2252 ix86_target_string (int isa, int flags, const char *arch, const char *tune,
2253 const char *fpmath, bool add_nl_p)
2255 struct ix86_target_opts
2257 const char *option; /* option string */
2258 int mask; /* isa mask options */
2261 /* This table is ordered so that options like -msse5 or -msse4.2 that imply
2262 preceding options while match those first. */
2263 static struct ix86_target_opts isa_opts[] =
2265 { "-m64", OPTION_MASK_ISA_64BIT },
2266 { "-msse5", OPTION_MASK_ISA_SSE5 },
2267 { "-msse4a", OPTION_MASK_ISA_SSE4A },
2268 { "-msse4.2", OPTION_MASK_ISA_SSE4_2 },
2269 { "-msse4.1", OPTION_MASK_ISA_SSE4_1 },
2270 { "-mssse3", OPTION_MASK_ISA_SSSE3 },
2271 { "-msse3", OPTION_MASK_ISA_SSE3 },
2272 { "-msse2", OPTION_MASK_ISA_SSE2 },
2273 { "-msse", OPTION_MASK_ISA_SSE },
2274 { "-m3dnow", OPTION_MASK_ISA_3DNOW },
2275 { "-m3dnowa", OPTION_MASK_ISA_3DNOW_A },
2276 { "-mmmx", OPTION_MASK_ISA_MMX },
2277 { "-mabm", OPTION_MASK_ISA_ABM },
2278 { "-mpopcnt", OPTION_MASK_ISA_POPCNT },
2279 { "-maes", OPTION_MASK_ISA_AES },
2280 { "-mpclmul", OPTION_MASK_ISA_PCLMUL },
2284 static struct ix86_target_opts flag_opts[] =
2286 { "-m128bit-long-double", MASK_128BIT_LONG_DOUBLE },
2287 { "-m80387", MASK_80387 },
2288 { "-maccumulate-outgoing-args", MASK_ACCUMULATE_OUTGOING_ARGS },
2289 { "-malign-double", MASK_ALIGN_DOUBLE },
2290 { "-mcld", MASK_CLD },
2291 { "-mfp-ret-in-387", MASK_FLOAT_RETURNS },
2292 { "-mieee-fp", MASK_IEEE_FP },
2293 { "-minline-all-stringops", MASK_INLINE_ALL_STRINGOPS },
2294 { "-minline-stringops-dynamically", MASK_INLINE_STRINGOPS_DYNAMICALLY },
2295 { "-mms-bitfields", MASK_MS_BITFIELD_LAYOUT },
2296 { "-mno-align-stringops", MASK_NO_ALIGN_STRINGOPS },
2297 { "-mno-fancy-math-387", MASK_NO_FANCY_MATH_387 },
2298 { "-mno-fused-madd", MASK_NO_FUSED_MADD },
2299 { "-mno-push-args", MASK_NO_PUSH_ARGS },
2300 { "-mno-red-zone", MASK_NO_RED_ZONE },
2301 { "-momit-leaf-frame-pointer", MASK_OMIT_LEAF_FRAME_POINTER },
2302 { "-mrecip", MASK_RECIP },
2303 { "-mrtd", MASK_RTD },
2304 { "-msseregparm", MASK_SSEREGPARM },
2305 { "-mstack-arg-probe", MASK_STACK_PROBE },
2306 { "-mtls-direct-seg-refs", MASK_TLS_DIRECT_SEG_REFS },
2309 const char *opts[ARRAY_SIZE (isa_opts) + ARRAY_SIZE (flag_opts) + 6][2];
2312 char target_other[40];
2321 memset (opts, '\0', sizeof (opts));
2323 /* Add -march= option. */
2326 opts[num][0] = "-march=";
2327 opts[num++][1] = arch;
2330 /* Add -mtune= option. */
2333 opts[num][0] = "-mtune=";
2334 opts[num++][1] = tune;
2337 /* Pick out the options in isa options. */
2338 for (i = 0; i < ARRAY_SIZE (isa_opts); i++)
2340 if ((isa & isa_opts[i].mask) != 0)
2342 opts[num++][0] = isa_opts[i].option;
2343 isa &= ~ isa_opts[i].mask;
2347 if (isa && add_nl_p)
2349 opts[num++][0] = isa_other;
2350 sprintf (isa_other, "(other isa: 0x%x)", isa);
2353 /* Add flag options. */
2354 for (i = 0; i < ARRAY_SIZE (flag_opts); i++)
2356 if ((flags & flag_opts[i].mask) != 0)
2358 opts[num++][0] = flag_opts[i].option;
2359 flags &= ~ flag_opts[i].mask;
2363 if (flags && add_nl_p)
2365 opts[num++][0] = target_other;
2366 sprintf (target_other, "(other flags: 0x%x)", isa);
2369 /* Add -fpmath= option. */
2372 opts[num][0] = "-mfpmath=";
2373 opts[num++][1] = fpmath;
2380 gcc_assert (num < ARRAY_SIZE (opts));
2382 /* Size the string. */
2384 sep_len = (add_nl_p) ? 3 : 1;
2385 for (i = 0; i < num; i++)
2388 for (j = 0; j < 2; j++)
2390 len += strlen (opts[i][j]);
2393 /* Build the string. */
2394 ret = ptr = (char *) xmalloc (len);
2397 for (i = 0; i < num; i++)
2401 for (j = 0; j < 2; j++)
2402 len2[j] = (opts[i][j]) ? strlen (opts[i][j]) : 0;
2409 if (add_nl_p && line_len + len2[0] + len2[1] > 70)
2417 for (j = 0; j < 2; j++)
2420 memcpy (ptr, opts[i][j], len2[j]);
2422 line_len += len2[j];
2427 gcc_assert (ret + len >= ptr);
2432 /* Function that is callable from the debugger to print the current
2435 ix86_debug_options (void)
2437 char *opts = ix86_target_string (ix86_isa_flags, target_flags,
2438 ix86_arch_string, ix86_tune_string,
2439 ix86_fpmath_string, true);
2443 fprintf (stderr, "%s\n\n", opts);
2447 fprintf (stderr, "<no options>\n\n");
2452 /* Sometimes certain combinations of command options do not make
2453 sense on a particular target machine. You can define a macro
2454 `OVERRIDE_OPTIONS' to take account of this. This macro, if
2455 defined, is executed once just after all the command options have
2458 Don't use this macro to turn on various extra optimizations for
2459 `-O'. That is what `OPTIMIZATION_OPTIONS' is for. */
2462 override_options (bool main_args_p)
2465 unsigned int ix86_arch_mask, ix86_tune_mask;
2470 /* Comes from final.c -- no real reason to change it. */
2471 #define MAX_CODE_ALIGN 16
2479 PTA_PREFETCH_SSE = 1 << 4,
2481 PTA_3DNOW_A = 1 << 6,
2485 PTA_POPCNT = 1 << 10,
2487 PTA_SSE4A = 1 << 12,
2488 PTA_NO_SAHF = 1 << 13,
2489 PTA_SSE4_1 = 1 << 14,
2490 PTA_SSE4_2 = 1 << 15,
2493 PTA_PCLMUL = 1 << 18,
2500 const char *const name; /* processor name or nickname. */
2501 const enum processor_type processor;
2502 const enum attr_cpu schedule;
2503 const unsigned /*enum pta_flags*/ flags;
2505 const processor_alias_table[] =
2507 {"i386", PROCESSOR_I386, CPU_NONE, 0},
2508 {"i486", PROCESSOR_I486, CPU_NONE, 0},
2509 {"i586", PROCESSOR_PENTIUM, CPU_PENTIUM, 0},
2510 {"pentium", PROCESSOR_PENTIUM, CPU_PENTIUM, 0},
2511 {"pentium-mmx", PROCESSOR_PENTIUM, CPU_PENTIUM, PTA_MMX},
2512 {"winchip-c6", PROCESSOR_I486, CPU_NONE, PTA_MMX},
2513 {"winchip2", PROCESSOR_I486, CPU_NONE, PTA_MMX | PTA_3DNOW},
2514 {"c3", PROCESSOR_I486, CPU_NONE, PTA_MMX | PTA_3DNOW},
2515 {"c3-2", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, PTA_MMX | PTA_SSE},
2516 {"i686", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, 0},
2517 {"pentiumpro", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, 0},
2518 {"pentium2", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, PTA_MMX},
2519 {"pentium3", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
2521 {"pentium3m", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
2523 {"pentium-m", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
2524 PTA_MMX | PTA_SSE | PTA_SSE2},
2525 {"pentium4", PROCESSOR_PENTIUM4, CPU_NONE,
2526 PTA_MMX |PTA_SSE | PTA_SSE2},
2527 {"pentium4m", PROCESSOR_PENTIUM4, CPU_NONE,
2528 PTA_MMX | PTA_SSE | PTA_SSE2},
2529 {"prescott", PROCESSOR_NOCONA, CPU_NONE,
2530 PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3},
2531 {"nocona", PROCESSOR_NOCONA, CPU_NONE,
2532 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
2533 | PTA_CX16 | PTA_NO_SAHF},
2534 {"core2", PROCESSOR_CORE2, CPU_CORE2,
2535 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
2536 | PTA_SSSE3 | PTA_CX16},
2537 {"geode", PROCESSOR_GEODE, CPU_GEODE,
2538 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A |PTA_PREFETCH_SSE},
2539 {"k6", PROCESSOR_K6, CPU_K6, PTA_MMX},
2540 {"k6-2", PROCESSOR_K6, CPU_K6, PTA_MMX | PTA_3DNOW},
2541 {"k6-3", PROCESSOR_K6, CPU_K6, PTA_MMX | PTA_3DNOW},
2542 {"athlon", PROCESSOR_ATHLON, CPU_ATHLON,
2543 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_PREFETCH_SSE},
2544 {"athlon-tbird", PROCESSOR_ATHLON, CPU_ATHLON,
2545 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_PREFETCH_SSE},
2546 {"athlon-4", PROCESSOR_ATHLON, CPU_ATHLON,
2547 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE},
2548 {"athlon-xp", PROCESSOR_ATHLON, CPU_ATHLON,
2549 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE},
2550 {"athlon-mp", PROCESSOR_ATHLON, CPU_ATHLON,
2551 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE},
2552 {"x86-64", PROCESSOR_K8, CPU_K8,
2553 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_NO_SAHF},
2554 {"k8", PROCESSOR_K8, CPU_K8,
2555 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
2556 | PTA_SSE2 | PTA_NO_SAHF},
2557 {"k8-sse3", PROCESSOR_K8, CPU_K8,
2558 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
2559 | PTA_SSE2 | PTA_SSE3 | PTA_NO_SAHF},
2560 {"opteron", PROCESSOR_K8, CPU_K8,
2561 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
2562 | PTA_SSE2 | PTA_NO_SAHF},
2563 {"opteron-sse3", PROCESSOR_K8, CPU_K8,
2564 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
2565 | PTA_SSE2 | PTA_SSE3 | PTA_NO_SAHF},
2566 {"athlon64", PROCESSOR_K8, CPU_K8,
2567 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
2568 | PTA_SSE2 | PTA_NO_SAHF},
2569 {"athlon64-sse3", PROCESSOR_K8, CPU_K8,
2570 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
2571 | PTA_SSE2 | PTA_SSE3 | PTA_NO_SAHF},
2572 {"athlon-fx", PROCESSOR_K8, CPU_K8,
2573 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
2574 | PTA_SSE2 | PTA_NO_SAHF},
2575 {"amdfam10", PROCESSOR_AMDFAM10, CPU_AMDFAM10,
2576 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
2577 | PTA_SSE2 | PTA_SSE3 | PTA_SSE4A | PTA_CX16 | PTA_ABM},
2578 {"barcelona", PROCESSOR_AMDFAM10, CPU_AMDFAM10,
2579 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
2580 | PTA_SSE2 | PTA_SSE3 | PTA_SSE4A | PTA_CX16 | PTA_ABM},
2581 {"generic32", PROCESSOR_GENERIC32, CPU_PENTIUMPRO,
2582 0 /* flags are only used for -march switch. */ },
2583 {"generic64", PROCESSOR_GENERIC64, CPU_GENERIC64,
2584 PTA_64BIT /* flags are only used for -march switch. */ },
2587 int const pta_size = ARRAY_SIZE (processor_alias_table);
2589 /* Set up prefix/suffix so the error messages refer to either the command
2590 line argument, or the attribute(target). */
2599 prefix = "option(\"";
2604 #ifdef SUBTARGET_OVERRIDE_OPTIONS
2605 SUBTARGET_OVERRIDE_OPTIONS;
2608 #ifdef SUBSUBTARGET_OVERRIDE_OPTIONS
2609 SUBSUBTARGET_OVERRIDE_OPTIONS;
2612 /* -fPIC is the default for x86_64. */
2613 if (TARGET_MACHO && TARGET_64BIT)
2616 /* Set the default values for switches whose default depends on TARGET_64BIT
2617 in case they weren't overwritten by command line options. */
2620 /* Mach-O doesn't support omitting the frame pointer for now. */
2621 if (flag_omit_frame_pointer == 2)
2622 flag_omit_frame_pointer = (TARGET_MACHO ? 0 : 1);
2623 if (flag_asynchronous_unwind_tables == 2)
2624 flag_asynchronous_unwind_tables = 1;
2625 if (flag_pcc_struct_return == 2)
2626 flag_pcc_struct_return = 0;
2630 if (flag_omit_frame_pointer == 2)
2631 flag_omit_frame_pointer = 0;
2632 if (flag_asynchronous_unwind_tables == 2)
2633 flag_asynchronous_unwind_tables = 0;
2634 if (flag_pcc_struct_return == 2)
2635 flag_pcc_struct_return = DEFAULT_PCC_STRUCT_RETURN;
2638 /* Need to check -mtune=generic first. */
2639 if (ix86_tune_string)
2641 if (!strcmp (ix86_tune_string, "generic")
2642 || !strcmp (ix86_tune_string, "i686")
2643 /* As special support for cross compilers we read -mtune=native
2644 as -mtune=generic. With native compilers we won't see the
2645 -mtune=native, as it was changed by the driver. */
2646 || !strcmp (ix86_tune_string, "native"))
2649 ix86_tune_string = "generic64";
2651 ix86_tune_string = "generic32";
2653 /* If this call is for setting the option attribute, allow the
2654 generic32/generic64 that was previously set. */
2655 else if (!main_args_p
2656 && (!strcmp (ix86_tune_string, "generic32")
2657 || !strcmp (ix86_tune_string, "generic64")))
2659 else if (!strncmp (ix86_tune_string, "generic", 7))
2660 error ("bad value (%s) for %stune=%s %s",
2661 ix86_tune_string, prefix, suffix, sw);
2665 if (ix86_arch_string)
2666 ix86_tune_string = ix86_arch_string;
2667 if (!ix86_tune_string)
2669 ix86_tune_string = cpu_names[TARGET_CPU_DEFAULT];
2670 ix86_tune_defaulted = 1;
2673 /* ix86_tune_string is set to ix86_arch_string or defaulted. We
2674 need to use a sensible tune option. */
2675 if (!strcmp (ix86_tune_string, "generic")
2676 || !strcmp (ix86_tune_string, "x86-64")
2677 || !strcmp (ix86_tune_string, "i686"))
2680 ix86_tune_string = "generic64";
2682 ix86_tune_string = "generic32";
2685 if (ix86_stringop_string)
2687 if (!strcmp (ix86_stringop_string, "rep_byte"))
2688 stringop_alg = rep_prefix_1_byte;
2689 else if (!strcmp (ix86_stringop_string, "libcall"))
2690 stringop_alg = libcall;
2691 else if (!strcmp (ix86_stringop_string, "rep_4byte"))
2692 stringop_alg = rep_prefix_4_byte;
2693 else if (!strcmp (ix86_stringop_string, "rep_8byte")
2695 /* rep; movq isn't available in 32-bit code. */
2696 stringop_alg = rep_prefix_8_byte;
2697 else if (!strcmp (ix86_stringop_string, "byte_loop"))
2698 stringop_alg = loop_1_byte;
2699 else if (!strcmp (ix86_stringop_string, "loop"))
2700 stringop_alg = loop;
2701 else if (!strcmp (ix86_stringop_string, "unrolled_loop"))
2702 stringop_alg = unrolled_loop;
2704 error ("bad value (%s) for %sstringop-strategy=%s %s",
2705 ix86_stringop_string, prefix, suffix, sw);
2707 if (!strcmp (ix86_tune_string, "x86-64"))
2708 warning (OPT_Wdeprecated, "%stune=x86-64%s is deprecated. Use "
2709 "%stune=k8%s or %stune=generic%s instead as appropriate.",
2710 prefix, suffix, prefix, suffix, prefix, suffix);
2712 if (!ix86_arch_string)
2713 ix86_arch_string = TARGET_64BIT ? "x86-64" : "i386";
2715 ix86_arch_specified = 1;
2717 if (!strcmp (ix86_arch_string, "generic"))
2718 error ("generic CPU can be used only for %stune=%s %s",
2719 prefix, suffix, sw);
2720 if (!strncmp (ix86_arch_string, "generic", 7))
2721 error ("bad value (%s) for %sarch=%s %s",
2722 ix86_arch_string, prefix, suffix, sw);
2724 if (ix86_cmodel_string != 0)
2726 if (!strcmp (ix86_cmodel_string, "small"))
2727 ix86_cmodel = flag_pic ? CM_SMALL_PIC : CM_SMALL;
2728 else if (!strcmp (ix86_cmodel_string, "medium"))
2729 ix86_cmodel = flag_pic ? CM_MEDIUM_PIC : CM_MEDIUM;
2730 else if (!strcmp (ix86_cmodel_string, "large"))
2731 ix86_cmodel = flag_pic ? CM_LARGE_PIC : CM_LARGE;
2733 error ("code model %s does not support PIC mode", ix86_cmodel_string);
2734 else if (!strcmp (ix86_cmodel_string, "32"))
2735 ix86_cmodel = CM_32;
2736 else if (!strcmp (ix86_cmodel_string, "kernel") && !flag_pic)
2737 ix86_cmodel = CM_KERNEL;
2739 error ("bad value (%s) for %scmodel=%s %s",
2740 ix86_cmodel_string, prefix, suffix, sw);
2744 /* For TARGET_64BIT and MS_ABI, force pic on, in order to enable the
2745 use of rip-relative addressing. This eliminates fixups that
2746 would otherwise be needed if this object is to be placed in a
2747 DLL, and is essentially just as efficient as direct addressing. */
2748 if (TARGET_64BIT && DEFAULT_ABI == MS_ABI)
2749 ix86_cmodel = CM_SMALL_PIC, flag_pic = 1;
2750 else if (TARGET_64BIT)
2751 ix86_cmodel = flag_pic ? CM_SMALL_PIC : CM_SMALL;
2753 ix86_cmodel = CM_32;
2755 if (ix86_asm_string != 0)
2758 && !strcmp (ix86_asm_string, "intel"))
2759 ix86_asm_dialect = ASM_INTEL;
2760 else if (!strcmp (ix86_asm_string, "att"))
2761 ix86_asm_dialect = ASM_ATT;
2763 error ("bad value (%s) for %sasm=%s %s",
2764 ix86_asm_string, prefix, suffix, sw);
2766 if ((TARGET_64BIT == 0) != (ix86_cmodel == CM_32))
2767 error ("code model %qs not supported in the %s bit mode",
2768 ix86_cmodel_string, TARGET_64BIT ? "64" : "32");
2769 if ((TARGET_64BIT != 0) != ((ix86_isa_flags & OPTION_MASK_ISA_64BIT) != 0))
2770 sorry ("%i-bit mode not compiled in",
2771 (ix86_isa_flags & OPTION_MASK_ISA_64BIT) ? 64 : 32);
2773 for (i = 0; i < pta_size; i++)
2774 if (! strcmp (ix86_arch_string, processor_alias_table[i].name))
2776 ix86_schedule = processor_alias_table[i].schedule;
2777 ix86_arch = processor_alias_table[i].processor;
2778 /* Default cpu tuning to the architecture. */
2779 ix86_tune = ix86_arch;
2781 if (TARGET_64BIT && !(processor_alias_table[i].flags & PTA_64BIT))
2782 error ("CPU you selected does not support x86-64 "
2785 if (processor_alias_table[i].flags & PTA_MMX
2786 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_MMX))
2787 ix86_isa_flags |= OPTION_MASK_ISA_MMX;
2788 if (processor_alias_table[i].flags & PTA_3DNOW
2789 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_3DNOW))
2790 ix86_isa_flags |= OPTION_MASK_ISA_3DNOW;
2791 if (processor_alias_table[i].flags & PTA_3DNOW_A
2792 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_3DNOW_A))
2793 ix86_isa_flags |= OPTION_MASK_ISA_3DNOW_A;
2794 if (processor_alias_table[i].flags & PTA_SSE
2795 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE))
2796 ix86_isa_flags |= OPTION_MASK_ISA_SSE;
2797 if (processor_alias_table[i].flags & PTA_SSE2
2798 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE2))
2799 ix86_isa_flags |= OPTION_MASK_ISA_SSE2;
2800 if (processor_alias_table[i].flags & PTA_SSE3
2801 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE3))
2802 ix86_isa_flags |= OPTION_MASK_ISA_SSE3;
2803 if (processor_alias_table[i].flags & PTA_SSSE3
2804 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSSE3))
2805 ix86_isa_flags |= OPTION_MASK_ISA_SSSE3;
2806 if (processor_alias_table[i].flags & PTA_SSE4_1
2807 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4_1))
2808 ix86_isa_flags |= OPTION_MASK_ISA_SSE4_1;
2809 if (processor_alias_table[i].flags & PTA_SSE4_2
2810 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4_2))
2811 ix86_isa_flags |= OPTION_MASK_ISA_SSE4_2;
2812 if (processor_alias_table[i].flags & PTA_AVX
2813 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX))
2814 ix86_isa_flags |= OPTION_MASK_ISA_AVX;
2815 if (processor_alias_table[i].flags & PTA_FMA
2816 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_FMA))
2817 ix86_isa_flags |= OPTION_MASK_ISA_FMA;
2818 if (processor_alias_table[i].flags & PTA_SSE4A
2819 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4A))
2820 ix86_isa_flags |= OPTION_MASK_ISA_SSE4A;
2821 if (processor_alias_table[i].flags & PTA_SSE5
2822 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE5))
2823 ix86_isa_flags |= OPTION_MASK_ISA_SSE5;
2824 if (processor_alias_table[i].flags & PTA_ABM
2825 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_ABM))
2826 ix86_isa_flags |= OPTION_MASK_ISA_ABM;
2827 if (processor_alias_table[i].flags & PTA_CX16
2828 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_CX16))
2829 ix86_isa_flags |= OPTION_MASK_ISA_CX16;
2830 if (processor_alias_table[i].flags & (PTA_POPCNT | PTA_ABM)
2831 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_POPCNT))
2832 ix86_isa_flags |= OPTION_MASK_ISA_POPCNT;
2833 if (!(TARGET_64BIT && (processor_alias_table[i].flags & PTA_NO_SAHF))
2834 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SAHF))
2835 ix86_isa_flags |= OPTION_MASK_ISA_SAHF;
2836 if (processor_alias_table[i].flags & PTA_AES
2837 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_AES))
2838 ix86_isa_flags |= OPTION_MASK_ISA_AES;
2839 if (processor_alias_table[i].flags & PTA_PCLMUL
2840 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_PCLMUL))
2841 ix86_isa_flags |= OPTION_MASK_ISA_PCLMUL;
2842 if (processor_alias_table[i].flags & (PTA_PREFETCH_SSE | PTA_SSE))
2843 x86_prefetch_sse = true;
2849 error ("bad value (%s) for %sarch=%s %s",
2850 ix86_arch_string, prefix, suffix, sw);
2852 ix86_arch_mask = 1u << ix86_arch;
2853 for (i = 0; i < X86_ARCH_LAST; ++i)
2854 ix86_arch_features[i] = !!(initial_ix86_arch_features[i] & ix86_arch_mask);
2856 for (i = 0; i < pta_size; i++)
2857 if (! strcmp (ix86_tune_string, processor_alias_table[i].name))
2859 ix86_schedule = processor_alias_table[i].schedule;
2860 ix86_tune = processor_alias_table[i].processor;
2861 if (TARGET_64BIT && !(processor_alias_table[i].flags & PTA_64BIT))
2863 if (ix86_tune_defaulted)
2865 ix86_tune_string = "x86-64";
2866 for (i = 0; i < pta_size; i++)
2867 if (! strcmp (ix86_tune_string,
2868 processor_alias_table[i].name))
2870 ix86_schedule = processor_alias_table[i].schedule;
2871 ix86_tune = processor_alias_table[i].processor;
2874 error ("CPU you selected does not support x86-64 "
2877 /* Intel CPUs have always interpreted SSE prefetch instructions as
2878 NOPs; so, we can enable SSE prefetch instructions even when
2879 -mtune (rather than -march) points us to a processor that has them.
2880 However, the VIA C3 gives a SIGILL, so we only do that for i686 and
2881 higher processors. */
2883 && (processor_alias_table[i].flags & (PTA_PREFETCH_SSE | PTA_SSE)))
2884 x86_prefetch_sse = true;
2888 error ("bad value (%s) for %stune=%s %s",
2889 ix86_tune_string, prefix, suffix, sw);
2891 ix86_tune_mask = 1u << ix86_tune;
2892 for (i = 0; i < X86_TUNE_LAST; ++i)
2893 ix86_tune_features[i] = !!(initial_ix86_tune_features[i] & ix86_tune_mask);
2896 ix86_cost = &ix86_size_cost;
2898 ix86_cost = processor_target_table[ix86_tune].cost;
2900 /* Arrange to set up i386_stack_locals for all functions. */
2901 init_machine_status = ix86_init_machine_status;
2903 /* Validate -mregparm= value. */
2904 if (ix86_regparm_string)
2907 warning (0, "%sregparm%s is ignored in 64-bit mode", prefix, suffix);
2908 i = atoi (ix86_regparm_string);
2909 if (i < 0 || i > REGPARM_MAX)
2910 error ("%sregparm=%d%s is not between 0 and %d",
2911 prefix, i, suffix, REGPARM_MAX);
2916 ix86_regparm = REGPARM_MAX;
2918 /* If the user has provided any of the -malign-* options,
2919 warn and use that value only if -falign-* is not set.
2920 Remove this code in GCC 3.2 or later. */
2921 if (ix86_align_loops_string)
2923 warning (0, "%salign-loops%s is obsolete, use %salign-loops%s",
2924 prefix, suffix, prefix, suffix);
2925 if (align_loops == 0)
2927 i = atoi (ix86_align_loops_string);
2928 if (i < 0 || i > MAX_CODE_ALIGN)
2929 error ("%salign-loops=%d%s is not between 0 and %d",
2930 prefix, i, suffix, MAX_CODE_ALIGN);
2932 align_loops = 1 << i;
2936 if (ix86_align_jumps_string)
2938 warning (0, "%salign-jumps%s is obsolete, use %salign-jumps%s",
2939 prefix, suffix, prefix, suffix);
2940 if (align_jumps == 0)
2942 i = atoi (ix86_align_jumps_string);
2943 if (i < 0 || i > MAX_CODE_ALIGN)
2944 error ("%salign-loops=%d%s is not between 0 and %d",
2945 prefix, i, suffix, MAX_CODE_ALIGN);
2947 align_jumps = 1 << i;
2951 if (ix86_align_funcs_string)
2953 warning (0, "%salign-functions%s is obsolete, use %salign-functions%s",
2954 prefix, suffix, prefix, suffix);
2955 if (align_functions == 0)
2957 i = atoi (ix86_align_funcs_string);
2958 if (i < 0 || i > MAX_CODE_ALIGN)
2959 error ("%salign-loops=%d%s is not between 0 and %d",
2960 prefix, i, suffix, MAX_CODE_ALIGN);
2962 align_functions = 1 << i;
2966 /* Default align_* from the processor table. */
2967 if (align_loops == 0)
2969 align_loops = processor_target_table[ix86_tune].align_loop;
2970 align_loops_max_skip = processor_target_table[ix86_tune].align_loop_max_skip;
2972 if (align_jumps == 0)
2974 align_jumps = processor_target_table[ix86_tune].align_jump;
2975 align_jumps_max_skip = processor_target_table[ix86_tune].align_jump_max_skip;
2977 if (align_functions == 0)
2979 align_functions = processor_target_table[ix86_tune].align_func;
2982 /* Validate -mbranch-cost= value, or provide default. */
2983 ix86_branch_cost = ix86_cost->branch_cost;
2984 if (ix86_branch_cost_string)
2986 i = atoi (ix86_branch_cost_string);
2988 error ("%sbranch-cost=%d%s is not between 0 and 5", prefix, i, suffix);
2990 ix86_branch_cost = i;
2992 if (ix86_section_threshold_string)
2994 i = atoi (ix86_section_threshold_string);
2996 error ("%slarge-data-threshold=%d%s is negative", prefix, i, suffix);
2998 ix86_section_threshold = i;
3001 if (ix86_tls_dialect_string)
3003 if (strcmp (ix86_tls_dialect_string, "gnu") == 0)
3004 ix86_tls_dialect = TLS_DIALECT_GNU;
3005 else if (strcmp (ix86_tls_dialect_string, "gnu2") == 0)
3006 ix86_tls_dialect = TLS_DIALECT_GNU2;
3007 else if (strcmp (ix86_tls_dialect_string, "sun") == 0)
3008 ix86_tls_dialect = TLS_DIALECT_SUN;
3010 error ("bad value (%s) for %stls-dialect=%s %s",
3011 ix86_tls_dialect_string, prefix, suffix, sw);
3014 if (ix87_precision_string)
3016 i = atoi (ix87_precision_string);
3017 if (i != 32 && i != 64 && i != 80)
3018 error ("pc%d is not valid precision setting (32, 64 or 80)", i);
3023 target_flags |= TARGET_SUBTARGET64_DEFAULT & ~target_flags_explicit;
3025 /* Enable by default the SSE and MMX builtins. Do allow the user to
3026 explicitly disable any of these. In particular, disabling SSE and
3027 MMX for kernel code is extremely useful. */
3028 if (!ix86_arch_specified)
3030 |= ((OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_MMX
3031 | TARGET_SUBTARGET64_ISA_DEFAULT) & ~ix86_isa_flags_explicit);
3034 warning (0, "%srtd%s is ignored in 64bit mode", prefix, suffix);
3038 target_flags |= TARGET_SUBTARGET32_DEFAULT & ~target_flags_explicit;
3040 if (!ix86_arch_specified)
3042 |= TARGET_SUBTARGET32_ISA_DEFAULT & ~ix86_isa_flags_explicit;
3044 /* i386 ABI does not specify red zone. It still makes sense to use it
3045 when programmer takes care to stack from being destroyed. */
3046 if (!(target_flags_explicit & MASK_NO_RED_ZONE))
3047 target_flags |= MASK_NO_RED_ZONE;
3050 /* Keep nonleaf frame pointers. */
3051 if (flag_omit_frame_pointer)
3052 target_flags &= ~MASK_OMIT_LEAF_FRAME_POINTER;
3053 else if (TARGET_OMIT_LEAF_FRAME_POINTER)
3054 flag_omit_frame_pointer = 1;
3056 /* If we're doing fast math, we don't care about comparison order
3057 wrt NaNs. This lets us use a shorter comparison sequence. */
3058 if (flag_finite_math_only)
3059 target_flags &= ~MASK_IEEE_FP;
3061 /* If the architecture always has an FPU, turn off NO_FANCY_MATH_387,
3062 since the insns won't need emulation. */
3063 if (x86_arch_always_fancy_math_387 & ix86_arch_mask)
3064 target_flags &= ~MASK_NO_FANCY_MATH_387;
3066 /* Likewise, if the target doesn't have a 387, or we've specified
3067 software floating point, don't use 387 inline intrinsics. */
3069 target_flags |= MASK_NO_FANCY_MATH_387;
3071 /* Turn on MMX builtins for -msse. */
3074 ix86_isa_flags |= OPTION_MASK_ISA_MMX & ~ix86_isa_flags_explicit;
3075 x86_prefetch_sse = true;
3078 /* Turn on popcnt instruction for -msse4.2 or -mabm. */
3079 if (TARGET_SSE4_2 || TARGET_ABM)
3080 ix86_isa_flags |= OPTION_MASK_ISA_POPCNT & ~ix86_isa_flags_explicit;
3082 /* Validate -mpreferred-stack-boundary= value or default it to
3083 PREFERRED_STACK_BOUNDARY_DEFAULT. */
3084 ix86_preferred_stack_boundary = PREFERRED_STACK_BOUNDARY_DEFAULT;
3085 if (ix86_preferred_stack_boundary_string)
3087 i = atoi (ix86_preferred_stack_boundary_string);
3088 if (i < (TARGET_64BIT ? 4 : 2) || i > 12)
3089 error ("%spreferred-stack-boundary=%d%s is not between %d and 12",
3090 prefix, i, suffix, TARGET_64BIT ? 4 : 2);
3092 ix86_preferred_stack_boundary = (1 << i) * BITS_PER_UNIT;
3095 /* Set the default value for -mstackrealign. */
3096 if (ix86_force_align_arg_pointer == -1)
3097 ix86_force_align_arg_pointer = STACK_REALIGN_DEFAULT;
3099 /* Validate -mincoming-stack-boundary= value or default it to
3100 MIN_STACK_BOUNDARY/PREFERRED_STACK_BOUNDARY. */
3101 if (ix86_force_align_arg_pointer)
3102 ix86_default_incoming_stack_boundary = MIN_STACK_BOUNDARY;
3104 ix86_default_incoming_stack_boundary = PREFERRED_STACK_BOUNDARY;
3105 ix86_incoming_stack_boundary = ix86_default_incoming_stack_boundary;
3106 if (ix86_incoming_stack_boundary_string)
3108 i = atoi (ix86_incoming_stack_boundary_string);
3109 if (i < (TARGET_64BIT ? 4 : 2) || i > 12)
3110 error ("-mincoming-stack-boundary=%d is not between %d and 12",
3111 i, TARGET_64BIT ? 4 : 2);
3114 ix86_user_incoming_stack_boundary = (1 << i) * BITS_PER_UNIT;
3115 ix86_incoming_stack_boundary
3116 = ix86_user_incoming_stack_boundary;
3120 /* Accept -msseregparm only if at least SSE support is enabled. */
3121 if (TARGET_SSEREGPARM
3123 error ("%ssseregparm%s used without SSE enabled", prefix, suffix);
3125 ix86_fpmath = TARGET_FPMATH_DEFAULT;
3126 if (ix86_fpmath_string != 0)
3128 if (! strcmp (ix86_fpmath_string, "387"))
3129 ix86_fpmath = FPMATH_387;
3130 else if (! strcmp (ix86_fpmath_string, "sse"))
3134 warning (0, "SSE instruction set disabled, using 387 arithmetics");
3135 ix86_fpmath = FPMATH_387;
3138 ix86_fpmath = FPMATH_SSE;
3140 else if (! strcmp (ix86_fpmath_string, "387,sse")
3141 || ! strcmp (ix86_fpmath_string, "387+sse")
3142 || ! strcmp (ix86_fpmath_string, "sse,387")
3143 || ! strcmp (ix86_fpmath_string, "sse+387")
3144 || ! strcmp (ix86_fpmath_string, "both"))
3148 warning (0, "SSE instruction set disabled, using 387 arithmetics");
3149 ix86_fpmath = FPMATH_387;
3151 else if (!TARGET_80387)
3153 warning (0, "387 instruction set disabled, using SSE arithmetics");
3154 ix86_fpmath = FPMATH_SSE;
3157 ix86_fpmath = (enum fpmath_unit) (FPMATH_SSE | FPMATH_387);
3160 error ("bad value (%s) for %sfpmath=%s %s",
3161 ix86_fpmath_string, prefix, suffix, sw);
3164 /* If the i387 is disabled, then do not return values in it. */
3166 target_flags &= ~MASK_FLOAT_RETURNS;
3168 /* Use external vectorized library in vectorizing intrinsics. */
3169 if (ix86_veclibabi_string)
3171 if (strcmp (ix86_veclibabi_string, "svml") == 0)
3172 ix86_veclib_handler = ix86_veclibabi_svml;
3173 else if (strcmp (ix86_veclibabi_string, "acml") == 0)
3174 ix86_veclib_handler = ix86_veclibabi_acml;
3176 error ("unknown vectorization library ABI type (%s) for "
3177 "%sveclibabi=%s %s", ix86_veclibabi_string,
3178 prefix, suffix, sw);
3181 if ((x86_accumulate_outgoing_args & ix86_tune_mask)
3182 && !(target_flags_explicit & MASK_ACCUMULATE_OUTGOING_ARGS)
3184 target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;
3186 /* ??? Unwind info is not correct around the CFG unless either a frame
3187 pointer is present or M_A_O_A is set. Fixing this requires rewriting
3188 unwind info generation to be aware of the CFG and propagating states
3190 if ((flag_unwind_tables || flag_asynchronous_unwind_tables
3191 || flag_exceptions || flag_non_call_exceptions)
3192 && flag_omit_frame_pointer
3193 && !(target_flags & MASK_ACCUMULATE_OUTGOING_ARGS))
3195 if (target_flags_explicit & MASK_ACCUMULATE_OUTGOING_ARGS)
3196 warning (0, "unwind tables currently require either a frame pointer "
3197 "or %saccumulate-outgoing-args%s for correctness",
3199 target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;
3202 /* If stack probes are required, the space used for large function
3203 arguments on the stack must also be probed, so enable
3204 -maccumulate-outgoing-args so this happens in the prologue. */
3205 if (TARGET_STACK_PROBE
3206 && !(target_flags & MASK_ACCUMULATE_OUTGOING_ARGS))
3208 if (target_flags_explicit & MASK_ACCUMULATE_OUTGOING_ARGS)
3209 warning (0, "stack probing requires %saccumulate-outgoing-args%s "
3210 "for correctness", prefix, suffix);
3211 target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;
3214 /* For sane SSE instruction set generation we need fcomi instruction.
3215 It is safe to enable all CMOVE instructions. */
3219 /* Figure out what ASM_GENERATE_INTERNAL_LABEL builds as a prefix. */
3222 ASM_GENERATE_INTERNAL_LABEL (internal_label_prefix, "LX", 0);
3223 p = strchr (internal_label_prefix, 'X');
3224 internal_label_prefix_len = p - internal_label_prefix;
3228 /* When scheduling description is not available, disable scheduler pass
3229 so it won't slow down the compilation and make x87 code slower. */
3230 if (!TARGET_SCHEDULE)
3231 flag_schedule_insns_after_reload = flag_schedule_insns = 0;
3233 if (!PARAM_SET_P (PARAM_SIMULTANEOUS_PREFETCHES))
3234 set_param_value ("simultaneous-prefetches",
3235 ix86_cost->simultaneous_prefetches);
3236 if (!PARAM_SET_P (PARAM_L1_CACHE_LINE_SIZE))
3237 set_param_value ("l1-cache-line-size", ix86_cost->prefetch_block);
3238 if (!PARAM_SET_P (PARAM_L1_CACHE_SIZE))
3239 set_param_value ("l1-cache-size", ix86_cost->l1_cache_size);
3240 if (!PARAM_SET_P (PARAM_L2_CACHE_SIZE))
3241 set_param_value ("l2-cache-size", ix86_cost->l2_cache_size);
3243 /* If using typedef char *va_list, signal that __builtin_va_start (&ap, 0)
3244 can be optimized to ap = __builtin_next_arg (0). */
3246 targetm.expand_builtin_va_start = NULL;
3250 ix86_gen_leave = gen_leave_rex64;
3251 ix86_gen_pop1 = gen_popdi1;
3252 ix86_gen_add3 = gen_adddi3;
3253 ix86_gen_sub3 = gen_subdi3;
3254 ix86_gen_sub3_carry = gen_subdi3_carry_rex64;
3255 ix86_gen_one_cmpl2 = gen_one_cmpldi2;
3256 ix86_gen_monitor = gen_sse3_monitor64;
3257 ix86_gen_andsp = gen_anddi3;
3261 ix86_gen_leave = gen_leave;
3262 ix86_gen_pop1 = gen_popsi1;
3263 ix86_gen_add3 = gen_addsi3;
3264 ix86_gen_sub3 = gen_subsi3;
3265 ix86_gen_sub3_carry = gen_subsi3_carry;
3266 ix86_gen_one_cmpl2 = gen_one_cmplsi2;
3267 ix86_gen_monitor = gen_sse3_monitor;
3268 ix86_gen_andsp = gen_andsi3;
3272 /* Use -mcld by default for 32-bit code if configured with --enable-cld. */
3274 target_flags |= MASK_CLD & ~target_flags_explicit;
3277 /* Save the initial options in case the user does function specific options */
3279 target_option_default_node = target_option_current_node
3280 = build_target_option_node ();
3283 /* Save the current options */
3286 ix86_function_specific_save (struct cl_target_option *ptr)
3288 gcc_assert (IN_RANGE (ix86_arch, 0, 255));
3289 gcc_assert (IN_RANGE (ix86_schedule, 0, 255));
3290 gcc_assert (IN_RANGE (ix86_tune, 0, 255));
3291 gcc_assert (IN_RANGE (ix86_fpmath, 0, 255));
3292 gcc_assert (IN_RANGE (ix86_branch_cost, 0, 255));
3294 ptr->arch = ix86_arch;
3295 ptr->schedule = ix86_schedule;
3296 ptr->tune = ix86_tune;
3297 ptr->fpmath = ix86_fpmath;
3298 ptr->branch_cost = ix86_branch_cost;
3299 ptr->tune_defaulted = ix86_tune_defaulted;
3300 ptr->arch_specified = ix86_arch_specified;
3301 ptr->ix86_isa_flags_explicit = ix86_isa_flags_explicit;
3302 ptr->target_flags_explicit = target_flags_explicit;
3305 /* Restore the current options */
3308 ix86_function_specific_restore (struct cl_target_option *ptr)
3310 enum processor_type old_tune = ix86_tune;
3311 enum processor_type old_arch = ix86_arch;
3312 unsigned int ix86_arch_mask, ix86_tune_mask;
3315 ix86_arch = ptr->arch;
3316 ix86_schedule = ptr->schedule;
3317 ix86_tune = ptr->tune;
3318 ix86_fpmath = ptr->fpmath;
3319 ix86_branch_cost = ptr->branch_cost;
3320 ix86_tune_defaulted = ptr->tune_defaulted;
3321 ix86_arch_specified = ptr->arch_specified;
3322 ix86_isa_flags_explicit = ptr->ix86_isa_flags_explicit;
3323 target_flags_explicit = ptr->target_flags_explicit;
3325 /* Recreate the arch feature tests if the arch changed */
3326 if (old_arch != ix86_arch)
3328 ix86_arch_mask = 1u << ix86_arch;
3329 for (i = 0; i < X86_ARCH_LAST; ++i)
3330 ix86_arch_features[i]
3331 = !!(initial_ix86_arch_features[i] & ix86_arch_mask);
3334 /* Recreate the tune optimization tests */
3335 if (old_tune != ix86_tune)
3337 ix86_tune_mask = 1u << ix86_tune;
3338 for (i = 0; i < X86_TUNE_LAST; ++i)
3339 ix86_tune_features[i]
3340 = !!(initial_ix86_tune_features[i] & ix86_tune_mask);
3344 /* Print the current options */
3347 ix86_function_specific_print (FILE *file, int indent,
3348 struct cl_target_option *ptr)
3351 = ix86_target_string (ptr->ix86_isa_flags, ptr->target_flags,
3352 NULL, NULL, NULL, false);
3354 fprintf (file, "%*sarch = %d (%s)\n",
3357 ((ptr->arch < TARGET_CPU_DEFAULT_max)
3358 ? cpu_names[ptr->arch]
3361 fprintf (file, "%*stune = %d (%s)\n",
3364 ((ptr->tune < TARGET_CPU_DEFAULT_max)
3365 ? cpu_names[ptr->tune]
3368 fprintf (file, "%*sfpmath = %d%s%s\n", indent, "", ptr->fpmath,
3369 (ptr->fpmath & FPMATH_387) ? ", 387" : "",
3370 (ptr->fpmath & FPMATH_SSE) ? ", sse" : "");
3371 fprintf (file, "%*sbranch_cost = %d\n", indent, "", ptr->branch_cost);
3375 fprintf (file, "%*s%s\n", indent, "", target_string);
3376 free (target_string);
3381 /* Inner function to process the attribute((target(...))), take an argument and
3382 set the current options from the argument. If we have a list, recursively go
3386 ix86_valid_target_attribute_inner_p (tree args, char *p_strings[])
3391 #define IX86_ATTR_ISA(S,O) { S, sizeof (S)-1, ix86_opt_isa, O, 0 }
3392 #define IX86_ATTR_STR(S,O) { S, sizeof (S)-1, ix86_opt_str, O, 0 }
3393 #define IX86_ATTR_YES(S,O,M) { S, sizeof (S)-1, ix86_opt_yes, O, M }
3394 #define IX86_ATTR_NO(S,O,M) { S, sizeof (S)-1, ix86_opt_no, O, M }
3409 enum ix86_opt_type type;
3414 IX86_ATTR_ISA ("3dnow", OPT_m3dnow),
3415 IX86_ATTR_ISA ("abm", OPT_mabm),
3416 IX86_ATTR_ISA ("aes", OPT_maes),
3417 IX86_ATTR_ISA ("avx", OPT_mavx),
3418 IX86_ATTR_ISA ("mmx", OPT_mmmx),
3419 IX86_ATTR_ISA ("pclmul", OPT_mpclmul),
3420 IX86_ATTR_ISA ("popcnt", OPT_mpopcnt),
3421 IX86_ATTR_ISA ("sse", OPT_msse),
3422 IX86_ATTR_ISA ("sse2", OPT_msse2),
3423 IX86_ATTR_ISA ("sse3", OPT_msse3),
3424 IX86_ATTR_ISA ("sse4", OPT_msse4),
3425 IX86_ATTR_ISA ("sse4.1", OPT_msse4_1),
3426 IX86_ATTR_ISA ("sse4.2", OPT_msse4_2),
3427 IX86_ATTR_ISA ("sse4a", OPT_msse4a),
3428 IX86_ATTR_ISA ("sse5", OPT_msse5),
3429 IX86_ATTR_ISA ("ssse3", OPT_mssse3),
3431 /* string options */
3432 IX86_ATTR_STR ("arch=", IX86_FUNCTION_SPECIFIC_ARCH),
3433 IX86_ATTR_STR ("fpmath=", IX86_FUNCTION_SPECIFIC_FPMATH),
3434 IX86_ATTR_STR ("tune=", IX86_FUNCTION_SPECIFIC_TUNE),
3437 IX86_ATTR_YES ("cld",
3441 IX86_ATTR_NO ("fancy-math-387",
3442 OPT_mfancy_math_387,
3443 MASK_NO_FANCY_MATH_387),
3445 IX86_ATTR_NO ("fused-madd",
3447 MASK_NO_FUSED_MADD),
3449 IX86_ATTR_YES ("ieee-fp",
3453 IX86_ATTR_YES ("inline-all-stringops",
3454 OPT_minline_all_stringops,
3455 MASK_INLINE_ALL_STRINGOPS),
3457 IX86_ATTR_YES ("inline-stringops-dynamically",
3458 OPT_minline_stringops_dynamically,
3459 MASK_INLINE_STRINGOPS_DYNAMICALLY),
3461 IX86_ATTR_NO ("align-stringops",
3462 OPT_mno_align_stringops,
3463 MASK_NO_ALIGN_STRINGOPS),
3465 IX86_ATTR_YES ("recip",
3471 /* If this is a list, recurse to get the options. */
3472 if (TREE_CODE (args) == TREE_LIST)
3476 for (; args; args = TREE_CHAIN (args))
3477 if (TREE_VALUE (args)
3478 && !ix86_valid_target_attribute_inner_p (TREE_VALUE (args), p_strings))
3484 else if (TREE_CODE (args) != STRING_CST)
3487 /* Handle multiple arguments separated by commas. */
3488 next_optstr = ASTRDUP (TREE_STRING_POINTER (args));
3490 while (next_optstr && *next_optstr != '\0')
3492 char *p = next_optstr;
3494 char *comma = strchr (next_optstr, ',');
3495 const char *opt_string;
3496 size_t len, opt_len;
3501 enum ix86_opt_type type = ix86_opt_unknown;
3507 len = comma - next_optstr;
3508 next_optstr = comma + 1;
3516 /* Recognize no-xxx. */
3517 if (len > 3 && p[0] == 'n' && p[1] == 'o' && p[2] == '-')
3526 /* Find the option. */
3529 for (i = 0; i < ARRAY_SIZE (attrs); i++)
3531 type = attrs[i].type;
3532 opt_len = attrs[i].len;
3533 if (ch == attrs[i].string[0]
3534 && ((type != ix86_opt_str) ? len == opt_len : len > opt_len)
3535 && memcmp (p, attrs[i].string, opt_len) == 0)
3538 mask = attrs[i].mask;
3539 opt_string = attrs[i].string;
3544 /* Process the option. */
3547 error ("attribute(target(\"%s\")) is unknown", orig_p);
3551 else if (type == ix86_opt_isa)
3552 ix86_handle_option (opt, p, opt_set_p);
3554 else if (type == ix86_opt_yes || type == ix86_opt_no)
3556 if (type == ix86_opt_no)
3557 opt_set_p = !opt_set_p;
3560 target_flags |= mask;
3562 target_flags &= ~mask;
3565 else if (type == ix86_opt_str)
3569 error ("option(\"%s\") was already specified", opt_string);
3573 p_strings[opt] = xstrdup (p + opt_len);
3583 /* Return a TARGET_OPTION_NODE tree of the target options listed or NULL. */
3586 ix86_valid_target_attribute_tree (tree args)
3588 const char *orig_arch_string = ix86_arch_string;
3589 const char *orig_tune_string = ix86_tune_string;
3590 const char *orig_fpmath_string = ix86_fpmath_string;
3591 int orig_tune_defaulted = ix86_tune_defaulted;
3592 int orig_arch_specified = ix86_arch_specified;
3593 char *option_strings[IX86_FUNCTION_SPECIFIC_MAX] = { NULL, NULL, NULL };
3596 struct cl_target_option *def
3597 = TREE_TARGET_OPTION (target_option_default_node);
3599 /* Process each of the options on the chain. */
3600 if (! ix86_valid_target_attribute_inner_p (args, option_strings))
3603 /* If the changed options are different from the default, rerun override_options,
3604 and then save the options away. The string options are are attribute options,
3605 and will be undone when we copy the save structure. */
3606 if (ix86_isa_flags != def->ix86_isa_flags
3607 || target_flags != def->target_flags
3608 || option_strings[IX86_FUNCTION_SPECIFIC_ARCH]
3609 || option_strings[IX86_FUNCTION_SPECIFIC_TUNE]
3610 || option_strings[IX86_FUNCTION_SPECIFIC_FPMATH])
3612 /* If we are using the default tune= or arch=, undo the string assigned,
3613 and use the default. */
3614 if (option_strings[IX86_FUNCTION_SPECIFIC_ARCH])
3615 ix86_arch_string = option_strings[IX86_FUNCTION_SPECIFIC_ARCH];
3616 else if (!orig_arch_specified)
3617 ix86_arch_string = NULL;
3619 if (option_strings[IX86_FUNCTION_SPECIFIC_TUNE])
3620 ix86_tune_string = option_strings[IX86_FUNCTION_SPECIFIC_TUNE];
3621 else if (orig_tune_defaulted)
3622 ix86_tune_string = NULL;
3624 /* If fpmath= is not set, and we now have sse2 on 32-bit, use it. */
3625 if (option_strings[IX86_FUNCTION_SPECIFIC_FPMATH])
3626 ix86_fpmath_string = option_strings[IX86_FUNCTION_SPECIFIC_FPMATH];
3627 else if (!TARGET_64BIT && TARGET_SSE)
3628 ix86_fpmath_string = "sse,387";
3630 /* Do any overrides, such as arch=xxx, or tune=xxx support. */
3631 override_options (false);
3633 /* Add any builtin functions with the new isa if any. */
3634 ix86_add_new_builtins (ix86_isa_flags);
3636 /* Save the current options unless we are validating options for
3638 t = build_target_option_node ();
3640 ix86_arch_string = orig_arch_string;
3641 ix86_tune_string = orig_tune_string;
3642 ix86_fpmath_string = orig_fpmath_string;
3644 /* Free up memory allocated to hold the strings */
3645 for (i = 0; i < IX86_FUNCTION_SPECIFIC_MAX; i++)
3646 if (option_strings[i])
3647 free (option_strings[i]);
3653 /* Hook to validate attribute((target("string"))). */
3656 ix86_valid_target_attribute_p (tree fndecl,
3657 tree ARG_UNUSED (name),
3659 int ARG_UNUSED (flags))
3661 struct cl_target_option cur_target;
3663 tree old_optimize = build_optimization_node ();
3664 tree new_target, new_optimize;
3665 tree func_optimize = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (fndecl);
3667 /* If the function changed the optimization levels as well as setting target
3668 options, start with the optimizations specified. */
3669 if (func_optimize && func_optimize != old_optimize)
3670 cl_optimization_restore (TREE_OPTIMIZATION (func_optimize));
3672 /* The target attributes may also change some optimization flags, so update
3673 the optimization options if necessary. */
3674 cl_target_option_save (&cur_target);
3675 new_target = ix86_valid_target_attribute_tree (args);
3676 new_optimize = build_optimization_node ();
3683 DECL_FUNCTION_SPECIFIC_TARGET (fndecl) = new_target;
3685 if (old_optimize != new_optimize)
3686 DECL_FUNCTION_SPECIFIC_OPTIMIZATION (fndecl) = new_optimize;
3689 cl_target_option_restore (&cur_target);
3691 if (old_optimize != new_optimize)
3692 cl_optimization_restore (TREE_OPTIMIZATION (old_optimize));
3698 /* Hook to determine if one function can safely inline another. */
3701 ix86_can_inline_p (tree caller, tree callee)
3704 tree caller_tree = DECL_FUNCTION_SPECIFIC_TARGET (caller);
3705 tree callee_tree = DECL_FUNCTION_SPECIFIC_TARGET (callee);
3707 /* If callee has no option attributes, then it is ok to inline. */
3711 /* If caller has no option attributes, but callee does then it is not ok to
3713 else if (!caller_tree)
3718 struct cl_target_option *caller_opts = TREE_TARGET_OPTION (caller_tree);
3719 struct cl_target_option *callee_opts = TREE_TARGET_OPTION (callee_tree);
3721 /* Callee's isa options should a subset of the caller's, i.e. a SSE5 function
3722 can inline a SSE2 function but a SSE2 function can't inline a SSE5
3724 if ((caller_opts->ix86_isa_flags & callee_opts->ix86_isa_flags)
3725 != callee_opts->ix86_isa_flags)
3728 /* See if we have the same non-isa options. */
3729 else if (caller_opts->target_flags != callee_opts->target_flags)
3732 /* See if arch, tune, etc. are the same. */
3733 else if (caller_opts->arch != callee_opts->arch)
3736 else if (caller_opts->tune != callee_opts->tune)
3739 else if (caller_opts->fpmath != callee_opts->fpmath)
3742 else if (caller_opts->branch_cost != callee_opts->branch_cost)
3753 /* Remember the last target of ix86_set_current_function. */
3754 static GTY(()) tree ix86_previous_fndecl;
3756 /* Establish appropriate back-end context for processing the function
3757 FNDECL. The argument might be NULL to indicate processing at top
3758 level, outside of any function scope. */
3760 ix86_set_current_function (tree fndecl)
3762 /* Only change the context if the function changes. This hook is called
3763 several times in the course of compiling a function, and we don't want to
3764 slow things down too much or call target_reinit when it isn't safe. */
3765 if (fndecl && fndecl != ix86_previous_fndecl)
3767 tree old_tree = (ix86_previous_fndecl
3768 ? DECL_FUNCTION_SPECIFIC_TARGET (ix86_previous_fndecl)
3771 tree new_tree = (fndecl
3772 ? DECL_FUNCTION_SPECIFIC_TARGET (fndecl)
3775 ix86_previous_fndecl = fndecl;
3776 if (old_tree == new_tree)
3781 cl_target_option_restore (TREE_TARGET_OPTION (new_tree));
3787 struct cl_target_option *def
3788 = TREE_TARGET_OPTION (target_option_current_node);
3790 cl_target_option_restore (def);
3797 /* Return true if this goes in large data/bss. */
3800 ix86_in_large_data_p (tree exp)
3802 if (ix86_cmodel != CM_MEDIUM && ix86_cmodel != CM_MEDIUM_PIC)
3805 /* Functions are never large data. */
3806 if (TREE_CODE (exp) == FUNCTION_DECL)
3809 if (TREE_CODE (exp) == VAR_DECL && DECL_SECTION_NAME (exp))
3811 const char *section = TREE_STRING_POINTER (DECL_SECTION_NAME (exp));
3812 if (strcmp (section, ".ldata") == 0
3813 || strcmp (section, ".lbss") == 0)
3819 HOST_WIDE_INT size = int_size_in_bytes (TREE_TYPE (exp));
3821 /* If this is an incomplete type with size 0, then we can't put it
3822 in data because it might be too big when completed. */
3823 if (!size || size > ix86_section_threshold)
3830 /* Switch to the appropriate section for output of DECL.
3831 DECL is either a `VAR_DECL' node or a constant of some sort.
3832 RELOC indicates whether forming the initial value of DECL requires
3833 link-time relocations. */
3835 static section * x86_64_elf_select_section (tree, int, unsigned HOST_WIDE_INT)
3839 x86_64_elf_select_section (tree decl, int reloc,
3840 unsigned HOST_WIDE_INT align)
3842 if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC)
3843 && ix86_in_large_data_p (decl))
3845 const char *sname = NULL;
3846 unsigned int flags = SECTION_WRITE;
3847 switch (categorize_decl_for_section (decl, reloc))
3852 case SECCAT_DATA_REL:
3853 sname = ".ldata.rel";
3855 case SECCAT_DATA_REL_LOCAL:
3856 sname = ".ldata.rel.local";
3858 case SECCAT_DATA_REL_RO:
3859 sname = ".ldata.rel.ro";
3861 case SECCAT_DATA_REL_RO_LOCAL:
3862 sname = ".ldata.rel.ro.local";
3866 flags |= SECTION_BSS;
3869 case SECCAT_RODATA_MERGE_STR:
3870 case SECCAT_RODATA_MERGE_STR_INIT:
3871 case SECCAT_RODATA_MERGE_CONST:
3875 case SECCAT_SRODATA:
3882 /* We don't split these for medium model. Place them into
3883 default sections and hope for best. */
3885 case SECCAT_EMUTLS_VAR:
3886 case SECCAT_EMUTLS_TMPL:
3891 /* We might get called with string constants, but get_named_section
3892 doesn't like them as they are not DECLs. Also, we need to set
3893 flags in that case. */
3895 return get_section (sname, flags, NULL);
3896 return get_named_section (decl, sname, reloc);
3899 return default_elf_select_section (decl, reloc, align);
3902 /* Build up a unique section name, expressed as a
3903 STRING_CST node, and assign it to DECL_SECTION_NAME (decl).
3904 RELOC indicates whether the initial value of EXP requires
3905 link-time relocations. */
3907 static void ATTRIBUTE_UNUSED
3908 x86_64_elf_unique_section (tree decl, int reloc)
3910 if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC)
3911 && ix86_in_large_data_p (decl))
3913 const char *prefix = NULL;
3914 /* We only need to use .gnu.linkonce if we don't have COMDAT groups. */
3915 bool one_only = DECL_ONE_ONLY (decl) && !HAVE_COMDAT_GROUP;
3917 switch (categorize_decl_for_section (decl, reloc))
3920 case SECCAT_DATA_REL:
3921 case SECCAT_DATA_REL_LOCAL:
3922 case SECCAT_DATA_REL_RO:
3923 case SECCAT_DATA_REL_RO_LOCAL:
3924 prefix = one_only ? ".ld" : ".ldata";
3927 prefix = one_only ? ".lb" : ".lbss";
3930 case SECCAT_RODATA_MERGE_STR:
3931 case SECCAT_RODATA_MERGE_STR_INIT:
3932 case SECCAT_RODATA_MERGE_CONST:
3933 prefix = one_only ? ".lr" : ".lrodata";
3935 case SECCAT_SRODATA:
3942 /* We don't split these for medium model. Place them into
3943 default sections and hope for best. */
3945 case SECCAT_EMUTLS_VAR:
3946 prefix = targetm.emutls.var_section;
3948 case SECCAT_EMUTLS_TMPL:
3949 prefix = targetm.emutls.tmpl_section;
3954 const char *name, *linkonce;
3957 name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
3958 name = targetm.strip_name_encoding (name);
3960 /* If we're using one_only, then there needs to be a .gnu.linkonce
3961 prefix to the section name. */
3962 linkonce = one_only ? ".gnu.linkonce" : "";
3964 string = ACONCAT ((linkonce, prefix, ".", name, NULL));
3966 DECL_SECTION_NAME (decl) = build_string (strlen (string), string);
3970 default_unique_section (decl, reloc);
3973 #ifdef COMMON_ASM_OP
3974 /* This says how to output assembler code to declare an
3975 uninitialized external linkage data object.
3977 For medium model x86-64 we need to use .largecomm opcode for
3980 x86_elf_aligned_common (FILE *file,
3981 const char *name, unsigned HOST_WIDE_INT size,
3984 if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC)
3985 && size > (unsigned int)ix86_section_threshold)
3986 fprintf (file, ".largecomm\t");
3988 fprintf (file, "%s", COMMON_ASM_OP);
3989 assemble_name (file, name);
3990 fprintf (file, ","HOST_WIDE_INT_PRINT_UNSIGNED",%u\n",
3991 size, align / BITS_PER_UNIT);
3995 /* Utility function for targets to use in implementing
3996 ASM_OUTPUT_ALIGNED_BSS. */
3999 x86_output_aligned_bss (FILE *file, tree decl ATTRIBUTE_UNUSED,
4000 const char *name, unsigned HOST_WIDE_INT size,
4003 if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC)
4004 && size > (unsigned int)ix86_section_threshold)
4005 switch_to_section (get_named_section (decl, ".lbss", 0));
4007 switch_to_section (bss_section);
4008 ASM_OUTPUT_ALIGN (file, floor_log2 (align / BITS_PER_UNIT));
4009 #ifdef ASM_DECLARE_OBJECT_NAME
4010 last_assemble_variable_decl = decl;
4011 ASM_DECLARE_OBJECT_NAME (file, name, decl);
4013 /* Standard thing is just output label for the object. */
4014 ASM_OUTPUT_LABEL (file, name);
4015 #endif /* ASM_DECLARE_OBJECT_NAME */
4016 ASM_OUTPUT_SKIP (file, size ? size : 1);
4020 optimization_options (int level, int size ATTRIBUTE_UNUSED)
4022 /* For -O2 and beyond, turn off -fschedule-insns by default. It tends to
4023 make the problem with not enough registers even worse. */
4024 #ifdef INSN_SCHEDULING
4026 flag_schedule_insns = 0;
4030 /* The Darwin libraries never set errno, so we might as well
4031 avoid calling them when that's the only reason we would. */
4032 flag_errno_math = 0;
4034 /* The default values of these switches depend on the TARGET_64BIT
4035 that is not known at this moment. Mark these values with 2 and
4036 let user the to override these. In case there is no command line option
4037 specifying them, we will set the defaults in override_options. */
4039 flag_omit_frame_pointer = 2;
4040 flag_pcc_struct_return = 2;
4041 flag_asynchronous_unwind_tables = 2;
4042 flag_vect_cost_model = 1;
4043 #ifdef SUBTARGET_OPTIMIZATION_OPTIONS
4044 SUBTARGET_OPTIMIZATION_OPTIONS;
4048 /* Decide whether we can make a sibling call to a function. DECL is the
4049 declaration of the function being targeted by the call and EXP is the
4050 CALL_EXPR representing the call. */
4053 ix86_function_ok_for_sibcall (tree decl, tree exp)
4058 /* If we are generating position-independent code, we cannot sibcall
4059 optimize any indirect call, or a direct call to a global function,
4060 as the PLT requires %ebx be live. */
4061 if (!TARGET_64BIT && flag_pic && (!decl || !targetm.binds_local_p (decl)))
4068 func = TREE_TYPE (CALL_EXPR_FN (exp));
4069 if (POINTER_TYPE_P (func))
4070 func = TREE_TYPE (func);
4073 /* Check that the return value locations are the same. Like
4074 if we are returning floats on the 80387 register stack, we cannot
4075 make a sibcall from a function that doesn't return a float to a
4076 function that does or, conversely, from a function that does return
4077 a float to a function that doesn't; the necessary stack adjustment
4078 would not be executed. This is also the place we notice
4079 differences in the return value ABI. Note that it is ok for one
4080 of the functions to have void return type as long as the return
4081 value of the other is passed in a register. */
4082 a = ix86_function_value (TREE_TYPE (exp), func, false);
4083 b = ix86_function_value (TREE_TYPE (DECL_RESULT (cfun->decl)),
4085 if (STACK_REG_P (a) || STACK_REG_P (b))
4087 if (!rtx_equal_p (a, b))
4090 else if (VOID_TYPE_P (TREE_TYPE (DECL_RESULT (cfun->decl))))
4092 else if (!rtx_equal_p (a, b))
4095 /* If this call is indirect, we'll need to be able to use a call-clobbered
4096 register for the address of the target function. Make sure that all
4097 such registers are not used for passing parameters. */
4098 if (!decl && !TARGET_64BIT)
4102 /* We're looking at the CALL_EXPR, we need the type of the function. */
4103 type = CALL_EXPR_FN (exp); /* pointer expression */
4104 type = TREE_TYPE (type); /* pointer type */
4105 type = TREE_TYPE (type); /* function type */
4107 if (ix86_function_regparm (type, NULL) >= 3)
4109 /* ??? Need to count the actual number of registers to be used,
4110 not the possible number of registers. Fix later. */
4115 /* Dllimport'd functions are also called indirectly. */
4116 if (TARGET_DLLIMPORT_DECL_ATTRIBUTES
4118 && decl && DECL_DLLIMPORT_P (decl)
4119 && ix86_function_regparm (TREE_TYPE (decl), NULL) >= 3)
4122 /* If we need to align the outgoing stack, then sibcalling would
4123 unalign the stack, which may break the called function. */
4124 if (ix86_incoming_stack_boundary < PREFERRED_STACK_BOUNDARY)
4127 /* Otherwise okay. That also includes certain types of indirect calls. */
4131 /* Handle "cdecl", "stdcall", "fastcall", "regparm" and "sseregparm"
4132 calling convention attributes;
4133 arguments as in struct attribute_spec.handler. */
4136 ix86_handle_cconv_attribute (tree *node, tree name,
4138 int flags ATTRIBUTE_UNUSED,
4141 if (TREE_CODE (*node) != FUNCTION_TYPE
4142 && TREE_CODE (*node) != METHOD_TYPE
4143 && TREE_CODE (*node) != FIELD_DECL
4144 && TREE_CODE (*node) != TYPE_DECL)
4146 warning (OPT_Wattributes, "%qs attribute only applies to functions",
4147 IDENTIFIER_POINTER (name));
4148 *no_add_attrs = true;
4152 /* Can combine regparm with all attributes but fastcall. */
4153 if (is_attribute_p ("regparm", name))
4157 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node)))
4159 error ("fastcall and regparm attributes are not compatible");
4162 cst = TREE_VALUE (args);
4163 if (TREE_CODE (cst) != INTEGER_CST)
4165 warning (OPT_Wattributes,
4166 "%qs attribute requires an integer constant argument",
4167 IDENTIFIER_POINTER (name));
4168 *no_add_attrs = true;
4170 else if (compare_tree_int (cst, REGPARM_MAX) > 0)
4172 warning (OPT_Wattributes, "argument to %qs attribute larger than %d",
4173 IDENTIFIER_POINTER (name), REGPARM_MAX);
4174 *no_add_attrs = true;
4182 /* Do not warn when emulating the MS ABI. */
4183 if (TREE_CODE (*node) != FUNCTION_TYPE || ix86_function_type_abi (*node)!=MS_ABI)
4184 warning (OPT_Wattributes, "%qs attribute ignored",
4185 IDENTIFIER_POINTER (name));
4186 *no_add_attrs = true;
4190 /* Can combine fastcall with stdcall (redundant) and sseregparm. */
4191 if (is_attribute_p ("fastcall", name))
4193 if (lookup_attribute ("cdecl", TYPE_ATTRIBUTES (*node)))
4195 error ("fastcall and cdecl attributes are not compatible");
4197 if (lookup_attribute ("stdcall", TYPE_ATTRIBUTES (*node)))
4199 error ("fastcall and stdcall attributes are not compatible");
4201 if (lookup_attribute ("regparm", TYPE_ATTRIBUTES (*node)))
4203 error ("fastcall and regparm attributes are not compatible");
4207 /* Can combine stdcall with fastcall (redundant), regparm and
4209 else if (is_attribute_p ("stdcall", name))
4211 if (lookup_attribute ("cdecl", TYPE_ATTRIBUTES (*node)))
4213 error ("stdcall and cdecl attributes are not compatible");
4215 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node)))
4217 error ("stdcall and fastcall attributes are not compatible");
4221 /* Can combine cdecl with regparm and sseregparm. */
4222 else if (is_attribute_p ("cdecl", name))
4224 if (lookup_attribute ("stdcall", TYPE_ATTRIBUTES (*node)))
4226 error ("stdcall and cdecl attributes are not compatible");
4228 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node)))
4230 error ("fastcall and cdecl attributes are not compatible");
4234 /* Can combine sseregparm with all attributes. */
4239 /* Return 0 if the attributes for two types are incompatible, 1 if they
4240 are compatible, and 2 if they are nearly compatible (which causes a
4241 warning to be generated). */
4244 ix86_comp_type_attributes (const_tree type1, const_tree type2)
4246 /* Check for mismatch of non-default calling convention. */
4247 const char *const rtdstr = TARGET_RTD ? "cdecl" : "stdcall";
4249 if (TREE_CODE (type1) != FUNCTION_TYPE
4250 && TREE_CODE (type1) != METHOD_TYPE)
4253 /* Check for mismatched fastcall/regparm types. */
4254 if ((!lookup_attribute ("fastcall", TYPE_ATTRIBUTES (type1))
4255 != !lookup_attribute ("fastcall", TYPE_ATTRIBUTES (type2)))
4256 || (ix86_function_regparm (type1, NULL)
4257 != ix86_function_regparm (type2, NULL)))
4260 /* Check for mismatched sseregparm types. */
4261 if (!lookup_attribute ("sseregparm", TYPE_ATTRIBUTES (type1))
4262 != !lookup_attribute ("sseregparm", TYPE_ATTRIBUTES (type2)))
4265 /* Check for mismatched return types (cdecl vs stdcall). */
4266 if (!lookup_attribute (rtdstr, TYPE_ATTRIBUTES (type1))
4267 != !lookup_attribute (rtdstr, TYPE_ATTRIBUTES (type2)))
4273 /* Return the regparm value for a function with the indicated TYPE and DECL.
4274 DECL may be NULL when calling function indirectly
4275 or considering a libcall. */
4278 ix86_function_regparm (const_tree type, const_tree decl)
4281 int regparm = ix86_regparm;
4283 static bool error_issued;
4287 if (ix86_function_type_abi (type) == DEFAULT_ABI)
4289 return DEFAULT_ABI != SYSV_ABI ? X86_64_REGPARM_MAX : X64_REGPARM_MAX;
4292 attr = lookup_attribute ("regparm", TYPE_ATTRIBUTES (type));
4296 = TREE_INT_CST_LOW (TREE_VALUE (TREE_VALUE (attr)));
4298 if (decl && TREE_CODE (decl) == FUNCTION_DECL)
4300 /* We can't use regparm(3) for nested functions because
4301 these pass static chain pointer in %ecx register. */
4302 if (!error_issued && regparm == 3
4303 && decl_function_context (decl)
4304 && !DECL_NO_STATIC_CHAIN (decl))
4306 error ("nested functions are limited to 2 register parameters");
4307 error_issued = true;
4315 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (type)))
4318 /* Use register calling convention for local functions when possible. */
4319 if (decl && TREE_CODE (decl) == FUNCTION_DECL
4322 /* FIXME: remove this CONST_CAST when cgraph.[ch] is constified. */
4323 struct cgraph_local_info *i = cgraph_local_info (CONST_CAST_TREE(decl));
4326 int local_regparm, globals = 0, regno;
4329 /* Make sure no regparm register is taken by a
4330 fixed register variable. */
4331 for (local_regparm = 0; local_regparm < REGPARM_MAX; local_regparm++)
4332 if (fixed_regs[local_regparm])
4335 /* We can't use regparm(3) for nested functions as these use
4336 static chain pointer in third argument. */
4337 if (local_regparm == 3
4338 && decl_function_context (decl)
4339 && !DECL_NO_STATIC_CHAIN (decl))
4342 /* If the function realigns its stackpointer, the prologue will
4343 clobber %ecx. If we've already generated code for the callee,
4344 the callee DECL_STRUCT_FUNCTION is gone, so we fall back to
4345 scanning the attributes for the self-realigning property. */
4346 f = DECL_STRUCT_FUNCTION (decl);
4347 /* Since current internal arg pointer won't conflict with
4348 parameter passing regs, so no need to change stack
4349 realignment and adjust regparm number.
4351 Each fixed register usage increases register pressure,
4352 so less registers should be used for argument passing.
4353 This functionality can be overriden by an explicit
4355 for (regno = 0; regno <= DI_REG; regno++)
4356 if (fixed_regs[regno])
4360 = globals < local_regparm ? local_regparm - globals : 0;
4362 if (local_regparm > regparm)
4363 regparm = local_regparm;
4370 /* Return 1 or 2, if we can pass up to SSE_REGPARM_MAX SFmode (1) and
4371 DFmode (2) arguments in SSE registers for a function with the
4372 indicated TYPE and DECL. DECL may be NULL when calling function
4373 indirectly or considering a libcall. Otherwise return 0. */
4376 ix86_function_sseregparm (const_tree type, const_tree decl, bool warn)
4378 gcc_assert (!TARGET_64BIT);
4380 /* Use SSE registers to pass SFmode and DFmode arguments if requested
4381 by the sseregparm attribute. */
4382 if (TARGET_SSEREGPARM
4383 || (type && lookup_attribute ("sseregparm", TYPE_ATTRIBUTES (type))))
4390 error ("Calling %qD with attribute sseregparm without "
4391 "SSE/SSE2 enabled", decl);
4393 error ("Calling %qT with attribute sseregparm without "
4394 "SSE/SSE2 enabled", type);
4402 /* For local functions, pass up to SSE_REGPARM_MAX SFmode
4403 (and DFmode for SSE2) arguments in SSE registers. */
4404 if (decl && TARGET_SSE_MATH && !profile_flag)
4406 /* FIXME: remove this CONST_CAST when cgraph.[ch] is constified. */
4407 struct cgraph_local_info *i = cgraph_local_info (CONST_CAST_TREE(decl));
4409 return TARGET_SSE2 ? 2 : 1;
4415 /* Return true if EAX is live at the start of the function. Used by
4416 ix86_expand_prologue to determine if we need special help before
4417 calling allocate_stack_worker. */
4420 ix86_eax_live_at_start_p (void)
4422 /* Cheat. Don't bother working forward from ix86_function_regparm
4423 to the function type to whether an actual argument is located in
4424 eax. Instead just look at cfg info, which is still close enough
4425 to correct at this point. This gives false positives for broken
4426 functions that might use uninitialized data that happens to be
4427 allocated in eax, but who cares? */
4428 return REGNO_REG_SET_P (df_get_live_out (ENTRY_BLOCK_PTR), 0);
4431 /* Value is the number of bytes of arguments automatically
4432 popped when returning from a subroutine call.
4433 FUNDECL is the declaration node of the function (as a tree),
4434 FUNTYPE is the data type of the function (as a tree),
4435 or for a library call it is an identifier node for the subroutine name.
4436 SIZE is the number of bytes of arguments passed on the stack.
4438 On the 80386, the RTD insn may be used to pop them if the number
4439 of args is fixed, but if the number is variable then the caller
4440 must pop them all. RTD can't be used for library calls now
4441 because the library is compiled with the Unix compiler.
4442 Use of RTD is a selectable option, since it is incompatible with
4443 standard Unix calling sequences. If the option is not selected,
4444 the caller must always pop the args.
4446 The attribute stdcall is equivalent to RTD on a per module basis. */
4449 ix86_return_pops_args (tree fundecl, tree funtype, int size)
4453 /* None of the 64-bit ABIs pop arguments. */
4457 rtd = TARGET_RTD && (!fundecl || TREE_CODE (fundecl) != IDENTIFIER_NODE);
4459 /* Cdecl functions override -mrtd, and never pop the stack. */
4460 if (! lookup_attribute ("cdecl", TYPE_ATTRIBUTES (funtype)))
4462 /* Stdcall and fastcall functions will pop the stack if not
4464 if (lookup_attribute ("stdcall", TYPE_ATTRIBUTES (funtype))
4465 || lookup_attribute ("fastcall", TYPE_ATTRIBUTES (funtype)))
4468 if (rtd && ! stdarg_p (funtype))
4472 /* Lose any fake structure return argument if it is passed on the stack. */
4473 if (aggregate_value_p (TREE_TYPE (funtype), fundecl)
4474 && !KEEP_AGGREGATE_RETURN_POINTER)
4476 int nregs = ix86_function_regparm (funtype, fundecl);
4478 return GET_MODE_SIZE (Pmode);
4484 /* Argument support functions. */
4486 /* Return true when register may be used to pass function parameters. */
4488 ix86_function_arg_regno_p (int regno)
4491 const int *parm_regs;
4496 return (regno < REGPARM_MAX
4497 || (TARGET_SSE && SSE_REGNO_P (regno) && !fixed_regs[regno]));
4499 return (regno < REGPARM_MAX
4500 || (TARGET_MMX && MMX_REGNO_P (regno)
4501 && (regno < FIRST_MMX_REG + MMX_REGPARM_MAX))
4502 || (TARGET_SSE && SSE_REGNO_P (regno)
4503 && (regno < FIRST_SSE_REG + SSE_REGPARM_MAX)));
4508 if (SSE_REGNO_P (regno) && TARGET_SSE)
4513 if (TARGET_SSE && SSE_REGNO_P (regno)
4514 && (regno < FIRST_SSE_REG + SSE_REGPARM_MAX))
4518 /* TODO: The function should depend on current function ABI but
4519 builtins.c would need updating then. Therefore we use the
4522 /* RAX is used as hidden argument to va_arg functions. */
4523 if (DEFAULT_ABI == SYSV_ABI && regno == AX_REG)
4526 if (DEFAULT_ABI == MS_ABI)
4527 parm_regs = x86_64_ms_abi_int_parameter_registers;
4529 parm_regs = x86_64_int_parameter_registers;
4530 for (i = 0; i < (DEFAULT_ABI == MS_ABI ? X64_REGPARM_MAX
4531 : X86_64_REGPARM_MAX); i++)
4532 if (regno == parm_regs[i])
4537 /* Return if we do not know how to pass TYPE solely in registers. */
4540 ix86_must_pass_in_stack (enum machine_mode mode, const_tree type)
4542 if (must_pass_in_stack_var_size_or_pad (mode, type))
4545 /* For 32-bit, we want TImode aggregates to go on the stack. But watch out!
4546 The layout_type routine is crafty and tries to trick us into passing
4547 currently unsupported vector types on the stack by using TImode. */
4548 return (!TARGET_64BIT && mode == TImode
4549 && type && TREE_CODE (type) != VECTOR_TYPE);
4552 /* It returns the size, in bytes, of the area reserved for arguments passed
4553 in registers for the function represented by fndecl dependent to the used
4556 ix86_reg_parm_stack_space (const_tree fndecl)
4558 int call_abi = SYSV_ABI;
4559 if (fndecl != NULL_TREE && TREE_CODE (fndecl) == FUNCTION_DECL)
4560 call_abi = ix86_function_abi (fndecl);
4562 call_abi = ix86_function_type_abi (fndecl);
4563 if (call_abi == MS_ABI)
4568 /* Returns value SYSV_ABI, MS_ABI dependent on fntype, specifying the
4571 ix86_function_type_abi (const_tree fntype)
4573 if (TARGET_64BIT && fntype != NULL)
4576 if (DEFAULT_ABI == SYSV_ABI)
4577 abi = lookup_attribute ("ms_abi", TYPE_ATTRIBUTES (fntype)) ? MS_ABI : SYSV_ABI;
4579 abi = lookup_attribute ("sysv_abi", TYPE_ATTRIBUTES (fntype)) ? SYSV_ABI : MS_ABI;
4587 ix86_function_abi (const_tree fndecl)
4591 return ix86_function_type_abi (TREE_TYPE (fndecl));
4594 /* Returns value SYSV_ABI, MS_ABI dependent on cfun, specifying the
4597 ix86_cfun_abi (void)
4599 if (! cfun || ! TARGET_64BIT)
4601 return cfun->machine->call_abi;
4605 extern void init_regs (void);
4607 /* Implementation of call abi switching target hook. Specific to FNDECL
4608 the specific call register sets are set. See also CONDITIONAL_REGISTER_USAGE
4609 for more details. */
4611 ix86_call_abi_override (const_tree fndecl)
4613 if (fndecl == NULL_TREE)
4614 cfun->machine->call_abi = DEFAULT_ABI;
4616 cfun->machine->call_abi = ix86_function_type_abi (TREE_TYPE (fndecl));
4619 /* MS and SYSV ABI have different set of call used registers. Avoid expensive
4620 re-initialization of init_regs each time we switch function context since
4621 this is needed only during RTL expansion. */
4623 ix86_maybe_switch_abi (void)
4626 call_used_regs[4 /*RSI*/] == (cfun->machine->call_abi == MS_ABI))
4630 /* Initialize a variable CUM of type CUMULATIVE_ARGS
4631 for a call to a function whose data type is FNTYPE.
4632 For a library call, FNTYPE is 0. */
4635 init_cumulative_args (CUMULATIVE_ARGS *cum, /* Argument info to initialize */
4636 tree fntype, /* tree ptr for function decl */
4637 rtx libname, /* SYMBOL_REF of library name or 0 */
4640 struct cgraph_local_info *i = fndecl ? cgraph_local_info (fndecl) : NULL;
4641 memset (cum, 0, sizeof (*cum));
4644 cum->call_abi = ix86_function_abi (fndecl);
4646 cum->call_abi = ix86_function_type_abi (fntype);
4647 /* Set up the number of registers to use for passing arguments. */
4649 if (cum->call_abi == MS_ABI && !ACCUMULATE_OUTGOING_ARGS)
4650 sorry ("ms_abi attribute require -maccumulate-outgoing-args or subtarget optimization implying it");
4651 cum->nregs = ix86_regparm;
4654 if (cum->call_abi != DEFAULT_ABI)
4655 cum->nregs = DEFAULT_ABI != SYSV_ABI ? X86_64_REGPARM_MAX
4660 cum->sse_nregs = SSE_REGPARM_MAX;
4663 if (cum->call_abi != DEFAULT_ABI)
4664 cum->sse_nregs = DEFAULT_ABI != SYSV_ABI ? X86_64_SSE_REGPARM_MAX
4665 : X64_SSE_REGPARM_MAX;
4669 cum->mmx_nregs = MMX_REGPARM_MAX;
4670 cum->warn_avx = true;
4671 cum->warn_sse = true;
4672 cum->warn_mmx = true;
4674 /* Because type might mismatch in between caller and callee, we need to
4675 use actual type of function for local calls.
4676 FIXME: cgraph_analyze can be told to actually record if function uses
4677 va_start so for local functions maybe_vaarg can be made aggressive
4679 FIXME: once typesytem is fixed, we won't need this code anymore. */
4681 fntype = TREE_TYPE (fndecl);
4682 cum->maybe_vaarg = (fntype
4683 ? (!prototype_p (fntype) || stdarg_p (fntype))
4688 /* If there are variable arguments, then we won't pass anything
4689 in registers in 32-bit mode. */
4690 if (stdarg_p (fntype))
4701 /* Use ecx and edx registers if function has fastcall attribute,
4702 else look for regparm information. */
4705 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (fntype)))
4711 cum->nregs = ix86_function_regparm (fntype, fndecl);
4714 /* Set up the number of SSE registers used for passing SFmode
4715 and DFmode arguments. Warn for mismatching ABI. */
4716 cum->float_in_sse = ix86_function_sseregparm (fntype, fndecl, true);
4720 /* Return the "natural" mode for TYPE. In most cases, this is just TYPE_MODE.
4721 But in the case of vector types, it is some vector mode.
4723 When we have only some of our vector isa extensions enabled, then there
4724 are some modes for which vector_mode_supported_p is false. For these
4725 modes, the generic vector support in gcc will choose some non-vector mode
4726 in order to implement the type. By computing the natural mode, we'll
4727 select the proper ABI location for the operand and not depend on whatever
4728 the middle-end decides to do with these vector types. */
4730 static enum machine_mode
4731 type_natural_mode (const_tree type)
4733 enum machine_mode mode = TYPE_MODE (type);
4735 if (TREE_CODE (type) == VECTOR_TYPE && !VECTOR_MODE_P (mode))
4737 HOST_WIDE_INT size = int_size_in_bytes (type);
4738 if ((size == 8 || size == 16)
4739 /* ??? Generic code allows us to create width 1 vectors. Ignore. */
4740 && TYPE_VECTOR_SUBPARTS (type) > 1)
4742 enum machine_mode innermode = TYPE_MODE (TREE_TYPE (type));
4744 if (TREE_CODE (TREE_TYPE (type)) == REAL_TYPE)
4745 mode = MIN_MODE_VECTOR_FLOAT;
4747 mode = MIN_MODE_VECTOR_INT;
4749 /* Get the mode which has this inner mode and number of units. */
4750 for (; mode != VOIDmode; mode = GET_MODE_WIDER_MODE (mode))
4751 if (GET_MODE_NUNITS (mode) == TYPE_VECTOR_SUBPARTS (type)
4752 && GET_MODE_INNER (mode) == innermode)
4762 /* We want to pass a value in REGNO whose "natural" mode is MODE. However,
4763 this may not agree with the mode that the type system has chosen for the
4764 register, which is ORIG_MODE. If ORIG_MODE is not BLKmode, then we can
4765 go ahead and use it. Otherwise we have to build a PARALLEL instead. */
4768 gen_reg_or_parallel (enum machine_mode mode, enum machine_mode orig_mode,
4773 if (orig_mode != BLKmode)
4774 tmp = gen_rtx_REG (orig_mode, regno);
4777 tmp = gen_rtx_REG (mode, regno);
4778 tmp = gen_rtx_EXPR_LIST (VOIDmode, tmp, const0_rtx);
4779 tmp = gen_rtx_PARALLEL (orig_mode, gen_rtvec (1, tmp));
4785 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
4786 of this code is to classify each 8bytes of incoming argument by the register
4787 class and assign registers accordingly. */
4789 /* Return the union class of CLASS1 and CLASS2.
4790 See the x86-64 PS ABI for details. */
4792 static enum x86_64_reg_class
4793 merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
4795 /* Rule #1: If both classes are equal, this is the resulting class. */
4796 if (class1 == class2)
4799 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
4801 if (class1 == X86_64_NO_CLASS)
4803 if (class2 == X86_64_NO_CLASS)
4806 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
4807 if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
4808 return X86_64_MEMORY_CLASS;
4810 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
4811 if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
4812 || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
4813 return X86_64_INTEGERSI_CLASS;
4814 if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
4815 || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
4816 return X86_64_INTEGER_CLASS;
4818 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
4820 if (class1 == X86_64_X87_CLASS
4821 || class1 == X86_64_X87UP_CLASS
4822 || class1 == X86_64_COMPLEX_X87_CLASS
4823 || class2 == X86_64_X87_CLASS
4824 || class2 == X86_64_X87UP_CLASS
4825 || class2 == X86_64_COMPLEX_X87_CLASS)
4826 return X86_64_MEMORY_CLASS;
4828 /* Rule #6: Otherwise class SSE is used. */
4829 return X86_64_SSE_CLASS;
4832 /* Classify the argument of type TYPE and mode MODE.
4833 CLASSES will be filled by the register class used to pass each word
4834 of the operand. The number of words is returned. In case the parameter
4835 should be passed in memory, 0 is returned. As a special case for zero
4836 sized containers, classes[0] will be NO_CLASS and 1 is returned.
4838 BIT_OFFSET is used internally for handling records and specifies offset
4839 of the offset in bits modulo 256 to avoid overflow cases.
4841 See the x86-64 PS ABI for details.
4845 classify_argument (enum machine_mode mode, const_tree type,
4846 enum x86_64_reg_class classes[MAX_CLASSES], int bit_offset)
4848 HOST_WIDE_INT bytes =
4849 (mode == BLKmode) ? int_size_in_bytes (type) : (int) GET_MODE_SIZE (mode);
4850 int words = (bytes + (bit_offset % 64) / 8 + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
4852 /* Variable sized entities are always passed/returned in memory. */
4856 if (mode != VOIDmode
4857 && targetm.calls.must_pass_in_stack (mode, type))
4860 if (type && AGGREGATE_TYPE_P (type))
4864 enum x86_64_reg_class subclasses[MAX_CLASSES];
4866 /* On x86-64 we pass structures larger than 16 bytes on the stack. */
4870 for (i = 0; i < words; i++)
4871 classes[i] = X86_64_NO_CLASS;
4873 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
4874 signalize memory class, so handle it as special case. */
4877 classes[0] = X86_64_NO_CLASS;
4881 /* Classify each field of record and merge classes. */
4882 switch (TREE_CODE (type))
4885 /* And now merge the fields of structure. */
4886 for (field = TYPE_FIELDS (type); field; field = TREE_CHAIN (field))
4888 if (TREE_CODE (field) == FIELD_DECL)
4892 if (TREE_TYPE (field) == error_mark_node)
4895 /* Bitfields are always classified as integer. Handle them
4896 early, since later code would consider them to be
4897 misaligned integers. */
4898 if (DECL_BIT_FIELD (field))
4900 for (i = (int_bit_position (field) + (bit_offset % 64)) / 8 / 8;
4901 i < ((int_bit_position (field) + (bit_offset % 64))
4902 + tree_low_cst (DECL_SIZE (field), 0)
4905 merge_classes (X86_64_INTEGER_CLASS,
4910 num = classify_argument (TYPE_MODE (TREE_TYPE (field)),
4911 TREE_TYPE (field), subclasses,
4912 (int_bit_position (field)
4913 + bit_offset) % 256);
4916 for (i = 0; i < num; i++)
4919 (int_bit_position (field) + (bit_offset % 64)) / 8 / 8;
4921 merge_classes (subclasses[i], classes[i + pos]);
4929 /* Arrays are handled as small records. */
4932 num = classify_argument (TYPE_MODE (TREE_TYPE (type)),
4933 TREE_TYPE (type), subclasses, bit_offset);
4937 /* The partial classes are now full classes. */
4938 if (subclasses[0] == X86_64_SSESF_CLASS && bytes != 4)
4939 subclasses[0] = X86_64_SSE_CLASS;
4940 if (subclasses[0] == X86_64_INTEGERSI_CLASS
4941 && !((bit_offset % 64) == 0 && bytes == 4))
4942 subclasses[0] = X86_64_INTEGER_CLASS;
4944 for (i = 0; i < words; i++)
4945 classes[i] = subclasses[i % num];
4950 case QUAL_UNION_TYPE:
4951 /* Unions are similar to RECORD_TYPE but offset is always 0.
4953 for (field = TYPE_FIELDS (type); field; field = TREE_CHAIN (field))
4955 if (TREE_CODE (field) == FIELD_DECL)
4959 if (TREE_TYPE (field) == error_mark_node)
4962 num = classify_argument (TYPE_MODE (TREE_TYPE (field)),
4963 TREE_TYPE (field), subclasses,
4967 for (i = 0; i < num; i++)
4968 classes[i] = merge_classes (subclasses[i], classes[i]);
4977 /* Final merger cleanup. */
4978 for (i = 0; i < words; i++)
4980 /* If one class is MEMORY, everything should be passed in
4982 if (classes[i] == X86_64_MEMORY_CLASS)
4985 /* The X86_64_SSEUP_CLASS should be always preceded by
4986 X86_64_SSE_CLASS. */
4987 if (classes[i] == X86_64_SSEUP_CLASS
4988 && (i == 0 || classes[i - 1] != X86_64_SSE_CLASS))
4989 classes[i] = X86_64_SSE_CLASS;
4991 /* X86_64_X87UP_CLASS should be preceded by X86_64_X87_CLASS. */
4992 if (classes[i] == X86_64_X87UP_CLASS
4993 && (i == 0 || classes[i - 1] != X86_64_X87_CLASS))
4994 classes[i] = X86_64_SSE_CLASS;
4999 /* Compute alignment needed. We align all types to natural boundaries with
5000 exception of XFmode that is aligned to 64bits. */
5001 if (mode != VOIDmode && mode != BLKmode)
5003 int mode_alignment = GET_MODE_BITSIZE (mode);
5006 mode_alignment = 128;
5007 else if (mode == XCmode)
5008 mode_alignment = 256;
5009 if (COMPLEX_MODE_P (mode))
5010 mode_alignment /= 2;
5011 /* Misaligned fields are always returned in memory. */
5012 if (bit_offset % mode_alignment)
5016 /* for V1xx modes, just use the base mode */
5017 if (VECTOR_MODE_P (mode) && mode != V1DImode
5018 && GET_MODE_SIZE (GET_MODE_INNER (mode)) == bytes)
5019 mode = GET_MODE_INNER (mode);
5021 /* Classification of atomic types. */
5026 classes[0] = X86_64_SSE_CLASS;
5029 classes[0] = X86_64_SSE_CLASS;
5030 classes[1] = X86_64_SSEUP_CLASS;
5040 int size = (bit_offset % 64)+ (int) GET_MODE_BITSIZE (mode);
5044 classes[0] = X86_64_INTEGERSI_CLASS;
5047 else if (size <= 64)
5049 classes[0] = X86_64_INTEGER_CLASS;
5052 else if (size <= 64+32)
5054 classes[0] = X86_64_INTEGER_CLASS;
5055 classes[1] = X86_64_INTEGERSI_CLASS;
5058 else if (size <= 64+64)
5060 classes[0] = classes[1] = X86_64_INTEGER_CLASS;
5068 classes[0] = classes[1] = X86_64_INTEGER_CLASS;
5075 if (!(bit_offset % 64))
5076 classes[0] = X86_64_SSESF_CLASS;
5078 classes[0] = X86_64_SSE_CLASS;
5081 classes[0] = X86_64_SSEDF_CLASS;
5084 classes[0] = X86_64_X87_CLASS;
5085 classes[1] = X86_64_X87UP_CLASS;
5088 classes[0] = X86_64_SSE_CLASS;
5089 classes[1] = X86_64_SSEUP_CLASS;
5092 classes[0] = X86_64_SSE_CLASS;
5095 classes[0] = X86_64_SSEDF_CLASS;
5096 classes[1] = X86_64_SSEDF_CLASS;
5099 classes[0] = X86_64_COMPLEX_X87_CLASS;
5102 /* This modes is larger than 16 bytes. */
5110 classes[0] = X86_64_AVX_CLASS;
5118 classes[0] = X86_64_SSE_CLASS;
5119 classes[1] = X86_64_SSEUP_CLASS;
5126 classes[0] = X86_64_SSE_CLASS;
5132 gcc_assert (VECTOR_MODE_P (mode));
5137 gcc_assert (GET_MODE_CLASS (GET_MODE_INNER (mode)) == MODE_INT);
5139 if (bit_offset + GET_MODE_BITSIZE (mode) <= 32)
5140 classes[0] = X86_64_INTEGERSI_CLASS;
5142 classes[0] = X86_64_INTEGER_CLASS;
5143 classes[1] = X86_64_INTEGER_CLASS;
5144 return 1 + (bytes > 8);
5148 /* Examine the argument and return set number of register required in each
5149 class. Return 0 iff parameter should be passed in memory. */
5151 examine_argument (enum machine_mode mode, const_tree type, int in_return,
5152 int *int_nregs, int *sse_nregs)
5154 enum x86_64_reg_class regclass[MAX_CLASSES];
5155 int n = classify_argument (mode, type, regclass, 0);
5161 for (n--; n >= 0; n--)
5162 switch (regclass[n])
5164 case X86_64_INTEGER_CLASS:
5165 case X86_64_INTEGERSI_CLASS:
5168 case X86_64_AVX_CLASS:
5169 case X86_64_SSE_CLASS:
5170 case X86_64_SSESF_CLASS:
5171 case X86_64_SSEDF_CLASS:
5174 case X86_64_NO_CLASS:
5175 case X86_64_SSEUP_CLASS:
5177 case X86_64_X87_CLASS:
5178 case X86_64_X87UP_CLASS:
5182 case X86_64_COMPLEX_X87_CLASS:
5183 return in_return ? 2 : 0;
5184 case X86_64_MEMORY_CLASS:
5190 /* Construct container for the argument used by GCC interface. See
5191 FUNCTION_ARG for the detailed description. */
5194 construct_container (enum machine_mode mode, enum machine_mode orig_mode,
5195 const_tree type, int in_return, int nintregs, int nsseregs,
5196 const int *intreg, int sse_regno)
5198 /* The following variables hold the static issued_error state. */
5199 static bool issued_sse_arg_error;
5200 static bool issued_sse_ret_error;
5201 static bool issued_x87_ret_error;
5203 enum machine_mode tmpmode;
5205 (mode == BLKmode) ? int_size_in_bytes (type) : (int) GET_MODE_SIZE (mode);
5206 enum x86_64_reg_class regclass[MAX_CLASSES];
5210 int needed_sseregs, needed_intregs;
5211 rtx exp[MAX_CLASSES];
5214 n = classify_argument (mode, type, regclass, 0);
5217 if (!examine_argument (mode, type, in_return, &needed_intregs,
5220 if (needed_intregs > nintregs || needed_sseregs > nsseregs)
5223 /* We allowed the user to turn off SSE for kernel mode. Don't crash if
5224 some less clueful developer tries to use floating-point anyway. */
5225 if (needed_sseregs && !TARGET_SSE)
5229 if (!issued_sse_ret_error)
5231 error ("SSE register return with SSE disabled");
5232 issued_sse_ret_error = true;
5235 else if (!issued_sse_arg_error)
5237 error ("SSE register argument with SSE disabled");
5238 issued_sse_arg_error = true;
5243 /* Likewise, error if the ABI requires us to return values in the
5244 x87 registers and the user specified -mno-80387. */
5245 if (!TARGET_80387 && in_return)
5246 for (i = 0; i < n; i++)
5247 if (regclass[i] == X86_64_X87_CLASS
5248 || regclass[i] == X86_64_X87UP_CLASS
5249 || regclass[i] == X86_64_COMPLEX_X87_CLASS)
5251 if (!issued_x87_ret_error)
5253 error ("x87 register return with x87 disabled");
5254 issued_x87_ret_error = true;
5259 /* First construct simple cases. Avoid SCmode, since we want to use
5260 single register to pass this type. */
5261 if (n == 1 && mode != SCmode)
5262 switch (regclass[0])
5264 case X86_64_INTEGER_CLASS:
5265 case X86_64_INTEGERSI_CLASS:
5266 return gen_rtx_REG (mode, intreg[0]);
5267 case X86_64_AVX_CLASS:
5268 case X86_64_SSE_CLASS:
5269 case X86_64_SSESF_CLASS:
5270 case X86_64_SSEDF_CLASS:
5271 return gen_reg_or_parallel (mode, orig_mode, SSE_REGNO (sse_regno));
5272 case X86_64_X87_CLASS:
5273 case X86_64_COMPLEX_X87_CLASS:
5274 return gen_rtx_REG (mode, FIRST_STACK_REG);
5275 case X86_64_NO_CLASS:
5276 /* Zero sized array, struct or class. */
5281 if (n == 2 && regclass[0] == X86_64_SSE_CLASS
5282 && regclass[1] == X86_64_SSEUP_CLASS && mode != BLKmode)
5283 return gen_rtx_REG (mode, SSE_REGNO (sse_regno));
5286 && regclass[0] == X86_64_X87_CLASS && regclass[1] == X86_64_X87UP_CLASS)
5287 return gen_rtx_REG (XFmode, FIRST_STACK_REG);
5288 if (n == 2 && regclass[0] == X86_64_INTEGER_CLASS
5289 && regclass[1] == X86_64_INTEGER_CLASS
5290 && (mode == CDImode || mode == TImode || mode == TFmode)
5291 && intreg[0] + 1 == intreg[1])
5292 return gen_rtx_REG (mode, intreg[0]);
5294 /* Otherwise figure out the entries of the PARALLEL. */
5295 for (i = 0; i < n; i++)
5297 switch (regclass[i])
5299 case X86_64_NO_CLASS:
5301 case X86_64_INTEGER_CLASS:
5302 case X86_64_INTEGERSI_CLASS:
5303 /* Merge TImodes on aligned occasions here too. */
5304 if (i * 8 + 8 > bytes)
5305 tmpmode = mode_for_size ((bytes - i * 8) * BITS_PER_UNIT, MODE_INT, 0);
5306 else if (regclass[i] == X86_64_INTEGERSI_CLASS)
5310 /* We've requested 24 bytes we don't have mode for. Use DImode. */
5311 if (tmpmode == BLKmode)
5313 exp [nexps++] = gen_rtx_EXPR_LIST (VOIDmode,
5314 gen_rtx_REG (tmpmode, *intreg),
5318 case X86_64_SSESF_CLASS:
5319 exp [nexps++] = gen_rtx_EXPR_LIST (VOIDmode,
5320 gen_rtx_REG (SFmode,
5321 SSE_REGNO (sse_regno)),
5325 case X86_64_SSEDF_CLASS:
5326 exp [nexps++] = gen_rtx_EXPR_LIST (VOIDmode,
5327 gen_rtx_REG (DFmode,
5328 SSE_REGNO (sse_regno)),
5332 case X86_64_SSE_CLASS:
5333 if (i < n - 1 && regclass[i + 1] == X86_64_SSEUP_CLASS)
5337 exp [nexps++] = gen_rtx_EXPR_LIST (VOIDmode,
5338 gen_rtx_REG (tmpmode,
5339 SSE_REGNO (sse_regno)),
5341 if (tmpmode == TImode)
5350 /* Empty aligned struct, union or class. */
5354 ret = gen_rtx_PARALLEL (mode, rtvec_alloc (nexps));
5355 for (i = 0; i < nexps; i++)
5356 XVECEXP (ret, 0, i) = exp [i];
5360 /* Update the data in CUM to advance over an argument of mode MODE
5361 and data type TYPE. (TYPE is null for libcalls where that information
5362 may not be available.) */
5365 function_arg_advance_32 (CUMULATIVE_ARGS *cum, enum machine_mode mode,
5366 tree type, HOST_WIDE_INT bytes, HOST_WIDE_INT words)
5382 cum->words += words;
5383 cum->nregs -= words;
5384 cum->regno += words;
5386 if (cum->nregs <= 0)
5394 if (cum->float_in_sse < 2)
5397 if (cum->float_in_sse < 1)
5415 if (!type || !AGGREGATE_TYPE_P (type))
5417 cum->sse_words += words;
5418 cum->sse_nregs -= 1;
5419 cum->sse_regno += 1;
5420 if (cum->sse_nregs <= 0)
5433 if (!type || !AGGREGATE_TYPE_P (type))
5435 cum->mmx_words += words;
5436 cum->mmx_nregs -= 1;
5437 cum->mmx_regno += 1;
5438 if (cum->mmx_nregs <= 0)
5449 function_arg_advance_64 (CUMULATIVE_ARGS *cum, enum machine_mode mode,
5450 tree type, HOST_WIDE_INT words, int named)
5452 int int_nregs, sse_nregs;
5454 /* Unnamed 256bit vector mode parameters are passed on stack. */
5455 if (!named && VALID_AVX256_REG_MODE (mode))
5458 if (!examine_argument (mode, type, 0, &int_nregs, &sse_nregs))
5459 cum->words += words;
5460 else if (sse_nregs <= cum->sse_nregs && int_nregs <= cum->nregs)
5462 cum->nregs -= int_nregs;
5463 cum->sse_nregs -= sse_nregs;
5464 cum->regno += int_nregs;
5465 cum->sse_regno += sse_nregs;
5468 cum->words += words;
5472 function_arg_advance_ms_64 (CUMULATIVE_ARGS *cum, HOST_WIDE_INT bytes,
5473 HOST_WIDE_INT words)
5475 /* Otherwise, this should be passed indirect. */
5476 gcc_assert (bytes == 1 || bytes == 2 || bytes == 4 || bytes == 8);
5478 cum->words += words;
5487 function_arg_advance (CUMULATIVE_ARGS *cum, enum machine_mode mode,
5488 tree type, int named)
5490 HOST_WIDE_INT bytes, words;
5492 if (mode == BLKmode)
5493 bytes = int_size_in_bytes (type);
5495 bytes = GET_MODE_SIZE (mode);
5496 words = (bytes + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
5499 mode = type_natural_mode (type);
5501 if (TARGET_64BIT && (cum ? cum->call_abi : DEFAULT_ABI) == MS_ABI)
5502 function_arg_advance_ms_64 (cum, bytes, words);
5503 else if (TARGET_64BIT)
5504 function_arg_advance_64 (cum, mode, type, words, named);
5506 function_arg_advance_32 (cum, mode, type, bytes, words);
5509 /* Define where to put the arguments to a function.
5510 Value is zero to push the argument on the stack,
5511 or a hard register in which to store the argument.
5513 MODE is the argument's machine mode.
5514 TYPE is the data type of the argument (as a tree).
5515 This is null for libcalls where that information may
5517 CUM is a variable of type CUMULATIVE_ARGS which gives info about
5518 the preceding args and about the function being called.
5519 NAMED is nonzero if this argument is a named parameter
5520 (otherwise it is an extra parameter matching an ellipsis). */
5523 function_arg_32 (CUMULATIVE_ARGS *cum, enum machine_mode mode,
5524 enum machine_mode orig_mode, tree type,
5525 HOST_WIDE_INT bytes, HOST_WIDE_INT words)
5527 static bool warnedavx, warnedsse, warnedmmx;
5529 /* Avoid the AL settings for the Unix64 ABI. */
5530 if (mode == VOIDmode)
5546 if (words <= cum->nregs)
5548 int regno = cum->regno;
5550 /* Fastcall allocates the first two DWORD (SImode) or
5551 smaller arguments to ECX and EDX if it isn't an
5557 || (type && AGGREGATE_TYPE_P (type)))
5560 /* ECX not EAX is the first allocated register. */
5561 if (regno == AX_REG)
5564 return gen_rtx_REG (mode, regno);
5569 if (cum->float_in_sse < 2)
5572 if (cum->float_in_sse < 1)
5576 /* In 32bit, we pass TImode in xmm registers. */
5583 if (!type || !AGGREGATE_TYPE_P (type))
5585 if (!TARGET_SSE && !warnedsse && cum->warn_sse)
5588 warning (0, "SSE vector argument without SSE enabled "
5592 return gen_reg_or_parallel (mode, orig_mode,
5593 cum->sse_regno + FIRST_SSE_REG);
5598 /* In 32bit, we pass OImode in ymm registers. */
5605 if (!type || !AGGREGATE_TYPE_P (type))
5607 if (!TARGET_AVX && !warnedavx && cum->warn_avx)
5610 warning (0, "AVX vector argument without AVX enabled "
5614 return gen_reg_or_parallel (mode, orig_mode,
5615 cum->sse_regno + FIRST_SSE_REG);
5624 if (!type || !AGGREGATE_TYPE_P (type))
5626 if (!TARGET_MMX && !warnedmmx && cum->warn_mmx)
5629 warning (0, "MMX vector argument without MMX enabled "
5633 return gen_reg_or_parallel (mode, orig_mode,
5634 cum->mmx_regno + FIRST_MMX_REG);
5643 function_arg_64 (CUMULATIVE_ARGS *cum, enum machine_mode mode,
5644 enum machine_mode orig_mode, tree type, int named)
5646 static bool warnedavx;
5648 /* Handle a hidden AL argument containing number of registers
5649 for varargs x86-64 functions. */
5650 if (mode == VOIDmode)
5651 return GEN_INT (cum->maybe_vaarg
5652 ? (cum->sse_nregs < 0
5653 ? (cum->call_abi == DEFAULT_ABI
5655 : (DEFAULT_ABI != SYSV_ABI ? X86_64_SSE_REGPARM_MAX
5656 : X64_SSE_REGPARM_MAX))
5671 /* In 64bit, we pass TImode in interger registers and OImode on
5673 if (!type || !AGGREGATE_TYPE_P (type))
5675 if (!TARGET_AVX && !warnedavx && cum->warn_avx)
5678 warning (0, "AVX vector argument without AVX enabled "
5683 /* Unnamed 256bit vector mode parameters are passed on stack. */
5689 return construct_container (mode, orig_mode, type, 0, cum->nregs,
5691 &x86_64_int_parameter_registers [cum->regno],
5696 function_arg_ms_64 (CUMULATIVE_ARGS *cum, enum machine_mode mode,
5697 enum machine_mode orig_mode, int named,
5698 HOST_WIDE_INT bytes)
5702 /* We need to add clobber for MS_ABI->SYSV ABI calls in expand_call.
5703 We use value of -2 to specify that current function call is MSABI. */
5704 if (mode == VOIDmode)
5705 return GEN_INT (-2);
5707 /* If we've run out of registers, it goes on the stack. */
5708 if (cum->nregs == 0)
5711 regno = x86_64_ms_abi_int_parameter_registers[cum->regno];
5713 /* Only floating point modes are passed in anything but integer regs. */
5714 if (TARGET_SSE && (mode == SFmode || mode == DFmode))
5717 regno = cum->regno + FIRST_SSE_REG;
5722 /* Unnamed floating parameters are passed in both the
5723 SSE and integer registers. */
5724 t1 = gen_rtx_REG (mode, cum->regno + FIRST_SSE_REG);
5725 t2 = gen_rtx_REG (mode, regno);
5726 t1 = gen_rtx_EXPR_LIST (VOIDmode, t1, const0_rtx);
5727 t2 = gen_rtx_EXPR_LIST (VOIDmode, t2, const0_rtx);
5728 return gen_rtx_PARALLEL (mode, gen_rtvec (2, t1, t2));
5731 /* Handle aggregated types passed in register. */
5732 if (orig_mode == BLKmode)
5734 if (bytes > 0 && bytes <= 8)
5735 mode = (bytes > 4 ? DImode : SImode);
5736 if (mode == BLKmode)
5740 return gen_reg_or_parallel (mode, orig_mode, regno);
5744 function_arg (CUMULATIVE_ARGS *cum, enum machine_mode omode,
5745 tree type, int named)
5747 enum machine_mode mode = omode;
5748 HOST_WIDE_INT bytes, words;
5750 if (mode == BLKmode)
5751 bytes = int_size_in_bytes (type);
5753 bytes = GET_MODE_SIZE (mode);
5754 words = (bytes + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
5756 /* To simplify the code below, represent vector types with a vector mode
5757 even if MMX/SSE are not active. */
5758 if (type && TREE_CODE (type) == VECTOR_TYPE)
5759 mode = type_natural_mode (type);
5761 if (TARGET_64BIT && (cum ? cum->call_abi : DEFAULT_ABI) == MS_ABI)
5762 return function_arg_ms_64 (cum, mode, omode, named, bytes);
5763 else if (TARGET_64BIT)
5764 return function_arg_64 (cum, mode, omode, type, named);
5766 return function_arg_32 (cum, mode, omode, type, bytes, words);
5769 /* A C expression that indicates when an argument must be passed by
5770 reference. If nonzero for an argument, a copy of that argument is
5771 made in memory and a pointer to the argument is passed instead of
5772 the argument itself. The pointer is passed in whatever way is
5773 appropriate for passing a pointer to that type. */
5776 ix86_pass_by_reference (CUMULATIVE_ARGS *cum ATTRIBUTE_UNUSED,
5777 enum machine_mode mode ATTRIBUTE_UNUSED,
5778 const_tree type, bool named ATTRIBUTE_UNUSED)
5780 /* See Windows x64 Software Convention. */
5781 if (TARGET_64BIT && (cum ? cum->call_abi : DEFAULT_ABI) == MS_ABI)
5783 int msize = (int) GET_MODE_SIZE (mode);
5786 /* Arrays are passed by reference. */
5787 if (TREE_CODE (type) == ARRAY_TYPE)
5790 if (AGGREGATE_TYPE_P (type))
5792 /* Structs/unions of sizes other than 8, 16, 32, or 64 bits
5793 are passed by reference. */
5794 msize = int_size_in_bytes (type);
5798 /* __m128 is passed by reference. */
5800 case 1: case 2: case 4: case 8:
5806 else if (TARGET_64BIT && type && int_size_in_bytes (type) == -1)
5812 /* Return true when TYPE should be 128bit aligned for 32bit argument passing
5815 contains_aligned_value_p (tree type)
5817 enum machine_mode mode = TYPE_MODE (type);
5818 if (((TARGET_SSE && SSE_REG_MODE_P (mode))
5822 && (!TYPE_USER_ALIGN (type) || TYPE_ALIGN (type) > 128))
5824 if (TYPE_ALIGN (type) < 128)
5827 if (AGGREGATE_TYPE_P (type))
5829 /* Walk the aggregates recursively. */
5830 switch (TREE_CODE (type))
5834 case QUAL_UNION_TYPE:
5838 /* Walk all the structure fields. */
5839 for (field = TYPE_FIELDS (type); field; field = TREE_CHAIN (field))
5841 if (TREE_CODE (field) == FIELD_DECL
5842 && contains_aligned_value_p (TREE_TYPE (field)))
5849 /* Just for use if some languages passes arrays by value. */
5850 if (contains_aligned_value_p (TREE_TYPE (type)))
5861 /* Gives the alignment boundary, in bits, of an argument with the
5862 specified mode and type. */
5865 ix86_function_arg_boundary (enum machine_mode mode, tree type)
5870 /* Since canonical type is used for call, we convert it to
5871 canonical type if needed. */
5872 if (!TYPE_STRUCTURAL_EQUALITY_P (type))
5873 type = TYPE_CANONICAL (type);
5874 align = TYPE_ALIGN (type);
5877 align = GET_MODE_ALIGNMENT (mode);
5878 if (align < PARM_BOUNDARY)
5879 align = PARM_BOUNDARY;
5880 /* In 32bit, only _Decimal128 and __float128 are aligned to their
5881 natural boundaries. */
5882 if (!TARGET_64BIT && mode != TDmode && mode != TFmode)
5884 /* i386 ABI defines all arguments to be 4 byte aligned. We have to
5885 make an exception for SSE modes since these require 128bit
5888 The handling here differs from field_alignment. ICC aligns MMX
5889 arguments to 4 byte boundaries, while structure fields are aligned
5890 to 8 byte boundaries. */
5893 if (!(TARGET_SSE && SSE_REG_MODE_P (mode)))
5894 align = PARM_BOUNDARY;
5898 if (!contains_aligned_value_p (type))
5899 align = PARM_BOUNDARY;
5902 if (align > BIGGEST_ALIGNMENT)
5903 align = BIGGEST_ALIGNMENT;
5907 /* Return true if N is a possible register number of function value. */
5910 ix86_function_value_regno_p (int regno)
5917 case FIRST_FLOAT_REG:
5918 /* TODO: The function should depend on current function ABI but
5919 builtins.c would need updating then. Therefore we use the
5921 if (TARGET_64BIT && DEFAULT_ABI == MS_ABI)
5923 return TARGET_FLOAT_RETURNS_IN_80387;
5929 if (TARGET_MACHO || TARGET_64BIT)
5937 /* Define how to find the value returned by a function.
5938 VALTYPE is the data type of the value (as a tree).
5939 If the precise function being called is known, FUNC is its FUNCTION_DECL;
5940 otherwise, FUNC is 0. */
5943 function_value_32 (enum machine_mode orig_mode, enum machine_mode mode,
5944 const_tree fntype, const_tree fn)
5948 /* 8-byte vector modes in %mm0. See ix86_return_in_memory for where
5949 we normally prevent this case when mmx is not available. However
5950 some ABIs may require the result to be returned like DImode. */
5951 if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 8)
5952 regno = TARGET_MMX ? FIRST_MMX_REG : 0;
5954 /* 16-byte vector modes in %xmm0. See ix86_return_in_memory for where
5955 we prevent this case when sse is not available. However some ABIs
5956 may require the result to be returned like integer TImode. */
5957 else if (mode == TImode
5958 || (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 16))
5959 regno = TARGET_SSE ? FIRST_SSE_REG : 0;
5961 /* Floating point return values in %st(0) (unless -mno-fp-ret-in-387). */
5962 else if (X87_FLOAT_MODE_P (mode) && TARGET_FLOAT_RETURNS_IN_80387)
5963 regno = FIRST_FLOAT_REG;
5965 /* Most things go in %eax. */
5968 /* Override FP return register with %xmm0 for local functions when
5969 SSE math is enabled or for functions with sseregparm attribute. */
5970 if ((fn || fntype) && (mode == SFmode || mode == DFmode))
5972 int sse_level = ix86_function_sseregparm (fntype, fn, false);
5973 if ((sse_level >= 1 && mode == SFmode)
5974 || (sse_level == 2 && mode == DFmode))
5975 regno = FIRST_SSE_REG;
5978 return gen_rtx_REG (orig_mode, regno);
5982 function_value_64 (enum machine_mode orig_mode, enum machine_mode mode,
5987 /* Handle libcalls, which don't provide a type node. */
5988 if (valtype == NULL)
6000 return gen_rtx_REG (mode, FIRST_SSE_REG);
6003 return gen_rtx_REG (mode, FIRST_FLOAT_REG);
6007 return gen_rtx_REG (mode, AX_REG);
6011 ret = construct_container (mode, orig_mode, valtype, 1,
6012 X86_64_REGPARM_MAX, X86_64_SSE_REGPARM_MAX,
6013 x86_64_int_return_registers, 0);
6015 /* For zero sized structures, construct_container returns NULL, but we
6016 need to keep rest of compiler happy by returning meaningful value. */
6018 ret = gen_rtx_REG (orig_mode, AX_REG);
6024 function_value_ms_64 (enum machine_mode orig_mode, enum machine_mode mode)
6026 unsigned int regno = AX_REG;
6030 switch (GET_MODE_SIZE (mode))
6033 if((SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
6034 && !COMPLEX_MODE_P (mode))
6035 regno = FIRST_SSE_REG;
6039 if (mode == SFmode || mode == DFmode)
6040 regno = FIRST_SSE_REG;
6046 return gen_rtx_REG (orig_mode, regno);
6050 ix86_function_value_1 (const_tree valtype, const_tree fntype_or_decl,
6051 enum machine_mode orig_mode, enum machine_mode mode)
6053 const_tree fn, fntype;
6056 if (fntype_or_decl && DECL_P (fntype_or_decl))
6057 fn = fntype_or_decl;
6058 fntype = fn ? TREE_TYPE (fn) : fntype_or_decl;
6060 if (TARGET_64BIT && ix86_function_type_abi (fntype) == MS_ABI)
6061 return function_value_ms_64 (orig_mode, mode);
6062 else if (TARGET_64BIT)
6063 return function_value_64 (orig_mode, mode, valtype);
6065 return function_value_32 (orig_mode, mode, fntype, fn);
6069 ix86_function_value (const_tree valtype, const_tree fntype_or_decl,
6070 bool outgoing ATTRIBUTE_UNUSED)
6072 enum machine_mode mode, orig_mode;
6074 orig_mode = TYPE_MODE (valtype);
6075 mode = type_natural_mode (valtype);
6076 return ix86_function_value_1 (valtype, fntype_or_decl, orig_mode, mode);
6080 ix86_libcall_value (enum machine_mode mode)
6082 return ix86_function_value_1 (NULL, NULL, mode, mode);
6085 /* Return true iff type is returned in memory. */
6087 static int ATTRIBUTE_UNUSED
6088 return_in_memory_32 (const_tree type, enum machine_mode mode)
6092 if (mode == BLKmode)
6095 size = int_size_in_bytes (type);
6097 if (MS_AGGREGATE_RETURN && AGGREGATE_TYPE_P (type) && size <= 8)
6100 if (VECTOR_MODE_P (mode) || mode == TImode)
6102 /* User-created vectors small enough to fit in EAX. */
6106 /* MMX/3dNow values are returned in MM0,
6107 except when it doesn't exits. */
6109 return (TARGET_MMX ? 0 : 1);
6111 /* SSE values are returned in XMM0, except when it doesn't exist. */
6113 return (TARGET_SSE ? 0 : 1);
6124 static int ATTRIBUTE_UNUSED
6125 return_in_memory_64 (const_tree type, enum machine_mode mode)
6127 int needed_intregs, needed_sseregs;
6128 return !examine_argument (mode, type, 1, &needed_intregs, &needed_sseregs);
6131 static int ATTRIBUTE_UNUSED
6132 return_in_memory_ms_64 (const_tree type, enum machine_mode mode)
6134 HOST_WIDE_INT size = int_size_in_bytes (type);
6136 /* __m128 is returned in xmm0. */
6137 if ((SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
6138 && !COMPLEX_MODE_P (mode) && (GET_MODE_SIZE (mode) == 16 || size == 16))
6141 /* Otherwise, the size must be exactly in [1248]. */
6142 return (size != 1 && size != 2 && size != 4 && size != 8);
6146 ix86_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
6148 #ifdef SUBTARGET_RETURN_IN_MEMORY
6149 return SUBTARGET_RETURN_IN_MEMORY (type, fntype);
6151 const enum machine_mode mode = type_natural_mode (type);
6153 if (TARGET_64BIT_MS_ABI)
6154 return return_in_memory_ms_64 (type, mode);
6155 else if (TARGET_64BIT)
6156 return return_in_memory_64 (type, mode);
6158 return return_in_memory_32 (type, mode);
6162 /* Return false iff TYPE is returned in memory. This version is used
6163 on Solaris 10. It is similar to the generic ix86_return_in_memory,
6164 but differs notably in that when MMX is available, 8-byte vectors
6165 are returned in memory, rather than in MMX registers. */
6168 ix86_sol10_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
6171 enum machine_mode mode = type_natural_mode (type);
6174 return return_in_memory_64 (type, mode);
6176 if (mode == BLKmode)
6179 size = int_size_in_bytes (type);
6181 if (VECTOR_MODE_P (mode))
6183 /* Return in memory only if MMX registers *are* available. This
6184 seems backwards, but it is consistent with the existing
6191 else if (mode == TImode)
6193 else if (mode == XFmode)
6199 /* When returning SSE vector types, we have a choice of either
6200 (1) being abi incompatible with a -march switch, or
6201 (2) generating an error.
6202 Given no good solution, I think the safest thing is one warning.
6203 The user won't be able to use -Werror, but....
6205 Choose the STRUCT_VALUE_RTX hook because that's (at present) only
6206 called in response to actually generating a caller or callee that
6207 uses such a type. As opposed to TARGET_RETURN_IN_MEMORY, which is called
6208 via aggregate_value_p for general type probing from tree-ssa. */
6211 ix86_struct_value_rtx (tree type, int incoming ATTRIBUTE_UNUSED)
6213 static bool warnedsse, warnedmmx;
6215 if (!TARGET_64BIT && type)
6217 /* Look at the return type of the function, not the function type. */
6218 enum machine_mode mode = TYPE_MODE (TREE_TYPE (type));
6220 if (!TARGET_SSE && !warnedsse)
6223 || (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 16))
6226 warning (0, "SSE vector return without SSE enabled "
6231 if (!TARGET_MMX && !warnedmmx)
6233 if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 8)
6236 warning (0, "MMX vector return without MMX enabled "
6246 /* Create the va_list data type. */
6248 /* Returns the calling convention specific va_list date type.
6249 The argument ABI can be DEFAULT_ABI, MS_ABI, or SYSV_ABI. */
6252 ix86_build_builtin_va_list_abi (enum calling_abi abi)
6254 tree f_gpr, f_fpr, f_ovf, f_sav, record, type_decl;
6256 /* For i386 we use plain pointer to argument area. */
6257 if (!TARGET_64BIT || abi == MS_ABI)
6258 return build_pointer_type (char_type_node);
6260 record = (*lang_hooks.types.make_type) (RECORD_TYPE);
6261 type_decl = build_decl (TYPE_DECL, get_identifier ("__va_list_tag"), record);
6263 f_gpr = build_decl (FIELD_DECL, get_identifier ("gp_offset"),
6264 unsigned_type_node);
6265 f_fpr = build_decl (FIELD_DECL, get_identifier ("fp_offset"),
6266 unsigned_type_node);
6267 f_ovf = build_decl (FIELD_DECL, get_identifier ("overflow_arg_area"),
6269 f_sav = build_decl (FIELD_DECL, get_identifier ("reg_save_area"),
6272 va_list_gpr_counter_field = f_gpr;
6273 va_list_fpr_counter_field = f_fpr;
6275 DECL_FIELD_CONTEXT (f_gpr) = record;
6276 DECL_FIELD_CONTEXT (f_fpr) = record;
6277 DECL_FIELD_CONTEXT (f_ovf) = record;
6278 DECL_FIELD_CONTEXT (f_sav) = record;
6280 TREE_CHAIN (record) = type_decl;
6281 TYPE_NAME (record) = type_decl;
6282 TYPE_FIELDS (record) = f_gpr;
6283 TREE_CHAIN (f_gpr) = f_fpr;
6284 TREE_CHAIN (f_fpr) = f_ovf;
6285 TREE_CHAIN (f_ovf) = f_sav;
6287 layout_type (record);
6289 /* The correct type is an array type of one element. */
6290 return build_array_type (record, build_index_type (size_zero_node));
6293 /* Setup the builtin va_list data type and for 64-bit the additional
6294 calling convention specific va_list data types. */
6297 ix86_build_builtin_va_list (void)
6299 tree ret = ix86_build_builtin_va_list_abi (DEFAULT_ABI);
6301 /* Initialize abi specific va_list builtin types. */
6305 if (DEFAULT_ABI == MS_ABI)
6307 t = ix86_build_builtin_va_list_abi (SYSV_ABI);
6308 if (TREE_CODE (t) != RECORD_TYPE)
6309 t = build_variant_type_copy (t);
6310 sysv_va_list_type_node = t;
6315 if (TREE_CODE (t) != RECORD_TYPE)
6316 t = build_variant_type_copy (t);
6317 sysv_va_list_type_node = t;
6319 if (DEFAULT_ABI != MS_ABI)
6321 t = ix86_build_builtin_va_list_abi (MS_ABI);
6322 if (TREE_CODE (t) != RECORD_TYPE)
6323 t = build_variant_type_copy (t);
6324 ms_va_list_type_node = t;
6329 if (TREE_CODE (t) != RECORD_TYPE)
6330 t = build_variant_type_copy (t);
6331 ms_va_list_type_node = t;
6338 /* Worker function for TARGET_SETUP_INCOMING_VARARGS. */
6341 setup_incoming_varargs_64 (CUMULATIVE_ARGS *cum)
6350 int regparm = ix86_regparm;
6352 if (cum->call_abi != DEFAULT_ABI)
6353 regparm = DEFAULT_ABI != SYSV_ABI ? X86_64_REGPARM_MAX : X64_REGPARM_MAX;
6355 /* GPR size of varargs save area. */
6356 if (cfun->va_list_gpr_size)
6357 ix86_varargs_gpr_size = X86_64_REGPARM_MAX * UNITS_PER_WORD;
6359 ix86_varargs_gpr_size = 0;
6361 /* FPR size of varargs save area. We don't need it if we don't pass
6362 anything in SSE registers. */
6363 if (cum->sse_nregs && cfun->va_list_fpr_size)
6364 ix86_varargs_fpr_size = X86_64_SSE_REGPARM_MAX * 16;
6366 ix86_varargs_fpr_size = 0;
6368 if (! ix86_varargs_gpr_size && ! ix86_varargs_fpr_size)
6371 save_area = frame_pointer_rtx;
6372 set = get_varargs_alias_set ();
6374 for (i = cum->regno;
6376 && i < cum->regno + cfun->va_list_gpr_size / UNITS_PER_WORD;
6379 mem = gen_rtx_MEM (Pmode,
6380 plus_constant (save_area, i * UNITS_PER_WORD));
6381 MEM_NOTRAP_P (mem) = 1;
6382 set_mem_alias_set (mem, set);
6383 emit_move_insn (mem, gen_rtx_REG (Pmode,
6384 x86_64_int_parameter_registers[i]));
6387 if (ix86_varargs_fpr_size)
6389 /* Now emit code to save SSE registers. The AX parameter contains number
6390 of SSE parameter registers used to call this function. We use
6391 sse_prologue_save insn template that produces computed jump across
6392 SSE saves. We need some preparation work to get this working. */
6394 label = gen_label_rtx ();
6395 label_ref = gen_rtx_LABEL_REF (Pmode, label);
6397 /* Compute address to jump to :
6398 label - eax*4 + nnamed_sse_arguments*4 Or
6399 label - eax*5 + nnamed_sse_arguments*5 for AVX. */
6400 tmp_reg = gen_reg_rtx (Pmode);
6401 nsse_reg = gen_reg_rtx (Pmode);
6402 emit_insn (gen_zero_extendqidi2 (nsse_reg, gen_rtx_REG (QImode, AX_REG)));
6403 emit_insn (gen_rtx_SET (VOIDmode, tmp_reg,
6404 gen_rtx_MULT (Pmode, nsse_reg,
6407 /* vmovaps is one byte longer than movaps. */
6409 emit_insn (gen_rtx_SET (VOIDmode, tmp_reg,
6410 gen_rtx_PLUS (Pmode, tmp_reg,
6416 gen_rtx_CONST (DImode,
6417 gen_rtx_PLUS (DImode,
6419 GEN_INT (cum->sse_regno
6420 * (TARGET_AVX ? 5 : 4)))));
6422 emit_move_insn (nsse_reg, label_ref);
6423 emit_insn (gen_subdi3 (nsse_reg, nsse_reg, tmp_reg));
6425 /* Compute address of memory block we save into. We always use pointer
6426 pointing 127 bytes after first byte to store - this is needed to keep
6427 instruction size limited by 4 bytes (5 bytes for AVX) with one
6428 byte displacement. */
6429 tmp_reg = gen_reg_rtx (Pmode);
6430 emit_insn (gen_rtx_SET (VOIDmode, tmp_reg,
6431 plus_constant (save_area,
6432 ix86_varargs_gpr_size + 127)));
6433 mem = gen_rtx_MEM (BLKmode, plus_constant (tmp_reg, -127));
6434 MEM_NOTRAP_P (mem) = 1;
6435 set_mem_alias_set (mem, set);
6436 set_mem_align (mem, BITS_PER_WORD);
6438 /* And finally do the dirty job! */
6439 emit_insn (gen_sse_prologue_save (mem, nsse_reg,
6440 GEN_INT (cum->sse_regno), label));
6445 setup_incoming_varargs_ms_64 (CUMULATIVE_ARGS *cum)
6447 alias_set_type set = get_varargs_alias_set ();
6450 for (i = cum->regno; i < X64_REGPARM_MAX; i++)
6454 mem = gen_rtx_MEM (Pmode,
6455 plus_constant (virtual_incoming_args_rtx,
6456 i * UNITS_PER_WORD));
6457 MEM_NOTRAP_P (mem) = 1;
6458 set_mem_alias_set (mem, set);
6460 reg = gen_rtx_REG (Pmode, x86_64_ms_abi_int_parameter_registers[i]);
6461 emit_move_insn (mem, reg);
6466 ix86_setup_incoming_varargs (CUMULATIVE_ARGS *cum, enum machine_mode mode,
6467 tree type, int *pretend_size ATTRIBUTE_UNUSED,
6470 CUMULATIVE_ARGS next_cum;
6473 /* This argument doesn't appear to be used anymore. Which is good,
6474 because the old code here didn't suppress rtl generation. */
6475 gcc_assert (!no_rtl);
6480 fntype = TREE_TYPE (current_function_decl);
6482 /* For varargs, we do not want to skip the dummy va_dcl argument.
6483 For stdargs, we do want to skip the last named argument. */
6485 if (stdarg_p (fntype))
6486 function_arg_advance (&next_cum, mode, type, 1);
6488 if (cum->call_abi == MS_ABI)
6489 setup_incoming_varargs_ms_64 (&next_cum);
6491 setup_incoming_varargs_64 (&next_cum);
6494 /* Checks if TYPE is of kind va_list char *. */
6497 is_va_list_char_pointer (tree type)
6501 /* For 32-bit it is always true. */
6504 canonic = ix86_canonical_va_list_type (type);
6505 return (canonic == ms_va_list_type_node
6506 || (DEFAULT_ABI == MS_ABI && canonic == va_list_type_node));
6509 /* Implement va_start. */
6512 ix86_va_start (tree valist, rtx nextarg)
6514 HOST_WIDE_INT words, n_gpr, n_fpr;
6515 tree f_gpr, f_fpr, f_ovf, f_sav;
6516 tree gpr, fpr, ovf, sav, t;
6519 /* Only 64bit target needs something special. */
6520 if (!TARGET_64BIT || is_va_list_char_pointer (TREE_TYPE (valist)))
6522 std_expand_builtin_va_start (valist, nextarg);
6526 f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node));
6527 f_fpr = TREE_CHAIN (f_gpr);
6528 f_ovf = TREE_CHAIN (f_fpr);
6529 f_sav = TREE_CHAIN (f_ovf);
6531 valist = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (valist)), valist);
6532 gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr), valist, f_gpr, NULL_TREE);
6533 fpr = build3 (COMPONENT_REF, TREE_TYPE (f_fpr), valist, f_fpr, NULL_TREE);
6534 ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), valist, f_ovf, NULL_TREE);
6535 sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), valist, f_sav, NULL_TREE);
6537 /* Count number of gp and fp argument registers used. */
6538 words = crtl->args.info.words;
6539 n_gpr = crtl->args.info.regno;
6540 n_fpr = crtl->args.info.sse_regno;
6542 if (cfun->va_list_gpr_size)
6544 type = TREE_TYPE (gpr);
6545 t = build2 (MODIFY_EXPR, type,
6546 gpr, build_int_cst (type, n_gpr * 8));
6547 TREE_SIDE_EFFECTS (t) = 1;
6548 expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
6551 if (TARGET_SSE && cfun->va_list_fpr_size)
6553 type = TREE_TYPE (fpr);
6554 t = build2 (MODIFY_EXPR, type, fpr,
6555 build_int_cst (type, n_fpr * 16 + 8*X86_64_REGPARM_MAX));
6556 TREE_SIDE_EFFECTS (t) = 1;
6557 expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
6560 /* Find the overflow area. */
6561 type = TREE_TYPE (ovf);
6562 t = make_tree (type, crtl->args.internal_arg_pointer);
6564 t = build2 (POINTER_PLUS_EXPR, type, t,
6565 size_int (words * UNITS_PER_WORD));
6566 t = build2 (MODIFY_EXPR, type, ovf, t);
6567 TREE_SIDE_EFFECTS (t) = 1;
6568 expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
6570 if (ix86_varargs_gpr_size || ix86_varargs_fpr_size)
6572 /* Find the register save area.
6573 Prologue of the function save it right above stack frame. */
6574 type = TREE_TYPE (sav);
6575 t = make_tree (type, frame_pointer_rtx);
6576 if (!ix86_varargs_gpr_size)
6577 t = build2 (POINTER_PLUS_EXPR, type, t,
6578 size_int (-8 * X86_64_REGPARM_MAX));
6579 t = build2 (MODIFY_EXPR, type, sav, t);
6580 TREE_SIDE_EFFECTS (t) = 1;
6581 expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
6585 /* Implement va_arg. */
6588 ix86_gimplify_va_arg (tree valist, tree type, gimple_seq *pre_p,
6591 static const int intreg[6] = { 0, 1, 2, 3, 4, 5 };
6592 tree f_gpr, f_fpr, f_ovf, f_sav;
6593 tree gpr, fpr, ovf, sav, t;
6595 tree lab_false, lab_over = NULL_TREE;
6600 enum machine_mode nat_mode;
6603 /* Only 64bit target needs something special. */
6604 if (!TARGET_64BIT || is_va_list_char_pointer (TREE_TYPE (valist)))
6605 return std_gimplify_va_arg_expr (valist, type, pre_p, post_p);
6607 f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node));
6608 f_fpr = TREE_CHAIN (f_gpr);
6609 f_ovf = TREE_CHAIN (f_fpr);
6610 f_sav = TREE_CHAIN (f_ovf);
6612 gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr),
6613 build_va_arg_indirect_ref (valist), f_gpr, NULL_TREE);
6614 valist = build_va_arg_indirect_ref (valist);
6615 fpr = build3 (COMPONENT_REF, TREE_TYPE (f_fpr), valist, f_fpr, NULL_TREE);
6616 ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), valist, f_ovf, NULL_TREE);
6617 sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), valist, f_sav, NULL_TREE);
6619 indirect_p = pass_by_reference (NULL, TYPE_MODE (type), type, false);
6621 type = build_pointer_type (type);
6622 size = int_size_in_bytes (type);
6623 rsize = (size + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
6625 nat_mode = type_natural_mode (type);
6634 /* Unnamed 256bit vector mode parameters are passed on stack. */
6635 if (ix86_cfun_abi () == SYSV_ABI)
6642 container = construct_container (nat_mode, TYPE_MODE (type),
6643 type, 0, X86_64_REGPARM_MAX,
6644 X86_64_SSE_REGPARM_MAX, intreg,
6649 /* Pull the value out of the saved registers. */
6651 addr = create_tmp_var (ptr_type_node, "addr");
6652 DECL_POINTER_ALIAS_SET (addr) = get_varargs_alias_set ();
6656 int needed_intregs, needed_sseregs;
6658 tree int_addr, sse_addr;
6660 lab_false = create_artificial_label ();
6661 lab_over = create_artificial_label ();
6663 examine_argument (nat_mode, type, 0, &needed_intregs, &needed_sseregs);
6665 need_temp = (!REG_P (container)
6666 && ((needed_intregs && TYPE_ALIGN (type) > 64)
6667 || TYPE_ALIGN (type) > 128));
6669 /* In case we are passing structure, verify that it is consecutive block
6670 on the register save area. If not we need to do moves. */
6671 if (!need_temp && !REG_P (container))
6673 /* Verify that all registers are strictly consecutive */
6674 if (SSE_REGNO_P (REGNO (XEXP (XVECEXP (container, 0, 0), 0))))
6678 for (i = 0; i < XVECLEN (container, 0) && !need_temp; i++)
6680 rtx slot = XVECEXP (container, 0, i);
6681 if (REGNO (XEXP (slot, 0)) != FIRST_SSE_REG + (unsigned int) i
6682 || INTVAL (XEXP (slot, 1)) != i * 16)
6690 for (i = 0; i < XVECLEN (container, 0) && !need_temp; i++)
6692 rtx slot = XVECEXP (container, 0, i);
6693 if (REGNO (XEXP (slot, 0)) != (unsigned int) i
6694 || INTVAL (XEXP (slot, 1)) != i * 8)
6706 int_addr = create_tmp_var (ptr_type_node, "int_addr");
6707 DECL_POINTER_ALIAS_SET (int_addr) = get_varargs_alias_set ();
6708 sse_addr = create_tmp_var (ptr_type_node, "sse_addr");
6709 DECL_POINTER_ALIAS_SET (sse_addr) = get_varargs_alias_set ();
6712 /* First ensure that we fit completely in registers. */
6715 t = build_int_cst (TREE_TYPE (gpr),
6716 (X86_64_REGPARM_MAX - needed_intregs + 1) * 8);
6717 t = build2 (GE_EXPR, boolean_type_node, gpr, t);
6718 t2 = build1 (GOTO_EXPR, void_type_node, lab_false);
6719 t = build3 (COND_EXPR, void_type_node, t, t2, NULL_TREE);
6720 gimplify_and_add (t, pre_p);
6724 t = build_int_cst (TREE_TYPE (fpr),
6725 (X86_64_SSE_REGPARM_MAX - needed_sseregs + 1) * 16
6726 + X86_64_REGPARM_MAX * 8);
6727 t = build2 (GE_EXPR, boolean_type_node, fpr, t);
6728 t2 = build1 (GOTO_EXPR, void_type_node, lab_false);
6729 t = build3 (COND_EXPR, void_type_node, t, t2, NULL_TREE);
6730 gimplify_and_add (t, pre_p);
6733 /* Compute index to start of area used for integer regs. */
6736 /* int_addr = gpr + sav; */
6737 t = fold_convert (sizetype, gpr);
6738 t = build2 (POINTER_PLUS_EXPR, ptr_type_node, sav, t);
6739 gimplify_assign (int_addr, t, pre_p);
6743 /* sse_addr = fpr + sav; */
6744 t = fold_convert (sizetype, fpr);
6745 t = build2 (POINTER_PLUS_EXPR, ptr_type_node, sav, t);
6746 gimplify_assign (sse_addr, t, pre_p);
6751 tree temp = create_tmp_var (type, "va_arg_tmp");
6754 t = build1 (ADDR_EXPR, build_pointer_type (type), temp);
6755 gimplify_assign (addr, t, pre_p);
6757 for (i = 0; i < XVECLEN (container, 0); i++)
6759 rtx slot = XVECEXP (container, 0, i);
6760 rtx reg = XEXP (slot, 0);
6761 enum machine_mode mode = GET_MODE (reg);
6762 tree piece_type = lang_hooks.types.type_for_mode (mode, 1);
6763 tree addr_type = build_pointer_type (piece_type);
6764 tree daddr_type = build_pointer_type_for_mode (piece_type,
6768 tree dest_addr, dest;
6770 if (SSE_REGNO_P (REGNO (reg)))
6772 src_addr = sse_addr;
6773 src_offset = (REGNO (reg) - FIRST_SSE_REG) * 16;
6777 src_addr = int_addr;
6778 src_offset = REGNO (reg) * 8;
6780 src_addr = fold_convert (addr_type, src_addr);
6781 src_addr = fold_build2 (POINTER_PLUS_EXPR, addr_type, src_addr,
6782 size_int (src_offset));
6783 src = build_va_arg_indirect_ref (src_addr);
6785 dest_addr = fold_convert (daddr_type, addr);
6786 dest_addr = fold_build2 (POINTER_PLUS_EXPR, daddr_type, dest_addr,
6787 size_int (INTVAL (XEXP (slot, 1))));
6788 dest = build_va_arg_indirect_ref (dest_addr);
6790 gimplify_assign (dest, src, pre_p);
6796 t = build2 (PLUS_EXPR, TREE_TYPE (gpr), gpr,
6797 build_int_cst (TREE_TYPE (gpr), needed_intregs * 8));
6798 gimplify_assign (gpr, t, pre_p);
6803 t = build2 (PLUS_EXPR, TREE_TYPE (fpr), fpr,
6804 build_int_cst (TREE_TYPE (fpr), needed_sseregs * 16));
6805 gimplify_assign (fpr, t, pre_p);
6808 gimple_seq_add_stmt (pre_p, gimple_build_goto (lab_over));
6810 gimple_seq_add_stmt (pre_p, gimple_build_label (lab_false));
6813 /* ... otherwise out of the overflow area. */
6815 /* When we align parameter on stack for caller, if the parameter
6816 alignment is beyond MAX_SUPPORTED_STACK_ALIGNMENT, it will be
6817 aligned at MAX_SUPPORTED_STACK_ALIGNMENT. We will match callee
6818 here with caller. */
6819 arg_boundary = FUNCTION_ARG_BOUNDARY (VOIDmode, type);
6820 if ((unsigned int) arg_boundary > MAX_SUPPORTED_STACK_ALIGNMENT)
6821 arg_boundary = MAX_SUPPORTED_STACK_ALIGNMENT;
6823 /* Care for on-stack alignment if needed. */
6824 if (arg_boundary <= 64
6825 || integer_zerop (TYPE_SIZE (type)))
6829 HOST_WIDE_INT align = arg_boundary / 8;
6830 t = build2 (POINTER_PLUS_EXPR, TREE_TYPE (ovf), ovf,
6831 size_int (align - 1));
6832 t = fold_convert (sizetype, t);
6833 t = build2 (BIT_AND_EXPR, TREE_TYPE (t), t,
6835 t = fold_convert (TREE_TYPE (ovf), t);
6837 gimplify_expr (&t, pre_p, NULL, is_gimple_val, fb_rvalue);
6838 gimplify_assign (addr, t, pre_p);
6840 t = build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t,
6841 size_int (rsize * UNITS_PER_WORD));
6842 gimplify_assign (unshare_expr (ovf), t, pre_p);
6845 gimple_seq_add_stmt (pre_p, gimple_build_label (lab_over));
6847 ptrtype = build_pointer_type (type);
6848 addr = fold_convert (ptrtype, addr);
6851 addr = build_va_arg_indirect_ref (addr);
6852 return build_va_arg_indirect_ref (addr);
6855 /* Return nonzero if OPNUM's MEM should be matched
6856 in movabs* patterns. */
6859 ix86_check_movabs (rtx insn, int opnum)
6863 set = PATTERN (insn);
6864 if (GET_CODE (set) == PARALLEL)
6865 set = XVECEXP (set, 0, 0);
6866 gcc_assert (GET_CODE (set) == SET);
6867 mem = XEXP (set, opnum);
6868 while (GET_CODE (mem) == SUBREG)
6869 mem = SUBREG_REG (mem);
6870 gcc_assert (MEM_P (mem));
6871 return (volatile_ok || !MEM_VOLATILE_P (mem));
6874 /* Initialize the table of extra 80387 mathematical constants. */
6877 init_ext_80387_constants (void)
6879 static const char * cst[5] =
6881 "0.3010299956639811952256464283594894482", /* 0: fldlg2 */
6882 "0.6931471805599453094286904741849753009", /* 1: fldln2 */
6883 "1.4426950408889634073876517827983434472", /* 2: fldl2e */
6884 "3.3219280948873623478083405569094566090", /* 3: fldl2t */
6885 "3.1415926535897932385128089594061862044", /* 4: fldpi */
6889 for (i = 0; i < 5; i++)
6891 real_from_string (&ext_80387_constants_table[i], cst[i]);
6892 /* Ensure each constant is rounded to XFmode precision. */
6893 real_convert (&ext_80387_constants_table[i],
6894 XFmode, &ext_80387_constants_table[i]);
6897 ext_80387_constants_init = 1;
6900 /* Return true if the constant is something that can be loaded with
6901 a special instruction. */
6904 standard_80387_constant_p (rtx x)
6906 enum machine_mode mode = GET_MODE (x);
6910 if (!(X87_FLOAT_MODE_P (mode) && (GET_CODE (x) == CONST_DOUBLE)))
6913 if (x == CONST0_RTX (mode))
6915 if (x == CONST1_RTX (mode))
6918 REAL_VALUE_FROM_CONST_DOUBLE (r, x);
6920 /* For XFmode constants, try to find a special 80387 instruction when
6921 optimizing for size or on those CPUs that benefit from them. */
6923 && (optimize_function_for_size_p (cfun) || TARGET_EXT_80387_CONSTANTS))
6927 if (! ext_80387_constants_init)
6928 init_ext_80387_constants ();
6930 for (i = 0; i < 5; i++)
6931 if (real_identical (&r, &ext_80387_constants_table[i]))
6935 /* Load of the constant -0.0 or -1.0 will be split as
6936 fldz;fchs or fld1;fchs sequence. */
6937 if (real_isnegzero (&r))
6939 if (real_identical (&r, &dconstm1))
6945 /* Return the opcode of the special instruction to be used to load
6949 standard_80387_constant_opcode (rtx x)
6951 switch (standard_80387_constant_p (x))
6975 /* Return the CONST_DOUBLE representing the 80387 constant that is
6976 loaded by the specified special instruction. The argument IDX
6977 matches the return value from standard_80387_constant_p. */
6980 standard_80387_constant_rtx (int idx)
6984 if (! ext_80387_constants_init)
6985 init_ext_80387_constants ();
7001 return CONST_DOUBLE_FROM_REAL_VALUE (ext_80387_constants_table[i],
7005 /* Return 1 if mode is a valid mode for sse. */
7007 standard_sse_mode_p (enum machine_mode mode)
7024 /* Return 1 if X is all 0s. For all 1s, return 2 if X is in 128bit
7025 SSE modes and SSE2 is enabled, return 3 if X is in 256bit AVX
7026 modes and AVX is enabled. */
7029 standard_sse_constant_p (rtx x)
7031 enum machine_mode mode = GET_MODE (x);
7033 if (x == const0_rtx || x == CONST0_RTX (GET_MODE (x)))
7035 if (vector_all_ones_operand (x, mode))
7037 if (standard_sse_mode_p (mode))
7038 return TARGET_SSE2 ? 2 : -2;
7039 else if (VALID_AVX256_REG_MODE (mode))
7040 return TARGET_AVX ? 3 : -3;
7046 /* Return the opcode of the special instruction to be used to load
7050 standard_sse_constant_opcode (rtx insn, rtx x)
7052 switch (standard_sse_constant_p (x))
7055 switch (get_attr_mode (insn))
7058 return TARGET_AVX ? "vxorps\t%0, %0, %0" : "xorps\t%0, %0";
7060 return TARGET_AVX ? "vxorpd\t%0, %0, %0" : "xorpd\t%0, %0";
7062 return TARGET_AVX ? "vpxor\t%0, %0, %0" : "pxor\t%0, %0";
7064 return "vxorps\t%x0, %x0, %x0";
7066 return "vxorpd\t%x0, %x0, %x0";
7068 return "vpxor\t%x0, %x0, %x0";
7074 switch (get_attr_mode (insn))
7079 return "vpcmpeqd\t%0, %0, %0";
7085 return "pcmpeqd\t%0, %0";
7090 /* Returns 1 if OP contains a symbol reference */
7093 symbolic_reference_mentioned_p (rtx op)
7098 if (GET_CODE (op) == SYMBOL_REF || GET_CODE (op) == LABEL_REF)
7101 fmt = GET_RTX_FORMAT (GET_CODE (op));
7102 for (i = GET_RTX_LENGTH (GET_CODE (op)) - 1; i >= 0; i--)
7108 for (j = XVECLEN (op, i) - 1; j >= 0; j--)
7109 if (symbolic_reference_mentioned_p (XVECEXP (op, i, j)))
7113 else if (fmt[i] == 'e' && symbolic_reference_mentioned_p (XEXP (op, i)))
7120 /* Return 1 if it is appropriate to emit `ret' instructions in the
7121 body of a function. Do this only if the epilogue is simple, needing a
7122 couple of insns. Prior to reloading, we can't tell how many registers
7123 must be saved, so return 0 then. Return 0 if there is no frame
7124 marker to de-allocate. */
7127 ix86_can_use_return_insn_p (void)
7129 struct ix86_frame frame;
7131 if (! reload_completed || frame_pointer_needed)
7134 /* Don't allow more than 32 pop, since that's all we can do
7135 with one instruction. */
7136 if (crtl->args.pops_args
7137 && crtl->args.size >= 32768)
7140 ix86_compute_frame_layout (&frame);
7141 return frame.to_allocate == 0 && frame.nregs == 0;
7144 /* Value should be nonzero if functions must have frame pointers.
7145 Zero means the frame pointer need not be set up (and parms may
7146 be accessed via the stack pointer) in functions that seem suitable. */
7149 ix86_frame_pointer_required (void)
7151 /* If we accessed previous frames, then the generated code expects
7152 to be able to access the saved ebp value in our frame. */
7153 if (cfun->machine->accesses_prev_frame)
7156 /* Several x86 os'es need a frame pointer for other reasons,
7157 usually pertaining to setjmp. */
7158 if (SUBTARGET_FRAME_POINTER_REQUIRED)
7161 /* In override_options, TARGET_OMIT_LEAF_FRAME_POINTER turns off
7162 the frame pointer by default. Turn it back on now if we've not
7163 got a leaf function. */
7164 if (TARGET_OMIT_LEAF_FRAME_POINTER
7165 && (!current_function_is_leaf
7166 || ix86_current_function_calls_tls_descriptor))
7175 /* Record that the current function accesses previous call frames. */
7178 ix86_setup_frame_addresses (void)
7180 cfun->machine->accesses_prev_frame = 1;
7183 #if (defined(HAVE_GAS_HIDDEN) && (SUPPORTS_ONE_ONLY - 0)) || TARGET_MACHO
7184 # define USE_HIDDEN_LINKONCE 1
7186 # define USE_HIDDEN_LINKONCE 0
7189 static int pic_labels_used;
7191 /* Fills in the label name that should be used for a pc thunk for
7192 the given register. */
7195 get_pc_thunk_name (char name[32], unsigned int regno)
7197 gcc_assert (!TARGET_64BIT);
7199 if (USE_HIDDEN_LINKONCE)
7200 sprintf (name, "__i686.get_pc_thunk.%s", reg_names[regno]);
7202 ASM_GENERATE_INTERNAL_LABEL (name, "LPR", regno);
7206 /* This function generates code for -fpic that loads %ebx with
7207 the return address of the caller and then returns. */
7210 ix86_file_end (void)
7215 for (regno = 0; regno < 8; ++regno)
7219 if (! ((pic_labels_used >> regno) & 1))
7222 get_pc_thunk_name (name, regno);
7227 switch_to_section (darwin_sections[text_coal_section]);
7228 fputs ("\t.weak_definition\t", asm_out_file);
7229 assemble_name (asm_out_file, name);
7230 fputs ("\n\t.private_extern\t", asm_out_file);
7231 assemble_name (asm_out_file, name);
7232 fputs ("\n", asm_out_file);
7233 ASM_OUTPUT_LABEL (asm_out_file, name);
7237 if (USE_HIDDEN_LINKONCE)
7241 decl = build_decl (FUNCTION_DECL, get_identifier (name),
7243 TREE_PUBLIC (decl) = 1;
7244 TREE_STATIC (decl) = 1;
7245 DECL_ONE_ONLY (decl) = 1;
7247 (*targetm.asm_out.unique_section) (decl, 0);
7248 switch_to_section (get_named_section (decl, NULL, 0));
7250 (*targetm.asm_out.globalize_label) (asm_out_file, name);
7251 fputs ("\t.hidden\t", asm_out_file);
7252 assemble_name (asm_out_file, name);
7253 fputc ('\n', asm_out_file);
7254 ASM_DECLARE_FUNCTION_NAME (asm_out_file, name, decl);
7258 switch_to_section (text_section);
7259 ASM_OUTPUT_LABEL (asm_out_file, name);
7262 xops[0] = gen_rtx_REG (Pmode, regno);
7263 xops[1] = gen_rtx_MEM (Pmode, stack_pointer_rtx);
7264 output_asm_insn ("mov%z0\t{%1, %0|%0, %1}", xops);
7265 output_asm_insn ("ret", xops);
7268 if (NEED_INDICATE_EXEC_STACK)
7269 file_end_indicate_exec_stack ();
7272 /* Emit code for the SET_GOT patterns. */
7275 output_set_got (rtx dest, rtx label ATTRIBUTE_UNUSED)
7281 if (TARGET_VXWORKS_RTP && flag_pic)
7283 /* Load (*VXWORKS_GOTT_BASE) into the PIC register. */
7284 xops[2] = gen_rtx_MEM (Pmode,
7285 gen_rtx_SYMBOL_REF (Pmode, VXWORKS_GOTT_BASE));
7286 output_asm_insn ("mov{l}\t{%2, %0|%0, %2}", xops);
7288 /* Load (*VXWORKS_GOTT_BASE)[VXWORKS_GOTT_INDEX] into the PIC register.
7289 Use %P and a local symbol in order to print VXWORKS_GOTT_INDEX as
7290 an unadorned address. */
7291 xops[2] = gen_rtx_SYMBOL_REF (Pmode, VXWORKS_GOTT_INDEX);
7292 SYMBOL_REF_FLAGS (xops[2]) |= SYMBOL_FLAG_LOCAL;
7293 output_asm_insn ("mov{l}\t{%P2(%0), %0|%0, DWORD PTR %P2[%0]}", xops);
7297 xops[1] = gen_rtx_SYMBOL_REF (Pmode, GOT_SYMBOL_NAME);
7299 if (! TARGET_DEEP_BRANCH_PREDICTION || !flag_pic)
7301 xops[2] = gen_rtx_LABEL_REF (Pmode, label ? label : gen_label_rtx ());
7304 output_asm_insn ("mov%z0\t{%2, %0|%0, %2}", xops);
7306 output_asm_insn ("call\t%a2", xops);
7309 /* Output the Mach-O "canonical" label name ("Lxx$pb") here too. This
7310 is what will be referenced by the Mach-O PIC subsystem. */
7312 ASM_OUTPUT_LABEL (asm_out_file, MACHOPIC_FUNCTION_BASE_NAME);
7315 (*targetm.asm_out.internal_label) (asm_out_file, "L",
7316 CODE_LABEL_NUMBER (XEXP (xops[2], 0)));
7319 output_asm_insn ("pop%z0\t%0", xops);
7324 get_pc_thunk_name (name, REGNO (dest));
7325 pic_labels_used |= 1 << REGNO (dest);
7327 xops[2] = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (name));
7328 xops[2] = gen_rtx_MEM (QImode, xops[2]);
7329 output_asm_insn ("call\t%X2", xops);
7330 /* Output the Mach-O "canonical" label name ("Lxx$pb") here too. This
7331 is what will be referenced by the Mach-O PIC subsystem. */
7334 ASM_OUTPUT_LABEL (asm_out_file, MACHOPIC_FUNCTION_BASE_NAME);
7336 targetm.asm_out.internal_label (asm_out_file, "L",
7337 CODE_LABEL_NUMBER (label));
7344 if (!flag_pic || TARGET_DEEP_BRANCH_PREDICTION)
7345 output_asm_insn ("add%z0\t{%1, %0|%0, %1}", xops);
7347 output_asm_insn ("add%z0\t{%1+[.-%a2], %0|%0, %1+(.-%a2)}", xops);
7352 /* Generate an "push" pattern for input ARG. */
7357 return gen_rtx_SET (VOIDmode,
7359 gen_rtx_PRE_DEC (Pmode,
7360 stack_pointer_rtx)),
7364 /* Return >= 0 if there is an unused call-clobbered register available
7365 for the entire function. */
7368 ix86_select_alt_pic_regnum (void)
7370 if (current_function_is_leaf && !crtl->profile
7371 && !ix86_current_function_calls_tls_descriptor)
7374 /* Can't use the same register for both PIC and DRAP. */
7376 drap = REGNO (crtl->drap_reg);
7379 for (i = 2; i >= 0; --i)
7380 if (i != drap && !df_regs_ever_live_p (i))
7384 return INVALID_REGNUM;
7387 /* Return 1 if we need to save REGNO. */
7389 ix86_save_reg (unsigned int regno, int maybe_eh_return)
7391 if (pic_offset_table_rtx
7392 && regno == REAL_PIC_OFFSET_TABLE_REGNUM
7393 && (df_regs_ever_live_p (REAL_PIC_OFFSET_TABLE_REGNUM)
7395 || crtl->calls_eh_return
7396 || crtl->uses_const_pool))
7398 if (ix86_select_alt_pic_regnum () != INVALID_REGNUM)
7403 if (crtl->calls_eh_return && maybe_eh_return)
7408 unsigned test = EH_RETURN_DATA_REGNO (i);
7409 if (test == INVALID_REGNUM)
7417 && regno == REGNO (crtl->drap_reg))
7420 return (df_regs_ever_live_p (regno)
7421 && !call_used_regs[regno]
7422 && !fixed_regs[regno]
7423 && (regno != HARD_FRAME_POINTER_REGNUM || !frame_pointer_needed));
7426 /* Return number of saved general prupose registers. */
7429 ix86_nsaved_regs (void)
7434 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
7435 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
7440 /* Return number of saved SSE registrers. */
7443 ix86_nsaved_sseregs (void)
7448 if (ix86_cfun_abi () != MS_ABI)
7450 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
7451 if (SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
7456 /* Given FROM and TO register numbers, say whether this elimination is
7457 allowed. If stack alignment is needed, we can only replace argument
7458 pointer with hard frame pointer, or replace frame pointer with stack
7459 pointer. Otherwise, frame pointer elimination is automatically
7460 handled and all other eliminations are valid. */
7463 ix86_can_eliminate (int from, int to)
7465 if (stack_realign_fp)
7466 return ((from == ARG_POINTER_REGNUM
7467 && to == HARD_FRAME_POINTER_REGNUM)
7468 || (from == FRAME_POINTER_REGNUM
7469 && to == STACK_POINTER_REGNUM));
7471 return to == STACK_POINTER_REGNUM ? !frame_pointer_needed : 1;
7474 /* Return the offset between two registers, one to be eliminated, and the other
7475 its replacement, at the start of a routine. */
7478 ix86_initial_elimination_offset (int from, int to)
7480 struct ix86_frame frame;
7481 ix86_compute_frame_layout (&frame);
7483 if (from == ARG_POINTER_REGNUM && to == HARD_FRAME_POINTER_REGNUM)
7484 return frame.hard_frame_pointer_offset;
7485 else if (from == FRAME_POINTER_REGNUM
7486 && to == HARD_FRAME_POINTER_REGNUM)
7487 return frame.hard_frame_pointer_offset - frame.frame_pointer_offset;
7490 gcc_assert (to == STACK_POINTER_REGNUM);
7492 if (from == ARG_POINTER_REGNUM)
7493 return frame.stack_pointer_offset;
7495 gcc_assert (from == FRAME_POINTER_REGNUM);
7496 return frame.stack_pointer_offset - frame.frame_pointer_offset;
7500 /* Fill structure ix86_frame about frame of currently computed function. */
7503 ix86_compute_frame_layout (struct ix86_frame *frame)
7505 HOST_WIDE_INT total_size;
7506 unsigned int stack_alignment_needed;
7507 HOST_WIDE_INT offset;
7508 unsigned int preferred_alignment;
7509 HOST_WIDE_INT size = get_frame_size ();
7511 frame->nregs = ix86_nsaved_regs ();
7512 frame->nsseregs = ix86_nsaved_sseregs ();
7515 stack_alignment_needed = crtl->stack_alignment_needed / BITS_PER_UNIT;
7516 preferred_alignment = crtl->preferred_stack_boundary / BITS_PER_UNIT;
7518 /* MS ABI seem to require stack alignment to be always 16 except for function
7520 if (ix86_cfun_abi () == MS_ABI && preferred_alignment < 16)
7522 preferred_alignment = 16;
7523 stack_alignment_needed = 16;
7524 crtl->preferred_stack_boundary = 128;
7525 crtl->stack_alignment_needed = 128;
7528 gcc_assert (!size || stack_alignment_needed);
7529 gcc_assert (preferred_alignment >= STACK_BOUNDARY / BITS_PER_UNIT);
7530 gcc_assert (preferred_alignment <= stack_alignment_needed);
7532 /* During reload iteration the amount of registers saved can change.
7533 Recompute the value as needed. Do not recompute when amount of registers
7534 didn't change as reload does multiple calls to the function and does not
7535 expect the decision to change within single iteration. */
7536 if (!optimize_function_for_size_p (cfun)
7537 && cfun->machine->use_fast_prologue_epilogue_nregs != frame->nregs)
7539 int count = frame->nregs;
7541 cfun->machine->use_fast_prologue_epilogue_nregs = count;
7542 /* The fast prologue uses move instead of push to save registers. This
7543 is significantly longer, but also executes faster as modern hardware
7544 can execute the moves in parallel, but can't do that for push/pop.
7546 Be careful about choosing what prologue to emit: When function takes
7547 many instructions to execute we may use slow version as well as in
7548 case function is known to be outside hot spot (this is known with
7549 feedback only). Weight the size of function by number of registers
7550 to save as it is cheap to use one or two push instructions but very
7551 slow to use many of them. */
7553 count = (count - 1) * FAST_PROLOGUE_INSN_COUNT;
7554 if (cfun->function_frequency < FUNCTION_FREQUENCY_NORMAL
7555 || (flag_branch_probabilities
7556 && cfun->function_frequency < FUNCTION_FREQUENCY_HOT))
7557 cfun->machine->use_fast_prologue_epilogue = false;
7559 cfun->machine->use_fast_prologue_epilogue
7560 = !expensive_function_p (count);
7562 if (TARGET_PROLOGUE_USING_MOVE
7563 && cfun->machine->use_fast_prologue_epilogue)
7564 frame->save_regs_using_mov = true;
7566 frame->save_regs_using_mov = false;
7569 /* Skip return address and saved base pointer. */
7570 offset = frame_pointer_needed ? UNITS_PER_WORD * 2 : UNITS_PER_WORD;
7572 frame->hard_frame_pointer_offset = offset;
7574 /* Set offset to aligned because the realigned frame starts from
7576 if (stack_realign_fp)
7577 offset = (offset + stack_alignment_needed -1) & -stack_alignment_needed;
7579 /* Register save area */
7580 offset += frame->nregs * UNITS_PER_WORD;
7582 /* Align SSE reg save area. */
7583 if (frame->nsseregs)
7584 frame->padding0 = ((offset + 16 - 1) & -16) - offset;
7586 frame->padding0 = 0;
7588 /* SSE register save area. */
7589 offset += frame->padding0 + frame->nsseregs * 16;
7592 frame->va_arg_size = ix86_varargs_gpr_size + ix86_varargs_fpr_size;
7593 offset += frame->va_arg_size;
7595 /* Align start of frame for local function. */
7596 frame->padding1 = ((offset + stack_alignment_needed - 1)
7597 & -stack_alignment_needed) - offset;
7599 offset += frame->padding1;
7601 /* Frame pointer points here. */
7602 frame->frame_pointer_offset = offset;
7606 /* Add outgoing arguments area. Can be skipped if we eliminated
7607 all the function calls as dead code.
7608 Skipping is however impossible when function calls alloca. Alloca
7609 expander assumes that last crtl->outgoing_args_size
7610 of stack frame are unused. */
7611 if (ACCUMULATE_OUTGOING_ARGS
7612 && (!current_function_is_leaf || cfun->calls_alloca
7613 || ix86_current_function_calls_tls_descriptor))
7615 offset += crtl->outgoing_args_size;
7616 frame->outgoing_arguments_size = crtl->outgoing_args_size;
7619 frame->outgoing_arguments_size = 0;
7621 /* Align stack boundary. Only needed if we're calling another function
7623 if (!current_function_is_leaf || cfun->calls_alloca
7624 || ix86_current_function_calls_tls_descriptor)
7625 frame->padding2 = ((offset + preferred_alignment - 1)
7626 & -preferred_alignment) - offset;
7628 frame->padding2 = 0;
7630 offset += frame->padding2;
7632 /* We've reached end of stack frame. */
7633 frame->stack_pointer_offset = offset;
7635 /* Size prologue needs to allocate. */
7636 frame->to_allocate =
7637 (size + frame->padding1 + frame->padding2
7638 + frame->outgoing_arguments_size + frame->va_arg_size);
7640 if ((!frame->to_allocate && frame->nregs <= 1)
7641 || (TARGET_64BIT && frame->to_allocate >= (HOST_WIDE_INT) 0x80000000))
7642 frame->save_regs_using_mov = false;
7644 if (!TARGET_64BIT_MS_ABI && TARGET_RED_ZONE && current_function_sp_is_unchanging
7645 && current_function_is_leaf
7646 && !ix86_current_function_calls_tls_descriptor)
7648 frame->red_zone_size = frame->to_allocate;
7649 if (frame->save_regs_using_mov)
7650 frame->red_zone_size += frame->nregs * UNITS_PER_WORD;
7651 if (frame->red_zone_size > RED_ZONE_SIZE - RED_ZONE_RESERVE)
7652 frame->red_zone_size = RED_ZONE_SIZE - RED_ZONE_RESERVE;
7655 frame->red_zone_size = 0;
7656 frame->to_allocate -= frame->red_zone_size;
7657 frame->stack_pointer_offset -= frame->red_zone_size;
7659 fprintf (stderr, "\n");
7660 fprintf (stderr, "size: %ld\n", (long)size);
7661 fprintf (stderr, "nregs: %ld\n", (long)frame->nregs);
7662 fprintf (stderr, "nsseregs: %ld\n", (long)frame->nsseregs);
7663 fprintf (stderr, "padding0: %ld\n", (long)frame->padding0);
7664 fprintf (stderr, "alignment1: %ld\n", (long)stack_alignment_needed);
7665 fprintf (stderr, "padding1: %ld\n", (long)frame->padding1);
7666 fprintf (stderr, "va_arg: %ld\n", (long)frame->va_arg_size);
7667 fprintf (stderr, "padding2: %ld\n", (long)frame->padding2);
7668 fprintf (stderr, "to_allocate: %ld\n", (long)frame->to_allocate);
7669 fprintf (stderr, "red_zone_size: %ld\n", (long)frame->red_zone_size);
7670 fprintf (stderr, "frame_pointer_offset: %ld\n", (long)frame->frame_pointer_offset);
7671 fprintf (stderr, "hard_frame_pointer_offset: %ld\n",
7672 (long)frame->hard_frame_pointer_offset);
7673 fprintf (stderr, "stack_pointer_offset: %ld\n", (long)frame->stack_pointer_offset);
7674 fprintf (stderr, "current_function_is_leaf: %ld\n", (long)current_function_is_leaf);
7675 fprintf (stderr, "cfun->calls_alloca: %ld\n", (long)cfun->calls_alloca);
7676 fprintf (stderr, "x86_current_function_calls_tls_descriptor: %ld\n", (long)ix86_current_function_calls_tls_descriptor);
7680 /* Emit code to save registers in the prologue. */
7683 ix86_emit_save_regs (void)
7688 for (regno = FIRST_PSEUDO_REGISTER - 1; regno-- > 0; )
7689 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
7691 insn = emit_insn (gen_push (gen_rtx_REG (Pmode, regno)));
7692 RTX_FRAME_RELATED_P (insn) = 1;
7696 /* Emit code to save registers using MOV insns. First register
7697 is restored from POINTER + OFFSET. */
7699 ix86_emit_save_regs_using_mov (rtx pointer, HOST_WIDE_INT offset)
7704 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
7705 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
7707 insn = emit_move_insn (adjust_address (gen_rtx_MEM (Pmode, pointer),
7709 gen_rtx_REG (Pmode, regno));
7710 RTX_FRAME_RELATED_P (insn) = 1;
7711 offset += UNITS_PER_WORD;
7715 /* Emit code to save registers using MOV insns. First register
7716 is restored from POINTER + OFFSET. */
7718 ix86_emit_save_sse_regs_using_mov (rtx pointer, HOST_WIDE_INT offset)
7724 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
7725 if (SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
7727 mem = adjust_address (gen_rtx_MEM (TImode, pointer), TImode, offset);
7728 set_mem_align (mem, 128);
7729 insn = emit_move_insn (mem, gen_rtx_REG (TImode, regno));
7730 RTX_FRAME_RELATED_P (insn) = 1;
7735 /* Expand prologue or epilogue stack adjustment.
7736 The pattern exist to put a dependency on all ebp-based memory accesses.
7737 STYLE should be negative if instructions should be marked as frame related,
7738 zero if %r11 register is live and cannot be freely used and positive
7742 pro_epilogue_adjust_stack (rtx dest, rtx src, rtx offset, int style)
7747 insn = emit_insn (gen_pro_epilogue_adjust_stack_1 (dest, src, offset));
7748 else if (x86_64_immediate_operand (offset, DImode))
7749 insn = emit_insn (gen_pro_epilogue_adjust_stack_rex64 (dest, src, offset));
7753 /* r11 is used by indirect sibcall return as well, set before the
7754 epilogue and used after the epilogue. ATM indirect sibcall
7755 shouldn't be used together with huge frame sizes in one
7756 function because of the frame_size check in sibcall.c. */
7758 r11 = gen_rtx_REG (DImode, R11_REG);
7759 insn = emit_insn (gen_rtx_SET (DImode, r11, offset));
7761 RTX_FRAME_RELATED_P (insn) = 1;
7762 insn = emit_insn (gen_pro_epilogue_adjust_stack_rex64_2 (dest, src, r11,
7766 RTX_FRAME_RELATED_P (insn) = 1;
7769 /* Find an available register to be used as dynamic realign argument
7770 pointer regsiter. Such a register will be written in prologue and
7771 used in begin of body, so it must not be
7772 1. parameter passing register.
7774 We reuse static-chain register if it is available. Otherwise, we
7775 use DI for i386 and R13 for x86-64. We chose R13 since it has
7778 Return: the regno of chosen register. */
7781 find_drap_reg (void)
7783 tree decl = cfun->decl;
7787 /* Use R13 for nested function or function need static chain.
7788 Since function with tail call may use any caller-saved
7789 registers in epilogue, DRAP must not use caller-saved
7790 register in such case. */
7791 if ((decl_function_context (decl)
7792 && !DECL_NO_STATIC_CHAIN (decl))
7793 || crtl->tail_call_emit)
7800 /* Use DI for nested function or function need static chain.
7801 Since function with tail call may use any caller-saved
7802 registers in epilogue, DRAP must not use caller-saved
7803 register in such case. */
7804 if ((decl_function_context (decl)
7805 && !DECL_NO_STATIC_CHAIN (decl))
7806 || crtl->tail_call_emit)
7809 /* Reuse static chain register if it isn't used for parameter
7811 if (ix86_function_regparm (TREE_TYPE (decl), decl) <= 2
7812 && !lookup_attribute ("fastcall",
7813 TYPE_ATTRIBUTES (TREE_TYPE (decl))))
7820 /* Update incoming stack boundary and estimated stack alignment. */
7823 ix86_update_stack_boundary (void)
7825 /* Prefer the one specified at command line. */
7826 ix86_incoming_stack_boundary
7827 = (ix86_user_incoming_stack_boundary
7828 ? ix86_user_incoming_stack_boundary
7829 : ix86_default_incoming_stack_boundary);
7831 /* Incoming stack alignment can be changed on individual functions
7832 via force_align_arg_pointer attribute. We use the smallest
7833 incoming stack boundary. */
7834 if (ix86_incoming_stack_boundary > MIN_STACK_BOUNDARY
7835 && lookup_attribute (ix86_force_align_arg_pointer_string,
7836 TYPE_ATTRIBUTES (TREE_TYPE (current_function_decl))))
7837 ix86_incoming_stack_boundary = MIN_STACK_BOUNDARY;
7839 /* The incoming stack frame has to be aligned at least at
7840 parm_stack_boundary. */
7841 if (ix86_incoming_stack_boundary < crtl->parm_stack_boundary)
7842 ix86_incoming_stack_boundary = crtl->parm_stack_boundary;
7844 /* Stack at entrance of main is aligned by runtime. We use the
7845 smallest incoming stack boundary. */
7846 if (ix86_incoming_stack_boundary > MAIN_STACK_BOUNDARY
7847 && DECL_NAME (current_function_decl)
7848 && MAIN_NAME_P (DECL_NAME (current_function_decl))
7849 && DECL_FILE_SCOPE_P (current_function_decl))
7850 ix86_incoming_stack_boundary = MAIN_STACK_BOUNDARY;
7852 /* x86_64 vararg needs 16byte stack alignment for register save
7856 && crtl->stack_alignment_estimated < 128)
7857 crtl->stack_alignment_estimated = 128;
7860 /* Handle the TARGET_GET_DRAP_RTX hook. Return NULL if no DRAP is
7861 needed or an rtx for DRAP otherwise. */
7864 ix86_get_drap_rtx (void)
7866 if (ix86_force_drap || !ACCUMULATE_OUTGOING_ARGS)
7867 crtl->need_drap = true;
7869 if (stack_realign_drap)
7871 /* Assign DRAP to vDRAP and returns vDRAP */
7872 unsigned int regno = find_drap_reg ();
7877 arg_ptr = gen_rtx_REG (Pmode, regno);
7878 crtl->drap_reg = arg_ptr;
7881 drap_vreg = copy_to_reg (arg_ptr);
7885 insn = emit_insn_before (seq, NEXT_INSN (entry_of_function ()));
7886 RTX_FRAME_RELATED_P (insn) = 1;
7893 /* Handle the TARGET_INTERNAL_ARG_POINTER hook. */
7896 ix86_internal_arg_pointer (void)
7898 return virtual_incoming_args_rtx;
7901 /* Handle the TARGET_DWARF_HANDLE_FRAME_UNSPEC hook.
7902 This is called from dwarf2out.c to emit call frame instructions
7903 for frame-related insns containing UNSPECs and UNSPEC_VOLATILEs. */
7905 ix86_dwarf_handle_frame_unspec (const char *label, rtx pattern, int index)
7907 rtx unspec = SET_SRC (pattern);
7908 gcc_assert (GET_CODE (unspec) == UNSPEC);
7912 case UNSPEC_REG_SAVE:
7913 dwarf2out_reg_save_reg (label, XVECEXP (unspec, 0, 0),
7914 SET_DEST (pattern));
7916 case UNSPEC_DEF_CFA:
7917 dwarf2out_def_cfa (label, REGNO (SET_DEST (pattern)),
7918 INTVAL (XVECEXP (unspec, 0, 0)));
7925 /* Finalize stack_realign_needed flag, which will guide prologue/epilogue
7926 to be generated in correct form. */
7928 ix86_finalize_stack_realign_flags (void)
7930 /* Check if stack realign is really needed after reload, and
7931 stores result in cfun */
7932 unsigned int incoming_stack_boundary
7933 = (crtl->parm_stack_boundary > ix86_incoming_stack_boundary
7934 ? crtl->parm_stack_boundary : ix86_incoming_stack_boundary);
7935 unsigned int stack_realign = (incoming_stack_boundary
7936 < (current_function_is_leaf
7937 ? crtl->max_used_stack_slot_alignment
7938 : crtl->stack_alignment_needed));
7940 if (crtl->stack_realign_finalized)
7942 /* After stack_realign_needed is finalized, we can't no longer
7944 gcc_assert (crtl->stack_realign_needed == stack_realign);
7948 crtl->stack_realign_needed = stack_realign;
7949 crtl->stack_realign_finalized = true;
7953 /* Expand the prologue into a bunch of separate insns. */
7956 ix86_expand_prologue (void)
7960 struct ix86_frame frame;
7961 HOST_WIDE_INT allocate;
7963 ix86_finalize_stack_realign_flags ();
7965 /* DRAP should not coexist with stack_realign_fp */
7966 gcc_assert (!(crtl->drap_reg && stack_realign_fp));
7968 ix86_compute_frame_layout (&frame);
7970 /* Emit prologue code to adjust stack alignment and setup DRAP, in case
7971 of DRAP is needed and stack realignment is really needed after reload */
7972 if (crtl->drap_reg && crtl->stack_realign_needed)
7975 int align_bytes = crtl->stack_alignment_needed / BITS_PER_UNIT;
7976 int param_ptr_offset = (call_used_regs[REGNO (crtl->drap_reg)]
7977 ? 0 : UNITS_PER_WORD);
7979 gcc_assert (stack_realign_drap);
7981 /* Grab the argument pointer. */
7982 x = plus_constant (stack_pointer_rtx,
7983 (UNITS_PER_WORD + param_ptr_offset));
7986 /* Only need to push parameter pointer reg if it is caller
7988 if (!call_used_regs[REGNO (crtl->drap_reg)])
7990 /* Push arg pointer reg */
7991 insn = emit_insn (gen_push (y));
7992 RTX_FRAME_RELATED_P (insn) = 1;
7995 insn = emit_insn (gen_rtx_SET (VOIDmode, y, x));
7996 RTX_FRAME_RELATED_P (insn) = 1;
7998 /* Align the stack. */
7999 insn = emit_insn ((*ix86_gen_andsp) (stack_pointer_rtx,
8001 GEN_INT (-align_bytes)));
8002 RTX_FRAME_RELATED_P (insn) = 1;
8004 /* Replicate the return address on the stack so that return
8005 address can be reached via (argp - 1) slot. This is needed
8006 to implement macro RETURN_ADDR_RTX and intrinsic function
8007 expand_builtin_return_addr etc. */
8009 x = gen_frame_mem (Pmode,
8010 plus_constant (x, -UNITS_PER_WORD));
8011 insn = emit_insn (gen_push (x));
8012 RTX_FRAME_RELATED_P (insn) = 1;
8015 /* Note: AT&T enter does NOT have reversed args. Enter is probably
8016 slower on all targets. Also sdb doesn't like it. */
8018 if (frame_pointer_needed)
8020 insn = emit_insn (gen_push (hard_frame_pointer_rtx));
8021 RTX_FRAME_RELATED_P (insn) = 1;
8023 insn = emit_move_insn (hard_frame_pointer_rtx, stack_pointer_rtx);
8024 RTX_FRAME_RELATED_P (insn) = 1;
8027 if (stack_realign_fp)
8029 int align_bytes = crtl->stack_alignment_needed / BITS_PER_UNIT;
8030 gcc_assert (align_bytes > MIN_STACK_BOUNDARY / BITS_PER_UNIT);
8032 /* Align the stack. */
8033 insn = emit_insn ((*ix86_gen_andsp) (stack_pointer_rtx,
8035 GEN_INT (-align_bytes)));
8036 RTX_FRAME_RELATED_P (insn) = 1;
8039 allocate = frame.to_allocate + frame.nsseregs * 16 + frame.padding0;
8041 if (!frame.save_regs_using_mov)
8042 ix86_emit_save_regs ();
8044 allocate += frame.nregs * UNITS_PER_WORD;
8046 /* When using red zone we may start register saving before allocating
8047 the stack frame saving one cycle of the prologue. However I will
8048 avoid doing this if I am going to have to probe the stack since
8049 at least on x86_64 the stack probe can turn into a call that clobbers
8050 a red zone location */
8051 if (!TARGET_64BIT_MS_ABI && TARGET_RED_ZONE && frame.save_regs_using_mov
8052 && (! TARGET_STACK_PROBE || allocate < CHECK_STACK_LIMIT))
8053 ix86_emit_save_regs_using_mov ((frame_pointer_needed
8054 && !crtl->stack_realign_needed)
8055 ? hard_frame_pointer_rtx
8056 : stack_pointer_rtx,
8057 -frame.nregs * UNITS_PER_WORD);
8061 else if (! TARGET_STACK_PROBE || allocate < CHECK_STACK_LIMIT)
8062 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
8063 GEN_INT (-allocate), -1);
8066 /* Only valid for Win32. */
8067 rtx eax = gen_rtx_REG (Pmode, AX_REG);
8071 gcc_assert (!TARGET_64BIT || cfun->machine->call_abi == MS_ABI);
8073 if (cfun->machine->call_abi == MS_ABI)
8076 eax_live = ix86_eax_live_at_start_p ();
8080 emit_insn (gen_push (eax));
8081 allocate -= UNITS_PER_WORD;
8084 emit_move_insn (eax, GEN_INT (allocate));
8087 insn = gen_allocate_stack_worker_64 (eax, eax);
8089 insn = gen_allocate_stack_worker_32 (eax, eax);
8090 insn = emit_insn (insn);
8091 RTX_FRAME_RELATED_P (insn) = 1;
8092 t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, GEN_INT (-allocate));
8093 t = gen_rtx_SET (VOIDmode, stack_pointer_rtx, t);
8094 REG_NOTES (insn) = gen_rtx_EXPR_LIST (REG_FRAME_RELATED_EXPR,
8095 t, REG_NOTES (insn));
8099 if (frame_pointer_needed)
8100 t = plus_constant (hard_frame_pointer_rtx,
8103 - frame.nregs * UNITS_PER_WORD);
8105 t = plus_constant (stack_pointer_rtx, allocate);
8106 emit_move_insn (eax, gen_rtx_MEM (Pmode, t));
8110 if (frame.save_regs_using_mov
8111 && !(!TARGET_64BIT_MS_ABI && TARGET_RED_ZONE
8112 && (! TARGET_STACK_PROBE || allocate < CHECK_STACK_LIMIT)))
8114 if (!frame_pointer_needed
8115 || !frame.to_allocate
8116 || crtl->stack_realign_needed)
8117 ix86_emit_save_regs_using_mov (stack_pointer_rtx,
8119 + frame.nsseregs * 16 + frame.padding0);
8121 ix86_emit_save_regs_using_mov (hard_frame_pointer_rtx,
8122 -frame.nregs * UNITS_PER_WORD);
8124 if (!frame_pointer_needed
8125 || !frame.to_allocate
8126 || crtl->stack_realign_needed)
8127 ix86_emit_save_sse_regs_using_mov (stack_pointer_rtx,
8130 ix86_emit_save_sse_regs_using_mov (hard_frame_pointer_rtx,
8131 - frame.nregs * UNITS_PER_WORD
8132 - frame.nsseregs * 16
8135 pic_reg_used = false;
8136 if (pic_offset_table_rtx
8137 && (df_regs_ever_live_p (REAL_PIC_OFFSET_TABLE_REGNUM)
8140 unsigned int alt_pic_reg_used = ix86_select_alt_pic_regnum ();
8142 if (alt_pic_reg_used != INVALID_REGNUM)
8143 SET_REGNO (pic_offset_table_rtx, alt_pic_reg_used);
8145 pic_reg_used = true;
8152 if (ix86_cmodel == CM_LARGE_PIC)
8154 rtx tmp_reg = gen_rtx_REG (DImode, R11_REG);
8155 rtx label = gen_label_rtx ();
8157 LABEL_PRESERVE_P (label) = 1;
8158 gcc_assert (REGNO (pic_offset_table_rtx) != REGNO (tmp_reg));
8159 insn = emit_insn (gen_set_rip_rex64 (pic_offset_table_rtx, label));
8160 insn = emit_insn (gen_set_got_offset_rex64 (tmp_reg, label));
8161 insn = emit_insn (gen_adddi3 (pic_offset_table_rtx,
8162 pic_offset_table_rtx, tmp_reg));
8165 insn = emit_insn (gen_set_got_rex64 (pic_offset_table_rtx));
8168 insn = emit_insn (gen_set_got (pic_offset_table_rtx));
8171 /* Prevent function calls from being scheduled before the call to mcount.
8172 In the pic_reg_used case, make sure that the got load isn't deleted. */
8176 emit_insn (gen_prologue_use (pic_offset_table_rtx));
8177 emit_insn (gen_blockage ());
8180 if (crtl->drap_reg && !crtl->stack_realign_needed)
8182 /* vDRAP is setup but after reload it turns out stack realign
8183 isn't necessary, here we will emit prologue to setup DRAP
8184 without stack realign adjustment */
8185 int drap_bp_offset = UNITS_PER_WORD * 2;
8186 rtx x = plus_constant (hard_frame_pointer_rtx, drap_bp_offset);
8187 insn = emit_insn (gen_rtx_SET (VOIDmode, crtl->drap_reg, x));
8190 /* Emit cld instruction if stringops are used in the function. */
8191 if (TARGET_CLD && ix86_current_function_needs_cld)
8192 emit_insn (gen_cld ());
8195 /* Emit code to restore saved registers using MOV insns. First register
8196 is restored from POINTER + OFFSET. */
8198 ix86_emit_restore_regs_using_mov (rtx pointer, HOST_WIDE_INT offset,
8199 int maybe_eh_return)
8202 rtx base_address = gen_rtx_MEM (Pmode, pointer);
8204 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
8205 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, maybe_eh_return))
8207 /* Ensure that adjust_address won't be forced to produce pointer
8208 out of range allowed by x86-64 instruction set. */
8209 if (TARGET_64BIT && offset != trunc_int_for_mode (offset, SImode))
8213 r11 = gen_rtx_REG (DImode, R11_REG);
8214 emit_move_insn (r11, GEN_INT (offset));
8215 emit_insn (gen_adddi3 (r11, r11, pointer));
8216 base_address = gen_rtx_MEM (Pmode, r11);
8219 emit_move_insn (gen_rtx_REG (Pmode, regno),
8220 adjust_address (base_address, Pmode, offset));
8221 offset += UNITS_PER_WORD;
8225 /* Emit code to restore saved registers using MOV insns. First register
8226 is restored from POINTER + OFFSET. */
8228 ix86_emit_restore_sse_regs_using_mov (rtx pointer, HOST_WIDE_INT offset,
8229 int maybe_eh_return)
8232 rtx base_address = gen_rtx_MEM (TImode, pointer);
8235 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
8236 if (SSE_REGNO_P (regno) && ix86_save_reg (regno, maybe_eh_return))
8238 /* Ensure that adjust_address won't be forced to produce pointer
8239 out of range allowed by x86-64 instruction set. */
8240 if (TARGET_64BIT && offset != trunc_int_for_mode (offset, SImode))
8244 r11 = gen_rtx_REG (DImode, R11_REG);
8245 emit_move_insn (r11, GEN_INT (offset));
8246 emit_insn (gen_adddi3 (r11, r11, pointer));
8247 base_address = gen_rtx_MEM (TImode, r11);
8250 mem = adjust_address (base_address, TImode, offset);
8251 set_mem_align (mem, 128);
8252 emit_move_insn (gen_rtx_REG (TImode, regno), mem);
8257 /* Restore function stack, frame, and registers. */
8260 ix86_expand_epilogue (int style)
8264 struct ix86_frame frame;
8265 HOST_WIDE_INT offset;
8267 ix86_finalize_stack_realign_flags ();
8269 /* When stack is realigned, SP must be valid. */
8270 sp_valid = (!frame_pointer_needed
8271 || current_function_sp_is_unchanging
8272 || stack_realign_fp);
8274 ix86_compute_frame_layout (&frame);
8276 /* Calculate start of saved registers relative to ebp. Special care
8277 must be taken for the normal return case of a function using
8278 eh_return: the eax and edx registers are marked as saved, but not
8279 restored along this path. */
8280 offset = frame.nregs;
8281 if (crtl->calls_eh_return && style != 2)
8283 offset *= -UNITS_PER_WORD;
8284 offset -= frame.nsseregs * 16 + frame.padding0;
8286 /* If we're only restoring one register and sp is not valid then
8287 using a move instruction to restore the register since it's
8288 less work than reloading sp and popping the register.
8290 The default code result in stack adjustment using add/lea instruction,
8291 while this code results in LEAVE instruction (or discrete equivalent),
8292 so it is profitable in some other cases as well. Especially when there
8293 are no registers to restore. We also use this code when TARGET_USE_LEAVE
8294 and there is exactly one register to pop. This heuristic may need some
8295 tuning in future. */
8296 if ((!sp_valid && frame.nregs <= 1)
8297 || (TARGET_EPILOGUE_USING_MOVE
8298 && cfun->machine->use_fast_prologue_epilogue
8299 && (frame.nregs > 1 || frame.to_allocate))
8300 || (frame_pointer_needed && !frame.nregs && frame.to_allocate)
8301 || (frame_pointer_needed && TARGET_USE_LEAVE
8302 && cfun->machine->use_fast_prologue_epilogue
8303 && frame.nregs == 1)
8304 || crtl->calls_eh_return)
8306 /* Restore registers. We can use ebp or esp to address the memory
8307 locations. If both are available, default to ebp, since offsets
8308 are known to be small. Only exception is esp pointing directly
8309 to the end of block of saved registers, where we may simplify
8312 If we are realigning stack with bp and sp, regs restore can't
8313 be addressed by bp. sp must be used instead. */
8315 if (!frame_pointer_needed
8316 || (sp_valid && !frame.to_allocate)
8317 || stack_realign_fp)
8319 ix86_emit_restore_sse_regs_using_mov (stack_pointer_rtx,
8320 frame.to_allocate, style == 2);
8321 ix86_emit_restore_regs_using_mov (stack_pointer_rtx,
8323 + frame.nsseregs * 16
8324 + frame.padding0, style == 2);
8328 ix86_emit_restore_sse_regs_using_mov (hard_frame_pointer_rtx,
8329 offset, style == 2);
8330 ix86_emit_restore_regs_using_mov (hard_frame_pointer_rtx,
8332 + frame.nsseregs * 16
8333 + frame.padding0, style == 2);
8336 /* eh_return epilogues need %ecx added to the stack pointer. */
8339 rtx tmp, sa = EH_RETURN_STACKADJ_RTX;
8341 /* Stack align doesn't work with eh_return. */
8342 gcc_assert (!crtl->stack_realign_needed);
8344 if (frame_pointer_needed)
8346 tmp = gen_rtx_PLUS (Pmode, hard_frame_pointer_rtx, sa);
8347 tmp = plus_constant (tmp, UNITS_PER_WORD);
8348 emit_insn (gen_rtx_SET (VOIDmode, sa, tmp));
8350 tmp = gen_rtx_MEM (Pmode, hard_frame_pointer_rtx);
8351 emit_move_insn (hard_frame_pointer_rtx, tmp);
8353 pro_epilogue_adjust_stack (stack_pointer_rtx, sa,
8358 tmp = gen_rtx_PLUS (Pmode, stack_pointer_rtx, sa);
8359 tmp = plus_constant (tmp, (frame.to_allocate
8360 + frame.nregs * UNITS_PER_WORD
8361 + frame.nsseregs * 16
8363 emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx, tmp));
8366 else if (!frame_pointer_needed)
8367 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
8368 GEN_INT (frame.to_allocate
8369 + frame.nregs * UNITS_PER_WORD
8370 + frame.nsseregs * 16
8373 /* If not an i386, mov & pop is faster than "leave". */
8374 else if (TARGET_USE_LEAVE || optimize_function_for_size_p (cfun)
8375 || !cfun->machine->use_fast_prologue_epilogue)
8376 emit_insn ((*ix86_gen_leave) ());
8379 pro_epilogue_adjust_stack (stack_pointer_rtx,
8380 hard_frame_pointer_rtx,
8383 emit_insn ((*ix86_gen_pop1) (hard_frame_pointer_rtx));
8388 /* First step is to deallocate the stack frame so that we can
8391 If we realign stack with frame pointer, then stack pointer
8392 won't be able to recover via lea $offset(%bp), %sp, because
8393 there is a padding area between bp and sp for realign.
8394 "add $to_allocate, %sp" must be used instead. */
8397 gcc_assert (frame_pointer_needed);
8398 gcc_assert (!stack_realign_fp);
8399 pro_epilogue_adjust_stack (stack_pointer_rtx,
8400 hard_frame_pointer_rtx,
8401 GEN_INT (offset), style);
8402 ix86_emit_restore_sse_regs_using_mov (stack_pointer_rtx,
8403 frame.to_allocate, style == 2);
8404 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
8405 GEN_INT (frame.nsseregs * 16), style);
8407 else if (frame.to_allocate || frame.nsseregs)
8409 ix86_emit_restore_sse_regs_using_mov (stack_pointer_rtx,
8412 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
8413 GEN_INT (frame.to_allocate
8414 + frame.nsseregs * 16
8415 + frame.padding0), style);
8418 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
8419 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, false))
8420 emit_insn ((*ix86_gen_pop1) (gen_rtx_REG (Pmode, regno)));
8421 if (frame_pointer_needed)
8423 /* Leave results in shorter dependency chains on CPUs that are
8424 able to grok it fast. */
8425 if (TARGET_USE_LEAVE)
8426 emit_insn ((*ix86_gen_leave) ());
8429 /* For stack realigned really happens, recover stack
8430 pointer to hard frame pointer is a must, if not using
8432 if (stack_realign_fp)
8433 pro_epilogue_adjust_stack (stack_pointer_rtx,
8434 hard_frame_pointer_rtx,
8436 emit_insn ((*ix86_gen_pop1) (hard_frame_pointer_rtx));
8441 if (crtl->drap_reg && crtl->stack_realign_needed)
8443 int param_ptr_offset = (call_used_regs[REGNO (crtl->drap_reg)]
8444 ? 0 : UNITS_PER_WORD);
8445 gcc_assert (stack_realign_drap);
8446 emit_insn ((*ix86_gen_add3) (stack_pointer_rtx,
8448 GEN_INT (-(UNITS_PER_WORD
8449 + param_ptr_offset))));
8450 if (!call_used_regs[REGNO (crtl->drap_reg)])
8451 emit_insn ((*ix86_gen_pop1) (crtl->drap_reg));
8455 /* Sibcall epilogues don't want a return instruction. */
8459 if (crtl->args.pops_args && crtl->args.size)
8461 rtx popc = GEN_INT (crtl->args.pops_args);
8463 /* i386 can only pop 64K bytes. If asked to pop more, pop
8464 return address, do explicit add, and jump indirectly to the
8467 if (crtl->args.pops_args >= 65536)
8469 rtx ecx = gen_rtx_REG (SImode, CX_REG);
8471 /* There is no "pascal" calling convention in any 64bit ABI. */
8472 gcc_assert (!TARGET_64BIT);
8474 emit_insn (gen_popsi1 (ecx));
8475 emit_insn (gen_addsi3 (stack_pointer_rtx, stack_pointer_rtx, popc));
8476 emit_jump_insn (gen_return_indirect_internal (ecx));
8479 emit_jump_insn (gen_return_pop_internal (popc));
8482 emit_jump_insn (gen_return_internal ());
8485 /* Reset from the function's potential modifications. */
8488 ix86_output_function_epilogue (FILE *file ATTRIBUTE_UNUSED,
8489 HOST_WIDE_INT size ATTRIBUTE_UNUSED)
8491 if (pic_offset_table_rtx)
8492 SET_REGNO (pic_offset_table_rtx, REAL_PIC_OFFSET_TABLE_REGNUM);
8494 /* Mach-O doesn't support labels at the end of objects, so if
8495 it looks like we might want one, insert a NOP. */
8497 rtx insn = get_last_insn ();
8500 && NOTE_KIND (insn) != NOTE_INSN_DELETED_LABEL)
8501 insn = PREV_INSN (insn);
8505 && NOTE_KIND (insn) == NOTE_INSN_DELETED_LABEL)))
8506 fputs ("\tnop\n", file);
8512 /* Extract the parts of an RTL expression that is a valid memory address
8513 for an instruction. Return 0 if the structure of the address is
8514 grossly off. Return -1 if the address contains ASHIFT, so it is not
8515 strictly valid, but still used for computing length of lea instruction. */
8518 ix86_decompose_address (rtx addr, struct ix86_address *out)
8520 rtx base = NULL_RTX, index = NULL_RTX, disp = NULL_RTX;
8521 rtx base_reg, index_reg;
8522 HOST_WIDE_INT scale = 1;
8523 rtx scale_rtx = NULL_RTX;
8525 enum ix86_address_seg seg = SEG_DEFAULT;
8527 if (REG_P (addr) || GET_CODE (addr) == SUBREG)
8529 else if (GET_CODE (addr) == PLUS)
8539 addends[n++] = XEXP (op, 1);
8542 while (GET_CODE (op) == PLUS);
8547 for (i = n; i >= 0; --i)
8550 switch (GET_CODE (op))
8555 index = XEXP (op, 0);
8556 scale_rtx = XEXP (op, 1);
8560 if (XINT (op, 1) == UNSPEC_TP
8561 && TARGET_TLS_DIRECT_SEG_REFS
8562 && seg == SEG_DEFAULT)
8563 seg = TARGET_64BIT ? SEG_FS : SEG_GS;
8592 else if (GET_CODE (addr) == MULT)
8594 index = XEXP (addr, 0); /* index*scale */
8595 scale_rtx = XEXP (addr, 1);
8597 else if (GET_CODE (addr) == ASHIFT)
8601 /* We're called for lea too, which implements ashift on occasion. */
8602 index = XEXP (addr, 0);
8603 tmp = XEXP (addr, 1);
8604 if (!CONST_INT_P (tmp))
8606 scale = INTVAL (tmp);
8607 if ((unsigned HOST_WIDE_INT) scale > 3)
8613 disp = addr; /* displacement */
8615 /* Extract the integral value of scale. */
8618 if (!CONST_INT_P (scale_rtx))
8620 scale = INTVAL (scale_rtx);
8623 base_reg = base && GET_CODE (base) == SUBREG ? SUBREG_REG (base) : base;
8624 index_reg = index && GET_CODE (index) == SUBREG ? SUBREG_REG (index) : index;
8626 /* Allow arg pointer and stack pointer as index if there is not scaling. */
8627 if (base_reg && index_reg && scale == 1
8628 && (index_reg == arg_pointer_rtx
8629 || index_reg == frame_pointer_rtx
8630 || (REG_P (index_reg) && REGNO (index_reg) == STACK_POINTER_REGNUM)))
8633 tmp = base, base = index, index = tmp;
8634 tmp = base_reg, base_reg = index_reg, index_reg = tmp;
8637 /* Special case: %ebp cannot be encoded as a base without a displacement. */
8638 if ((base_reg == hard_frame_pointer_rtx
8639 || base_reg == frame_pointer_rtx
8640 || base_reg == arg_pointer_rtx) && !disp)
8643 /* Special case: on K6, [%esi] makes the instruction vector decoded.
8644 Avoid this by transforming to [%esi+0].
8645 Reload calls address legitimization without cfun defined, so we need
8646 to test cfun for being non-NULL. */
8647 if (TARGET_K6 && cfun && optimize_function_for_speed_p (cfun)
8648 && base_reg && !index_reg && !disp
8650 && REGNO_REG_CLASS (REGNO (base_reg)) == SIREG)
8653 /* Special case: encode reg+reg instead of reg*2. */
8654 if (!base && index && scale && scale == 2)
8655 base = index, base_reg = index_reg, scale = 1;
8657 /* Special case: scaling cannot be encoded without base or displacement. */
8658 if (!base && !disp && index && scale != 1)
8670 /* Return cost of the memory address x.
8671 For i386, it is better to use a complex address than let gcc copy
8672 the address into a reg and make a new pseudo. But not if the address
8673 requires to two regs - that would mean more pseudos with longer
8676 ix86_address_cost (rtx x, bool speed ATTRIBUTE_UNUSED)
8678 struct ix86_address parts;
8680 int ok = ix86_decompose_address (x, &parts);
8684 if (parts.base && GET_CODE (parts.base) == SUBREG)
8685 parts.base = SUBREG_REG (parts.base);
8686 if (parts.index && GET_CODE (parts.index) == SUBREG)
8687 parts.index = SUBREG_REG (parts.index);
8689 /* Attempt to minimize number of registers in the address. */
8691 && (!REG_P (parts.base) || REGNO (parts.base) >= FIRST_PSEUDO_REGISTER))
8693 && (!REG_P (parts.index)
8694 || REGNO (parts.index) >= FIRST_PSEUDO_REGISTER)))
8698 && (!REG_P (parts.base) || REGNO (parts.base) >= FIRST_PSEUDO_REGISTER)
8700 && (!REG_P (parts.index) || REGNO (parts.index) >= FIRST_PSEUDO_REGISTER)
8701 && parts.base != parts.index)
8704 /* AMD-K6 don't like addresses with ModR/M set to 00_xxx_100b,
8705 since it's predecode logic can't detect the length of instructions
8706 and it degenerates to vector decoded. Increase cost of such
8707 addresses here. The penalty is minimally 2 cycles. It may be worthwhile
8708 to split such addresses or even refuse such addresses at all.
8710 Following addressing modes are affected:
8715 The first and last case may be avoidable by explicitly coding the zero in
8716 memory address, but I don't have AMD-K6 machine handy to check this
8720 && ((!parts.disp && parts.base && parts.index && parts.scale != 1)
8721 || (parts.disp && !parts.base && parts.index && parts.scale != 1)
8722 || (!parts.disp && parts.base && parts.index && parts.scale == 1)))
8728 /* Allow {LABEL | SYMBOL}_REF - SYMBOL_REF-FOR-PICBASE for Mach-O as
8729 this is used for to form addresses to local data when -fPIC is in
8733 darwin_local_data_pic (rtx disp)
8735 return (GET_CODE (disp) == UNSPEC
8736 && XINT (disp, 1) == UNSPEC_MACHOPIC_OFFSET);
8739 /* Determine if a given RTX is a valid constant. We already know this
8740 satisfies CONSTANT_P. */
8743 legitimate_constant_p (rtx x)
8745 switch (GET_CODE (x))
8750 if (GET_CODE (x) == PLUS)
8752 if (!CONST_INT_P (XEXP (x, 1)))
8757 if (TARGET_MACHO && darwin_local_data_pic (x))
8760 /* Only some unspecs are valid as "constants". */
8761 if (GET_CODE (x) == UNSPEC)
8762 switch (XINT (x, 1))
8767 return TARGET_64BIT;
8770 x = XVECEXP (x, 0, 0);
8771 return (GET_CODE (x) == SYMBOL_REF
8772 && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
8774 x = XVECEXP (x, 0, 0);
8775 return (GET_CODE (x) == SYMBOL_REF
8776 && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_DYNAMIC);
8781 /* We must have drilled down to a symbol. */
8782 if (GET_CODE (x) == LABEL_REF)
8784 if (GET_CODE (x) != SYMBOL_REF)
8789 /* TLS symbols are never valid. */
8790 if (SYMBOL_REF_TLS_MODEL (x))
8793 /* DLLIMPORT symbols are never valid. */
8794 if (TARGET_DLLIMPORT_DECL_ATTRIBUTES
8795 && SYMBOL_REF_DLLIMPORT_P (x))
8800 if (GET_MODE (x) == TImode
8801 && x != CONST0_RTX (TImode)
8807 if (x == CONST0_RTX (GET_MODE (x)))
8815 /* Otherwise we handle everything else in the move patterns. */
8819 /* Determine if it's legal to put X into the constant pool. This
8820 is not possible for the address of thread-local symbols, which
8821 is checked above. */
8824 ix86_cannot_force_const_mem (rtx x)
8826 /* We can always put integral constants and vectors in memory. */
8827 switch (GET_CODE (x))
8837 return !legitimate_constant_p (x);
8840 /* Determine if a given RTX is a valid constant address. */
8843 constant_address_p (rtx x)
8845 return CONSTANT_P (x) && legitimate_address_p (Pmode, x, 1);
8848 /* Nonzero if the constant value X is a legitimate general operand
8849 when generating PIC code. It is given that flag_pic is on and
8850 that X satisfies CONSTANT_P or is a CONST_DOUBLE. */
8853 legitimate_pic_operand_p (rtx x)
8857 switch (GET_CODE (x))
8860 inner = XEXP (x, 0);
8861 if (GET_CODE (inner) == PLUS
8862 && CONST_INT_P (XEXP (inner, 1)))
8863 inner = XEXP (inner, 0);
8865 /* Only some unspecs are valid as "constants". */
8866 if (GET_CODE (inner) == UNSPEC)
8867 switch (XINT (inner, 1))
8872 return TARGET_64BIT;
8874 x = XVECEXP (inner, 0, 0);
8875 return (GET_CODE (x) == SYMBOL_REF
8876 && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
8877 case UNSPEC_MACHOPIC_OFFSET:
8878 return legitimate_pic_address_disp_p (x);
8886 return legitimate_pic_address_disp_p (x);
8893 /* Determine if a given CONST RTX is a valid memory displacement
8897 legitimate_pic_address_disp_p (rtx disp)
8901 /* In 64bit mode we can allow direct addresses of symbols and labels
8902 when they are not dynamic symbols. */
8905 rtx op0 = disp, op1;
8907 switch (GET_CODE (disp))
8913 if (GET_CODE (XEXP (disp, 0)) != PLUS)
8915 op0 = XEXP (XEXP (disp, 0), 0);
8916 op1 = XEXP (XEXP (disp, 0), 1);
8917 if (!CONST_INT_P (op1)
8918 || INTVAL (op1) >= 16*1024*1024
8919 || INTVAL (op1) < -16*1024*1024)
8921 if (GET_CODE (op0) == LABEL_REF)
8923 if (GET_CODE (op0) != SYMBOL_REF)
8928 /* TLS references should always be enclosed in UNSPEC. */
8929 if (SYMBOL_REF_TLS_MODEL (op0))
8931 if (!SYMBOL_REF_FAR_ADDR_P (op0) && SYMBOL_REF_LOCAL_P (op0)
8932 && ix86_cmodel != CM_LARGE_PIC)
8940 if (GET_CODE (disp) != CONST)
8942 disp = XEXP (disp, 0);
8946 /* We are unsafe to allow PLUS expressions. This limit allowed distance
8947 of GOT tables. We should not need these anyway. */
8948 if (GET_CODE (disp) != UNSPEC
8949 || (XINT (disp, 1) != UNSPEC_GOTPCREL
8950 && XINT (disp, 1) != UNSPEC_GOTOFF
8951 && XINT (disp, 1) != UNSPEC_PLTOFF))
8954 if (GET_CODE (XVECEXP (disp, 0, 0)) != SYMBOL_REF
8955 && GET_CODE (XVECEXP (disp, 0, 0)) != LABEL_REF)
8961 if (GET_CODE (disp) == PLUS)
8963 if (!CONST_INT_P (XEXP (disp, 1)))
8965 disp = XEXP (disp, 0);
8969 if (TARGET_MACHO && darwin_local_data_pic (disp))
8972 if (GET_CODE (disp) != UNSPEC)
8975 switch (XINT (disp, 1))
8980 /* We need to check for both symbols and labels because VxWorks loads
8981 text labels with @GOT rather than @GOTOFF. See gotoff_operand for
8983 return (GET_CODE (XVECEXP (disp, 0, 0)) == SYMBOL_REF
8984 || GET_CODE (XVECEXP (disp, 0, 0)) == LABEL_REF);
8986 /* Refuse GOTOFF in 64bit mode since it is always 64bit when used.
8987 While ABI specify also 32bit relocation but we don't produce it in
8988 small PIC model at all. */
8989 if ((GET_CODE (XVECEXP (disp, 0, 0)) == SYMBOL_REF
8990 || GET_CODE (XVECEXP (disp, 0, 0)) == LABEL_REF)
8992 return gotoff_operand (XVECEXP (disp, 0, 0), Pmode);
8994 case UNSPEC_GOTTPOFF:
8995 case UNSPEC_GOTNTPOFF:
8996 case UNSPEC_INDNTPOFF:
8999 disp = XVECEXP (disp, 0, 0);
9000 return (GET_CODE (disp) == SYMBOL_REF
9001 && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_INITIAL_EXEC);
9003 disp = XVECEXP (disp, 0, 0);
9004 return (GET_CODE (disp) == SYMBOL_REF
9005 && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_EXEC);
9007 disp = XVECEXP (disp, 0, 0);
9008 return (GET_CODE (disp) == SYMBOL_REF
9009 && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_DYNAMIC);
9015 /* GO_IF_LEGITIMATE_ADDRESS recognizes an RTL expression that is a valid
9016 memory address for an instruction. The MODE argument is the machine mode
9017 for the MEM expression that wants to use this address.
9019 It only recognizes address in canonical form. LEGITIMIZE_ADDRESS should
9020 convert common non-canonical forms to canonical form so that they will
9024 legitimate_address_p (enum machine_mode mode ATTRIBUTE_UNUSED,
9025 rtx addr, int strict)
9027 struct ix86_address parts;
9028 rtx base, index, disp;
9029 HOST_WIDE_INT scale;
9030 const char *reason = NULL;
9031 rtx reason_rtx = NULL_RTX;
9033 if (ix86_decompose_address (addr, &parts) <= 0)
9035 reason = "decomposition failed";
9040 index = parts.index;
9042 scale = parts.scale;
9044 /* Validate base register.
9046 Don't allow SUBREG's that span more than a word here. It can lead to spill
9047 failures when the base is one word out of a two word structure, which is
9048 represented internally as a DImode int. */
9057 else if (GET_CODE (base) == SUBREG
9058 && REG_P (SUBREG_REG (base))
9059 && GET_MODE_SIZE (GET_MODE (SUBREG_REG (base)))
9061 reg = SUBREG_REG (base);
9064 reason = "base is not a register";
9068 if (GET_MODE (base) != Pmode)
9070 reason = "base is not in Pmode";
9074 if ((strict && ! REG_OK_FOR_BASE_STRICT_P (reg))
9075 || (! strict && ! REG_OK_FOR_BASE_NONSTRICT_P (reg)))
9077 reason = "base is not valid";
9082 /* Validate index register.
9084 Don't allow SUBREG's that span more than a word here -- same as above. */
9093 else if (GET_CODE (index) == SUBREG
9094 && REG_P (SUBREG_REG (index))
9095 && GET_MODE_SIZE (GET_MODE (SUBREG_REG (index)))
9097 reg = SUBREG_REG (index);
9100 reason = "index is not a register";
9104 if (GET_MODE (index) != Pmode)
9106 reason = "index is not in Pmode";
9110 if ((strict && ! REG_OK_FOR_INDEX_STRICT_P (reg))
9111 || (! strict && ! REG_OK_FOR_INDEX_NONSTRICT_P (reg)))
9113 reason = "index is not valid";
9118 /* Validate scale factor. */
9121 reason_rtx = GEN_INT (scale);
9124 reason = "scale without index";
9128 if (scale != 2 && scale != 4 && scale != 8)
9130 reason = "scale is not a valid multiplier";
9135 /* Validate displacement. */
9140 if (GET_CODE (disp) == CONST
9141 && GET_CODE (XEXP (disp, 0)) == UNSPEC
9142 && XINT (XEXP (disp, 0), 1) != UNSPEC_MACHOPIC_OFFSET)
9143 switch (XINT (XEXP (disp, 0), 1))
9145 /* Refuse GOTOFF and GOT in 64bit mode since it is always 64bit when
9146 used. While ABI specify also 32bit relocations, we don't produce
9147 them at all and use IP relative instead. */
9150 gcc_assert (flag_pic);
9152 goto is_legitimate_pic;
9153 reason = "64bit address unspec";
9156 case UNSPEC_GOTPCREL:
9157 gcc_assert (flag_pic);
9158 goto is_legitimate_pic;
9160 case UNSPEC_GOTTPOFF:
9161 case UNSPEC_GOTNTPOFF:
9162 case UNSPEC_INDNTPOFF:
9168 reason = "invalid address unspec";
9172 else if (SYMBOLIC_CONST (disp)
9176 && MACHOPIC_INDIRECT
9177 && !machopic_operand_p (disp)
9183 if (TARGET_64BIT && (index || base))
9185 /* foo@dtpoff(%rX) is ok. */
9186 if (GET_CODE (disp) != CONST
9187 || GET_CODE (XEXP (disp, 0)) != PLUS
9188 || GET_CODE (XEXP (XEXP (disp, 0), 0)) != UNSPEC
9189 || !CONST_INT_P (XEXP (XEXP (disp, 0), 1))
9190 || (XINT (XEXP (XEXP (disp, 0), 0), 1) != UNSPEC_DTPOFF
9191 && XINT (XEXP (XEXP (disp, 0), 0), 1) != UNSPEC_NTPOFF))
9193 reason = "non-constant pic memory reference";
9197 else if (! legitimate_pic_address_disp_p (disp))
9199 reason = "displacement is an invalid pic construct";
9203 /* This code used to verify that a symbolic pic displacement
9204 includes the pic_offset_table_rtx register.
9206 While this is good idea, unfortunately these constructs may
9207 be created by "adds using lea" optimization for incorrect
9216 This code is nonsensical, but results in addressing
9217 GOT table with pic_offset_table_rtx base. We can't
9218 just refuse it easily, since it gets matched by
9219 "addsi3" pattern, that later gets split to lea in the
9220 case output register differs from input. While this
9221 can be handled by separate addsi pattern for this case
9222 that never results in lea, this seems to be easier and
9223 correct fix for crash to disable this test. */
9225 else if (GET_CODE (disp) != LABEL_REF
9226 && !CONST_INT_P (disp)
9227 && (GET_CODE (disp) != CONST
9228 || !legitimate_constant_p (disp))
9229 && (GET_CODE (disp) != SYMBOL_REF
9230 || !legitimate_constant_p (disp)))
9232 reason = "displacement is not constant";
9235 else if (TARGET_64BIT
9236 && !x86_64_immediate_operand (disp, VOIDmode))
9238 reason = "displacement is out of range";
9243 /* Everything looks valid. */
9250 /* Return a unique alias set for the GOT. */
9252 static alias_set_type
9253 ix86_GOT_alias_set (void)
9255 static alias_set_type set = -1;
9257 set = new_alias_set ();
9261 /* Return a legitimate reference for ORIG (an address) using the
9262 register REG. If REG is 0, a new pseudo is generated.
9264 There are two types of references that must be handled:
9266 1. Global data references must load the address from the GOT, via
9267 the PIC reg. An insn is emitted to do this load, and the reg is
9270 2. Static data references, constant pool addresses, and code labels
9271 compute the address as an offset from the GOT, whose base is in
9272 the PIC reg. Static data objects have SYMBOL_FLAG_LOCAL set to
9273 differentiate them from global data objects. The returned
9274 address is the PIC reg + an unspec constant.
9276 GO_IF_LEGITIMATE_ADDRESS rejects symbolic references unless the PIC
9277 reg also appears in the address. */
9280 legitimize_pic_address (rtx orig, rtx reg)
9287 if (TARGET_MACHO && !TARGET_64BIT)
9290 reg = gen_reg_rtx (Pmode);
9291 /* Use the generic Mach-O PIC machinery. */
9292 return machopic_legitimize_pic_address (orig, GET_MODE (orig), reg);
9296 if (TARGET_64BIT && legitimate_pic_address_disp_p (addr))
9298 else if (TARGET_64BIT
9299 && ix86_cmodel != CM_SMALL_PIC
9300 && gotoff_operand (addr, Pmode))
9303 /* This symbol may be referenced via a displacement from the PIC
9304 base address (@GOTOFF). */
9306 if (reload_in_progress)
9307 df_set_regs_ever_live (PIC_OFFSET_TABLE_REGNUM, true);
9308 if (GET_CODE (addr) == CONST)
9309 addr = XEXP (addr, 0);
9310 if (GET_CODE (addr) == PLUS)
9312 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, XEXP (addr, 0)),
9314 new_rtx = gen_rtx_PLUS (Pmode, new_rtx, XEXP (addr, 1));
9317 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOTOFF);
9318 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
9320 tmpreg = gen_reg_rtx (Pmode);
9323 emit_move_insn (tmpreg, new_rtx);
9327 new_rtx = expand_simple_binop (Pmode, PLUS, reg, pic_offset_table_rtx,
9328 tmpreg, 1, OPTAB_DIRECT);
9331 else new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, tmpreg);
9333 else if (!TARGET_64BIT && gotoff_operand (addr, Pmode))
9335 /* This symbol may be referenced via a displacement from the PIC
9336 base address (@GOTOFF). */
9338 if (reload_in_progress)
9339 df_set_regs_ever_live (PIC_OFFSET_TABLE_REGNUM, true);
9340 if (GET_CODE (addr) == CONST)
9341 addr = XEXP (addr, 0);
9342 if (GET_CODE (addr) == PLUS)
9344 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, XEXP (addr, 0)),
9346 new_rtx = gen_rtx_PLUS (Pmode, new_rtx, XEXP (addr, 1));
9349 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOTOFF);
9350 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
9351 new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
9355 emit_move_insn (reg, new_rtx);
9359 else if ((GET_CODE (addr) == SYMBOL_REF && SYMBOL_REF_TLS_MODEL (addr) == 0)
9360 /* We can't use @GOTOFF for text labels on VxWorks;
9361 see gotoff_operand. */
9362 || (TARGET_VXWORKS_RTP && GET_CODE (addr) == LABEL_REF))
9364 if (TARGET_DLLIMPORT_DECL_ATTRIBUTES)
9366 if (GET_CODE (addr) == SYMBOL_REF && SYMBOL_REF_DLLIMPORT_P (addr))
9367 return legitimize_dllimport_symbol (addr, true);
9368 if (GET_CODE (addr) == CONST && GET_CODE (XEXP (addr, 0)) == PLUS
9369 && GET_CODE (XEXP (XEXP (addr, 0), 0)) == SYMBOL_REF
9370 && SYMBOL_REF_DLLIMPORT_P (XEXP (XEXP (addr, 0), 0)))
9372 rtx t = legitimize_dllimport_symbol (XEXP (XEXP (addr, 0), 0), true);
9373 return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (addr, 0), 1));
9377 if (TARGET_64BIT && ix86_cmodel != CM_LARGE_PIC)
9379 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOTPCREL);
9380 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
9381 new_rtx = gen_const_mem (Pmode, new_rtx);
9382 set_mem_alias_set (new_rtx, ix86_GOT_alias_set ());
9385 reg = gen_reg_rtx (Pmode);
9386 /* Use directly gen_movsi, otherwise the address is loaded
9387 into register for CSE. We don't want to CSE this addresses,
9388 instead we CSE addresses from the GOT table, so skip this. */
9389 emit_insn (gen_movsi (reg, new_rtx));
9394 /* This symbol must be referenced via a load from the
9395 Global Offset Table (@GOT). */
9397 if (reload_in_progress)
9398 df_set_regs_ever_live (PIC_OFFSET_TABLE_REGNUM, true);
9399 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOT);
9400 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
9402 new_rtx = force_reg (Pmode, new_rtx);
9403 new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
9404 new_rtx = gen_const_mem (Pmode, new_rtx);
9405 set_mem_alias_set (new_rtx, ix86_GOT_alias_set ());
9408 reg = gen_reg_rtx (Pmode);
9409 emit_move_insn (reg, new_rtx);
9415 if (CONST_INT_P (addr)
9416 && !x86_64_immediate_operand (addr, VOIDmode))
9420 emit_move_insn (reg, addr);
9424 new_rtx = force_reg (Pmode, addr);
9426 else if (GET_CODE (addr) == CONST)
9428 addr = XEXP (addr, 0);
9430 /* We must match stuff we generate before. Assume the only
9431 unspecs that can get here are ours. Not that we could do
9432 anything with them anyway.... */
9433 if (GET_CODE (addr) == UNSPEC
9434 || (GET_CODE (addr) == PLUS
9435 && GET_CODE (XEXP (addr, 0)) == UNSPEC))
9437 gcc_assert (GET_CODE (addr) == PLUS);
9439 if (GET_CODE (addr) == PLUS)
9441 rtx op0 = XEXP (addr, 0), op1 = XEXP (addr, 1);
9443 /* Check first to see if this is a constant offset from a @GOTOFF
9444 symbol reference. */
9445 if (gotoff_operand (op0, Pmode)
9446 && CONST_INT_P (op1))
9450 if (reload_in_progress)
9451 df_set_regs_ever_live (PIC_OFFSET_TABLE_REGNUM, true);
9452 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, op0),
9454 new_rtx = gen_rtx_PLUS (Pmode, new_rtx, op1);
9455 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
9456 new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
9460 emit_move_insn (reg, new_rtx);
9466 if (INTVAL (op1) < -16*1024*1024
9467 || INTVAL (op1) >= 16*1024*1024)
9469 if (!x86_64_immediate_operand (op1, Pmode))
9470 op1 = force_reg (Pmode, op1);
9471 new_rtx = gen_rtx_PLUS (Pmode, force_reg (Pmode, op0), op1);
9477 base = legitimize_pic_address (XEXP (addr, 0), reg);
9478 new_rtx = legitimize_pic_address (XEXP (addr, 1),
9479 base == reg ? NULL_RTX : reg);
9481 if (CONST_INT_P (new_rtx))
9482 new_rtx = plus_constant (base, INTVAL (new_rtx));
9485 if (GET_CODE (new_rtx) == PLUS && CONSTANT_P (XEXP (new_rtx, 1)))
9487 base = gen_rtx_PLUS (Pmode, base, XEXP (new_rtx, 0));
9488 new_rtx = XEXP (new_rtx, 1);
9490 new_rtx = gen_rtx_PLUS (Pmode, base, new_rtx);
9498 /* Load the thread pointer. If TO_REG is true, force it into a register. */
9501 get_thread_pointer (int to_reg)
9505 tp = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx), UNSPEC_TP);
9509 reg = gen_reg_rtx (Pmode);
9510 insn = gen_rtx_SET (VOIDmode, reg, tp);
9511 insn = emit_insn (insn);
9516 /* A subroutine of legitimize_address and ix86_expand_move. FOR_MOV is
9517 false if we expect this to be used for a memory address and true if
9518 we expect to load the address into a register. */
9521 legitimize_tls_address (rtx x, enum tls_model model, int for_mov)
9523 rtx dest, base, off, pic, tp;
9528 case TLS_MODEL_GLOBAL_DYNAMIC:
9529 dest = gen_reg_rtx (Pmode);
9530 tp = TARGET_GNU2_TLS ? get_thread_pointer (1) : 0;
9532 if (TARGET_64BIT && ! TARGET_GNU2_TLS)
9534 rtx rax = gen_rtx_REG (Pmode, AX_REG), insns;
9537 emit_call_insn (gen_tls_global_dynamic_64 (rax, x));
9538 insns = get_insns ();
9541 RTL_CONST_CALL_P (insns) = 1;
9542 emit_libcall_block (insns, dest, rax, x);
9544 else if (TARGET_64BIT && TARGET_GNU2_TLS)
9545 emit_insn (gen_tls_global_dynamic_64 (dest, x));
9547 emit_insn (gen_tls_global_dynamic_32 (dest, x));
9549 if (TARGET_GNU2_TLS)
9551 dest = force_reg (Pmode, gen_rtx_PLUS (Pmode, tp, dest));
9553 set_unique_reg_note (get_last_insn (), REG_EQUIV, x);
9557 case TLS_MODEL_LOCAL_DYNAMIC:
9558 base = gen_reg_rtx (Pmode);
9559 tp = TARGET_GNU2_TLS ? get_thread_pointer (1) : 0;
9561 if (TARGET_64BIT && ! TARGET_GNU2_TLS)
9563 rtx rax = gen_rtx_REG (Pmode, AX_REG), insns, note;
9566 emit_call_insn (gen_tls_local_dynamic_base_64 (rax));
9567 insns = get_insns ();
9570 note = gen_rtx_EXPR_LIST (VOIDmode, const0_rtx, NULL);
9571 note = gen_rtx_EXPR_LIST (VOIDmode, ix86_tls_get_addr (), note);
9572 RTL_CONST_CALL_P (insns) = 1;
9573 emit_libcall_block (insns, base, rax, note);
9575 else if (TARGET_64BIT && TARGET_GNU2_TLS)
9576 emit_insn (gen_tls_local_dynamic_base_64 (base));
9578 emit_insn (gen_tls_local_dynamic_base_32 (base));
9580 if (TARGET_GNU2_TLS)
9582 rtx x = ix86_tls_module_base ();
9584 set_unique_reg_note (get_last_insn (), REG_EQUIV,
9585 gen_rtx_MINUS (Pmode, x, tp));
9588 off = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x), UNSPEC_DTPOFF);
9589 off = gen_rtx_CONST (Pmode, off);
9591 dest = force_reg (Pmode, gen_rtx_PLUS (Pmode, base, off));
9593 if (TARGET_GNU2_TLS)
9595 dest = force_reg (Pmode, gen_rtx_PLUS (Pmode, dest, tp));
9597 set_unique_reg_note (get_last_insn (), REG_EQUIV, x);
9602 case TLS_MODEL_INITIAL_EXEC:
9606 type = UNSPEC_GOTNTPOFF;
9610 if (reload_in_progress)
9611 df_set_regs_ever_live (PIC_OFFSET_TABLE_REGNUM, true);
9612 pic = pic_offset_table_rtx;
9613 type = TARGET_ANY_GNU_TLS ? UNSPEC_GOTNTPOFF : UNSPEC_GOTTPOFF;
9615 else if (!TARGET_ANY_GNU_TLS)
9617 pic = gen_reg_rtx (Pmode);
9618 emit_insn (gen_set_got (pic));
9619 type = UNSPEC_GOTTPOFF;
9624 type = UNSPEC_INDNTPOFF;
9627 off = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x), type);
9628 off = gen_rtx_CONST (Pmode, off);
9630 off = gen_rtx_PLUS (Pmode, pic, off);
9631 off = gen_const_mem (Pmode, off);
9632 set_mem_alias_set (off, ix86_GOT_alias_set ());
9634 if (TARGET_64BIT || TARGET_ANY_GNU_TLS)
9636 base = get_thread_pointer (for_mov || !TARGET_TLS_DIRECT_SEG_REFS);
9637 off = force_reg (Pmode, off);
9638 return gen_rtx_PLUS (Pmode, base, off);
9642 base = get_thread_pointer (true);
9643 dest = gen_reg_rtx (Pmode);
9644 emit_insn (gen_subsi3 (dest, base, off));
9648 case TLS_MODEL_LOCAL_EXEC:
9649 off = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x),
9650 (TARGET_64BIT || TARGET_ANY_GNU_TLS)
9651 ? UNSPEC_NTPOFF : UNSPEC_TPOFF);
9652 off = gen_rtx_CONST (Pmode, off);
9654 if (TARGET_64BIT || TARGET_ANY_GNU_TLS)
9656 base = get_thread_pointer (for_mov || !TARGET_TLS_DIRECT_SEG_REFS);
9657 return gen_rtx_PLUS (Pmode, base, off);
9661 base = get_thread_pointer (true);
9662 dest = gen_reg_rtx (Pmode);
9663 emit_insn (gen_subsi3 (dest, base, off));
9674 /* Create or return the unique __imp_DECL dllimport symbol corresponding
9677 static GTY((if_marked ("tree_map_marked_p"), param_is (struct tree_map)))
9678 htab_t dllimport_map;
9681 get_dllimport_decl (tree decl)
9683 struct tree_map *h, in;
9687 size_t namelen, prefixlen;
9693 dllimport_map = htab_create_ggc (512, tree_map_hash, tree_map_eq, 0);
9695 in.hash = htab_hash_pointer (decl);
9696 in.base.from = decl;
9697 loc = htab_find_slot_with_hash (dllimport_map, &in, in.hash, INSERT);
9698 h = (struct tree_map *) *loc;
9702 *loc = h = GGC_NEW (struct tree_map);
9704 h->base.from = decl;
9705 h->to = to = build_decl (VAR_DECL, NULL, ptr_type_node);
9706 DECL_ARTIFICIAL (to) = 1;
9707 DECL_IGNORED_P (to) = 1;
9708 DECL_EXTERNAL (to) = 1;
9709 TREE_READONLY (to) = 1;
9711 name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
9712 name = targetm.strip_name_encoding (name);
9713 prefix = name[0] == FASTCALL_PREFIX || user_label_prefix[0] == 0
9714 ? "*__imp_" : "*__imp__";
9715 namelen = strlen (name);
9716 prefixlen = strlen (prefix);
9717 imp_name = (char *) alloca (namelen + prefixlen + 1);
9718 memcpy (imp_name, prefix, prefixlen);
9719 memcpy (imp_name + prefixlen, name, namelen + 1);
9721 name = ggc_alloc_string (imp_name, namelen + prefixlen);
9722 rtl = gen_rtx_SYMBOL_REF (Pmode, name);
9723 SET_SYMBOL_REF_DECL (rtl, to);
9724 SYMBOL_REF_FLAGS (rtl) = SYMBOL_FLAG_LOCAL;
9726 rtl = gen_const_mem (Pmode, rtl);
9727 set_mem_alias_set (rtl, ix86_GOT_alias_set ());
9729 SET_DECL_RTL (to, rtl);
9730 SET_DECL_ASSEMBLER_NAME (to, get_identifier (name));
9735 /* Expand SYMBOL into its corresponding dllimport symbol. WANT_REG is
9736 true if we require the result be a register. */
9739 legitimize_dllimport_symbol (rtx symbol, bool want_reg)
9744 gcc_assert (SYMBOL_REF_DECL (symbol));
9745 imp_decl = get_dllimport_decl (SYMBOL_REF_DECL (symbol));
9747 x = DECL_RTL (imp_decl);
9749 x = force_reg (Pmode, x);
9753 /* Try machine-dependent ways of modifying an illegitimate address
9754 to be legitimate. If we find one, return the new, valid address.
9755 This macro is used in only one place: `memory_address' in explow.c.
9757 OLDX is the address as it was before break_out_memory_refs was called.
9758 In some cases it is useful to look at this to decide what needs to be done.
9760 MODE and WIN are passed so that this macro can use
9761 GO_IF_LEGITIMATE_ADDRESS.
9763 It is always safe for this macro to do nothing. It exists to recognize
9764 opportunities to optimize the output.
9766 For the 80386, we handle X+REG by loading X into a register R and
9767 using R+REG. R will go in a general reg and indexing will be used.
9768 However, if REG is a broken-out memory address or multiplication,
9769 nothing needs to be done because REG can certainly go in a general reg.
9771 When -fpic is used, special handling is needed for symbolic references.
9772 See comments by legitimize_pic_address in i386.c for details. */
9775 legitimize_address (rtx x, rtx oldx ATTRIBUTE_UNUSED, enum machine_mode mode)
9780 log = GET_CODE (x) == SYMBOL_REF ? SYMBOL_REF_TLS_MODEL (x) : 0;
9782 return legitimize_tls_address (x, (enum tls_model) log, false);
9783 if (GET_CODE (x) == CONST
9784 && GET_CODE (XEXP (x, 0)) == PLUS
9785 && GET_CODE (XEXP (XEXP (x, 0), 0)) == SYMBOL_REF
9786 && (log = SYMBOL_REF_TLS_MODEL (XEXP (XEXP (x, 0), 0))))
9788 rtx t = legitimize_tls_address (XEXP (XEXP (x, 0), 0),
9789 (enum tls_model) log, false);
9790 return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (x, 0), 1));
9793 if (TARGET_DLLIMPORT_DECL_ATTRIBUTES)
9795 if (GET_CODE (x) == SYMBOL_REF && SYMBOL_REF_DLLIMPORT_P (x))
9796 return legitimize_dllimport_symbol (x, true);
9797 if (GET_CODE (x) == CONST
9798 && GET_CODE (XEXP (x, 0)) == PLUS
9799 && GET_CODE (XEXP (XEXP (x, 0), 0)) == SYMBOL_REF
9800 && SYMBOL_REF_DLLIMPORT_P (XEXP (XEXP (x, 0), 0)))
9802 rtx t = legitimize_dllimport_symbol (XEXP (XEXP (x, 0), 0), true);
9803 return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (x, 0), 1));
9807 if (flag_pic && SYMBOLIC_CONST (x))
9808 return legitimize_pic_address (x, 0);
9810 /* Canonicalize shifts by 0, 1, 2, 3 into multiply */
9811 if (GET_CODE (x) == ASHIFT
9812 && CONST_INT_P (XEXP (x, 1))
9813 && (unsigned HOST_WIDE_INT) INTVAL (XEXP (x, 1)) < 4)
9816 log = INTVAL (XEXP (x, 1));
9817 x = gen_rtx_MULT (Pmode, force_reg (Pmode, XEXP (x, 0)),
9818 GEN_INT (1 << log));
9821 if (GET_CODE (x) == PLUS)
9823 /* Canonicalize shifts by 0, 1, 2, 3 into multiply. */
9825 if (GET_CODE (XEXP (x, 0)) == ASHIFT
9826 && CONST_INT_P (XEXP (XEXP (x, 0), 1))
9827 && (unsigned HOST_WIDE_INT) INTVAL (XEXP (XEXP (x, 0), 1)) < 4)
9830 log = INTVAL (XEXP (XEXP (x, 0), 1));
9831 XEXP (x, 0) = gen_rtx_MULT (Pmode,
9832 force_reg (Pmode, XEXP (XEXP (x, 0), 0)),
9833 GEN_INT (1 << log));
9836 if (GET_CODE (XEXP (x, 1)) == ASHIFT
9837 && CONST_INT_P (XEXP (XEXP (x, 1), 1))
9838 && (unsigned HOST_WIDE_INT) INTVAL (XEXP (XEXP (x, 1), 1)) < 4)
9841 log = INTVAL (XEXP (XEXP (x, 1), 1));
9842 XEXP (x, 1) = gen_rtx_MULT (Pmode,
9843 force_reg (Pmode, XEXP (XEXP (x, 1), 0)),
9844 GEN_INT (1 << log));
9847 /* Put multiply first if it isn't already. */
9848 if (GET_CODE (XEXP (x, 1)) == MULT)
9850 rtx tmp = XEXP (x, 0);
9851 XEXP (x, 0) = XEXP (x, 1);
9856 /* Canonicalize (plus (mult (reg) (const)) (plus (reg) (const)))
9857 into (plus (plus (mult (reg) (const)) (reg)) (const)). This can be
9858 created by virtual register instantiation, register elimination, and
9859 similar optimizations. */
9860 if (GET_CODE (XEXP (x, 0)) == MULT && GET_CODE (XEXP (x, 1)) == PLUS)
9863 x = gen_rtx_PLUS (Pmode,
9864 gen_rtx_PLUS (Pmode, XEXP (x, 0),
9865 XEXP (XEXP (x, 1), 0)),
9866 XEXP (XEXP (x, 1), 1));
9870 (plus (plus (mult (reg) (const)) (plus (reg) (const))) const)
9871 into (plus (plus (mult (reg) (const)) (reg)) (const)). */
9872 else if (GET_CODE (x) == PLUS && GET_CODE (XEXP (x, 0)) == PLUS
9873 && GET_CODE (XEXP (XEXP (x, 0), 0)) == MULT
9874 && GET_CODE (XEXP (XEXP (x, 0), 1)) == PLUS
9875 && CONSTANT_P (XEXP (x, 1)))
9878 rtx other = NULL_RTX;
9880 if (CONST_INT_P (XEXP (x, 1)))
9882 constant = XEXP (x, 1);
9883 other = XEXP (XEXP (XEXP (x, 0), 1), 1);
9885 else if (CONST_INT_P (XEXP (XEXP (XEXP (x, 0), 1), 1)))
9887 constant = XEXP (XEXP (XEXP (x, 0), 1), 1);
9888 other = XEXP (x, 1);
9896 x = gen_rtx_PLUS (Pmode,
9897 gen_rtx_PLUS (Pmode, XEXP (XEXP (x, 0), 0),
9898 XEXP (XEXP (XEXP (x, 0), 1), 0)),
9899 plus_constant (other, INTVAL (constant)));
9903 if (changed && legitimate_address_p (mode, x, FALSE))
9906 if (GET_CODE (XEXP (x, 0)) == MULT)
9909 XEXP (x, 0) = force_operand (XEXP (x, 0), 0);
9912 if (GET_CODE (XEXP (x, 1)) == MULT)
9915 XEXP (x, 1) = force_operand (XEXP (x, 1), 0);
9919 && REG_P (XEXP (x, 1))
9920 && REG_P (XEXP (x, 0)))
9923 if (flag_pic && SYMBOLIC_CONST (XEXP (x, 1)))
9926 x = legitimize_pic_address (x, 0);
9929 if (changed && legitimate_address_p (mode, x, FALSE))
9932 if (REG_P (XEXP (x, 0)))
9934 rtx temp = gen_reg_rtx (Pmode);
9935 rtx val = force_operand (XEXP (x, 1), temp);
9937 emit_move_insn (temp, val);
9943 else if (REG_P (XEXP (x, 1)))
9945 rtx temp = gen_reg_rtx (Pmode);
9946 rtx val = force_operand (XEXP (x, 0), temp);
9948 emit_move_insn (temp, val);
9958 /* Print an integer constant expression in assembler syntax. Addition
9959 and subtraction are the only arithmetic that may appear in these
9960 expressions. FILE is the stdio stream to write to, X is the rtx, and
9961 CODE is the operand print code from the output string. */
9964 output_pic_addr_const (FILE *file, rtx x, int code)
9968 switch (GET_CODE (x))
9971 gcc_assert (flag_pic);
9976 if (! TARGET_MACHO || TARGET_64BIT)
9977 output_addr_const (file, x);
9980 const char *name = XSTR (x, 0);
9982 /* Mark the decl as referenced so that cgraph will
9983 output the function. */
9984 if (SYMBOL_REF_DECL (x))
9985 mark_decl_referenced (SYMBOL_REF_DECL (x));
9988 if (MACHOPIC_INDIRECT
9989 && machopic_classify_symbol (x) == MACHOPIC_UNDEFINED_FUNCTION)
9990 name = machopic_indirection_name (x, /*stub_p=*/true);
9992 assemble_name (file, name);
9994 if (!TARGET_MACHO && !(TARGET_64BIT && DEFAULT_ABI == MS_ABI)
9995 && code == 'P' && ! SYMBOL_REF_LOCAL_P (x))
9996 fputs ("@PLT", file);
10003 ASM_GENERATE_INTERNAL_LABEL (buf, "L", CODE_LABEL_NUMBER (x));
10004 assemble_name (asm_out_file, buf);
10008 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (x));
10012 /* This used to output parentheses around the expression,
10013 but that does not work on the 386 (either ATT or BSD assembler). */
10014 output_pic_addr_const (file, XEXP (x, 0), code);
10018 if (GET_MODE (x) == VOIDmode)
10020 /* We can use %d if the number is <32 bits and positive. */
10021 if (CONST_DOUBLE_HIGH (x) || CONST_DOUBLE_LOW (x) < 0)
10022 fprintf (file, "0x%lx%08lx",
10023 (unsigned long) CONST_DOUBLE_HIGH (x),
10024 (unsigned long) CONST_DOUBLE_LOW (x));
10026 fprintf (file, HOST_WIDE_INT_PRINT_DEC, CONST_DOUBLE_LOW (x));
10029 /* We can't handle floating point constants;
10030 PRINT_OPERAND must handle them. */
10031 output_operand_lossage ("floating constant misused");
10035 /* Some assemblers need integer constants to appear first. */
10036 if (CONST_INT_P (XEXP (x, 0)))
10038 output_pic_addr_const (file, XEXP (x, 0), code);
10040 output_pic_addr_const (file, XEXP (x, 1), code);
10044 gcc_assert (CONST_INT_P (XEXP (x, 1)));
10045 output_pic_addr_const (file, XEXP (x, 1), code);
10047 output_pic_addr_const (file, XEXP (x, 0), code);
10053 putc (ASSEMBLER_DIALECT == ASM_INTEL ? '(' : '[', file);
10054 output_pic_addr_const (file, XEXP (x, 0), code);
10056 output_pic_addr_const (file, XEXP (x, 1), code);
10058 putc (ASSEMBLER_DIALECT == ASM_INTEL ? ')' : ']', file);
10062 gcc_assert (XVECLEN (x, 0) == 1);
10063 output_pic_addr_const (file, XVECEXP (x, 0, 0), code);
10064 switch (XINT (x, 1))
10067 fputs ("@GOT", file);
10069 case UNSPEC_GOTOFF:
10070 fputs ("@GOTOFF", file);
10072 case UNSPEC_PLTOFF:
10073 fputs ("@PLTOFF", file);
10075 case UNSPEC_GOTPCREL:
10076 fputs (ASSEMBLER_DIALECT == ASM_ATT ?
10077 "@GOTPCREL(%rip)" : "@GOTPCREL[rip]", file);
10079 case UNSPEC_GOTTPOFF:
10080 /* FIXME: This might be @TPOFF in Sun ld too. */
10081 fputs ("@GOTTPOFF", file);
10084 fputs ("@TPOFF", file);
10086 case UNSPEC_NTPOFF:
10088 fputs ("@TPOFF", file);
10090 fputs ("@NTPOFF", file);
10092 case UNSPEC_DTPOFF:
10093 fputs ("@DTPOFF", file);
10095 case UNSPEC_GOTNTPOFF:
10097 fputs (ASSEMBLER_DIALECT == ASM_ATT ?
10098 "@GOTTPOFF(%rip)": "@GOTTPOFF[rip]", file);
10100 fputs ("@GOTNTPOFF", file);
10102 case UNSPEC_INDNTPOFF:
10103 fputs ("@INDNTPOFF", file);
10106 case UNSPEC_MACHOPIC_OFFSET:
10108 machopic_output_function_base_name (file);
10112 output_operand_lossage ("invalid UNSPEC as operand");
10118 output_operand_lossage ("invalid expression as operand");
10122 /* This is called from dwarf2out.c via TARGET_ASM_OUTPUT_DWARF_DTPREL.
10123 We need to emit DTP-relative relocations. */
10125 static void ATTRIBUTE_UNUSED
10126 i386_output_dwarf_dtprel (FILE *file, int size, rtx x)
10128 fputs (ASM_LONG, file);
10129 output_addr_const (file, x);
10130 fputs ("@DTPOFF", file);
10136 fputs (", 0", file);
10139 gcc_unreachable ();
10143 /* Return true if X is a representation of the PIC register. This copes
10144 with calls from ix86_find_base_term, where the register might have
10145 been replaced by a cselib value. */
10148 ix86_pic_register_p (rtx x)
10150 if (GET_CODE (x) == VALUE)
10151 return (pic_offset_table_rtx
10152 && rtx_equal_for_cselib_p (x, pic_offset_table_rtx));
10154 return REG_P (x) && REGNO (x) == PIC_OFFSET_TABLE_REGNUM;
10157 /* In the name of slightly smaller debug output, and to cater to
10158 general assembler lossage, recognize PIC+GOTOFF and turn it back
10159 into a direct symbol reference.
10161 On Darwin, this is necessary to avoid a crash, because Darwin
10162 has a different PIC label for each routine but the DWARF debugging
10163 information is not associated with any particular routine, so it's
10164 necessary to remove references to the PIC label from RTL stored by
10165 the DWARF output code. */
10168 ix86_delegitimize_address (rtx orig_x)
10171 /* reg_addend is NULL or a multiple of some register. */
10172 rtx reg_addend = NULL_RTX;
10173 /* const_addend is NULL or a const_int. */
10174 rtx const_addend = NULL_RTX;
10175 /* This is the result, or NULL. */
10176 rtx result = NULL_RTX;
10183 if (GET_CODE (x) != CONST
10184 || GET_CODE (XEXP (x, 0)) != UNSPEC
10185 || XINT (XEXP (x, 0), 1) != UNSPEC_GOTPCREL
10186 || !MEM_P (orig_x))
10188 return XVECEXP (XEXP (x, 0), 0, 0);
10191 if (GET_CODE (x) != PLUS
10192 || GET_CODE (XEXP (x, 1)) != CONST)
10195 if (ix86_pic_register_p (XEXP (x, 0)))
10196 /* %ebx + GOT/GOTOFF */
10198 else if (GET_CODE (XEXP (x, 0)) == PLUS)
10200 /* %ebx + %reg * scale + GOT/GOTOFF */
10201 reg_addend = XEXP (x, 0);
10202 if (ix86_pic_register_p (XEXP (reg_addend, 0)))
10203 reg_addend = XEXP (reg_addend, 1);
10204 else if (ix86_pic_register_p (XEXP (reg_addend, 1)))
10205 reg_addend = XEXP (reg_addend, 0);
10208 if (!REG_P (reg_addend)
10209 && GET_CODE (reg_addend) != MULT
10210 && GET_CODE (reg_addend) != ASHIFT)
10216 x = XEXP (XEXP (x, 1), 0);
10217 if (GET_CODE (x) == PLUS
10218 && CONST_INT_P (XEXP (x, 1)))
10220 const_addend = XEXP (x, 1);
10224 if (GET_CODE (x) == UNSPEC
10225 && ((XINT (x, 1) == UNSPEC_GOT && MEM_P (orig_x))
10226 || (XINT (x, 1) == UNSPEC_GOTOFF && !MEM_P (orig_x))))
10227 result = XVECEXP (x, 0, 0);
10229 if (TARGET_MACHO && darwin_local_data_pic (x)
10230 && !MEM_P (orig_x))
10231 result = XVECEXP (x, 0, 0);
10237 result = gen_rtx_CONST (Pmode, gen_rtx_PLUS (Pmode, result, const_addend));
10239 result = gen_rtx_PLUS (Pmode, reg_addend, result);
10243 /* If X is a machine specific address (i.e. a symbol or label being
10244 referenced as a displacement from the GOT implemented using an
10245 UNSPEC), then return the base term. Otherwise return X. */
10248 ix86_find_base_term (rtx x)
10254 if (GET_CODE (x) != CONST)
10256 term = XEXP (x, 0);
10257 if (GET_CODE (term) == PLUS
10258 && (CONST_INT_P (XEXP (term, 1))
10259 || GET_CODE (XEXP (term, 1)) == CONST_DOUBLE))
10260 term = XEXP (term, 0);
10261 if (GET_CODE (term) != UNSPEC
10262 || XINT (term, 1) != UNSPEC_GOTPCREL)
10265 return XVECEXP (term, 0, 0);
10268 return ix86_delegitimize_address (x);
10272 put_condition_code (enum rtx_code code, enum machine_mode mode, int reverse,
10273 int fp, FILE *file)
10275 const char *suffix;
10277 if (mode == CCFPmode || mode == CCFPUmode)
10279 enum rtx_code second_code, bypass_code;
10280 ix86_fp_comparison_codes (code, &bypass_code, &code, &second_code);
10281 gcc_assert (bypass_code == UNKNOWN && second_code == UNKNOWN);
10282 code = ix86_fp_compare_code_to_integer (code);
10286 code = reverse_condition (code);
10337 gcc_assert (mode == CCmode || mode == CCNOmode || mode == CCGCmode);
10341 /* ??? Use "nbe" instead of "a" for fcmov lossage on some assemblers.
10342 Those same assemblers have the same but opposite lossage on cmov. */
10343 if (mode == CCmode)
10344 suffix = fp ? "nbe" : "a";
10345 else if (mode == CCCmode)
10348 gcc_unreachable ();
10364 gcc_unreachable ();
10368 gcc_assert (mode == CCmode || mode == CCCmode);
10385 gcc_unreachable ();
10389 /* ??? As above. */
10390 gcc_assert (mode == CCmode || mode == CCCmode);
10391 suffix = fp ? "nb" : "ae";
10394 gcc_assert (mode == CCmode || mode == CCGCmode || mode == CCNOmode);
10398 /* ??? As above. */
10399 if (mode == CCmode)
10401 else if (mode == CCCmode)
10402 suffix = fp ? "nb" : "ae";
10404 gcc_unreachable ();
10407 suffix = fp ? "u" : "p";
10410 suffix = fp ? "nu" : "np";
10413 gcc_unreachable ();
10415 fputs (suffix, file);
10418 /* Print the name of register X to FILE based on its machine mode and number.
10419 If CODE is 'w', pretend the mode is HImode.
10420 If CODE is 'b', pretend the mode is QImode.
10421 If CODE is 'k', pretend the mode is SImode.
10422 If CODE is 'q', pretend the mode is DImode.
10423 If CODE is 'x', pretend the mode is V4SFmode.
10424 If CODE is 't', pretend the mode is V8SFmode.
10425 If CODE is 'h', pretend the reg is the 'high' byte register.
10426 If CODE is 'y', print "st(0)" instead of "st", if the reg is stack op.
10427 If CODE is 'd', duplicate the operand for AVX instruction.
10431 print_reg (rtx x, int code, FILE *file)
10434 bool duplicated = code == 'd' && TARGET_AVX;
10436 gcc_assert (x == pc_rtx
10437 || (REGNO (x) != ARG_POINTER_REGNUM
10438 && REGNO (x) != FRAME_POINTER_REGNUM
10439 && REGNO (x) != FLAGS_REG
10440 && REGNO (x) != FPSR_REG
10441 && REGNO (x) != FPCR_REG));
10443 if (ASSEMBLER_DIALECT == ASM_ATT)
10448 gcc_assert (TARGET_64BIT);
10449 fputs ("rip", file);
10453 if (code == 'w' || MMX_REG_P (x))
10455 else if (code == 'b')
10457 else if (code == 'k')
10459 else if (code == 'q')
10461 else if (code == 'y')
10463 else if (code == 'h')
10465 else if (code == 'x')
10467 else if (code == 't')
10470 code = GET_MODE_SIZE (GET_MODE (x));
10472 /* Irritatingly, AMD extended registers use different naming convention
10473 from the normal registers. */
10474 if (REX_INT_REG_P (x))
10476 gcc_assert (TARGET_64BIT);
10480 error ("extended registers have no high halves");
10483 fprintf (file, "r%ib", REGNO (x) - FIRST_REX_INT_REG + 8);
10486 fprintf (file, "r%iw", REGNO (x) - FIRST_REX_INT_REG + 8);
10489 fprintf (file, "r%id", REGNO (x) - FIRST_REX_INT_REG + 8);
10492 fprintf (file, "r%i", REGNO (x) - FIRST_REX_INT_REG + 8);
10495 error ("unsupported operand size for extended register");
10505 if (STACK_TOP_P (x))
10514 if (! ANY_FP_REG_P (x))
10515 putc (code == 8 && TARGET_64BIT ? 'r' : 'e', file);
10520 reg = hi_reg_name[REGNO (x)];
10523 if (REGNO (x) >= ARRAY_SIZE (qi_reg_name))
10525 reg = qi_reg_name[REGNO (x)];
10528 if (REGNO (x) >= ARRAY_SIZE (qi_high_reg_name))
10530 reg = qi_high_reg_name[REGNO (x)];
10535 gcc_assert (!duplicated);
10537 fputs (hi_reg_name[REGNO (x)] + 1, file);
10542 gcc_unreachable ();
10548 if (ASSEMBLER_DIALECT == ASM_ATT)
10549 fprintf (file, ", %%%s", reg);
10551 fprintf (file, ", %s", reg);
10555 /* Locate some local-dynamic symbol still in use by this function
10556 so that we can print its name in some tls_local_dynamic_base
10560 get_some_local_dynamic_name_1 (rtx *px, void *data ATTRIBUTE_UNUSED)
10564 if (GET_CODE (x) == SYMBOL_REF
10565 && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_DYNAMIC)
10567 cfun->machine->some_ld_name = XSTR (x, 0);
10574 static const char *
10575 get_some_local_dynamic_name (void)
10579 if (cfun->machine->some_ld_name)
10580 return cfun->machine->some_ld_name;
10582 for (insn = get_insns (); insn ; insn = NEXT_INSN (insn))
10584 && for_each_rtx (&PATTERN (insn), get_some_local_dynamic_name_1, 0))
10585 return cfun->machine->some_ld_name;
10587 gcc_unreachable ();
10590 /* Meaning of CODE:
10591 L,W,B,Q,S,T -- print the opcode suffix for specified size of operand.
10592 C -- print opcode suffix for set/cmov insn.
10593 c -- like C, but print reversed condition
10594 E,e -- likewise, but for compare-and-branch fused insn.
10595 F,f -- likewise, but for floating-point.
10596 O -- if HAVE_AS_IX86_CMOV_SUN_SYNTAX, expand to "w.", "l." or "q.",
10598 R -- print the prefix for register names.
10599 z -- print the opcode suffix for the size of the current operand.
10600 * -- print a star (in certain assembler syntax)
10601 A -- print an absolute memory reference.
10602 w -- print the operand as if it's a "word" (HImode) even if it isn't.
10603 s -- print a shift double count, followed by the assemblers argument
10605 b -- print the QImode name of the register for the indicated operand.
10606 %b0 would print %al if operands[0] is reg 0.
10607 w -- likewise, print the HImode name of the register.
10608 k -- likewise, print the SImode name of the register.
10609 q -- likewise, print the DImode name of the register.
10610 x -- likewise, print the V4SFmode name of the register.
10611 t -- likewise, print the V8SFmode name of the register.
10612 h -- print the QImode name for a "high" register, either ah, bh, ch or dh.
10613 y -- print "st(0)" instead of "st" as a register.
10614 d -- print duplicated register operand for AVX instruction.
10615 D -- print condition for SSE cmp instruction.
10616 P -- if PIC, print an @PLT suffix.
10617 X -- don't print any sort of PIC '@' suffix for a symbol.
10618 & -- print some in-use local-dynamic symbol name.
10619 H -- print a memory address offset by 8; used for sse high-parts
10620 Y -- print condition for SSE5 com* instruction.
10621 + -- print a branch hint as 'cs' or 'ds' prefix
10622 ; -- print a semicolon (after prefixes due to bug in older gas).
10626 print_operand (FILE *file, rtx x, int code)
10633 if (ASSEMBLER_DIALECT == ASM_ATT)
10638 assemble_name (file, get_some_local_dynamic_name ());
10642 switch (ASSEMBLER_DIALECT)
10649 /* Intel syntax. For absolute addresses, registers should not
10650 be surrounded by braces. */
10654 PRINT_OPERAND (file, x, 0);
10661 gcc_unreachable ();
10664 PRINT_OPERAND (file, x, 0);
10669 if (ASSEMBLER_DIALECT == ASM_ATT)
10674 if (ASSEMBLER_DIALECT == ASM_ATT)
10679 if (ASSEMBLER_DIALECT == ASM_ATT)
10684 if (ASSEMBLER_DIALECT == ASM_ATT)
10689 if (ASSEMBLER_DIALECT == ASM_ATT)
10694 if (ASSEMBLER_DIALECT == ASM_ATT)
10699 /* 387 opcodes don't get size suffixes if the operands are
10701 if (STACK_REG_P (x))
10704 /* Likewise if using Intel opcodes. */
10705 if (ASSEMBLER_DIALECT == ASM_INTEL)
10708 /* This is the size of op from size of operand. */
10709 switch (GET_MODE_SIZE (GET_MODE (x)))
10718 #ifdef HAVE_GAS_FILDS_FISTS
10728 if (GET_MODE (x) == SFmode)
10743 if (GET_MODE_CLASS (GET_MODE (x)) == MODE_INT)
10747 #ifdef GAS_MNEMONICS
10762 gcc_unreachable ();
10779 if (CONST_INT_P (x) || ! SHIFT_DOUBLE_OMITS_COUNT)
10781 PRINT_OPERAND (file, x, 0);
10782 fputs (", ", file);
10787 /* Little bit of braindamage here. The SSE compare instructions
10788 does use completely different names for the comparisons that the
10789 fp conditional moves. */
10792 switch (GET_CODE (x))
10795 fputs ("eq", file);
10798 fputs ("eq_us", file);
10801 fputs ("lt", file);
10804 fputs ("nge", file);
10807 fputs ("le", file);
10810 fputs ("ngt", file);
10813 fputs ("unord", file);
10816 fputs ("neq", file);
10819 fputs ("neq_oq", file);
10822 fputs ("ge", file);
10825 fputs ("nlt", file);
10828 fputs ("gt", file);
10831 fputs ("nle", file);
10834 fputs ("ord", file);
10837 output_operand_lossage ("operand is not a condition code, invalid operand code 'D'");
10843 switch (GET_CODE (x))
10847 fputs ("eq", file);
10851 fputs ("lt", file);
10855 fputs ("le", file);
10858 fputs ("unord", file);
10862 fputs ("neq", file);
10866 fputs ("nlt", file);
10870 fputs ("nle", file);
10873 fputs ("ord", file);
10876 output_operand_lossage ("operand is not a condition code, invalid operand code 'D'");
10882 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
10883 if (ASSEMBLER_DIALECT == ASM_ATT)
10885 switch (GET_MODE (x))
10887 case HImode: putc ('w', file); break;
10889 case SFmode: putc ('l', file); break;
10891 case DFmode: putc ('q', file); break;
10892 default: gcc_unreachable ();
10899 if (!COMPARISON_P (x))
10901 output_operand_lossage ("operand is neither a constant nor a "
10902 "condition code, invalid operand code "
10906 put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)), 0, 0, file);
10909 if (!COMPARISON_P (x))
10911 output_operand_lossage ("operand is neither a constant nor a "
10912 "condition code, invalid operand code "
10916 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
10917 if (ASSEMBLER_DIALECT == ASM_ATT)
10920 put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)), 0, 1, file);
10923 /* Like above, but reverse condition */
10925 /* Check to see if argument to %c is really a constant
10926 and not a condition code which needs to be reversed. */
10927 if (!COMPARISON_P (x))
10929 output_operand_lossage ("operand is neither a constant nor a "
10930 "condition code, invalid operand "
10934 put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)), 1, 0, file);
10937 if (!COMPARISON_P (x))
10939 output_operand_lossage ("operand is neither a constant nor a "
10940 "condition code, invalid operand "
10944 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
10945 if (ASSEMBLER_DIALECT == ASM_ATT)
10948 put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)), 1, 1, file);
10952 put_condition_code (GET_CODE (x), CCmode, 0, 0, file);
10956 put_condition_code (GET_CODE (x), CCmode, 1, 0, file);
10960 /* It doesn't actually matter what mode we use here, as we're
10961 only going to use this for printing. */
10962 x = adjust_address_nv (x, DImode, 8);
10970 || optimize_function_for_size_p (cfun) || !TARGET_BRANCH_PREDICTION_HINTS)
10973 x = find_reg_note (current_output_insn, REG_BR_PROB, 0);
10976 int pred_val = INTVAL (XEXP (x, 0));
10978 if (pred_val < REG_BR_PROB_BASE * 45 / 100
10979 || pred_val > REG_BR_PROB_BASE * 55 / 100)
10981 int taken = pred_val > REG_BR_PROB_BASE / 2;
10982 int cputaken = final_forward_branch_p (current_output_insn) == 0;
10984 /* Emit hints only in the case default branch prediction
10985 heuristics would fail. */
10986 if (taken != cputaken)
10988 /* We use 3e (DS) prefix for taken branches and
10989 2e (CS) prefix for not taken branches. */
10991 fputs ("ds ; ", file);
10993 fputs ("cs ; ", file);
11001 switch (GET_CODE (x))
11004 fputs ("neq", file);
11007 fputs ("eq", file);
11011 fputs (INTEGRAL_MODE_P (GET_MODE (x)) ? "ge" : "unlt", file);
11015 fputs (INTEGRAL_MODE_P (GET_MODE (x)) ? "gt" : "unle", file);
11019 fputs ("le", file);
11023 fputs ("lt", file);
11026 fputs ("unord", file);
11029 fputs ("ord", file);
11032 fputs ("ueq", file);
11035 fputs ("nlt", file);
11038 fputs ("nle", file);
11041 fputs ("ule", file);
11044 fputs ("ult", file);
11047 fputs ("une", file);
11050 output_operand_lossage ("operand is not a condition code, invalid operand code 'D'");
11057 fputs (" ; ", file);
11064 output_operand_lossage ("invalid operand code '%c'", code);
11069 print_reg (x, code, file);
11071 else if (MEM_P (x))
11073 /* No `byte ptr' prefix for call instructions or BLKmode operands. */
11074 if (ASSEMBLER_DIALECT == ASM_INTEL && code != 'X' && code != 'P'
11075 && GET_MODE (x) != BLKmode)
11078 switch (GET_MODE_SIZE (GET_MODE (x)))
11080 case 1: size = "BYTE"; break;
11081 case 2: size = "WORD"; break;
11082 case 4: size = "DWORD"; break;
11083 case 8: size = "QWORD"; break;
11084 case 12: size = "XWORD"; break;
11086 if (GET_MODE (x) == XFmode)
11092 gcc_unreachable ();
11095 /* Check for explicit size override (codes 'b', 'w' and 'k') */
11098 else if (code == 'w')
11100 else if (code == 'k')
11103 fputs (size, file);
11104 fputs (" PTR ", file);
11108 /* Avoid (%rip) for call operands. */
11109 if (CONSTANT_ADDRESS_P (x) && code == 'P'
11110 && !CONST_INT_P (x))
11111 output_addr_const (file, x);
11112 else if (this_is_asm_operands && ! address_operand (x, VOIDmode))
11113 output_operand_lossage ("invalid constraints for operand");
11115 output_address (x);
11118 else if (GET_CODE (x) == CONST_DOUBLE && GET_MODE (x) == SFmode)
11123 REAL_VALUE_FROM_CONST_DOUBLE (r, x);
11124 REAL_VALUE_TO_TARGET_SINGLE (r, l);
11126 if (ASSEMBLER_DIALECT == ASM_ATT)
11128 fprintf (file, "0x%08lx", (long unsigned int) l);
11131 /* These float cases don't actually occur as immediate operands. */
11132 else if (GET_CODE (x) == CONST_DOUBLE && GET_MODE (x) == DFmode)
11136 real_to_decimal (dstr, CONST_DOUBLE_REAL_VALUE (x), sizeof (dstr), 0, 1);
11137 fprintf (file, "%s", dstr);
11140 else if (GET_CODE (x) == CONST_DOUBLE
11141 && GET_MODE (x) == XFmode)
11145 real_to_decimal (dstr, CONST_DOUBLE_REAL_VALUE (x), sizeof (dstr), 0, 1);
11146 fprintf (file, "%s", dstr);
11151 /* We have patterns that allow zero sets of memory, for instance.
11152 In 64-bit mode, we should probably support all 8-byte vectors,
11153 since we can in fact encode that into an immediate. */
11154 if (GET_CODE (x) == CONST_VECTOR)
11156 gcc_assert (x == CONST0_RTX (GET_MODE (x)));
11162 if (CONST_INT_P (x) || GET_CODE (x) == CONST_DOUBLE)
11164 if (ASSEMBLER_DIALECT == ASM_ATT)
11167 else if (GET_CODE (x) == CONST || GET_CODE (x) == SYMBOL_REF
11168 || GET_CODE (x) == LABEL_REF)
11170 if (ASSEMBLER_DIALECT == ASM_ATT)
11173 fputs ("OFFSET FLAT:", file);
11176 if (CONST_INT_P (x))
11177 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (x));
11179 output_pic_addr_const (file, x, code);
11181 output_addr_const (file, x);
11185 /* Print a memory operand whose address is ADDR. */
11188 print_operand_address (FILE *file, rtx addr)
11190 struct ix86_address parts;
11191 rtx base, index, disp;
11193 int ok = ix86_decompose_address (addr, &parts);
11198 index = parts.index;
11200 scale = parts.scale;
11208 if (ASSEMBLER_DIALECT == ASM_ATT)
11210 fputs ((parts.seg == SEG_FS ? "fs:" : "gs:"), file);
11213 gcc_unreachable ();
11216 /* Use one byte shorter RIP relative addressing for 64bit mode. */
11217 if (TARGET_64BIT && !base && !index)
11221 if (GET_CODE (disp) == CONST
11222 && GET_CODE (XEXP (disp, 0)) == PLUS
11223 && CONST_INT_P (XEXP (XEXP (disp, 0), 1)))
11224 symbol = XEXP (XEXP (disp, 0), 0);
11226 if (GET_CODE (symbol) == LABEL_REF
11227 || (GET_CODE (symbol) == SYMBOL_REF
11228 && SYMBOL_REF_TLS_MODEL (symbol) == 0))
11231 if (!base && !index)
11233 /* Displacement only requires special attention. */
11235 if (CONST_INT_P (disp))
11237 if (ASSEMBLER_DIALECT == ASM_INTEL && parts.seg == SEG_DEFAULT)
11238 fputs ("ds:", file);
11239 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (disp));
11242 output_pic_addr_const (file, disp, 0);
11244 output_addr_const (file, disp);
11248 if (ASSEMBLER_DIALECT == ASM_ATT)
11253 output_pic_addr_const (file, disp, 0);
11254 else if (GET_CODE (disp) == LABEL_REF)
11255 output_asm_label (disp);
11257 output_addr_const (file, disp);
11262 print_reg (base, 0, file);
11266 print_reg (index, 0, file);
11268 fprintf (file, ",%d", scale);
11274 rtx offset = NULL_RTX;
11278 /* Pull out the offset of a symbol; print any symbol itself. */
11279 if (GET_CODE (disp) == CONST
11280 && GET_CODE (XEXP (disp, 0)) == PLUS
11281 && CONST_INT_P (XEXP (XEXP (disp, 0), 1)))
11283 offset = XEXP (XEXP (disp, 0), 1);
11284 disp = gen_rtx_CONST (VOIDmode,
11285 XEXP (XEXP (disp, 0), 0));
11289 output_pic_addr_const (file, disp, 0);
11290 else if (GET_CODE (disp) == LABEL_REF)
11291 output_asm_label (disp);
11292 else if (CONST_INT_P (disp))
11295 output_addr_const (file, disp);
11301 print_reg (base, 0, file);
11304 if (INTVAL (offset) >= 0)
11306 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (offset));
11310 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (offset));
11317 print_reg (index, 0, file);
11319 fprintf (file, "*%d", scale);
11327 output_addr_const_extra (FILE *file, rtx x)
11331 if (GET_CODE (x) != UNSPEC)
11334 op = XVECEXP (x, 0, 0);
11335 switch (XINT (x, 1))
11337 case UNSPEC_GOTTPOFF:
11338 output_addr_const (file, op);
11339 /* FIXME: This might be @TPOFF in Sun ld. */
11340 fputs ("@GOTTPOFF", file);
11343 output_addr_const (file, op);
11344 fputs ("@TPOFF", file);
11346 case UNSPEC_NTPOFF:
11347 output_addr_const (file, op);
11349 fputs ("@TPOFF", file);
11351 fputs ("@NTPOFF", file);
11353 case UNSPEC_DTPOFF:
11354 output_addr_const (file, op);
11355 fputs ("@DTPOFF", file);
11357 case UNSPEC_GOTNTPOFF:
11358 output_addr_const (file, op);
11360 fputs (ASSEMBLER_DIALECT == ASM_ATT ?
11361 "@GOTTPOFF(%rip)" : "@GOTTPOFF[rip]", file);
11363 fputs ("@GOTNTPOFF", file);
11365 case UNSPEC_INDNTPOFF:
11366 output_addr_const (file, op);
11367 fputs ("@INDNTPOFF", file);
11370 case UNSPEC_MACHOPIC_OFFSET:
11371 output_addr_const (file, op);
11373 machopic_output_function_base_name (file);
11384 /* Split one or more DImode RTL references into pairs of SImode
11385 references. The RTL can be REG, offsettable MEM, integer constant, or
11386 CONST_DOUBLE. "operands" is a pointer to an array of DImode RTL to
11387 split and "num" is its length. lo_half and hi_half are output arrays
11388 that parallel "operands". */
11391 split_di (rtx operands[], int num, rtx lo_half[], rtx hi_half[])
11395 rtx op = operands[num];
11397 /* simplify_subreg refuse to split volatile memory addresses,
11398 but we still have to handle it. */
11401 lo_half[num] = adjust_address (op, SImode, 0);
11402 hi_half[num] = adjust_address (op, SImode, 4);
11406 lo_half[num] = simplify_gen_subreg (SImode, op,
11407 GET_MODE (op) == VOIDmode
11408 ? DImode : GET_MODE (op), 0);
11409 hi_half[num] = simplify_gen_subreg (SImode, op,
11410 GET_MODE (op) == VOIDmode
11411 ? DImode : GET_MODE (op), 4);
11415 /* Split one or more TImode RTL references into pairs of DImode
11416 references. The RTL can be REG, offsettable MEM, integer constant, or
11417 CONST_DOUBLE. "operands" is a pointer to an array of DImode RTL to
11418 split and "num" is its length. lo_half and hi_half are output arrays
11419 that parallel "operands". */
11422 split_ti (rtx operands[], int num, rtx lo_half[], rtx hi_half[])
11426 rtx op = operands[num];
11428 /* simplify_subreg refuse to split volatile memory addresses, but we
11429 still have to handle it. */
11432 lo_half[num] = adjust_address (op, DImode, 0);
11433 hi_half[num] = adjust_address (op, DImode, 8);
11437 lo_half[num] = simplify_gen_subreg (DImode, op, TImode, 0);
11438 hi_half[num] = simplify_gen_subreg (DImode, op, TImode, 8);
11443 /* Output code to perform a 387 binary operation in INSN, one of PLUS,
11444 MINUS, MULT or DIV. OPERANDS are the insn operands, where operands[3]
11445 is the expression of the binary operation. The output may either be
11446 emitted here, or returned to the caller, like all output_* functions.
11448 There is no guarantee that the operands are the same mode, as they
11449 might be within FLOAT or FLOAT_EXTEND expressions. */
11451 #ifndef SYSV386_COMPAT
11452 /* Set to 1 for compatibility with brain-damaged assemblers. No-one
11453 wants to fix the assemblers because that causes incompatibility
11454 with gcc. No-one wants to fix gcc because that causes
11455 incompatibility with assemblers... You can use the option of
11456 -DSYSV386_COMPAT=0 if you recompile both gcc and gas this way. */
11457 #define SYSV386_COMPAT 1
11461 output_387_binary_op (rtx insn, rtx *operands)
11463 static char buf[40];
11466 int is_sse = SSE_REG_P (operands[0]) || SSE_REG_P (operands[1]) || SSE_REG_P (operands[2]);
11468 #ifdef ENABLE_CHECKING
11469 /* Even if we do not want to check the inputs, this documents input
11470 constraints. Which helps in understanding the following code. */
11471 if (STACK_REG_P (operands[0])
11472 && ((REG_P (operands[1])
11473 && REGNO (operands[0]) == REGNO (operands[1])
11474 && (STACK_REG_P (operands[2]) || MEM_P (operands[2])))
11475 || (REG_P (operands[2])
11476 && REGNO (operands[0]) == REGNO (operands[2])
11477 && (STACK_REG_P (operands[1]) || MEM_P (operands[1]))))
11478 && (STACK_TOP_P (operands[1]) || STACK_TOP_P (operands[2])))
11481 gcc_assert (is_sse);
11484 switch (GET_CODE (operands[3]))
11487 if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
11488 || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
11496 if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
11497 || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
11505 if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
11506 || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
11514 if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
11515 || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
11523 gcc_unreachable ();
11530 strcpy (buf, ssep);
11531 if (GET_MODE (operands[0]) == SFmode)
11532 strcat (buf, "ss\t{%2, %1, %0|%0, %1, %2}");
11534 strcat (buf, "sd\t{%2, %1, %0|%0, %1, %2}");
11538 strcpy (buf, ssep + 1);
11539 if (GET_MODE (operands[0]) == SFmode)
11540 strcat (buf, "ss\t{%2, %0|%0, %2}");
11542 strcat (buf, "sd\t{%2, %0|%0, %2}");
11548 switch (GET_CODE (operands[3]))
11552 if (REG_P (operands[2]) && REGNO (operands[0]) == REGNO (operands[2]))
11554 rtx temp = operands[2];
11555 operands[2] = operands[1];
11556 operands[1] = temp;
11559 /* know operands[0] == operands[1]. */
11561 if (MEM_P (operands[2]))
11567 if (find_regno_note (insn, REG_DEAD, REGNO (operands[2])))
11569 if (STACK_TOP_P (operands[0]))
11570 /* How is it that we are storing to a dead operand[2]?
11571 Well, presumably operands[1] is dead too. We can't
11572 store the result to st(0) as st(0) gets popped on this
11573 instruction. Instead store to operands[2] (which I
11574 think has to be st(1)). st(1) will be popped later.
11575 gcc <= 2.8.1 didn't have this check and generated
11576 assembly code that the Unixware assembler rejected. */
11577 p = "p\t{%0, %2|%2, %0}"; /* st(1) = st(0) op st(1); pop */
11579 p = "p\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0); pop */
11583 if (STACK_TOP_P (operands[0]))
11584 p = "\t{%y2, %0|%0, %y2}"; /* st(0) = st(0) op st(r2) */
11586 p = "\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0) */
11591 if (MEM_P (operands[1]))
11597 if (MEM_P (operands[2]))
11603 if (find_regno_note (insn, REG_DEAD, REGNO (operands[2])))
11606 /* The SystemV/386 SVR3.2 assembler, and probably all AT&T
11607 derived assemblers, confusingly reverse the direction of
11608 the operation for fsub{r} and fdiv{r} when the
11609 destination register is not st(0). The Intel assembler
11610 doesn't have this brain damage. Read !SYSV386_COMPAT to
11611 figure out what the hardware really does. */
11612 if (STACK_TOP_P (operands[0]))
11613 p = "{p\t%0, %2|rp\t%2, %0}";
11615 p = "{rp\t%2, %0|p\t%0, %2}";
11617 if (STACK_TOP_P (operands[0]))
11618 /* As above for fmul/fadd, we can't store to st(0). */
11619 p = "rp\t{%0, %2|%2, %0}"; /* st(1) = st(0) op st(1); pop */
11621 p = "p\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0); pop */
11626 if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
11629 if (STACK_TOP_P (operands[0]))
11630 p = "{rp\t%0, %1|p\t%1, %0}";
11632 p = "{p\t%1, %0|rp\t%0, %1}";
11634 if (STACK_TOP_P (operands[0]))
11635 p = "p\t{%0, %1|%1, %0}"; /* st(1) = st(1) op st(0); pop */
11637 p = "rp\t{%1, %0|%0, %1}"; /* st(r2) = st(0) op st(r2); pop */
11642 if (STACK_TOP_P (operands[0]))
11644 if (STACK_TOP_P (operands[1]))
11645 p = "\t{%y2, %0|%0, %y2}"; /* st(0) = st(0) op st(r2) */
11647 p = "r\t{%y1, %0|%0, %y1}"; /* st(0) = st(r1) op st(0) */
11650 else if (STACK_TOP_P (operands[1]))
11653 p = "{\t%1, %0|r\t%0, %1}";
11655 p = "r\t{%1, %0|%0, %1}"; /* st(r2) = st(0) op st(r2) */
11661 p = "{r\t%2, %0|\t%0, %2}";
11663 p = "\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0) */
11669 gcc_unreachable ();
11676 /* Return needed mode for entity in optimize_mode_switching pass. */
11679 ix86_mode_needed (int entity, rtx insn)
11681 enum attr_i387_cw mode;
11683 /* The mode UNINITIALIZED is used to store control word after a
11684 function call or ASM pattern. The mode ANY specify that function
11685 has no requirements on the control word and make no changes in the
11686 bits we are interested in. */
11689 || (NONJUMP_INSN_P (insn)
11690 && (asm_noperands (PATTERN (insn)) >= 0
11691 || GET_CODE (PATTERN (insn)) == ASM_INPUT)))
11692 return I387_CW_UNINITIALIZED;
11694 if (recog_memoized (insn) < 0)
11695 return I387_CW_ANY;
11697 mode = get_attr_i387_cw (insn);
11702 if (mode == I387_CW_TRUNC)
11707 if (mode == I387_CW_FLOOR)
11712 if (mode == I387_CW_CEIL)
11717 if (mode == I387_CW_MASK_PM)
11722 gcc_unreachable ();
11725 return I387_CW_ANY;
11728 /* Output code to initialize control word copies used by trunc?f?i and
11729 rounding patterns. CURRENT_MODE is set to current control word,
11730 while NEW_MODE is set to new control word. */
11733 emit_i387_cw_initialization (int mode)
11735 rtx stored_mode = assign_386_stack_local (HImode, SLOT_CW_STORED);
11738 enum ix86_stack_slot slot;
11740 rtx reg = gen_reg_rtx (HImode);
11742 emit_insn (gen_x86_fnstcw_1 (stored_mode));
11743 emit_move_insn (reg, copy_rtx (stored_mode));
11745 if (TARGET_64BIT || TARGET_PARTIAL_REG_STALL
11746 || optimize_function_for_size_p (cfun))
11750 case I387_CW_TRUNC:
11751 /* round toward zero (truncate) */
11752 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0c00)));
11753 slot = SLOT_CW_TRUNC;
11756 case I387_CW_FLOOR:
11757 /* round down toward -oo */
11758 emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
11759 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0400)));
11760 slot = SLOT_CW_FLOOR;
11764 /* round up toward +oo */
11765 emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
11766 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0800)));
11767 slot = SLOT_CW_CEIL;
11770 case I387_CW_MASK_PM:
11771 /* mask precision exception for nearbyint() */
11772 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0020)));
11773 slot = SLOT_CW_MASK_PM;
11777 gcc_unreachable ();
11784 case I387_CW_TRUNC:
11785 /* round toward zero (truncate) */
11786 emit_insn (gen_movsi_insv_1 (reg, GEN_INT (0xc)));
11787 slot = SLOT_CW_TRUNC;
11790 case I387_CW_FLOOR:
11791 /* round down toward -oo */
11792 emit_insn (gen_movsi_insv_1 (reg, GEN_INT (0x4)));
11793 slot = SLOT_CW_FLOOR;
11797 /* round up toward +oo */
11798 emit_insn (gen_movsi_insv_1 (reg, GEN_INT (0x8)));
11799 slot = SLOT_CW_CEIL;
11802 case I387_CW_MASK_PM:
11803 /* mask precision exception for nearbyint() */
11804 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0020)));
11805 slot = SLOT_CW_MASK_PM;
11809 gcc_unreachable ();
11813 gcc_assert (slot < MAX_386_STACK_LOCALS);
11815 new_mode = assign_386_stack_local (HImode, slot);
11816 emit_move_insn (new_mode, reg);
11819 /* Output code for INSN to convert a float to a signed int. OPERANDS
11820 are the insn operands. The output may be [HSD]Imode and the input
11821 operand may be [SDX]Fmode. */
11824 output_fix_trunc (rtx insn, rtx *operands, int fisttp)
11826 int stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG) != 0;
11827 int dimode_p = GET_MODE (operands[0]) == DImode;
11828 int round_mode = get_attr_i387_cw (insn);
11830 /* Jump through a hoop or two for DImode, since the hardware has no
11831 non-popping instruction. We used to do this a different way, but
11832 that was somewhat fragile and broke with post-reload splitters. */
11833 if ((dimode_p || fisttp) && !stack_top_dies)
11834 output_asm_insn ("fld\t%y1", operands);
11836 gcc_assert (STACK_TOP_P (operands[1]));
11837 gcc_assert (MEM_P (operands[0]));
11838 gcc_assert (GET_MODE (operands[1]) != TFmode);
11841 output_asm_insn ("fisttp%z0\t%0", operands);
11844 if (round_mode != I387_CW_ANY)
11845 output_asm_insn ("fldcw\t%3", operands);
11846 if (stack_top_dies || dimode_p)
11847 output_asm_insn ("fistp%z0\t%0", operands);
11849 output_asm_insn ("fist%z0\t%0", operands);
11850 if (round_mode != I387_CW_ANY)
11851 output_asm_insn ("fldcw\t%2", operands);
11857 /* Output code for x87 ffreep insn. The OPNO argument, which may only
11858 have the values zero or one, indicates the ffreep insn's operand
11859 from the OPERANDS array. */
11861 static const char *
11862 output_387_ffreep (rtx *operands ATTRIBUTE_UNUSED, int opno)
11864 if (TARGET_USE_FFREEP)
11865 #if HAVE_AS_IX86_FFREEP
11866 return opno ? "ffreep\t%y1" : "ffreep\t%y0";
11869 static char retval[] = ".word\t0xc_df";
11870 int regno = REGNO (operands[opno]);
11872 gcc_assert (FP_REGNO_P (regno));
11874 retval[9] = '0' + (regno - FIRST_STACK_REG);
11879 return opno ? "fstp\t%y1" : "fstp\t%y0";
11883 /* Output code for INSN to compare OPERANDS. EFLAGS_P is 1 when fcomi
11884 should be used. UNORDERED_P is true when fucom should be used. */
11887 output_fp_compare (rtx insn, rtx *operands, int eflags_p, int unordered_p)
11889 int stack_top_dies;
11890 rtx cmp_op0, cmp_op1;
11891 int is_sse = SSE_REG_P (operands[0]) || SSE_REG_P (operands[1]);
11895 cmp_op0 = operands[0];
11896 cmp_op1 = operands[1];
11900 cmp_op0 = operands[1];
11901 cmp_op1 = operands[2];
11906 static const char ucomiss[] = "vucomiss\t{%1, %0|%0, %1}";
11907 static const char ucomisd[] = "vucomisd\t{%1, %0|%0, %1}";
11908 static const char comiss[] = "vcomiss\t{%1, %0|%0, %1}";
11909 static const char comisd[] = "vcomisd\t{%1, %0|%0, %1}";
11911 if (GET_MODE (operands[0]) == SFmode)
11913 return &ucomiss[TARGET_AVX ? 0 : 1];
11915 return &comiss[TARGET_AVX ? 0 : 1];
11918 return &ucomisd[TARGET_AVX ? 0 : 1];
11920 return &comisd[TARGET_AVX ? 0 : 1];
11923 gcc_assert (STACK_TOP_P (cmp_op0));
11925 stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG) != 0;
11927 if (cmp_op1 == CONST0_RTX (GET_MODE (cmp_op1)))
11929 if (stack_top_dies)
11931 output_asm_insn ("ftst\n\tfnstsw\t%0", operands);
11932 return output_387_ffreep (operands, 1);
11935 return "ftst\n\tfnstsw\t%0";
11938 if (STACK_REG_P (cmp_op1)
11940 && find_regno_note (insn, REG_DEAD, REGNO (cmp_op1))
11941 && REGNO (cmp_op1) != FIRST_STACK_REG)
11943 /* If both the top of the 387 stack dies, and the other operand
11944 is also a stack register that dies, then this must be a
11945 `fcompp' float compare */
11949 /* There is no double popping fcomi variant. Fortunately,
11950 eflags is immune from the fstp's cc clobbering. */
11952 output_asm_insn ("fucomip\t{%y1, %0|%0, %y1}", operands);
11954 output_asm_insn ("fcomip\t{%y1, %0|%0, %y1}", operands);
11955 return output_387_ffreep (operands, 0);
11960 return "fucompp\n\tfnstsw\t%0";
11962 return "fcompp\n\tfnstsw\t%0";
11967 /* Encoded here as eflags_p | intmode | unordered_p | stack_top_dies. */
11969 static const char * const alt[16] =
11971 "fcom%z2\t%y2\n\tfnstsw\t%0",
11972 "fcomp%z2\t%y2\n\tfnstsw\t%0",
11973 "fucom%z2\t%y2\n\tfnstsw\t%0",
11974 "fucomp%z2\t%y2\n\tfnstsw\t%0",
11976 "ficom%z2\t%y2\n\tfnstsw\t%0",
11977 "ficomp%z2\t%y2\n\tfnstsw\t%0",
11981 "fcomi\t{%y1, %0|%0, %y1}",
11982 "fcomip\t{%y1, %0|%0, %y1}",
11983 "fucomi\t{%y1, %0|%0, %y1}",
11984 "fucomip\t{%y1, %0|%0, %y1}",
11995 mask = eflags_p << 3;
11996 mask |= (GET_MODE_CLASS (GET_MODE (cmp_op1)) == MODE_INT) << 2;
11997 mask |= unordered_p << 1;
11998 mask |= stack_top_dies;
12000 gcc_assert (mask < 16);
12009 ix86_output_addr_vec_elt (FILE *file, int value)
12011 const char *directive = ASM_LONG;
12015 directive = ASM_QUAD;
12017 gcc_assert (!TARGET_64BIT);
12020 fprintf (file, "%s%s%d\n", directive, LPREFIX, value);
12024 ix86_output_addr_diff_elt (FILE *file, int value, int rel)
12026 const char *directive = ASM_LONG;
12029 if (TARGET_64BIT && CASE_VECTOR_MODE == DImode)
12030 directive = ASM_QUAD;
12032 gcc_assert (!TARGET_64BIT);
12034 /* We can't use @GOTOFF for text labels on VxWorks; see gotoff_operand. */
12035 if (TARGET_64BIT || TARGET_VXWORKS_RTP)
12036 fprintf (file, "%s%s%d-%s%d\n",
12037 directive, LPREFIX, value, LPREFIX, rel);
12038 else if (HAVE_AS_GOTOFF_IN_DATA)
12039 fprintf (file, "%s%s%d@GOTOFF\n", ASM_LONG, LPREFIX, value);
12041 else if (TARGET_MACHO)
12043 fprintf (file, "%s%s%d-", ASM_LONG, LPREFIX, value);
12044 machopic_output_function_base_name (file);
12045 fprintf(file, "\n");
12049 asm_fprintf (file, "%s%U%s+[.-%s%d]\n",
12050 ASM_LONG, GOT_SYMBOL_NAME, LPREFIX, value);
12053 /* Generate either "mov $0, reg" or "xor reg, reg", as appropriate
12057 ix86_expand_clear (rtx dest)
12061 /* We play register width games, which are only valid after reload. */
12062 gcc_assert (reload_completed);
12064 /* Avoid HImode and its attendant prefix byte. */
12065 if (GET_MODE_SIZE (GET_MODE (dest)) < 4)
12066 dest = gen_rtx_REG (SImode, REGNO (dest));
12067 tmp = gen_rtx_SET (VOIDmode, dest, const0_rtx);
12069 /* This predicate should match that for movsi_xor and movdi_xor_rex64. */
12070 if (reload_completed && (!TARGET_USE_MOV0 || optimize_insn_for_speed_p ()))
12072 rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
12073 tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, clob));
12079 /* X is an unchanging MEM. If it is a constant pool reference, return
12080 the constant pool rtx, else NULL. */
12083 maybe_get_pool_constant (rtx x)
12085 x = ix86_delegitimize_address (XEXP (x, 0));
12087 if (GET_CODE (x) == SYMBOL_REF && CONSTANT_POOL_ADDRESS_P (x))
12088 return get_pool_constant (x);
12094 ix86_expand_move (enum machine_mode mode, rtx operands[])
12097 enum tls_model model;
12102 if (GET_CODE (op1) == SYMBOL_REF)
12104 model = SYMBOL_REF_TLS_MODEL (op1);
12107 op1 = legitimize_tls_address (op1, model, true);
12108 op1 = force_operand (op1, op0);
12112 else if (TARGET_DLLIMPORT_DECL_ATTRIBUTES
12113 && SYMBOL_REF_DLLIMPORT_P (op1))
12114 op1 = legitimize_dllimport_symbol (op1, false);
12116 else if (GET_CODE (op1) == CONST
12117 && GET_CODE (XEXP (op1, 0)) == PLUS
12118 && GET_CODE (XEXP (XEXP (op1, 0), 0)) == SYMBOL_REF)
12120 rtx addend = XEXP (XEXP (op1, 0), 1);
12121 rtx symbol = XEXP (XEXP (op1, 0), 0);
12124 model = SYMBOL_REF_TLS_MODEL (symbol);
12126 tmp = legitimize_tls_address (symbol, model, true);
12127 else if (TARGET_DLLIMPORT_DECL_ATTRIBUTES
12128 && SYMBOL_REF_DLLIMPORT_P (symbol))
12129 tmp = legitimize_dllimport_symbol (symbol, true);
12133 tmp = force_operand (tmp, NULL);
12134 tmp = expand_simple_binop (Pmode, PLUS, tmp, addend,
12135 op0, 1, OPTAB_DIRECT);
12141 if (flag_pic && mode == Pmode && symbolic_operand (op1, Pmode))
12143 if (TARGET_MACHO && !TARGET_64BIT)
12148 rtx temp = ((reload_in_progress
12149 || ((op0 && REG_P (op0))
12151 ? op0 : gen_reg_rtx (Pmode));
12152 op1 = machopic_indirect_data_reference (op1, temp);
12153 op1 = machopic_legitimize_pic_address (op1, mode,
12154 temp == op1 ? 0 : temp);
12156 else if (MACHOPIC_INDIRECT)
12157 op1 = machopic_indirect_data_reference (op1, 0);
12165 op1 = force_reg (Pmode, op1);
12166 else if (!TARGET_64BIT || !x86_64_movabs_operand (op1, Pmode))
12168 rtx reg = !can_create_pseudo_p () ? op0 : NULL_RTX;
12169 op1 = legitimize_pic_address (op1, reg);
12178 && (PUSH_ROUNDING (GET_MODE_SIZE (mode)) != GET_MODE_SIZE (mode)
12179 || !push_operand (op0, mode))
12181 op1 = force_reg (mode, op1);
12183 if (push_operand (op0, mode)
12184 && ! general_no_elim_operand (op1, mode))
12185 op1 = copy_to_mode_reg (mode, op1);
12187 /* Force large constants in 64bit compilation into register
12188 to get them CSEed. */
12189 if (can_create_pseudo_p ()
12190 && (mode == DImode) && TARGET_64BIT
12191 && immediate_operand (op1, mode)
12192 && !x86_64_zext_immediate_operand (op1, VOIDmode)
12193 && !register_operand (op0, mode)
12195 op1 = copy_to_mode_reg (mode, op1);
12197 if (can_create_pseudo_p ()
12198 && FLOAT_MODE_P (mode)
12199 && GET_CODE (op1) == CONST_DOUBLE)
12201 /* If we are loading a floating point constant to a register,
12202 force the value to memory now, since we'll get better code
12203 out the back end. */
12205 op1 = validize_mem (force_const_mem (mode, op1));
12206 if (!register_operand (op0, mode))
12208 rtx temp = gen_reg_rtx (mode);
12209 emit_insn (gen_rtx_SET (VOIDmode, temp, op1));
12210 emit_move_insn (op0, temp);
12216 emit_insn (gen_rtx_SET (VOIDmode, op0, op1));
12220 ix86_expand_vector_move (enum machine_mode mode, rtx operands[])
12222 rtx op0 = operands[0], op1 = operands[1];
12223 unsigned int align = GET_MODE_ALIGNMENT (mode);
12225 /* Force constants other than zero into memory. We do not know how
12226 the instructions used to build constants modify the upper 64 bits
12227 of the register, once we have that information we may be able
12228 to handle some of them more efficiently. */
12229 if (can_create_pseudo_p ()
12230 && register_operand (op0, mode)
12231 && (CONSTANT_P (op1)
12232 || (GET_CODE (op1) == SUBREG
12233 && CONSTANT_P (SUBREG_REG (op1))))
12234 && standard_sse_constant_p (op1) <= 0)
12235 op1 = validize_mem (force_const_mem (mode, op1));
12237 /* We need to check memory alignment for SSE mode since attribute
12238 can make operands unaligned. */
12239 if (can_create_pseudo_p ()
12240 && SSE_REG_MODE_P (mode)
12241 && ((MEM_P (op0) && (MEM_ALIGN (op0) < align))
12242 || (MEM_P (op1) && (MEM_ALIGN (op1) < align))))
12246 /* ix86_expand_vector_move_misalign() does not like constants ... */
12247 if (CONSTANT_P (op1)
12248 || (GET_CODE (op1) == SUBREG
12249 && CONSTANT_P (SUBREG_REG (op1))))
12250 op1 = validize_mem (force_const_mem (mode, op1));
12252 /* ... nor both arguments in memory. */
12253 if (!register_operand (op0, mode)
12254 && !register_operand (op1, mode))
12255 op1 = force_reg (mode, op1);
12257 tmp[0] = op0; tmp[1] = op1;
12258 ix86_expand_vector_move_misalign (mode, tmp);
12262 /* Make operand1 a register if it isn't already. */
12263 if (can_create_pseudo_p ()
12264 && !register_operand (op0, mode)
12265 && !register_operand (op1, mode))
12267 emit_move_insn (op0, force_reg (GET_MODE (op0), op1));
12271 emit_insn (gen_rtx_SET (VOIDmode, op0, op1));
12274 /* Implement the movmisalign patterns for SSE. Non-SSE modes go
12275 straight to ix86_expand_vector_move. */
12276 /* Code generation for scalar reg-reg moves of single and double precision data:
12277 if (x86_sse_partial_reg_dependency == true | x86_sse_split_regs == true)
12281 if (x86_sse_partial_reg_dependency == true)
12286 Code generation for scalar loads of double precision data:
12287 if (x86_sse_split_regs == true)
12288 movlpd mem, reg (gas syntax)
12292 Code generation for unaligned packed loads of single precision data
12293 (x86_sse_unaligned_move_optimal overrides x86_sse_partial_reg_dependency):
12294 if (x86_sse_unaligned_move_optimal)
12297 if (x86_sse_partial_reg_dependency == true)
12309 Code generation for unaligned packed loads of double precision data
12310 (x86_sse_unaligned_move_optimal overrides x86_sse_split_regs):
12311 if (x86_sse_unaligned_move_optimal)
12314 if (x86_sse_split_regs == true)
12327 ix86_expand_vector_move_misalign (enum machine_mode mode, rtx operands[])
12336 switch (GET_MODE_CLASS (mode))
12338 case MODE_VECTOR_INT:
12340 switch (GET_MODE_SIZE (mode))
12343 op0 = gen_lowpart (V16QImode, op0);
12344 op1 = gen_lowpart (V16QImode, op1);
12345 emit_insn (gen_avx_movdqu (op0, op1));
12348 op0 = gen_lowpart (V32QImode, op0);
12349 op1 = gen_lowpart (V32QImode, op1);
12350 emit_insn (gen_avx_movdqu256 (op0, op1));
12353 gcc_unreachable ();
12356 case MODE_VECTOR_FLOAT:
12357 op0 = gen_lowpart (mode, op0);
12358 op1 = gen_lowpart (mode, op1);
12363 emit_insn (gen_avx_movups (op0, op1));
12366 emit_insn (gen_avx_movups256 (op0, op1));
12369 emit_insn (gen_avx_movupd (op0, op1));
12372 emit_insn (gen_avx_movupd256 (op0, op1));
12375 gcc_unreachable ();
12380 gcc_unreachable ();
12388 /* If we're optimizing for size, movups is the smallest. */
12389 if (optimize_insn_for_size_p ())
12391 op0 = gen_lowpart (V4SFmode, op0);
12392 op1 = gen_lowpart (V4SFmode, op1);
12393 emit_insn (gen_sse_movups (op0, op1));
12397 /* ??? If we have typed data, then it would appear that using
12398 movdqu is the only way to get unaligned data loaded with
12400 if (TARGET_SSE2 && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
12402 op0 = gen_lowpart (V16QImode, op0);
12403 op1 = gen_lowpart (V16QImode, op1);
12404 emit_insn (gen_sse2_movdqu (op0, op1));
12408 if (TARGET_SSE2 && mode == V2DFmode)
12412 if (TARGET_SSE_UNALIGNED_MOVE_OPTIMAL)
12414 op0 = gen_lowpart (V2DFmode, op0);
12415 op1 = gen_lowpart (V2DFmode, op1);
12416 emit_insn (gen_sse2_movupd (op0, op1));
12420 /* When SSE registers are split into halves, we can avoid
12421 writing to the top half twice. */
12422 if (TARGET_SSE_SPLIT_REGS)
12424 emit_clobber (op0);
12429 /* ??? Not sure about the best option for the Intel chips.
12430 The following would seem to satisfy; the register is
12431 entirely cleared, breaking the dependency chain. We
12432 then store to the upper half, with a dependency depth
12433 of one. A rumor has it that Intel recommends two movsd
12434 followed by an unpacklpd, but this is unconfirmed. And
12435 given that the dependency depth of the unpacklpd would
12436 still be one, I'm not sure why this would be better. */
12437 zero = CONST0_RTX (V2DFmode);
12440 m = adjust_address (op1, DFmode, 0);
12441 emit_insn (gen_sse2_loadlpd (op0, zero, m));
12442 m = adjust_address (op1, DFmode, 8);
12443 emit_insn (gen_sse2_loadhpd (op0, op0, m));
12447 if (TARGET_SSE_UNALIGNED_MOVE_OPTIMAL)
12449 op0 = gen_lowpart (V4SFmode, op0);
12450 op1 = gen_lowpart (V4SFmode, op1);
12451 emit_insn (gen_sse_movups (op0, op1));
12455 if (TARGET_SSE_PARTIAL_REG_DEPENDENCY)
12456 emit_move_insn (op0, CONST0_RTX (mode));
12458 emit_clobber (op0);
12460 if (mode != V4SFmode)
12461 op0 = gen_lowpart (V4SFmode, op0);
12462 m = adjust_address (op1, V2SFmode, 0);
12463 emit_insn (gen_sse_loadlps (op0, op0, m));
12464 m = adjust_address (op1, V2SFmode, 8);
12465 emit_insn (gen_sse_loadhps (op0, op0, m));
12468 else if (MEM_P (op0))
12470 /* If we're optimizing for size, movups is the smallest. */
12471 if (optimize_insn_for_size_p ())
12473 op0 = gen_lowpart (V4SFmode, op0);
12474 op1 = gen_lowpart (V4SFmode, op1);
12475 emit_insn (gen_sse_movups (op0, op1));
12479 /* ??? Similar to above, only less clear because of quote
12480 typeless stores unquote. */
12481 if (TARGET_SSE2 && !TARGET_SSE_TYPELESS_STORES
12482 && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
12484 op0 = gen_lowpart (V16QImode, op0);
12485 op1 = gen_lowpart (V16QImode, op1);
12486 emit_insn (gen_sse2_movdqu (op0, op1));
12490 if (TARGET_SSE2 && mode == V2DFmode)
12492 m = adjust_address (op0, DFmode, 0);
12493 emit_insn (gen_sse2_storelpd (m, op1));
12494 m = adjust_address (op0, DFmode, 8);
12495 emit_insn (gen_sse2_storehpd (m, op1));
12499 if (mode != V4SFmode)
12500 op1 = gen_lowpart (V4SFmode, op1);
12501 m = adjust_address (op0, V2SFmode, 0);
12502 emit_insn (gen_sse_storelps (m, op1));
12503 m = adjust_address (op0, V2SFmode, 8);
12504 emit_insn (gen_sse_storehps (m, op1));
12508 gcc_unreachable ();
12511 /* Expand a push in MODE. This is some mode for which we do not support
12512 proper push instructions, at least from the registers that we expect
12513 the value to live in. */
12516 ix86_expand_push (enum machine_mode mode, rtx x)
12520 tmp = expand_simple_binop (Pmode, PLUS, stack_pointer_rtx,
12521 GEN_INT (-GET_MODE_SIZE (mode)),
12522 stack_pointer_rtx, 1, OPTAB_DIRECT);
12523 if (tmp != stack_pointer_rtx)
12524 emit_move_insn (stack_pointer_rtx, tmp);
12526 tmp = gen_rtx_MEM (mode, stack_pointer_rtx);
12527 emit_move_insn (tmp, x);
12530 /* Helper function of ix86_fixup_binary_operands to canonicalize
12531 operand order. Returns true if the operands should be swapped. */
12534 ix86_swap_binary_operands_p (enum rtx_code code, enum machine_mode mode,
12537 rtx dst = operands[0];
12538 rtx src1 = operands[1];
12539 rtx src2 = operands[2];
12541 /* If the operation is not commutative, we can't do anything. */
12542 if (GET_RTX_CLASS (code) != RTX_COMM_ARITH)
12545 /* Highest priority is that src1 should match dst. */
12546 if (rtx_equal_p (dst, src1))
12548 if (rtx_equal_p (dst, src2))
12551 /* Next highest priority is that immediate constants come second. */
12552 if (immediate_operand (src2, mode))
12554 if (immediate_operand (src1, mode))
12557 /* Lowest priority is that memory references should come second. */
12567 /* Fix up OPERANDS to satisfy ix86_binary_operator_ok. Return the
12568 destination to use for the operation. If different from the true
12569 destination in operands[0], a copy operation will be required. */
12572 ix86_fixup_binary_operands (enum rtx_code code, enum machine_mode mode,
12575 rtx dst = operands[0];
12576 rtx src1 = operands[1];
12577 rtx src2 = operands[2];
12579 /* Canonicalize operand order. */
12580 if (ix86_swap_binary_operands_p (code, mode, operands))
12584 /* It is invalid to swap operands of different modes. */
12585 gcc_assert (GET_MODE (src1) == GET_MODE (src2));
12592 /* Both source operands cannot be in memory. */
12593 if (MEM_P (src1) && MEM_P (src2))
12595 /* Optimization: Only read from memory once. */
12596 if (rtx_equal_p (src1, src2))
12598 src2 = force_reg (mode, src2);
12602 src2 = force_reg (mode, src2);
12605 /* If the destination is memory, and we do not have matching source
12606 operands, do things in registers. */
12607 if (MEM_P (dst) && !rtx_equal_p (dst, src1))
12608 dst = gen_reg_rtx (mode);
12610 /* Source 1 cannot be a constant. */
12611 if (CONSTANT_P (src1))
12612 src1 = force_reg (mode, src1);
12614 /* Source 1 cannot be a non-matching memory. */
12615 if (MEM_P (src1) && !rtx_equal_p (dst, src1))
12616 src1 = force_reg (mode, src1);
12618 operands[1] = src1;
12619 operands[2] = src2;
12623 /* Similarly, but assume that the destination has already been
12624 set up properly. */
12627 ix86_fixup_binary_operands_no_copy (enum rtx_code code,
12628 enum machine_mode mode, rtx operands[])
12630 rtx dst = ix86_fixup_binary_operands (code, mode, operands);
12631 gcc_assert (dst == operands[0]);
12634 /* Attempt to expand a binary operator. Make the expansion closer to the
12635 actual machine, then just general_operand, which will allow 3 separate
12636 memory references (one output, two input) in a single insn. */
12639 ix86_expand_binary_operator (enum rtx_code code, enum machine_mode mode,
12642 rtx src1, src2, dst, op, clob;
12644 dst = ix86_fixup_binary_operands (code, mode, operands);
12645 src1 = operands[1];
12646 src2 = operands[2];
12648 /* Emit the instruction. */
12650 op = gen_rtx_SET (VOIDmode, dst, gen_rtx_fmt_ee (code, mode, src1, src2));
12651 if (reload_in_progress)
12653 /* Reload doesn't know about the flags register, and doesn't know that
12654 it doesn't want to clobber it. We can only do this with PLUS. */
12655 gcc_assert (code == PLUS);
12660 clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
12661 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
12664 /* Fix up the destination if needed. */
12665 if (dst != operands[0])
12666 emit_move_insn (operands[0], dst);
12669 /* Return TRUE or FALSE depending on whether the binary operator meets the
12670 appropriate constraints. */
12673 ix86_binary_operator_ok (enum rtx_code code, enum machine_mode mode,
12676 rtx dst = operands[0];
12677 rtx src1 = operands[1];
12678 rtx src2 = operands[2];
12680 /* Both source operands cannot be in memory. */
12681 if (MEM_P (src1) && MEM_P (src2))
12684 /* Canonicalize operand order for commutative operators. */
12685 if (ix86_swap_binary_operands_p (code, mode, operands))
12692 /* If the destination is memory, we must have a matching source operand. */
12693 if (MEM_P (dst) && !rtx_equal_p (dst, src1))
12696 /* Source 1 cannot be a constant. */
12697 if (CONSTANT_P (src1))
12700 /* Source 1 cannot be a non-matching memory. */
12701 if (MEM_P (src1) && !rtx_equal_p (dst, src1))
12707 /* Attempt to expand a unary operator. Make the expansion closer to the
12708 actual machine, then just general_operand, which will allow 2 separate
12709 memory references (one output, one input) in a single insn. */
12712 ix86_expand_unary_operator (enum rtx_code code, enum machine_mode mode,
12715 int matching_memory;
12716 rtx src, dst, op, clob;
12721 /* If the destination is memory, and we do not have matching source
12722 operands, do things in registers. */
12723 matching_memory = 0;
12726 if (rtx_equal_p (dst, src))
12727 matching_memory = 1;
12729 dst = gen_reg_rtx (mode);
12732 /* When source operand is memory, destination must match. */
12733 if (MEM_P (src) && !matching_memory)
12734 src = force_reg (mode, src);
12736 /* Emit the instruction. */
12738 op = gen_rtx_SET (VOIDmode, dst, gen_rtx_fmt_e (code, mode, src));
12739 if (reload_in_progress || code == NOT)
12741 /* Reload doesn't know about the flags register, and doesn't know that
12742 it doesn't want to clobber it. */
12743 gcc_assert (code == NOT);
12748 clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
12749 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
12752 /* Fix up the destination if needed. */
12753 if (dst != operands[0])
12754 emit_move_insn (operands[0], dst);
12757 /* Return TRUE or FALSE depending on whether the unary operator meets the
12758 appropriate constraints. */
12761 ix86_unary_operator_ok (enum rtx_code code ATTRIBUTE_UNUSED,
12762 enum machine_mode mode ATTRIBUTE_UNUSED,
12763 rtx operands[2] ATTRIBUTE_UNUSED)
12765 /* If one of operands is memory, source and destination must match. */
12766 if ((MEM_P (operands[0])
12767 || MEM_P (operands[1]))
12768 && ! rtx_equal_p (operands[0], operands[1]))
12773 /* Post-reload splitter for converting an SF or DFmode value in an
12774 SSE register into an unsigned SImode. */
12777 ix86_split_convert_uns_si_sse (rtx operands[])
12779 enum machine_mode vecmode;
12780 rtx value, large, zero_or_two31, input, two31, x;
12782 large = operands[1];
12783 zero_or_two31 = operands[2];
12784 input = operands[3];
12785 two31 = operands[4];
12786 vecmode = GET_MODE (large);
12787 value = gen_rtx_REG (vecmode, REGNO (operands[0]));
12789 /* Load up the value into the low element. We must ensure that the other
12790 elements are valid floats -- zero is the easiest such value. */
12793 if (vecmode == V4SFmode)
12794 emit_insn (gen_vec_setv4sf_0 (value, CONST0_RTX (V4SFmode), input));
12796 emit_insn (gen_sse2_loadlpd (value, CONST0_RTX (V2DFmode), input));
12800 input = gen_rtx_REG (vecmode, REGNO (input));
12801 emit_move_insn (value, CONST0_RTX (vecmode));
12802 if (vecmode == V4SFmode)
12803 emit_insn (gen_sse_movss (value, value, input));
12805 emit_insn (gen_sse2_movsd (value, value, input));
12808 emit_move_insn (large, two31);
12809 emit_move_insn (zero_or_two31, MEM_P (two31) ? large : two31);
12811 x = gen_rtx_fmt_ee (LE, vecmode, large, value);
12812 emit_insn (gen_rtx_SET (VOIDmode, large, x));
12814 x = gen_rtx_AND (vecmode, zero_or_two31, large);
12815 emit_insn (gen_rtx_SET (VOIDmode, zero_or_two31, x));
12817 x = gen_rtx_MINUS (vecmode, value, zero_or_two31);
12818 emit_insn (gen_rtx_SET (VOIDmode, value, x));
12820 large = gen_rtx_REG (V4SImode, REGNO (large));
12821 emit_insn (gen_ashlv4si3 (large, large, GEN_INT (31)));
12823 x = gen_rtx_REG (V4SImode, REGNO (value));
12824 if (vecmode == V4SFmode)
12825 emit_insn (gen_sse2_cvttps2dq (x, value));
12827 emit_insn (gen_sse2_cvttpd2dq (x, value));
12830 emit_insn (gen_xorv4si3 (value, value, large));
12833 /* Convert an unsigned DImode value into a DFmode, using only SSE.
12834 Expects the 64-bit DImode to be supplied in a pair of integral
12835 registers. Requires SSE2; will use SSE3 if available. For x86_32,
12836 -mfpmath=sse, !optimize_size only. */
12839 ix86_expand_convert_uns_didf_sse (rtx target, rtx input)
12841 REAL_VALUE_TYPE bias_lo_rvt, bias_hi_rvt;
12842 rtx int_xmm, fp_xmm;
12843 rtx biases, exponents;
12846 int_xmm = gen_reg_rtx (V4SImode);
12847 if (TARGET_INTER_UNIT_MOVES)
12848 emit_insn (gen_movdi_to_sse (int_xmm, input));
12849 else if (TARGET_SSE_SPLIT_REGS)
12851 emit_clobber (int_xmm);
12852 emit_move_insn (gen_lowpart (DImode, int_xmm), input);
12856 x = gen_reg_rtx (V2DImode);
12857 ix86_expand_vector_init_one_nonzero (false, V2DImode, x, input, 0);
12858 emit_move_insn (int_xmm, gen_lowpart (V4SImode, x));
12861 x = gen_rtx_CONST_VECTOR (V4SImode,
12862 gen_rtvec (4, GEN_INT (0x43300000UL),
12863 GEN_INT (0x45300000UL),
12864 const0_rtx, const0_rtx));
12865 exponents = validize_mem (force_const_mem (V4SImode, x));
12867 /* int_xmm = {0x45300000UL, fp_xmm/hi, 0x43300000, fp_xmm/lo } */
12868 emit_insn (gen_sse2_punpckldq (int_xmm, int_xmm, exponents));
12870 /* Concatenating (juxtaposing) (0x43300000UL ## fp_value_low_xmm)
12871 yields a valid DF value equal to (0x1.0p52 + double(fp_value_lo_xmm)).
12872 Similarly (0x45300000UL ## fp_value_hi_xmm) yields
12873 (0x1.0p84 + double(fp_value_hi_xmm)).
12874 Note these exponents differ by 32. */
12876 fp_xmm = copy_to_mode_reg (V2DFmode, gen_lowpart (V2DFmode, int_xmm));
12878 /* Subtract off those 0x1.0p52 and 0x1.0p84 biases, to produce values
12879 in [0,2**32-1] and [0]+[2**32,2**64-1] respectively. */
12880 real_ldexp (&bias_lo_rvt, &dconst1, 52);
12881 real_ldexp (&bias_hi_rvt, &dconst1, 84);
12882 biases = const_double_from_real_value (bias_lo_rvt, DFmode);
12883 x = const_double_from_real_value (bias_hi_rvt, DFmode);
12884 biases = gen_rtx_CONST_VECTOR (V2DFmode, gen_rtvec (2, biases, x));
12885 biases = validize_mem (force_const_mem (V2DFmode, biases));
12886 emit_insn (gen_subv2df3 (fp_xmm, fp_xmm, biases));
12888 /* Add the upper and lower DFmode values together. */
12890 emit_insn (gen_sse3_haddv2df3 (fp_xmm, fp_xmm, fp_xmm));
12893 x = copy_to_mode_reg (V2DFmode, fp_xmm);
12894 emit_insn (gen_sse2_unpckhpd (fp_xmm, fp_xmm, fp_xmm));
12895 emit_insn (gen_addv2df3 (fp_xmm, fp_xmm, x));
12898 ix86_expand_vector_extract (false, target, fp_xmm, 0);
12901 /* Not used, but eases macroization of patterns. */
12903 ix86_expand_convert_uns_sixf_sse (rtx target ATTRIBUTE_UNUSED,
12904 rtx input ATTRIBUTE_UNUSED)
12906 gcc_unreachable ();
12909 /* Convert an unsigned SImode value into a DFmode. Only currently used
12910 for SSE, but applicable anywhere. */
12913 ix86_expand_convert_uns_sidf_sse (rtx target, rtx input)
12915 REAL_VALUE_TYPE TWO31r;
12918 x = expand_simple_binop (SImode, PLUS, input, GEN_INT (-2147483647 - 1),
12919 NULL, 1, OPTAB_DIRECT);
12921 fp = gen_reg_rtx (DFmode);
12922 emit_insn (gen_floatsidf2 (fp, x));
12924 real_ldexp (&TWO31r, &dconst1, 31);
12925 x = const_double_from_real_value (TWO31r, DFmode);
12927 x = expand_simple_binop (DFmode, PLUS, fp, x, target, 0, OPTAB_DIRECT);
12929 emit_move_insn (target, x);
12932 /* Convert a signed DImode value into a DFmode. Only used for SSE in
12933 32-bit mode; otherwise we have a direct convert instruction. */
12936 ix86_expand_convert_sign_didf_sse (rtx target, rtx input)
12938 REAL_VALUE_TYPE TWO32r;
12939 rtx fp_lo, fp_hi, x;
12941 fp_lo = gen_reg_rtx (DFmode);
12942 fp_hi = gen_reg_rtx (DFmode);
12944 emit_insn (gen_floatsidf2 (fp_hi, gen_highpart (SImode, input)));
12946 real_ldexp (&TWO32r, &dconst1, 32);
12947 x = const_double_from_real_value (TWO32r, DFmode);
12948 fp_hi = expand_simple_binop (DFmode, MULT, fp_hi, x, fp_hi, 0, OPTAB_DIRECT);
12950 ix86_expand_convert_uns_sidf_sse (fp_lo, gen_lowpart (SImode, input));
12952 x = expand_simple_binop (DFmode, PLUS, fp_hi, fp_lo, target,
12955 emit_move_insn (target, x);
12958 /* Convert an unsigned SImode value into a SFmode, using only SSE.
12959 For x86_32, -mfpmath=sse, !optimize_size only. */
12961 ix86_expand_convert_uns_sisf_sse (rtx target, rtx input)
12963 REAL_VALUE_TYPE ONE16r;
12964 rtx fp_hi, fp_lo, int_hi, int_lo, x;
12966 real_ldexp (&ONE16r, &dconst1, 16);
12967 x = const_double_from_real_value (ONE16r, SFmode);
12968 int_lo = expand_simple_binop (SImode, AND, input, GEN_INT(0xffff),
12969 NULL, 0, OPTAB_DIRECT);
12970 int_hi = expand_simple_binop (SImode, LSHIFTRT, input, GEN_INT(16),
12971 NULL, 0, OPTAB_DIRECT);
12972 fp_hi = gen_reg_rtx (SFmode);
12973 fp_lo = gen_reg_rtx (SFmode);
12974 emit_insn (gen_floatsisf2 (fp_hi, int_hi));
12975 emit_insn (gen_floatsisf2 (fp_lo, int_lo));
12976 fp_hi = expand_simple_binop (SFmode, MULT, fp_hi, x, fp_hi,
12978 fp_hi = expand_simple_binop (SFmode, PLUS, fp_hi, fp_lo, target,
12980 if (!rtx_equal_p (target, fp_hi))
12981 emit_move_insn (target, fp_hi);
12984 /* A subroutine of ix86_build_signbit_mask_vector. If VECT is true,
12985 then replicate the value for all elements of the vector
12989 ix86_build_const_vector (enum machine_mode mode, bool vect, rtx value)
12996 v = gen_rtvec (4, value, value, value, value);
12997 return gen_rtx_CONST_VECTOR (V4SImode, v);
13001 v = gen_rtvec (2, value, value);
13002 return gen_rtx_CONST_VECTOR (V2DImode, v);
13006 v = gen_rtvec (4, value, value, value, value);
13008 v = gen_rtvec (4, value, CONST0_RTX (SFmode),
13009 CONST0_RTX (SFmode), CONST0_RTX (SFmode));
13010 return gen_rtx_CONST_VECTOR (V4SFmode, v);
13014 v = gen_rtvec (2, value, value);
13016 v = gen_rtvec (2, value, CONST0_RTX (DFmode));
13017 return gen_rtx_CONST_VECTOR (V2DFmode, v);
13020 gcc_unreachable ();
13024 /* A subroutine of ix86_expand_fp_absneg_operator, copysign expanders
13025 and ix86_expand_int_vcond. Create a mask for the sign bit in MODE
13026 for an SSE register. If VECT is true, then replicate the mask for
13027 all elements of the vector register. If INVERT is true, then create
13028 a mask excluding the sign bit. */
13031 ix86_build_signbit_mask (enum machine_mode mode, bool vect, bool invert)
13033 enum machine_mode vec_mode, imode;
13034 HOST_WIDE_INT hi, lo;
13039 /* Find the sign bit, sign extended to 2*HWI. */
13045 vec_mode = (mode == SImode) ? V4SImode : V4SFmode;
13046 lo = 0x80000000, hi = lo < 0;
13052 vec_mode = (mode == DImode) ? V2DImode : V2DFmode;
13053 if (HOST_BITS_PER_WIDE_INT >= 64)
13054 lo = (HOST_WIDE_INT)1 << shift, hi = -1;
13056 lo = 0, hi = (HOST_WIDE_INT)1 << (shift - HOST_BITS_PER_WIDE_INT);
13061 vec_mode = VOIDmode;
13062 if (HOST_BITS_PER_WIDE_INT >= 64)
13065 lo = 0, hi = (HOST_WIDE_INT)1 << shift;
13072 lo = 0, hi = (HOST_WIDE_INT)1 << (shift - HOST_BITS_PER_WIDE_INT);
13076 lo = ~lo, hi = ~hi;
13082 mask = immed_double_const (lo, hi, imode);
13084 vec = gen_rtvec (2, v, mask);
13085 v = gen_rtx_CONST_VECTOR (V2DImode, vec);
13086 v = copy_to_mode_reg (mode, gen_lowpart (mode, v));
13093 gcc_unreachable ();
13097 lo = ~lo, hi = ~hi;
13099 /* Force this value into the low part of a fp vector constant. */
13100 mask = immed_double_const (lo, hi, imode);
13101 mask = gen_lowpart (mode, mask);
13103 if (vec_mode == VOIDmode)
13104 return force_reg (mode, mask);
13106 v = ix86_build_const_vector (mode, vect, mask);
13107 return force_reg (vec_mode, v);
13110 /* Generate code for floating point ABS or NEG. */
13113 ix86_expand_fp_absneg_operator (enum rtx_code code, enum machine_mode mode,
13116 rtx mask, set, use, clob, dst, src;
13117 bool use_sse = false;
13118 bool vector_mode = VECTOR_MODE_P (mode);
13119 enum machine_mode elt_mode = mode;
13123 elt_mode = GET_MODE_INNER (mode);
13126 else if (mode == TFmode)
13128 else if (TARGET_SSE_MATH)
13129 use_sse = SSE_FLOAT_MODE_P (mode);
13131 /* NEG and ABS performed with SSE use bitwise mask operations.
13132 Create the appropriate mask now. */
13134 mask = ix86_build_signbit_mask (elt_mode, vector_mode, code == ABS);
13143 set = gen_rtx_fmt_ee (code == NEG ? XOR : AND, mode, src, mask);
13144 set = gen_rtx_SET (VOIDmode, dst, set);
13149 set = gen_rtx_fmt_e (code, mode, src);
13150 set = gen_rtx_SET (VOIDmode, dst, set);
13153 use = gen_rtx_USE (VOIDmode, mask);
13154 clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
13155 emit_insn (gen_rtx_PARALLEL (VOIDmode,
13156 gen_rtvec (3, set, use, clob)));
13163 /* Expand a copysign operation. Special case operand 0 being a constant. */
13166 ix86_expand_copysign (rtx operands[])
13168 enum machine_mode mode;
13169 rtx dest, op0, op1, mask, nmask;
13171 dest = operands[0];
13175 mode = GET_MODE (dest);
13177 if (GET_CODE (op0) == CONST_DOUBLE)
13179 rtx (*copysign_insn)(rtx, rtx, rtx, rtx);
13181 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
13182 op0 = simplify_unary_operation (ABS, mode, op0, mode);
13184 if (mode == SFmode || mode == DFmode)
13186 enum machine_mode vmode;
13188 vmode = mode == SFmode ? V4SFmode : V2DFmode;
13190 if (op0 == CONST0_RTX (mode))
13191 op0 = CONST0_RTX (vmode);
13196 if (mode == SFmode)
13197 v = gen_rtvec (4, op0, CONST0_RTX (SFmode),
13198 CONST0_RTX (SFmode), CONST0_RTX (SFmode));
13200 v = gen_rtvec (2, op0, CONST0_RTX (DFmode));
13202 op0 = force_reg (vmode, gen_rtx_CONST_VECTOR (vmode, v));
13205 else if (op0 != CONST0_RTX (mode))
13206 op0 = force_reg (mode, op0);
13208 mask = ix86_build_signbit_mask (mode, 0, 0);
13210 if (mode == SFmode)
13211 copysign_insn = gen_copysignsf3_const;
13212 else if (mode == DFmode)
13213 copysign_insn = gen_copysigndf3_const;
13215 copysign_insn = gen_copysigntf3_const;
13217 emit_insn (copysign_insn (dest, op0, op1, mask));
13221 rtx (*copysign_insn)(rtx, rtx, rtx, rtx, rtx, rtx);
13223 nmask = ix86_build_signbit_mask (mode, 0, 1);
13224 mask = ix86_build_signbit_mask (mode, 0, 0);
13226 if (mode == SFmode)
13227 copysign_insn = gen_copysignsf3_var;
13228 else if (mode == DFmode)
13229 copysign_insn = gen_copysigndf3_var;
13231 copysign_insn = gen_copysigntf3_var;
13233 emit_insn (copysign_insn (dest, NULL_RTX, op0, op1, nmask, mask));
13237 /* Deconstruct a copysign operation into bit masks. Operand 0 is known to
13238 be a constant, and so has already been expanded into a vector constant. */
13241 ix86_split_copysign_const (rtx operands[])
13243 enum machine_mode mode, vmode;
13244 rtx dest, op0, op1, mask, x;
13246 dest = operands[0];
13249 mask = operands[3];
13251 mode = GET_MODE (dest);
13252 vmode = GET_MODE (mask);
13254 dest = simplify_gen_subreg (vmode, dest, mode, 0);
13255 x = gen_rtx_AND (vmode, dest, mask);
13256 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
13258 if (op0 != CONST0_RTX (vmode))
13260 x = gen_rtx_IOR (vmode, dest, op0);
13261 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
13265 /* Deconstruct a copysign operation into bit masks. Operand 0 is variable,
13266 so we have to do two masks. */
13269 ix86_split_copysign_var (rtx operands[])
13271 enum machine_mode mode, vmode;
13272 rtx dest, scratch, op0, op1, mask, nmask, x;
13274 dest = operands[0];
13275 scratch = operands[1];
13278 nmask = operands[4];
13279 mask = operands[5];
13281 mode = GET_MODE (dest);
13282 vmode = GET_MODE (mask);
13284 if (rtx_equal_p (op0, op1))
13286 /* Shouldn't happen often (it's useless, obviously), but when it does
13287 we'd generate incorrect code if we continue below. */
13288 emit_move_insn (dest, op0);
13292 if (REG_P (mask) && REGNO (dest) == REGNO (mask)) /* alternative 0 */
13294 gcc_assert (REGNO (op1) == REGNO (scratch));
13296 x = gen_rtx_AND (vmode, scratch, mask);
13297 emit_insn (gen_rtx_SET (VOIDmode, scratch, x));
13300 op0 = simplify_gen_subreg (vmode, op0, mode, 0);
13301 x = gen_rtx_NOT (vmode, dest);
13302 x = gen_rtx_AND (vmode, x, op0);
13303 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
13307 if (REGNO (op1) == REGNO (scratch)) /* alternative 1,3 */
13309 x = gen_rtx_AND (vmode, scratch, mask);
13311 else /* alternative 2,4 */
13313 gcc_assert (REGNO (mask) == REGNO (scratch));
13314 op1 = simplify_gen_subreg (vmode, op1, mode, 0);
13315 x = gen_rtx_AND (vmode, scratch, op1);
13317 emit_insn (gen_rtx_SET (VOIDmode, scratch, x));
13319 if (REGNO (op0) == REGNO (dest)) /* alternative 1,2 */
13321 dest = simplify_gen_subreg (vmode, op0, mode, 0);
13322 x = gen_rtx_AND (vmode, dest, nmask);
13324 else /* alternative 3,4 */
13326 gcc_assert (REGNO (nmask) == REGNO (dest));
13328 op0 = simplify_gen_subreg (vmode, op0, mode, 0);
13329 x = gen_rtx_AND (vmode, dest, op0);
13331 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
13334 x = gen_rtx_IOR (vmode, dest, scratch);
13335 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
13338 /* Return TRUE or FALSE depending on whether the first SET in INSN
13339 has source and destination with matching CC modes, and that the
13340 CC mode is at least as constrained as REQ_MODE. */
13343 ix86_match_ccmode (rtx insn, enum machine_mode req_mode)
13346 enum machine_mode set_mode;
13348 set = PATTERN (insn);
13349 if (GET_CODE (set) == PARALLEL)
13350 set = XVECEXP (set, 0, 0);
13351 gcc_assert (GET_CODE (set) == SET);
13352 gcc_assert (GET_CODE (SET_SRC (set)) == COMPARE);
13354 set_mode = GET_MODE (SET_DEST (set));
13358 if (req_mode != CCNOmode
13359 && (req_mode != CCmode
13360 || XEXP (SET_SRC (set), 1) != const0_rtx))
13364 if (req_mode == CCGCmode)
13368 if (req_mode == CCGOCmode || req_mode == CCNOmode)
13372 if (req_mode == CCZmode)
13383 gcc_unreachable ();
13386 return (GET_MODE (SET_SRC (set)) == set_mode);
13389 /* Generate insn patterns to do an integer compare of OPERANDS. */
13392 ix86_expand_int_compare (enum rtx_code code, rtx op0, rtx op1)
13394 enum machine_mode cmpmode;
13397 cmpmode = SELECT_CC_MODE (code, op0, op1);
13398 flags = gen_rtx_REG (cmpmode, FLAGS_REG);
13400 /* This is very simple, but making the interface the same as in the
13401 FP case makes the rest of the code easier. */
13402 tmp = gen_rtx_COMPARE (cmpmode, op0, op1);
13403 emit_insn (gen_rtx_SET (VOIDmode, flags, tmp));
13405 /* Return the test that should be put into the flags user, i.e.
13406 the bcc, scc, or cmov instruction. */
13407 return gen_rtx_fmt_ee (code, VOIDmode, flags, const0_rtx);
13410 /* Figure out whether to use ordered or unordered fp comparisons.
13411 Return the appropriate mode to use. */
13414 ix86_fp_compare_mode (enum rtx_code code ATTRIBUTE_UNUSED)
13416 /* ??? In order to make all comparisons reversible, we do all comparisons
13417 non-trapping when compiling for IEEE. Once gcc is able to distinguish
13418 all forms trapping and nontrapping comparisons, we can make inequality
13419 comparisons trapping again, since it results in better code when using
13420 FCOM based compares. */
13421 return TARGET_IEEE_FP ? CCFPUmode : CCFPmode;
13425 ix86_cc_mode (enum rtx_code code, rtx op0, rtx op1)
13427 enum machine_mode mode = GET_MODE (op0);
13429 if (SCALAR_FLOAT_MODE_P (mode))
13431 gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));
13432 return ix86_fp_compare_mode (code);
13437 /* Only zero flag is needed. */
13438 case EQ: /* ZF=0 */
13439 case NE: /* ZF!=0 */
13441 /* Codes needing carry flag. */
13442 case GEU: /* CF=0 */
13443 case LTU: /* CF=1 */
13444 /* Detect overflow checks. They need just the carry flag. */
13445 if (GET_CODE (op0) == PLUS
13446 && rtx_equal_p (op1, XEXP (op0, 0)))
13450 case GTU: /* CF=0 & ZF=0 */
13451 case LEU: /* CF=1 | ZF=1 */
13452 /* Detect overflow checks. They need just the carry flag. */
13453 if (GET_CODE (op0) == MINUS
13454 && rtx_equal_p (op1, XEXP (op0, 0)))
13458 /* Codes possibly doable only with sign flag when
13459 comparing against zero. */
13460 case GE: /* SF=OF or SF=0 */
13461 case LT: /* SF<>OF or SF=1 */
13462 if (op1 == const0_rtx)
13465 /* For other cases Carry flag is not required. */
13467 /* Codes doable only with sign flag when comparing
13468 against zero, but we miss jump instruction for it
13469 so we need to use relational tests against overflow
13470 that thus needs to be zero. */
13471 case GT: /* ZF=0 & SF=OF */
13472 case LE: /* ZF=1 | SF<>OF */
13473 if (op1 == const0_rtx)
13477 /* strcmp pattern do (use flags) and combine may ask us for proper
13482 gcc_unreachable ();
13486 /* Return the fixed registers used for condition codes. */
13489 ix86_fixed_condition_code_regs (unsigned int *p1, unsigned int *p2)
13496 /* If two condition code modes are compatible, return a condition code
13497 mode which is compatible with both. Otherwise, return
13500 static enum machine_mode
13501 ix86_cc_modes_compatible (enum machine_mode m1, enum machine_mode m2)
13506 if (GET_MODE_CLASS (m1) != MODE_CC || GET_MODE_CLASS (m2) != MODE_CC)
13509 if ((m1 == CCGCmode && m2 == CCGOCmode)
13510 || (m1 == CCGOCmode && m2 == CCGCmode))
13516 gcc_unreachable ();
13546 /* These are only compatible with themselves, which we already
13552 /* Split comparison code CODE into comparisons we can do using branch
13553 instructions. BYPASS_CODE is comparison code for branch that will
13554 branch around FIRST_CODE and SECOND_CODE. If some of branches
13555 is not required, set value to UNKNOWN.
13556 We never require more than two branches. */
13559 ix86_fp_comparison_codes (enum rtx_code code, enum rtx_code *bypass_code,
13560 enum rtx_code *first_code,
13561 enum rtx_code *second_code)
13563 *first_code = code;
13564 *bypass_code = UNKNOWN;
13565 *second_code = UNKNOWN;
13567 /* The fcomi comparison sets flags as follows:
13577 case GT: /* GTU - CF=0 & ZF=0 */
13578 case GE: /* GEU - CF=0 */
13579 case ORDERED: /* PF=0 */
13580 case UNORDERED: /* PF=1 */
13581 case UNEQ: /* EQ - ZF=1 */
13582 case UNLT: /* LTU - CF=1 */
13583 case UNLE: /* LEU - CF=1 | ZF=1 */
13584 case LTGT: /* EQ - ZF=0 */
13586 case LT: /* LTU - CF=1 - fails on unordered */
13587 *first_code = UNLT;
13588 *bypass_code = UNORDERED;
13590 case LE: /* LEU - CF=1 | ZF=1 - fails on unordered */
13591 *first_code = UNLE;
13592 *bypass_code = UNORDERED;
13594 case EQ: /* EQ - ZF=1 - fails on unordered */
13595 *first_code = UNEQ;
13596 *bypass_code = UNORDERED;
13598 case NE: /* NE - ZF=0 - fails on unordered */
13599 *first_code = LTGT;
13600 *second_code = UNORDERED;
13602 case UNGE: /* GEU - CF=0 - fails on unordered */
13604 *second_code = UNORDERED;
13606 case UNGT: /* GTU - CF=0 & ZF=0 - fails on unordered */
13608 *second_code = UNORDERED;
13611 gcc_unreachable ();
13613 if (!TARGET_IEEE_FP)
13615 *second_code = UNKNOWN;
13616 *bypass_code = UNKNOWN;
13620 /* Return cost of comparison done fcom + arithmetics operations on AX.
13621 All following functions do use number of instructions as a cost metrics.
13622 In future this should be tweaked to compute bytes for optimize_size and
13623 take into account performance of various instructions on various CPUs. */
13625 ix86_fp_comparison_arithmetics_cost (enum rtx_code code)
13627 if (!TARGET_IEEE_FP)
13629 /* The cost of code output by ix86_expand_fp_compare. */
13653 gcc_unreachable ();
13657 /* Return cost of comparison done using fcomi operation.
13658 See ix86_fp_comparison_arithmetics_cost for the metrics. */
13660 ix86_fp_comparison_fcomi_cost (enum rtx_code code)
13662 enum rtx_code bypass_code, first_code, second_code;
13663 /* Return arbitrarily high cost when instruction is not supported - this
13664 prevents gcc from using it. */
13667 ix86_fp_comparison_codes (code, &bypass_code, &first_code, &second_code);
13668 return (bypass_code != UNKNOWN || second_code != UNKNOWN) + 2;
13671 /* Return cost of comparison done using sahf operation.
13672 See ix86_fp_comparison_arithmetics_cost for the metrics. */
13674 ix86_fp_comparison_sahf_cost (enum rtx_code code)
13676 enum rtx_code bypass_code, first_code, second_code;
13677 /* Return arbitrarily high cost when instruction is not preferred - this
13678 avoids gcc from using it. */
13679 if (!(TARGET_SAHF && (TARGET_USE_SAHF || optimize_insn_for_size_p ())))
13681 ix86_fp_comparison_codes (code, &bypass_code, &first_code, &second_code);
13682 return (bypass_code != UNKNOWN || second_code != UNKNOWN) + 3;
13685 /* Compute cost of the comparison done using any method.
13686 See ix86_fp_comparison_arithmetics_cost for the metrics. */
13688 ix86_fp_comparison_cost (enum rtx_code code)
13690 int fcomi_cost, sahf_cost, arithmetics_cost = 1024;
13693 fcomi_cost = ix86_fp_comparison_fcomi_cost (code);
13694 sahf_cost = ix86_fp_comparison_sahf_cost (code);
13696 min = arithmetics_cost = ix86_fp_comparison_arithmetics_cost (code);
13697 if (min > sahf_cost)
13699 if (min > fcomi_cost)
13704 /* Return true if we should use an FCOMI instruction for this
13708 ix86_use_fcomi_compare (enum rtx_code code ATTRIBUTE_UNUSED)
13710 enum rtx_code swapped_code = swap_condition (code);
13712 return ((ix86_fp_comparison_cost (code)
13713 == ix86_fp_comparison_fcomi_cost (code))
13714 || (ix86_fp_comparison_cost (swapped_code)
13715 == ix86_fp_comparison_fcomi_cost (swapped_code)));
13718 /* Swap, force into registers, or otherwise massage the two operands
13719 to a fp comparison. The operands are updated in place; the new
13720 comparison code is returned. */
13722 static enum rtx_code
13723 ix86_prepare_fp_compare_args (enum rtx_code code, rtx *pop0, rtx *pop1)
13725 enum machine_mode fpcmp_mode = ix86_fp_compare_mode (code);
13726 rtx op0 = *pop0, op1 = *pop1;
13727 enum machine_mode op_mode = GET_MODE (op0);
13728 int is_sse = TARGET_SSE_MATH && SSE_FLOAT_MODE_P (op_mode);
13730 /* All of the unordered compare instructions only work on registers.
13731 The same is true of the fcomi compare instructions. The XFmode
13732 compare instructions require registers except when comparing
13733 against zero or when converting operand 1 from fixed point to
13737 && (fpcmp_mode == CCFPUmode
13738 || (op_mode == XFmode
13739 && ! (standard_80387_constant_p (op0) == 1
13740 || standard_80387_constant_p (op1) == 1)
13741 && GET_CODE (op1) != FLOAT)
13742 || ix86_use_fcomi_compare (code)))
13744 op0 = force_reg (op_mode, op0);
13745 op1 = force_reg (op_mode, op1);
13749 /* %%% We only allow op1 in memory; op0 must be st(0). So swap
13750 things around if they appear profitable, otherwise force op0
13751 into a register. */
13753 if (standard_80387_constant_p (op0) == 0
13755 && ! (standard_80387_constant_p (op1) == 0
13759 tmp = op0, op0 = op1, op1 = tmp;
13760 code = swap_condition (code);
13764 op0 = force_reg (op_mode, op0);
13766 if (CONSTANT_P (op1))
13768 int tmp = standard_80387_constant_p (op1);
13770 op1 = validize_mem (force_const_mem (op_mode, op1));
13774 op1 = force_reg (op_mode, op1);
13777 op1 = force_reg (op_mode, op1);
13781 /* Try to rearrange the comparison to make it cheaper. */
13782 if (ix86_fp_comparison_cost (code)
13783 > ix86_fp_comparison_cost (swap_condition (code))
13784 && (REG_P (op1) || can_create_pseudo_p ()))
13787 tmp = op0, op0 = op1, op1 = tmp;
13788 code = swap_condition (code);
13790 op0 = force_reg (op_mode, op0);
13798 /* Convert comparison codes we use to represent FP comparison to integer
13799 code that will result in proper branch. Return UNKNOWN if no such code
13803 ix86_fp_compare_code_to_integer (enum rtx_code code)
13832 /* Generate insn patterns to do a floating point compare of OPERANDS. */
13835 ix86_expand_fp_compare (enum rtx_code code, rtx op0, rtx op1, rtx scratch,
13836 rtx *second_test, rtx *bypass_test)
13838 enum machine_mode fpcmp_mode, intcmp_mode;
13840 int cost = ix86_fp_comparison_cost (code);
13841 enum rtx_code bypass_code, first_code, second_code;
13843 fpcmp_mode = ix86_fp_compare_mode (code);
13844 code = ix86_prepare_fp_compare_args (code, &op0, &op1);
13847 *second_test = NULL_RTX;
13849 *bypass_test = NULL_RTX;
13851 ix86_fp_comparison_codes (code, &bypass_code, &first_code, &second_code);
13853 /* Do fcomi/sahf based test when profitable. */
13854 if (ix86_fp_comparison_arithmetics_cost (code) > cost
13855 && (bypass_code == UNKNOWN || bypass_test)
13856 && (second_code == UNKNOWN || second_test))
13858 tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1);
13859 tmp = gen_rtx_SET (VOIDmode, gen_rtx_REG (fpcmp_mode, FLAGS_REG),
13865 gcc_assert (TARGET_SAHF);
13868 scratch = gen_reg_rtx (HImode);
13869 tmp2 = gen_rtx_CLOBBER (VOIDmode, scratch);
13871 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, tmp2)));
13874 /* The FP codes work out to act like unsigned. */
13875 intcmp_mode = fpcmp_mode;
13877 if (bypass_code != UNKNOWN)
13878 *bypass_test = gen_rtx_fmt_ee (bypass_code, VOIDmode,
13879 gen_rtx_REG (intcmp_mode, FLAGS_REG),
13881 if (second_code != UNKNOWN)
13882 *second_test = gen_rtx_fmt_ee (second_code, VOIDmode,
13883 gen_rtx_REG (intcmp_mode, FLAGS_REG),
13888 /* Sadness wrt reg-stack pops killing fpsr -- gotta get fnstsw first. */
13889 tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1);
13890 tmp2 = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW);
13892 scratch = gen_reg_rtx (HImode);
13893 emit_insn (gen_rtx_SET (VOIDmode, scratch, tmp2));
13895 /* In the unordered case, we have to check C2 for NaN's, which
13896 doesn't happen to work out to anything nice combination-wise.
13897 So do some bit twiddling on the value we've got in AH to come
13898 up with an appropriate set of condition codes. */
13900 intcmp_mode = CCNOmode;
13905 if (code == GT || !TARGET_IEEE_FP)
13907 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x45)));
13912 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
13913 emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
13914 emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x44)));
13915 intcmp_mode = CCmode;
13921 if (code == LT && TARGET_IEEE_FP)
13923 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
13924 emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x01)));
13925 intcmp_mode = CCmode;
13930 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x01)));
13936 if (code == GE || !TARGET_IEEE_FP)
13938 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x05)));
13943 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
13944 emit_insn (gen_xorqi_cc_ext_1 (scratch, scratch,
13951 if (code == LE && TARGET_IEEE_FP)
13953 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
13954 emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
13955 emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
13956 intcmp_mode = CCmode;
13961 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x45)));
13967 if (code == EQ && TARGET_IEEE_FP)
13969 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
13970 emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
13971 intcmp_mode = CCmode;
13976 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x40)));
13983 if (code == NE && TARGET_IEEE_FP)
13985 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
13986 emit_insn (gen_xorqi_cc_ext_1 (scratch, scratch,
13992 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x40)));
13998 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x04)));
14002 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x04)));
14007 gcc_unreachable ();
14011 /* Return the test that should be put into the flags user, i.e.
14012 the bcc, scc, or cmov instruction. */
14013 return gen_rtx_fmt_ee (code, VOIDmode,
14014 gen_rtx_REG (intcmp_mode, FLAGS_REG),
14019 ix86_expand_compare (enum rtx_code code, rtx *second_test, rtx *bypass_test)
14022 op0 = ix86_compare_op0;
14023 op1 = ix86_compare_op1;
14026 *second_test = NULL_RTX;
14028 *bypass_test = NULL_RTX;
14030 if (ix86_compare_emitted)
14032 ret = gen_rtx_fmt_ee (code, VOIDmode, ix86_compare_emitted, const0_rtx);
14033 ix86_compare_emitted = NULL_RTX;
14035 else if (SCALAR_FLOAT_MODE_P (GET_MODE (op0)))
14037 gcc_assert (!DECIMAL_FLOAT_MODE_P (GET_MODE (op0)));
14038 ret = ix86_expand_fp_compare (code, op0, op1, NULL_RTX,
14039 second_test, bypass_test);
14042 ret = ix86_expand_int_compare (code, op0, op1);
14047 /* Return true if the CODE will result in nontrivial jump sequence. */
14049 ix86_fp_jump_nontrivial_p (enum rtx_code code)
14051 enum rtx_code bypass_code, first_code, second_code;
14054 ix86_fp_comparison_codes (code, &bypass_code, &first_code, &second_code);
14055 return bypass_code != UNKNOWN || second_code != UNKNOWN;
14059 ix86_expand_branch (enum rtx_code code, rtx label)
14063 /* If we have emitted a compare insn, go straight to simple.
14064 ix86_expand_compare won't emit anything if ix86_compare_emitted
14066 if (ix86_compare_emitted)
14069 switch (GET_MODE (ix86_compare_op0))
14075 tmp = ix86_expand_compare (code, NULL, NULL);
14076 tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
14077 gen_rtx_LABEL_REF (VOIDmode, label),
14079 emit_jump_insn (gen_rtx_SET (VOIDmode, pc_rtx, tmp));
14088 enum rtx_code bypass_code, first_code, second_code;
14090 code = ix86_prepare_fp_compare_args (code, &ix86_compare_op0,
14091 &ix86_compare_op1);
14093 ix86_fp_comparison_codes (code, &bypass_code, &first_code, &second_code);
14095 /* Check whether we will use the natural sequence with one jump. If
14096 so, we can expand jump early. Otherwise delay expansion by
14097 creating compound insn to not confuse optimizers. */
14098 if (bypass_code == UNKNOWN && second_code == UNKNOWN)
14100 ix86_split_fp_branch (code, ix86_compare_op0, ix86_compare_op1,
14101 gen_rtx_LABEL_REF (VOIDmode, label),
14102 pc_rtx, NULL_RTX, NULL_RTX);
14106 tmp = gen_rtx_fmt_ee (code, VOIDmode,
14107 ix86_compare_op0, ix86_compare_op1);
14108 tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
14109 gen_rtx_LABEL_REF (VOIDmode, label),
14111 tmp = gen_rtx_SET (VOIDmode, pc_rtx, tmp);
14113 use_fcomi = ix86_use_fcomi_compare (code);
14114 vec = rtvec_alloc (3 + !use_fcomi);
14115 RTVEC_ELT (vec, 0) = tmp;
14117 = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCFPmode, FPSR_REG));
14119 = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCFPmode, FLAGS_REG));
14122 = gen_rtx_CLOBBER (VOIDmode, gen_rtx_SCRATCH (HImode));
14124 emit_jump_insn (gen_rtx_PARALLEL (VOIDmode, vec));
14133 /* Expand DImode branch into multiple compare+branch. */
14135 rtx lo[2], hi[2], label2;
14136 enum rtx_code code1, code2, code3;
14137 enum machine_mode submode;
14139 if (CONSTANT_P (ix86_compare_op0) && ! CONSTANT_P (ix86_compare_op1))
14141 tmp = ix86_compare_op0;
14142 ix86_compare_op0 = ix86_compare_op1;
14143 ix86_compare_op1 = tmp;
14144 code = swap_condition (code);
14146 if (GET_MODE (ix86_compare_op0) == DImode)
14148 split_di (&ix86_compare_op0, 1, lo+0, hi+0);
14149 split_di (&ix86_compare_op1, 1, lo+1, hi+1);
14154 split_ti (&ix86_compare_op0, 1, lo+0, hi+0);
14155 split_ti (&ix86_compare_op1, 1, lo+1, hi+1);
14159 /* When comparing for equality, we can use (hi0^hi1)|(lo0^lo1) to
14160 avoid two branches. This costs one extra insn, so disable when
14161 optimizing for size. */
14163 if ((code == EQ || code == NE)
14164 && (!optimize_insn_for_size_p ()
14165 || hi[1] == const0_rtx || lo[1] == const0_rtx))
14170 if (hi[1] != const0_rtx)
14171 xor1 = expand_binop (submode, xor_optab, xor1, hi[1],
14172 NULL_RTX, 0, OPTAB_WIDEN);
14175 if (lo[1] != const0_rtx)
14176 xor0 = expand_binop (submode, xor_optab, xor0, lo[1],
14177 NULL_RTX, 0, OPTAB_WIDEN);
14179 tmp = expand_binop (submode, ior_optab, xor1, xor0,
14180 NULL_RTX, 0, OPTAB_WIDEN);
14182 ix86_compare_op0 = tmp;
14183 ix86_compare_op1 = const0_rtx;
14184 ix86_expand_branch (code, label);
14188 /* Otherwise, if we are doing less-than or greater-or-equal-than,
14189 op1 is a constant and the low word is zero, then we can just
14190 examine the high word. Similarly for low word -1 and
14191 less-or-equal-than or greater-than. */
14193 if (CONST_INT_P (hi[1]))
14196 case LT: case LTU: case GE: case GEU:
14197 if (lo[1] == const0_rtx)
14199 ix86_compare_op0 = hi[0];
14200 ix86_compare_op1 = hi[1];
14201 ix86_expand_branch (code, label);
14205 case LE: case LEU: case GT: case GTU:
14206 if (lo[1] == constm1_rtx)
14208 ix86_compare_op0 = hi[0];
14209 ix86_compare_op1 = hi[1];
14210 ix86_expand_branch (code, label);
14218 /* Otherwise, we need two or three jumps. */
14220 label2 = gen_label_rtx ();
14223 code2 = swap_condition (code);
14224 code3 = unsigned_condition (code);
14228 case LT: case GT: case LTU: case GTU:
14231 case LE: code1 = LT; code2 = GT; break;
14232 case GE: code1 = GT; code2 = LT; break;
14233 case LEU: code1 = LTU; code2 = GTU; break;
14234 case GEU: code1 = GTU; code2 = LTU; break;
14236 case EQ: code1 = UNKNOWN; code2 = NE; break;
14237 case NE: code2 = UNKNOWN; break;
14240 gcc_unreachable ();
14245 * if (hi(a) < hi(b)) goto true;
14246 * if (hi(a) > hi(b)) goto false;
14247 * if (lo(a) < lo(b)) goto true;
14251 ix86_compare_op0 = hi[0];
14252 ix86_compare_op1 = hi[1];
14254 if (code1 != UNKNOWN)
14255 ix86_expand_branch (code1, label);
14256 if (code2 != UNKNOWN)
14257 ix86_expand_branch (code2, label2);
14259 ix86_compare_op0 = lo[0];
14260 ix86_compare_op1 = lo[1];
14261 ix86_expand_branch (code3, label);
14263 if (code2 != UNKNOWN)
14264 emit_label (label2);
14269 gcc_unreachable ();
14273 /* Split branch based on floating point condition. */
14275 ix86_split_fp_branch (enum rtx_code code, rtx op1, rtx op2,
14276 rtx target1, rtx target2, rtx tmp, rtx pushed)
14278 rtx second, bypass;
14279 rtx label = NULL_RTX;
14281 int bypass_probability = -1, second_probability = -1, probability = -1;
14284 if (target2 != pc_rtx)
14287 code = reverse_condition_maybe_unordered (code);
14292 condition = ix86_expand_fp_compare (code, op1, op2,
14293 tmp, &second, &bypass);
14295 /* Remove pushed operand from stack. */
14297 ix86_free_from_memory (GET_MODE (pushed));
14299 if (split_branch_probability >= 0)
14301 /* Distribute the probabilities across the jumps.
14302 Assume the BYPASS and SECOND to be always test
14304 probability = split_branch_probability;
14306 /* Value of 1 is low enough to make no need for probability
14307 to be updated. Later we may run some experiments and see
14308 if unordered values are more frequent in practice. */
14310 bypass_probability = 1;
14312 second_probability = 1;
14314 if (bypass != NULL_RTX)
14316 label = gen_label_rtx ();
14317 i = emit_jump_insn (gen_rtx_SET
14319 gen_rtx_IF_THEN_ELSE (VOIDmode,
14321 gen_rtx_LABEL_REF (VOIDmode,
14324 if (bypass_probability >= 0)
14326 = gen_rtx_EXPR_LIST (REG_BR_PROB,
14327 GEN_INT (bypass_probability),
14330 i = emit_jump_insn (gen_rtx_SET
14332 gen_rtx_IF_THEN_ELSE (VOIDmode,
14333 condition, target1, target2)));
14334 if (probability >= 0)
14336 = gen_rtx_EXPR_LIST (REG_BR_PROB,
14337 GEN_INT (probability),
14339 if (second != NULL_RTX)
14341 i = emit_jump_insn (gen_rtx_SET
14343 gen_rtx_IF_THEN_ELSE (VOIDmode, second, target1,
14345 if (second_probability >= 0)
14347 = gen_rtx_EXPR_LIST (REG_BR_PROB,
14348 GEN_INT (second_probability),
14351 if (label != NULL_RTX)
14352 emit_label (label);
14356 ix86_expand_setcc (enum rtx_code code, rtx dest)
14358 rtx ret, tmp, tmpreg, equiv;
14359 rtx second_test, bypass_test;
14361 if (GET_MODE (ix86_compare_op0) == (TARGET_64BIT ? TImode : DImode))
14362 return 0; /* FAIL */
14364 gcc_assert (GET_MODE (dest) == QImode);
14366 ret = ix86_expand_compare (code, &second_test, &bypass_test);
14367 PUT_MODE (ret, QImode);
14372 emit_insn (gen_rtx_SET (VOIDmode, tmp, ret));
14373 if (bypass_test || second_test)
14375 rtx test = second_test;
14377 rtx tmp2 = gen_reg_rtx (QImode);
14380 gcc_assert (!second_test);
14381 test = bypass_test;
14383 PUT_CODE (test, reverse_condition_maybe_unordered (GET_CODE (test)));
14385 PUT_MODE (test, QImode);
14386 emit_insn (gen_rtx_SET (VOIDmode, tmp2, test));
14389 emit_insn (gen_andqi3 (tmp, tmpreg, tmp2));
14391 emit_insn (gen_iorqi3 (tmp, tmpreg, tmp2));
14394 /* Attach a REG_EQUAL note describing the comparison result. */
14395 if (ix86_compare_op0 && ix86_compare_op1)
14397 equiv = simplify_gen_relational (code, QImode,
14398 GET_MODE (ix86_compare_op0),
14399 ix86_compare_op0, ix86_compare_op1);
14400 set_unique_reg_note (get_last_insn (), REG_EQUAL, equiv);
14403 return 1; /* DONE */
14406 /* Expand comparison setting or clearing carry flag. Return true when
14407 successful and set pop for the operation. */
14409 ix86_expand_carry_flag_compare (enum rtx_code code, rtx op0, rtx op1, rtx *pop)
14411 enum machine_mode mode =
14412 GET_MODE (op0) != VOIDmode ? GET_MODE (op0) : GET_MODE (op1);
14414 /* Do not handle DImode compares that go through special path. */
14415 if (mode == (TARGET_64BIT ? TImode : DImode))
14418 if (SCALAR_FLOAT_MODE_P (mode))
14420 rtx second_test = NULL, bypass_test = NULL;
14421 rtx compare_op, compare_seq;
14423 gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));
14425 /* Shortcut: following common codes never translate
14426 into carry flag compares. */
14427 if (code == EQ || code == NE || code == UNEQ || code == LTGT
14428 || code == ORDERED || code == UNORDERED)
14431 /* These comparisons require zero flag; swap operands so they won't. */
14432 if ((code == GT || code == UNLE || code == LE || code == UNGT)
14433 && !TARGET_IEEE_FP)
14438 code = swap_condition (code);
14441 /* Try to expand the comparison and verify that we end up with
14442 carry flag based comparison. This fails to be true only when
14443 we decide to expand comparison using arithmetic that is not
14444 too common scenario. */
14446 compare_op = ix86_expand_fp_compare (code, op0, op1, NULL_RTX,
14447 &second_test, &bypass_test);
14448 compare_seq = get_insns ();
14451 if (second_test || bypass_test)
14454 if (GET_MODE (XEXP (compare_op, 0)) == CCFPmode
14455 || GET_MODE (XEXP (compare_op, 0)) == CCFPUmode)
14456 code = ix86_fp_compare_code_to_integer (GET_CODE (compare_op));
14458 code = GET_CODE (compare_op);
14460 if (code != LTU && code != GEU)
14463 emit_insn (compare_seq);
14468 if (!INTEGRAL_MODE_P (mode))
14477 /* Convert a==0 into (unsigned)a<1. */
14480 if (op1 != const0_rtx)
14483 code = (code == EQ ? LTU : GEU);
14486 /* Convert a>b into b<a or a>=b-1. */
14489 if (CONST_INT_P (op1))
14491 op1 = gen_int_mode (INTVAL (op1) + 1, GET_MODE (op0));
14492 /* Bail out on overflow. We still can swap operands but that
14493 would force loading of the constant into register. */
14494 if (op1 == const0_rtx
14495 || !x86_64_immediate_operand (op1, GET_MODE (op1)))
14497 code = (code == GTU ? GEU : LTU);
14504 code = (code == GTU ? LTU : GEU);
14508 /* Convert a>=0 into (unsigned)a<0x80000000. */
14511 if (mode == DImode || op1 != const0_rtx)
14513 op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
14514 code = (code == LT ? GEU : LTU);
14518 if (mode == DImode || op1 != constm1_rtx)
14520 op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
14521 code = (code == LE ? GEU : LTU);
14527 /* Swapping operands may cause constant to appear as first operand. */
14528 if (!nonimmediate_operand (op0, VOIDmode))
14530 if (!can_create_pseudo_p ())
14532 op0 = force_reg (mode, op0);
14534 ix86_compare_op0 = op0;
14535 ix86_compare_op1 = op1;
14536 *pop = ix86_expand_compare (code, NULL, NULL);
14537 gcc_assert (GET_CODE (*pop) == LTU || GET_CODE (*pop) == GEU);
14542 ix86_expand_int_movcc (rtx operands[])
14544 enum rtx_code code = GET_CODE (operands[1]), compare_code;
14545 rtx compare_seq, compare_op;
14546 rtx second_test, bypass_test;
14547 enum machine_mode mode = GET_MODE (operands[0]);
14548 bool sign_bit_compare_p = false;;
14551 compare_op = ix86_expand_compare (code, &second_test, &bypass_test);
14552 compare_seq = get_insns ();
14555 compare_code = GET_CODE (compare_op);
14557 if ((ix86_compare_op1 == const0_rtx && (code == GE || code == LT))
14558 || (ix86_compare_op1 == constm1_rtx && (code == GT || code == LE)))
14559 sign_bit_compare_p = true;
14561 /* Don't attempt mode expansion here -- if we had to expand 5 or 6
14562 HImode insns, we'd be swallowed in word prefix ops. */
14564 if ((mode != HImode || TARGET_FAST_PREFIX)
14565 && (mode != (TARGET_64BIT ? TImode : DImode))
14566 && CONST_INT_P (operands[2])
14567 && CONST_INT_P (operands[3]))
14569 rtx out = operands[0];
14570 HOST_WIDE_INT ct = INTVAL (operands[2]);
14571 HOST_WIDE_INT cf = INTVAL (operands[3]);
14572 HOST_WIDE_INT diff;
14575 /* Sign bit compares are better done using shifts than we do by using
14577 if (sign_bit_compare_p
14578 || ix86_expand_carry_flag_compare (code, ix86_compare_op0,
14579 ix86_compare_op1, &compare_op))
14581 /* Detect overlap between destination and compare sources. */
14584 if (!sign_bit_compare_p)
14586 bool fpcmp = false;
14588 compare_code = GET_CODE (compare_op);
14590 if (GET_MODE (XEXP (compare_op, 0)) == CCFPmode
14591 || GET_MODE (XEXP (compare_op, 0)) == CCFPUmode)
14594 compare_code = ix86_fp_compare_code_to_integer (compare_code);
14597 /* To simplify rest of code, restrict to the GEU case. */
14598 if (compare_code == LTU)
14600 HOST_WIDE_INT tmp = ct;
14603 compare_code = reverse_condition (compare_code);
14604 code = reverse_condition (code);
14609 PUT_CODE (compare_op,
14610 reverse_condition_maybe_unordered
14611 (GET_CODE (compare_op)));
14613 PUT_CODE (compare_op, reverse_condition (GET_CODE (compare_op)));
14617 if (reg_overlap_mentioned_p (out, ix86_compare_op0)
14618 || reg_overlap_mentioned_p (out, ix86_compare_op1))
14619 tmp = gen_reg_rtx (mode);
14621 if (mode == DImode)
14622 emit_insn (gen_x86_movdicc_0_m1_rex64 (tmp, compare_op));
14624 emit_insn (gen_x86_movsicc_0_m1 (gen_lowpart (SImode, tmp), compare_op));
14628 if (code == GT || code == GE)
14629 code = reverse_condition (code);
14632 HOST_WIDE_INT tmp = ct;
14637 tmp = emit_store_flag (tmp, code, ix86_compare_op0,
14638 ix86_compare_op1, VOIDmode, 0, -1);
14651 tmp = expand_simple_binop (mode, PLUS,
14653 copy_rtx (tmp), 1, OPTAB_DIRECT);
14664 tmp = expand_simple_binop (mode, IOR,
14666 copy_rtx (tmp), 1, OPTAB_DIRECT);
14668 else if (diff == -1 && ct)
14678 tmp = expand_simple_unop (mode, NOT, tmp, copy_rtx (tmp), 1);
14680 tmp = expand_simple_binop (mode, PLUS,
14681 copy_rtx (tmp), GEN_INT (cf),
14682 copy_rtx (tmp), 1, OPTAB_DIRECT);
14690 * andl cf - ct, dest
14700 tmp = expand_simple_unop (mode, NOT, tmp, copy_rtx (tmp), 1);
14703 tmp = expand_simple_binop (mode, AND,
14705 gen_int_mode (cf - ct, mode),
14706 copy_rtx (tmp), 1, OPTAB_DIRECT);
14708 tmp = expand_simple_binop (mode, PLUS,
14709 copy_rtx (tmp), GEN_INT (ct),
14710 copy_rtx (tmp), 1, OPTAB_DIRECT);
14713 if (!rtx_equal_p (tmp, out))
14714 emit_move_insn (copy_rtx (out), copy_rtx (tmp));
14716 return 1; /* DONE */
14721 enum machine_mode cmp_mode = GET_MODE (ix86_compare_op0);
14724 tmp = ct, ct = cf, cf = tmp;
14727 if (SCALAR_FLOAT_MODE_P (cmp_mode))
14729 gcc_assert (!DECIMAL_FLOAT_MODE_P (cmp_mode));
14731 /* We may be reversing unordered compare to normal compare, that
14732 is not valid in general (we may convert non-trapping condition
14733 to trapping one), however on i386 we currently emit all
14734 comparisons unordered. */
14735 compare_code = reverse_condition_maybe_unordered (compare_code);
14736 code = reverse_condition_maybe_unordered (code);
14740 compare_code = reverse_condition (compare_code);
14741 code = reverse_condition (code);
14745 compare_code = UNKNOWN;
14746 if (GET_MODE_CLASS (GET_MODE (ix86_compare_op0)) == MODE_INT
14747 && CONST_INT_P (ix86_compare_op1))
14749 if (ix86_compare_op1 == const0_rtx
14750 && (code == LT || code == GE))
14751 compare_code = code;
14752 else if (ix86_compare_op1 == constm1_rtx)
14756 else if (code == GT)
14761 /* Optimize dest = (op0 < 0) ? -1 : cf. */
14762 if (compare_code != UNKNOWN
14763 && GET_MODE (ix86_compare_op0) == GET_MODE (out)
14764 && (cf == -1 || ct == -1))
14766 /* If lea code below could be used, only optimize
14767 if it results in a 2 insn sequence. */
14769 if (! (diff == 1 || diff == 2 || diff == 4 || diff == 8
14770 || diff == 3 || diff == 5 || diff == 9)
14771 || (compare_code == LT && ct == -1)
14772 || (compare_code == GE && cf == -1))
14775 * notl op1 (if necessary)
14783 code = reverse_condition (code);
14786 out = emit_store_flag (out, code, ix86_compare_op0,
14787 ix86_compare_op1, VOIDmode, 0, -1);
14789 out = expand_simple_binop (mode, IOR,
14791 out, 1, OPTAB_DIRECT);
14792 if (out != operands[0])
14793 emit_move_insn (operands[0], out);
14795 return 1; /* DONE */
14800 if ((diff == 1 || diff == 2 || diff == 4 || diff == 8
14801 || diff == 3 || diff == 5 || diff == 9)
14802 && ((mode != QImode && mode != HImode) || !TARGET_PARTIAL_REG_STALL)
14804 || x86_64_immediate_operand (GEN_INT (cf), VOIDmode)))
14810 * lea cf(dest*(ct-cf)),dest
14814 * This also catches the degenerate setcc-only case.
14820 out = emit_store_flag (out, code, ix86_compare_op0,
14821 ix86_compare_op1, VOIDmode, 0, 1);
14824 /* On x86_64 the lea instruction operates on Pmode, so we need
14825 to get arithmetics done in proper mode to match. */
14827 tmp = copy_rtx (out);
14831 out1 = copy_rtx (out);
14832 tmp = gen_rtx_MULT (mode, out1, GEN_INT (diff & ~1));
14836 tmp = gen_rtx_PLUS (mode, tmp, out1);
14842 tmp = gen_rtx_PLUS (mode, tmp, GEN_INT (cf));
14845 if (!rtx_equal_p (tmp, out))
14848 out = force_operand (tmp, copy_rtx (out));
14850 emit_insn (gen_rtx_SET (VOIDmode, copy_rtx (out), copy_rtx (tmp)));
14852 if (!rtx_equal_p (out, operands[0]))
14853 emit_move_insn (operands[0], copy_rtx (out));
14855 return 1; /* DONE */
14859 * General case: Jumpful:
14860 * xorl dest,dest cmpl op1, op2
14861 * cmpl op1, op2 movl ct, dest
14862 * setcc dest jcc 1f
14863 * decl dest movl cf, dest
14864 * andl (cf-ct),dest 1:
14867 * Size 20. Size 14.
14869 * This is reasonably steep, but branch mispredict costs are
14870 * high on modern cpus, so consider failing only if optimizing
14874 if ((!TARGET_CMOVE || (mode == QImode && TARGET_PARTIAL_REG_STALL))
14875 && BRANCH_COST (optimize_insn_for_speed_p (),
14880 enum machine_mode cmp_mode = GET_MODE (ix86_compare_op0);
14885 if (SCALAR_FLOAT_MODE_P (cmp_mode))
14887 gcc_assert (!DECIMAL_FLOAT_MODE_P (cmp_mode));
14889 /* We may be reversing unordered compare to normal compare,
14890 that is not valid in general (we may convert non-trapping
14891 condition to trapping one), however on i386 we currently
14892 emit all comparisons unordered. */
14893 code = reverse_condition_maybe_unordered (code);
14897 code = reverse_condition (code);
14898 if (compare_code != UNKNOWN)
14899 compare_code = reverse_condition (compare_code);
14903 if (compare_code != UNKNOWN)
14905 /* notl op1 (if needed)
14910 For x < 0 (resp. x <= -1) there will be no notl,
14911 so if possible swap the constants to get rid of the
14913 True/false will be -1/0 while code below (store flag
14914 followed by decrement) is 0/-1, so the constants need
14915 to be exchanged once more. */
14917 if (compare_code == GE || !cf)
14919 code = reverse_condition (code);
14924 HOST_WIDE_INT tmp = cf;
14929 out = emit_store_flag (out, code, ix86_compare_op0,
14930 ix86_compare_op1, VOIDmode, 0, -1);
14934 out = emit_store_flag (out, code, ix86_compare_op0,
14935 ix86_compare_op1, VOIDmode, 0, 1);
14937 out = expand_simple_binop (mode, PLUS, copy_rtx (out), constm1_rtx,
14938 copy_rtx (out), 1, OPTAB_DIRECT);
14941 out = expand_simple_binop (mode, AND, copy_rtx (out),
14942 gen_int_mode (cf - ct, mode),
14943 copy_rtx (out), 1, OPTAB_DIRECT);
14945 out = expand_simple_binop (mode, PLUS, copy_rtx (out), GEN_INT (ct),
14946 copy_rtx (out), 1, OPTAB_DIRECT);
14947 if (!rtx_equal_p (out, operands[0]))
14948 emit_move_insn (operands[0], copy_rtx (out));
14950 return 1; /* DONE */
14954 if (!TARGET_CMOVE || (mode == QImode && TARGET_PARTIAL_REG_STALL))
14956 /* Try a few things more with specific constants and a variable. */
14959 rtx var, orig_out, out, tmp;
14961 if (BRANCH_COST (optimize_insn_for_speed_p (), false) <= 2)
14962 return 0; /* FAIL */
14964 /* If one of the two operands is an interesting constant, load a
14965 constant with the above and mask it in with a logical operation. */
14967 if (CONST_INT_P (operands[2]))
14970 if (INTVAL (operands[2]) == 0 && operands[3] != constm1_rtx)
14971 operands[3] = constm1_rtx, op = and_optab;
14972 else if (INTVAL (operands[2]) == -1 && operands[3] != const0_rtx)
14973 operands[3] = const0_rtx, op = ior_optab;
14975 return 0; /* FAIL */
14977 else if (CONST_INT_P (operands[3]))
14980 if (INTVAL (operands[3]) == 0 && operands[2] != constm1_rtx)
14981 operands[2] = constm1_rtx, op = and_optab;
14982 else if (INTVAL (operands[3]) == -1 && operands[3] != const0_rtx)
14983 operands[2] = const0_rtx, op = ior_optab;
14985 return 0; /* FAIL */
14988 return 0; /* FAIL */
14990 orig_out = operands[0];
14991 tmp = gen_reg_rtx (mode);
14994 /* Recurse to get the constant loaded. */
14995 if (ix86_expand_int_movcc (operands) == 0)
14996 return 0; /* FAIL */
14998 /* Mask in the interesting variable. */
14999 out = expand_binop (mode, op, var, tmp, orig_out, 0,
15001 if (!rtx_equal_p (out, orig_out))
15002 emit_move_insn (copy_rtx (orig_out), copy_rtx (out));
15004 return 1; /* DONE */
15008 * For comparison with above,
15018 if (! nonimmediate_operand (operands[2], mode))
15019 operands[2] = force_reg (mode, operands[2]);
15020 if (! nonimmediate_operand (operands[3], mode))
15021 operands[3] = force_reg (mode, operands[3]);
15023 if (bypass_test && reg_overlap_mentioned_p (operands[0], operands[3]))
15025 rtx tmp = gen_reg_rtx (mode);
15026 emit_move_insn (tmp, operands[3]);
15029 if (second_test && reg_overlap_mentioned_p (operands[0], operands[2]))
15031 rtx tmp = gen_reg_rtx (mode);
15032 emit_move_insn (tmp, operands[2]);
15036 if (! register_operand (operands[2], VOIDmode)
15038 || ! register_operand (operands[3], VOIDmode)))
15039 operands[2] = force_reg (mode, operands[2]);
15042 && ! register_operand (operands[3], VOIDmode))
15043 operands[3] = force_reg (mode, operands[3]);
15045 emit_insn (compare_seq);
15046 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
15047 gen_rtx_IF_THEN_ELSE (mode,
15048 compare_op, operands[2],
15051 emit_insn (gen_rtx_SET (VOIDmode, copy_rtx (operands[0]),
15052 gen_rtx_IF_THEN_ELSE (mode,
15054 copy_rtx (operands[3]),
15055 copy_rtx (operands[0]))));
15057 emit_insn (gen_rtx_SET (VOIDmode, copy_rtx (operands[0]),
15058 gen_rtx_IF_THEN_ELSE (mode,
15060 copy_rtx (operands[2]),
15061 copy_rtx (operands[0]))));
15063 return 1; /* DONE */
15066 /* Swap, force into registers, or otherwise massage the two operands
15067 to an sse comparison with a mask result. Thus we differ a bit from
15068 ix86_prepare_fp_compare_args which expects to produce a flags result.
15070 The DEST operand exists to help determine whether to commute commutative
15071 operators. The POP0/POP1 operands are updated in place. The new
15072 comparison code is returned, or UNKNOWN if not implementable. */
15074 static enum rtx_code
15075 ix86_prepare_sse_fp_compare_args (rtx dest, enum rtx_code code,
15076 rtx *pop0, rtx *pop1)
15084 /* We have no LTGT as an operator. We could implement it with
15085 NE & ORDERED, but this requires an extra temporary. It's
15086 not clear that it's worth it. */
15093 /* These are supported directly. */
15100 /* For commutative operators, try to canonicalize the destination
15101 operand to be first in the comparison - this helps reload to
15102 avoid extra moves. */
15103 if (!dest || !rtx_equal_p (dest, *pop1))
15111 /* These are not supported directly. Swap the comparison operands
15112 to transform into something that is supported. */
15116 code = swap_condition (code);
15120 gcc_unreachable ();
15126 /* Detect conditional moves that exactly match min/max operational
15127 semantics. Note that this is IEEE safe, as long as we don't
15128 interchange the operands.
15130 Returns FALSE if this conditional move doesn't match a MIN/MAX,
15131 and TRUE if the operation is successful and instructions are emitted. */
15134 ix86_expand_sse_fp_minmax (rtx dest, enum rtx_code code, rtx cmp_op0,
15135 rtx cmp_op1, rtx if_true, rtx if_false)
15137 enum machine_mode mode;
15143 else if (code == UNGE)
15146 if_true = if_false;
15152 if (rtx_equal_p (cmp_op0, if_true) && rtx_equal_p (cmp_op1, if_false))
15154 else if (rtx_equal_p (cmp_op1, if_true) && rtx_equal_p (cmp_op0, if_false))
15159 mode = GET_MODE (dest);
15161 /* We want to check HONOR_NANS and HONOR_SIGNED_ZEROS here,
15162 but MODE may be a vector mode and thus not appropriate. */
15163 if (!flag_finite_math_only || !flag_unsafe_math_optimizations)
15165 int u = is_min ? UNSPEC_IEEE_MIN : UNSPEC_IEEE_MAX;
15168 if_true = force_reg (mode, if_true);
15169 v = gen_rtvec (2, if_true, if_false);
15170 tmp = gen_rtx_UNSPEC (mode, v, u);
15174 code = is_min ? SMIN : SMAX;
15175 tmp = gen_rtx_fmt_ee (code, mode, if_true, if_false);
15178 emit_insn (gen_rtx_SET (VOIDmode, dest, tmp));
15182 /* Expand an sse vector comparison. Return the register with the result. */
15185 ix86_expand_sse_cmp (rtx dest, enum rtx_code code, rtx cmp_op0, rtx cmp_op1,
15186 rtx op_true, rtx op_false)
15188 enum machine_mode mode = GET_MODE (dest);
15191 cmp_op0 = force_reg (mode, cmp_op0);
15192 if (!nonimmediate_operand (cmp_op1, mode))
15193 cmp_op1 = force_reg (mode, cmp_op1);
15196 || reg_overlap_mentioned_p (dest, op_true)
15197 || reg_overlap_mentioned_p (dest, op_false))
15198 dest = gen_reg_rtx (mode);
15200 x = gen_rtx_fmt_ee (code, mode, cmp_op0, cmp_op1);
15201 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
15206 /* Expand DEST = CMP ? OP_TRUE : OP_FALSE into a sequence of logical
15207 operations. This is used for both scalar and vector conditional moves. */
15210 ix86_expand_sse_movcc (rtx dest, rtx cmp, rtx op_true, rtx op_false)
15212 enum machine_mode mode = GET_MODE (dest);
15215 if (op_false == CONST0_RTX (mode))
15217 op_true = force_reg (mode, op_true);
15218 x = gen_rtx_AND (mode, cmp, op_true);
15219 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
15221 else if (op_true == CONST0_RTX (mode))
15223 op_false = force_reg (mode, op_false);
15224 x = gen_rtx_NOT (mode, cmp);
15225 x = gen_rtx_AND (mode, x, op_false);
15226 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
15228 else if (TARGET_SSE5)
15230 rtx pcmov = gen_rtx_SET (mode, dest,
15231 gen_rtx_IF_THEN_ELSE (mode, cmp,
15238 op_true = force_reg (mode, op_true);
15239 op_false = force_reg (mode, op_false);
15241 t2 = gen_reg_rtx (mode);
15243 t3 = gen_reg_rtx (mode);
15247 x = gen_rtx_AND (mode, op_true, cmp);
15248 emit_insn (gen_rtx_SET (VOIDmode, t2, x));
15250 x = gen_rtx_NOT (mode, cmp);
15251 x = gen_rtx_AND (mode, x, op_false);
15252 emit_insn (gen_rtx_SET (VOIDmode, t3, x));
15254 x = gen_rtx_IOR (mode, t3, t2);
15255 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
15259 /* Expand a floating-point conditional move. Return true if successful. */
15262 ix86_expand_fp_movcc (rtx operands[])
15264 enum machine_mode mode = GET_MODE (operands[0]);
15265 enum rtx_code code = GET_CODE (operands[1]);
15266 rtx tmp, compare_op, second_test, bypass_test;
15268 if (TARGET_SSE_MATH && SSE_FLOAT_MODE_P (mode))
15270 enum machine_mode cmode;
15272 /* Since we've no cmove for sse registers, don't force bad register
15273 allocation just to gain access to it. Deny movcc when the
15274 comparison mode doesn't match the move mode. */
15275 cmode = GET_MODE (ix86_compare_op0);
15276 if (cmode == VOIDmode)
15277 cmode = GET_MODE (ix86_compare_op1);
15281 code = ix86_prepare_sse_fp_compare_args (operands[0], code,
15283 &ix86_compare_op1);
15284 if (code == UNKNOWN)
15287 if (ix86_expand_sse_fp_minmax (operands[0], code, ix86_compare_op0,
15288 ix86_compare_op1, operands[2],
15292 tmp = ix86_expand_sse_cmp (operands[0], code, ix86_compare_op0,
15293 ix86_compare_op1, operands[2], operands[3]);
15294 ix86_expand_sse_movcc (operands[0], tmp, operands[2], operands[3]);
15298 /* The floating point conditional move instructions don't directly
15299 support conditions resulting from a signed integer comparison. */
15301 compare_op = ix86_expand_compare (code, &second_test, &bypass_test);
15303 /* The floating point conditional move instructions don't directly
15304 support signed integer comparisons. */
15306 if (!fcmov_comparison_operator (compare_op, VOIDmode))
15308 gcc_assert (!second_test && !bypass_test);
15309 tmp = gen_reg_rtx (QImode);
15310 ix86_expand_setcc (code, tmp);
15312 ix86_compare_op0 = tmp;
15313 ix86_compare_op1 = const0_rtx;
15314 compare_op = ix86_expand_compare (code, &second_test, &bypass_test);
15316 if (bypass_test && reg_overlap_mentioned_p (operands[0], operands[3]))
15318 tmp = gen_reg_rtx (mode);
15319 emit_move_insn (tmp, operands[3]);
15322 if (second_test && reg_overlap_mentioned_p (operands[0], operands[2]))
15324 tmp = gen_reg_rtx (mode);
15325 emit_move_insn (tmp, operands[2]);
15329 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
15330 gen_rtx_IF_THEN_ELSE (mode, compare_op,
15331 operands[2], operands[3])));
15333 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
15334 gen_rtx_IF_THEN_ELSE (mode, bypass_test,
15335 operands[3], operands[0])));
15337 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
15338 gen_rtx_IF_THEN_ELSE (mode, second_test,
15339 operands[2], operands[0])));
15344 /* Expand a floating-point vector conditional move; a vcond operation
15345 rather than a movcc operation. */
15348 ix86_expand_fp_vcond (rtx operands[])
15350 enum rtx_code code = GET_CODE (operands[3]);
15353 code = ix86_prepare_sse_fp_compare_args (operands[0], code,
15354 &operands[4], &operands[5]);
15355 if (code == UNKNOWN)
15358 if (ix86_expand_sse_fp_minmax (operands[0], code, operands[4],
15359 operands[5], operands[1], operands[2]))
15362 cmp = ix86_expand_sse_cmp (operands[0], code, operands[4], operands[5],
15363 operands[1], operands[2]);
15364 ix86_expand_sse_movcc (operands[0], cmp, operands[1], operands[2]);
15368 /* Expand a signed/unsigned integral vector conditional move. */
15371 ix86_expand_int_vcond (rtx operands[])
15373 enum machine_mode mode = GET_MODE (operands[0]);
15374 enum rtx_code code = GET_CODE (operands[3]);
15375 bool negate = false;
15378 cop0 = operands[4];
15379 cop1 = operands[5];
15381 /* SSE5 supports all of the comparisons on all vector int types. */
15384 /* Canonicalize the comparison to EQ, GT, GTU. */
15395 code = reverse_condition (code);
15401 code = reverse_condition (code);
15407 code = swap_condition (code);
15408 x = cop0, cop0 = cop1, cop1 = x;
15412 gcc_unreachable ();
15415 /* Only SSE4.1/SSE4.2 supports V2DImode. */
15416 if (mode == V2DImode)
15421 /* SSE4.1 supports EQ. */
15422 if (!TARGET_SSE4_1)
15428 /* SSE4.2 supports GT/GTU. */
15429 if (!TARGET_SSE4_2)
15434 gcc_unreachable ();
15438 /* Unsigned parallel compare is not supported by the hardware. Play some
15439 tricks to turn this into a signed comparison against 0. */
15442 cop0 = force_reg (mode, cop0);
15451 /* Perform a parallel modulo subtraction. */
15452 t1 = gen_reg_rtx (mode);
15453 emit_insn ((mode == V4SImode
15455 : gen_subv2di3) (t1, cop0, cop1));
15457 /* Extract the original sign bit of op0. */
15458 mask = ix86_build_signbit_mask (GET_MODE_INNER (mode),
15460 t2 = gen_reg_rtx (mode);
15461 emit_insn ((mode == V4SImode
15463 : gen_andv2di3) (t2, cop0, mask));
15465 /* XOR it back into the result of the subtraction. This results
15466 in the sign bit set iff we saw unsigned underflow. */
15467 x = gen_reg_rtx (mode);
15468 emit_insn ((mode == V4SImode
15470 : gen_xorv2di3) (x, t1, t2));
15478 /* Perform a parallel unsigned saturating subtraction. */
15479 x = gen_reg_rtx (mode);
15480 emit_insn (gen_rtx_SET (VOIDmode, x,
15481 gen_rtx_US_MINUS (mode, cop0, cop1)));
15488 gcc_unreachable ();
15492 cop1 = CONST0_RTX (mode);
15496 x = ix86_expand_sse_cmp (operands[0], code, cop0, cop1,
15497 operands[1+negate], operands[2-negate]);
15499 ix86_expand_sse_movcc (operands[0], x, operands[1+negate],
15500 operands[2-negate]);
15504 /* Unpack OP[1] into the next wider integer vector type. UNSIGNED_P is
15505 true if we should do zero extension, else sign extension. HIGH_P is
15506 true if we want the N/2 high elements, else the low elements. */
15509 ix86_expand_sse_unpack (rtx operands[2], bool unsigned_p, bool high_p)
15511 enum machine_mode imode = GET_MODE (operands[1]);
15512 rtx (*unpack)(rtx, rtx, rtx);
15519 unpack = gen_vec_interleave_highv16qi;
15521 unpack = gen_vec_interleave_lowv16qi;
15525 unpack = gen_vec_interleave_highv8hi;
15527 unpack = gen_vec_interleave_lowv8hi;
15531 unpack = gen_vec_interleave_highv4si;
15533 unpack = gen_vec_interleave_lowv4si;
15536 gcc_unreachable ();
15539 dest = gen_lowpart (imode, operands[0]);
15542 se = force_reg (imode, CONST0_RTX (imode));
15544 se = ix86_expand_sse_cmp (gen_reg_rtx (imode), GT, CONST0_RTX (imode),
15545 operands[1], pc_rtx, pc_rtx);
15547 emit_insn (unpack (dest, operands[1], se));
15550 /* This function performs the same task as ix86_expand_sse_unpack,
15551 but with SSE4.1 instructions. */
15554 ix86_expand_sse4_unpack (rtx operands[2], bool unsigned_p, bool high_p)
15556 enum machine_mode imode = GET_MODE (operands[1]);
15557 rtx (*unpack)(rtx, rtx);
15564 unpack = gen_sse4_1_zero_extendv8qiv8hi2;
15566 unpack = gen_sse4_1_extendv8qiv8hi2;
15570 unpack = gen_sse4_1_zero_extendv4hiv4si2;
15572 unpack = gen_sse4_1_extendv4hiv4si2;
15576 unpack = gen_sse4_1_zero_extendv2siv2di2;
15578 unpack = gen_sse4_1_extendv2siv2di2;
15581 gcc_unreachable ();
15584 dest = operands[0];
15587 /* Shift higher 8 bytes to lower 8 bytes. */
15588 src = gen_reg_rtx (imode);
15589 emit_insn (gen_sse2_lshrti3 (gen_lowpart (TImode, src),
15590 gen_lowpart (TImode, operands[1]),
15596 emit_insn (unpack (dest, src));
15599 /* This function performs the same task as ix86_expand_sse_unpack,
15600 but with sse5 instructions. */
15603 ix86_expand_sse5_unpack (rtx operands[2], bool unsigned_p, bool high_p)
15605 enum machine_mode imode = GET_MODE (operands[1]);
15606 int pperm_bytes[16];
15608 int h = (high_p) ? 8 : 0;
15611 rtvec v = rtvec_alloc (16);
15614 rtx op0 = operands[0], op1 = operands[1];
15619 vs = rtvec_alloc (8);
15620 h2 = (high_p) ? 8 : 0;
15621 for (i = 0; i < 8; i++)
15623 pperm_bytes[2*i+0] = PPERM_SRC | PPERM_SRC2 | i | h;
15624 pperm_bytes[2*i+1] = ((unsigned_p)
15626 : PPERM_SIGN | PPERM_SRC2 | i | h);
15629 for (i = 0; i < 16; i++)
15630 RTVEC_ELT (v, i) = GEN_INT (pperm_bytes[i]);
15632 for (i = 0; i < 8; i++)
15633 RTVEC_ELT (vs, i) = GEN_INT (i + h2);
15635 p = gen_rtx_PARALLEL (VOIDmode, vs);
15636 x = force_reg (V16QImode, gen_rtx_CONST_VECTOR (V16QImode, v));
15638 emit_insn (gen_sse5_pperm_zero_v16qi_v8hi (op0, op1, p, x));
15640 emit_insn (gen_sse5_pperm_sign_v16qi_v8hi (op0, op1, p, x));
15644 vs = rtvec_alloc (4);
15645 h2 = (high_p) ? 4 : 0;
15646 for (i = 0; i < 4; i++)
15648 sign_extend = ((unsigned_p)
15650 : PPERM_SIGN | PPERM_SRC2 | ((2*i) + 1 + h));
15651 pperm_bytes[4*i+0] = PPERM_SRC | PPERM_SRC2 | ((2*i) + 0 + h);
15652 pperm_bytes[4*i+1] = PPERM_SRC | PPERM_SRC2 | ((2*i) + 1 + h);
15653 pperm_bytes[4*i+2] = sign_extend;
15654 pperm_bytes[4*i+3] = sign_extend;
15657 for (i = 0; i < 16; i++)
15658 RTVEC_ELT (v, i) = GEN_INT (pperm_bytes[i]);
15660 for (i = 0; i < 4; i++)
15661 RTVEC_ELT (vs, i) = GEN_INT (i + h2);
15663 p = gen_rtx_PARALLEL (VOIDmode, vs);
15664 x = force_reg (V16QImode, gen_rtx_CONST_VECTOR (V16QImode, v));
15666 emit_insn (gen_sse5_pperm_zero_v8hi_v4si (op0, op1, p, x));
15668 emit_insn (gen_sse5_pperm_sign_v8hi_v4si (op0, op1, p, x));
15672 vs = rtvec_alloc (2);
15673 h2 = (high_p) ? 2 : 0;
15674 for (i = 0; i < 2; i++)
15676 sign_extend = ((unsigned_p)
15678 : PPERM_SIGN | PPERM_SRC2 | ((4*i) + 3 + h));
15679 pperm_bytes[8*i+0] = PPERM_SRC | PPERM_SRC2 | ((4*i) + 0 + h);
15680 pperm_bytes[8*i+1] = PPERM_SRC | PPERM_SRC2 | ((4*i) + 1 + h);
15681 pperm_bytes[8*i+2] = PPERM_SRC | PPERM_SRC2 | ((4*i) + 2 + h);
15682 pperm_bytes[8*i+3] = PPERM_SRC | PPERM_SRC2 | ((4*i) + 3 + h);
15683 pperm_bytes[8*i+4] = sign_extend;
15684 pperm_bytes[8*i+5] = sign_extend;
15685 pperm_bytes[8*i+6] = sign_extend;
15686 pperm_bytes[8*i+7] = sign_extend;
15689 for (i = 0; i < 16; i++)
15690 RTVEC_ELT (v, i) = GEN_INT (pperm_bytes[i]);
15692 for (i = 0; i < 2; i++)
15693 RTVEC_ELT (vs, i) = GEN_INT (i + h2);
15695 p = gen_rtx_PARALLEL (VOIDmode, vs);
15696 x = force_reg (V16QImode, gen_rtx_CONST_VECTOR (V16QImode, v));
15698 emit_insn (gen_sse5_pperm_zero_v4si_v2di (op0, op1, p, x));
15700 emit_insn (gen_sse5_pperm_sign_v4si_v2di (op0, op1, p, x));
15704 gcc_unreachable ();
15710 /* Pack the high bits from OPERANDS[1] and low bits from OPERANDS[2] into the
15711 next narrower integer vector type */
15713 ix86_expand_sse5_pack (rtx operands[3])
15715 enum machine_mode imode = GET_MODE (operands[0]);
15716 int pperm_bytes[16];
15718 rtvec v = rtvec_alloc (16);
15720 rtx op0 = operands[0];
15721 rtx op1 = operands[1];
15722 rtx op2 = operands[2];
15727 for (i = 0; i < 8; i++)
15729 pperm_bytes[i+0] = PPERM_SRC | PPERM_SRC1 | (i*2);
15730 pperm_bytes[i+8] = PPERM_SRC | PPERM_SRC2 | (i*2);
15733 for (i = 0; i < 16; i++)
15734 RTVEC_ELT (v, i) = GEN_INT (pperm_bytes[i]);
15736 x = force_reg (V16QImode, gen_rtx_CONST_VECTOR (V16QImode, v));
15737 emit_insn (gen_sse5_pperm_pack_v8hi_v16qi (op0, op1, op2, x));
15741 for (i = 0; i < 4; i++)
15743 pperm_bytes[(2*i)+0] = PPERM_SRC | PPERM_SRC1 | ((i*4) + 0);
15744 pperm_bytes[(2*i)+1] = PPERM_SRC | PPERM_SRC1 | ((i*4) + 1);
15745 pperm_bytes[(2*i)+8] = PPERM_SRC | PPERM_SRC2 | ((i*4) + 0);
15746 pperm_bytes[(2*i)+9] = PPERM_SRC | PPERM_SRC2 | ((i*4) + 1);
15749 for (i = 0; i < 16; i++)
15750 RTVEC_ELT (v, i) = GEN_INT (pperm_bytes[i]);
15752 x = force_reg (V16QImode, gen_rtx_CONST_VECTOR (V16QImode, v));
15753 emit_insn (gen_sse5_pperm_pack_v4si_v8hi (op0, op1, op2, x));
15757 for (i = 0; i < 2; i++)
15759 pperm_bytes[(4*i)+0] = PPERM_SRC | PPERM_SRC1 | ((i*8) + 0);
15760 pperm_bytes[(4*i)+1] = PPERM_SRC | PPERM_SRC1 | ((i*8) + 1);
15761 pperm_bytes[(4*i)+2] = PPERM_SRC | PPERM_SRC1 | ((i*8) + 2);
15762 pperm_bytes[(4*i)+3] = PPERM_SRC | PPERM_SRC1 | ((i*8) + 3);
15763 pperm_bytes[(4*i)+8] = PPERM_SRC | PPERM_SRC2 | ((i*8) + 0);
15764 pperm_bytes[(4*i)+9] = PPERM_SRC | PPERM_SRC2 | ((i*8) + 1);
15765 pperm_bytes[(4*i)+10] = PPERM_SRC | PPERM_SRC2 | ((i*8) + 2);
15766 pperm_bytes[(4*i)+11] = PPERM_SRC | PPERM_SRC2 | ((i*8) + 3);
15769 for (i = 0; i < 16; i++)
15770 RTVEC_ELT (v, i) = GEN_INT (pperm_bytes[i]);
15772 x = force_reg (V16QImode, gen_rtx_CONST_VECTOR (V16QImode, v));
15773 emit_insn (gen_sse5_pperm_pack_v2di_v4si (op0, op1, op2, x));
15777 gcc_unreachable ();
15783 /* Expand conditional increment or decrement using adb/sbb instructions.
15784 The default case using setcc followed by the conditional move can be
15785 done by generic code. */
15787 ix86_expand_int_addcc (rtx operands[])
15789 enum rtx_code code = GET_CODE (operands[1]);
15791 rtx val = const0_rtx;
15792 bool fpcmp = false;
15793 enum machine_mode mode = GET_MODE (operands[0]);
15795 if (operands[3] != const1_rtx
15796 && operands[3] != constm1_rtx)
15798 if (!ix86_expand_carry_flag_compare (code, ix86_compare_op0,
15799 ix86_compare_op1, &compare_op))
15801 code = GET_CODE (compare_op);
15803 if (GET_MODE (XEXP (compare_op, 0)) == CCFPmode
15804 || GET_MODE (XEXP (compare_op, 0)) == CCFPUmode)
15807 code = ix86_fp_compare_code_to_integer (code);
15814 PUT_CODE (compare_op,
15815 reverse_condition_maybe_unordered
15816 (GET_CODE (compare_op)));
15818 PUT_CODE (compare_op, reverse_condition (GET_CODE (compare_op)));
15820 PUT_MODE (compare_op, mode);
15822 /* Construct either adc or sbb insn. */
15823 if ((code == LTU) == (operands[3] == constm1_rtx))
15825 switch (GET_MODE (operands[0]))
15828 emit_insn (gen_subqi3_carry (operands[0], operands[2], val, compare_op));
15831 emit_insn (gen_subhi3_carry (operands[0], operands[2], val, compare_op));
15834 emit_insn (gen_subsi3_carry (operands[0], operands[2], val, compare_op));
15837 emit_insn (gen_subdi3_carry_rex64 (operands[0], operands[2], val, compare_op));
15840 gcc_unreachable ();
15845 switch (GET_MODE (operands[0]))
15848 emit_insn (gen_addqi3_carry (operands[0], operands[2], val, compare_op));
15851 emit_insn (gen_addhi3_carry (operands[0], operands[2], val, compare_op));
15854 emit_insn (gen_addsi3_carry (operands[0], operands[2], val, compare_op));
15857 emit_insn (gen_adddi3_carry_rex64 (operands[0], operands[2], val, compare_op));
15860 gcc_unreachable ();
15863 return 1; /* DONE */
15867 /* Split operands 0 and 1 into SImode parts. Similar to split_di, but
15868 works for floating pointer parameters and nonoffsetable memories.
15869 For pushes, it returns just stack offsets; the values will be saved
15870 in the right order. Maximally three parts are generated. */
15873 ix86_split_to_parts (rtx operand, rtx *parts, enum machine_mode mode)
15878 size = mode==XFmode ? 3 : GET_MODE_SIZE (mode) / 4;
15880 size = (GET_MODE_SIZE (mode) + 4) / 8;
15882 gcc_assert (!REG_P (operand) || !MMX_REGNO_P (REGNO (operand)));
15883 gcc_assert (size >= 2 && size <= 4);
15885 /* Optimize constant pool reference to immediates. This is used by fp
15886 moves, that force all constants to memory to allow combining. */
15887 if (MEM_P (operand) && MEM_READONLY_P (operand))
15889 rtx tmp = maybe_get_pool_constant (operand);
15894 if (MEM_P (operand) && !offsettable_memref_p (operand))
15896 /* The only non-offsetable memories we handle are pushes. */
15897 int ok = push_operand (operand, VOIDmode);
15901 operand = copy_rtx (operand);
15902 PUT_MODE (operand, Pmode);
15903 parts[0] = parts[1] = parts[2] = parts[3] = operand;
15907 if (GET_CODE (operand) == CONST_VECTOR)
15909 enum machine_mode imode = int_mode_for_mode (mode);
15910 /* Caution: if we looked through a constant pool memory above,
15911 the operand may actually have a different mode now. That's
15912 ok, since we want to pun this all the way back to an integer. */
15913 operand = simplify_subreg (imode, operand, GET_MODE (operand), 0);
15914 gcc_assert (operand != NULL);
15920 if (mode == DImode)
15921 split_di (&operand, 1, &parts[0], &parts[1]);
15926 if (REG_P (operand))
15928 gcc_assert (reload_completed);
15929 for (i = 0; i < size; i++)
15930 parts[i] = gen_rtx_REG (SImode, REGNO (operand) + i);
15932 else if (offsettable_memref_p (operand))
15934 operand = adjust_address (operand, SImode, 0);
15935 parts[0] = operand;
15936 for (i = 1; i < size; i++)
15937 parts[i] = adjust_address (operand, SImode, 4 * i);
15939 else if (GET_CODE (operand) == CONST_DOUBLE)
15944 REAL_VALUE_FROM_CONST_DOUBLE (r, operand);
15948 real_to_target (l, &r, mode);
15949 parts[3] = gen_int_mode (l[3], SImode);
15950 parts[2] = gen_int_mode (l[2], SImode);
15953 REAL_VALUE_TO_TARGET_LONG_DOUBLE (r, l);
15954 parts[2] = gen_int_mode (l[2], SImode);
15957 REAL_VALUE_TO_TARGET_DOUBLE (r, l);
15960 gcc_unreachable ();
15962 parts[1] = gen_int_mode (l[1], SImode);
15963 parts[0] = gen_int_mode (l[0], SImode);
15966 gcc_unreachable ();
15971 if (mode == TImode)
15972 split_ti (&operand, 1, &parts[0], &parts[1]);
15973 if (mode == XFmode || mode == TFmode)
15975 enum machine_mode upper_mode = mode==XFmode ? SImode : DImode;
15976 if (REG_P (operand))
15978 gcc_assert (reload_completed);
15979 parts[0] = gen_rtx_REG (DImode, REGNO (operand) + 0);
15980 parts[1] = gen_rtx_REG (upper_mode, REGNO (operand) + 1);
15982 else if (offsettable_memref_p (operand))
15984 operand = adjust_address (operand, DImode, 0);
15985 parts[0] = operand;
15986 parts[1] = adjust_address (operand, upper_mode, 8);
15988 else if (GET_CODE (operand) == CONST_DOUBLE)
15993 REAL_VALUE_FROM_CONST_DOUBLE (r, operand);
15994 real_to_target (l, &r, mode);
15996 /* Do not use shift by 32 to avoid warning on 32bit systems. */
15997 if (HOST_BITS_PER_WIDE_INT >= 64)
16000 ((l[0] & (((HOST_WIDE_INT) 2 << 31) - 1))
16001 + ((((HOST_WIDE_INT) l[1]) << 31) << 1),
16004 parts[0] = immed_double_const (l[0], l[1], DImode);
16006 if (upper_mode == SImode)
16007 parts[1] = gen_int_mode (l[2], SImode);
16008 else if (HOST_BITS_PER_WIDE_INT >= 64)
16011 ((l[2] & (((HOST_WIDE_INT) 2 << 31) - 1))
16012 + ((((HOST_WIDE_INT) l[3]) << 31) << 1),
16015 parts[1] = immed_double_const (l[2], l[3], DImode);
16018 gcc_unreachable ();
16025 /* Emit insns to perform a move or push of DI, DF, XF, and TF values.
16026 Return false when normal moves are needed; true when all required
16027 insns have been emitted. Operands 2-4 contain the input values
16028 int the correct order; operands 5-7 contain the output values. */
16031 ix86_split_long_move (rtx operands[])
16036 int collisions = 0;
16037 enum machine_mode mode = GET_MODE (operands[0]);
16038 bool collisionparts[4];
16040 /* The DFmode expanders may ask us to move double.
16041 For 64bit target this is single move. By hiding the fact
16042 here we simplify i386.md splitters. */
16043 if (GET_MODE_SIZE (GET_MODE (operands[0])) == 8 && TARGET_64BIT)
16045 /* Optimize constant pool reference to immediates. This is used by
16046 fp moves, that force all constants to memory to allow combining. */
16048 if (MEM_P (operands[1])
16049 && GET_CODE (XEXP (operands[1], 0)) == SYMBOL_REF
16050 && CONSTANT_POOL_ADDRESS_P (XEXP (operands[1], 0)))
16051 operands[1] = get_pool_constant (XEXP (operands[1], 0));
16052 if (push_operand (operands[0], VOIDmode))
16054 operands[0] = copy_rtx (operands[0]);
16055 PUT_MODE (operands[0], Pmode);
16058 operands[0] = gen_lowpart (DImode, operands[0]);
16059 operands[1] = gen_lowpart (DImode, operands[1]);
16060 emit_move_insn (operands[0], operands[1]);
16064 /* The only non-offsettable memory we handle is push. */
16065 if (push_operand (operands[0], VOIDmode))
16068 gcc_assert (!MEM_P (operands[0])
16069 || offsettable_memref_p (operands[0]));
16071 nparts = ix86_split_to_parts (operands[1], part[1], GET_MODE (operands[0]));
16072 ix86_split_to_parts (operands[0], part[0], GET_MODE (operands[0]));
16074 /* When emitting push, take care for source operands on the stack. */
16075 if (push && MEM_P (operands[1])
16076 && reg_overlap_mentioned_p (stack_pointer_rtx, operands[1]))
16077 for (i = 0; i < nparts - 1; i++)
16078 part[1][i] = change_address (part[1][i],
16079 GET_MODE (part[1][i]),
16080 XEXP (part[1][i + 1], 0));
16082 /* We need to do copy in the right order in case an address register
16083 of the source overlaps the destination. */
16084 if (REG_P (part[0][0]) && MEM_P (part[1][0]))
16088 for (i = 0; i < nparts; i++)
16091 = reg_overlap_mentioned_p (part[0][i], XEXP (part[1][0], 0));
16092 if (collisionparts[i])
16096 /* Collision in the middle part can be handled by reordering. */
16097 if (collisions == 1 && nparts == 3 && collisionparts [1])
16099 tmp = part[0][1]; part[0][1] = part[0][2]; part[0][2] = tmp;
16100 tmp = part[1][1]; part[1][1] = part[1][2]; part[1][2] = tmp;
16102 else if (collisions == 1
16104 && (collisionparts [1] || collisionparts [2]))
16106 if (collisionparts [1])
16108 tmp = part[0][1]; part[0][1] = part[0][2]; part[0][2] = tmp;
16109 tmp = part[1][1]; part[1][1] = part[1][2]; part[1][2] = tmp;
16113 tmp = part[0][2]; part[0][2] = part[0][3]; part[0][3] = tmp;
16114 tmp = part[1][2]; part[1][2] = part[1][3]; part[1][3] = tmp;
16118 /* If there are more collisions, we can't handle it by reordering.
16119 Do an lea to the last part and use only one colliding move. */
16120 else if (collisions > 1)
16126 base = part[0][nparts - 1];
16128 /* Handle the case when the last part isn't valid for lea.
16129 Happens in 64-bit mode storing the 12-byte XFmode. */
16130 if (GET_MODE (base) != Pmode)
16131 base = gen_rtx_REG (Pmode, REGNO (base));
16133 emit_insn (gen_rtx_SET (VOIDmode, base, XEXP (part[1][0], 0)));
16134 part[1][0] = replace_equiv_address (part[1][0], base);
16135 for (i = 1; i < nparts; i++)
16137 tmp = plus_constant (base, UNITS_PER_WORD * i);
16138 part[1][i] = replace_equiv_address (part[1][i], tmp);
16149 if (TARGET_128BIT_LONG_DOUBLE && mode == XFmode)
16150 emit_insn (gen_addsi3 (stack_pointer_rtx, stack_pointer_rtx, GEN_INT (-4)));
16151 emit_move_insn (part[0][2], part[1][2]);
16153 else if (nparts == 4)
16155 emit_move_insn (part[0][3], part[1][3]);
16156 emit_move_insn (part[0][2], part[1][2]);
16161 /* In 64bit mode we don't have 32bit push available. In case this is
16162 register, it is OK - we will just use larger counterpart. We also
16163 retype memory - these comes from attempt to avoid REX prefix on
16164 moving of second half of TFmode value. */
16165 if (GET_MODE (part[1][1]) == SImode)
16167 switch (GET_CODE (part[1][1]))
16170 part[1][1] = adjust_address (part[1][1], DImode, 0);
16174 part[1][1] = gen_rtx_REG (DImode, REGNO (part[1][1]));
16178 gcc_unreachable ();
16181 if (GET_MODE (part[1][0]) == SImode)
16182 part[1][0] = part[1][1];
16185 emit_move_insn (part[0][1], part[1][1]);
16186 emit_move_insn (part[0][0], part[1][0]);
16190 /* Choose correct order to not overwrite the source before it is copied. */
16191 if ((REG_P (part[0][0])
16192 && REG_P (part[1][1])
16193 && (REGNO (part[0][0]) == REGNO (part[1][1])
16195 && REGNO (part[0][0]) == REGNO (part[1][2]))
16197 && REGNO (part[0][0]) == REGNO (part[1][3]))))
16199 && reg_overlap_mentioned_p (part[0][0], XEXP (part[1][0], 0))))
16201 for (i = 0, j = nparts - 1; i < nparts; i++, j--)
16203 operands[2 + i] = part[0][j];
16204 operands[6 + i] = part[1][j];
16209 for (i = 0; i < nparts; i++)
16211 operands[2 + i] = part[0][i];
16212 operands[6 + i] = part[1][i];
16216 /* If optimizing for size, attempt to locally unCSE nonzero constants. */
16217 if (optimize_insn_for_size_p ())
16219 for (j = 0; j < nparts - 1; j++)
16220 if (CONST_INT_P (operands[6 + j])
16221 && operands[6 + j] != const0_rtx
16222 && REG_P (operands[2 + j]))
16223 for (i = j; i < nparts - 1; i++)
16224 if (CONST_INT_P (operands[7 + i])
16225 && INTVAL (operands[7 + i]) == INTVAL (operands[6 + j]))
16226 operands[7 + i] = operands[2 + j];
16229 for (i = 0; i < nparts; i++)
16230 emit_move_insn (operands[2 + i], operands[6 + i]);
16235 /* Helper function of ix86_split_ashl used to generate an SImode/DImode
16236 left shift by a constant, either using a single shift or
16237 a sequence of add instructions. */
16240 ix86_expand_ashl_const (rtx operand, int count, enum machine_mode mode)
16244 emit_insn ((mode == DImode
16246 : gen_adddi3) (operand, operand, operand));
16248 else if (!optimize_insn_for_size_p ()
16249 && count * ix86_cost->add <= ix86_cost->shift_const)
16252 for (i=0; i<count; i++)
16254 emit_insn ((mode == DImode
16256 : gen_adddi3) (operand, operand, operand));
16260 emit_insn ((mode == DImode
16262 : gen_ashldi3) (operand, operand, GEN_INT (count)));
16266 ix86_split_ashl (rtx *operands, rtx scratch, enum machine_mode mode)
16268 rtx low[2], high[2];
16270 const int single_width = mode == DImode ? 32 : 64;
16272 if (CONST_INT_P (operands[2]))
16274 (mode == DImode ? split_di : split_ti) (operands, 2, low, high);
16275 count = INTVAL (operands[2]) & (single_width * 2 - 1);
16277 if (count >= single_width)
16279 emit_move_insn (high[0], low[1]);
16280 emit_move_insn (low[0], const0_rtx);
16282 if (count > single_width)
16283 ix86_expand_ashl_const (high[0], count - single_width, mode);
16287 if (!rtx_equal_p (operands[0], operands[1]))
16288 emit_move_insn (operands[0], operands[1]);
16289 emit_insn ((mode == DImode
16291 : gen_x86_64_shld) (high[0], low[0], GEN_INT (count)));
16292 ix86_expand_ashl_const (low[0], count, mode);
16297 (mode == DImode ? split_di : split_ti) (operands, 1, low, high);
16299 if (operands[1] == const1_rtx)
16301 /* Assuming we've chosen a QImode capable registers, then 1 << N
16302 can be done with two 32/64-bit shifts, no branches, no cmoves. */
16303 if (ANY_QI_REG_P (low[0]) && ANY_QI_REG_P (high[0]))
16305 rtx s, d, flags = gen_rtx_REG (CCZmode, FLAGS_REG);
16307 ix86_expand_clear (low[0]);
16308 ix86_expand_clear (high[0]);
16309 emit_insn (gen_testqi_ccz_1 (operands[2], GEN_INT (single_width)));
16311 d = gen_lowpart (QImode, low[0]);
16312 d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
16313 s = gen_rtx_EQ (QImode, flags, const0_rtx);
16314 emit_insn (gen_rtx_SET (VOIDmode, d, s));
16316 d = gen_lowpart (QImode, high[0]);
16317 d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
16318 s = gen_rtx_NE (QImode, flags, const0_rtx);
16319 emit_insn (gen_rtx_SET (VOIDmode, d, s));
16322 /* Otherwise, we can get the same results by manually performing
16323 a bit extract operation on bit 5/6, and then performing the two
16324 shifts. The two methods of getting 0/1 into low/high are exactly
16325 the same size. Avoiding the shift in the bit extract case helps
16326 pentium4 a bit; no one else seems to care much either way. */
16331 if (TARGET_PARTIAL_REG_STALL && !optimize_insn_for_size_p ())
16332 x = gen_rtx_ZERO_EXTEND (mode == DImode ? SImode : DImode, operands[2]);
16334 x = gen_lowpart (mode == DImode ? SImode : DImode, operands[2]);
16335 emit_insn (gen_rtx_SET (VOIDmode, high[0], x));
16337 emit_insn ((mode == DImode
16339 : gen_lshrdi3) (high[0], high[0], GEN_INT (mode == DImode ? 5 : 6)));
16340 emit_insn ((mode == DImode
16342 : gen_anddi3) (high[0], high[0], GEN_INT (1)));
16343 emit_move_insn (low[0], high[0]);
16344 emit_insn ((mode == DImode
16346 : gen_xordi3) (low[0], low[0], GEN_INT (1)));
16349 emit_insn ((mode == DImode
16351 : gen_ashldi3) (low[0], low[0], operands[2]));
16352 emit_insn ((mode == DImode
16354 : gen_ashldi3) (high[0], high[0], operands[2]));
16358 if (operands[1] == constm1_rtx)
16360 /* For -1 << N, we can avoid the shld instruction, because we
16361 know that we're shifting 0...31/63 ones into a -1. */
16362 emit_move_insn (low[0], constm1_rtx);
16363 if (optimize_insn_for_size_p ())
16364 emit_move_insn (high[0], low[0]);
16366 emit_move_insn (high[0], constm1_rtx);
16370 if (!rtx_equal_p (operands[0], operands[1]))
16371 emit_move_insn (operands[0], operands[1]);
16373 (mode == DImode ? split_di : split_ti) (operands, 1, low, high);
16374 emit_insn ((mode == DImode
16376 : gen_x86_64_shld) (high[0], low[0], operands[2]));
16379 emit_insn ((mode == DImode ? gen_ashlsi3 : gen_ashldi3) (low[0], low[0], operands[2]));
16381 if (TARGET_CMOVE && scratch)
16383 ix86_expand_clear (scratch);
16384 emit_insn ((mode == DImode
16385 ? gen_x86_shift_adj_1
16386 : gen_x86_64_shift_adj_1) (high[0], low[0], operands[2],
16390 emit_insn ((mode == DImode
16391 ? gen_x86_shift_adj_2
16392 : gen_x86_64_shift_adj_2) (high[0], low[0], operands[2]));
16396 ix86_split_ashr (rtx *operands, rtx scratch, enum machine_mode mode)
16398 rtx low[2], high[2];
16400 const int single_width = mode == DImode ? 32 : 64;
16402 if (CONST_INT_P (operands[2]))
16404 (mode == DImode ? split_di : split_ti) (operands, 2, low, high);
16405 count = INTVAL (operands[2]) & (single_width * 2 - 1);
16407 if (count == single_width * 2 - 1)
16409 emit_move_insn (high[0], high[1]);
16410 emit_insn ((mode == DImode
16412 : gen_ashrdi3) (high[0], high[0],
16413 GEN_INT (single_width - 1)));
16414 emit_move_insn (low[0], high[0]);
16417 else if (count >= single_width)
16419 emit_move_insn (low[0], high[1]);
16420 emit_move_insn (high[0], low[0]);
16421 emit_insn ((mode == DImode
16423 : gen_ashrdi3) (high[0], high[0],
16424 GEN_INT (single_width - 1)));
16425 if (count > single_width)
16426 emit_insn ((mode == DImode
16428 : gen_ashrdi3) (low[0], low[0],
16429 GEN_INT (count - single_width)));
16433 if (!rtx_equal_p (operands[0], operands[1]))
16434 emit_move_insn (operands[0], operands[1]);
16435 emit_insn ((mode == DImode
16437 : gen_x86_64_shrd) (low[0], high[0], GEN_INT (count)));
16438 emit_insn ((mode == DImode
16440 : gen_ashrdi3) (high[0], high[0], GEN_INT (count)));
16445 if (!rtx_equal_p (operands[0], operands[1]))
16446 emit_move_insn (operands[0], operands[1]);
16448 (mode == DImode ? split_di : split_ti) (operands, 1, low, high);
16450 emit_insn ((mode == DImode
16452 : gen_x86_64_shrd) (low[0], high[0], operands[2]));
16453 emit_insn ((mode == DImode
16455 : gen_ashrdi3) (high[0], high[0], operands[2]));
16457 if (TARGET_CMOVE && scratch)
16459 emit_move_insn (scratch, high[0]);
16460 emit_insn ((mode == DImode
16462 : gen_ashrdi3) (scratch, scratch,
16463 GEN_INT (single_width - 1)));
16464 emit_insn ((mode == DImode
16465 ? gen_x86_shift_adj_1
16466 : gen_x86_64_shift_adj_1) (low[0], high[0], operands[2],
16470 emit_insn ((mode == DImode
16471 ? gen_x86_shift_adj_3
16472 : gen_x86_64_shift_adj_3) (low[0], high[0], operands[2]));
16477 ix86_split_lshr (rtx *operands, rtx scratch, enum machine_mode mode)
16479 rtx low[2], high[2];
16481 const int single_width = mode == DImode ? 32 : 64;
16483 if (CONST_INT_P (operands[2]))
16485 (mode == DImode ? split_di : split_ti) (operands, 2, low, high);
16486 count = INTVAL (operands[2]) & (single_width * 2 - 1);
16488 if (count >= single_width)
16490 emit_move_insn (low[0], high[1]);
16491 ix86_expand_clear (high[0]);
16493 if (count > single_width)
16494 emit_insn ((mode == DImode
16496 : gen_lshrdi3) (low[0], low[0],
16497 GEN_INT (count - single_width)));
16501 if (!rtx_equal_p (operands[0], operands[1]))
16502 emit_move_insn (operands[0], operands[1]);
16503 emit_insn ((mode == DImode
16505 : gen_x86_64_shrd) (low[0], high[0], GEN_INT (count)));
16506 emit_insn ((mode == DImode
16508 : gen_lshrdi3) (high[0], high[0], GEN_INT (count)));
16513 if (!rtx_equal_p (operands[0], operands[1]))
16514 emit_move_insn (operands[0], operands[1]);
16516 (mode == DImode ? split_di : split_ti) (operands, 1, low, high);
16518 emit_insn ((mode == DImode
16520 : gen_x86_64_shrd) (low[0], high[0], operands[2]));
16521 emit_insn ((mode == DImode
16523 : gen_lshrdi3) (high[0], high[0], operands[2]));
16525 /* Heh. By reversing the arguments, we can reuse this pattern. */
16526 if (TARGET_CMOVE && scratch)
16528 ix86_expand_clear (scratch);
16529 emit_insn ((mode == DImode
16530 ? gen_x86_shift_adj_1
16531 : gen_x86_64_shift_adj_1) (low[0], high[0], operands[2],
16535 emit_insn ((mode == DImode
16536 ? gen_x86_shift_adj_2
16537 : gen_x86_64_shift_adj_2) (low[0], high[0], operands[2]));
16541 /* Predict just emitted jump instruction to be taken with probability PROB. */
16543 predict_jump (int prob)
16545 rtx insn = get_last_insn ();
16546 gcc_assert (JUMP_P (insn));
16548 = gen_rtx_EXPR_LIST (REG_BR_PROB,
16553 /* Helper function for the string operations below. Dest VARIABLE whether
16554 it is aligned to VALUE bytes. If true, jump to the label. */
16556 ix86_expand_aligntest (rtx variable, int value, bool epilogue)
16558 rtx label = gen_label_rtx ();
16559 rtx tmpcount = gen_reg_rtx (GET_MODE (variable));
16560 if (GET_MODE (variable) == DImode)
16561 emit_insn (gen_anddi3 (tmpcount, variable, GEN_INT (value)));
16563 emit_insn (gen_andsi3 (tmpcount, variable, GEN_INT (value)));
16564 emit_cmp_and_jump_insns (tmpcount, const0_rtx, EQ, 0, GET_MODE (variable),
16567 predict_jump (REG_BR_PROB_BASE * 50 / 100);
16569 predict_jump (REG_BR_PROB_BASE * 90 / 100);
16573 /* Adjust COUNTER by the VALUE. */
16575 ix86_adjust_counter (rtx countreg, HOST_WIDE_INT value)
16577 if (GET_MODE (countreg) == DImode)
16578 emit_insn (gen_adddi3 (countreg, countreg, GEN_INT (-value)));
16580 emit_insn (gen_addsi3 (countreg, countreg, GEN_INT (-value)));
16583 /* Zero extend possibly SImode EXP to Pmode register. */
16585 ix86_zero_extend_to_Pmode (rtx exp)
16588 if (GET_MODE (exp) == VOIDmode)
16589 return force_reg (Pmode, exp);
16590 if (GET_MODE (exp) == Pmode)
16591 return copy_to_mode_reg (Pmode, exp);
16592 r = gen_reg_rtx (Pmode);
16593 emit_insn (gen_zero_extendsidi2 (r, exp));
16597 /* Divide COUNTREG by SCALE. */
16599 scale_counter (rtx countreg, int scale)
16602 rtx piece_size_mask;
16606 if (CONST_INT_P (countreg))
16607 return GEN_INT (INTVAL (countreg) / scale);
16608 gcc_assert (REG_P (countreg));
16610 piece_size_mask = GEN_INT (scale - 1);
16611 sc = expand_simple_binop (GET_MODE (countreg), LSHIFTRT, countreg,
16612 GEN_INT (exact_log2 (scale)),
16613 NULL, 1, OPTAB_DIRECT);
16617 /* Return mode for the memcpy/memset loop counter. Prefer SImode over
16618 DImode for constant loop counts. */
16620 static enum machine_mode
16621 counter_mode (rtx count_exp)
16623 if (GET_MODE (count_exp) != VOIDmode)
16624 return GET_MODE (count_exp);
16625 if (GET_CODE (count_exp) != CONST_INT)
16627 if (TARGET_64BIT && (INTVAL (count_exp) & ~0xffffffff))
16632 /* When SRCPTR is non-NULL, output simple loop to move memory
16633 pointer to SRCPTR to DESTPTR via chunks of MODE unrolled UNROLL times,
16634 overall size is COUNT specified in bytes. When SRCPTR is NULL, output the
16635 equivalent loop to set memory by VALUE (supposed to be in MODE).
16637 The size is rounded down to whole number of chunk size moved at once.
16638 SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info. */
16642 expand_set_or_movmem_via_loop (rtx destmem, rtx srcmem,
16643 rtx destptr, rtx srcptr, rtx value,
16644 rtx count, enum machine_mode mode, int unroll,
16647 rtx out_label, top_label, iter, tmp;
16648 enum machine_mode iter_mode = counter_mode (count);
16649 rtx piece_size = GEN_INT (GET_MODE_SIZE (mode) * unroll);
16650 rtx piece_size_mask = GEN_INT (~((GET_MODE_SIZE (mode) * unroll) - 1));
16656 top_label = gen_label_rtx ();
16657 out_label = gen_label_rtx ();
16658 iter = gen_reg_rtx (iter_mode);
16660 size = expand_simple_binop (iter_mode, AND, count, piece_size_mask,
16661 NULL, 1, OPTAB_DIRECT);
16662 /* Those two should combine. */
16663 if (piece_size == const1_rtx)
16665 emit_cmp_and_jump_insns (size, const0_rtx, EQ, NULL_RTX, iter_mode,
16667 predict_jump (REG_BR_PROB_BASE * 10 / 100);
16669 emit_move_insn (iter, const0_rtx);
16671 emit_label (top_label);
16673 tmp = convert_modes (Pmode, iter_mode, iter, true);
16674 x_addr = gen_rtx_PLUS (Pmode, destptr, tmp);
16675 destmem = change_address (destmem, mode, x_addr);
16679 y_addr = gen_rtx_PLUS (Pmode, srcptr, copy_rtx (tmp));
16680 srcmem = change_address (srcmem, mode, y_addr);
16682 /* When unrolling for chips that reorder memory reads and writes,
16683 we can save registers by using single temporary.
16684 Also using 4 temporaries is overkill in 32bit mode. */
16685 if (!TARGET_64BIT && 0)
16687 for (i = 0; i < unroll; i++)
16692 adjust_address (copy_rtx (destmem), mode, GET_MODE_SIZE (mode));
16694 adjust_address (copy_rtx (srcmem), mode, GET_MODE_SIZE (mode));
16696 emit_move_insn (destmem, srcmem);
16702 gcc_assert (unroll <= 4);
16703 for (i = 0; i < unroll; i++)
16705 tmpreg[i] = gen_reg_rtx (mode);
16709 adjust_address (copy_rtx (srcmem), mode, GET_MODE_SIZE (mode));
16711 emit_move_insn (tmpreg[i], srcmem);
16713 for (i = 0; i < unroll; i++)
16718 adjust_address (copy_rtx (destmem), mode, GET_MODE_SIZE (mode));
16720 emit_move_insn (destmem, tmpreg[i]);
16725 for (i = 0; i < unroll; i++)
16729 adjust_address (copy_rtx (destmem), mode, GET_MODE_SIZE (mode));
16730 emit_move_insn (destmem, value);
16733 tmp = expand_simple_binop (iter_mode, PLUS, iter, piece_size, iter,
16734 true, OPTAB_LIB_WIDEN);
16736 emit_move_insn (iter, tmp);
16738 emit_cmp_and_jump_insns (iter, size, LT, NULL_RTX, iter_mode,
16740 if (expected_size != -1)
16742 expected_size /= GET_MODE_SIZE (mode) * unroll;
16743 if (expected_size == 0)
16745 else if (expected_size > REG_BR_PROB_BASE)
16746 predict_jump (REG_BR_PROB_BASE - 1);
16748 predict_jump (REG_BR_PROB_BASE - (REG_BR_PROB_BASE + expected_size / 2) / expected_size);
16751 predict_jump (REG_BR_PROB_BASE * 80 / 100);
16752 iter = ix86_zero_extend_to_Pmode (iter);
16753 tmp = expand_simple_binop (Pmode, PLUS, destptr, iter, destptr,
16754 true, OPTAB_LIB_WIDEN);
16755 if (tmp != destptr)
16756 emit_move_insn (destptr, tmp);
16759 tmp = expand_simple_binop (Pmode, PLUS, srcptr, iter, srcptr,
16760 true, OPTAB_LIB_WIDEN);
16762 emit_move_insn (srcptr, tmp);
16764 emit_label (out_label);
16767 /* Output "rep; mov" instruction.
16768 Arguments have same meaning as for previous function */
16770 expand_movmem_via_rep_mov (rtx destmem, rtx srcmem,
16771 rtx destptr, rtx srcptr,
16773 enum machine_mode mode)
16779 /* If the size is known, it is shorter to use rep movs. */
16780 if (mode == QImode && CONST_INT_P (count)
16781 && !(INTVAL (count) & 3))
16784 if (destptr != XEXP (destmem, 0) || GET_MODE (destmem) != BLKmode)
16785 destmem = adjust_automodify_address_nv (destmem, BLKmode, destptr, 0);
16786 if (srcptr != XEXP (srcmem, 0) || GET_MODE (srcmem) != BLKmode)
16787 srcmem = adjust_automodify_address_nv (srcmem, BLKmode, srcptr, 0);
16788 countreg = ix86_zero_extend_to_Pmode (scale_counter (count, GET_MODE_SIZE (mode)));
16789 if (mode != QImode)
16791 destexp = gen_rtx_ASHIFT (Pmode, countreg,
16792 GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
16793 destexp = gen_rtx_PLUS (Pmode, destexp, destptr);
16794 srcexp = gen_rtx_ASHIFT (Pmode, countreg,
16795 GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
16796 srcexp = gen_rtx_PLUS (Pmode, srcexp, srcptr);
16800 destexp = gen_rtx_PLUS (Pmode, destptr, countreg);
16801 srcexp = gen_rtx_PLUS (Pmode, srcptr, countreg);
16803 if (CONST_INT_P (count))
16805 count = GEN_INT (INTVAL (count)
16806 & ~((HOST_WIDE_INT) GET_MODE_SIZE (mode) - 1));
16807 destmem = shallow_copy_rtx (destmem);
16808 srcmem = shallow_copy_rtx (srcmem);
16809 set_mem_size (destmem, count);
16810 set_mem_size (srcmem, count);
16814 if (MEM_SIZE (destmem))
16815 set_mem_size (destmem, NULL_RTX);
16816 if (MEM_SIZE (srcmem))
16817 set_mem_size (srcmem, NULL_RTX);
16819 emit_insn (gen_rep_mov (destptr, destmem, srcptr, srcmem, countreg,
16823 /* Output "rep; stos" instruction.
16824 Arguments have same meaning as for previous function */
16826 expand_setmem_via_rep_stos (rtx destmem, rtx destptr, rtx value,
16827 rtx count, enum machine_mode mode,
16833 if (destptr != XEXP (destmem, 0) || GET_MODE (destmem) != BLKmode)
16834 destmem = adjust_automodify_address_nv (destmem, BLKmode, destptr, 0);
16835 value = force_reg (mode, gen_lowpart (mode, value));
16836 countreg = ix86_zero_extend_to_Pmode (scale_counter (count, GET_MODE_SIZE (mode)));
16837 if (mode != QImode)
16839 destexp = gen_rtx_ASHIFT (Pmode, countreg,
16840 GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
16841 destexp = gen_rtx_PLUS (Pmode, destexp, destptr);
16844 destexp = gen_rtx_PLUS (Pmode, destptr, countreg);
16845 if (orig_value == const0_rtx && CONST_INT_P (count))
16847 count = GEN_INT (INTVAL (count)
16848 & ~((HOST_WIDE_INT) GET_MODE_SIZE (mode) - 1));
16849 destmem = shallow_copy_rtx (destmem);
16850 set_mem_size (destmem, count);
16852 else if (MEM_SIZE (destmem))
16853 set_mem_size (destmem, NULL_RTX);
16854 emit_insn (gen_rep_stos (destptr, countreg, destmem, value, destexp));
16858 emit_strmov (rtx destmem, rtx srcmem,
16859 rtx destptr, rtx srcptr, enum machine_mode mode, int offset)
16861 rtx src = adjust_automodify_address_nv (srcmem, mode, srcptr, offset);
16862 rtx dest = adjust_automodify_address_nv (destmem, mode, destptr, offset);
16863 emit_insn (gen_strmov (destptr, dest, srcptr, src));
16866 /* Output code to copy at most count & (max_size - 1) bytes from SRC to DEST. */
16868 expand_movmem_epilogue (rtx destmem, rtx srcmem,
16869 rtx destptr, rtx srcptr, rtx count, int max_size)
16872 if (CONST_INT_P (count))
16874 HOST_WIDE_INT countval = INTVAL (count);
16877 if ((countval & 0x10) && max_size > 16)
16881 emit_strmov (destmem, srcmem, destptr, srcptr, DImode, offset);
16882 emit_strmov (destmem, srcmem, destptr, srcptr, DImode, offset + 8);
16885 gcc_unreachable ();
16888 if ((countval & 0x08) && max_size > 8)
16891 emit_strmov (destmem, srcmem, destptr, srcptr, DImode, offset);
16894 emit_strmov (destmem, srcmem, destptr, srcptr, SImode, offset);
16895 emit_strmov (destmem, srcmem, destptr, srcptr, SImode, offset + 4);
16899 if ((countval & 0x04) && max_size > 4)
16901 emit_strmov (destmem, srcmem, destptr, srcptr, SImode, offset);
16904 if ((countval & 0x02) && max_size > 2)
16906 emit_strmov (destmem, srcmem, destptr, srcptr, HImode, offset);
16909 if ((countval & 0x01) && max_size > 1)
16911 emit_strmov (destmem, srcmem, destptr, srcptr, QImode, offset);
16918 count = expand_simple_binop (GET_MODE (count), AND, count, GEN_INT (max_size - 1),
16919 count, 1, OPTAB_DIRECT);
16920 expand_set_or_movmem_via_loop (destmem, srcmem, destptr, srcptr, NULL,
16921 count, QImode, 1, 4);
16925 /* When there are stringops, we can cheaply increase dest and src pointers.
16926 Otherwise we save code size by maintaining offset (zero is readily
16927 available from preceding rep operation) and using x86 addressing modes.
16929 if (TARGET_SINGLE_STRINGOP)
16933 rtx label = ix86_expand_aligntest (count, 4, true);
16934 src = change_address (srcmem, SImode, srcptr);
16935 dest = change_address (destmem, SImode, destptr);
16936 emit_insn (gen_strmov (destptr, dest, srcptr, src));
16937 emit_label (label);
16938 LABEL_NUSES (label) = 1;
16942 rtx label = ix86_expand_aligntest (count, 2, true);
16943 src = change_address (srcmem, HImode, srcptr);
16944 dest = change_address (destmem, HImode, destptr);
16945 emit_insn (gen_strmov (destptr, dest, srcptr, src));
16946 emit_label (label);
16947 LABEL_NUSES (label) = 1;
16951 rtx label = ix86_expand_aligntest (count, 1, true);
16952 src = change_address (srcmem, QImode, srcptr);
16953 dest = change_address (destmem, QImode, destptr);
16954 emit_insn (gen_strmov (destptr, dest, srcptr, src));
16955 emit_label (label);
16956 LABEL_NUSES (label) = 1;
16961 rtx offset = force_reg (Pmode, const0_rtx);
16966 rtx label = ix86_expand_aligntest (count, 4, true);
16967 src = change_address (srcmem, SImode, srcptr);
16968 dest = change_address (destmem, SImode, destptr);
16969 emit_move_insn (dest, src);
16970 tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (4), NULL,
16971 true, OPTAB_LIB_WIDEN);
16973 emit_move_insn (offset, tmp);
16974 emit_label (label);
16975 LABEL_NUSES (label) = 1;
16979 rtx label = ix86_expand_aligntest (count, 2, true);
16980 tmp = gen_rtx_PLUS (Pmode, srcptr, offset);
16981 src = change_address (srcmem, HImode, tmp);
16982 tmp = gen_rtx_PLUS (Pmode, destptr, offset);
16983 dest = change_address (destmem, HImode, tmp);
16984 emit_move_insn (dest, src);
16985 tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (2), tmp,
16986 true, OPTAB_LIB_WIDEN);
16988 emit_move_insn (offset, tmp);
16989 emit_label (label);
16990 LABEL_NUSES (label) = 1;
16994 rtx label = ix86_expand_aligntest (count, 1, true);
16995 tmp = gen_rtx_PLUS (Pmode, srcptr, offset);
16996 src = change_address (srcmem, QImode, tmp);
16997 tmp = gen_rtx_PLUS (Pmode, destptr, offset);
16998 dest = change_address (destmem, QImode, tmp);
16999 emit_move_insn (dest, src);
17000 emit_label (label);
17001 LABEL_NUSES (label) = 1;
17006 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
17008 expand_setmem_epilogue_via_loop (rtx destmem, rtx destptr, rtx value,
17009 rtx count, int max_size)
17012 expand_simple_binop (counter_mode (count), AND, count,
17013 GEN_INT (max_size - 1), count, 1, OPTAB_DIRECT);
17014 expand_set_or_movmem_via_loop (destmem, NULL, destptr, NULL,
17015 gen_lowpart (QImode, value), count, QImode,
17019 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
17021 expand_setmem_epilogue (rtx destmem, rtx destptr, rtx value, rtx count, int max_size)
17025 if (CONST_INT_P (count))
17027 HOST_WIDE_INT countval = INTVAL (count);
17030 if ((countval & 0x10) && max_size > 16)
17034 dest = adjust_automodify_address_nv (destmem, DImode, destptr, offset);
17035 emit_insn (gen_strset (destptr, dest, value));
17036 dest = adjust_automodify_address_nv (destmem, DImode, destptr, offset + 8);
17037 emit_insn (gen_strset (destptr, dest, value));
17040 gcc_unreachable ();
17043 if ((countval & 0x08) && max_size > 8)
17047 dest = adjust_automodify_address_nv (destmem, DImode, destptr, offset);
17048 emit_insn (gen_strset (destptr, dest, value));
17052 dest = adjust_automodify_address_nv (destmem, SImode, destptr, offset);
17053 emit_insn (gen_strset (destptr, dest, value));
17054 dest = adjust_automodify_address_nv (destmem, SImode, destptr, offset + 4);
17055 emit_insn (gen_strset (destptr, dest, value));
17059 if ((countval & 0x04) && max_size > 4)
17061 dest = adjust_automodify_address_nv (destmem, SImode, destptr, offset);
17062 emit_insn (gen_strset (destptr, dest, gen_lowpart (SImode, value)));
17065 if ((countval & 0x02) && max_size > 2)
17067 dest = adjust_automodify_address_nv (destmem, HImode, destptr, offset);
17068 emit_insn (gen_strset (destptr, dest, gen_lowpart (HImode, value)));
17071 if ((countval & 0x01) && max_size > 1)
17073 dest = adjust_automodify_address_nv (destmem, QImode, destptr, offset);
17074 emit_insn (gen_strset (destptr, dest, gen_lowpart (QImode, value)));
17081 expand_setmem_epilogue_via_loop (destmem, destptr, value, count, max_size);
17086 rtx label = ix86_expand_aligntest (count, 16, true);
17089 dest = change_address (destmem, DImode, destptr);
17090 emit_insn (gen_strset (destptr, dest, value));
17091 emit_insn (gen_strset (destptr, dest, value));
17095 dest = change_address (destmem, SImode, destptr);
17096 emit_insn (gen_strset (destptr, dest, value));
17097 emit_insn (gen_strset (destptr, dest, value));
17098 emit_insn (gen_strset (destptr, dest, value));
17099 emit_insn (gen_strset (destptr, dest, value));
17101 emit_label (label);
17102 LABEL_NUSES (label) = 1;
17106 rtx label = ix86_expand_aligntest (count, 8, true);
17109 dest = change_address (destmem, DImode, destptr);
17110 emit_insn (gen_strset (destptr, dest, value));
17114 dest = change_address (destmem, SImode, destptr);
17115 emit_insn (gen_strset (destptr, dest, value));
17116 emit_insn (gen_strset (destptr, dest, value));
17118 emit_label (label);
17119 LABEL_NUSES (label) = 1;
17123 rtx label = ix86_expand_aligntest (count, 4, true);
17124 dest = change_address (destmem, SImode, destptr);
17125 emit_insn (gen_strset (destptr, dest, gen_lowpart (SImode, value)));
17126 emit_label (label);
17127 LABEL_NUSES (label) = 1;
17131 rtx label = ix86_expand_aligntest (count, 2, true);
17132 dest = change_address (destmem, HImode, destptr);
17133 emit_insn (gen_strset (destptr, dest, gen_lowpart (HImode, value)));
17134 emit_label (label);
17135 LABEL_NUSES (label) = 1;
17139 rtx label = ix86_expand_aligntest (count, 1, true);
17140 dest = change_address (destmem, QImode, destptr);
17141 emit_insn (gen_strset (destptr, dest, gen_lowpart (QImode, value)));
17142 emit_label (label);
17143 LABEL_NUSES (label) = 1;
17147 /* Copy enough from DEST to SRC to align DEST known to by aligned by ALIGN to
17148 DESIRED_ALIGNMENT. */
17150 expand_movmem_prologue (rtx destmem, rtx srcmem,
17151 rtx destptr, rtx srcptr, rtx count,
17152 int align, int desired_alignment)
17154 if (align <= 1 && desired_alignment > 1)
17156 rtx label = ix86_expand_aligntest (destptr, 1, false);
17157 srcmem = change_address (srcmem, QImode, srcptr);
17158 destmem = change_address (destmem, QImode, destptr);
17159 emit_insn (gen_strmov (destptr, destmem, srcptr, srcmem));
17160 ix86_adjust_counter (count, 1);
17161 emit_label (label);
17162 LABEL_NUSES (label) = 1;
17164 if (align <= 2 && desired_alignment > 2)
17166 rtx label = ix86_expand_aligntest (destptr, 2, false);
17167 srcmem = change_address (srcmem, HImode, srcptr);
17168 destmem = change_address (destmem, HImode, destptr);
17169 emit_insn (gen_strmov (destptr, destmem, srcptr, srcmem));
17170 ix86_adjust_counter (count, 2);
17171 emit_label (label);
17172 LABEL_NUSES (label) = 1;
17174 if (align <= 4 && desired_alignment > 4)
17176 rtx label = ix86_expand_aligntest (destptr, 4, false);
17177 srcmem = change_address (srcmem, SImode, srcptr);
17178 destmem = change_address (destmem, SImode, destptr);
17179 emit_insn (gen_strmov (destptr, destmem, srcptr, srcmem));
17180 ix86_adjust_counter (count, 4);
17181 emit_label (label);
17182 LABEL_NUSES (label) = 1;
17184 gcc_assert (desired_alignment <= 8);
17187 /* Copy enough from DST to SRC to align DST known to DESIRED_ALIGN.
17188 ALIGN_BYTES is how many bytes need to be copied. */
17190 expand_constant_movmem_prologue (rtx dst, rtx *srcp, rtx destreg, rtx srcreg,
17191 int desired_align, int align_bytes)
17194 rtx src_size, dst_size;
17196 int src_align_bytes = get_mem_align_offset (src, desired_align * BITS_PER_UNIT);
17197 if (src_align_bytes >= 0)
17198 src_align_bytes = desired_align - src_align_bytes;
17199 src_size = MEM_SIZE (src);
17200 dst_size = MEM_SIZE (dst);
17201 if (align_bytes & 1)
17203 dst = adjust_automodify_address_nv (dst, QImode, destreg, 0);
17204 src = adjust_automodify_address_nv (src, QImode, srcreg, 0);
17206 emit_insn (gen_strmov (destreg, dst, srcreg, src));
17208 if (align_bytes & 2)
17210 dst = adjust_automodify_address_nv (dst, HImode, destreg, off);
17211 src = adjust_automodify_address_nv (src, HImode, srcreg, off);
17212 if (MEM_ALIGN (dst) < 2 * BITS_PER_UNIT)
17213 set_mem_align (dst, 2 * BITS_PER_UNIT);
17214 if (src_align_bytes >= 0
17215 && (src_align_bytes & 1) == (align_bytes & 1)
17216 && MEM_ALIGN (src) < 2 * BITS_PER_UNIT)
17217 set_mem_align (src, 2 * BITS_PER_UNIT);
17219 emit_insn (gen_strmov (destreg, dst, srcreg, src));
17221 if (align_bytes & 4)
17223 dst = adjust_automodify_address_nv (dst, SImode, destreg, off);
17224 src = adjust_automodify_address_nv (src, SImode, srcreg, off);
17225 if (MEM_ALIGN (dst) < 4 * BITS_PER_UNIT)
17226 set_mem_align (dst, 4 * BITS_PER_UNIT);
17227 if (src_align_bytes >= 0)
17229 unsigned int src_align = 0;
17230 if ((src_align_bytes & 3) == (align_bytes & 3))
17232 else if ((src_align_bytes & 1) == (align_bytes & 1))
17234 if (MEM_ALIGN (src) < src_align * BITS_PER_UNIT)
17235 set_mem_align (src, src_align * BITS_PER_UNIT);
17238 emit_insn (gen_strmov (destreg, dst, srcreg, src));
17240 dst = adjust_automodify_address_nv (dst, BLKmode, destreg, off);
17241 src = adjust_automodify_address_nv (src, BLKmode, srcreg, off);
17242 if (MEM_ALIGN (dst) < (unsigned int) desired_align * BITS_PER_UNIT)
17243 set_mem_align (dst, desired_align * BITS_PER_UNIT);
17244 if (src_align_bytes >= 0)
17246 unsigned int src_align = 0;
17247 if ((src_align_bytes & 7) == (align_bytes & 7))
17249 else if ((src_align_bytes & 3) == (align_bytes & 3))
17251 else if ((src_align_bytes & 1) == (align_bytes & 1))
17253 if (src_align > (unsigned int) desired_align)
17254 src_align = desired_align;
17255 if (MEM_ALIGN (src) < src_align * BITS_PER_UNIT)
17256 set_mem_align (src, src_align * BITS_PER_UNIT);
17259 set_mem_size (dst, GEN_INT (INTVAL (dst_size) - align_bytes));
17261 set_mem_size (dst, GEN_INT (INTVAL (src_size) - align_bytes));
17266 /* Set enough from DEST to align DEST known to by aligned by ALIGN to
17267 DESIRED_ALIGNMENT. */
17269 expand_setmem_prologue (rtx destmem, rtx destptr, rtx value, rtx count,
17270 int align, int desired_alignment)
17272 if (align <= 1 && desired_alignment > 1)
17274 rtx label = ix86_expand_aligntest (destptr, 1, false);
17275 destmem = change_address (destmem, QImode, destptr);
17276 emit_insn (gen_strset (destptr, destmem, gen_lowpart (QImode, value)));
17277 ix86_adjust_counter (count, 1);
17278 emit_label (label);
17279 LABEL_NUSES (label) = 1;
17281 if (align <= 2 && desired_alignment > 2)
17283 rtx label = ix86_expand_aligntest (destptr, 2, false);
17284 destmem = change_address (destmem, HImode, destptr);
17285 emit_insn (gen_strset (destptr, destmem, gen_lowpart (HImode, value)));
17286 ix86_adjust_counter (count, 2);
17287 emit_label (label);
17288 LABEL_NUSES (label) = 1;
17290 if (align <= 4 && desired_alignment > 4)
17292 rtx label = ix86_expand_aligntest (destptr, 4, false);
17293 destmem = change_address (destmem, SImode, destptr);
17294 emit_insn (gen_strset (destptr, destmem, gen_lowpart (SImode, value)));
17295 ix86_adjust_counter (count, 4);
17296 emit_label (label);
17297 LABEL_NUSES (label) = 1;
17299 gcc_assert (desired_alignment <= 8);
17302 /* Set enough from DST to align DST known to by aligned by ALIGN to
17303 DESIRED_ALIGN. ALIGN_BYTES is how many bytes need to be stored. */
17305 expand_constant_setmem_prologue (rtx dst, rtx destreg, rtx value,
17306 int desired_align, int align_bytes)
17309 rtx dst_size = MEM_SIZE (dst);
17310 if (align_bytes & 1)
17312 dst = adjust_automodify_address_nv (dst, QImode, destreg, 0);
17314 emit_insn (gen_strset (destreg, dst,
17315 gen_lowpart (QImode, value)));
17317 if (align_bytes & 2)
17319 dst = adjust_automodify_address_nv (dst, HImode, destreg, off);
17320 if (MEM_ALIGN (dst) < 2 * BITS_PER_UNIT)
17321 set_mem_align (dst, 2 * BITS_PER_UNIT);
17323 emit_insn (gen_strset (destreg, dst,
17324 gen_lowpart (HImode, value)));
17326 if (align_bytes & 4)
17328 dst = adjust_automodify_address_nv (dst, SImode, destreg, off);
17329 if (MEM_ALIGN (dst) < 4 * BITS_PER_UNIT)
17330 set_mem_align (dst, 4 * BITS_PER_UNIT);
17332 emit_insn (gen_strset (destreg, dst,
17333 gen_lowpart (SImode, value)));
17335 dst = adjust_automodify_address_nv (dst, BLKmode, destreg, off);
17336 if (MEM_ALIGN (dst) < (unsigned int) desired_align * BITS_PER_UNIT)
17337 set_mem_align (dst, desired_align * BITS_PER_UNIT);
17339 set_mem_size (dst, GEN_INT (INTVAL (dst_size) - align_bytes));
17343 /* Given COUNT and EXPECTED_SIZE, decide on codegen of string operation. */
17344 static enum stringop_alg
17345 decide_alg (HOST_WIDE_INT count, HOST_WIDE_INT expected_size, bool memset,
17346 int *dynamic_check)
17348 const struct stringop_algs * algs;
17349 bool optimize_for_speed;
17350 /* Algorithms using the rep prefix want at least edi and ecx;
17351 additionally, memset wants eax and memcpy wants esi. Don't
17352 consider such algorithms if the user has appropriated those
17353 registers for their own purposes. */
17354 bool rep_prefix_usable = !(fixed_regs[CX_REG] || fixed_regs[DI_REG]
17356 ? fixed_regs[AX_REG] : fixed_regs[SI_REG]));
17358 #define ALG_USABLE_P(alg) (rep_prefix_usable \
17359 || (alg != rep_prefix_1_byte \
17360 && alg != rep_prefix_4_byte \
17361 && alg != rep_prefix_8_byte))
17362 const struct processor_costs *cost;
17364 /* Even if the string operation call is cold, we still might spend a lot
17365 of time processing large blocks. */
17366 if (optimize_function_for_size_p (cfun)
17367 || (optimize_insn_for_size_p ()
17368 && expected_size != -1 && expected_size < 256))
17369 optimize_for_speed = false;
17371 optimize_for_speed = true;
17373 cost = optimize_for_speed ? ix86_cost : &ix86_size_cost;
17375 *dynamic_check = -1;
17377 algs = &cost->memset[TARGET_64BIT != 0];
17379 algs = &cost->memcpy[TARGET_64BIT != 0];
17380 if (stringop_alg != no_stringop && ALG_USABLE_P (stringop_alg))
17381 return stringop_alg;
17382 /* rep; movq or rep; movl is the smallest variant. */
17383 else if (!optimize_for_speed)
17385 if (!count || (count & 3))
17386 return rep_prefix_usable ? rep_prefix_1_byte : loop_1_byte;
17388 return rep_prefix_usable ? rep_prefix_4_byte : loop;
17390 /* Very tiny blocks are best handled via the loop, REP is expensive to setup.
17392 else if (expected_size != -1 && expected_size < 4)
17393 return loop_1_byte;
17394 else if (expected_size != -1)
17397 enum stringop_alg alg = libcall;
17398 for (i = 0; i < NAX_STRINGOP_ALGS; i++)
17400 /* We get here if the algorithms that were not libcall-based
17401 were rep-prefix based and we are unable to use rep prefixes
17402 based on global register usage. Break out of the loop and
17403 use the heuristic below. */
17404 if (algs->size[i].max == 0)
17406 if (algs->size[i].max >= expected_size || algs->size[i].max == -1)
17408 enum stringop_alg candidate = algs->size[i].alg;
17410 if (candidate != libcall && ALG_USABLE_P (candidate))
17412 /* Honor TARGET_INLINE_ALL_STRINGOPS by picking
17413 last non-libcall inline algorithm. */
17414 if (TARGET_INLINE_ALL_STRINGOPS)
17416 /* When the current size is best to be copied by a libcall,
17417 but we are still forced to inline, run the heuristic below
17418 that will pick code for medium sized blocks. */
17419 if (alg != libcall)
17423 else if (ALG_USABLE_P (candidate))
17427 gcc_assert (TARGET_INLINE_ALL_STRINGOPS || !rep_prefix_usable);
17429 /* When asked to inline the call anyway, try to pick meaningful choice.
17430 We look for maximal size of block that is faster to copy by hand and
17431 take blocks of at most of that size guessing that average size will
17432 be roughly half of the block.
17434 If this turns out to be bad, we might simply specify the preferred
17435 choice in ix86_costs. */
17436 if ((TARGET_INLINE_ALL_STRINGOPS || TARGET_INLINE_STRINGOPS_DYNAMICALLY)
17437 && (algs->unknown_size == libcall || !ALG_USABLE_P (algs->unknown_size)))
17440 enum stringop_alg alg;
17442 bool any_alg_usable_p = true;
17444 for (i = 0; i < NAX_STRINGOP_ALGS; i++)
17446 enum stringop_alg candidate = algs->size[i].alg;
17447 any_alg_usable_p = any_alg_usable_p && ALG_USABLE_P (candidate);
17449 if (candidate != libcall && candidate
17450 && ALG_USABLE_P (candidate))
17451 max = algs->size[i].max;
17453 /* If there aren't any usable algorithms, then recursing on
17454 smaller sizes isn't going to find anything. Just return the
17455 simple byte-at-a-time copy loop. */
17456 if (!any_alg_usable_p)
17458 /* Pick something reasonable. */
17459 if (TARGET_INLINE_STRINGOPS_DYNAMICALLY)
17460 *dynamic_check = 128;
17461 return loop_1_byte;
17465 alg = decide_alg (count, max / 2, memset, dynamic_check);
17466 gcc_assert (*dynamic_check == -1);
17467 gcc_assert (alg != libcall);
17468 if (TARGET_INLINE_STRINGOPS_DYNAMICALLY)
17469 *dynamic_check = max;
17472 return ALG_USABLE_P (algs->unknown_size) ? algs->unknown_size : libcall;
17473 #undef ALG_USABLE_P
17476 /* Decide on alignment. We know that the operand is already aligned to ALIGN
17477 (ALIGN can be based on profile feedback and thus it is not 100% guaranteed). */
17479 decide_alignment (int align,
17480 enum stringop_alg alg,
17483 int desired_align = 0;
17487 gcc_unreachable ();
17489 case unrolled_loop:
17490 desired_align = GET_MODE_SIZE (Pmode);
17492 case rep_prefix_8_byte:
17495 case rep_prefix_4_byte:
17496 /* PentiumPro has special logic triggering for 8 byte aligned blocks.
17497 copying whole cacheline at once. */
17498 if (TARGET_PENTIUMPRO)
17503 case rep_prefix_1_byte:
17504 /* PentiumPro has special logic triggering for 8 byte aligned blocks.
17505 copying whole cacheline at once. */
17506 if (TARGET_PENTIUMPRO)
17520 if (desired_align < align)
17521 desired_align = align;
17522 if (expected_size != -1 && expected_size < 4)
17523 desired_align = align;
17524 return desired_align;
17527 /* Return the smallest power of 2 greater than VAL. */
17529 smallest_pow2_greater_than (int val)
17537 /* Expand string move (memcpy) operation. Use i386 string operations when
17538 profitable. expand_setmem contains similar code. The code depends upon
17539 architecture, block size and alignment, but always has the same
17542 1) Prologue guard: Conditional that jumps up to epilogues for small
17543 blocks that can be handled by epilogue alone. This is faster but
17544 also needed for correctness, since prologue assume the block is larger
17545 than the desired alignment.
17547 Optional dynamic check for size and libcall for large
17548 blocks is emitted here too, with -minline-stringops-dynamically.
17550 2) Prologue: copy first few bytes in order to get destination aligned
17551 to DESIRED_ALIGN. It is emitted only when ALIGN is less than
17552 DESIRED_ALIGN and and up to DESIRED_ALIGN - ALIGN bytes can be copied.
17553 We emit either a jump tree on power of two sized blocks, or a byte loop.
17555 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
17556 with specified algorithm.
17558 4) Epilogue: code copying tail of the block that is too small to be
17559 handled by main body (or up to size guarded by prologue guard). */
17562 ix86_expand_movmem (rtx dst, rtx src, rtx count_exp, rtx align_exp,
17563 rtx expected_align_exp, rtx expected_size_exp)
17569 rtx jump_around_label = NULL;
17570 HOST_WIDE_INT align = 1;
17571 unsigned HOST_WIDE_INT count = 0;
17572 HOST_WIDE_INT expected_size = -1;
17573 int size_needed = 0, epilogue_size_needed;
17574 int desired_align = 0, align_bytes = 0;
17575 enum stringop_alg alg;
17577 bool need_zero_guard = false;
17579 if (CONST_INT_P (align_exp))
17580 align = INTVAL (align_exp);
17581 /* i386 can do misaligned access on reasonably increased cost. */
17582 if (CONST_INT_P (expected_align_exp)
17583 && INTVAL (expected_align_exp) > align)
17584 align = INTVAL (expected_align_exp);
17585 /* ALIGN is the minimum of destination and source alignment, but we care here
17586 just about destination alignment. */
17587 else if (MEM_ALIGN (dst) > (unsigned HOST_WIDE_INT) align * BITS_PER_UNIT)
17588 align = MEM_ALIGN (dst) / BITS_PER_UNIT;
17590 if (CONST_INT_P (count_exp))
17591 count = expected_size = INTVAL (count_exp);
17592 if (CONST_INT_P (expected_size_exp) && count == 0)
17593 expected_size = INTVAL (expected_size_exp);
17595 /* Make sure we don't need to care about overflow later on. */
17596 if (count > ((unsigned HOST_WIDE_INT) 1 << 30))
17599 /* Step 0: Decide on preferred algorithm, desired alignment and
17600 size of chunks to be copied by main loop. */
17602 alg = decide_alg (count, expected_size, false, &dynamic_check);
17603 desired_align = decide_alignment (align, alg, expected_size);
17605 if (!TARGET_ALIGN_STRINGOPS)
17606 align = desired_align;
17608 if (alg == libcall)
17610 gcc_assert (alg != no_stringop);
17612 count_exp = copy_to_mode_reg (GET_MODE (count_exp), count_exp);
17613 destreg = copy_to_mode_reg (Pmode, XEXP (dst, 0));
17614 srcreg = copy_to_mode_reg (Pmode, XEXP (src, 0));
17619 gcc_unreachable ();
17621 need_zero_guard = true;
17622 size_needed = GET_MODE_SIZE (Pmode);
17624 case unrolled_loop:
17625 need_zero_guard = true;
17626 size_needed = GET_MODE_SIZE (Pmode) * (TARGET_64BIT ? 4 : 2);
17628 case rep_prefix_8_byte:
17631 case rep_prefix_4_byte:
17634 case rep_prefix_1_byte:
17638 need_zero_guard = true;
17643 epilogue_size_needed = size_needed;
17645 /* Step 1: Prologue guard. */
17647 /* Alignment code needs count to be in register. */
17648 if (CONST_INT_P (count_exp) && desired_align > align)
17650 if (INTVAL (count_exp) > desired_align
17651 && INTVAL (count_exp) > size_needed)
17654 = get_mem_align_offset (dst, desired_align * BITS_PER_UNIT);
17655 if (align_bytes <= 0)
17658 align_bytes = desired_align - align_bytes;
17660 if (align_bytes == 0)
17661 count_exp = force_reg (counter_mode (count_exp), count_exp);
17663 gcc_assert (desired_align >= 1 && align >= 1);
17665 /* Ensure that alignment prologue won't copy past end of block. */
17666 if (size_needed > 1 || (desired_align > 1 && desired_align > align))
17668 epilogue_size_needed = MAX (size_needed - 1, desired_align - align);
17669 /* Epilogue always copies COUNT_EXP & EPILOGUE_SIZE_NEEDED bytes.
17670 Make sure it is power of 2. */
17671 epilogue_size_needed = smallest_pow2_greater_than (epilogue_size_needed);
17675 if (count < (unsigned HOST_WIDE_INT)epilogue_size_needed)
17677 /* If main algorithm works on QImode, no epilogue is needed.
17678 For small sizes just don't align anything. */
17679 if (size_needed == 1)
17680 desired_align = align;
17687 label = gen_label_rtx ();
17688 emit_cmp_and_jump_insns (count_exp,
17689 GEN_INT (epilogue_size_needed),
17690 LTU, 0, counter_mode (count_exp), 1, label);
17691 if (expected_size == -1 || expected_size < epilogue_size_needed)
17692 predict_jump (REG_BR_PROB_BASE * 60 / 100);
17694 predict_jump (REG_BR_PROB_BASE * 20 / 100);
17698 /* Emit code to decide on runtime whether library call or inline should be
17700 if (dynamic_check != -1)
17702 if (CONST_INT_P (count_exp))
17704 if (UINTVAL (count_exp) >= (unsigned HOST_WIDE_INT)dynamic_check)
17706 emit_block_move_via_libcall (dst, src, count_exp, false);
17707 count_exp = const0_rtx;
17713 rtx hot_label = gen_label_rtx ();
17714 jump_around_label = gen_label_rtx ();
17715 emit_cmp_and_jump_insns (count_exp, GEN_INT (dynamic_check - 1),
17716 LEU, 0, GET_MODE (count_exp), 1, hot_label);
17717 predict_jump (REG_BR_PROB_BASE * 90 / 100);
17718 emit_block_move_via_libcall (dst, src, count_exp, false);
17719 emit_jump (jump_around_label);
17720 emit_label (hot_label);
17724 /* Step 2: Alignment prologue. */
17726 if (desired_align > align)
17728 if (align_bytes == 0)
17730 /* Except for the first move in epilogue, we no longer know
17731 constant offset in aliasing info. It don't seems to worth
17732 the pain to maintain it for the first move, so throw away
17734 src = change_address (src, BLKmode, srcreg);
17735 dst = change_address (dst, BLKmode, destreg);
17736 expand_movmem_prologue (dst, src, destreg, srcreg, count_exp, align,
17741 /* If we know how many bytes need to be stored before dst is
17742 sufficiently aligned, maintain aliasing info accurately. */
17743 dst = expand_constant_movmem_prologue (dst, &src, destreg, srcreg,
17744 desired_align, align_bytes);
17745 count_exp = plus_constant (count_exp, -align_bytes);
17746 count -= align_bytes;
17748 if (need_zero_guard
17749 && (count < (unsigned HOST_WIDE_INT) size_needed
17750 || (align_bytes == 0
17751 && count < ((unsigned HOST_WIDE_INT) size_needed
17752 + desired_align - align))))
17754 /* It is possible that we copied enough so the main loop will not
17756 gcc_assert (size_needed > 1);
17757 if (label == NULL_RTX)
17758 label = gen_label_rtx ();
17759 emit_cmp_and_jump_insns (count_exp,
17760 GEN_INT (size_needed),
17761 LTU, 0, counter_mode (count_exp), 1, label);
17762 if (expected_size == -1
17763 || expected_size < (desired_align - align) / 2 + size_needed)
17764 predict_jump (REG_BR_PROB_BASE * 20 / 100);
17766 predict_jump (REG_BR_PROB_BASE * 60 / 100);
17769 if (label && size_needed == 1)
17771 emit_label (label);
17772 LABEL_NUSES (label) = 1;
17774 epilogue_size_needed = 1;
17776 else if (label == NULL_RTX)
17777 epilogue_size_needed = size_needed;
17779 /* Step 3: Main loop. */
17785 gcc_unreachable ();
17787 expand_set_or_movmem_via_loop (dst, src, destreg, srcreg, NULL,
17788 count_exp, QImode, 1, expected_size);
17791 expand_set_or_movmem_via_loop (dst, src, destreg, srcreg, NULL,
17792 count_exp, Pmode, 1, expected_size);
17794 case unrolled_loop:
17795 /* Unroll only by factor of 2 in 32bit mode, since we don't have enough
17796 registers for 4 temporaries anyway. */
17797 expand_set_or_movmem_via_loop (dst, src, destreg, srcreg, NULL,
17798 count_exp, Pmode, TARGET_64BIT ? 4 : 2,
17801 case rep_prefix_8_byte:
17802 expand_movmem_via_rep_mov (dst, src, destreg, srcreg, count_exp,
17805 case rep_prefix_4_byte:
17806 expand_movmem_via_rep_mov (dst, src, destreg, srcreg, count_exp,
17809 case rep_prefix_1_byte:
17810 expand_movmem_via_rep_mov (dst, src, destreg, srcreg, count_exp,
17814 /* Adjust properly the offset of src and dest memory for aliasing. */
17815 if (CONST_INT_P (count_exp))
17817 src = adjust_automodify_address_nv (src, BLKmode, srcreg,
17818 (count / size_needed) * size_needed);
17819 dst = adjust_automodify_address_nv (dst, BLKmode, destreg,
17820 (count / size_needed) * size_needed);
17824 src = change_address (src, BLKmode, srcreg);
17825 dst = change_address (dst, BLKmode, destreg);
17828 /* Step 4: Epilogue to copy the remaining bytes. */
17832 /* When the main loop is done, COUNT_EXP might hold original count,
17833 while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
17834 Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
17835 bytes. Compensate if needed. */
17837 if (size_needed < epilogue_size_needed)
17840 expand_simple_binop (counter_mode (count_exp), AND, count_exp,
17841 GEN_INT (size_needed - 1), count_exp, 1,
17843 if (tmp != count_exp)
17844 emit_move_insn (count_exp, tmp);
17846 emit_label (label);
17847 LABEL_NUSES (label) = 1;
17850 if (count_exp != const0_rtx && epilogue_size_needed > 1)
17851 expand_movmem_epilogue (dst, src, destreg, srcreg, count_exp,
17852 epilogue_size_needed);
17853 if (jump_around_label)
17854 emit_label (jump_around_label);
17858 /* Helper function for memcpy. For QImode value 0xXY produce
17859 0xXYXYXYXY of wide specified by MODE. This is essentially
17860 a * 0x10101010, but we can do slightly better than
17861 synth_mult by unwinding the sequence by hand on CPUs with
17864 promote_duplicated_reg (enum machine_mode mode, rtx val)
17866 enum machine_mode valmode = GET_MODE (val);
17868 int nops = mode == DImode ? 3 : 2;
17870 gcc_assert (mode == SImode || mode == DImode);
17871 if (val == const0_rtx)
17872 return copy_to_mode_reg (mode, const0_rtx);
17873 if (CONST_INT_P (val))
17875 HOST_WIDE_INT v = INTVAL (val) & 255;
17879 if (mode == DImode)
17880 v |= (v << 16) << 16;
17881 return copy_to_mode_reg (mode, gen_int_mode (v, mode));
17884 if (valmode == VOIDmode)
17886 if (valmode != QImode)
17887 val = gen_lowpart (QImode, val);
17888 if (mode == QImode)
17890 if (!TARGET_PARTIAL_REG_STALL)
17892 if (ix86_cost->mult_init[mode == DImode ? 3 : 2]
17893 + ix86_cost->mult_bit * (mode == DImode ? 8 : 4)
17894 <= (ix86_cost->shift_const + ix86_cost->add) * nops
17895 + (COSTS_N_INSNS (TARGET_PARTIAL_REG_STALL == 0)))
17897 rtx reg = convert_modes (mode, QImode, val, true);
17898 tmp = promote_duplicated_reg (mode, const1_rtx);
17899 return expand_simple_binop (mode, MULT, reg, tmp, NULL, 1,
17904 rtx reg = convert_modes (mode, QImode, val, true);
17906 if (!TARGET_PARTIAL_REG_STALL)
17907 if (mode == SImode)
17908 emit_insn (gen_movsi_insv_1 (reg, reg));
17910 emit_insn (gen_movdi_insv_1_rex64 (reg, reg));
17913 tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (8),
17914 NULL, 1, OPTAB_DIRECT);
17916 expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
17918 tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (16),
17919 NULL, 1, OPTAB_DIRECT);
17920 reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
17921 if (mode == SImode)
17923 tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (32),
17924 NULL, 1, OPTAB_DIRECT);
17925 reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
17930 /* Duplicate value VAL using promote_duplicated_reg into maximal size that will
17931 be needed by main loop copying SIZE_NEEDED chunks and prologue getting
17932 alignment from ALIGN to DESIRED_ALIGN. */
17934 promote_duplicated_reg_to_size (rtx val, int size_needed, int desired_align, int align)
17939 && (size_needed > 4 || (desired_align > align && desired_align > 4)))
17940 promoted_val = promote_duplicated_reg (DImode, val);
17941 else if (size_needed > 2 || (desired_align > align && desired_align > 2))
17942 promoted_val = promote_duplicated_reg (SImode, val);
17943 else if (size_needed > 1 || (desired_align > align && desired_align > 1))
17944 promoted_val = promote_duplicated_reg (HImode, val);
17946 promoted_val = val;
17948 return promoted_val;
17951 /* Expand string clear operation (bzero). Use i386 string operations when
17952 profitable. See expand_movmem comment for explanation of individual
17953 steps performed. */
17955 ix86_expand_setmem (rtx dst, rtx count_exp, rtx val_exp, rtx align_exp,
17956 rtx expected_align_exp, rtx expected_size_exp)
17961 rtx jump_around_label = NULL;
17962 HOST_WIDE_INT align = 1;
17963 unsigned HOST_WIDE_INT count = 0;
17964 HOST_WIDE_INT expected_size = -1;
17965 int size_needed = 0, epilogue_size_needed;
17966 int desired_align = 0, align_bytes = 0;
17967 enum stringop_alg alg;
17968 rtx promoted_val = NULL;
17969 bool force_loopy_epilogue = false;
17971 bool need_zero_guard = false;
17973 if (CONST_INT_P (align_exp))
17974 align = INTVAL (align_exp);
17975 /* i386 can do misaligned access on reasonably increased cost. */
17976 if (CONST_INT_P (expected_align_exp)
17977 && INTVAL (expected_align_exp) > align)
17978 align = INTVAL (expected_align_exp);
17979 if (CONST_INT_P (count_exp))
17980 count = expected_size = INTVAL (count_exp);
17981 if (CONST_INT_P (expected_size_exp) && count == 0)
17982 expected_size = INTVAL (expected_size_exp);
17984 /* Make sure we don't need to care about overflow later on. */
17985 if (count > ((unsigned HOST_WIDE_INT) 1 << 30))
17988 /* Step 0: Decide on preferred algorithm, desired alignment and
17989 size of chunks to be copied by main loop. */
17991 alg = decide_alg (count, expected_size, true, &dynamic_check);
17992 desired_align = decide_alignment (align, alg, expected_size);
17994 if (!TARGET_ALIGN_STRINGOPS)
17995 align = desired_align;
17997 if (alg == libcall)
17999 gcc_assert (alg != no_stringop);
18001 count_exp = copy_to_mode_reg (counter_mode (count_exp), count_exp);
18002 destreg = copy_to_mode_reg (Pmode, XEXP (dst, 0));
18007 gcc_unreachable ();
18009 need_zero_guard = true;
18010 size_needed = GET_MODE_SIZE (Pmode);
18012 case unrolled_loop:
18013 need_zero_guard = true;
18014 size_needed = GET_MODE_SIZE (Pmode) * 4;
18016 case rep_prefix_8_byte:
18019 case rep_prefix_4_byte:
18022 case rep_prefix_1_byte:
18026 need_zero_guard = true;
18030 epilogue_size_needed = size_needed;
18032 /* Step 1: Prologue guard. */
18034 /* Alignment code needs count to be in register. */
18035 if (CONST_INT_P (count_exp) && desired_align > align)
18037 if (INTVAL (count_exp) > desired_align
18038 && INTVAL (count_exp) > size_needed)
18041 = get_mem_align_offset (dst, desired_align * BITS_PER_UNIT);
18042 if (align_bytes <= 0)
18045 align_bytes = desired_align - align_bytes;
18047 if (align_bytes == 0)
18049 enum machine_mode mode = SImode;
18050 if (TARGET_64BIT && (count & ~0xffffffff))
18052 count_exp = force_reg (mode, count_exp);
18055 /* Do the cheap promotion to allow better CSE across the
18056 main loop and epilogue (ie one load of the big constant in the
18057 front of all code. */
18058 if (CONST_INT_P (val_exp))
18059 promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
18060 desired_align, align);
18061 /* Ensure that alignment prologue won't copy past end of block. */
18062 if (size_needed > 1 || (desired_align > 1 && desired_align > align))
18064 epilogue_size_needed = MAX (size_needed - 1, desired_align - align);
18065 /* Epilogue always copies COUNT_EXP & (EPILOGUE_SIZE_NEEDED - 1) bytes.
18066 Make sure it is power of 2. */
18067 epilogue_size_needed = smallest_pow2_greater_than (epilogue_size_needed);
18069 /* To improve performance of small blocks, we jump around the VAL
18070 promoting mode. This mean that if the promoted VAL is not constant,
18071 we might not use it in the epilogue and have to use byte
18073 if (epilogue_size_needed > 2 && !promoted_val)
18074 force_loopy_epilogue = true;
18077 if (count < (unsigned HOST_WIDE_INT)epilogue_size_needed)
18079 /* If main algorithm works on QImode, no epilogue is needed.
18080 For small sizes just don't align anything. */
18081 if (size_needed == 1)
18082 desired_align = align;
18089 label = gen_label_rtx ();
18090 emit_cmp_and_jump_insns (count_exp,
18091 GEN_INT (epilogue_size_needed),
18092 LTU, 0, counter_mode (count_exp), 1, label);
18093 if (expected_size == -1 || expected_size <= epilogue_size_needed)
18094 predict_jump (REG_BR_PROB_BASE * 60 / 100);
18096 predict_jump (REG_BR_PROB_BASE * 20 / 100);
18099 if (dynamic_check != -1)
18101 rtx hot_label = gen_label_rtx ();
18102 jump_around_label = gen_label_rtx ();
18103 emit_cmp_and_jump_insns (count_exp, GEN_INT (dynamic_check - 1),
18104 LEU, 0, counter_mode (count_exp), 1, hot_label);
18105 predict_jump (REG_BR_PROB_BASE * 90 / 100);
18106 set_storage_via_libcall (dst, count_exp, val_exp, false);
18107 emit_jump (jump_around_label);
18108 emit_label (hot_label);
18111 /* Step 2: Alignment prologue. */
18113 /* Do the expensive promotion once we branched off the small blocks. */
18115 promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
18116 desired_align, align);
18117 gcc_assert (desired_align >= 1 && align >= 1);
18119 if (desired_align > align)
18121 if (align_bytes == 0)
18123 /* Except for the first move in epilogue, we no longer know
18124 constant offset in aliasing info. It don't seems to worth
18125 the pain to maintain it for the first move, so throw away
18127 dst = change_address (dst, BLKmode, destreg);
18128 expand_setmem_prologue (dst, destreg, promoted_val, count_exp, align,
18133 /* If we know how many bytes need to be stored before dst is
18134 sufficiently aligned, maintain aliasing info accurately. */
18135 dst = expand_constant_setmem_prologue (dst, destreg, promoted_val,
18136 desired_align, align_bytes);
18137 count_exp = plus_constant (count_exp, -align_bytes);
18138 count -= align_bytes;
18140 if (need_zero_guard
18141 && (count < (unsigned HOST_WIDE_INT) size_needed
18142 || (align_bytes == 0
18143 && count < ((unsigned HOST_WIDE_INT) size_needed
18144 + desired_align - align))))
18146 /* It is possible that we copied enough so the main loop will not
18148 gcc_assert (size_needed > 1);
18149 if (label == NULL_RTX)
18150 label = gen_label_rtx ();
18151 emit_cmp_and_jump_insns (count_exp,
18152 GEN_INT (size_needed),
18153 LTU, 0, counter_mode (count_exp), 1, label);
18154 if (expected_size == -1
18155 || expected_size < (desired_align - align) / 2 + size_needed)
18156 predict_jump (REG_BR_PROB_BASE * 20 / 100);
18158 predict_jump (REG_BR_PROB_BASE * 60 / 100);
18161 if (label && size_needed == 1)
18163 emit_label (label);
18164 LABEL_NUSES (label) = 1;
18166 promoted_val = val_exp;
18167 epilogue_size_needed = 1;
18169 else if (label == NULL_RTX)
18170 epilogue_size_needed = size_needed;
18172 /* Step 3: Main loop. */
18178 gcc_unreachable ();
18180 expand_set_or_movmem_via_loop (dst, NULL, destreg, NULL, promoted_val,
18181 count_exp, QImode, 1, expected_size);
18184 expand_set_or_movmem_via_loop (dst, NULL, destreg, NULL, promoted_val,
18185 count_exp, Pmode, 1, expected_size);
18187 case unrolled_loop:
18188 expand_set_or_movmem_via_loop (dst, NULL, destreg, NULL, promoted_val,
18189 count_exp, Pmode, 4, expected_size);
18191 case rep_prefix_8_byte:
18192 expand_setmem_via_rep_stos (dst, destreg, promoted_val, count_exp,
18195 case rep_prefix_4_byte:
18196 expand_setmem_via_rep_stos (dst, destreg, promoted_val, count_exp,
18199 case rep_prefix_1_byte:
18200 expand_setmem_via_rep_stos (dst, destreg, promoted_val, count_exp,
18204 /* Adjust properly the offset of src and dest memory for aliasing. */
18205 if (CONST_INT_P (count_exp))
18206 dst = adjust_automodify_address_nv (dst, BLKmode, destreg,
18207 (count / size_needed) * size_needed);
18209 dst = change_address (dst, BLKmode, destreg);
18211 /* Step 4: Epilogue to copy the remaining bytes. */
18215 /* When the main loop is done, COUNT_EXP might hold original count,
18216 while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
18217 Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
18218 bytes. Compensate if needed. */
18220 if (size_needed < epilogue_size_needed)
18223 expand_simple_binop (counter_mode (count_exp), AND, count_exp,
18224 GEN_INT (size_needed - 1), count_exp, 1,
18226 if (tmp != count_exp)
18227 emit_move_insn (count_exp, tmp);
18229 emit_label (label);
18230 LABEL_NUSES (label) = 1;
18233 if (count_exp != const0_rtx && epilogue_size_needed > 1)
18235 if (force_loopy_epilogue)
18236 expand_setmem_epilogue_via_loop (dst, destreg, val_exp, count_exp,
18237 epilogue_size_needed);
18239 expand_setmem_epilogue (dst, destreg, promoted_val, count_exp,
18240 epilogue_size_needed);
18242 if (jump_around_label)
18243 emit_label (jump_around_label);
18247 /* Expand the appropriate insns for doing strlen if not just doing
18250 out = result, initialized with the start address
18251 align_rtx = alignment of the address.
18252 scratch = scratch register, initialized with the startaddress when
18253 not aligned, otherwise undefined
18255 This is just the body. It needs the initializations mentioned above and
18256 some address computing at the end. These things are done in i386.md. */
18259 ix86_expand_strlensi_unroll_1 (rtx out, rtx src, rtx align_rtx)
18263 rtx align_2_label = NULL_RTX;
18264 rtx align_3_label = NULL_RTX;
18265 rtx align_4_label = gen_label_rtx ();
18266 rtx end_0_label = gen_label_rtx ();
18268 rtx tmpreg = gen_reg_rtx (SImode);
18269 rtx scratch = gen_reg_rtx (SImode);
18273 if (CONST_INT_P (align_rtx))
18274 align = INTVAL (align_rtx);
18276 /* Loop to check 1..3 bytes for null to get an aligned pointer. */
18278 /* Is there a known alignment and is it less than 4? */
18281 rtx scratch1 = gen_reg_rtx (Pmode);
18282 emit_move_insn (scratch1, out);
18283 /* Is there a known alignment and is it not 2? */
18286 align_3_label = gen_label_rtx (); /* Label when aligned to 3-byte */
18287 align_2_label = gen_label_rtx (); /* Label when aligned to 2-byte */
18289 /* Leave just the 3 lower bits. */
18290 align_rtx = expand_binop (Pmode, and_optab, scratch1, GEN_INT (3),
18291 NULL_RTX, 0, OPTAB_WIDEN);
18293 emit_cmp_and_jump_insns (align_rtx, const0_rtx, EQ, NULL,
18294 Pmode, 1, align_4_label);
18295 emit_cmp_and_jump_insns (align_rtx, const2_rtx, EQ, NULL,
18296 Pmode, 1, align_2_label);
18297 emit_cmp_and_jump_insns (align_rtx, const2_rtx, GTU, NULL,
18298 Pmode, 1, align_3_label);
18302 /* Since the alignment is 2, we have to check 2 or 0 bytes;
18303 check if is aligned to 4 - byte. */
18305 align_rtx = expand_binop (Pmode, and_optab, scratch1, const2_rtx,
18306 NULL_RTX, 0, OPTAB_WIDEN);
18308 emit_cmp_and_jump_insns (align_rtx, const0_rtx, EQ, NULL,
18309 Pmode, 1, align_4_label);
18312 mem = change_address (src, QImode, out);
18314 /* Now compare the bytes. */
18316 /* Compare the first n unaligned byte on a byte per byte basis. */
18317 emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL,
18318 QImode, 1, end_0_label);
18320 /* Increment the address. */
18321 emit_insn ((*ix86_gen_add3) (out, out, const1_rtx));
18323 /* Not needed with an alignment of 2 */
18326 emit_label (align_2_label);
18328 emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1,
18331 emit_insn ((*ix86_gen_add3) (out, out, const1_rtx));
18333 emit_label (align_3_label);
18336 emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1,
18339 emit_insn ((*ix86_gen_add3) (out, out, const1_rtx));
18342 /* Generate loop to check 4 bytes at a time. It is not a good idea to
18343 align this loop. It gives only huge programs, but does not help to
18345 emit_label (align_4_label);
18347 mem = change_address (src, SImode, out);
18348 emit_move_insn (scratch, mem);
18349 emit_insn ((*ix86_gen_add3) (out, out, GEN_INT (4)));
18351 /* This formula yields a nonzero result iff one of the bytes is zero.
18352 This saves three branches inside loop and many cycles. */
18354 emit_insn (gen_addsi3 (tmpreg, scratch, GEN_INT (-0x01010101)));
18355 emit_insn (gen_one_cmplsi2 (scratch, scratch));
18356 emit_insn (gen_andsi3 (tmpreg, tmpreg, scratch));
18357 emit_insn (gen_andsi3 (tmpreg, tmpreg,
18358 gen_int_mode (0x80808080, SImode)));
18359 emit_cmp_and_jump_insns (tmpreg, const0_rtx, EQ, 0, SImode, 1,
18364 rtx reg = gen_reg_rtx (SImode);
18365 rtx reg2 = gen_reg_rtx (Pmode);
18366 emit_move_insn (reg, tmpreg);
18367 emit_insn (gen_lshrsi3 (reg, reg, GEN_INT (16)));
18369 /* If zero is not in the first two bytes, move two bytes forward. */
18370 emit_insn (gen_testsi_ccno_1 (tmpreg, GEN_INT (0x8080)));
18371 tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
18372 tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
18373 emit_insn (gen_rtx_SET (VOIDmode, tmpreg,
18374 gen_rtx_IF_THEN_ELSE (SImode, tmp,
18377 /* Emit lea manually to avoid clobbering of flags. */
18378 emit_insn (gen_rtx_SET (SImode, reg2,
18379 gen_rtx_PLUS (Pmode, out, const2_rtx)));
18381 tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
18382 tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
18383 emit_insn (gen_rtx_SET (VOIDmode, out,
18384 gen_rtx_IF_THEN_ELSE (Pmode, tmp,
18391 rtx end_2_label = gen_label_rtx ();
18392 /* Is zero in the first two bytes? */
18394 emit_insn (gen_testsi_ccno_1 (tmpreg, GEN_INT (0x8080)));
18395 tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
18396 tmp = gen_rtx_NE (VOIDmode, tmp, const0_rtx);
18397 tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
18398 gen_rtx_LABEL_REF (VOIDmode, end_2_label),
18400 tmp = emit_jump_insn (gen_rtx_SET (VOIDmode, pc_rtx, tmp));
18401 JUMP_LABEL (tmp) = end_2_label;
18403 /* Not in the first two. Move two bytes forward. */
18404 emit_insn (gen_lshrsi3 (tmpreg, tmpreg, GEN_INT (16)));
18405 emit_insn ((*ix86_gen_add3) (out, out, const2_rtx));
18407 emit_label (end_2_label);
18411 /* Avoid branch in fixing the byte. */
18412 tmpreg = gen_lowpart (QImode, tmpreg);
18413 emit_insn (gen_addqi3_cc (tmpreg, tmpreg, tmpreg));
18414 cmp = gen_rtx_LTU (Pmode, gen_rtx_REG (CCmode, FLAGS_REG), const0_rtx);
18415 emit_insn ((*ix86_gen_sub3_carry) (out, out, GEN_INT (3), cmp));
18417 emit_label (end_0_label);
18420 /* Expand strlen. */
18423 ix86_expand_strlen (rtx out, rtx src, rtx eoschar, rtx align)
18425 rtx addr, scratch1, scratch2, scratch3, scratch4;
18427 /* The generic case of strlen expander is long. Avoid it's
18428 expanding unless TARGET_INLINE_ALL_STRINGOPS. */
18430 if (TARGET_UNROLL_STRLEN && eoschar == const0_rtx && optimize > 1
18431 && !TARGET_INLINE_ALL_STRINGOPS
18432 && !optimize_insn_for_size_p ()
18433 && (!CONST_INT_P (align) || INTVAL (align) < 4))
18436 addr = force_reg (Pmode, XEXP (src, 0));
18437 scratch1 = gen_reg_rtx (Pmode);
18439 if (TARGET_UNROLL_STRLEN && eoschar == const0_rtx && optimize > 1
18440 && !optimize_insn_for_size_p ())
18442 /* Well it seems that some optimizer does not combine a call like
18443 foo(strlen(bar), strlen(bar));
18444 when the move and the subtraction is done here. It does calculate
18445 the length just once when these instructions are done inside of
18446 output_strlen_unroll(). But I think since &bar[strlen(bar)] is
18447 often used and I use one fewer register for the lifetime of
18448 output_strlen_unroll() this is better. */
18450 emit_move_insn (out, addr);
18452 ix86_expand_strlensi_unroll_1 (out, src, align);
18454 /* strlensi_unroll_1 returns the address of the zero at the end of
18455 the string, like memchr(), so compute the length by subtracting
18456 the start address. */
18457 emit_insn ((*ix86_gen_sub3) (out, out, addr));
18463 /* Can't use this if the user has appropriated eax, ecx, or edi. */
18464 if (fixed_regs[AX_REG] || fixed_regs[CX_REG] || fixed_regs[DI_REG])
18467 scratch2 = gen_reg_rtx (Pmode);
18468 scratch3 = gen_reg_rtx (Pmode);
18469 scratch4 = force_reg (Pmode, constm1_rtx);
18471 emit_move_insn (scratch3, addr);
18472 eoschar = force_reg (QImode, eoschar);
18474 src = replace_equiv_address_nv (src, scratch3);
18476 /* If .md starts supporting :P, this can be done in .md. */
18477 unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (4, src, eoschar, align,
18478 scratch4), UNSPEC_SCAS);
18479 emit_insn (gen_strlenqi_1 (scratch1, scratch3, unspec));
18480 emit_insn ((*ix86_gen_one_cmpl2) (scratch2, scratch1));
18481 emit_insn ((*ix86_gen_add3) (out, scratch2, constm1_rtx));
18486 /* For given symbol (function) construct code to compute address of it's PLT
18487 entry in large x86-64 PIC model. */
18489 construct_plt_address (rtx symbol)
18491 rtx tmp = gen_reg_rtx (Pmode);
18492 rtx unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, symbol), UNSPEC_PLTOFF);
18494 gcc_assert (GET_CODE (symbol) == SYMBOL_REF);
18495 gcc_assert (ix86_cmodel == CM_LARGE_PIC);
18497 emit_move_insn (tmp, gen_rtx_CONST (Pmode, unspec));
18498 emit_insn (gen_adddi3 (tmp, tmp, pic_offset_table_rtx));
18503 ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
18505 rtx pop, int sibcall)
18507 rtx use = NULL, call;
18508 enum calling_abi function_call_abi;
18510 if (callarg2 && INTVAL (callarg2) == -2)
18511 function_call_abi = MS_ABI;
18513 function_call_abi = SYSV_ABI;
18514 if (pop == const0_rtx)
18516 gcc_assert (!TARGET_64BIT || !pop);
18518 if (TARGET_MACHO && !TARGET_64BIT)
18521 if (flag_pic && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF)
18522 fnaddr = machopic_indirect_call_target (fnaddr);
18527 /* Static functions and indirect calls don't need the pic register. */
18528 if (flag_pic && (!TARGET_64BIT || ix86_cmodel == CM_LARGE_PIC)
18529 && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF
18530 && ! SYMBOL_REF_LOCAL_P (XEXP (fnaddr, 0)))
18531 use_reg (&use, pic_offset_table_rtx);
18534 if (TARGET_64BIT && INTVAL (callarg2) >= 0)
18536 rtx al = gen_rtx_REG (QImode, AX_REG);
18537 emit_move_insn (al, callarg2);
18538 use_reg (&use, al);
18541 if (ix86_cmodel == CM_LARGE_PIC
18542 && GET_CODE (fnaddr) == MEM
18543 && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF
18544 && !local_symbolic_operand (XEXP (fnaddr, 0), VOIDmode))
18545 fnaddr = gen_rtx_MEM (QImode, construct_plt_address (XEXP (fnaddr, 0)));
18546 else if (! call_insn_operand (XEXP (fnaddr, 0), Pmode))
18548 fnaddr = copy_to_mode_reg (Pmode, XEXP (fnaddr, 0));
18549 fnaddr = gen_rtx_MEM (QImode, fnaddr);
18551 if (sibcall && TARGET_64BIT
18552 && !constant_call_address_operand (XEXP (fnaddr, 0), Pmode))
18555 addr = copy_to_mode_reg (Pmode, XEXP (fnaddr, 0));
18556 fnaddr = gen_rtx_REG (Pmode, R11_REG);
18557 emit_move_insn (fnaddr, addr);
18558 fnaddr = gen_rtx_MEM (QImode, fnaddr);
18561 call = gen_rtx_CALL (VOIDmode, fnaddr, callarg1);
18563 call = gen_rtx_SET (VOIDmode, retval, call);
18566 pop = gen_rtx_PLUS (Pmode, stack_pointer_rtx, pop);
18567 pop = gen_rtx_SET (VOIDmode, stack_pointer_rtx, pop);
18568 call = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, call, pop));
18569 gcc_assert (ix86_cfun_abi () != MS_ABI || function_call_abi != SYSV_ABI);
18571 /* We need to represent that SI and DI registers are clobbered
18573 if (ix86_cfun_abi () == MS_ABI && function_call_abi == SYSV_ABI)
18575 static int clobbered_registers[] = {27, 28, 45, 46, 47, 48, 49, 50, 51,
18576 52, SI_REG, DI_REG};
18578 rtx vec[ARRAY_SIZE (clobbered_registers) + 2];
18579 rtx unspec = gen_rtx_UNSPEC (VOIDmode, gen_rtvec (1, const0_rtx),
18580 UNSPEC_MS_TO_SYSV_CALL);
18584 for (i = 0; i < ARRAY_SIZE (clobbered_registers); i++)
18585 vec[i + 2] = gen_rtx_CLOBBER (SSE_REGNO_P (clobbered_registers[i])
18588 (SSE_REGNO_P (clobbered_registers[i])
18590 clobbered_registers[i]));
18592 call = gen_rtx_PARALLEL (VOIDmode,
18593 gen_rtvec_v (ARRAY_SIZE (clobbered_registers)
18597 call = emit_call_insn (call);
18599 CALL_INSN_FUNCTION_USAGE (call) = use;
18603 /* Clear stack slot assignments remembered from previous functions.
18604 This is called from INIT_EXPANDERS once before RTL is emitted for each
18607 static struct machine_function *
18608 ix86_init_machine_status (void)
18610 struct machine_function *f;
18612 f = GGC_CNEW (struct machine_function);
18613 f->use_fast_prologue_epilogue_nregs = -1;
18614 f->tls_descriptor_call_expanded_p = 0;
18615 f->call_abi = DEFAULT_ABI;
18620 /* Return a MEM corresponding to a stack slot with mode MODE.
18621 Allocate a new slot if necessary.
18623 The RTL for a function can have several slots available: N is
18624 which slot to use. */
18627 assign_386_stack_local (enum machine_mode mode, enum ix86_stack_slot n)
18629 struct stack_local_entry *s;
18631 gcc_assert (n < MAX_386_STACK_LOCALS);
18633 /* Virtual slot is valid only before vregs are instantiated. */
18634 gcc_assert ((n == SLOT_VIRTUAL) == !virtuals_instantiated);
18636 for (s = ix86_stack_locals; s; s = s->next)
18637 if (s->mode == mode && s->n == n)
18638 return copy_rtx (s->rtl);
18640 s = (struct stack_local_entry *)
18641 ggc_alloc (sizeof (struct stack_local_entry));
18644 s->rtl = assign_stack_local (mode, GET_MODE_SIZE (mode), 0);
18646 s->next = ix86_stack_locals;
18647 ix86_stack_locals = s;
18651 /* Construct the SYMBOL_REF for the tls_get_addr function. */
18653 static GTY(()) rtx ix86_tls_symbol;
18655 ix86_tls_get_addr (void)
18658 if (!ix86_tls_symbol)
18660 ix86_tls_symbol = gen_rtx_SYMBOL_REF (Pmode,
18661 (TARGET_ANY_GNU_TLS
18663 ? "___tls_get_addr"
18664 : "__tls_get_addr");
18667 return ix86_tls_symbol;
18670 /* Construct the SYMBOL_REF for the _TLS_MODULE_BASE_ symbol. */
18672 static GTY(()) rtx ix86_tls_module_base_symbol;
18674 ix86_tls_module_base (void)
18677 if (!ix86_tls_module_base_symbol)
18679 ix86_tls_module_base_symbol = gen_rtx_SYMBOL_REF (Pmode,
18680 "_TLS_MODULE_BASE_");
18681 SYMBOL_REF_FLAGS (ix86_tls_module_base_symbol)
18682 |= TLS_MODEL_GLOBAL_DYNAMIC << SYMBOL_FLAG_TLS_SHIFT;
18685 return ix86_tls_module_base_symbol;
18688 /* Calculate the length of the memory address in the instruction
18689 encoding. Does not include the one-byte modrm, opcode, or prefix. */
18692 memory_address_length (rtx addr)
18694 struct ix86_address parts;
18695 rtx base, index, disp;
18699 if (GET_CODE (addr) == PRE_DEC
18700 || GET_CODE (addr) == POST_INC
18701 || GET_CODE (addr) == PRE_MODIFY
18702 || GET_CODE (addr) == POST_MODIFY)
18705 ok = ix86_decompose_address (addr, &parts);
18708 if (parts.base && GET_CODE (parts.base) == SUBREG)
18709 parts.base = SUBREG_REG (parts.base);
18710 if (parts.index && GET_CODE (parts.index) == SUBREG)
18711 parts.index = SUBREG_REG (parts.index);
18714 index = parts.index;
18719 - esp as the base always wants an index,
18720 - ebp as the base always wants a displacement. */
18722 /* Register Indirect. */
18723 if (base && !index && !disp)
18725 /* esp (for its index) and ebp (for its displacement) need
18726 the two-byte modrm form. */
18727 if (addr == stack_pointer_rtx
18728 || addr == arg_pointer_rtx
18729 || addr == frame_pointer_rtx
18730 || addr == hard_frame_pointer_rtx)
18734 /* Direct Addressing. */
18735 else if (disp && !base && !index)
18740 /* Find the length of the displacement constant. */
18743 if (base && satisfies_constraint_K (disp))
18748 /* ebp always wants a displacement. */
18749 else if (base == hard_frame_pointer_rtx)
18752 /* An index requires the two-byte modrm form.... */
18754 /* ...like esp, which always wants an index. */
18755 || base == stack_pointer_rtx
18756 || base == arg_pointer_rtx
18757 || base == frame_pointer_rtx)
18764 /* Compute default value for "length_immediate" attribute. When SHORTFORM
18765 is set, expect that insn have 8bit immediate alternative. */
18767 ix86_attr_length_immediate_default (rtx insn, int shortform)
18771 extract_insn_cached (insn);
18772 for (i = recog_data.n_operands - 1; i >= 0; --i)
18773 if (CONSTANT_P (recog_data.operand[i]))
18776 if (shortform && satisfies_constraint_K (recog_data.operand[i]))
18780 switch (get_attr_mode (insn))
18791 /* Immediates for DImode instructions are encoded as 32bit sign extended values. */
18796 fatal_insn ("unknown insn mode", insn);
18802 /* Compute default value for "length_address" attribute. */
18804 ix86_attr_length_address_default (rtx insn)
18808 if (get_attr_type (insn) == TYPE_LEA)
18810 rtx set = PATTERN (insn);
18812 if (GET_CODE (set) == PARALLEL)
18813 set = XVECEXP (set, 0, 0);
18815 gcc_assert (GET_CODE (set) == SET);
18817 return memory_address_length (SET_SRC (set));
18820 extract_insn_cached (insn);
18821 for (i = recog_data.n_operands - 1; i >= 0; --i)
18822 if (MEM_P (recog_data.operand[i]))
18824 return memory_address_length (XEXP (recog_data.operand[i], 0));
18830 /* Compute default value for "length_vex" attribute. It includes
18831 2 or 3 byte VEX prefix and 1 opcode byte. */
18834 ix86_attr_length_vex_default (rtx insn, int has_0f_opcode,
18839 /* Only 0f opcode can use 2 byte VEX prefix and VEX W bit uses 3
18840 byte VEX prefix. */
18841 if (!has_0f_opcode || has_vex_w)
18844 /* We can always use 2 byte VEX prefix in 32bit. */
18848 extract_insn_cached (insn);
18850 for (i = recog_data.n_operands - 1; i >= 0; --i)
18851 if (REG_P (recog_data.operand[i]))
18853 /* REX.W bit uses 3 byte VEX prefix. */
18854 if (GET_MODE (recog_data.operand[i]) == DImode)
18859 /* REX.X or REX.B bits use 3 byte VEX prefix. */
18860 if (MEM_P (recog_data.operand[i])
18861 && x86_extended_reg_mentioned_p (recog_data.operand[i]))
18868 /* Return the maximum number of instructions a cpu can issue. */
18871 ix86_issue_rate (void)
18875 case PROCESSOR_PENTIUM:
18879 case PROCESSOR_PENTIUMPRO:
18880 case PROCESSOR_PENTIUM4:
18881 case PROCESSOR_ATHLON:
18883 case PROCESSOR_AMDFAM10:
18884 case PROCESSOR_NOCONA:
18885 case PROCESSOR_GENERIC32:
18886 case PROCESSOR_GENERIC64:
18889 case PROCESSOR_CORE2:
18897 /* A subroutine of ix86_adjust_cost -- return true iff INSN reads flags set
18898 by DEP_INSN and nothing set by DEP_INSN. */
18901 ix86_flags_dependent (rtx insn, rtx dep_insn, enum attr_type insn_type)
18905 /* Simplify the test for uninteresting insns. */
18906 if (insn_type != TYPE_SETCC
18907 && insn_type != TYPE_ICMOV
18908 && insn_type != TYPE_FCMOV
18909 && insn_type != TYPE_IBR)
18912 if ((set = single_set (dep_insn)) != 0)
18914 set = SET_DEST (set);
18917 else if (GET_CODE (PATTERN (dep_insn)) == PARALLEL
18918 && XVECLEN (PATTERN (dep_insn), 0) == 2
18919 && GET_CODE (XVECEXP (PATTERN (dep_insn), 0, 0)) == SET
18920 && GET_CODE (XVECEXP (PATTERN (dep_insn), 0, 1)) == SET)
18922 set = SET_DEST (XVECEXP (PATTERN (dep_insn), 0, 0));
18923 set2 = SET_DEST (XVECEXP (PATTERN (dep_insn), 0, 0));
18928 if (!REG_P (set) || REGNO (set) != FLAGS_REG)
18931 /* This test is true if the dependent insn reads the flags but
18932 not any other potentially set register. */
18933 if (!reg_overlap_mentioned_p (set, PATTERN (insn)))
18936 if (set2 && reg_overlap_mentioned_p (set2, PATTERN (insn)))
18942 /* A subroutine of ix86_adjust_cost -- return true iff INSN has a memory
18943 address with operands set by DEP_INSN. */
18946 ix86_agi_dependent (rtx insn, rtx dep_insn, enum attr_type insn_type)
18950 if (insn_type == TYPE_LEA
18953 addr = PATTERN (insn);
18955 if (GET_CODE (addr) == PARALLEL)
18956 addr = XVECEXP (addr, 0, 0);
18958 gcc_assert (GET_CODE (addr) == SET);
18960 addr = SET_SRC (addr);
18965 extract_insn_cached (insn);
18966 for (i = recog_data.n_operands - 1; i >= 0; --i)
18967 if (MEM_P (recog_data.operand[i]))
18969 addr = XEXP (recog_data.operand[i], 0);
18976 return modified_in_p (addr, dep_insn);
18980 ix86_adjust_cost (rtx insn, rtx link, rtx dep_insn, int cost)
18982 enum attr_type insn_type, dep_insn_type;
18983 enum attr_memory memory;
18985 int dep_insn_code_number;
18987 /* Anti and output dependencies have zero cost on all CPUs. */
18988 if (REG_NOTE_KIND (link) != 0)
18991 dep_insn_code_number = recog_memoized (dep_insn);
18993 /* If we can't recognize the insns, we can't really do anything. */
18994 if (dep_insn_code_number < 0 || recog_memoized (insn) < 0)
18997 insn_type = get_attr_type (insn);
18998 dep_insn_type = get_attr_type (dep_insn);
19002 case PROCESSOR_PENTIUM:
19003 /* Address Generation Interlock adds a cycle of latency. */
19004 if (ix86_agi_dependent (insn, dep_insn, insn_type))
19007 /* ??? Compares pair with jump/setcc. */
19008 if (ix86_flags_dependent (insn, dep_insn, insn_type))
19011 /* Floating point stores require value to be ready one cycle earlier. */
19012 if (insn_type == TYPE_FMOV
19013 && get_attr_memory (insn) == MEMORY_STORE
19014 && !ix86_agi_dependent (insn, dep_insn, insn_type))
19018 case PROCESSOR_PENTIUMPRO:
19019 memory = get_attr_memory (insn);
19021 /* INT->FP conversion is expensive. */
19022 if (get_attr_fp_int_src (dep_insn))
19025 /* There is one cycle extra latency between an FP op and a store. */
19026 if (insn_type == TYPE_FMOV
19027 && (set = single_set (dep_insn)) != NULL_RTX
19028 && (set2 = single_set (insn)) != NULL_RTX
19029 && rtx_equal_p (SET_DEST (set), SET_SRC (set2))
19030 && MEM_P (SET_DEST (set2)))
19033 /* Show ability of reorder buffer to hide latency of load by executing
19034 in parallel with previous instruction in case
19035 previous instruction is not needed to compute the address. */
19036 if ((memory == MEMORY_LOAD || memory == MEMORY_BOTH)
19037 && !ix86_agi_dependent (insn, dep_insn, insn_type))
19039 /* Claim moves to take one cycle, as core can issue one load
19040 at time and the next load can start cycle later. */
19041 if (dep_insn_type == TYPE_IMOV
19042 || dep_insn_type == TYPE_FMOV)
19050 memory = get_attr_memory (insn);
19052 /* The esp dependency is resolved before the instruction is really
19054 if ((insn_type == TYPE_PUSH || insn_type == TYPE_POP)
19055 && (dep_insn_type == TYPE_PUSH || dep_insn_type == TYPE_POP))
19058 /* INT->FP conversion is expensive. */
19059 if (get_attr_fp_int_src (dep_insn))
19062 /* Show ability of reorder buffer to hide latency of load by executing
19063 in parallel with previous instruction in case
19064 previous instruction is not needed to compute the address. */
19065 if ((memory == MEMORY_LOAD || memory == MEMORY_BOTH)
19066 && !ix86_agi_dependent (insn, dep_insn, insn_type))
19068 /* Claim moves to take one cycle, as core can issue one load
19069 at time and the next load can start cycle later. */
19070 if (dep_insn_type == TYPE_IMOV
19071 || dep_insn_type == TYPE_FMOV)
19080 case PROCESSOR_ATHLON:
19082 case PROCESSOR_AMDFAM10:
19083 case PROCESSOR_GENERIC32:
19084 case PROCESSOR_GENERIC64:
19085 memory = get_attr_memory (insn);
19087 /* Show ability of reorder buffer to hide latency of load by executing
19088 in parallel with previous instruction in case
19089 previous instruction is not needed to compute the address. */
19090 if ((memory == MEMORY_LOAD || memory == MEMORY_BOTH)
19091 && !ix86_agi_dependent (insn, dep_insn, insn_type))
19093 enum attr_unit unit = get_attr_unit (insn);
19096 /* Because of the difference between the length of integer and
19097 floating unit pipeline preparation stages, the memory operands
19098 for floating point are cheaper.
19100 ??? For Athlon it the difference is most probably 2. */
19101 if (unit == UNIT_INTEGER || unit == UNIT_UNKNOWN)
19104 loadcost = TARGET_ATHLON ? 2 : 0;
19106 if (cost >= loadcost)
19119 /* How many alternative schedules to try. This should be as wide as the
19120 scheduling freedom in the DFA, but no wider. Making this value too
19121 large results extra work for the scheduler. */
19124 ia32_multipass_dfa_lookahead (void)
19128 case PROCESSOR_PENTIUM:
19131 case PROCESSOR_PENTIUMPRO:
19141 /* Compute the alignment given to a constant that is being placed in memory.
19142 EXP is the constant and ALIGN is the alignment that the object would
19144 The value of this function is used instead of that alignment to align
19148 ix86_constant_alignment (tree exp, int align)
19150 if (TREE_CODE (exp) == REAL_CST || TREE_CODE (exp) == VECTOR_CST
19151 || TREE_CODE (exp) == INTEGER_CST)
19153 if (TYPE_MODE (TREE_TYPE (exp)) == DFmode && align < 64)
19155 else if (ALIGN_MODE_128 (TYPE_MODE (TREE_TYPE (exp))) && align < 128)
19158 else if (!optimize_size && TREE_CODE (exp) == STRING_CST
19159 && TREE_STRING_LENGTH (exp) >= 31 && align < BITS_PER_WORD)
19160 return BITS_PER_WORD;
19165 /* Compute the alignment for a static variable.
19166 TYPE is the data type, and ALIGN is the alignment that
19167 the object would ordinarily have. The value of this function is used
19168 instead of that alignment to align the object. */
19171 ix86_data_alignment (tree type, int align)
19173 int max_align = optimize_size ? BITS_PER_WORD : MIN (256, MAX_OFILE_ALIGNMENT);
19175 if (AGGREGATE_TYPE_P (type)
19176 && TYPE_SIZE (type)
19177 && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST
19178 && (TREE_INT_CST_LOW (TYPE_SIZE (type)) >= (unsigned) max_align
19179 || TREE_INT_CST_HIGH (TYPE_SIZE (type)))
19180 && align < max_align)
19183 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
19184 to 16byte boundary. */
19187 if (AGGREGATE_TYPE_P (type)
19188 && TYPE_SIZE (type)
19189 && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST
19190 && (TREE_INT_CST_LOW (TYPE_SIZE (type)) >= 128
19191 || TREE_INT_CST_HIGH (TYPE_SIZE (type))) && align < 128)
19195 if (TREE_CODE (type) == ARRAY_TYPE)
19197 if (TYPE_MODE (TREE_TYPE (type)) == DFmode && align < 64)
19199 if (ALIGN_MODE_128 (TYPE_MODE (TREE_TYPE (type))) && align < 128)
19202 else if (TREE_CODE (type) == COMPLEX_TYPE)
19205 if (TYPE_MODE (type) == DCmode && align < 64)
19207 if ((TYPE_MODE (type) == XCmode
19208 || TYPE_MODE (type) == TCmode) && align < 128)
19211 else if ((TREE_CODE (type) == RECORD_TYPE
19212 || TREE_CODE (type) == UNION_TYPE
19213 || TREE_CODE (type) == QUAL_UNION_TYPE)
19214 && TYPE_FIELDS (type))
19216 if (DECL_MODE (TYPE_FIELDS (type)) == DFmode && align < 64)
19218 if (ALIGN_MODE_128 (DECL_MODE (TYPE_FIELDS (type))) && align < 128)
19221 else if (TREE_CODE (type) == REAL_TYPE || TREE_CODE (type) == VECTOR_TYPE
19222 || TREE_CODE (type) == INTEGER_TYPE)
19224 if (TYPE_MODE (type) == DFmode && align < 64)
19226 if (ALIGN_MODE_128 (TYPE_MODE (type)) && align < 128)
19233 /* Compute the alignment for a local variable or a stack slot. TYPE is
19234 the data type, MODE is the widest mode available and ALIGN is the
19235 alignment that the object would ordinarily have. The value of this
19236 macro is used instead of that alignment to align the object. */
19239 ix86_local_alignment (tree type, enum machine_mode mode,
19240 unsigned int align)
19242 /* If TYPE is NULL, we are allocating a stack slot for caller-save
19243 register in MODE. We will return the largest alignment of XF
19247 if (mode == XFmode && align < GET_MODE_ALIGNMENT (DFmode))
19248 align = GET_MODE_ALIGNMENT (DFmode);
19252 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
19253 to 16byte boundary. */
19256 if (AGGREGATE_TYPE_P (type)
19257 && TYPE_SIZE (type)
19258 && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST
19259 && (TREE_INT_CST_LOW (TYPE_SIZE (type)) >= 16
19260 || TREE_INT_CST_HIGH (TYPE_SIZE (type))) && align < 128)
19263 if (TREE_CODE (type) == ARRAY_TYPE)
19265 if (TYPE_MODE (TREE_TYPE (type)) == DFmode && align < 64)
19267 if (ALIGN_MODE_128 (TYPE_MODE (TREE_TYPE (type))) && align < 128)
19270 else if (TREE_CODE (type) == COMPLEX_TYPE)
19272 if (TYPE_MODE (type) == DCmode && align < 64)
19274 if ((TYPE_MODE (type) == XCmode
19275 || TYPE_MODE (type) == TCmode) && align < 128)
19278 else if ((TREE_CODE (type) == RECORD_TYPE
19279 || TREE_CODE (type) == UNION_TYPE
19280 || TREE_CODE (type) == QUAL_UNION_TYPE)
19281 && TYPE_FIELDS (type))
19283 if (DECL_MODE (TYPE_FIELDS (type)) == DFmode && align < 64)
19285 if (ALIGN_MODE_128 (DECL_MODE (TYPE_FIELDS (type))) && align < 128)
19288 else if (TREE_CODE (type) == REAL_TYPE || TREE_CODE (type) == VECTOR_TYPE
19289 || TREE_CODE (type) == INTEGER_TYPE)
19292 if (TYPE_MODE (type) == DFmode && align < 64)
19294 if (ALIGN_MODE_128 (TYPE_MODE (type)) && align < 128)
19300 /* Emit RTL insns to initialize the variable parts of a trampoline.
19301 FNADDR is an RTX for the address of the function's pure code.
19302 CXT is an RTX for the static chain value for the function. */
19304 x86_initialize_trampoline (rtx tramp, rtx fnaddr, rtx cxt)
19308 /* Compute offset from the end of the jmp to the target function. */
19309 rtx disp = expand_binop (SImode, sub_optab, fnaddr,
19310 plus_constant (tramp, 10),
19311 NULL_RTX, 1, OPTAB_DIRECT);
19312 emit_move_insn (gen_rtx_MEM (QImode, tramp),
19313 gen_int_mode (0xb9, QImode));
19314 emit_move_insn (gen_rtx_MEM (SImode, plus_constant (tramp, 1)), cxt);
19315 emit_move_insn (gen_rtx_MEM (QImode, plus_constant (tramp, 5)),
19316 gen_int_mode (0xe9, QImode));
19317 emit_move_insn (gen_rtx_MEM (SImode, plus_constant (tramp, 6)), disp);
19322 /* Try to load address using shorter movl instead of movabs.
19323 We may want to support movq for kernel mode, but kernel does not use
19324 trampolines at the moment. */
19325 if (x86_64_zext_immediate_operand (fnaddr, VOIDmode))
19327 fnaddr = copy_to_mode_reg (DImode, fnaddr);
19328 emit_move_insn (gen_rtx_MEM (HImode, plus_constant (tramp, offset)),
19329 gen_int_mode (0xbb41, HImode));
19330 emit_move_insn (gen_rtx_MEM (SImode, plus_constant (tramp, offset + 2)),
19331 gen_lowpart (SImode, fnaddr));
19336 emit_move_insn (gen_rtx_MEM (HImode, plus_constant (tramp, offset)),
19337 gen_int_mode (0xbb49, HImode));
19338 emit_move_insn (gen_rtx_MEM (DImode, plus_constant (tramp, offset + 2)),
19342 /* Load static chain using movabs to r10. */
19343 emit_move_insn (gen_rtx_MEM (HImode, plus_constant (tramp, offset)),
19344 gen_int_mode (0xba49, HImode));
19345 emit_move_insn (gen_rtx_MEM (DImode, plus_constant (tramp, offset + 2)),
19348 /* Jump to the r11 */
19349 emit_move_insn (gen_rtx_MEM (HImode, plus_constant (tramp, offset)),
19350 gen_int_mode (0xff49, HImode));
19351 emit_move_insn (gen_rtx_MEM (QImode, plus_constant (tramp, offset+2)),
19352 gen_int_mode (0xe3, QImode));
19354 gcc_assert (offset <= TRAMPOLINE_SIZE);
19357 #ifdef ENABLE_EXECUTE_STACK
19358 emit_library_call (gen_rtx_SYMBOL_REF (Pmode, "__enable_execute_stack"),
19359 LCT_NORMAL, VOIDmode, 1, tramp, Pmode);
19363 /* Codes for all the SSE/MMX builtins. */
19366 IX86_BUILTIN_ADDPS,
19367 IX86_BUILTIN_ADDSS,
19368 IX86_BUILTIN_DIVPS,
19369 IX86_BUILTIN_DIVSS,
19370 IX86_BUILTIN_MULPS,
19371 IX86_BUILTIN_MULSS,
19372 IX86_BUILTIN_SUBPS,
19373 IX86_BUILTIN_SUBSS,
19375 IX86_BUILTIN_CMPEQPS,
19376 IX86_BUILTIN_CMPLTPS,
19377 IX86_BUILTIN_CMPLEPS,
19378 IX86_BUILTIN_CMPGTPS,
19379 IX86_BUILTIN_CMPGEPS,
19380 IX86_BUILTIN_CMPNEQPS,
19381 IX86_BUILTIN_CMPNLTPS,
19382 IX86_BUILTIN_CMPNLEPS,
19383 IX86_BUILTIN_CMPNGTPS,
19384 IX86_BUILTIN_CMPNGEPS,
19385 IX86_BUILTIN_CMPORDPS,
19386 IX86_BUILTIN_CMPUNORDPS,
19387 IX86_BUILTIN_CMPEQSS,
19388 IX86_BUILTIN_CMPLTSS,
19389 IX86_BUILTIN_CMPLESS,
19390 IX86_BUILTIN_CMPNEQSS,
19391 IX86_BUILTIN_CMPNLTSS,
19392 IX86_BUILTIN_CMPNLESS,
19393 IX86_BUILTIN_CMPNGTSS,
19394 IX86_BUILTIN_CMPNGESS,
19395 IX86_BUILTIN_CMPORDSS,
19396 IX86_BUILTIN_CMPUNORDSS,
19398 IX86_BUILTIN_COMIEQSS,
19399 IX86_BUILTIN_COMILTSS,
19400 IX86_BUILTIN_COMILESS,
19401 IX86_BUILTIN_COMIGTSS,
19402 IX86_BUILTIN_COMIGESS,
19403 IX86_BUILTIN_COMINEQSS,
19404 IX86_BUILTIN_UCOMIEQSS,
19405 IX86_BUILTIN_UCOMILTSS,
19406 IX86_BUILTIN_UCOMILESS,
19407 IX86_BUILTIN_UCOMIGTSS,
19408 IX86_BUILTIN_UCOMIGESS,
19409 IX86_BUILTIN_UCOMINEQSS,
19411 IX86_BUILTIN_CVTPI2PS,
19412 IX86_BUILTIN_CVTPS2PI,
19413 IX86_BUILTIN_CVTSI2SS,
19414 IX86_BUILTIN_CVTSI642SS,
19415 IX86_BUILTIN_CVTSS2SI,
19416 IX86_BUILTIN_CVTSS2SI64,
19417 IX86_BUILTIN_CVTTPS2PI,
19418 IX86_BUILTIN_CVTTSS2SI,
19419 IX86_BUILTIN_CVTTSS2SI64,
19421 IX86_BUILTIN_MAXPS,
19422 IX86_BUILTIN_MAXSS,
19423 IX86_BUILTIN_MINPS,
19424 IX86_BUILTIN_MINSS,
19426 IX86_BUILTIN_LOADUPS,
19427 IX86_BUILTIN_STOREUPS,
19428 IX86_BUILTIN_MOVSS,
19430 IX86_BUILTIN_MOVHLPS,
19431 IX86_BUILTIN_MOVLHPS,
19432 IX86_BUILTIN_LOADHPS,
19433 IX86_BUILTIN_LOADLPS,
19434 IX86_BUILTIN_STOREHPS,
19435 IX86_BUILTIN_STORELPS,
19437 IX86_BUILTIN_MASKMOVQ,
19438 IX86_BUILTIN_MOVMSKPS,
19439 IX86_BUILTIN_PMOVMSKB,
19441 IX86_BUILTIN_MOVNTPS,
19442 IX86_BUILTIN_MOVNTQ,
19444 IX86_BUILTIN_LOADDQU,
19445 IX86_BUILTIN_STOREDQU,
19447 IX86_BUILTIN_PACKSSWB,
19448 IX86_BUILTIN_PACKSSDW,
19449 IX86_BUILTIN_PACKUSWB,
19451 IX86_BUILTIN_PADDB,
19452 IX86_BUILTIN_PADDW,
19453 IX86_BUILTIN_PADDD,
19454 IX86_BUILTIN_PADDQ,
19455 IX86_BUILTIN_PADDSB,
19456 IX86_BUILTIN_PADDSW,
19457 IX86_BUILTIN_PADDUSB,
19458 IX86_BUILTIN_PADDUSW,
19459 IX86_BUILTIN_PSUBB,
19460 IX86_BUILTIN_PSUBW,
19461 IX86_BUILTIN_PSUBD,
19462 IX86_BUILTIN_PSUBQ,
19463 IX86_BUILTIN_PSUBSB,
19464 IX86_BUILTIN_PSUBSW,
19465 IX86_BUILTIN_PSUBUSB,
19466 IX86_BUILTIN_PSUBUSW,
19469 IX86_BUILTIN_PANDN,
19473 IX86_BUILTIN_PAVGB,
19474 IX86_BUILTIN_PAVGW,
19476 IX86_BUILTIN_PCMPEQB,
19477 IX86_BUILTIN_PCMPEQW,
19478 IX86_BUILTIN_PCMPEQD,
19479 IX86_BUILTIN_PCMPGTB,
19480 IX86_BUILTIN_PCMPGTW,
19481 IX86_BUILTIN_PCMPGTD,
19483 IX86_BUILTIN_PMADDWD,
19485 IX86_BUILTIN_PMAXSW,
19486 IX86_BUILTIN_PMAXUB,
19487 IX86_BUILTIN_PMINSW,
19488 IX86_BUILTIN_PMINUB,
19490 IX86_BUILTIN_PMULHUW,
19491 IX86_BUILTIN_PMULHW,
19492 IX86_BUILTIN_PMULLW,
19494 IX86_BUILTIN_PSADBW,
19495 IX86_BUILTIN_PSHUFW,
19497 IX86_BUILTIN_PSLLW,
19498 IX86_BUILTIN_PSLLD,
19499 IX86_BUILTIN_PSLLQ,
19500 IX86_BUILTIN_PSRAW,
19501 IX86_BUILTIN_PSRAD,
19502 IX86_BUILTIN_PSRLW,
19503 IX86_BUILTIN_PSRLD,
19504 IX86_BUILTIN_PSRLQ,
19505 IX86_BUILTIN_PSLLWI,
19506 IX86_BUILTIN_PSLLDI,
19507 IX86_BUILTIN_PSLLQI,
19508 IX86_BUILTIN_PSRAWI,
19509 IX86_BUILTIN_PSRADI,
19510 IX86_BUILTIN_PSRLWI,
19511 IX86_BUILTIN_PSRLDI,
19512 IX86_BUILTIN_PSRLQI,
19514 IX86_BUILTIN_PUNPCKHBW,
19515 IX86_BUILTIN_PUNPCKHWD,
19516 IX86_BUILTIN_PUNPCKHDQ,
19517 IX86_BUILTIN_PUNPCKLBW,
19518 IX86_BUILTIN_PUNPCKLWD,
19519 IX86_BUILTIN_PUNPCKLDQ,
19521 IX86_BUILTIN_SHUFPS,
19523 IX86_BUILTIN_RCPPS,
19524 IX86_BUILTIN_RCPSS,
19525 IX86_BUILTIN_RSQRTPS,
19526 IX86_BUILTIN_RSQRTPS_NR,
19527 IX86_BUILTIN_RSQRTSS,
19528 IX86_BUILTIN_RSQRTF,
19529 IX86_BUILTIN_SQRTPS,
19530 IX86_BUILTIN_SQRTPS_NR,
19531 IX86_BUILTIN_SQRTSS,
19533 IX86_BUILTIN_UNPCKHPS,
19534 IX86_BUILTIN_UNPCKLPS,
19536 IX86_BUILTIN_ANDPS,
19537 IX86_BUILTIN_ANDNPS,
19539 IX86_BUILTIN_XORPS,
19542 IX86_BUILTIN_LDMXCSR,
19543 IX86_BUILTIN_STMXCSR,
19544 IX86_BUILTIN_SFENCE,
19546 /* 3DNow! Original */
19547 IX86_BUILTIN_FEMMS,
19548 IX86_BUILTIN_PAVGUSB,
19549 IX86_BUILTIN_PF2ID,
19550 IX86_BUILTIN_PFACC,
19551 IX86_BUILTIN_PFADD,
19552 IX86_BUILTIN_PFCMPEQ,
19553 IX86_BUILTIN_PFCMPGE,
19554 IX86_BUILTIN_PFCMPGT,
19555 IX86_BUILTIN_PFMAX,
19556 IX86_BUILTIN_PFMIN,
19557 IX86_BUILTIN_PFMUL,
19558 IX86_BUILTIN_PFRCP,
19559 IX86_BUILTIN_PFRCPIT1,
19560 IX86_BUILTIN_PFRCPIT2,
19561 IX86_BUILTIN_PFRSQIT1,
19562 IX86_BUILTIN_PFRSQRT,
19563 IX86_BUILTIN_PFSUB,
19564 IX86_BUILTIN_PFSUBR,
19565 IX86_BUILTIN_PI2FD,
19566 IX86_BUILTIN_PMULHRW,
19568 /* 3DNow! Athlon Extensions */
19569 IX86_BUILTIN_PF2IW,
19570 IX86_BUILTIN_PFNACC,
19571 IX86_BUILTIN_PFPNACC,
19572 IX86_BUILTIN_PI2FW,
19573 IX86_BUILTIN_PSWAPDSI,
19574 IX86_BUILTIN_PSWAPDSF,
19577 IX86_BUILTIN_ADDPD,
19578 IX86_BUILTIN_ADDSD,
19579 IX86_BUILTIN_DIVPD,
19580 IX86_BUILTIN_DIVSD,
19581 IX86_BUILTIN_MULPD,
19582 IX86_BUILTIN_MULSD,
19583 IX86_BUILTIN_SUBPD,
19584 IX86_BUILTIN_SUBSD,
19586 IX86_BUILTIN_CMPEQPD,
19587 IX86_BUILTIN_CMPLTPD,
19588 IX86_BUILTIN_CMPLEPD,
19589 IX86_BUILTIN_CMPGTPD,
19590 IX86_BUILTIN_CMPGEPD,
19591 IX86_BUILTIN_CMPNEQPD,
19592 IX86_BUILTIN_CMPNLTPD,
19593 IX86_BUILTIN_CMPNLEPD,
19594 IX86_BUILTIN_CMPNGTPD,
19595 IX86_BUILTIN_CMPNGEPD,
19596 IX86_BUILTIN_CMPORDPD,
19597 IX86_BUILTIN_CMPUNORDPD,
19598 IX86_BUILTIN_CMPEQSD,
19599 IX86_BUILTIN_CMPLTSD,
19600 IX86_BUILTIN_CMPLESD,
19601 IX86_BUILTIN_CMPNEQSD,
19602 IX86_BUILTIN_CMPNLTSD,
19603 IX86_BUILTIN_CMPNLESD,
19604 IX86_BUILTIN_CMPORDSD,
19605 IX86_BUILTIN_CMPUNORDSD,
19607 IX86_BUILTIN_COMIEQSD,
19608 IX86_BUILTIN_COMILTSD,
19609 IX86_BUILTIN_COMILESD,
19610 IX86_BUILTIN_COMIGTSD,
19611 IX86_BUILTIN_COMIGESD,
19612 IX86_BUILTIN_COMINEQSD,
19613 IX86_BUILTIN_UCOMIEQSD,
19614 IX86_BUILTIN_UCOMILTSD,
19615 IX86_BUILTIN_UCOMILESD,
19616 IX86_BUILTIN_UCOMIGTSD,
19617 IX86_BUILTIN_UCOMIGESD,
19618 IX86_BUILTIN_UCOMINEQSD,
19620 IX86_BUILTIN_MAXPD,
19621 IX86_BUILTIN_MAXSD,
19622 IX86_BUILTIN_MINPD,
19623 IX86_BUILTIN_MINSD,
19625 IX86_BUILTIN_ANDPD,
19626 IX86_BUILTIN_ANDNPD,
19628 IX86_BUILTIN_XORPD,
19630 IX86_BUILTIN_SQRTPD,
19631 IX86_BUILTIN_SQRTSD,
19633 IX86_BUILTIN_UNPCKHPD,
19634 IX86_BUILTIN_UNPCKLPD,
19636 IX86_BUILTIN_SHUFPD,
19638 IX86_BUILTIN_LOADUPD,
19639 IX86_BUILTIN_STOREUPD,
19640 IX86_BUILTIN_MOVSD,
19642 IX86_BUILTIN_LOADHPD,
19643 IX86_BUILTIN_LOADLPD,
19645 IX86_BUILTIN_CVTDQ2PD,
19646 IX86_BUILTIN_CVTDQ2PS,
19648 IX86_BUILTIN_CVTPD2DQ,
19649 IX86_BUILTIN_CVTPD2PI,
19650 IX86_BUILTIN_CVTPD2PS,
19651 IX86_BUILTIN_CVTTPD2DQ,
19652 IX86_BUILTIN_CVTTPD2PI,
19654 IX86_BUILTIN_CVTPI2PD,
19655 IX86_BUILTIN_CVTSI2SD,
19656 IX86_BUILTIN_CVTSI642SD,
19658 IX86_BUILTIN_CVTSD2SI,
19659 IX86_BUILTIN_CVTSD2SI64,
19660 IX86_BUILTIN_CVTSD2SS,
19661 IX86_BUILTIN_CVTSS2SD,
19662 IX86_BUILTIN_CVTTSD2SI,
19663 IX86_BUILTIN_CVTTSD2SI64,
19665 IX86_BUILTIN_CVTPS2DQ,
19666 IX86_BUILTIN_CVTPS2PD,
19667 IX86_BUILTIN_CVTTPS2DQ,
19669 IX86_BUILTIN_MOVNTI,
19670 IX86_BUILTIN_MOVNTPD,
19671 IX86_BUILTIN_MOVNTDQ,
19673 IX86_BUILTIN_MOVQ128,
19676 IX86_BUILTIN_MASKMOVDQU,
19677 IX86_BUILTIN_MOVMSKPD,
19678 IX86_BUILTIN_PMOVMSKB128,
19680 IX86_BUILTIN_PACKSSWB128,
19681 IX86_BUILTIN_PACKSSDW128,
19682 IX86_BUILTIN_PACKUSWB128,
19684 IX86_BUILTIN_PADDB128,
19685 IX86_BUILTIN_PADDW128,
19686 IX86_BUILTIN_PADDD128,
19687 IX86_BUILTIN_PADDQ128,
19688 IX86_BUILTIN_PADDSB128,
19689 IX86_BUILTIN_PADDSW128,
19690 IX86_BUILTIN_PADDUSB128,
19691 IX86_BUILTIN_PADDUSW128,
19692 IX86_BUILTIN_PSUBB128,
19693 IX86_BUILTIN_PSUBW128,
19694 IX86_BUILTIN_PSUBD128,
19695 IX86_BUILTIN_PSUBQ128,
19696 IX86_BUILTIN_PSUBSB128,
19697 IX86_BUILTIN_PSUBSW128,
19698 IX86_BUILTIN_PSUBUSB128,
19699 IX86_BUILTIN_PSUBUSW128,
19701 IX86_BUILTIN_PAND128,
19702 IX86_BUILTIN_PANDN128,
19703 IX86_BUILTIN_POR128,
19704 IX86_BUILTIN_PXOR128,
19706 IX86_BUILTIN_PAVGB128,
19707 IX86_BUILTIN_PAVGW128,
19709 IX86_BUILTIN_PCMPEQB128,
19710 IX86_BUILTIN_PCMPEQW128,
19711 IX86_BUILTIN_PCMPEQD128,
19712 IX86_BUILTIN_PCMPGTB128,
19713 IX86_BUILTIN_PCMPGTW128,
19714 IX86_BUILTIN_PCMPGTD128,
19716 IX86_BUILTIN_PMADDWD128,
19718 IX86_BUILTIN_PMAXSW128,
19719 IX86_BUILTIN_PMAXUB128,
19720 IX86_BUILTIN_PMINSW128,
19721 IX86_BUILTIN_PMINUB128,
19723 IX86_BUILTIN_PMULUDQ,
19724 IX86_BUILTIN_PMULUDQ128,
19725 IX86_BUILTIN_PMULHUW128,
19726 IX86_BUILTIN_PMULHW128,
19727 IX86_BUILTIN_PMULLW128,
19729 IX86_BUILTIN_PSADBW128,
19730 IX86_BUILTIN_PSHUFHW,
19731 IX86_BUILTIN_PSHUFLW,
19732 IX86_BUILTIN_PSHUFD,
19734 IX86_BUILTIN_PSLLDQI128,
19735 IX86_BUILTIN_PSLLWI128,
19736 IX86_BUILTIN_PSLLDI128,
19737 IX86_BUILTIN_PSLLQI128,
19738 IX86_BUILTIN_PSRAWI128,
19739 IX86_BUILTIN_PSRADI128,
19740 IX86_BUILTIN_PSRLDQI128,
19741 IX86_BUILTIN_PSRLWI128,
19742 IX86_BUILTIN_PSRLDI128,
19743 IX86_BUILTIN_PSRLQI128,
19745 IX86_BUILTIN_PSLLDQ128,
19746 IX86_BUILTIN_PSLLW128,
19747 IX86_BUILTIN_PSLLD128,
19748 IX86_BUILTIN_PSLLQ128,
19749 IX86_BUILTIN_PSRAW128,
19750 IX86_BUILTIN_PSRAD128,
19751 IX86_BUILTIN_PSRLW128,
19752 IX86_BUILTIN_PSRLD128,
19753 IX86_BUILTIN_PSRLQ128,
19755 IX86_BUILTIN_PUNPCKHBW128,
19756 IX86_BUILTIN_PUNPCKHWD128,
19757 IX86_BUILTIN_PUNPCKHDQ128,
19758 IX86_BUILTIN_PUNPCKHQDQ128,
19759 IX86_BUILTIN_PUNPCKLBW128,
19760 IX86_BUILTIN_PUNPCKLWD128,
19761 IX86_BUILTIN_PUNPCKLDQ128,
19762 IX86_BUILTIN_PUNPCKLQDQ128,
19764 IX86_BUILTIN_CLFLUSH,
19765 IX86_BUILTIN_MFENCE,
19766 IX86_BUILTIN_LFENCE,
19769 IX86_BUILTIN_ADDSUBPS,
19770 IX86_BUILTIN_HADDPS,
19771 IX86_BUILTIN_HSUBPS,
19772 IX86_BUILTIN_MOVSHDUP,
19773 IX86_BUILTIN_MOVSLDUP,
19774 IX86_BUILTIN_ADDSUBPD,
19775 IX86_BUILTIN_HADDPD,
19776 IX86_BUILTIN_HSUBPD,
19777 IX86_BUILTIN_LDDQU,
19779 IX86_BUILTIN_MONITOR,
19780 IX86_BUILTIN_MWAIT,
19783 IX86_BUILTIN_PHADDW,
19784 IX86_BUILTIN_PHADDD,
19785 IX86_BUILTIN_PHADDSW,
19786 IX86_BUILTIN_PHSUBW,
19787 IX86_BUILTIN_PHSUBD,
19788 IX86_BUILTIN_PHSUBSW,
19789 IX86_BUILTIN_PMADDUBSW,
19790 IX86_BUILTIN_PMULHRSW,
19791 IX86_BUILTIN_PSHUFB,
19792 IX86_BUILTIN_PSIGNB,
19793 IX86_BUILTIN_PSIGNW,
19794 IX86_BUILTIN_PSIGND,
19795 IX86_BUILTIN_PALIGNR,
19796 IX86_BUILTIN_PABSB,
19797 IX86_BUILTIN_PABSW,
19798 IX86_BUILTIN_PABSD,
19800 IX86_BUILTIN_PHADDW128,
19801 IX86_BUILTIN_PHADDD128,
19802 IX86_BUILTIN_PHADDSW128,
19803 IX86_BUILTIN_PHSUBW128,
19804 IX86_BUILTIN_PHSUBD128,
19805 IX86_BUILTIN_PHSUBSW128,
19806 IX86_BUILTIN_PMADDUBSW128,
19807 IX86_BUILTIN_PMULHRSW128,
19808 IX86_BUILTIN_PSHUFB128,
19809 IX86_BUILTIN_PSIGNB128,
19810 IX86_BUILTIN_PSIGNW128,
19811 IX86_BUILTIN_PSIGND128,
19812 IX86_BUILTIN_PALIGNR128,
19813 IX86_BUILTIN_PABSB128,
19814 IX86_BUILTIN_PABSW128,
19815 IX86_BUILTIN_PABSD128,
19817 /* AMDFAM10 - SSE4A New Instructions. */
19818 IX86_BUILTIN_MOVNTSD,
19819 IX86_BUILTIN_MOVNTSS,
19820 IX86_BUILTIN_EXTRQI,
19821 IX86_BUILTIN_EXTRQ,
19822 IX86_BUILTIN_INSERTQI,
19823 IX86_BUILTIN_INSERTQ,
19826 IX86_BUILTIN_BLENDPD,
19827 IX86_BUILTIN_BLENDPS,
19828 IX86_BUILTIN_BLENDVPD,
19829 IX86_BUILTIN_BLENDVPS,
19830 IX86_BUILTIN_PBLENDVB128,
19831 IX86_BUILTIN_PBLENDW128,
19836 IX86_BUILTIN_INSERTPS128,
19838 IX86_BUILTIN_MOVNTDQA,
19839 IX86_BUILTIN_MPSADBW128,
19840 IX86_BUILTIN_PACKUSDW128,
19841 IX86_BUILTIN_PCMPEQQ,
19842 IX86_BUILTIN_PHMINPOSUW128,
19844 IX86_BUILTIN_PMAXSB128,
19845 IX86_BUILTIN_PMAXSD128,
19846 IX86_BUILTIN_PMAXUD128,
19847 IX86_BUILTIN_PMAXUW128,
19849 IX86_BUILTIN_PMINSB128,
19850 IX86_BUILTIN_PMINSD128,
19851 IX86_BUILTIN_PMINUD128,
19852 IX86_BUILTIN_PMINUW128,
19854 IX86_BUILTIN_PMOVSXBW128,
19855 IX86_BUILTIN_PMOVSXBD128,
19856 IX86_BUILTIN_PMOVSXBQ128,
19857 IX86_BUILTIN_PMOVSXWD128,
19858 IX86_BUILTIN_PMOVSXWQ128,
19859 IX86_BUILTIN_PMOVSXDQ128,
19861 IX86_BUILTIN_PMOVZXBW128,
19862 IX86_BUILTIN_PMOVZXBD128,
19863 IX86_BUILTIN_PMOVZXBQ128,
19864 IX86_BUILTIN_PMOVZXWD128,
19865 IX86_BUILTIN_PMOVZXWQ128,
19866 IX86_BUILTIN_PMOVZXDQ128,
19868 IX86_BUILTIN_PMULDQ128,
19869 IX86_BUILTIN_PMULLD128,
19871 IX86_BUILTIN_ROUNDPD,
19872 IX86_BUILTIN_ROUNDPS,
19873 IX86_BUILTIN_ROUNDSD,
19874 IX86_BUILTIN_ROUNDSS,
19876 IX86_BUILTIN_PTESTZ,
19877 IX86_BUILTIN_PTESTC,
19878 IX86_BUILTIN_PTESTNZC,
19880 IX86_BUILTIN_VEC_INIT_V2SI,
19881 IX86_BUILTIN_VEC_INIT_V4HI,
19882 IX86_BUILTIN_VEC_INIT_V8QI,
19883 IX86_BUILTIN_VEC_EXT_V2DF,
19884 IX86_BUILTIN_VEC_EXT_V2DI,
19885 IX86_BUILTIN_VEC_EXT_V4SF,
19886 IX86_BUILTIN_VEC_EXT_V4SI,
19887 IX86_BUILTIN_VEC_EXT_V8HI,
19888 IX86_BUILTIN_VEC_EXT_V2SI,
19889 IX86_BUILTIN_VEC_EXT_V4HI,
19890 IX86_BUILTIN_VEC_EXT_V16QI,
19891 IX86_BUILTIN_VEC_SET_V2DI,
19892 IX86_BUILTIN_VEC_SET_V4SF,
19893 IX86_BUILTIN_VEC_SET_V4SI,
19894 IX86_BUILTIN_VEC_SET_V8HI,
19895 IX86_BUILTIN_VEC_SET_V4HI,
19896 IX86_BUILTIN_VEC_SET_V16QI,
19898 IX86_BUILTIN_VEC_PACK_SFIX,
19901 IX86_BUILTIN_CRC32QI,
19902 IX86_BUILTIN_CRC32HI,
19903 IX86_BUILTIN_CRC32SI,
19904 IX86_BUILTIN_CRC32DI,
19906 IX86_BUILTIN_PCMPESTRI128,
19907 IX86_BUILTIN_PCMPESTRM128,
19908 IX86_BUILTIN_PCMPESTRA128,
19909 IX86_BUILTIN_PCMPESTRC128,
19910 IX86_BUILTIN_PCMPESTRO128,
19911 IX86_BUILTIN_PCMPESTRS128,
19912 IX86_BUILTIN_PCMPESTRZ128,
19913 IX86_BUILTIN_PCMPISTRI128,
19914 IX86_BUILTIN_PCMPISTRM128,
19915 IX86_BUILTIN_PCMPISTRA128,
19916 IX86_BUILTIN_PCMPISTRC128,
19917 IX86_BUILTIN_PCMPISTRO128,
19918 IX86_BUILTIN_PCMPISTRS128,
19919 IX86_BUILTIN_PCMPISTRZ128,
19921 IX86_BUILTIN_PCMPGTQ,
19923 /* AES instructions */
19924 IX86_BUILTIN_AESENC128,
19925 IX86_BUILTIN_AESENCLAST128,
19926 IX86_BUILTIN_AESDEC128,
19927 IX86_BUILTIN_AESDECLAST128,
19928 IX86_BUILTIN_AESIMC128,
19929 IX86_BUILTIN_AESKEYGENASSIST128,
19931 /* PCLMUL instruction */
19932 IX86_BUILTIN_PCLMULQDQ128,
19935 IX86_BUILTIN_ADDPD256,
19936 IX86_BUILTIN_ADDPS256,
19937 IX86_BUILTIN_ADDSUBPD256,
19938 IX86_BUILTIN_ADDSUBPS256,
19939 IX86_BUILTIN_ANDPD256,
19940 IX86_BUILTIN_ANDPS256,
19941 IX86_BUILTIN_ANDNPD256,
19942 IX86_BUILTIN_ANDNPS256,
19943 IX86_BUILTIN_BLENDPD256,
19944 IX86_BUILTIN_BLENDPS256,
19945 IX86_BUILTIN_BLENDVPD256,
19946 IX86_BUILTIN_BLENDVPS256,
19947 IX86_BUILTIN_DIVPD256,
19948 IX86_BUILTIN_DIVPS256,
19949 IX86_BUILTIN_DPPS256,
19950 IX86_BUILTIN_HADDPD256,
19951 IX86_BUILTIN_HADDPS256,
19952 IX86_BUILTIN_HSUBPD256,
19953 IX86_BUILTIN_HSUBPS256,
19954 IX86_BUILTIN_MAXPD256,
19955 IX86_BUILTIN_MAXPS256,
19956 IX86_BUILTIN_MINPD256,
19957 IX86_BUILTIN_MINPS256,
19958 IX86_BUILTIN_MULPD256,
19959 IX86_BUILTIN_MULPS256,
19960 IX86_BUILTIN_ORPD256,
19961 IX86_BUILTIN_ORPS256,
19962 IX86_BUILTIN_SHUFPD256,
19963 IX86_BUILTIN_SHUFPS256,
19964 IX86_BUILTIN_SUBPD256,
19965 IX86_BUILTIN_SUBPS256,
19966 IX86_BUILTIN_XORPD256,
19967 IX86_BUILTIN_XORPS256,
19968 IX86_BUILTIN_CMPSD,
19969 IX86_BUILTIN_CMPSS,
19970 IX86_BUILTIN_CMPPD,
19971 IX86_BUILTIN_CMPPS,
19972 IX86_BUILTIN_CMPPD256,
19973 IX86_BUILTIN_CMPPS256,
19974 IX86_BUILTIN_CVTDQ2PD256,
19975 IX86_BUILTIN_CVTDQ2PS256,
19976 IX86_BUILTIN_CVTPD2PS256,
19977 IX86_BUILTIN_CVTPS2DQ256,
19978 IX86_BUILTIN_CVTPS2PD256,
19979 IX86_BUILTIN_CVTTPD2DQ256,
19980 IX86_BUILTIN_CVTPD2DQ256,
19981 IX86_BUILTIN_CVTTPS2DQ256,
19982 IX86_BUILTIN_EXTRACTF128PD256,
19983 IX86_BUILTIN_EXTRACTF128PS256,
19984 IX86_BUILTIN_EXTRACTF128SI256,
19985 IX86_BUILTIN_VZEROALL,
19986 IX86_BUILTIN_VZEROUPPER,
19987 IX86_BUILTIN_VZEROUPPER_REX64,
19988 IX86_BUILTIN_VPERMILVARPD,
19989 IX86_BUILTIN_VPERMILVARPS,
19990 IX86_BUILTIN_VPERMILVARPD256,
19991 IX86_BUILTIN_VPERMILVARPS256,
19992 IX86_BUILTIN_VPERMILPD,
19993 IX86_BUILTIN_VPERMILPS,
19994 IX86_BUILTIN_VPERMILPD256,
19995 IX86_BUILTIN_VPERMILPS256,
19996 IX86_BUILTIN_VPERM2F128PD256,
19997 IX86_BUILTIN_VPERM2F128PS256,
19998 IX86_BUILTIN_VPERM2F128SI256,
19999 IX86_BUILTIN_VBROADCASTSS,
20000 IX86_BUILTIN_VBROADCASTSD256,
20001 IX86_BUILTIN_VBROADCASTSS256,
20002 IX86_BUILTIN_VBROADCASTPD256,
20003 IX86_BUILTIN_VBROADCASTPS256,
20004 IX86_BUILTIN_VINSERTF128PD256,
20005 IX86_BUILTIN_VINSERTF128PS256,
20006 IX86_BUILTIN_VINSERTF128SI256,
20007 IX86_BUILTIN_LOADUPD256,
20008 IX86_BUILTIN_LOADUPS256,
20009 IX86_BUILTIN_STOREUPD256,
20010 IX86_BUILTIN_STOREUPS256,
20011 IX86_BUILTIN_LDDQU256,
20012 IX86_BUILTIN_MOVNTDQ256,
20013 IX86_BUILTIN_MOVNTPD256,
20014 IX86_BUILTIN_MOVNTPS256,
20015 IX86_BUILTIN_LOADDQU256,
20016 IX86_BUILTIN_STOREDQU256,
20017 IX86_BUILTIN_MASKLOADPD,
20018 IX86_BUILTIN_MASKLOADPS,
20019 IX86_BUILTIN_MASKSTOREPD,
20020 IX86_BUILTIN_MASKSTOREPS,
20021 IX86_BUILTIN_MASKLOADPD256,
20022 IX86_BUILTIN_MASKLOADPS256,
20023 IX86_BUILTIN_MASKSTOREPD256,
20024 IX86_BUILTIN_MASKSTOREPS256,
20025 IX86_BUILTIN_MOVSHDUP256,
20026 IX86_BUILTIN_MOVSLDUP256,
20027 IX86_BUILTIN_MOVDDUP256,
20029 IX86_BUILTIN_SQRTPD256,
20030 IX86_BUILTIN_SQRTPS256,
20031 IX86_BUILTIN_SQRTPS_NR256,
20032 IX86_BUILTIN_RSQRTPS256,
20033 IX86_BUILTIN_RSQRTPS_NR256,
20035 IX86_BUILTIN_RCPPS256,
20037 IX86_BUILTIN_ROUNDPD256,
20038 IX86_BUILTIN_ROUNDPS256,
20040 IX86_BUILTIN_UNPCKHPD256,
20041 IX86_BUILTIN_UNPCKLPD256,
20042 IX86_BUILTIN_UNPCKHPS256,
20043 IX86_BUILTIN_UNPCKLPS256,
20045 IX86_BUILTIN_SI256_SI,
20046 IX86_BUILTIN_PS256_PS,
20047 IX86_BUILTIN_PD256_PD,
20048 IX86_BUILTIN_SI_SI256,
20049 IX86_BUILTIN_PS_PS256,
20050 IX86_BUILTIN_PD_PD256,
20052 IX86_BUILTIN_VTESTZPD,
20053 IX86_BUILTIN_VTESTCPD,
20054 IX86_BUILTIN_VTESTNZCPD,
20055 IX86_BUILTIN_VTESTZPS,
20056 IX86_BUILTIN_VTESTCPS,
20057 IX86_BUILTIN_VTESTNZCPS,
20058 IX86_BUILTIN_VTESTZPD256,
20059 IX86_BUILTIN_VTESTCPD256,
20060 IX86_BUILTIN_VTESTNZCPD256,
20061 IX86_BUILTIN_VTESTZPS256,
20062 IX86_BUILTIN_VTESTCPS256,
20063 IX86_BUILTIN_VTESTNZCPS256,
20064 IX86_BUILTIN_PTESTZ256,
20065 IX86_BUILTIN_PTESTC256,
20066 IX86_BUILTIN_PTESTNZC256,
20068 IX86_BUILTIN_MOVMSKPD256,
20069 IX86_BUILTIN_MOVMSKPS256,
20071 /* TFmode support builtins. */
20073 IX86_BUILTIN_FABSQ,
20074 IX86_BUILTIN_COPYSIGNQ,
20076 /* SSE5 instructions */
20077 IX86_BUILTIN_FMADDSS,
20078 IX86_BUILTIN_FMADDSD,
20079 IX86_BUILTIN_FMADDPS,
20080 IX86_BUILTIN_FMADDPD,
20081 IX86_BUILTIN_FMSUBSS,
20082 IX86_BUILTIN_FMSUBSD,
20083 IX86_BUILTIN_FMSUBPS,
20084 IX86_BUILTIN_FMSUBPD,
20085 IX86_BUILTIN_FNMADDSS,
20086 IX86_BUILTIN_FNMADDSD,
20087 IX86_BUILTIN_FNMADDPS,
20088 IX86_BUILTIN_FNMADDPD,
20089 IX86_BUILTIN_FNMSUBSS,
20090 IX86_BUILTIN_FNMSUBSD,
20091 IX86_BUILTIN_FNMSUBPS,
20092 IX86_BUILTIN_FNMSUBPD,
20093 IX86_BUILTIN_PCMOV,
20094 IX86_BUILTIN_PCMOV_V2DI,
20095 IX86_BUILTIN_PCMOV_V4SI,
20096 IX86_BUILTIN_PCMOV_V8HI,
20097 IX86_BUILTIN_PCMOV_V16QI,
20098 IX86_BUILTIN_PCMOV_V4SF,
20099 IX86_BUILTIN_PCMOV_V2DF,
20100 IX86_BUILTIN_PPERM,
20101 IX86_BUILTIN_PERMPS,
20102 IX86_BUILTIN_PERMPD,
20103 IX86_BUILTIN_PMACSSWW,
20104 IX86_BUILTIN_PMACSWW,
20105 IX86_BUILTIN_PMACSSWD,
20106 IX86_BUILTIN_PMACSWD,
20107 IX86_BUILTIN_PMACSSDD,
20108 IX86_BUILTIN_PMACSDD,
20109 IX86_BUILTIN_PMACSSDQL,
20110 IX86_BUILTIN_PMACSSDQH,
20111 IX86_BUILTIN_PMACSDQL,
20112 IX86_BUILTIN_PMACSDQH,
20113 IX86_BUILTIN_PMADCSSWD,
20114 IX86_BUILTIN_PMADCSWD,
20115 IX86_BUILTIN_PHADDBW,
20116 IX86_BUILTIN_PHADDBD,
20117 IX86_BUILTIN_PHADDBQ,
20118 IX86_BUILTIN_PHADDWD,
20119 IX86_BUILTIN_PHADDWQ,
20120 IX86_BUILTIN_PHADDDQ,
20121 IX86_BUILTIN_PHADDUBW,
20122 IX86_BUILTIN_PHADDUBD,
20123 IX86_BUILTIN_PHADDUBQ,
20124 IX86_BUILTIN_PHADDUWD,
20125 IX86_BUILTIN_PHADDUWQ,
20126 IX86_BUILTIN_PHADDUDQ,
20127 IX86_BUILTIN_PHSUBBW,
20128 IX86_BUILTIN_PHSUBWD,
20129 IX86_BUILTIN_PHSUBDQ,
20130 IX86_BUILTIN_PROTB,
20131 IX86_BUILTIN_PROTW,
20132 IX86_BUILTIN_PROTD,
20133 IX86_BUILTIN_PROTQ,
20134 IX86_BUILTIN_PROTB_IMM,
20135 IX86_BUILTIN_PROTW_IMM,
20136 IX86_BUILTIN_PROTD_IMM,
20137 IX86_BUILTIN_PROTQ_IMM,
20138 IX86_BUILTIN_PSHLB,
20139 IX86_BUILTIN_PSHLW,
20140 IX86_BUILTIN_PSHLD,
20141 IX86_BUILTIN_PSHLQ,
20142 IX86_BUILTIN_PSHAB,
20143 IX86_BUILTIN_PSHAW,
20144 IX86_BUILTIN_PSHAD,
20145 IX86_BUILTIN_PSHAQ,
20146 IX86_BUILTIN_FRCZSS,
20147 IX86_BUILTIN_FRCZSD,
20148 IX86_BUILTIN_FRCZPS,
20149 IX86_BUILTIN_FRCZPD,
20150 IX86_BUILTIN_CVTPH2PS,
20151 IX86_BUILTIN_CVTPS2PH,
20153 IX86_BUILTIN_COMEQSS,
20154 IX86_BUILTIN_COMNESS,
20155 IX86_BUILTIN_COMLTSS,
20156 IX86_BUILTIN_COMLESS,
20157 IX86_BUILTIN_COMGTSS,
20158 IX86_BUILTIN_COMGESS,
20159 IX86_BUILTIN_COMUEQSS,
20160 IX86_BUILTIN_COMUNESS,
20161 IX86_BUILTIN_COMULTSS,
20162 IX86_BUILTIN_COMULESS,
20163 IX86_BUILTIN_COMUGTSS,
20164 IX86_BUILTIN_COMUGESS,
20165 IX86_BUILTIN_COMORDSS,
20166 IX86_BUILTIN_COMUNORDSS,
20167 IX86_BUILTIN_COMFALSESS,
20168 IX86_BUILTIN_COMTRUESS,
20170 IX86_BUILTIN_COMEQSD,
20171 IX86_BUILTIN_COMNESD,
20172 IX86_BUILTIN_COMLTSD,
20173 IX86_BUILTIN_COMLESD,
20174 IX86_BUILTIN_COMGTSD,
20175 IX86_BUILTIN_COMGESD,
20176 IX86_BUILTIN_COMUEQSD,
20177 IX86_BUILTIN_COMUNESD,
20178 IX86_BUILTIN_COMULTSD,
20179 IX86_BUILTIN_COMULESD,
20180 IX86_BUILTIN_COMUGTSD,
20181 IX86_BUILTIN_COMUGESD,
20182 IX86_BUILTIN_COMORDSD,
20183 IX86_BUILTIN_COMUNORDSD,
20184 IX86_BUILTIN_COMFALSESD,
20185 IX86_BUILTIN_COMTRUESD,
20187 IX86_BUILTIN_COMEQPS,
20188 IX86_BUILTIN_COMNEPS,
20189 IX86_BUILTIN_COMLTPS,
20190 IX86_BUILTIN_COMLEPS,
20191 IX86_BUILTIN_COMGTPS,
20192 IX86_BUILTIN_COMGEPS,
20193 IX86_BUILTIN_COMUEQPS,
20194 IX86_BUILTIN_COMUNEPS,
20195 IX86_BUILTIN_COMULTPS,
20196 IX86_BUILTIN_COMULEPS,
20197 IX86_BUILTIN_COMUGTPS,
20198 IX86_BUILTIN_COMUGEPS,
20199 IX86_BUILTIN_COMORDPS,
20200 IX86_BUILTIN_COMUNORDPS,
20201 IX86_BUILTIN_COMFALSEPS,
20202 IX86_BUILTIN_COMTRUEPS,
20204 IX86_BUILTIN_COMEQPD,
20205 IX86_BUILTIN_COMNEPD,
20206 IX86_BUILTIN_COMLTPD,
20207 IX86_BUILTIN_COMLEPD,
20208 IX86_BUILTIN_COMGTPD,
20209 IX86_BUILTIN_COMGEPD,
20210 IX86_BUILTIN_COMUEQPD,
20211 IX86_BUILTIN_COMUNEPD,
20212 IX86_BUILTIN_COMULTPD,
20213 IX86_BUILTIN_COMULEPD,
20214 IX86_BUILTIN_COMUGTPD,
20215 IX86_BUILTIN_COMUGEPD,
20216 IX86_BUILTIN_COMORDPD,
20217 IX86_BUILTIN_COMUNORDPD,
20218 IX86_BUILTIN_COMFALSEPD,
20219 IX86_BUILTIN_COMTRUEPD,
20221 IX86_BUILTIN_PCOMEQUB,
20222 IX86_BUILTIN_PCOMNEUB,
20223 IX86_BUILTIN_PCOMLTUB,
20224 IX86_BUILTIN_PCOMLEUB,
20225 IX86_BUILTIN_PCOMGTUB,
20226 IX86_BUILTIN_PCOMGEUB,
20227 IX86_BUILTIN_PCOMFALSEUB,
20228 IX86_BUILTIN_PCOMTRUEUB,
20229 IX86_BUILTIN_PCOMEQUW,
20230 IX86_BUILTIN_PCOMNEUW,
20231 IX86_BUILTIN_PCOMLTUW,
20232 IX86_BUILTIN_PCOMLEUW,
20233 IX86_BUILTIN_PCOMGTUW,
20234 IX86_BUILTIN_PCOMGEUW,
20235 IX86_BUILTIN_PCOMFALSEUW,
20236 IX86_BUILTIN_PCOMTRUEUW,
20237 IX86_BUILTIN_PCOMEQUD,
20238 IX86_BUILTIN_PCOMNEUD,
20239 IX86_BUILTIN_PCOMLTUD,
20240 IX86_BUILTIN_PCOMLEUD,
20241 IX86_BUILTIN_PCOMGTUD,
20242 IX86_BUILTIN_PCOMGEUD,
20243 IX86_BUILTIN_PCOMFALSEUD,
20244 IX86_BUILTIN_PCOMTRUEUD,
20245 IX86_BUILTIN_PCOMEQUQ,
20246 IX86_BUILTIN_PCOMNEUQ,
20247 IX86_BUILTIN_PCOMLTUQ,
20248 IX86_BUILTIN_PCOMLEUQ,
20249 IX86_BUILTIN_PCOMGTUQ,
20250 IX86_BUILTIN_PCOMGEUQ,
20251 IX86_BUILTIN_PCOMFALSEUQ,
20252 IX86_BUILTIN_PCOMTRUEUQ,
20254 IX86_BUILTIN_PCOMEQB,
20255 IX86_BUILTIN_PCOMNEB,
20256 IX86_BUILTIN_PCOMLTB,
20257 IX86_BUILTIN_PCOMLEB,
20258 IX86_BUILTIN_PCOMGTB,
20259 IX86_BUILTIN_PCOMGEB,
20260 IX86_BUILTIN_PCOMFALSEB,
20261 IX86_BUILTIN_PCOMTRUEB,
20262 IX86_BUILTIN_PCOMEQW,
20263 IX86_BUILTIN_PCOMNEW,
20264 IX86_BUILTIN_PCOMLTW,
20265 IX86_BUILTIN_PCOMLEW,
20266 IX86_BUILTIN_PCOMGTW,
20267 IX86_BUILTIN_PCOMGEW,
20268 IX86_BUILTIN_PCOMFALSEW,
20269 IX86_BUILTIN_PCOMTRUEW,
20270 IX86_BUILTIN_PCOMEQD,
20271 IX86_BUILTIN_PCOMNED,
20272 IX86_BUILTIN_PCOMLTD,
20273 IX86_BUILTIN_PCOMLED,
20274 IX86_BUILTIN_PCOMGTD,
20275 IX86_BUILTIN_PCOMGED,
20276 IX86_BUILTIN_PCOMFALSED,
20277 IX86_BUILTIN_PCOMTRUED,
20278 IX86_BUILTIN_PCOMEQQ,
20279 IX86_BUILTIN_PCOMNEQ,
20280 IX86_BUILTIN_PCOMLTQ,
20281 IX86_BUILTIN_PCOMLEQ,
20282 IX86_BUILTIN_PCOMGTQ,
20283 IX86_BUILTIN_PCOMGEQ,
20284 IX86_BUILTIN_PCOMFALSEQ,
20285 IX86_BUILTIN_PCOMTRUEQ,
20290 /* Table for the ix86 builtin decls. */
20291 static GTY(()) tree ix86_builtins[(int) IX86_BUILTIN_MAX];
20293 /* Table of all of the builtin functions that are possible with different ISA's
20294 but are waiting to be built until a function is declared to use that
20296 struct builtin_isa GTY(())
20298 tree type; /* builtin type to use in the declaration */
20299 const char *name; /* function name */
20300 int isa; /* isa_flags this builtin is defined for */
20301 bool const_p; /* true if the declaration is constant */
20304 static GTY(()) struct builtin_isa ix86_builtins_isa[(int) IX86_BUILTIN_MAX];
20307 /* Add an ix86 target builtin function with CODE, NAME and TYPE. Save the MASK
20308 * of which isa_flags to use in the ix86_builtins_isa array. Stores the
20309 * function decl in the ix86_builtins array. Returns the function decl or
20310 * NULL_TREE, if the builtin was not added.
20312 * If the front end has a special hook for builtin functions, delay adding
20313 * builtin functions that aren't in the current ISA until the ISA is changed
20314 * with function specific optimization. Doing so, can save about 300K for the
20315 * default compiler. When the builtin is expanded, check at that time whether
20318 * If the front end doesn't have a special hook, record all builtins, even if
20319 * it isn't an instruction set in the current ISA in case the user uses
20320 * function specific options for a different ISA, so that we don't get scope
20321 * errors if a builtin is added in the middle of a function scope. */
20324 def_builtin (int mask, const char *name, tree type, enum ix86_builtins code)
20326 tree decl = NULL_TREE;
20328 if (!(mask & OPTION_MASK_ISA_64BIT) || TARGET_64BIT)
20330 ix86_builtins_isa[(int) code].isa = mask;
20332 if ((mask & ix86_isa_flags) != 0
20333 || (lang_hooks.builtin_function
20334 == lang_hooks.builtin_function_ext_scope))
20337 decl = add_builtin_function (name, type, code, BUILT_IN_MD, NULL,
20339 ix86_builtins[(int) code] = decl;
20340 ix86_builtins_isa[(int) code].type = NULL_TREE;
20344 ix86_builtins[(int) code] = NULL_TREE;
20345 ix86_builtins_isa[(int) code].const_p = false;
20346 ix86_builtins_isa[(int) code].type = type;
20347 ix86_builtins_isa[(int) code].name = name;
20354 /* Like def_builtin, but also marks the function decl "const". */
20357 def_builtin_const (int mask, const char *name, tree type,
20358 enum ix86_builtins code)
20360 tree decl = def_builtin (mask, name, type, code);
20362 TREE_READONLY (decl) = 1;
20364 ix86_builtins_isa[(int) code].const_p = true;
20369 /* Add any new builtin functions for a given ISA that may not have been
20370 declared. This saves a bit of space compared to adding all of the
20371 declarations to the tree, even if we didn't use them. */
20374 ix86_add_new_builtins (int isa)
20379 for (i = 0; i < (int)IX86_BUILTIN_MAX; i++)
20381 if ((ix86_builtins_isa[i].isa & isa) != 0
20382 && ix86_builtins_isa[i].type != NULL_TREE)
20384 decl = add_builtin_function_ext_scope (ix86_builtins_isa[i].name,
20385 ix86_builtins_isa[i].type,
20386 i, BUILT_IN_MD, NULL,
20389 ix86_builtins[i] = decl;
20390 ix86_builtins_isa[i].type = NULL_TREE;
20391 if (ix86_builtins_isa[i].const_p)
20392 TREE_READONLY (decl) = 1;
20397 /* Bits for builtin_description.flag. */
20399 /* Set when we don't support the comparison natively, and should
20400 swap_comparison in order to support it. */
20401 #define BUILTIN_DESC_SWAP_OPERANDS 1
20403 struct builtin_description
20405 const unsigned int mask;
20406 const enum insn_code icode;
20407 const char *const name;
20408 const enum ix86_builtins code;
20409 const enum rtx_code comparison;
20413 static const struct builtin_description bdesc_comi[] =
20415 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comieq", IX86_BUILTIN_COMIEQSS, UNEQ, 0 },
20416 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comilt", IX86_BUILTIN_COMILTSS, UNLT, 0 },
20417 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comile", IX86_BUILTIN_COMILESS, UNLE, 0 },
20418 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comigt", IX86_BUILTIN_COMIGTSS, GT, 0 },
20419 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comige", IX86_BUILTIN_COMIGESS, GE, 0 },
20420 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comineq", IX86_BUILTIN_COMINEQSS, LTGT, 0 },
20421 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomieq", IX86_BUILTIN_UCOMIEQSS, UNEQ, 0 },
20422 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomilt", IX86_BUILTIN_UCOMILTSS, UNLT, 0 },
20423 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomile", IX86_BUILTIN_UCOMILESS, UNLE, 0 },
20424 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomigt", IX86_BUILTIN_UCOMIGTSS, GT, 0 },
20425 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomige", IX86_BUILTIN_UCOMIGESS, GE, 0 },
20426 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomineq", IX86_BUILTIN_UCOMINEQSS, LTGT, 0 },
20427 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdeq", IX86_BUILTIN_COMIEQSD, UNEQ, 0 },
20428 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdlt", IX86_BUILTIN_COMILTSD, UNLT, 0 },
20429 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdle", IX86_BUILTIN_COMILESD, UNLE, 0 },
20430 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdgt", IX86_BUILTIN_COMIGTSD, GT, 0 },
20431 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdge", IX86_BUILTIN_COMIGESD, GE, 0 },
20432 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdneq", IX86_BUILTIN_COMINEQSD, LTGT, 0 },
20433 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdeq", IX86_BUILTIN_UCOMIEQSD, UNEQ, 0 },
20434 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdlt", IX86_BUILTIN_UCOMILTSD, UNLT, 0 },
20435 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdle", IX86_BUILTIN_UCOMILESD, UNLE, 0 },
20436 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdgt", IX86_BUILTIN_UCOMIGTSD, GT, 0 },
20437 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdge", IX86_BUILTIN_UCOMIGESD, GE, 0 },
20438 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdneq", IX86_BUILTIN_UCOMINEQSD, LTGT, 0 },
20441 static const struct builtin_description bdesc_pcmpestr[] =
20444 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestri128", IX86_BUILTIN_PCMPESTRI128, UNKNOWN, 0 },
20445 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrm128", IX86_BUILTIN_PCMPESTRM128, UNKNOWN, 0 },
20446 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestria128", IX86_BUILTIN_PCMPESTRA128, UNKNOWN, (int) CCAmode },
20447 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestric128", IX86_BUILTIN_PCMPESTRC128, UNKNOWN, (int) CCCmode },
20448 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrio128", IX86_BUILTIN_PCMPESTRO128, UNKNOWN, (int) CCOmode },
20449 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestris128", IX86_BUILTIN_PCMPESTRS128, UNKNOWN, (int) CCSmode },
20450 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestriz128", IX86_BUILTIN_PCMPESTRZ128, UNKNOWN, (int) CCZmode },
20453 static const struct builtin_description bdesc_pcmpistr[] =
20456 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistri128", IX86_BUILTIN_PCMPISTRI128, UNKNOWN, 0 },
20457 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrm128", IX86_BUILTIN_PCMPISTRM128, UNKNOWN, 0 },
20458 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistria128", IX86_BUILTIN_PCMPISTRA128, UNKNOWN, (int) CCAmode },
20459 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistric128", IX86_BUILTIN_PCMPISTRC128, UNKNOWN, (int) CCCmode },
20460 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrio128", IX86_BUILTIN_PCMPISTRO128, UNKNOWN, (int) CCOmode },
20461 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistris128", IX86_BUILTIN_PCMPISTRS128, UNKNOWN, (int) CCSmode },
20462 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistriz128", IX86_BUILTIN_PCMPISTRZ128, UNKNOWN, (int) CCZmode },
20465 /* Special builtin types */
20466 enum ix86_special_builtin_type
20468 SPECIAL_FTYPE_UNKNOWN,
20470 V32QI_FTYPE_PCCHAR,
20471 V16QI_FTYPE_PCCHAR,
20473 V8SF_FTYPE_PCFLOAT,
20475 V4DF_FTYPE_PCDOUBLE,
20476 V4SF_FTYPE_PCFLOAT,
20477 V2DF_FTYPE_PCDOUBLE,
20478 V8SF_FTYPE_PCV8SF_V8SF,
20479 V4DF_FTYPE_PCV4DF_V4DF,
20480 V4SF_FTYPE_V4SF_PCV2SF,
20481 V4SF_FTYPE_PCV4SF_V4SF,
20482 V2DF_FTYPE_V2DF_PCDOUBLE,
20483 V2DF_FTYPE_PCV2DF_V2DF,
20485 VOID_FTYPE_PV2SF_V4SF,
20486 VOID_FTYPE_PV4DI_V4DI,
20487 VOID_FTYPE_PV2DI_V2DI,
20488 VOID_FTYPE_PCHAR_V32QI,
20489 VOID_FTYPE_PCHAR_V16QI,
20490 VOID_FTYPE_PFLOAT_V8SF,
20491 VOID_FTYPE_PFLOAT_V4SF,
20492 VOID_FTYPE_PDOUBLE_V4DF,
20493 VOID_FTYPE_PDOUBLE_V2DF,
20495 VOID_FTYPE_PINT_INT,
20496 VOID_FTYPE_PV8SF_V8SF_V8SF,
20497 VOID_FTYPE_PV4DF_V4DF_V4DF,
20498 VOID_FTYPE_PV4SF_V4SF_V4SF,
20499 VOID_FTYPE_PV2DF_V2DF_V2DF
20502 /* Builtin types */
20503 enum ix86_builtin_type
20506 FLOAT128_FTYPE_FLOAT128,
20508 FLOAT128_FTYPE_FLOAT128_FLOAT128,
20509 INT_FTYPE_V8SF_V8SF_PTEST,
20510 INT_FTYPE_V4DI_V4DI_PTEST,
20511 INT_FTYPE_V4DF_V4DF_PTEST,
20512 INT_FTYPE_V4SF_V4SF_PTEST,
20513 INT_FTYPE_V2DI_V2DI_PTEST,
20514 INT_FTYPE_V2DF_V2DF_PTEST,
20546 V4SF_FTYPE_V4SF_VEC_MERGE,
20555 V2DF_FTYPE_V2DF_VEC_MERGE,
20566 V16QI_FTYPE_V16QI_V16QI,
20567 V16QI_FTYPE_V8HI_V8HI,
20568 V8QI_FTYPE_V8QI_V8QI,
20569 V8QI_FTYPE_V4HI_V4HI,
20570 V8HI_FTYPE_V8HI_V8HI,
20571 V8HI_FTYPE_V8HI_V8HI_COUNT,
20572 V8HI_FTYPE_V16QI_V16QI,
20573 V8HI_FTYPE_V4SI_V4SI,
20574 V8HI_FTYPE_V8HI_SI_COUNT,
20575 V8SF_FTYPE_V8SF_V8SF,
20576 V8SF_FTYPE_V8SF_V8SI,
20577 V4SI_FTYPE_V4SI_V4SI,
20578 V4SI_FTYPE_V4SI_V4SI_COUNT,
20579 V4SI_FTYPE_V8HI_V8HI,
20580 V4SI_FTYPE_V4SF_V4SF,
20581 V4SI_FTYPE_V2DF_V2DF,
20582 V4SI_FTYPE_V4SI_SI_COUNT,
20583 V4HI_FTYPE_V4HI_V4HI,
20584 V4HI_FTYPE_V4HI_V4HI_COUNT,
20585 V4HI_FTYPE_V8QI_V8QI,
20586 V4HI_FTYPE_V2SI_V2SI,
20587 V4HI_FTYPE_V4HI_SI_COUNT,
20588 V4DF_FTYPE_V4DF_V4DF,
20589 V4DF_FTYPE_V4DF_V4DI,
20590 V4SF_FTYPE_V4SF_V4SF,
20591 V4SF_FTYPE_V4SF_V4SF_SWAP,
20592 V4SF_FTYPE_V4SF_V4SI,
20593 V4SF_FTYPE_V4SF_V2SI,
20594 V4SF_FTYPE_V4SF_V2DF,
20595 V4SF_FTYPE_V4SF_DI,
20596 V4SF_FTYPE_V4SF_SI,
20597 V2DI_FTYPE_V2DI_V2DI,
20598 V2DI_FTYPE_V2DI_V2DI_COUNT,
20599 V2DI_FTYPE_V16QI_V16QI,
20600 V2DI_FTYPE_V4SI_V4SI,
20601 V2DI_FTYPE_V2DI_V16QI,
20602 V2DI_FTYPE_V2DF_V2DF,
20603 V2DI_FTYPE_V2DI_SI_COUNT,
20604 V2SI_FTYPE_V2SI_V2SI,
20605 V2SI_FTYPE_V2SI_V2SI_COUNT,
20606 V2SI_FTYPE_V4HI_V4HI,
20607 V2SI_FTYPE_V2SF_V2SF,
20608 V2SI_FTYPE_V2SI_SI_COUNT,
20609 V2DF_FTYPE_V2DF_V2DF,
20610 V2DF_FTYPE_V2DF_V2DF_SWAP,
20611 V2DF_FTYPE_V2DF_V4SF,
20612 V2DF_FTYPE_V2DF_V2DI,
20613 V2DF_FTYPE_V2DF_DI,
20614 V2DF_FTYPE_V2DF_SI,
20615 V2SF_FTYPE_V2SF_V2SF,
20616 V1DI_FTYPE_V1DI_V1DI,
20617 V1DI_FTYPE_V1DI_V1DI_COUNT,
20618 V1DI_FTYPE_V8QI_V8QI,
20619 V1DI_FTYPE_V2SI_V2SI,
20620 V1DI_FTYPE_V1DI_SI_COUNT,
20621 UINT64_FTYPE_UINT64_UINT64,
20622 UINT_FTYPE_UINT_UINT,
20623 UINT_FTYPE_UINT_USHORT,
20624 UINT_FTYPE_UINT_UCHAR,
20625 V8HI_FTYPE_V8HI_INT,
20626 V4SI_FTYPE_V4SI_INT,
20627 V4HI_FTYPE_V4HI_INT,
20628 V8SF_FTYPE_V8SF_INT,
20629 V4SI_FTYPE_V8SI_INT,
20630 V4SF_FTYPE_V8SF_INT,
20631 V2DF_FTYPE_V4DF_INT,
20632 V4DF_FTYPE_V4DF_INT,
20633 V4SF_FTYPE_V4SF_INT,
20634 V2DI_FTYPE_V2DI_INT,
20635 V2DI2TI_FTYPE_V2DI_INT,
20636 V2DF_FTYPE_V2DF_INT,
20637 V16QI_FTYPE_V16QI_V16QI_V16QI,
20638 V8SF_FTYPE_V8SF_V8SF_V8SF,
20639 V4DF_FTYPE_V4DF_V4DF_V4DF,
20640 V4SF_FTYPE_V4SF_V4SF_V4SF,
20641 V2DF_FTYPE_V2DF_V2DF_V2DF,
20642 V16QI_FTYPE_V16QI_V16QI_INT,
20643 V8SI_FTYPE_V8SI_V8SI_INT,
20644 V8SI_FTYPE_V8SI_V4SI_INT,
20645 V8HI_FTYPE_V8HI_V8HI_INT,
20646 V8SF_FTYPE_V8SF_V8SF_INT,
20647 V8SF_FTYPE_V8SF_V4SF_INT,
20648 V4SI_FTYPE_V4SI_V4SI_INT,
20649 V4DF_FTYPE_V4DF_V4DF_INT,
20650 V4DF_FTYPE_V4DF_V2DF_INT,
20651 V4SF_FTYPE_V4SF_V4SF_INT,
20652 V2DI_FTYPE_V2DI_V2DI_INT,
20653 V2DI2TI_FTYPE_V2DI_V2DI_INT,
20654 V1DI2DI_FTYPE_V1DI_V1DI_INT,
20655 V2DF_FTYPE_V2DF_V2DF_INT,
20656 V2DI_FTYPE_V2DI_UINT_UINT,
20657 V2DI_FTYPE_V2DI_V2DI_UINT_UINT
20660 /* Special builtins with variable number of arguments. */
20661 static const struct builtin_description bdesc_special_args[] =
20664 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_emms, "__builtin_ia32_emms", IX86_BUILTIN_EMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
20667 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_femms, "__builtin_ia32_femms", IX86_BUILTIN_FEMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
20670 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movups, "__builtin_ia32_storeups", IX86_BUILTIN_STOREUPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
20671 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movntv4sf, "__builtin_ia32_movntps", IX86_BUILTIN_MOVNTPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
20672 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movups, "__builtin_ia32_loadups", IX86_BUILTIN_LOADUPS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
20674 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadhps_exp, "__builtin_ia32_loadhps", IX86_BUILTIN_LOADHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
20675 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadlps_exp, "__builtin_ia32_loadlps", IX86_BUILTIN_LOADLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
20676 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storehps, "__builtin_ia32_storehps", IX86_BUILTIN_STOREHPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
20677 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storelps, "__builtin_ia32_storelps", IX86_BUILTIN_STORELPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
20679 /* SSE or 3DNow!A */
20680 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_sfence, "__builtin_ia32_sfence", IX86_BUILTIN_SFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
20681 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_movntdi, "__builtin_ia32_movntq", IX86_BUILTIN_MOVNTQ, UNKNOWN, (int) VOID_FTYPE_PDI_DI },
20684 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lfence, "__builtin_ia32_lfence", IX86_BUILTIN_LFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
20685 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_mfence, 0, IX86_BUILTIN_MFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
20686 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movupd, "__builtin_ia32_storeupd", IX86_BUILTIN_STOREUPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
20687 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movdqu, "__builtin_ia32_storedqu", IX86_BUILTIN_STOREDQU, UNKNOWN, (int) VOID_FTYPE_PCHAR_V16QI },
20688 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2df, "__builtin_ia32_movntpd", IX86_BUILTIN_MOVNTPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
20689 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2di, "__builtin_ia32_movntdq", IX86_BUILTIN_MOVNTDQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI },
20690 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntsi, "__builtin_ia32_movnti", IX86_BUILTIN_MOVNTI, UNKNOWN, (int) VOID_FTYPE_PINT_INT },
20691 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movupd, "__builtin_ia32_loadupd", IX86_BUILTIN_LOADUPD, UNKNOWN, (int) V2DF_FTYPE_PCDOUBLE },
20692 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movdqu, "__builtin_ia32_loaddqu", IX86_BUILTIN_LOADDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
20694 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadhpd_exp, "__builtin_ia32_loadhpd", IX86_BUILTIN_LOADHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
20695 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadlpd_exp, "__builtin_ia32_loadlpd", IX86_BUILTIN_LOADLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
20698 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_lddqu, "__builtin_ia32_lddqu", IX86_BUILTIN_LDDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
20701 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_movntdqa, "__builtin_ia32_movntdqa", IX86_BUILTIN_MOVNTDQA, UNKNOWN, (int) V2DI_FTYPE_PV2DI },
20704 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv2df, "__builtin_ia32_movntsd", IX86_BUILTIN_MOVNTSD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
20705 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv4sf, "__builtin_ia32_movntss", IX86_BUILTIN_MOVNTSS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
20708 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroall, "__builtin_ia32_vzeroall", IX86_BUILTIN_VZEROALL, UNKNOWN, (int) VOID_FTYPE_VOID },
20709 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroupper, 0, IX86_BUILTIN_VZEROUPPER, UNKNOWN, (int) VOID_FTYPE_VOID },
20710 { OPTION_MASK_ISA_AVX | OPTION_MASK_ISA_64BIT, CODE_FOR_avx_vzeroupper_rex64, 0, IX86_BUILTIN_VZEROUPPER_REX64, UNKNOWN, (int) VOID_FTYPE_VOID },
20712 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastss, "__builtin_ia32_vbroadcastss", IX86_BUILTIN_VBROADCASTSS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
20713 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastsd256, "__builtin_ia32_vbroadcastsd256", IX86_BUILTIN_VBROADCASTSD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
20714 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastss256, "__builtin_ia32_vbroadcastss256", IX86_BUILTIN_VBROADCASTSS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
20715 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_pd256, "__builtin_ia32_vbroadcastf128_pd256", IX86_BUILTIN_VBROADCASTPD256, UNKNOWN, (int) V4DF_FTYPE_PCV2DF },
20716 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_ps256, "__builtin_ia32_vbroadcastf128_ps256", IX86_BUILTIN_VBROADCASTPS256, UNKNOWN, (int) V8SF_FTYPE_PCV4SF },
20718 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movupd256, "__builtin_ia32_loadupd256", IX86_BUILTIN_LOADUPD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
20719 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movups256, "__builtin_ia32_loadups256", IX86_BUILTIN_LOADUPS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
20720 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movupd256, "__builtin_ia32_storeupd256", IX86_BUILTIN_STOREUPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
20721 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movups256, "__builtin_ia32_storeups256", IX86_BUILTIN_STOREUPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
20722 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movdqu256, "__builtin_ia32_loaddqu256", IX86_BUILTIN_LOADDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
20723 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movdqu256, "__builtin_ia32_storedqu256", IX86_BUILTIN_STOREDQU256, UNKNOWN, (int) VOID_FTYPE_PCHAR_V32QI },
20724 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_lddqu256, "__builtin_ia32_lddqu256", IX86_BUILTIN_LDDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
20726 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4di, "__builtin_ia32_movntdq256", IX86_BUILTIN_MOVNTDQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI },
20727 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4df, "__builtin_ia32_movntpd256", IX86_BUILTIN_MOVNTPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
20728 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv8sf, "__builtin_ia32_movntps256", IX86_BUILTIN_MOVNTPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
20730 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd, "__builtin_ia32_maskloadpd", IX86_BUILTIN_MASKLOADPD, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DF },
20731 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps, "__builtin_ia32_maskloadps", IX86_BUILTIN_MASKLOADPS, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SF },
20732 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd256, "__builtin_ia32_maskloadpd256", IX86_BUILTIN_MASKLOADPD256, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DF },
20733 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps256, "__builtin_ia32_maskloadps256", IX86_BUILTIN_MASKLOADPS256, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SF },
20734 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd, "__builtin_ia32_maskstorepd", IX86_BUILTIN_MASKSTOREPD, UNKNOWN, (int) VOID_FTYPE_PV2DF_V2DF_V2DF },
20735 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps, "__builtin_ia32_maskstoreps", IX86_BUILTIN_MASKSTOREPS, UNKNOWN, (int) VOID_FTYPE_PV4SF_V4SF_V4SF },
20736 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd256, "__builtin_ia32_maskstorepd256", IX86_BUILTIN_MASKSTOREPD256, UNKNOWN, (int) VOID_FTYPE_PV4DF_V4DF_V4DF },
20737 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps256, "__builtin_ia32_maskstoreps256", IX86_BUILTIN_MASKSTOREPS256, UNKNOWN, (int) VOID_FTYPE_PV8SF_V8SF_V8SF },
20740 /* Builtins with variable number of arguments. */
20741 static const struct builtin_description bdesc_args[] =
20744 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv8qi3, "__builtin_ia32_paddb", IX86_BUILTIN_PADDB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
20745 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv4hi3, "__builtin_ia32_paddw", IX86_BUILTIN_PADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
20746 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv2si3, "__builtin_ia32_paddd", IX86_BUILTIN_PADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
20747 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv8qi3, "__builtin_ia32_psubb", IX86_BUILTIN_PSUBB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
20748 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv4hi3, "__builtin_ia32_psubw", IX86_BUILTIN_PSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
20749 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv2si3, "__builtin_ia32_psubd", IX86_BUILTIN_PSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
20751 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv8qi3, "__builtin_ia32_paddsb", IX86_BUILTIN_PADDSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
20752 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv4hi3, "__builtin_ia32_paddsw", IX86_BUILTIN_PADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
20753 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv8qi3, "__builtin_ia32_psubsb", IX86_BUILTIN_PSUBSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
20754 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv4hi3, "__builtin_ia32_psubsw", IX86_BUILTIN_PSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
20755 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv8qi3, "__builtin_ia32_paddusb", IX86_BUILTIN_PADDUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
20756 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv4hi3, "__builtin_ia32_paddusw", IX86_BUILTIN_PADDUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
20757 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv8qi3, "__builtin_ia32_psubusb", IX86_BUILTIN_PSUBUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
20758 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv4hi3, "__builtin_ia32_psubusw", IX86_BUILTIN_PSUBUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
20760 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_mulv4hi3, "__builtin_ia32_pmullw", IX86_BUILTIN_PMULLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
20761 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_smulv4hi3_highpart, "__builtin_ia32_pmulhw", IX86_BUILTIN_PMULHW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
20763 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andv2si3, "__builtin_ia32_pand", IX86_BUILTIN_PAND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
20764 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andnotv2si3, "__builtin_ia32_pandn", IX86_BUILTIN_PANDN, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
20765 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_iorv2si3, "__builtin_ia32_por", IX86_BUILTIN_POR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
20766 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_xorv2si3, "__builtin_ia32_pxor", IX86_BUILTIN_PXOR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
20768 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv8qi3, "__builtin_ia32_pcmpeqb", IX86_BUILTIN_PCMPEQB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
20769 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv4hi3, "__builtin_ia32_pcmpeqw", IX86_BUILTIN_PCMPEQW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
20770 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv2si3, "__builtin_ia32_pcmpeqd", IX86_BUILTIN_PCMPEQD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
20771 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv8qi3, "__builtin_ia32_pcmpgtb", IX86_BUILTIN_PCMPGTB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
20772 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv4hi3, "__builtin_ia32_pcmpgtw", IX86_BUILTIN_PCMPGTW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
20773 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv2si3, "__builtin_ia32_pcmpgtd", IX86_BUILTIN_PCMPGTD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
20775 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhbw, "__builtin_ia32_punpckhbw", IX86_BUILTIN_PUNPCKHBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
20776 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhwd, "__builtin_ia32_punpckhwd", IX86_BUILTIN_PUNPCKHWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
20777 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhdq, "__builtin_ia32_punpckhdq", IX86_BUILTIN_PUNPCKHDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
20778 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklbw, "__builtin_ia32_punpcklbw", IX86_BUILTIN_PUNPCKLBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
20779 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklwd, "__builtin_ia32_punpcklwd", IX86_BUILTIN_PUNPCKLWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI},
20780 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckldq, "__builtin_ia32_punpckldq", IX86_BUILTIN_PUNPCKLDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI},
20782 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packsswb, "__builtin_ia32_packsswb", IX86_BUILTIN_PACKSSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
20783 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packssdw, "__builtin_ia32_packssdw", IX86_BUILTIN_PACKSSDW, UNKNOWN, (int) V4HI_FTYPE_V2SI_V2SI },
20784 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packuswb, "__builtin_ia32_packuswb", IX86_BUILTIN_PACKUSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
20786 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_pmaddwd, "__builtin_ia32_pmaddwd", IX86_BUILTIN_PMADDWD, UNKNOWN, (int) V2SI_FTYPE_V4HI_V4HI },
20788 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllwi", IX86_BUILTIN_PSLLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
20789 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslldi", IX86_BUILTIN_PSLLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
20790 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllqi", IX86_BUILTIN_PSLLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
20791 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllw", IX86_BUILTIN_PSLLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
20792 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslld", IX86_BUILTIN_PSLLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
20793 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllq", IX86_BUILTIN_PSLLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
20795 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlwi", IX86_BUILTIN_PSRLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
20796 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrldi", IX86_BUILTIN_PSRLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
20797 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlqi", IX86_BUILTIN_PSRLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
20798 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlw", IX86_BUILTIN_PSRLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
20799 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrld", IX86_BUILTIN_PSRLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
20800 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlq", IX86_BUILTIN_PSRLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
20802 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psrawi", IX86_BUILTIN_PSRAWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
20803 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psradi", IX86_BUILTIN_PSRADI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
20804 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psraw", IX86_BUILTIN_PSRAW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
20805 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psrad", IX86_BUILTIN_PSRAD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
20808 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pf2id, "__builtin_ia32_pf2id", IX86_BUILTIN_PF2ID, UNKNOWN, (int) V2SI_FTYPE_V2SF },
20809 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_floatv2si2, "__builtin_ia32_pi2fd", IX86_BUILTIN_PI2FD, UNKNOWN, (int) V2SF_FTYPE_V2SI },
20810 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpv2sf2, "__builtin_ia32_pfrcp", IX86_BUILTIN_PFRCP, UNKNOWN, (int) V2SF_FTYPE_V2SF },
20811 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqrtv2sf2, "__builtin_ia32_pfrsqrt", IX86_BUILTIN_PFRSQRT, UNKNOWN, (int) V2SF_FTYPE_V2SF },
20813 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgusb", IX86_BUILTIN_PAVGUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
20814 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_haddv2sf3, "__builtin_ia32_pfacc", IX86_BUILTIN_PFACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
20815 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_addv2sf3, "__builtin_ia32_pfadd", IX86_BUILTIN_PFADD, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
20816 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_eqv2sf3, "__builtin_ia32_pfcmpeq", IX86_BUILTIN_PFCMPEQ, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
20817 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gev2sf3, "__builtin_ia32_pfcmpge", IX86_BUILTIN_PFCMPGE, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
20818 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gtv2sf3, "__builtin_ia32_pfcmpgt", IX86_BUILTIN_PFCMPGT, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
20819 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_smaxv2sf3, "__builtin_ia32_pfmax", IX86_BUILTIN_PFMAX, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
20820 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_sminv2sf3, "__builtin_ia32_pfmin", IX86_BUILTIN_PFMIN, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
20821 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_mulv2sf3, "__builtin_ia32_pfmul", IX86_BUILTIN_PFMUL, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
20822 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit1v2sf3, "__builtin_ia32_pfrcpit1", IX86_BUILTIN_PFRCPIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
20823 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit2v2sf3, "__builtin_ia32_pfrcpit2", IX86_BUILTIN_PFRCPIT2, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
20824 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqit1v2sf3, "__builtin_ia32_pfrsqit1", IX86_BUILTIN_PFRSQIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
20825 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subv2sf3, "__builtin_ia32_pfsub", IX86_BUILTIN_PFSUB, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
20826 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subrv2sf3, "__builtin_ia32_pfsubr", IX86_BUILTIN_PFSUBR, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
20827 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pmulhrwv4hi3, "__builtin_ia32_pmulhrw", IX86_BUILTIN_PMULHRW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
20830 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pf2iw, "__builtin_ia32_pf2iw", IX86_BUILTIN_PF2IW, UNKNOWN, (int) V2SI_FTYPE_V2SF },
20831 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pi2fw, "__builtin_ia32_pi2fw", IX86_BUILTIN_PI2FW, UNKNOWN, (int) V2SF_FTYPE_V2SI },
20832 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2si2, "__builtin_ia32_pswapdsi", IX86_BUILTIN_PSWAPDSI, UNKNOWN, (int) V2SI_FTYPE_V2SI },
20833 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2sf2, "__builtin_ia32_pswapdsf", IX86_BUILTIN_PSWAPDSF, UNKNOWN, (int) V2SF_FTYPE_V2SF },
20834 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_hsubv2sf3, "__builtin_ia32_pfnacc", IX86_BUILTIN_PFNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
20835 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_addsubv2sf3, "__builtin_ia32_pfpnacc", IX86_BUILTIN_PFPNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
20838 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movmskps, "__builtin_ia32_movmskps", IX86_BUILTIN_MOVMSKPS, UNKNOWN, (int) INT_FTYPE_V4SF },
20839 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_sqrtv4sf2, "__builtin_ia32_sqrtps", IX86_BUILTIN_SQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
20840 { OPTION_MASK_ISA_SSE, CODE_FOR_sqrtv4sf2, "__builtin_ia32_sqrtps_nr", IX86_BUILTIN_SQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
20841 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rsqrtv4sf2, "__builtin_ia32_rsqrtps", IX86_BUILTIN_RSQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
20842 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtv4sf2, "__builtin_ia32_rsqrtps_nr", IX86_BUILTIN_RSQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
20843 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rcpv4sf2, "__builtin_ia32_rcpps", IX86_BUILTIN_RCPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
20844 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtps2pi, "__builtin_ia32_cvtps2pi", IX86_BUILTIN_CVTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
20845 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtss2si, "__builtin_ia32_cvtss2si", IX86_BUILTIN_CVTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
20846 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtss2siq, "__builtin_ia32_cvtss2si64", IX86_BUILTIN_CVTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
20847 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttps2pi, "__builtin_ia32_cvttps2pi", IX86_BUILTIN_CVTTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
20848 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttss2si, "__builtin_ia32_cvttss2si", IX86_BUILTIN_CVTTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
20849 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvttss2siq, "__builtin_ia32_cvttss2si64", IX86_BUILTIN_CVTTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
20851 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_shufps, "__builtin_ia32_shufps", IX86_BUILTIN_SHUFPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
20853 { OPTION_MASK_ISA_SSE, CODE_FOR_addv4sf3, "__builtin_ia32_addps", IX86_BUILTIN_ADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20854 { OPTION_MASK_ISA_SSE, CODE_FOR_subv4sf3, "__builtin_ia32_subps", IX86_BUILTIN_SUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20855 { OPTION_MASK_ISA_SSE, CODE_FOR_mulv4sf3, "__builtin_ia32_mulps", IX86_BUILTIN_MULPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20856 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_divv4sf3, "__builtin_ia32_divps", IX86_BUILTIN_DIVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20857 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmaddv4sf3, "__builtin_ia32_addss", IX86_BUILTIN_ADDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20858 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsubv4sf3, "__builtin_ia32_subss", IX86_BUILTIN_SUBSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20859 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmulv4sf3, "__builtin_ia32_mulss", IX86_BUILTIN_MULSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20860 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmdivv4sf3, "__builtin_ia32_divss", IX86_BUILTIN_DIVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20862 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpeqps", IX86_BUILTIN_CMPEQPS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
20863 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpltps", IX86_BUILTIN_CMPLTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
20864 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpleps", IX86_BUILTIN_CMPLEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
20865 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgtps", IX86_BUILTIN_CMPGTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
20866 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgeps", IX86_BUILTIN_CMPGEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
20867 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpunordps", IX86_BUILTIN_CMPUNORDPS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
20868 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpneqps", IX86_BUILTIN_CMPNEQPS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
20869 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnltps", IX86_BUILTIN_CMPNLTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
20870 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnleps", IX86_BUILTIN_CMPNLEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
20871 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngtps", IX86_BUILTIN_CMPNGTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
20872 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngeps", IX86_BUILTIN_CMPNGEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP},
20873 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpordps", IX86_BUILTIN_CMPORDPS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
20874 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpeqss", IX86_BUILTIN_CMPEQSS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
20875 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpltss", IX86_BUILTIN_CMPLTSS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
20876 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpless", IX86_BUILTIN_CMPLESS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
20877 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpunordss", IX86_BUILTIN_CMPUNORDSS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
20878 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpneqss", IX86_BUILTIN_CMPNEQSS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
20879 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnltss", IX86_BUILTIN_CMPNLTSS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
20880 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnless", IX86_BUILTIN_CMPNLESS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
20881 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpngtss", IX86_BUILTIN_CMPNGTSS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
20882 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpngess", IX86_BUILTIN_CMPNGESS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
20883 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpordss", IX86_BUILTIN_CMPORDSS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
20885 { OPTION_MASK_ISA_SSE, CODE_FOR_sminv4sf3, "__builtin_ia32_minps", IX86_BUILTIN_MINPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20886 { OPTION_MASK_ISA_SSE, CODE_FOR_smaxv4sf3, "__builtin_ia32_maxps", IX86_BUILTIN_MAXPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20887 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsminv4sf3, "__builtin_ia32_minss", IX86_BUILTIN_MINSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20888 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsmaxv4sf3, "__builtin_ia32_maxss", IX86_BUILTIN_MAXSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20890 { OPTION_MASK_ISA_SSE, CODE_FOR_andv4sf3, "__builtin_ia32_andps", IX86_BUILTIN_ANDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20891 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_andnotv4sf3, "__builtin_ia32_andnps", IX86_BUILTIN_ANDNPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20892 { OPTION_MASK_ISA_SSE, CODE_FOR_iorv4sf3, "__builtin_ia32_orps", IX86_BUILTIN_ORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20893 { OPTION_MASK_ISA_SSE, CODE_FOR_xorv4sf3, "__builtin_ia32_xorps", IX86_BUILTIN_XORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20895 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movss, "__builtin_ia32_movss", IX86_BUILTIN_MOVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20896 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movhlps_exp, "__builtin_ia32_movhlps", IX86_BUILTIN_MOVHLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20897 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movlhps_exp, "__builtin_ia32_movlhps", IX86_BUILTIN_MOVLHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20898 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_unpckhps, "__builtin_ia32_unpckhps", IX86_BUILTIN_UNPCKHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20899 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_unpcklps, "__builtin_ia32_unpcklps", IX86_BUILTIN_UNPCKLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
20901 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtpi2ps, "__builtin_ia32_cvtpi2ps", IX86_BUILTIN_CVTPI2PS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2SI },
20902 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtsi2ss, "__builtin_ia32_cvtsi2ss", IX86_BUILTIN_CVTSI2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_SI },
20903 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtsi2ssq, "__builtin_ia32_cvtsi642ss", IX86_BUILTIN_CVTSI642SS, UNKNOWN, V4SF_FTYPE_V4SF_DI },
20905 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtsf2, "__builtin_ia32_rsqrtf", IX86_BUILTIN_RSQRTF, UNKNOWN, (int) FLOAT_FTYPE_FLOAT },
20907 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsqrtv4sf2, "__builtin_ia32_sqrtss", IX86_BUILTIN_SQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
20908 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrsqrtv4sf2, "__builtin_ia32_rsqrtss", IX86_BUILTIN_RSQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
20909 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrcpv4sf2, "__builtin_ia32_rcpss", IX86_BUILTIN_RCPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
20911 /* SSE MMX or 3Dnow!A */
20912 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgb", IX86_BUILTIN_PAVGB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
20913 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv4hi3, "__builtin_ia32_pavgw", IX86_BUILTIN_PAVGW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
20914 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umulv4hi3_highpart, "__builtin_ia32_pmulhuw", IX86_BUILTIN_PMULHUW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
20916 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umaxv8qi3, "__builtin_ia32_pmaxub", IX86_BUILTIN_PMAXUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
20917 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_smaxv4hi3, "__builtin_ia32_pmaxsw", IX86_BUILTIN_PMAXSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
20918 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uminv8qi3, "__builtin_ia32_pminub", IX86_BUILTIN_PMINUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
20919 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_sminv4hi3, "__builtin_ia32_pminsw", IX86_BUILTIN_PMINSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
20921 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_psadbw, "__builtin_ia32_psadbw", IX86_BUILTIN_PSADBW, UNKNOWN, (int) V1DI_FTYPE_V8QI_V8QI },
20922 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pmovmskb, "__builtin_ia32_pmovmskb", IX86_BUILTIN_PMOVMSKB, UNKNOWN, (int) INT_FTYPE_V8QI },
20924 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pshufw, "__builtin_ia32_pshufw", IX86_BUILTIN_PSHUFW, UNKNOWN, (int) V4HI_FTYPE_V4HI_INT },
20927 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_shufpd, "__builtin_ia32_shufpd", IX86_BUILTIN_SHUFPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
20929 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movmskpd, "__builtin_ia32_movmskpd", IX86_BUILTIN_MOVMSKPD, UNKNOWN, (int) INT_FTYPE_V2DF },
20930 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmovmskb, "__builtin_ia32_pmovmskb128", IX86_BUILTIN_PMOVMSKB128, UNKNOWN, (int) INT_FTYPE_V16QI },
20931 { OPTION_MASK_ISA_SSE2, CODE_FOR_sqrtv2df2, "__builtin_ia32_sqrtpd", IX86_BUILTIN_SQRTPD, UNKNOWN, (int) V2DF_FTYPE_V2DF },
20932 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtdq2pd, "__builtin_ia32_cvtdq2pd", IX86_BUILTIN_CVTDQ2PD, UNKNOWN, (int) V2DF_FTYPE_V4SI },
20933 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtdq2ps, "__builtin_ia32_cvtdq2ps", IX86_BUILTIN_CVTDQ2PS, UNKNOWN, (int) V4SF_FTYPE_V4SI },
20935 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2dq, "__builtin_ia32_cvtpd2dq", IX86_BUILTIN_CVTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
20936 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2pi, "__builtin_ia32_cvtpd2pi", IX86_BUILTIN_CVTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
20937 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2ps, "__builtin_ia32_cvtpd2ps", IX86_BUILTIN_CVTPD2PS, UNKNOWN, (int) V4SF_FTYPE_V2DF },
20938 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2dq, "__builtin_ia32_cvttpd2dq", IX86_BUILTIN_CVTTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
20939 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2pi, "__builtin_ia32_cvttpd2pi", IX86_BUILTIN_CVTTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
20941 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpi2pd, "__builtin_ia32_cvtpi2pd", IX86_BUILTIN_CVTPI2PD, UNKNOWN, (int) V2DF_FTYPE_V2SI },
20943 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2si, "__builtin_ia32_cvtsd2si", IX86_BUILTIN_CVTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
20944 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttsd2si, "__builtin_ia32_cvttsd2si", IX86_BUILTIN_CVTTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
20945 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsd2siq, "__builtin_ia32_cvtsd2si64", IX86_BUILTIN_CVTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
20946 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvttsd2siq, "__builtin_ia32_cvttsd2si64", IX86_BUILTIN_CVTTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
20948 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2dq, "__builtin_ia32_cvtps2dq", IX86_BUILTIN_CVTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
20949 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2pd, "__builtin_ia32_cvtps2pd", IX86_BUILTIN_CVTPS2PD, UNKNOWN, (int) V2DF_FTYPE_V4SF },
20950 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttps2dq, "__builtin_ia32_cvttps2dq", IX86_BUILTIN_CVTTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
20952 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2df3, "__builtin_ia32_addpd", IX86_BUILTIN_ADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20953 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2df3, "__builtin_ia32_subpd", IX86_BUILTIN_SUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20954 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv2df3, "__builtin_ia32_mulpd", IX86_BUILTIN_MULPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20955 { OPTION_MASK_ISA_SSE2, CODE_FOR_divv2df3, "__builtin_ia32_divpd", IX86_BUILTIN_DIVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20956 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmaddv2df3, "__builtin_ia32_addsd", IX86_BUILTIN_ADDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20957 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsubv2df3, "__builtin_ia32_subsd", IX86_BUILTIN_SUBSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20958 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmulv2df3, "__builtin_ia32_mulsd", IX86_BUILTIN_MULSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20959 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmdivv2df3, "__builtin_ia32_divsd", IX86_BUILTIN_DIVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20961 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpeqpd", IX86_BUILTIN_CMPEQPD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
20962 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpltpd", IX86_BUILTIN_CMPLTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
20963 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmplepd", IX86_BUILTIN_CMPLEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
20964 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgtpd", IX86_BUILTIN_CMPGTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
20965 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgepd", IX86_BUILTIN_CMPGEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP},
20966 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpunordpd", IX86_BUILTIN_CMPUNORDPD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
20967 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpneqpd", IX86_BUILTIN_CMPNEQPD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
20968 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnltpd", IX86_BUILTIN_CMPNLTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
20969 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnlepd", IX86_BUILTIN_CMPNLEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
20970 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngtpd", IX86_BUILTIN_CMPNGTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
20971 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngepd", IX86_BUILTIN_CMPNGEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
20972 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpordpd", IX86_BUILTIN_CMPORDPD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
20973 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpeqsd", IX86_BUILTIN_CMPEQSD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
20974 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpltsd", IX86_BUILTIN_CMPLTSD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
20975 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmplesd", IX86_BUILTIN_CMPLESD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
20976 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpunordsd", IX86_BUILTIN_CMPUNORDSD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
20977 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpneqsd", IX86_BUILTIN_CMPNEQSD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
20978 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnltsd", IX86_BUILTIN_CMPNLTSD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
20979 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnlesd", IX86_BUILTIN_CMPNLESD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
20980 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpordsd", IX86_BUILTIN_CMPORDSD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
20982 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv2df3, "__builtin_ia32_minpd", IX86_BUILTIN_MINPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20983 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv2df3, "__builtin_ia32_maxpd", IX86_BUILTIN_MAXPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20984 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsminv2df3, "__builtin_ia32_minsd", IX86_BUILTIN_MINSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20985 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsmaxv2df3, "__builtin_ia32_maxsd", IX86_BUILTIN_MAXSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20987 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2df3, "__builtin_ia32_andpd", IX86_BUILTIN_ANDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20988 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2df3, "__builtin_ia32_andnpd", IX86_BUILTIN_ANDNPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20989 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2df3, "__builtin_ia32_orpd", IX86_BUILTIN_ORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20990 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2df3, "__builtin_ia32_xorpd", IX86_BUILTIN_XORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20992 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movsd, "__builtin_ia32_movsd", IX86_BUILTIN_MOVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20993 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_unpckhpd_exp, "__builtin_ia32_unpckhpd", IX86_BUILTIN_UNPCKHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20994 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_unpcklpd_exp, "__builtin_ia32_unpcklpd", IX86_BUILTIN_UNPCKLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
20996 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_pack_sfix_v2df, "__builtin_ia32_vec_pack_sfix", IX86_BUILTIN_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
20998 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv16qi3, "__builtin_ia32_paddb128", IX86_BUILTIN_PADDB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
20999 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv8hi3, "__builtin_ia32_paddw128", IX86_BUILTIN_PADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21000 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv4si3, "__builtin_ia32_paddd128", IX86_BUILTIN_PADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21001 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2di3, "__builtin_ia32_paddq128", IX86_BUILTIN_PADDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21002 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv16qi3, "__builtin_ia32_psubb128", IX86_BUILTIN_PSUBB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21003 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv8hi3, "__builtin_ia32_psubw128", IX86_BUILTIN_PSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21004 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv4si3, "__builtin_ia32_psubd128", IX86_BUILTIN_PSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21005 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2di3, "__builtin_ia32_psubq128", IX86_BUILTIN_PSUBQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21007 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv16qi3, "__builtin_ia32_paddsb128", IX86_BUILTIN_PADDSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21008 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv8hi3, "__builtin_ia32_paddsw128", IX86_BUILTIN_PADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21009 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv16qi3, "__builtin_ia32_psubsb128", IX86_BUILTIN_PSUBSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21010 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv8hi3, "__builtin_ia32_psubsw128", IX86_BUILTIN_PSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21011 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv16qi3, "__builtin_ia32_paddusb128", IX86_BUILTIN_PADDUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21012 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv8hi3, "__builtin_ia32_paddusw128", IX86_BUILTIN_PADDUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21013 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv16qi3, "__builtin_ia32_psubusb128", IX86_BUILTIN_PSUBUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21014 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv8hi3, "__builtin_ia32_psubusw128", IX86_BUILTIN_PSUBUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21016 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv8hi3, "__builtin_ia32_pmullw128", IX86_BUILTIN_PMULLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21017 { OPTION_MASK_ISA_SSE2, CODE_FOR_smulv8hi3_highpart, "__builtin_ia32_pmulhw128", IX86_BUILTIN_PMULHW128, UNKNOWN,(int) V8HI_FTYPE_V8HI_V8HI },
21019 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2di3, "__builtin_ia32_pand128", IX86_BUILTIN_PAND128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21020 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2di3, "__builtin_ia32_pandn128", IX86_BUILTIN_PANDN128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21021 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2di3, "__builtin_ia32_por128", IX86_BUILTIN_POR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21022 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2di3, "__builtin_ia32_pxor128", IX86_BUILTIN_PXOR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21024 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv16qi3, "__builtin_ia32_pavgb128", IX86_BUILTIN_PAVGB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21025 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv8hi3, "__builtin_ia32_pavgw128", IX86_BUILTIN_PAVGW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21027 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv16qi3, "__builtin_ia32_pcmpeqb128", IX86_BUILTIN_PCMPEQB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21028 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv8hi3, "__builtin_ia32_pcmpeqw128", IX86_BUILTIN_PCMPEQW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21029 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv4si3, "__builtin_ia32_pcmpeqd128", IX86_BUILTIN_PCMPEQD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21030 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv16qi3, "__builtin_ia32_pcmpgtb128", IX86_BUILTIN_PCMPGTB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21031 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv8hi3, "__builtin_ia32_pcmpgtw128", IX86_BUILTIN_PCMPGTW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21032 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv4si3, "__builtin_ia32_pcmpgtd128", IX86_BUILTIN_PCMPGTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21034 { OPTION_MASK_ISA_SSE2, CODE_FOR_umaxv16qi3, "__builtin_ia32_pmaxub128", IX86_BUILTIN_PMAXUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21035 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv8hi3, "__builtin_ia32_pmaxsw128", IX86_BUILTIN_PMAXSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21036 { OPTION_MASK_ISA_SSE2, CODE_FOR_uminv16qi3, "__builtin_ia32_pminub128", IX86_BUILTIN_PMINUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21037 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv8hi3, "__builtin_ia32_pminsw128", IX86_BUILTIN_PMINSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21039 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_punpckhbw, "__builtin_ia32_punpckhbw128", IX86_BUILTIN_PUNPCKHBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21040 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_punpckhwd, "__builtin_ia32_punpckhwd128", IX86_BUILTIN_PUNPCKHWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21041 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_punpckhdq, "__builtin_ia32_punpckhdq128", IX86_BUILTIN_PUNPCKHDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21042 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_punpckhqdq, "__builtin_ia32_punpckhqdq128", IX86_BUILTIN_PUNPCKHQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21043 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_punpcklbw, "__builtin_ia32_punpcklbw128", IX86_BUILTIN_PUNPCKLBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21044 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_punpcklwd, "__builtin_ia32_punpcklwd128", IX86_BUILTIN_PUNPCKLWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21045 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_punpckldq, "__builtin_ia32_punpckldq128", IX86_BUILTIN_PUNPCKLDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21046 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_punpcklqdq, "__builtin_ia32_punpcklqdq128", IX86_BUILTIN_PUNPCKLQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21048 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packsswb, "__builtin_ia32_packsswb128", IX86_BUILTIN_PACKSSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
21049 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packssdw, "__builtin_ia32_packssdw128", IX86_BUILTIN_PACKSSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
21050 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packuswb, "__builtin_ia32_packuswb128", IX86_BUILTIN_PACKUSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
21052 { OPTION_MASK_ISA_SSE2, CODE_FOR_umulv8hi3_highpart, "__builtin_ia32_pmulhuw128", IX86_BUILTIN_PMULHUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21053 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_psadbw, "__builtin_ia32_psadbw128", IX86_BUILTIN_PSADBW128, UNKNOWN, (int) V2DI_FTYPE_V16QI_V16QI },
21055 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_umulv1siv1di3, "__builtin_ia32_pmuludq", IX86_BUILTIN_PMULUDQ, UNKNOWN, (int) V1DI_FTYPE_V2SI_V2SI },
21056 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_umulv2siv2di3, "__builtin_ia32_pmuludq128", IX86_BUILTIN_PMULUDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
21058 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmaddwd, "__builtin_ia32_pmaddwd128", IX86_BUILTIN_PMADDWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI_V8HI },
21060 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsi2sd, "__builtin_ia32_cvtsi2sd", IX86_BUILTIN_CVTSI2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_SI },
21061 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsi2sdq, "__builtin_ia32_cvtsi642sd", IX86_BUILTIN_CVTSI642SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_DI },
21062 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2ss, "__builtin_ia32_cvtsd2ss", IX86_BUILTIN_CVTSD2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2DF },
21063 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtss2sd, "__builtin_ia32_cvtss2sd", IX86_BUILTIN_CVTSS2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V4SF },
21065 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ashlti3, "__builtin_ia32_pslldqi128", IX86_BUILTIN_PSLLDQI128, UNKNOWN, (int) V2DI2TI_FTYPE_V2DI_INT },
21066 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllwi128", IX86_BUILTIN_PSLLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
21067 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslldi128", IX86_BUILTIN_PSLLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
21068 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllqi128", IX86_BUILTIN_PSLLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
21069 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllw128", IX86_BUILTIN_PSLLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
21070 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslld128", IX86_BUILTIN_PSLLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
21071 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllq128", IX86_BUILTIN_PSLLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
21073 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lshrti3, "__builtin_ia32_psrldqi128", IX86_BUILTIN_PSRLDQI128, UNKNOWN, (int) V2DI2TI_FTYPE_V2DI_INT },
21074 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlwi128", IX86_BUILTIN_PSRLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
21075 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrldi128", IX86_BUILTIN_PSRLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
21076 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlqi128", IX86_BUILTIN_PSRLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
21077 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlw128", IX86_BUILTIN_PSRLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
21078 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrld128", IX86_BUILTIN_PSRLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
21079 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlq128", IX86_BUILTIN_PSRLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
21081 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psrawi128", IX86_BUILTIN_PSRAWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
21082 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psradi128", IX86_BUILTIN_PSRADI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
21083 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psraw128", IX86_BUILTIN_PSRAW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
21084 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psrad128", IX86_BUILTIN_PSRAD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
21086 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufd, "__builtin_ia32_pshufd", IX86_BUILTIN_PSHUFD, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT },
21087 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshuflw, "__builtin_ia32_pshuflw", IX86_BUILTIN_PSHUFLW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
21088 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufhw, "__builtin_ia32_pshufhw", IX86_BUILTIN_PSHUFHW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
21090 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsqrtv2df2, "__builtin_ia32_sqrtsd", IX86_BUILTIN_SQRTSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_VEC_MERGE },
21092 { OPTION_MASK_ISA_SSE2, CODE_FOR_abstf2, 0, IX86_BUILTIN_FABSQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128 },
21093 { OPTION_MASK_ISA_SSE2, CODE_FOR_copysigntf3, 0, IX86_BUILTIN_COPYSIGNQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128_FLOAT128 },
21095 { OPTION_MASK_ISA_SSE, CODE_FOR_sse2_movq128, "__builtin_ia32_movq128", IX86_BUILTIN_MOVQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
21098 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_addv1di3, "__builtin_ia32_paddq", IX86_BUILTIN_PADDQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
21099 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_subv1di3, "__builtin_ia32_psubq", IX86_BUILTIN_PSUBQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
21102 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movshdup, "__builtin_ia32_movshdup", IX86_BUILTIN_MOVSHDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF},
21103 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movsldup, "__builtin_ia32_movsldup", IX86_BUILTIN_MOVSLDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF },
21105 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv4sf3, "__builtin_ia32_addsubps", IX86_BUILTIN_ADDSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21106 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv2df3, "__builtin_ia32_addsubpd", IX86_BUILTIN_ADDSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21107 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv4sf3, "__builtin_ia32_haddps", IX86_BUILTIN_HADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21108 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv2df3, "__builtin_ia32_haddpd", IX86_BUILTIN_HADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21109 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv4sf3, "__builtin_ia32_hsubps", IX86_BUILTIN_HSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21110 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv2df3, "__builtin_ia32_hsubpd", IX86_BUILTIN_HSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21113 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv16qi2, "__builtin_ia32_pabsb128", IX86_BUILTIN_PABSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI },
21114 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8qi2, "__builtin_ia32_pabsb", IX86_BUILTIN_PABSB, UNKNOWN, (int) V8QI_FTYPE_V8QI },
21115 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8hi2, "__builtin_ia32_pabsw128", IX86_BUILTIN_PABSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
21116 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4hi2, "__builtin_ia32_pabsw", IX86_BUILTIN_PABSW, UNKNOWN, (int) V4HI_FTYPE_V4HI },
21117 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4si2, "__builtin_ia32_pabsd128", IX86_BUILTIN_PABSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
21118 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv2si2, "__builtin_ia32_pabsd", IX86_BUILTIN_PABSD, UNKNOWN, (int) V2SI_FTYPE_V2SI },
21120 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv8hi3, "__builtin_ia32_phaddw128", IX86_BUILTIN_PHADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21121 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv4hi3, "__builtin_ia32_phaddw", IX86_BUILTIN_PHADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21122 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv4si3, "__builtin_ia32_phaddd128", IX86_BUILTIN_PHADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21123 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv2si3, "__builtin_ia32_phaddd", IX86_BUILTIN_PHADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
21124 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv8hi3, "__builtin_ia32_phaddsw128", IX86_BUILTIN_PHADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21125 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv4hi3, "__builtin_ia32_phaddsw", IX86_BUILTIN_PHADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21126 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv8hi3, "__builtin_ia32_phsubw128", IX86_BUILTIN_PHSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21127 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv4hi3, "__builtin_ia32_phsubw", IX86_BUILTIN_PHSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21128 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv4si3, "__builtin_ia32_phsubd128", IX86_BUILTIN_PHSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21129 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv2si3, "__builtin_ia32_phsubd", IX86_BUILTIN_PHSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
21130 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv8hi3, "__builtin_ia32_phsubsw128", IX86_BUILTIN_PHSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21131 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv4hi3, "__builtin_ia32_phsubsw", IX86_BUILTIN_PHSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21132 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw128, "__builtin_ia32_pmaddubsw128", IX86_BUILTIN_PMADDUBSW128, UNKNOWN, (int) V8HI_FTYPE_V16QI_V16QI },
21133 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw, "__builtin_ia32_pmaddubsw", IX86_BUILTIN_PMADDUBSW, UNKNOWN, (int) V4HI_FTYPE_V8QI_V8QI },
21134 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv8hi3, "__builtin_ia32_pmulhrsw128", IX86_BUILTIN_PMULHRSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21135 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv4hi3, "__builtin_ia32_pmulhrsw", IX86_BUILTIN_PMULHRSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21136 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv16qi3, "__builtin_ia32_pshufb128", IX86_BUILTIN_PSHUFB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21137 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv8qi3, "__builtin_ia32_pshufb", IX86_BUILTIN_PSHUFB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
21138 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv16qi3, "__builtin_ia32_psignb128", IX86_BUILTIN_PSIGNB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21139 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8qi3, "__builtin_ia32_psignb", IX86_BUILTIN_PSIGNB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
21140 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8hi3, "__builtin_ia32_psignw128", IX86_BUILTIN_PSIGNW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21141 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4hi3, "__builtin_ia32_psignw", IX86_BUILTIN_PSIGNW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21142 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4si3, "__builtin_ia32_psignd128", IX86_BUILTIN_PSIGND128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21143 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv2si3, "__builtin_ia32_psignd", IX86_BUILTIN_PSIGND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
21146 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrti, "__builtin_ia32_palignr128", IX86_BUILTIN_PALIGNR128, UNKNOWN, (int) V2DI2TI_FTYPE_V2DI_V2DI_INT },
21147 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrdi, "__builtin_ia32_palignr", IX86_BUILTIN_PALIGNR, UNKNOWN, (int) V1DI2DI_FTYPE_V1DI_V1DI_INT },
21150 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendpd, "__builtin_ia32_blendpd", IX86_BUILTIN_BLENDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
21151 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendps, "__builtin_ia32_blendps", IX86_BUILTIN_BLENDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
21152 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvpd, "__builtin_ia32_blendvpd", IX86_BUILTIN_BLENDVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF },
21153 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvps, "__builtin_ia32_blendvps", IX86_BUILTIN_BLENDVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF },
21154 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dppd, "__builtin_ia32_dppd", IX86_BUILTIN_DPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
21155 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dpps, "__builtin_ia32_dpps", IX86_BUILTIN_DPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
21156 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_insertps, "__builtin_ia32_insertps128", IX86_BUILTIN_INSERTPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
21157 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mpsadbw, "__builtin_ia32_mpsadbw128", IX86_BUILTIN_MPSADBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_INT },
21158 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendvb, "__builtin_ia32_pblendvb128", IX86_BUILTIN_PBLENDVB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI },
21159 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendw, "__builtin_ia32_pblendw128", IX86_BUILTIN_PBLENDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_INT },
21161 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_extendv8qiv8hi2, "__builtin_ia32_pmovsxbw128", IX86_BUILTIN_PMOVSXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
21162 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_extendv4qiv4si2, "__builtin_ia32_pmovsxbd128", IX86_BUILTIN_PMOVSXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
21163 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_extendv2qiv2di2, "__builtin_ia32_pmovsxbq128", IX86_BUILTIN_PMOVSXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
21164 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_extendv4hiv4si2, "__builtin_ia32_pmovsxwd128", IX86_BUILTIN_PMOVSXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
21165 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_extendv2hiv2di2, "__builtin_ia32_pmovsxwq128", IX86_BUILTIN_PMOVSXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
21166 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_extendv2siv2di2, "__builtin_ia32_pmovsxdq128", IX86_BUILTIN_PMOVSXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
21167 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv8qiv8hi2, "__builtin_ia32_pmovzxbw128", IX86_BUILTIN_PMOVZXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
21168 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4qiv4si2, "__builtin_ia32_pmovzxbd128", IX86_BUILTIN_PMOVZXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
21169 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2qiv2di2, "__builtin_ia32_pmovzxbq128", IX86_BUILTIN_PMOVZXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
21170 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4hiv4si2, "__builtin_ia32_pmovzxwd128", IX86_BUILTIN_PMOVZXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
21171 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2hiv2di2, "__builtin_ia32_pmovzxwq128", IX86_BUILTIN_PMOVZXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
21172 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2siv2di2, "__builtin_ia32_pmovzxdq128", IX86_BUILTIN_PMOVZXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
21173 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_phminposuw, "__builtin_ia32_phminposuw128", IX86_BUILTIN_PHMINPOSUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
21175 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_packusdw, "__builtin_ia32_packusdw128", IX86_BUILTIN_PACKUSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
21176 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_eqv2di3, "__builtin_ia32_pcmpeqq", IX86_BUILTIN_PCMPEQQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21177 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv16qi3, "__builtin_ia32_pmaxsb128", IX86_BUILTIN_PMAXSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21178 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv4si3, "__builtin_ia32_pmaxsd128", IX86_BUILTIN_PMAXSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21179 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv4si3, "__builtin_ia32_pmaxud128", IX86_BUILTIN_PMAXUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21180 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv8hi3, "__builtin_ia32_pmaxuw128", IX86_BUILTIN_PMAXUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21181 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv16qi3, "__builtin_ia32_pminsb128", IX86_BUILTIN_PMINSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21182 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv4si3, "__builtin_ia32_pminsd128", IX86_BUILTIN_PMINSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21183 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv4si3, "__builtin_ia32_pminud128", IX86_BUILTIN_PMINUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21184 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv8hi3, "__builtin_ia32_pminuw128", IX86_BUILTIN_PMINUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21185 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mulv2siv2di3, "__builtin_ia32_pmuldq128", IX86_BUILTIN_PMULDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
21186 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_mulv4si3, "__builtin_ia32_pmulld128", IX86_BUILTIN_PMULLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21188 /* SSE4.1 and SSE5 */
21189 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_roundpd", IX86_BUILTIN_ROUNDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
21190 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_roundps", IX86_BUILTIN_ROUNDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
21191 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundsd, "__builtin_ia32_roundsd", IX86_BUILTIN_ROUNDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
21192 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundss, "__builtin_ia32_roundss", IX86_BUILTIN_ROUNDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
21194 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestz128", IX86_BUILTIN_PTESTZ, EQ, (int) INT_FTYPE_V2DI_V2DI_PTEST },
21195 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestc128", IX86_BUILTIN_PTESTC, LTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
21196 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestnzc128", IX86_BUILTIN_PTESTNZC, GTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
21199 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_gtv2di3, "__builtin_ia32_pcmpgtq", IX86_BUILTIN_PCMPGTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21200 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_crc32qi, "__builtin_ia32_crc32qi", IX86_BUILTIN_CRC32QI, UNKNOWN, (int) UINT_FTYPE_UINT_UCHAR },
21201 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_crc32hi, "__builtin_ia32_crc32hi", IX86_BUILTIN_CRC32HI, UNKNOWN, (int) UINT_FTYPE_UINT_USHORT },
21202 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_crc32si, "__builtin_ia32_crc32si", IX86_BUILTIN_CRC32SI, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
21203 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse4_2_crc32di, "__builtin_ia32_crc32di", IX86_BUILTIN_CRC32DI, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
21206 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrqi, "__builtin_ia32_extrqi", IX86_BUILTIN_EXTRQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_UINT_UINT },
21207 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrq, "__builtin_ia32_extrq", IX86_BUILTIN_EXTRQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V16QI },
21208 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertqi, "__builtin_ia32_insertqi", IX86_BUILTIN_INSERTQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_UINT_UINT },
21209 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertq, "__builtin_ia32_insertq", IX86_BUILTIN_INSERTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21212 { OPTION_MASK_ISA_SSE2, CODE_FOR_aeskeygenassist, 0, IX86_BUILTIN_AESKEYGENASSIST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT },
21213 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesimc, 0, IX86_BUILTIN_AESIMC128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
21215 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenc, 0, IX86_BUILTIN_AESENC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21216 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenclast, 0, IX86_BUILTIN_AESENCLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21217 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdec, 0, IX86_BUILTIN_AESDEC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21218 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdeclast, 0, IX86_BUILTIN_AESDECLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21221 { OPTION_MASK_ISA_SSE2, CODE_FOR_pclmulqdq, 0, IX86_BUILTIN_PCLMULQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT },
21224 { OPTION_MASK_ISA_AVX, CODE_FOR_addv4df3, "__builtin_ia32_addpd256", IX86_BUILTIN_ADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21225 { OPTION_MASK_ISA_AVX, CODE_FOR_addv8sf3, "__builtin_ia32_addps256", IX86_BUILTIN_ADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21226 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv4df3, "__builtin_ia32_addsubpd256", IX86_BUILTIN_ADDSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21227 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv8sf3, "__builtin_ia32_addsubps256", IX86_BUILTIN_ADDSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21228 { OPTION_MASK_ISA_AVX, CODE_FOR_andv4df3, "__builtin_ia32_andpd256", IX86_BUILTIN_ANDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21229 { OPTION_MASK_ISA_AVX, CODE_FOR_andv8sf3, "__builtin_ia32_andps256", IX86_BUILTIN_ANDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21230 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv4df3, "__builtin_ia32_andnpd256", IX86_BUILTIN_ANDNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21231 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv8sf3, "__builtin_ia32_andnps256", IX86_BUILTIN_ANDNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21232 { OPTION_MASK_ISA_AVX, CODE_FOR_divv4df3, "__builtin_ia32_divpd256", IX86_BUILTIN_DIVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21233 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_divv8sf3, "__builtin_ia32_divps256", IX86_BUILTIN_DIVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21234 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv4df3, "__builtin_ia32_haddpd256", IX86_BUILTIN_HADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21235 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv8sf3, "__builtin_ia32_hsubps256", IX86_BUILTIN_HSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21236 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv4df3, "__builtin_ia32_hsubpd256", IX86_BUILTIN_HSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21237 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv8sf3, "__builtin_ia32_haddps256", IX86_BUILTIN_HADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21238 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv4df3, "__builtin_ia32_maxpd256", IX86_BUILTIN_MAXPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21239 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv8sf3, "__builtin_ia32_maxps256", IX86_BUILTIN_MAXPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21240 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv4df3, "__builtin_ia32_minpd256", IX86_BUILTIN_MINPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21241 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv8sf3, "__builtin_ia32_minps256", IX86_BUILTIN_MINPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21242 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv4df3, "__builtin_ia32_mulpd256", IX86_BUILTIN_MULPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21243 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv8sf3, "__builtin_ia32_mulps256", IX86_BUILTIN_MULPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21244 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv4df3, "__builtin_ia32_orpd256", IX86_BUILTIN_ORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21245 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv8sf3, "__builtin_ia32_orps256", IX86_BUILTIN_ORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21246 { OPTION_MASK_ISA_AVX, CODE_FOR_subv4df3, "__builtin_ia32_subpd256", IX86_BUILTIN_SUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21247 { OPTION_MASK_ISA_AVX, CODE_FOR_subv8sf3, "__builtin_ia32_subps256", IX86_BUILTIN_SUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21248 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv4df3, "__builtin_ia32_xorpd256", IX86_BUILTIN_XORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21249 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv8sf3, "__builtin_ia32_xorps256", IX86_BUILTIN_XORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21251 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv2df3, "__builtin_ia32_vpermilvarpd", IX86_BUILTIN_VPERMILVARPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DI },
21252 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4sf3, "__builtin_ia32_vpermilvarps", IX86_BUILTIN_VPERMILVARPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SI },
21253 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4df3, "__builtin_ia32_vpermilvarpd256", IX86_BUILTIN_VPERMILVARPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DI },
21254 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv8sf3, "__builtin_ia32_vpermilvarps256", IX86_BUILTIN_VPERMILVARPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI },
21256 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendpd256, "__builtin_ia32_blendpd256", IX86_BUILTIN_BLENDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
21257 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendps256, "__builtin_ia32_blendps256", IX86_BUILTIN_BLENDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
21258 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvpd256, "__builtin_ia32_blendvpd256", IX86_BUILTIN_BLENDVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF },
21259 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvps256, "__builtin_ia32_blendvps256", IX86_BUILTIN_BLENDVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF },
21260 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_dpps256, "__builtin_ia32_dpps256", IX86_BUILTIN_DPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
21261 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufpd256, "__builtin_ia32_shufpd256", IX86_BUILTIN_SHUFPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
21262 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufps256, "__builtin_ia32_shufps256", IX86_BUILTIN_SHUFPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
21263 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpsdv2df3, "__builtin_ia32_cmpsd", IX86_BUILTIN_CMPSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
21264 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpssv4sf3, "__builtin_ia32_cmpss", IX86_BUILTIN_CMPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
21265 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmppdv2df3, "__builtin_ia32_cmppd", IX86_BUILTIN_CMPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
21266 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmppsv4sf3, "__builtin_ia32_cmpps", IX86_BUILTIN_CMPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
21267 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmppdv4df3, "__builtin_ia32_cmppd256", IX86_BUILTIN_CMPPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
21268 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmppsv8sf3, "__builtin_ia32_cmpps256", IX86_BUILTIN_CMPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
21269 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v4df, "__builtin_ia32_vextractf128_pd256", IX86_BUILTIN_EXTRACTF128PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF_INT },
21270 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8sf, "__builtin_ia32_vextractf128_ps256", IX86_BUILTIN_EXTRACTF128PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF_INT },
21271 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8si, "__builtin_ia32_vextractf128_si256", IX86_BUILTIN_EXTRACTF128SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI_INT },
21272 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtdq2pd256, "__builtin_ia32_cvtdq2pd256", IX86_BUILTIN_CVTDQ2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SI },
21273 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtdq2ps256, "__builtin_ia32_cvtdq2ps256", IX86_BUILTIN_CVTDQ2PS256, UNKNOWN, (int) V8SF_FTYPE_V8SI },
21274 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2ps256, "__builtin_ia32_cvtpd2ps256", IX86_BUILTIN_CVTPD2PS256, UNKNOWN, (int) V4SF_FTYPE_V4DF },
21275 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2dq256, "__builtin_ia32_cvtps2dq256", IX86_BUILTIN_CVTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
21276 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2pd256, "__builtin_ia32_cvtps2pd256", IX86_BUILTIN_CVTPS2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SF },
21277 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvttpd2dq256, "__builtin_ia32_cvttpd2dq256", IX86_BUILTIN_CVTTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
21278 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2dq256, "__builtin_ia32_cvtpd2dq256", IX86_BUILTIN_CVTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
21279 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvttps2dq256, "__builtin_ia32_cvttps2dq256", IX86_BUILTIN_CVTTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
21280 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v4df3, "__builtin_ia32_vperm2f128_pd256", IX86_BUILTIN_VPERM2F128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
21281 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8sf3, "__builtin_ia32_vperm2f128_ps256", IX86_BUILTIN_VPERM2F128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
21282 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8si3, "__builtin_ia32_vperm2f128_si256", IX86_BUILTIN_VPERM2F128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
21283 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv2df, "__builtin_ia32_vpermilpd", IX86_BUILTIN_VPERMILPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
21284 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4sf, "__builtin_ia32_vpermilps", IX86_BUILTIN_VPERMILPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
21285 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4df, "__builtin_ia32_vpermilpd256", IX86_BUILTIN_VPERMILPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
21286 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv8sf, "__builtin_ia32_vpermilps256", IX86_BUILTIN_VPERMILPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
21287 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v4df, "__builtin_ia32_vinsertf128_pd256", IX86_BUILTIN_VINSERTF128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V2DF_INT },
21288 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8sf, "__builtin_ia32_vinsertf128_ps256", IX86_BUILTIN_VINSERTF128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V4SF_INT },
21289 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8si, "__builtin_ia32_vinsertf128_si256", IX86_BUILTIN_VINSERTF128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_INT },
21291 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movshdup256, "__builtin_ia32_movshdup256", IX86_BUILTIN_MOVSHDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
21292 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movsldup256, "__builtin_ia32_movsldup256", IX86_BUILTIN_MOVSLDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
21293 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movddup256, "__builtin_ia32_movddup256", IX86_BUILTIN_MOVDDUP256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
21295 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv4df2, "__builtin_ia32_sqrtpd256", IX86_BUILTIN_SQRTPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
21296 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_sqrtv8sf2, "__builtin_ia32_sqrtps256", IX86_BUILTIN_SQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
21297 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv8sf2, "__builtin_ia32_sqrtps_nr256", IX86_BUILTIN_SQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
21298 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rsqrtv8sf2, "__builtin_ia32_rsqrtps256", IX86_BUILTIN_RSQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
21299 { OPTION_MASK_ISA_AVX, CODE_FOR_rsqrtv8sf2, "__builtin_ia32_rsqrtps_nr256", IX86_BUILTIN_RSQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
21301 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rcpv8sf2, "__builtin_ia32_rcpps256", IX86_BUILTIN_RCPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
21303 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_roundpd256", IX86_BUILTIN_ROUNDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
21304 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_roundps256", IX86_BUILTIN_ROUNDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
21306 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhpd256, "__builtin_ia32_unpckhpd256", IX86_BUILTIN_UNPCKHPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21307 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklpd256, "__builtin_ia32_unpcklpd256", IX86_BUILTIN_UNPCKLPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21308 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhps256, "__builtin_ia32_unpckhps256", IX86_BUILTIN_UNPCKHPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21309 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklps256, "__builtin_ia32_unpcklps256", IX86_BUILTIN_UNPCKLPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21311 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_si256_si, "__builtin_ia32_si256_si", IX86_BUILTIN_SI256_SI, UNKNOWN, (int) V8SI_FTYPE_V4SI },
21312 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ps256_ps, "__builtin_ia32_ps256_ps", IX86_BUILTIN_PS256_PS, UNKNOWN, (int) V8SF_FTYPE_V4SF },
21313 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_pd256_pd, "__builtin_ia32_pd256_pd", IX86_BUILTIN_PD256_PD, UNKNOWN, (int) V4DF_FTYPE_V2DF },
21314 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_si_si256, "__builtin_ia32_si_si256", IX86_BUILTIN_SI_SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI },
21315 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ps_ps256, "__builtin_ia32_ps_ps256", IX86_BUILTIN_PS_PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF },
21316 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_pd_pd256, "__builtin_ia32_pd_pd256", IX86_BUILTIN_PD_PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF },
21318 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestzpd", IX86_BUILTIN_VTESTZPD, EQ, (int) INT_FTYPE_V2DF_V2DF_PTEST },
21319 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestcpd", IX86_BUILTIN_VTESTCPD, LTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
21320 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestnzcpd", IX86_BUILTIN_VTESTNZCPD, GTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
21321 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestzps", IX86_BUILTIN_VTESTZPS, EQ, (int) INT_FTYPE_V4SF_V4SF_PTEST },
21322 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestcps", IX86_BUILTIN_VTESTCPS, LTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
21323 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestnzcps", IX86_BUILTIN_VTESTNZCPS, GTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
21324 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestzpd256", IX86_BUILTIN_VTESTZPD256, EQ, (int) INT_FTYPE_V4DF_V4DF_PTEST },
21325 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestcpd256", IX86_BUILTIN_VTESTCPD256, LTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
21326 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestnzcpd256", IX86_BUILTIN_VTESTNZCPD256, GTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
21327 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestzps256", IX86_BUILTIN_VTESTZPS256, EQ, (int) INT_FTYPE_V8SF_V8SF_PTEST },
21328 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestcps256", IX86_BUILTIN_VTESTCPS256, LTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
21329 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestnzcps256", IX86_BUILTIN_VTESTNZCPS256, GTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
21330 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestz256", IX86_BUILTIN_PTESTZ256, EQ, (int) INT_FTYPE_V4DI_V4DI_PTEST },
21331 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestc256", IX86_BUILTIN_PTESTC256, LTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
21332 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestnzc256", IX86_BUILTIN_PTESTNZC256, GTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
21334 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskpd256, "__builtin_ia32_movmskpd256", IX86_BUILTIN_MOVMSKPD256, UNKNOWN, (int) INT_FTYPE_V4DF },
21335 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskps256, "__builtin_ia32_movmskps256", IX86_BUILTIN_MOVMSKPS256, UNKNOWN, (int) INT_FTYPE_V8SF },
21339 enum multi_arg_type {
21349 MULTI_ARG_3_PERMPS,
21350 MULTI_ARG_3_PERMPD,
21357 MULTI_ARG_2_DI_IMM,
21358 MULTI_ARG_2_SI_IMM,
21359 MULTI_ARG_2_HI_IMM,
21360 MULTI_ARG_2_QI_IMM,
21361 MULTI_ARG_2_SF_CMP,
21362 MULTI_ARG_2_DF_CMP,
21363 MULTI_ARG_2_DI_CMP,
21364 MULTI_ARG_2_SI_CMP,
21365 MULTI_ARG_2_HI_CMP,
21366 MULTI_ARG_2_QI_CMP,
21389 static const struct builtin_description bdesc_multi_arg[] =
21391 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_vmfmaddv4sf4, "__builtin_ia32_fmaddss", IX86_BUILTIN_FMADDSS, 0, (int)MULTI_ARG_3_SF },
21392 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_vmfmaddv2df4, "__builtin_ia32_fmaddsd", IX86_BUILTIN_FMADDSD, 0, (int)MULTI_ARG_3_DF },
21393 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_fmaddv4sf4, "__builtin_ia32_fmaddps", IX86_BUILTIN_FMADDPS, 0, (int)MULTI_ARG_3_SF },
21394 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_fmaddv2df4, "__builtin_ia32_fmaddpd", IX86_BUILTIN_FMADDPD, 0, (int)MULTI_ARG_3_DF },
21395 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_vmfmsubv4sf4, "__builtin_ia32_fmsubss", IX86_BUILTIN_FMSUBSS, 0, (int)MULTI_ARG_3_SF },
21396 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_vmfmsubv2df4, "__builtin_ia32_fmsubsd", IX86_BUILTIN_FMSUBSD, 0, (int)MULTI_ARG_3_DF },
21397 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_fmsubv4sf4, "__builtin_ia32_fmsubps", IX86_BUILTIN_FMSUBPS, 0, (int)MULTI_ARG_3_SF },
21398 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_fmsubv2df4, "__builtin_ia32_fmsubpd", IX86_BUILTIN_FMSUBPD, 0, (int)MULTI_ARG_3_DF },
21399 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_vmfnmaddv4sf4, "__builtin_ia32_fnmaddss", IX86_BUILTIN_FNMADDSS, 0, (int)MULTI_ARG_3_SF },
21400 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_vmfnmaddv2df4, "__builtin_ia32_fnmaddsd", IX86_BUILTIN_FNMADDSD, 0, (int)MULTI_ARG_3_DF },
21401 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_fnmaddv4sf4, "__builtin_ia32_fnmaddps", IX86_BUILTIN_FNMADDPS, 0, (int)MULTI_ARG_3_SF },
21402 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_fnmaddv2df4, "__builtin_ia32_fnmaddpd", IX86_BUILTIN_FNMADDPD, 0, (int)MULTI_ARG_3_DF },
21403 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_vmfnmsubv4sf4, "__builtin_ia32_fnmsubss", IX86_BUILTIN_FNMSUBSS, 0, (int)MULTI_ARG_3_SF },
21404 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_vmfnmsubv2df4, "__builtin_ia32_fnmsubsd", IX86_BUILTIN_FNMSUBSD, 0, (int)MULTI_ARG_3_DF },
21405 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_fnmsubv4sf4, "__builtin_ia32_fnmsubps", IX86_BUILTIN_FNMSUBPS, 0, (int)MULTI_ARG_3_SF },
21406 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_fnmsubv2df4, "__builtin_ia32_fnmsubpd", IX86_BUILTIN_FNMSUBPD, 0, (int)MULTI_ARG_3_DF },
21407 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcmov_v2di, "__builtin_ia32_pcmov", IX86_BUILTIN_PCMOV, 0, (int)MULTI_ARG_3_DI },
21408 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcmov_v2di, "__builtin_ia32_pcmov_v2di", IX86_BUILTIN_PCMOV_V2DI, 0, (int)MULTI_ARG_3_DI },
21409 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcmov_v4si, "__builtin_ia32_pcmov_v4si", IX86_BUILTIN_PCMOV_V4SI, 0, (int)MULTI_ARG_3_SI },
21410 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcmov_v8hi, "__builtin_ia32_pcmov_v8hi", IX86_BUILTIN_PCMOV_V8HI, 0, (int)MULTI_ARG_3_HI },
21411 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcmov_v16qi, "__builtin_ia32_pcmov_v16qi",IX86_BUILTIN_PCMOV_V16QI,0, (int)MULTI_ARG_3_QI },
21412 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcmov_v2df, "__builtin_ia32_pcmov_v2df", IX86_BUILTIN_PCMOV_V2DF, 0, (int)MULTI_ARG_3_DF },
21413 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcmov_v4sf, "__builtin_ia32_pcmov_v4sf", IX86_BUILTIN_PCMOV_V4SF, 0, (int)MULTI_ARG_3_SF },
21414 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pperm, "__builtin_ia32_pperm", IX86_BUILTIN_PPERM, 0, (int)MULTI_ARG_3_QI },
21415 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_permv4sf, "__builtin_ia32_permps", IX86_BUILTIN_PERMPS, 0, (int)MULTI_ARG_3_PERMPS },
21416 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_permv2df, "__builtin_ia32_permpd", IX86_BUILTIN_PERMPD, 0, (int)MULTI_ARG_3_PERMPD },
21417 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacssww, "__builtin_ia32_pmacssww", IX86_BUILTIN_PMACSSWW, 0, (int)MULTI_ARG_3_HI },
21418 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacsww, "__builtin_ia32_pmacsww", IX86_BUILTIN_PMACSWW, 0, (int)MULTI_ARG_3_HI },
21419 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacsswd, "__builtin_ia32_pmacsswd", IX86_BUILTIN_PMACSSWD, 0, (int)MULTI_ARG_3_HI_SI },
21420 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacswd, "__builtin_ia32_pmacswd", IX86_BUILTIN_PMACSWD, 0, (int)MULTI_ARG_3_HI_SI },
21421 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacssdd, "__builtin_ia32_pmacssdd", IX86_BUILTIN_PMACSSDD, 0, (int)MULTI_ARG_3_SI },
21422 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacsdd, "__builtin_ia32_pmacsdd", IX86_BUILTIN_PMACSDD, 0, (int)MULTI_ARG_3_SI },
21423 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacssdql, "__builtin_ia32_pmacssdql", IX86_BUILTIN_PMACSSDQL, 0, (int)MULTI_ARG_3_SI_DI },
21424 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacssdqh, "__builtin_ia32_pmacssdqh", IX86_BUILTIN_PMACSSDQH, 0, (int)MULTI_ARG_3_SI_DI },
21425 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacsdql, "__builtin_ia32_pmacsdql", IX86_BUILTIN_PMACSDQL, 0, (int)MULTI_ARG_3_SI_DI },
21426 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacsdqh, "__builtin_ia32_pmacsdqh", IX86_BUILTIN_PMACSDQH, 0, (int)MULTI_ARG_3_SI_DI },
21427 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmadcsswd, "__builtin_ia32_pmadcsswd", IX86_BUILTIN_PMADCSSWD, 0, (int)MULTI_ARG_3_HI_SI },
21428 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmadcswd, "__builtin_ia32_pmadcswd", IX86_BUILTIN_PMADCSWD, 0, (int)MULTI_ARG_3_HI_SI },
21429 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vrotlv2di3, "__builtin_ia32_protq", IX86_BUILTIN_PROTQ, 0, (int)MULTI_ARG_2_DI },
21430 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vrotlv4si3, "__builtin_ia32_protd", IX86_BUILTIN_PROTD, 0, (int)MULTI_ARG_2_SI },
21431 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vrotlv8hi3, "__builtin_ia32_protw", IX86_BUILTIN_PROTW, 0, (int)MULTI_ARG_2_HI },
21432 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vrotlv16qi3, "__builtin_ia32_protb", IX86_BUILTIN_PROTB, 0, (int)MULTI_ARG_2_QI },
21433 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_rotlv2di3, "__builtin_ia32_protqi", IX86_BUILTIN_PROTQ_IMM, 0, (int)MULTI_ARG_2_DI_IMM },
21434 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_rotlv4si3, "__builtin_ia32_protdi", IX86_BUILTIN_PROTD_IMM, 0, (int)MULTI_ARG_2_SI_IMM },
21435 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_rotlv8hi3, "__builtin_ia32_protwi", IX86_BUILTIN_PROTW_IMM, 0, (int)MULTI_ARG_2_HI_IMM },
21436 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_rotlv16qi3, "__builtin_ia32_protbi", IX86_BUILTIN_PROTB_IMM, 0, (int)MULTI_ARG_2_QI_IMM },
21437 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_ashlv2di3, "__builtin_ia32_pshaq", IX86_BUILTIN_PSHAQ, 0, (int)MULTI_ARG_2_DI },
21438 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_ashlv4si3, "__builtin_ia32_pshad", IX86_BUILTIN_PSHAD, 0, (int)MULTI_ARG_2_SI },
21439 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_ashlv8hi3, "__builtin_ia32_pshaw", IX86_BUILTIN_PSHAW, 0, (int)MULTI_ARG_2_HI },
21440 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_ashlv16qi3, "__builtin_ia32_pshab", IX86_BUILTIN_PSHAB, 0, (int)MULTI_ARG_2_QI },
21441 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_lshlv2di3, "__builtin_ia32_pshlq", IX86_BUILTIN_PSHLQ, 0, (int)MULTI_ARG_2_DI },
21442 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_lshlv4si3, "__builtin_ia32_pshld", IX86_BUILTIN_PSHLD, 0, (int)MULTI_ARG_2_SI },
21443 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_lshlv8hi3, "__builtin_ia32_pshlw", IX86_BUILTIN_PSHLW, 0, (int)MULTI_ARG_2_HI },
21444 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_lshlv16qi3, "__builtin_ia32_pshlb", IX86_BUILTIN_PSHLB, 0, (int)MULTI_ARG_2_QI },
21445 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmfrczv4sf2, "__builtin_ia32_frczss", IX86_BUILTIN_FRCZSS, 0, (int)MULTI_ARG_2_SF },
21446 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmfrczv2df2, "__builtin_ia32_frczsd", IX86_BUILTIN_FRCZSD, 0, (int)MULTI_ARG_2_DF },
21447 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_frczv4sf2, "__builtin_ia32_frczps", IX86_BUILTIN_FRCZPS, 0, (int)MULTI_ARG_1_SF },
21448 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_frczv2df2, "__builtin_ia32_frczpd", IX86_BUILTIN_FRCZPD, 0, (int)MULTI_ARG_1_DF },
21449 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_cvtph2ps, "__builtin_ia32_cvtph2ps", IX86_BUILTIN_CVTPH2PS, 0, (int)MULTI_ARG_1_PH2PS },
21450 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_cvtps2ph, "__builtin_ia32_cvtps2ph", IX86_BUILTIN_CVTPS2PH, 0, (int)MULTI_ARG_1_PS2PH },
21451 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phaddbw, "__builtin_ia32_phaddbw", IX86_BUILTIN_PHADDBW, 0, (int)MULTI_ARG_1_QI_HI },
21452 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phaddbd, "__builtin_ia32_phaddbd", IX86_BUILTIN_PHADDBD, 0, (int)MULTI_ARG_1_QI_SI },
21453 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phaddbq, "__builtin_ia32_phaddbq", IX86_BUILTIN_PHADDBQ, 0, (int)MULTI_ARG_1_QI_DI },
21454 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phaddwd, "__builtin_ia32_phaddwd", IX86_BUILTIN_PHADDWD, 0, (int)MULTI_ARG_1_HI_SI },
21455 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phaddwq, "__builtin_ia32_phaddwq", IX86_BUILTIN_PHADDWQ, 0, (int)MULTI_ARG_1_HI_DI },
21456 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phadddq, "__builtin_ia32_phadddq", IX86_BUILTIN_PHADDDQ, 0, (int)MULTI_ARG_1_SI_DI },
21457 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phaddubw, "__builtin_ia32_phaddubw", IX86_BUILTIN_PHADDUBW, 0, (int)MULTI_ARG_1_QI_HI },
21458 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phaddubd, "__builtin_ia32_phaddubd", IX86_BUILTIN_PHADDUBD, 0, (int)MULTI_ARG_1_QI_SI },
21459 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phaddubq, "__builtin_ia32_phaddubq", IX86_BUILTIN_PHADDUBQ, 0, (int)MULTI_ARG_1_QI_DI },
21460 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phadduwd, "__builtin_ia32_phadduwd", IX86_BUILTIN_PHADDUWD, 0, (int)MULTI_ARG_1_HI_SI },
21461 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phadduwq, "__builtin_ia32_phadduwq", IX86_BUILTIN_PHADDUWQ, 0, (int)MULTI_ARG_1_HI_DI },
21462 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phaddudq, "__builtin_ia32_phaddudq", IX86_BUILTIN_PHADDUDQ, 0, (int)MULTI_ARG_1_SI_DI },
21463 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phsubbw, "__builtin_ia32_phsubbw", IX86_BUILTIN_PHSUBBW, 0, (int)MULTI_ARG_1_QI_HI },
21464 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phsubwd, "__builtin_ia32_phsubwd", IX86_BUILTIN_PHSUBWD, 0, (int)MULTI_ARG_1_HI_SI },
21465 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phsubdq, "__builtin_ia32_phsubdq", IX86_BUILTIN_PHSUBDQ, 0, (int)MULTI_ARG_1_SI_DI },
21467 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comeqss", IX86_BUILTIN_COMEQSS, EQ, (int)MULTI_ARG_2_SF_CMP },
21468 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comness", IX86_BUILTIN_COMNESS, NE, (int)MULTI_ARG_2_SF_CMP },
21469 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comneqss", IX86_BUILTIN_COMNESS, NE, (int)MULTI_ARG_2_SF_CMP },
21470 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comltss", IX86_BUILTIN_COMLTSS, LT, (int)MULTI_ARG_2_SF_CMP },
21471 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comless", IX86_BUILTIN_COMLESS, LE, (int)MULTI_ARG_2_SF_CMP },
21472 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comgtss", IX86_BUILTIN_COMGTSS, GT, (int)MULTI_ARG_2_SF_CMP },
21473 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comgess", IX86_BUILTIN_COMGESS, GE, (int)MULTI_ARG_2_SF_CMP },
21474 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comueqss", IX86_BUILTIN_COMUEQSS, UNEQ, (int)MULTI_ARG_2_SF_CMP },
21475 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comuness", IX86_BUILTIN_COMUNESS, LTGT, (int)MULTI_ARG_2_SF_CMP },
21476 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comuneqss", IX86_BUILTIN_COMUNESS, LTGT, (int)MULTI_ARG_2_SF_CMP },
21477 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comunltss", IX86_BUILTIN_COMULTSS, UNLT, (int)MULTI_ARG_2_SF_CMP },
21478 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comunless", IX86_BUILTIN_COMULESS, UNLE, (int)MULTI_ARG_2_SF_CMP },
21479 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comungtss", IX86_BUILTIN_COMUGTSS, UNGT, (int)MULTI_ARG_2_SF_CMP },
21480 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comungess", IX86_BUILTIN_COMUGESS, UNGE, (int)MULTI_ARG_2_SF_CMP },
21481 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comordss", IX86_BUILTIN_COMORDSS, ORDERED, (int)MULTI_ARG_2_SF_CMP },
21482 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comunordss", IX86_BUILTIN_COMUNORDSS, UNORDERED, (int)MULTI_ARG_2_SF_CMP },
21484 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comeqsd", IX86_BUILTIN_COMEQSD, EQ, (int)MULTI_ARG_2_DF_CMP },
21485 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comnesd", IX86_BUILTIN_COMNESD, NE, (int)MULTI_ARG_2_DF_CMP },
21486 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comneqsd", IX86_BUILTIN_COMNESD, NE, (int)MULTI_ARG_2_DF_CMP },
21487 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comltsd", IX86_BUILTIN_COMLTSD, LT, (int)MULTI_ARG_2_DF_CMP },
21488 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comlesd", IX86_BUILTIN_COMLESD, LE, (int)MULTI_ARG_2_DF_CMP },
21489 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comgtsd", IX86_BUILTIN_COMGTSD, GT, (int)MULTI_ARG_2_DF_CMP },
21490 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comgesd", IX86_BUILTIN_COMGESD, GE, (int)MULTI_ARG_2_DF_CMP },
21491 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comueqsd", IX86_BUILTIN_COMUEQSD, UNEQ, (int)MULTI_ARG_2_DF_CMP },
21492 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comunesd", IX86_BUILTIN_COMUNESD, LTGT, (int)MULTI_ARG_2_DF_CMP },
21493 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comuneqsd", IX86_BUILTIN_COMUNESD, LTGT, (int)MULTI_ARG_2_DF_CMP },
21494 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comunltsd", IX86_BUILTIN_COMULTSD, UNLT, (int)MULTI_ARG_2_DF_CMP },
21495 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comunlesd", IX86_BUILTIN_COMULESD, UNLE, (int)MULTI_ARG_2_DF_CMP },
21496 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comungtsd", IX86_BUILTIN_COMUGTSD, UNGT, (int)MULTI_ARG_2_DF_CMP },
21497 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comungesd", IX86_BUILTIN_COMUGESD, UNGE, (int)MULTI_ARG_2_DF_CMP },
21498 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comordsd", IX86_BUILTIN_COMORDSD, ORDERED, (int)MULTI_ARG_2_DF_CMP },
21499 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comunordsd", IX86_BUILTIN_COMUNORDSD, UNORDERED, (int)MULTI_ARG_2_DF_CMP },
21501 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comeqps", IX86_BUILTIN_COMEQPS, EQ, (int)MULTI_ARG_2_SF_CMP },
21502 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comneps", IX86_BUILTIN_COMNEPS, NE, (int)MULTI_ARG_2_SF_CMP },
21503 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comneqps", IX86_BUILTIN_COMNEPS, NE, (int)MULTI_ARG_2_SF_CMP },
21504 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comltps", IX86_BUILTIN_COMLTPS, LT, (int)MULTI_ARG_2_SF_CMP },
21505 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comleps", IX86_BUILTIN_COMLEPS, LE, (int)MULTI_ARG_2_SF_CMP },
21506 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comgtps", IX86_BUILTIN_COMGTPS, GT, (int)MULTI_ARG_2_SF_CMP },
21507 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comgeps", IX86_BUILTIN_COMGEPS, GE, (int)MULTI_ARG_2_SF_CMP },
21508 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comueqps", IX86_BUILTIN_COMUEQPS, UNEQ, (int)MULTI_ARG_2_SF_CMP },
21509 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comuneps", IX86_BUILTIN_COMUNEPS, LTGT, (int)MULTI_ARG_2_SF_CMP },
21510 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comuneqps", IX86_BUILTIN_COMUNEPS, LTGT, (int)MULTI_ARG_2_SF_CMP },
21511 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comunltps", IX86_BUILTIN_COMULTPS, UNLT, (int)MULTI_ARG_2_SF_CMP },
21512 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comunleps", IX86_BUILTIN_COMULEPS, UNLE, (int)MULTI_ARG_2_SF_CMP },
21513 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comungtps", IX86_BUILTIN_COMUGTPS, UNGT, (int)MULTI_ARG_2_SF_CMP },
21514 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comungeps", IX86_BUILTIN_COMUGEPS, UNGE, (int)MULTI_ARG_2_SF_CMP },
21515 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comordps", IX86_BUILTIN_COMORDPS, ORDERED, (int)MULTI_ARG_2_SF_CMP },
21516 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comunordps", IX86_BUILTIN_COMUNORDPS, UNORDERED, (int)MULTI_ARG_2_SF_CMP },
21518 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comeqpd", IX86_BUILTIN_COMEQPD, EQ, (int)MULTI_ARG_2_DF_CMP },
21519 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comnepd", IX86_BUILTIN_COMNEPD, NE, (int)MULTI_ARG_2_DF_CMP },
21520 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comneqpd", IX86_BUILTIN_COMNEPD, NE, (int)MULTI_ARG_2_DF_CMP },
21521 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comltpd", IX86_BUILTIN_COMLTPD, LT, (int)MULTI_ARG_2_DF_CMP },
21522 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comlepd", IX86_BUILTIN_COMLEPD, LE, (int)MULTI_ARG_2_DF_CMP },
21523 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comgtpd", IX86_BUILTIN_COMGTPD, GT, (int)MULTI_ARG_2_DF_CMP },
21524 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comgepd", IX86_BUILTIN_COMGEPD, GE, (int)MULTI_ARG_2_DF_CMP },
21525 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comueqpd", IX86_BUILTIN_COMUEQPD, UNEQ, (int)MULTI_ARG_2_DF_CMP },
21526 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comunepd", IX86_BUILTIN_COMUNEPD, LTGT, (int)MULTI_ARG_2_DF_CMP },
21527 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comuneqpd", IX86_BUILTIN_COMUNEPD, LTGT, (int)MULTI_ARG_2_DF_CMP },
21528 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comunltpd", IX86_BUILTIN_COMULTPD, UNLT, (int)MULTI_ARG_2_DF_CMP },
21529 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comunlepd", IX86_BUILTIN_COMULEPD, UNLE, (int)MULTI_ARG_2_DF_CMP },
21530 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comungtpd", IX86_BUILTIN_COMUGTPD, UNGT, (int)MULTI_ARG_2_DF_CMP },
21531 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comungepd", IX86_BUILTIN_COMUGEPD, UNGE, (int)MULTI_ARG_2_DF_CMP },
21532 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comordpd", IX86_BUILTIN_COMORDPD, ORDERED, (int)MULTI_ARG_2_DF_CMP },
21533 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comunordpd", IX86_BUILTIN_COMUNORDPD, UNORDERED, (int)MULTI_ARG_2_DF_CMP },
21535 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv16qi3, "__builtin_ia32_pcomeqb", IX86_BUILTIN_PCOMEQB, EQ, (int)MULTI_ARG_2_QI_CMP },
21536 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv16qi3, "__builtin_ia32_pcomneb", IX86_BUILTIN_PCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
21537 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv16qi3, "__builtin_ia32_pcomneqb", IX86_BUILTIN_PCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
21538 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv16qi3, "__builtin_ia32_pcomltb", IX86_BUILTIN_PCOMLTB, LT, (int)MULTI_ARG_2_QI_CMP },
21539 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv16qi3, "__builtin_ia32_pcomleb", IX86_BUILTIN_PCOMLEB, LE, (int)MULTI_ARG_2_QI_CMP },
21540 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv16qi3, "__builtin_ia32_pcomgtb", IX86_BUILTIN_PCOMGTB, GT, (int)MULTI_ARG_2_QI_CMP },
21541 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv16qi3, "__builtin_ia32_pcomgeb", IX86_BUILTIN_PCOMGEB, GE, (int)MULTI_ARG_2_QI_CMP },
21543 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv8hi3, "__builtin_ia32_pcomeqw", IX86_BUILTIN_PCOMEQW, EQ, (int)MULTI_ARG_2_HI_CMP },
21544 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv8hi3, "__builtin_ia32_pcomnew", IX86_BUILTIN_PCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
21545 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv8hi3, "__builtin_ia32_pcomneqw", IX86_BUILTIN_PCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
21546 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv8hi3, "__builtin_ia32_pcomltw", IX86_BUILTIN_PCOMLTW, LT, (int)MULTI_ARG_2_HI_CMP },
21547 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv8hi3, "__builtin_ia32_pcomlew", IX86_BUILTIN_PCOMLEW, LE, (int)MULTI_ARG_2_HI_CMP },
21548 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv8hi3, "__builtin_ia32_pcomgtw", IX86_BUILTIN_PCOMGTW, GT, (int)MULTI_ARG_2_HI_CMP },
21549 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv8hi3, "__builtin_ia32_pcomgew", IX86_BUILTIN_PCOMGEW, GE, (int)MULTI_ARG_2_HI_CMP },
21551 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4si3, "__builtin_ia32_pcomeqd", IX86_BUILTIN_PCOMEQD, EQ, (int)MULTI_ARG_2_SI_CMP },
21552 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4si3, "__builtin_ia32_pcomned", IX86_BUILTIN_PCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
21553 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4si3, "__builtin_ia32_pcomneqd", IX86_BUILTIN_PCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
21554 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4si3, "__builtin_ia32_pcomltd", IX86_BUILTIN_PCOMLTD, LT, (int)MULTI_ARG_2_SI_CMP },
21555 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4si3, "__builtin_ia32_pcomled", IX86_BUILTIN_PCOMLED, LE, (int)MULTI_ARG_2_SI_CMP },
21556 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4si3, "__builtin_ia32_pcomgtd", IX86_BUILTIN_PCOMGTD, GT, (int)MULTI_ARG_2_SI_CMP },
21557 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4si3, "__builtin_ia32_pcomged", IX86_BUILTIN_PCOMGED, GE, (int)MULTI_ARG_2_SI_CMP },
21559 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2di3, "__builtin_ia32_pcomeqq", IX86_BUILTIN_PCOMEQQ, EQ, (int)MULTI_ARG_2_DI_CMP },
21560 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2di3, "__builtin_ia32_pcomneq", IX86_BUILTIN_PCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
21561 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2di3, "__builtin_ia32_pcomneqq", IX86_BUILTIN_PCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
21562 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2di3, "__builtin_ia32_pcomltq", IX86_BUILTIN_PCOMLTQ, LT, (int)MULTI_ARG_2_DI_CMP },
21563 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2di3, "__builtin_ia32_pcomleq", IX86_BUILTIN_PCOMLEQ, LE, (int)MULTI_ARG_2_DI_CMP },
21564 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2di3, "__builtin_ia32_pcomgtq", IX86_BUILTIN_PCOMGTQ, GT, (int)MULTI_ARG_2_DI_CMP },
21565 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2di3, "__builtin_ia32_pcomgeq", IX86_BUILTIN_PCOMGEQ, GE, (int)MULTI_ARG_2_DI_CMP },
21567 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v16qi3,"__builtin_ia32_pcomequb", IX86_BUILTIN_PCOMEQUB, EQ, (int)MULTI_ARG_2_QI_CMP },
21568 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v16qi3,"__builtin_ia32_pcomneub", IX86_BUILTIN_PCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
21569 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v16qi3,"__builtin_ia32_pcomnequb", IX86_BUILTIN_PCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
21570 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv16qi3, "__builtin_ia32_pcomltub", IX86_BUILTIN_PCOMLTUB, LTU, (int)MULTI_ARG_2_QI_CMP },
21571 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv16qi3, "__builtin_ia32_pcomleub", IX86_BUILTIN_PCOMLEUB, LEU, (int)MULTI_ARG_2_QI_CMP },
21572 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv16qi3, "__builtin_ia32_pcomgtub", IX86_BUILTIN_PCOMGTUB, GTU, (int)MULTI_ARG_2_QI_CMP },
21573 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv16qi3, "__builtin_ia32_pcomgeub", IX86_BUILTIN_PCOMGEUB, GEU, (int)MULTI_ARG_2_QI_CMP },
21575 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v8hi3, "__builtin_ia32_pcomequw", IX86_BUILTIN_PCOMEQUW, EQ, (int)MULTI_ARG_2_HI_CMP },
21576 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v8hi3, "__builtin_ia32_pcomneuw", IX86_BUILTIN_PCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
21577 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v8hi3, "__builtin_ia32_pcomnequw", IX86_BUILTIN_PCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
21578 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv8hi3, "__builtin_ia32_pcomltuw", IX86_BUILTIN_PCOMLTUW, LTU, (int)MULTI_ARG_2_HI_CMP },
21579 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv8hi3, "__builtin_ia32_pcomleuw", IX86_BUILTIN_PCOMLEUW, LEU, (int)MULTI_ARG_2_HI_CMP },
21580 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv8hi3, "__builtin_ia32_pcomgtuw", IX86_BUILTIN_PCOMGTUW, GTU, (int)MULTI_ARG_2_HI_CMP },
21581 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv8hi3, "__builtin_ia32_pcomgeuw", IX86_BUILTIN_PCOMGEUW, GEU, (int)MULTI_ARG_2_HI_CMP },
21583 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v4si3, "__builtin_ia32_pcomequd", IX86_BUILTIN_PCOMEQUD, EQ, (int)MULTI_ARG_2_SI_CMP },
21584 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v4si3, "__builtin_ia32_pcomneud", IX86_BUILTIN_PCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
21585 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v4si3, "__builtin_ia32_pcomnequd", IX86_BUILTIN_PCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
21586 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv4si3, "__builtin_ia32_pcomltud", IX86_BUILTIN_PCOMLTUD, LTU, (int)MULTI_ARG_2_SI_CMP },
21587 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv4si3, "__builtin_ia32_pcomleud", IX86_BUILTIN_PCOMLEUD, LEU, (int)MULTI_ARG_2_SI_CMP },
21588 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv4si3, "__builtin_ia32_pcomgtud", IX86_BUILTIN_PCOMGTUD, GTU, (int)MULTI_ARG_2_SI_CMP },
21589 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv4si3, "__builtin_ia32_pcomgeud", IX86_BUILTIN_PCOMGEUD, GEU, (int)MULTI_ARG_2_SI_CMP },
21591 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v2di3, "__builtin_ia32_pcomequq", IX86_BUILTIN_PCOMEQUQ, EQ, (int)MULTI_ARG_2_DI_CMP },
21592 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v2di3, "__builtin_ia32_pcomneuq", IX86_BUILTIN_PCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
21593 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v2di3, "__builtin_ia32_pcomnequq", IX86_BUILTIN_PCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
21594 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv2di3, "__builtin_ia32_pcomltuq", IX86_BUILTIN_PCOMLTUQ, LTU, (int)MULTI_ARG_2_DI_CMP },
21595 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv2di3, "__builtin_ia32_pcomleuq", IX86_BUILTIN_PCOMLEUQ, LEU, (int)MULTI_ARG_2_DI_CMP },
21596 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv2di3, "__builtin_ia32_pcomgtuq", IX86_BUILTIN_PCOMGTUQ, GTU, (int)MULTI_ARG_2_DI_CMP },
21597 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv2di3, "__builtin_ia32_pcomgeuq", IX86_BUILTIN_PCOMGEUQ, GEU, (int)MULTI_ARG_2_DI_CMP },
21599 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_com_tfv4sf3, "__builtin_ia32_comfalsess", IX86_BUILTIN_COMFALSESS, COM_FALSE_S, (int)MULTI_ARG_2_SF_TF },
21600 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_com_tfv4sf3, "__builtin_ia32_comtruess", IX86_BUILTIN_COMTRUESS, COM_TRUE_S, (int)MULTI_ARG_2_SF_TF },
21601 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_com_tfv4sf3, "__builtin_ia32_comfalseps", IX86_BUILTIN_COMFALSEPS, COM_FALSE_P, (int)MULTI_ARG_2_SF_TF },
21602 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_com_tfv4sf3, "__builtin_ia32_comtrueps", IX86_BUILTIN_COMTRUEPS, COM_TRUE_P, (int)MULTI_ARG_2_SF_TF },
21603 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_com_tfv2df3, "__builtin_ia32_comfalsesd", IX86_BUILTIN_COMFALSESD, COM_FALSE_S, (int)MULTI_ARG_2_DF_TF },
21604 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_com_tfv2df3, "__builtin_ia32_comtruesd", IX86_BUILTIN_COMTRUESD, COM_TRUE_S, (int)MULTI_ARG_2_DF_TF },
21605 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_com_tfv2df3, "__builtin_ia32_comfalsepd", IX86_BUILTIN_COMFALSEPD, COM_FALSE_P, (int)MULTI_ARG_2_DF_TF },
21606 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_com_tfv2df3, "__builtin_ia32_comtruepd", IX86_BUILTIN_COMTRUEPD, COM_TRUE_P, (int)MULTI_ARG_2_DF_TF },
21608 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv16qi3, "__builtin_ia32_pcomfalseb", IX86_BUILTIN_PCOMFALSEB, PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
21609 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv8hi3, "__builtin_ia32_pcomfalsew", IX86_BUILTIN_PCOMFALSEW, PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
21610 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv4si3, "__builtin_ia32_pcomfalsed", IX86_BUILTIN_PCOMFALSED, PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
21611 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv2di3, "__builtin_ia32_pcomfalseq", IX86_BUILTIN_PCOMFALSEQ, PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
21612 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv16qi3, "__builtin_ia32_pcomfalseub",IX86_BUILTIN_PCOMFALSEUB,PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
21613 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv8hi3, "__builtin_ia32_pcomfalseuw",IX86_BUILTIN_PCOMFALSEUW,PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
21614 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv4si3, "__builtin_ia32_pcomfalseud",IX86_BUILTIN_PCOMFALSEUD,PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
21615 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv2di3, "__builtin_ia32_pcomfalseuq",IX86_BUILTIN_PCOMFALSEUQ,PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
21617 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv16qi3, "__builtin_ia32_pcomtrueb", IX86_BUILTIN_PCOMTRUEB, PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
21618 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv8hi3, "__builtin_ia32_pcomtruew", IX86_BUILTIN_PCOMTRUEW, PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
21619 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv4si3, "__builtin_ia32_pcomtrued", IX86_BUILTIN_PCOMTRUED, PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
21620 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv2di3, "__builtin_ia32_pcomtrueq", IX86_BUILTIN_PCOMTRUEQ, PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
21621 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv16qi3, "__builtin_ia32_pcomtrueub", IX86_BUILTIN_PCOMTRUEUB, PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
21622 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv8hi3, "__builtin_ia32_pcomtrueuw", IX86_BUILTIN_PCOMTRUEUW, PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
21623 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv4si3, "__builtin_ia32_pcomtrueud", IX86_BUILTIN_PCOMTRUEUD, PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
21624 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv2di3, "__builtin_ia32_pcomtrueuq", IX86_BUILTIN_PCOMTRUEUQ, PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
21627 /* Set up all the MMX/SSE builtins, even builtins for instructions that are not
21628 in the current target ISA to allow the user to compile particular modules
21629 with different target specific options that differ from the command line
21632 ix86_init_mmx_sse_builtins (void)
21634 const struct builtin_description * d;
21637 tree V16QI_type_node = build_vector_type_for_mode (char_type_node, V16QImode);
21638 tree V2SI_type_node = build_vector_type_for_mode (intSI_type_node, V2SImode);
21639 tree V1DI_type_node
21640 = build_vector_type_for_mode (long_long_integer_type_node, V1DImode);
21641 tree V2SF_type_node = build_vector_type_for_mode (float_type_node, V2SFmode);
21642 tree V2DI_type_node
21643 = build_vector_type_for_mode (long_long_integer_type_node, V2DImode);
21644 tree V2DF_type_node = build_vector_type_for_mode (double_type_node, V2DFmode);
21645 tree V4SF_type_node = build_vector_type_for_mode (float_type_node, V4SFmode);
21646 tree V4SI_type_node = build_vector_type_for_mode (intSI_type_node, V4SImode);
21647 tree V4HI_type_node = build_vector_type_for_mode (intHI_type_node, V4HImode);
21648 tree V8QI_type_node = build_vector_type_for_mode (char_type_node, V8QImode);
21649 tree V8HI_type_node = build_vector_type_for_mode (intHI_type_node, V8HImode);
21651 tree pchar_type_node = build_pointer_type (char_type_node);
21652 tree pcchar_type_node
21653 = build_pointer_type (build_type_variant (char_type_node, 1, 0));
21654 tree pfloat_type_node = build_pointer_type (float_type_node);
21655 tree pcfloat_type_node
21656 = build_pointer_type (build_type_variant (float_type_node, 1, 0));
21657 tree pv2sf_type_node = build_pointer_type (V2SF_type_node);
21658 tree pcv2sf_type_node
21659 = build_pointer_type (build_type_variant (V2SF_type_node, 1, 0));
21660 tree pv2di_type_node = build_pointer_type (V2DI_type_node);
21661 tree pdi_type_node = build_pointer_type (long_long_unsigned_type_node);
21664 tree int_ftype_v4sf_v4sf
21665 = build_function_type_list (integer_type_node,
21666 V4SF_type_node, V4SF_type_node, NULL_TREE);
21667 tree v4si_ftype_v4sf_v4sf
21668 = build_function_type_list (V4SI_type_node,
21669 V4SF_type_node, V4SF_type_node, NULL_TREE);
21670 /* MMX/SSE/integer conversions. */
21671 tree int_ftype_v4sf
21672 = build_function_type_list (integer_type_node,
21673 V4SF_type_node, NULL_TREE);
21674 tree int64_ftype_v4sf
21675 = build_function_type_list (long_long_integer_type_node,
21676 V4SF_type_node, NULL_TREE);
21677 tree int_ftype_v8qi
21678 = build_function_type_list (integer_type_node, V8QI_type_node, NULL_TREE);
21679 tree v4sf_ftype_v4sf_int
21680 = build_function_type_list (V4SF_type_node,
21681 V4SF_type_node, integer_type_node, NULL_TREE);
21682 tree v4sf_ftype_v4sf_int64
21683 = build_function_type_list (V4SF_type_node,
21684 V4SF_type_node, long_long_integer_type_node,
21686 tree v4sf_ftype_v4sf_v2si
21687 = build_function_type_list (V4SF_type_node,
21688 V4SF_type_node, V2SI_type_node, NULL_TREE);
21690 /* Miscellaneous. */
21691 tree v8qi_ftype_v4hi_v4hi
21692 = build_function_type_list (V8QI_type_node,
21693 V4HI_type_node, V4HI_type_node, NULL_TREE);
21694 tree v4hi_ftype_v2si_v2si
21695 = build_function_type_list (V4HI_type_node,
21696 V2SI_type_node, V2SI_type_node, NULL_TREE);
21697 tree v4sf_ftype_v4sf_v4sf_int
21698 = build_function_type_list (V4SF_type_node,
21699 V4SF_type_node, V4SF_type_node,
21700 integer_type_node, NULL_TREE);
21701 tree v2si_ftype_v4hi_v4hi
21702 = build_function_type_list (V2SI_type_node,
21703 V4HI_type_node, V4HI_type_node, NULL_TREE);
21704 tree v4hi_ftype_v4hi_int
21705 = build_function_type_list (V4HI_type_node,
21706 V4HI_type_node, integer_type_node, NULL_TREE);
21707 tree v2si_ftype_v2si_int
21708 = build_function_type_list (V2SI_type_node,
21709 V2SI_type_node, integer_type_node, NULL_TREE);
21710 tree v1di_ftype_v1di_int
21711 = build_function_type_list (V1DI_type_node,
21712 V1DI_type_node, integer_type_node, NULL_TREE);
21714 tree void_ftype_void
21715 = build_function_type (void_type_node, void_list_node);
21716 tree void_ftype_unsigned
21717 = build_function_type_list (void_type_node, unsigned_type_node, NULL_TREE);
21718 tree void_ftype_unsigned_unsigned
21719 = build_function_type_list (void_type_node, unsigned_type_node,
21720 unsigned_type_node, NULL_TREE);
21721 tree void_ftype_pcvoid_unsigned_unsigned
21722 = build_function_type_list (void_type_node, const_ptr_type_node,
21723 unsigned_type_node, unsigned_type_node,
21725 tree unsigned_ftype_void
21726 = build_function_type (unsigned_type_node, void_list_node);
21727 tree v2si_ftype_v4sf
21728 = build_function_type_list (V2SI_type_node, V4SF_type_node, NULL_TREE);
21729 /* Loads/stores. */
21730 tree void_ftype_v8qi_v8qi_pchar
21731 = build_function_type_list (void_type_node,
21732 V8QI_type_node, V8QI_type_node,
21733 pchar_type_node, NULL_TREE);
21734 tree v4sf_ftype_pcfloat
21735 = build_function_type_list (V4SF_type_node, pcfloat_type_node, NULL_TREE);
21736 tree v4sf_ftype_v4sf_pcv2sf
21737 = build_function_type_list (V4SF_type_node,
21738 V4SF_type_node, pcv2sf_type_node, NULL_TREE);
21739 tree void_ftype_pv2sf_v4sf
21740 = build_function_type_list (void_type_node,
21741 pv2sf_type_node, V4SF_type_node, NULL_TREE);
21742 tree void_ftype_pfloat_v4sf
21743 = build_function_type_list (void_type_node,
21744 pfloat_type_node, V4SF_type_node, NULL_TREE);
21745 tree void_ftype_pdi_di
21746 = build_function_type_list (void_type_node,
21747 pdi_type_node, long_long_unsigned_type_node,
21749 tree void_ftype_pv2di_v2di
21750 = build_function_type_list (void_type_node,
21751 pv2di_type_node, V2DI_type_node, NULL_TREE);
21752 /* Normal vector unops. */
21753 tree v4sf_ftype_v4sf
21754 = build_function_type_list (V4SF_type_node, V4SF_type_node, NULL_TREE);
21755 tree v16qi_ftype_v16qi
21756 = build_function_type_list (V16QI_type_node, V16QI_type_node, NULL_TREE);
21757 tree v8hi_ftype_v8hi
21758 = build_function_type_list (V8HI_type_node, V8HI_type_node, NULL_TREE);
21759 tree v4si_ftype_v4si
21760 = build_function_type_list (V4SI_type_node, V4SI_type_node, NULL_TREE);
21761 tree v8qi_ftype_v8qi
21762 = build_function_type_list (V8QI_type_node, V8QI_type_node, NULL_TREE);
21763 tree v4hi_ftype_v4hi
21764 = build_function_type_list (V4HI_type_node, V4HI_type_node, NULL_TREE);
21766 /* Normal vector binops. */
21767 tree v4sf_ftype_v4sf_v4sf
21768 = build_function_type_list (V4SF_type_node,
21769 V4SF_type_node, V4SF_type_node, NULL_TREE);
21770 tree v8qi_ftype_v8qi_v8qi
21771 = build_function_type_list (V8QI_type_node,
21772 V8QI_type_node, V8QI_type_node, NULL_TREE);
21773 tree v4hi_ftype_v4hi_v4hi
21774 = build_function_type_list (V4HI_type_node,
21775 V4HI_type_node, V4HI_type_node, NULL_TREE);
21776 tree v2si_ftype_v2si_v2si
21777 = build_function_type_list (V2SI_type_node,
21778 V2SI_type_node, V2SI_type_node, NULL_TREE);
21779 tree v1di_ftype_v1di_v1di
21780 = build_function_type_list (V1DI_type_node,
21781 V1DI_type_node, V1DI_type_node, NULL_TREE);
21782 tree v1di_ftype_v1di_v1di_int
21783 = build_function_type_list (V1DI_type_node,
21784 V1DI_type_node, V1DI_type_node,
21785 integer_type_node, NULL_TREE);
21786 tree v2si_ftype_v2sf
21787 = build_function_type_list (V2SI_type_node, V2SF_type_node, NULL_TREE);
21788 tree v2sf_ftype_v2si
21789 = build_function_type_list (V2SF_type_node, V2SI_type_node, NULL_TREE);
21790 tree v2si_ftype_v2si
21791 = build_function_type_list (V2SI_type_node, V2SI_type_node, NULL_TREE);
21792 tree v2sf_ftype_v2sf
21793 = build_function_type_list (V2SF_type_node, V2SF_type_node, NULL_TREE);
21794 tree v2sf_ftype_v2sf_v2sf
21795 = build_function_type_list (V2SF_type_node,
21796 V2SF_type_node, V2SF_type_node, NULL_TREE);
21797 tree v2si_ftype_v2sf_v2sf
21798 = build_function_type_list (V2SI_type_node,
21799 V2SF_type_node, V2SF_type_node, NULL_TREE);
21800 tree pint_type_node = build_pointer_type (integer_type_node);
21801 tree pdouble_type_node = build_pointer_type (double_type_node);
21802 tree pcdouble_type_node = build_pointer_type (
21803 build_type_variant (double_type_node, 1, 0));
21804 tree int_ftype_v2df_v2df
21805 = build_function_type_list (integer_type_node,
21806 V2DF_type_node, V2DF_type_node, NULL_TREE);
21808 tree void_ftype_pcvoid
21809 = build_function_type_list (void_type_node, const_ptr_type_node, NULL_TREE);
21810 tree v4sf_ftype_v4si
21811 = build_function_type_list (V4SF_type_node, V4SI_type_node, NULL_TREE);
21812 tree v4si_ftype_v4sf
21813 = build_function_type_list (V4SI_type_node, V4SF_type_node, NULL_TREE);
21814 tree v2df_ftype_v4si
21815 = build_function_type_list (V2DF_type_node, V4SI_type_node, NULL_TREE);
21816 tree v4si_ftype_v2df
21817 = build_function_type_list (V4SI_type_node, V2DF_type_node, NULL_TREE);
21818 tree v4si_ftype_v2df_v2df
21819 = build_function_type_list (V4SI_type_node,
21820 V2DF_type_node, V2DF_type_node, NULL_TREE);
21821 tree v2si_ftype_v2df
21822 = build_function_type_list (V2SI_type_node, V2DF_type_node, NULL_TREE);
21823 tree v4sf_ftype_v2df
21824 = build_function_type_list (V4SF_type_node, V2DF_type_node, NULL_TREE);
21825 tree v2df_ftype_v2si
21826 = build_function_type_list (V2DF_type_node, V2SI_type_node, NULL_TREE);
21827 tree v2df_ftype_v4sf
21828 = build_function_type_list (V2DF_type_node, V4SF_type_node, NULL_TREE);
21829 tree int_ftype_v2df
21830 = build_function_type_list (integer_type_node, V2DF_type_node, NULL_TREE);
21831 tree int64_ftype_v2df
21832 = build_function_type_list (long_long_integer_type_node,
21833 V2DF_type_node, NULL_TREE);
21834 tree v2df_ftype_v2df_int
21835 = build_function_type_list (V2DF_type_node,
21836 V2DF_type_node, integer_type_node, NULL_TREE);
21837 tree v2df_ftype_v2df_int64
21838 = build_function_type_list (V2DF_type_node,
21839 V2DF_type_node, long_long_integer_type_node,
21841 tree v4sf_ftype_v4sf_v2df
21842 = build_function_type_list (V4SF_type_node,
21843 V4SF_type_node, V2DF_type_node, NULL_TREE);
21844 tree v2df_ftype_v2df_v4sf
21845 = build_function_type_list (V2DF_type_node,
21846 V2DF_type_node, V4SF_type_node, NULL_TREE);
21847 tree v2df_ftype_v2df_v2df_int
21848 = build_function_type_list (V2DF_type_node,
21849 V2DF_type_node, V2DF_type_node,
21852 tree v2df_ftype_v2df_pcdouble
21853 = build_function_type_list (V2DF_type_node,
21854 V2DF_type_node, pcdouble_type_node, NULL_TREE);
21855 tree void_ftype_pdouble_v2df
21856 = build_function_type_list (void_type_node,
21857 pdouble_type_node, V2DF_type_node, NULL_TREE);
21858 tree void_ftype_pint_int
21859 = build_function_type_list (void_type_node,
21860 pint_type_node, integer_type_node, NULL_TREE);
21861 tree void_ftype_v16qi_v16qi_pchar
21862 = build_function_type_list (void_type_node,
21863 V16QI_type_node, V16QI_type_node,
21864 pchar_type_node, NULL_TREE);
21865 tree v2df_ftype_pcdouble
21866 = build_function_type_list (V2DF_type_node, pcdouble_type_node, NULL_TREE);
21867 tree v2df_ftype_v2df_v2df
21868 = build_function_type_list (V2DF_type_node,
21869 V2DF_type_node, V2DF_type_node, NULL_TREE);
21870 tree v16qi_ftype_v16qi_v16qi
21871 = build_function_type_list (V16QI_type_node,
21872 V16QI_type_node, V16QI_type_node, NULL_TREE);
21873 tree v8hi_ftype_v8hi_v8hi
21874 = build_function_type_list (V8HI_type_node,
21875 V8HI_type_node, V8HI_type_node, NULL_TREE);
21876 tree v4si_ftype_v4si_v4si
21877 = build_function_type_list (V4SI_type_node,
21878 V4SI_type_node, V4SI_type_node, NULL_TREE);
21879 tree v2di_ftype_v2di_v2di
21880 = build_function_type_list (V2DI_type_node,
21881 V2DI_type_node, V2DI_type_node, NULL_TREE);
21882 tree v2di_ftype_v2df_v2df
21883 = build_function_type_list (V2DI_type_node,
21884 V2DF_type_node, V2DF_type_node, NULL_TREE);
21885 tree v2df_ftype_v2df
21886 = build_function_type_list (V2DF_type_node, V2DF_type_node, NULL_TREE);
21887 tree v2di_ftype_v2di_int
21888 = build_function_type_list (V2DI_type_node,
21889 V2DI_type_node, integer_type_node, NULL_TREE);
21890 tree v2di_ftype_v2di_v2di_int
21891 = build_function_type_list (V2DI_type_node, V2DI_type_node,
21892 V2DI_type_node, integer_type_node, NULL_TREE);
21893 tree v4si_ftype_v4si_int
21894 = build_function_type_list (V4SI_type_node,
21895 V4SI_type_node, integer_type_node, NULL_TREE);
21896 tree v8hi_ftype_v8hi_int
21897 = build_function_type_list (V8HI_type_node,
21898 V8HI_type_node, integer_type_node, NULL_TREE);
21899 tree v4si_ftype_v8hi_v8hi
21900 = build_function_type_list (V4SI_type_node,
21901 V8HI_type_node, V8HI_type_node, NULL_TREE);
21902 tree v1di_ftype_v8qi_v8qi
21903 = build_function_type_list (V1DI_type_node,
21904 V8QI_type_node, V8QI_type_node, NULL_TREE);
21905 tree v1di_ftype_v2si_v2si
21906 = build_function_type_list (V1DI_type_node,
21907 V2SI_type_node, V2SI_type_node, NULL_TREE);
21908 tree v2di_ftype_v16qi_v16qi
21909 = build_function_type_list (V2DI_type_node,
21910 V16QI_type_node, V16QI_type_node, NULL_TREE);
21911 tree v2di_ftype_v4si_v4si
21912 = build_function_type_list (V2DI_type_node,
21913 V4SI_type_node, V4SI_type_node, NULL_TREE);
21914 tree int_ftype_v16qi
21915 = build_function_type_list (integer_type_node, V16QI_type_node, NULL_TREE);
21916 tree v16qi_ftype_pcchar
21917 = build_function_type_list (V16QI_type_node, pcchar_type_node, NULL_TREE);
21918 tree void_ftype_pchar_v16qi
21919 = build_function_type_list (void_type_node,
21920 pchar_type_node, V16QI_type_node, NULL_TREE);
21922 tree v2di_ftype_v2di_unsigned_unsigned
21923 = build_function_type_list (V2DI_type_node, V2DI_type_node,
21924 unsigned_type_node, unsigned_type_node,
21926 tree v2di_ftype_v2di_v2di_unsigned_unsigned
21927 = build_function_type_list (V2DI_type_node, V2DI_type_node, V2DI_type_node,
21928 unsigned_type_node, unsigned_type_node,
21930 tree v2di_ftype_v2di_v16qi
21931 = build_function_type_list (V2DI_type_node, V2DI_type_node, V16QI_type_node,
21933 tree v2df_ftype_v2df_v2df_v2df
21934 = build_function_type_list (V2DF_type_node,
21935 V2DF_type_node, V2DF_type_node,
21936 V2DF_type_node, NULL_TREE);
21937 tree v4sf_ftype_v4sf_v4sf_v4sf
21938 = build_function_type_list (V4SF_type_node,
21939 V4SF_type_node, V4SF_type_node,
21940 V4SF_type_node, NULL_TREE);
21941 tree v8hi_ftype_v16qi
21942 = build_function_type_list (V8HI_type_node, V16QI_type_node,
21944 tree v4si_ftype_v16qi
21945 = build_function_type_list (V4SI_type_node, V16QI_type_node,
21947 tree v2di_ftype_v16qi
21948 = build_function_type_list (V2DI_type_node, V16QI_type_node,
21950 tree v4si_ftype_v8hi
21951 = build_function_type_list (V4SI_type_node, V8HI_type_node,
21953 tree v2di_ftype_v8hi
21954 = build_function_type_list (V2DI_type_node, V8HI_type_node,
21956 tree v2di_ftype_v4si
21957 = build_function_type_list (V2DI_type_node, V4SI_type_node,
21959 tree v2di_ftype_pv2di
21960 = build_function_type_list (V2DI_type_node, pv2di_type_node,
21962 tree v16qi_ftype_v16qi_v16qi_int
21963 = build_function_type_list (V16QI_type_node, V16QI_type_node,
21964 V16QI_type_node, integer_type_node,
21966 tree v16qi_ftype_v16qi_v16qi_v16qi
21967 = build_function_type_list (V16QI_type_node, V16QI_type_node,
21968 V16QI_type_node, V16QI_type_node,
21970 tree v8hi_ftype_v8hi_v8hi_int
21971 = build_function_type_list (V8HI_type_node, V8HI_type_node,
21972 V8HI_type_node, integer_type_node,
21974 tree v4si_ftype_v4si_v4si_int
21975 = build_function_type_list (V4SI_type_node, V4SI_type_node,
21976 V4SI_type_node, integer_type_node,
21978 tree int_ftype_v2di_v2di
21979 = build_function_type_list (integer_type_node,
21980 V2DI_type_node, V2DI_type_node,
21982 tree int_ftype_v16qi_int_v16qi_int_int
21983 = build_function_type_list (integer_type_node,
21990 tree v16qi_ftype_v16qi_int_v16qi_int_int
21991 = build_function_type_list (V16QI_type_node,
21998 tree int_ftype_v16qi_v16qi_int
21999 = build_function_type_list (integer_type_node,
22005 /* SSE5 instructions */
22006 tree v2di_ftype_v2di_v2di_v2di
22007 = build_function_type_list (V2DI_type_node,
22013 tree v4si_ftype_v4si_v4si_v4si
22014 = build_function_type_list (V4SI_type_node,
22020 tree v4si_ftype_v4si_v4si_v2di
22021 = build_function_type_list (V4SI_type_node,
22027 tree v8hi_ftype_v8hi_v8hi_v8hi
22028 = build_function_type_list (V8HI_type_node,
22034 tree v8hi_ftype_v8hi_v8hi_v4si
22035 = build_function_type_list (V8HI_type_node,
22041 tree v2df_ftype_v2df_v2df_v16qi
22042 = build_function_type_list (V2DF_type_node,
22048 tree v4sf_ftype_v4sf_v4sf_v16qi
22049 = build_function_type_list (V4SF_type_node,
22055 tree v2di_ftype_v2di_si
22056 = build_function_type_list (V2DI_type_node,
22061 tree v4si_ftype_v4si_si
22062 = build_function_type_list (V4SI_type_node,
22067 tree v8hi_ftype_v8hi_si
22068 = build_function_type_list (V8HI_type_node,
22073 tree v16qi_ftype_v16qi_si
22074 = build_function_type_list (V16QI_type_node,
22078 tree v4sf_ftype_v4hi
22079 = build_function_type_list (V4SF_type_node,
22083 tree v4hi_ftype_v4sf
22084 = build_function_type_list (V4HI_type_node,
22088 tree v2di_ftype_v2di
22089 = build_function_type_list (V2DI_type_node, V2DI_type_node, NULL_TREE);
22091 tree v16qi_ftype_v8hi_v8hi
22092 = build_function_type_list (V16QI_type_node,
22093 V8HI_type_node, V8HI_type_node,
22095 tree v8hi_ftype_v4si_v4si
22096 = build_function_type_list (V8HI_type_node,
22097 V4SI_type_node, V4SI_type_node,
22099 tree v8hi_ftype_v16qi_v16qi
22100 = build_function_type_list (V8HI_type_node,
22101 V16QI_type_node, V16QI_type_node,
22103 tree v4hi_ftype_v8qi_v8qi
22104 = build_function_type_list (V4HI_type_node,
22105 V8QI_type_node, V8QI_type_node,
22107 tree unsigned_ftype_unsigned_uchar
22108 = build_function_type_list (unsigned_type_node,
22109 unsigned_type_node,
22110 unsigned_char_type_node,
22112 tree unsigned_ftype_unsigned_ushort
22113 = build_function_type_list (unsigned_type_node,
22114 unsigned_type_node,
22115 short_unsigned_type_node,
22117 tree unsigned_ftype_unsigned_unsigned
22118 = build_function_type_list (unsigned_type_node,
22119 unsigned_type_node,
22120 unsigned_type_node,
22122 tree uint64_ftype_uint64_uint64
22123 = build_function_type_list (long_long_unsigned_type_node,
22124 long_long_unsigned_type_node,
22125 long_long_unsigned_type_node,
22127 tree float_ftype_float
22128 = build_function_type_list (float_type_node,
22133 tree V32QI_type_node = build_vector_type_for_mode (char_type_node,
22135 tree V8SI_type_node = build_vector_type_for_mode (intSI_type_node,
22137 tree V8SF_type_node = build_vector_type_for_mode (float_type_node,
22139 tree V4DI_type_node = build_vector_type_for_mode (long_long_integer_type_node,
22141 tree V4DF_type_node = build_vector_type_for_mode (double_type_node,
22143 tree v8sf_ftype_v8sf
22144 = build_function_type_list (V8SF_type_node,
22147 tree v8si_ftype_v8sf
22148 = build_function_type_list (V8SI_type_node,
22151 tree v8sf_ftype_v8si
22152 = build_function_type_list (V8SF_type_node,
22155 tree v4si_ftype_v4df
22156 = build_function_type_list (V4SI_type_node,
22159 tree v4df_ftype_v4df
22160 = build_function_type_list (V4DF_type_node,
22163 tree v4df_ftype_v4si
22164 = build_function_type_list (V4DF_type_node,
22167 tree v4df_ftype_v4sf
22168 = build_function_type_list (V4DF_type_node,
22171 tree v4sf_ftype_v4df
22172 = build_function_type_list (V4SF_type_node,
22175 tree v8sf_ftype_v8sf_v8sf
22176 = build_function_type_list (V8SF_type_node,
22177 V8SF_type_node, V8SF_type_node,
22179 tree v4df_ftype_v4df_v4df
22180 = build_function_type_list (V4DF_type_node,
22181 V4DF_type_node, V4DF_type_node,
22183 tree v8sf_ftype_v8sf_int
22184 = build_function_type_list (V8SF_type_node,
22185 V8SF_type_node, integer_type_node,
22187 tree v4si_ftype_v8si_int
22188 = build_function_type_list (V4SI_type_node,
22189 V8SI_type_node, integer_type_node,
22191 tree v4df_ftype_v4df_int
22192 = build_function_type_list (V4DF_type_node,
22193 V4DF_type_node, integer_type_node,
22195 tree v4sf_ftype_v8sf_int
22196 = build_function_type_list (V4SF_type_node,
22197 V8SF_type_node, integer_type_node,
22199 tree v2df_ftype_v4df_int
22200 = build_function_type_list (V2DF_type_node,
22201 V4DF_type_node, integer_type_node,
22203 tree v8sf_ftype_v8sf_v8sf_int
22204 = build_function_type_list (V8SF_type_node,
22205 V8SF_type_node, V8SF_type_node,
22208 tree v8sf_ftype_v8sf_v8sf_v8sf
22209 = build_function_type_list (V8SF_type_node,
22210 V8SF_type_node, V8SF_type_node,
22213 tree v4df_ftype_v4df_v4df_v4df
22214 = build_function_type_list (V4DF_type_node,
22215 V4DF_type_node, V4DF_type_node,
22218 tree v8si_ftype_v8si_v8si_int
22219 = build_function_type_list (V8SI_type_node,
22220 V8SI_type_node, V8SI_type_node,
22223 tree v4df_ftype_v4df_v4df_int
22224 = build_function_type_list (V4DF_type_node,
22225 V4DF_type_node, V4DF_type_node,
22228 tree v8sf_ftype_pcfloat
22229 = build_function_type_list (V8SF_type_node,
22232 tree v4df_ftype_pcdouble
22233 = build_function_type_list (V4DF_type_node,
22234 pcdouble_type_node,
22236 tree pcv4sf_type_node
22237 = build_pointer_type (build_type_variant (V4SF_type_node, 1, 0));
22238 tree pcv2df_type_node
22239 = build_pointer_type (build_type_variant (V2DF_type_node, 1, 0));
22240 tree v8sf_ftype_pcv4sf
22241 = build_function_type_list (V8SF_type_node,
22244 tree v4df_ftype_pcv2df
22245 = build_function_type_list (V4DF_type_node,
22248 tree v32qi_ftype_pcchar
22249 = build_function_type_list (V32QI_type_node,
22252 tree void_ftype_pchar_v32qi
22253 = build_function_type_list (void_type_node,
22254 pchar_type_node, V32QI_type_node,
22256 tree v8si_ftype_v8si_v4si_int
22257 = build_function_type_list (V8SI_type_node,
22258 V8SI_type_node, V4SI_type_node,
22261 tree pv4di_type_node = build_pointer_type (V4DI_type_node);
22262 tree void_ftype_pv4di_v4di
22263 = build_function_type_list (void_type_node,
22264 pv4di_type_node, V4DI_type_node,
22266 tree v8sf_ftype_v8sf_v4sf_int
22267 = build_function_type_list (V8SF_type_node,
22268 V8SF_type_node, V4SF_type_node,
22271 tree v4df_ftype_v4df_v2df_int
22272 = build_function_type_list (V4DF_type_node,
22273 V4DF_type_node, V2DF_type_node,
22276 tree void_ftype_pfloat_v8sf
22277 = build_function_type_list (void_type_node,
22278 pfloat_type_node, V8SF_type_node,
22280 tree void_ftype_pdouble_v4df
22281 = build_function_type_list (void_type_node,
22282 pdouble_type_node, V4DF_type_node,
22284 tree pv8sf_type_node = build_pointer_type (V8SF_type_node);
22285 tree pv4sf_type_node = build_pointer_type (V4SF_type_node);
22286 tree pv4df_type_node = build_pointer_type (V4DF_type_node);
22287 tree pv2df_type_node = build_pointer_type (V2DF_type_node);
22288 tree pcv8sf_type_node
22289 = build_pointer_type (build_type_variant (V8SF_type_node, 1, 0));
22290 tree pcv4df_type_node
22291 = build_pointer_type (build_type_variant (V4DF_type_node, 1, 0));
22292 tree v8sf_ftype_pcv8sf_v8sf
22293 = build_function_type_list (V8SF_type_node,
22294 pcv8sf_type_node, V8SF_type_node,
22296 tree v4df_ftype_pcv4df_v4df
22297 = build_function_type_list (V4DF_type_node,
22298 pcv4df_type_node, V4DF_type_node,
22300 tree v4sf_ftype_pcv4sf_v4sf
22301 = build_function_type_list (V4SF_type_node,
22302 pcv4sf_type_node, V4SF_type_node,
22304 tree v2df_ftype_pcv2df_v2df
22305 = build_function_type_list (V2DF_type_node,
22306 pcv2df_type_node, V2DF_type_node,
22308 tree void_ftype_pv8sf_v8sf_v8sf
22309 = build_function_type_list (void_type_node,
22310 pv8sf_type_node, V8SF_type_node,
22313 tree void_ftype_pv4df_v4df_v4df
22314 = build_function_type_list (void_type_node,
22315 pv4df_type_node, V4DF_type_node,
22318 tree void_ftype_pv4sf_v4sf_v4sf
22319 = build_function_type_list (void_type_node,
22320 pv4sf_type_node, V4SF_type_node,
22323 tree void_ftype_pv2df_v2df_v2df
22324 = build_function_type_list (void_type_node,
22325 pv2df_type_node, V2DF_type_node,
22328 tree v4df_ftype_v2df
22329 = build_function_type_list (V4DF_type_node,
22332 tree v8sf_ftype_v4sf
22333 = build_function_type_list (V8SF_type_node,
22336 tree v8si_ftype_v4si
22337 = build_function_type_list (V8SI_type_node,
22340 tree v2df_ftype_v4df
22341 = build_function_type_list (V2DF_type_node,
22344 tree v4sf_ftype_v8sf
22345 = build_function_type_list (V4SF_type_node,
22348 tree v4si_ftype_v8si
22349 = build_function_type_list (V4SI_type_node,
22352 tree int_ftype_v4df
22353 = build_function_type_list (integer_type_node,
22356 tree int_ftype_v8sf
22357 = build_function_type_list (integer_type_node,
22360 tree int_ftype_v8sf_v8sf
22361 = build_function_type_list (integer_type_node,
22362 V8SF_type_node, V8SF_type_node,
22364 tree int_ftype_v4di_v4di
22365 = build_function_type_list (integer_type_node,
22366 V4DI_type_node, V4DI_type_node,
22368 tree int_ftype_v4df_v4df
22369 = build_function_type_list (integer_type_node,
22370 V4DF_type_node, V4DF_type_node,
22372 tree v8sf_ftype_v8sf_v8si
22373 = build_function_type_list (V8SF_type_node,
22374 V8SF_type_node, V8SI_type_node,
22376 tree v4df_ftype_v4df_v4di
22377 = build_function_type_list (V4DF_type_node,
22378 V4DF_type_node, V4DI_type_node,
22380 tree v4sf_ftype_v4sf_v4si
22381 = build_function_type_list (V4SF_type_node,
22382 V4SF_type_node, V4SI_type_node, NULL_TREE);
22383 tree v2df_ftype_v2df_v2di
22384 = build_function_type_list (V2DF_type_node,
22385 V2DF_type_node, V2DI_type_node, NULL_TREE);
22389 /* Add all special builtins with variable number of operands. */
22390 for (i = 0, d = bdesc_special_args;
22391 i < ARRAY_SIZE (bdesc_special_args);
22399 switch ((enum ix86_special_builtin_type) d->flag)
22401 case VOID_FTYPE_VOID:
22402 type = void_ftype_void;
22404 case V32QI_FTYPE_PCCHAR:
22405 type = v32qi_ftype_pcchar;
22407 case V16QI_FTYPE_PCCHAR:
22408 type = v16qi_ftype_pcchar;
22410 case V8SF_FTYPE_PCV4SF:
22411 type = v8sf_ftype_pcv4sf;
22413 case V8SF_FTYPE_PCFLOAT:
22414 type = v8sf_ftype_pcfloat;
22416 case V4DF_FTYPE_PCV2DF:
22417 type = v4df_ftype_pcv2df;
22419 case V4DF_FTYPE_PCDOUBLE:
22420 type = v4df_ftype_pcdouble;
22422 case V4SF_FTYPE_PCFLOAT:
22423 type = v4sf_ftype_pcfloat;
22425 case V2DI_FTYPE_PV2DI:
22426 type = v2di_ftype_pv2di;
22428 case V2DF_FTYPE_PCDOUBLE:
22429 type = v2df_ftype_pcdouble;
22431 case V8SF_FTYPE_PCV8SF_V8SF:
22432 type = v8sf_ftype_pcv8sf_v8sf;
22434 case V4DF_FTYPE_PCV4DF_V4DF:
22435 type = v4df_ftype_pcv4df_v4df;
22437 case V4SF_FTYPE_V4SF_PCV2SF:
22438 type = v4sf_ftype_v4sf_pcv2sf;
22440 case V4SF_FTYPE_PCV4SF_V4SF:
22441 type = v4sf_ftype_pcv4sf_v4sf;
22443 case V2DF_FTYPE_V2DF_PCDOUBLE:
22444 type = v2df_ftype_v2df_pcdouble;
22446 case V2DF_FTYPE_PCV2DF_V2DF:
22447 type = v2df_ftype_pcv2df_v2df;
22449 case VOID_FTYPE_PV2SF_V4SF:
22450 type = void_ftype_pv2sf_v4sf;
22452 case VOID_FTYPE_PV4DI_V4DI:
22453 type = void_ftype_pv4di_v4di;
22455 case VOID_FTYPE_PV2DI_V2DI:
22456 type = void_ftype_pv2di_v2di;
22458 case VOID_FTYPE_PCHAR_V32QI:
22459 type = void_ftype_pchar_v32qi;
22461 case VOID_FTYPE_PCHAR_V16QI:
22462 type = void_ftype_pchar_v16qi;
22464 case VOID_FTYPE_PFLOAT_V8SF:
22465 type = void_ftype_pfloat_v8sf;
22467 case VOID_FTYPE_PFLOAT_V4SF:
22468 type = void_ftype_pfloat_v4sf;
22470 case VOID_FTYPE_PDOUBLE_V4DF:
22471 type = void_ftype_pdouble_v4df;
22473 case VOID_FTYPE_PDOUBLE_V2DF:
22474 type = void_ftype_pdouble_v2df;
22476 case VOID_FTYPE_PDI_DI:
22477 type = void_ftype_pdi_di;
22479 case VOID_FTYPE_PINT_INT:
22480 type = void_ftype_pint_int;
22482 case VOID_FTYPE_PV8SF_V8SF_V8SF:
22483 type = void_ftype_pv8sf_v8sf_v8sf;
22485 case VOID_FTYPE_PV4DF_V4DF_V4DF:
22486 type = void_ftype_pv4df_v4df_v4df;
22488 case VOID_FTYPE_PV4SF_V4SF_V4SF:
22489 type = void_ftype_pv4sf_v4sf_v4sf;
22491 case VOID_FTYPE_PV2DF_V2DF_V2DF:
22492 type = void_ftype_pv2df_v2df_v2df;
22495 gcc_unreachable ();
22498 def_builtin (d->mask, d->name, type, d->code);
22501 /* Add all builtins with variable number of operands. */
22502 for (i = 0, d = bdesc_args;
22503 i < ARRAY_SIZE (bdesc_args);
22511 switch ((enum ix86_builtin_type) d->flag)
22513 case FLOAT_FTYPE_FLOAT:
22514 type = float_ftype_float;
22516 case INT_FTYPE_V8SF_V8SF_PTEST:
22517 type = int_ftype_v8sf_v8sf;
22519 case INT_FTYPE_V4DI_V4DI_PTEST:
22520 type = int_ftype_v4di_v4di;
22522 case INT_FTYPE_V4DF_V4DF_PTEST:
22523 type = int_ftype_v4df_v4df;
22525 case INT_FTYPE_V4SF_V4SF_PTEST:
22526 type = int_ftype_v4sf_v4sf;
22528 case INT_FTYPE_V2DI_V2DI_PTEST:
22529 type = int_ftype_v2di_v2di;
22531 case INT_FTYPE_V2DF_V2DF_PTEST:
22532 type = int_ftype_v2df_v2df;
22534 case INT64_FTYPE_V4SF:
22535 type = int64_ftype_v4sf;
22537 case INT64_FTYPE_V2DF:
22538 type = int64_ftype_v2df;
22540 case INT_FTYPE_V16QI:
22541 type = int_ftype_v16qi;
22543 case INT_FTYPE_V8QI:
22544 type = int_ftype_v8qi;
22546 case INT_FTYPE_V8SF:
22547 type = int_ftype_v8sf;
22549 case INT_FTYPE_V4DF:
22550 type = int_ftype_v4df;
22552 case INT_FTYPE_V4SF:
22553 type = int_ftype_v4sf;
22555 case INT_FTYPE_V2DF:
22556 type = int_ftype_v2df;
22558 case V16QI_FTYPE_V16QI:
22559 type = v16qi_ftype_v16qi;
22561 case V8SI_FTYPE_V8SF:
22562 type = v8si_ftype_v8sf;
22564 case V8SI_FTYPE_V4SI:
22565 type = v8si_ftype_v4si;
22567 case V8HI_FTYPE_V8HI:
22568 type = v8hi_ftype_v8hi;
22570 case V8HI_FTYPE_V16QI:
22571 type = v8hi_ftype_v16qi;
22573 case V8QI_FTYPE_V8QI:
22574 type = v8qi_ftype_v8qi;
22576 case V8SF_FTYPE_V8SF:
22577 type = v8sf_ftype_v8sf;
22579 case V8SF_FTYPE_V8SI:
22580 type = v8sf_ftype_v8si;
22582 case V8SF_FTYPE_V4SF:
22583 type = v8sf_ftype_v4sf;
22585 case V4SI_FTYPE_V4DF:
22586 type = v4si_ftype_v4df;
22588 case V4SI_FTYPE_V4SI:
22589 type = v4si_ftype_v4si;
22591 case V4SI_FTYPE_V16QI:
22592 type = v4si_ftype_v16qi;
22594 case V4SI_FTYPE_V8SI:
22595 type = v4si_ftype_v8si;
22597 case V4SI_FTYPE_V8HI:
22598 type = v4si_ftype_v8hi;
22600 case V4SI_FTYPE_V4SF:
22601 type = v4si_ftype_v4sf;
22603 case V4SI_FTYPE_V2DF:
22604 type = v4si_ftype_v2df;
22606 case V4HI_FTYPE_V4HI:
22607 type = v4hi_ftype_v4hi;
22609 case V4DF_FTYPE_V4DF:
22610 type = v4df_ftype_v4df;
22612 case V4DF_FTYPE_V4SI:
22613 type = v4df_ftype_v4si;
22615 case V4DF_FTYPE_V4SF:
22616 type = v4df_ftype_v4sf;
22618 case V4DF_FTYPE_V2DF:
22619 type = v4df_ftype_v2df;
22621 case V4SF_FTYPE_V4SF:
22622 case V4SF_FTYPE_V4SF_VEC_MERGE:
22623 type = v4sf_ftype_v4sf;
22625 case V4SF_FTYPE_V8SF:
22626 type = v4sf_ftype_v8sf;
22628 case V4SF_FTYPE_V4SI:
22629 type = v4sf_ftype_v4si;
22631 case V4SF_FTYPE_V4DF:
22632 type = v4sf_ftype_v4df;
22634 case V4SF_FTYPE_V2DF:
22635 type = v4sf_ftype_v2df;
22637 case V2DI_FTYPE_V2DI:
22638 type = v2di_ftype_v2di;
22640 case V2DI_FTYPE_V16QI:
22641 type = v2di_ftype_v16qi;
22643 case V2DI_FTYPE_V8HI:
22644 type = v2di_ftype_v8hi;
22646 case V2DI_FTYPE_V4SI:
22647 type = v2di_ftype_v4si;
22649 case V2SI_FTYPE_V2SI:
22650 type = v2si_ftype_v2si;
22652 case V2SI_FTYPE_V4SF:
22653 type = v2si_ftype_v4sf;
22655 case V2SI_FTYPE_V2DF:
22656 type = v2si_ftype_v2df;
22658 case V2SI_FTYPE_V2SF:
22659 type = v2si_ftype_v2sf;
22661 case V2DF_FTYPE_V4DF:
22662 type = v2df_ftype_v4df;
22664 case V2DF_FTYPE_V4SF:
22665 type = v2df_ftype_v4sf;
22667 case V2DF_FTYPE_V2DF:
22668 case V2DF_FTYPE_V2DF_VEC_MERGE:
22669 type = v2df_ftype_v2df;
22671 case V2DF_FTYPE_V2SI:
22672 type = v2df_ftype_v2si;
22674 case V2DF_FTYPE_V4SI:
22675 type = v2df_ftype_v4si;
22677 case V2SF_FTYPE_V2SF:
22678 type = v2sf_ftype_v2sf;
22680 case V2SF_FTYPE_V2SI:
22681 type = v2sf_ftype_v2si;
22683 case V16QI_FTYPE_V16QI_V16QI:
22684 type = v16qi_ftype_v16qi_v16qi;
22686 case V16QI_FTYPE_V8HI_V8HI:
22687 type = v16qi_ftype_v8hi_v8hi;
22689 case V8QI_FTYPE_V8QI_V8QI:
22690 type = v8qi_ftype_v8qi_v8qi;
22692 case V8QI_FTYPE_V4HI_V4HI:
22693 type = v8qi_ftype_v4hi_v4hi;
22695 case V8HI_FTYPE_V8HI_V8HI:
22696 case V8HI_FTYPE_V8HI_V8HI_COUNT:
22697 type = v8hi_ftype_v8hi_v8hi;
22699 case V8HI_FTYPE_V16QI_V16QI:
22700 type = v8hi_ftype_v16qi_v16qi;
22702 case V8HI_FTYPE_V4SI_V4SI:
22703 type = v8hi_ftype_v4si_v4si;
22705 case V8HI_FTYPE_V8HI_SI_COUNT:
22706 type = v8hi_ftype_v8hi_int;
22708 case V8SF_FTYPE_V8SF_V8SF:
22709 type = v8sf_ftype_v8sf_v8sf;
22711 case V8SF_FTYPE_V8SF_V8SI:
22712 type = v8sf_ftype_v8sf_v8si;
22714 case V4SI_FTYPE_V4SI_V4SI:
22715 case V4SI_FTYPE_V4SI_V4SI_COUNT:
22716 type = v4si_ftype_v4si_v4si;
22718 case V4SI_FTYPE_V8HI_V8HI:
22719 type = v4si_ftype_v8hi_v8hi;
22721 case V4SI_FTYPE_V4SF_V4SF:
22722 type = v4si_ftype_v4sf_v4sf;
22724 case V4SI_FTYPE_V2DF_V2DF:
22725 type = v4si_ftype_v2df_v2df;
22727 case V4SI_FTYPE_V4SI_SI_COUNT:
22728 type = v4si_ftype_v4si_int;
22730 case V4HI_FTYPE_V4HI_V4HI:
22731 case V4HI_FTYPE_V4HI_V4HI_COUNT:
22732 type = v4hi_ftype_v4hi_v4hi;
22734 case V4HI_FTYPE_V8QI_V8QI:
22735 type = v4hi_ftype_v8qi_v8qi;
22737 case V4HI_FTYPE_V2SI_V2SI:
22738 type = v4hi_ftype_v2si_v2si;
22740 case V4HI_FTYPE_V4HI_SI_COUNT:
22741 type = v4hi_ftype_v4hi_int;
22743 case V4DF_FTYPE_V4DF_V4DF:
22744 type = v4df_ftype_v4df_v4df;
22746 case V4DF_FTYPE_V4DF_V4DI:
22747 type = v4df_ftype_v4df_v4di;
22749 case V4SF_FTYPE_V4SF_V4SF:
22750 case V4SF_FTYPE_V4SF_V4SF_SWAP:
22751 type = v4sf_ftype_v4sf_v4sf;
22753 case V4SF_FTYPE_V4SF_V4SI:
22754 type = v4sf_ftype_v4sf_v4si;
22756 case V4SF_FTYPE_V4SF_V2SI:
22757 type = v4sf_ftype_v4sf_v2si;
22759 case V4SF_FTYPE_V4SF_V2DF:
22760 type = v4sf_ftype_v4sf_v2df;
22762 case V4SF_FTYPE_V4SF_DI:
22763 type = v4sf_ftype_v4sf_int64;
22765 case V4SF_FTYPE_V4SF_SI:
22766 type = v4sf_ftype_v4sf_int;
22768 case V2DI_FTYPE_V2DI_V2DI:
22769 case V2DI_FTYPE_V2DI_V2DI_COUNT:
22770 type = v2di_ftype_v2di_v2di;
22772 case V2DI_FTYPE_V16QI_V16QI:
22773 type = v2di_ftype_v16qi_v16qi;
22775 case V2DI_FTYPE_V4SI_V4SI:
22776 type = v2di_ftype_v4si_v4si;
22778 case V2DI_FTYPE_V2DI_V16QI:
22779 type = v2di_ftype_v2di_v16qi;
22781 case V2DI_FTYPE_V2DF_V2DF:
22782 type = v2di_ftype_v2df_v2df;
22784 case V2DI_FTYPE_V2DI_SI_COUNT:
22785 type = v2di_ftype_v2di_int;
22787 case V2SI_FTYPE_V2SI_V2SI:
22788 case V2SI_FTYPE_V2SI_V2SI_COUNT:
22789 type = v2si_ftype_v2si_v2si;
22791 case V2SI_FTYPE_V4HI_V4HI:
22792 type = v2si_ftype_v4hi_v4hi;
22794 case V2SI_FTYPE_V2SF_V2SF:
22795 type = v2si_ftype_v2sf_v2sf;
22797 case V2SI_FTYPE_V2SI_SI_COUNT:
22798 type = v2si_ftype_v2si_int;
22800 case V2DF_FTYPE_V2DF_V2DF:
22801 case V2DF_FTYPE_V2DF_V2DF_SWAP:
22802 type = v2df_ftype_v2df_v2df;
22804 case V2DF_FTYPE_V2DF_V4SF:
22805 type = v2df_ftype_v2df_v4sf;
22807 case V2DF_FTYPE_V2DF_V2DI:
22808 type = v2df_ftype_v2df_v2di;
22810 case V2DF_FTYPE_V2DF_DI:
22811 type = v2df_ftype_v2df_int64;
22813 case V2DF_FTYPE_V2DF_SI:
22814 type = v2df_ftype_v2df_int;
22816 case V2SF_FTYPE_V2SF_V2SF:
22817 type = v2sf_ftype_v2sf_v2sf;
22819 case V1DI_FTYPE_V1DI_V1DI:
22820 case V1DI_FTYPE_V1DI_V1DI_COUNT:
22821 type = v1di_ftype_v1di_v1di;
22823 case V1DI_FTYPE_V8QI_V8QI:
22824 type = v1di_ftype_v8qi_v8qi;
22826 case V1DI_FTYPE_V2SI_V2SI:
22827 type = v1di_ftype_v2si_v2si;
22829 case V1DI_FTYPE_V1DI_SI_COUNT:
22830 type = v1di_ftype_v1di_int;
22832 case UINT64_FTYPE_UINT64_UINT64:
22833 type = uint64_ftype_uint64_uint64;
22835 case UINT_FTYPE_UINT_UINT:
22836 type = unsigned_ftype_unsigned_unsigned;
22838 case UINT_FTYPE_UINT_USHORT:
22839 type = unsigned_ftype_unsigned_ushort;
22841 case UINT_FTYPE_UINT_UCHAR:
22842 type = unsigned_ftype_unsigned_uchar;
22844 case V8HI_FTYPE_V8HI_INT:
22845 type = v8hi_ftype_v8hi_int;
22847 case V8SF_FTYPE_V8SF_INT:
22848 type = v8sf_ftype_v8sf_int;
22850 case V4SI_FTYPE_V4SI_INT:
22851 type = v4si_ftype_v4si_int;
22853 case V4SI_FTYPE_V8SI_INT:
22854 type = v4si_ftype_v8si_int;
22856 case V4HI_FTYPE_V4HI_INT:
22857 type = v4hi_ftype_v4hi_int;
22859 case V4DF_FTYPE_V4DF_INT:
22860 type = v4df_ftype_v4df_int;
22862 case V4SF_FTYPE_V4SF_INT:
22863 type = v4sf_ftype_v4sf_int;
22865 case V4SF_FTYPE_V8SF_INT:
22866 type = v4sf_ftype_v8sf_int;
22868 case V2DI_FTYPE_V2DI_INT:
22869 case V2DI2TI_FTYPE_V2DI_INT:
22870 type = v2di_ftype_v2di_int;
22872 case V2DF_FTYPE_V2DF_INT:
22873 type = v2df_ftype_v2df_int;
22875 case V2DF_FTYPE_V4DF_INT:
22876 type = v2df_ftype_v4df_int;
22878 case V16QI_FTYPE_V16QI_V16QI_V16QI:
22879 type = v16qi_ftype_v16qi_v16qi_v16qi;
22881 case V8SF_FTYPE_V8SF_V8SF_V8SF:
22882 type = v8sf_ftype_v8sf_v8sf_v8sf;
22884 case V4DF_FTYPE_V4DF_V4DF_V4DF:
22885 type = v4df_ftype_v4df_v4df_v4df;
22887 case V4SF_FTYPE_V4SF_V4SF_V4SF:
22888 type = v4sf_ftype_v4sf_v4sf_v4sf;
22890 case V2DF_FTYPE_V2DF_V2DF_V2DF:
22891 type = v2df_ftype_v2df_v2df_v2df;
22893 case V16QI_FTYPE_V16QI_V16QI_INT:
22894 type = v16qi_ftype_v16qi_v16qi_int;
22896 case V8SI_FTYPE_V8SI_V8SI_INT:
22897 type = v8si_ftype_v8si_v8si_int;
22899 case V8SI_FTYPE_V8SI_V4SI_INT:
22900 type = v8si_ftype_v8si_v4si_int;
22902 case V8HI_FTYPE_V8HI_V8HI_INT:
22903 type = v8hi_ftype_v8hi_v8hi_int;
22905 case V8SF_FTYPE_V8SF_V8SF_INT:
22906 type = v8sf_ftype_v8sf_v8sf_int;
22908 case V8SF_FTYPE_V8SF_V4SF_INT:
22909 type = v8sf_ftype_v8sf_v4sf_int;
22911 case V4SI_FTYPE_V4SI_V4SI_INT:
22912 type = v4si_ftype_v4si_v4si_int;
22914 case V4DF_FTYPE_V4DF_V4DF_INT:
22915 type = v4df_ftype_v4df_v4df_int;
22917 case V4DF_FTYPE_V4DF_V2DF_INT:
22918 type = v4df_ftype_v4df_v2df_int;
22920 case V4SF_FTYPE_V4SF_V4SF_INT:
22921 type = v4sf_ftype_v4sf_v4sf_int;
22923 case V2DI_FTYPE_V2DI_V2DI_INT:
22924 case V2DI2TI_FTYPE_V2DI_V2DI_INT:
22925 type = v2di_ftype_v2di_v2di_int;
22927 case V2DF_FTYPE_V2DF_V2DF_INT:
22928 type = v2df_ftype_v2df_v2df_int;
22930 case V2DI_FTYPE_V2DI_UINT_UINT:
22931 type = v2di_ftype_v2di_unsigned_unsigned;
22933 case V2DI_FTYPE_V2DI_V2DI_UINT_UINT:
22934 type = v2di_ftype_v2di_v2di_unsigned_unsigned;
22936 case V1DI2DI_FTYPE_V1DI_V1DI_INT:
22937 type = v1di_ftype_v1di_v1di_int;
22940 gcc_unreachable ();
22943 def_builtin_const (d->mask, d->name, type, d->code);
22946 /* pcmpestr[im] insns. */
22947 for (i = 0, d = bdesc_pcmpestr;
22948 i < ARRAY_SIZE (bdesc_pcmpestr);
22951 if (d->code == IX86_BUILTIN_PCMPESTRM128)
22952 ftype = v16qi_ftype_v16qi_int_v16qi_int_int;
22954 ftype = int_ftype_v16qi_int_v16qi_int_int;
22955 def_builtin_const (d->mask, d->name, ftype, d->code);
22958 /* pcmpistr[im] insns. */
22959 for (i = 0, d = bdesc_pcmpistr;
22960 i < ARRAY_SIZE (bdesc_pcmpistr);
22963 if (d->code == IX86_BUILTIN_PCMPISTRM128)
22964 ftype = v16qi_ftype_v16qi_v16qi_int;
22966 ftype = int_ftype_v16qi_v16qi_int;
22967 def_builtin_const (d->mask, d->name, ftype, d->code);
22970 /* comi/ucomi insns. */
22971 for (i = 0, d = bdesc_comi; i < ARRAY_SIZE (bdesc_comi); i++, d++)
22972 if (d->mask == OPTION_MASK_ISA_SSE2)
22973 def_builtin_const (d->mask, d->name, int_ftype_v2df_v2df, d->code);
22975 def_builtin_const (d->mask, d->name, int_ftype_v4sf_v4sf, d->code);
22978 def_builtin (OPTION_MASK_ISA_SSE, "__builtin_ia32_ldmxcsr", void_ftype_unsigned, IX86_BUILTIN_LDMXCSR);
22979 def_builtin (OPTION_MASK_ISA_SSE, "__builtin_ia32_stmxcsr", unsigned_ftype_void, IX86_BUILTIN_STMXCSR);
22981 /* SSE or 3DNow!A */
22982 def_builtin (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, "__builtin_ia32_maskmovq", void_ftype_v8qi_v8qi_pchar, IX86_BUILTIN_MASKMOVQ);
22985 def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_maskmovdqu", void_ftype_v16qi_v16qi_pchar, IX86_BUILTIN_MASKMOVDQU);
22987 def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_clflush", void_ftype_pcvoid, IX86_BUILTIN_CLFLUSH);
22988 x86_mfence = def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_mfence", void_ftype_void, IX86_BUILTIN_MFENCE);
22991 def_builtin (OPTION_MASK_ISA_SSE3, "__builtin_ia32_monitor", void_ftype_pcvoid_unsigned_unsigned, IX86_BUILTIN_MONITOR);
22992 def_builtin (OPTION_MASK_ISA_SSE3, "__builtin_ia32_mwait", void_ftype_unsigned_unsigned, IX86_BUILTIN_MWAIT);
22995 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesenc128", v2di_ftype_v2di_v2di, IX86_BUILTIN_AESENC128);
22996 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesenclast128", v2di_ftype_v2di_v2di, IX86_BUILTIN_AESENCLAST128);
22997 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesdec128", v2di_ftype_v2di_v2di, IX86_BUILTIN_AESDEC128);
22998 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesdeclast128", v2di_ftype_v2di_v2di, IX86_BUILTIN_AESDECLAST128);
22999 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesimc128", v2di_ftype_v2di, IX86_BUILTIN_AESIMC128);
23000 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aeskeygenassist128", v2di_ftype_v2di_int, IX86_BUILTIN_AESKEYGENASSIST128);
23003 def_builtin_const (OPTION_MASK_ISA_PCLMUL, "__builtin_ia32_pclmulqdq128", v2di_ftype_v2di_v2di_int, IX86_BUILTIN_PCLMULQDQ128);
23006 def_builtin (OPTION_MASK_ISA_AVX, "__builtin_ia32_vzeroupper", void_ftype_void,
23007 TARGET_64BIT ? IX86_BUILTIN_VZEROUPPER_REX64 : IX86_BUILTIN_VZEROUPPER);
23009 /* Access to the vec_init patterns. */
23010 ftype = build_function_type_list (V2SI_type_node, integer_type_node,
23011 integer_type_node, NULL_TREE);
23012 def_builtin_const (OPTION_MASK_ISA_MMX, "__builtin_ia32_vec_init_v2si", ftype, IX86_BUILTIN_VEC_INIT_V2SI);
23014 ftype = build_function_type_list (V4HI_type_node, short_integer_type_node,
23015 short_integer_type_node,
23016 short_integer_type_node,
23017 short_integer_type_node, NULL_TREE);
23018 def_builtin_const (OPTION_MASK_ISA_MMX, "__builtin_ia32_vec_init_v4hi", ftype, IX86_BUILTIN_VEC_INIT_V4HI);
23020 ftype = build_function_type_list (V8QI_type_node, char_type_node,
23021 char_type_node, char_type_node,
23022 char_type_node, char_type_node,
23023 char_type_node, char_type_node,
23024 char_type_node, NULL_TREE);
23025 def_builtin_const (OPTION_MASK_ISA_MMX, "__builtin_ia32_vec_init_v8qi", ftype, IX86_BUILTIN_VEC_INIT_V8QI);
23027 /* Access to the vec_extract patterns. */
23028 ftype = build_function_type_list (double_type_node, V2DF_type_node,
23029 integer_type_node, NULL_TREE);
23030 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v2df", ftype, IX86_BUILTIN_VEC_EXT_V2DF);
23032 ftype = build_function_type_list (long_long_integer_type_node,
23033 V2DI_type_node, integer_type_node,
23035 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v2di", ftype, IX86_BUILTIN_VEC_EXT_V2DI);
23037 ftype = build_function_type_list (float_type_node, V4SF_type_node,
23038 integer_type_node, NULL_TREE);
23039 def_builtin_const (OPTION_MASK_ISA_SSE, "__builtin_ia32_vec_ext_v4sf", ftype, IX86_BUILTIN_VEC_EXT_V4SF);
23041 ftype = build_function_type_list (intSI_type_node, V4SI_type_node,
23042 integer_type_node, NULL_TREE);
23043 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v4si", ftype, IX86_BUILTIN_VEC_EXT_V4SI);
23045 ftype = build_function_type_list (intHI_type_node, V8HI_type_node,
23046 integer_type_node, NULL_TREE);
23047 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v8hi", ftype, IX86_BUILTIN_VEC_EXT_V8HI);
23049 ftype = build_function_type_list (intHI_type_node, V4HI_type_node,
23050 integer_type_node, NULL_TREE);
23051 def_builtin_const (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, "__builtin_ia32_vec_ext_v4hi", ftype, IX86_BUILTIN_VEC_EXT_V4HI);
23053 ftype = build_function_type_list (intSI_type_node, V2SI_type_node,
23054 integer_type_node, NULL_TREE);
23055 def_builtin_const (OPTION_MASK_ISA_MMX, "__builtin_ia32_vec_ext_v2si", ftype, IX86_BUILTIN_VEC_EXT_V2SI);
23057 ftype = build_function_type_list (intQI_type_node, V16QI_type_node,
23058 integer_type_node, NULL_TREE);
23059 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v16qi", ftype, IX86_BUILTIN_VEC_EXT_V16QI);
23061 /* Access to the vec_set patterns. */
23062 ftype = build_function_type_list (V2DI_type_node, V2DI_type_node,
23064 integer_type_node, NULL_TREE);
23065 def_builtin_const (OPTION_MASK_ISA_SSE4_1 | OPTION_MASK_ISA_64BIT, "__builtin_ia32_vec_set_v2di", ftype, IX86_BUILTIN_VEC_SET_V2DI);
23067 ftype = build_function_type_list (V4SF_type_node, V4SF_type_node,
23069 integer_type_node, NULL_TREE);
23070 def_builtin_const (OPTION_MASK_ISA_SSE4_1, "__builtin_ia32_vec_set_v4sf", ftype, IX86_BUILTIN_VEC_SET_V4SF);
23072 ftype = build_function_type_list (V4SI_type_node, V4SI_type_node,
23074 integer_type_node, NULL_TREE);
23075 def_builtin_const (OPTION_MASK_ISA_SSE4_1, "__builtin_ia32_vec_set_v4si", ftype, IX86_BUILTIN_VEC_SET_V4SI);
23077 ftype = build_function_type_list (V8HI_type_node, V8HI_type_node,
23079 integer_type_node, NULL_TREE);
23080 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_set_v8hi", ftype, IX86_BUILTIN_VEC_SET_V8HI);
23082 ftype = build_function_type_list (V4HI_type_node, V4HI_type_node,
23084 integer_type_node, NULL_TREE);
23085 def_builtin_const (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, "__builtin_ia32_vec_set_v4hi", ftype, IX86_BUILTIN_VEC_SET_V4HI);
23087 ftype = build_function_type_list (V16QI_type_node, V16QI_type_node,
23089 integer_type_node, NULL_TREE);
23090 def_builtin_const (OPTION_MASK_ISA_SSE4_1, "__builtin_ia32_vec_set_v16qi", ftype, IX86_BUILTIN_VEC_SET_V16QI);
23092 /* Add SSE5 multi-arg argument instructions */
23093 for (i = 0, d = bdesc_multi_arg; i < ARRAY_SIZE (bdesc_multi_arg); i++, d++)
23095 tree mtype = NULL_TREE;
23100 switch ((enum multi_arg_type)d->flag)
23102 case MULTI_ARG_3_SF: mtype = v4sf_ftype_v4sf_v4sf_v4sf; break;
23103 case MULTI_ARG_3_DF: mtype = v2df_ftype_v2df_v2df_v2df; break;
23104 case MULTI_ARG_3_DI: mtype = v2di_ftype_v2di_v2di_v2di; break;
23105 case MULTI_ARG_3_SI: mtype = v4si_ftype_v4si_v4si_v4si; break;
23106 case MULTI_ARG_3_SI_DI: mtype = v4si_ftype_v4si_v4si_v2di; break;
23107 case MULTI_ARG_3_HI: mtype = v8hi_ftype_v8hi_v8hi_v8hi; break;
23108 case MULTI_ARG_3_HI_SI: mtype = v8hi_ftype_v8hi_v8hi_v4si; break;
23109 case MULTI_ARG_3_QI: mtype = v16qi_ftype_v16qi_v16qi_v16qi; break;
23110 case MULTI_ARG_3_PERMPS: mtype = v4sf_ftype_v4sf_v4sf_v16qi; break;
23111 case MULTI_ARG_3_PERMPD: mtype = v2df_ftype_v2df_v2df_v16qi; break;
23112 case MULTI_ARG_2_SF: mtype = v4sf_ftype_v4sf_v4sf; break;
23113 case MULTI_ARG_2_DF: mtype = v2df_ftype_v2df_v2df; break;
23114 case MULTI_ARG_2_DI: mtype = v2di_ftype_v2di_v2di; break;
23115 case MULTI_ARG_2_SI: mtype = v4si_ftype_v4si_v4si; break;
23116 case MULTI_ARG_2_HI: mtype = v8hi_ftype_v8hi_v8hi; break;
23117 case MULTI_ARG_2_QI: mtype = v16qi_ftype_v16qi_v16qi; break;
23118 case MULTI_ARG_2_DI_IMM: mtype = v2di_ftype_v2di_si; break;
23119 case MULTI_ARG_2_SI_IMM: mtype = v4si_ftype_v4si_si; break;
23120 case MULTI_ARG_2_HI_IMM: mtype = v8hi_ftype_v8hi_si; break;
23121 case MULTI_ARG_2_QI_IMM: mtype = v16qi_ftype_v16qi_si; break;
23122 case MULTI_ARG_2_SF_CMP: mtype = v4sf_ftype_v4sf_v4sf; break;
23123 case MULTI_ARG_2_DF_CMP: mtype = v2df_ftype_v2df_v2df; break;
23124 case MULTI_ARG_2_DI_CMP: mtype = v2di_ftype_v2di_v2di; break;
23125 case MULTI_ARG_2_SI_CMP: mtype = v4si_ftype_v4si_v4si; break;
23126 case MULTI_ARG_2_HI_CMP: mtype = v8hi_ftype_v8hi_v8hi; break;
23127 case MULTI_ARG_2_QI_CMP: mtype = v16qi_ftype_v16qi_v16qi; break;
23128 case MULTI_ARG_2_SF_TF: mtype = v4sf_ftype_v4sf_v4sf; break;
23129 case MULTI_ARG_2_DF_TF: mtype = v2df_ftype_v2df_v2df; break;
23130 case MULTI_ARG_2_DI_TF: mtype = v2di_ftype_v2di_v2di; break;
23131 case MULTI_ARG_2_SI_TF: mtype = v4si_ftype_v4si_v4si; break;
23132 case MULTI_ARG_2_HI_TF: mtype = v8hi_ftype_v8hi_v8hi; break;
23133 case MULTI_ARG_2_QI_TF: mtype = v16qi_ftype_v16qi_v16qi; break;
23134 case MULTI_ARG_1_SF: mtype = v4sf_ftype_v4sf; break;
23135 case MULTI_ARG_1_DF: mtype = v2df_ftype_v2df; break;
23136 case MULTI_ARG_1_DI: mtype = v2di_ftype_v2di; break;
23137 case MULTI_ARG_1_SI: mtype = v4si_ftype_v4si; break;
23138 case MULTI_ARG_1_HI: mtype = v8hi_ftype_v8hi; break;
23139 case MULTI_ARG_1_QI: mtype = v16qi_ftype_v16qi; break;
23140 case MULTI_ARG_1_SI_DI: mtype = v2di_ftype_v4si; break;
23141 case MULTI_ARG_1_HI_DI: mtype = v2di_ftype_v8hi; break;
23142 case MULTI_ARG_1_HI_SI: mtype = v4si_ftype_v8hi; break;
23143 case MULTI_ARG_1_QI_DI: mtype = v2di_ftype_v16qi; break;
23144 case MULTI_ARG_1_QI_SI: mtype = v4si_ftype_v16qi; break;
23145 case MULTI_ARG_1_QI_HI: mtype = v8hi_ftype_v16qi; break;
23146 case MULTI_ARG_1_PH2PS: mtype = v4sf_ftype_v4hi; break;
23147 case MULTI_ARG_1_PS2PH: mtype = v4hi_ftype_v4sf; break;
23148 case MULTI_ARG_UNKNOWN:
23150 gcc_unreachable ();
23154 def_builtin_const (d->mask, d->name, mtype, d->code);
23158 /* Internal method for ix86_init_builtins. */
23161 ix86_init_builtins_va_builtins_abi (void)
23163 tree ms_va_ref, sysv_va_ref;
23164 tree fnvoid_va_end_ms, fnvoid_va_end_sysv;
23165 tree fnvoid_va_start_ms, fnvoid_va_start_sysv;
23166 tree fnvoid_va_copy_ms, fnvoid_va_copy_sysv;
23167 tree fnattr_ms = NULL_TREE, fnattr_sysv = NULL_TREE;
23171 fnattr_ms = build_tree_list (get_identifier ("ms_abi"), NULL_TREE);
23172 fnattr_sysv = build_tree_list (get_identifier ("sysv_abi"), NULL_TREE);
23173 ms_va_ref = build_reference_type (ms_va_list_type_node);
23175 build_pointer_type (TREE_TYPE (sysv_va_list_type_node));
23178 build_function_type_list (void_type_node, ms_va_ref, NULL_TREE);
23179 fnvoid_va_start_ms =
23180 build_varargs_function_type_list (void_type_node, ms_va_ref, NULL_TREE);
23181 fnvoid_va_end_sysv =
23182 build_function_type_list (void_type_node, sysv_va_ref, NULL_TREE);
23183 fnvoid_va_start_sysv =
23184 build_varargs_function_type_list (void_type_node, sysv_va_ref,
23186 fnvoid_va_copy_ms =
23187 build_function_type_list (void_type_node, ms_va_ref, ms_va_list_type_node,
23189 fnvoid_va_copy_sysv =
23190 build_function_type_list (void_type_node, sysv_va_ref,
23191 sysv_va_ref, NULL_TREE);
23193 add_builtin_function ("__builtin_ms_va_start", fnvoid_va_start_ms,
23194 BUILT_IN_VA_START, BUILT_IN_NORMAL, NULL, fnattr_ms);
23195 add_builtin_function ("__builtin_ms_va_end", fnvoid_va_end_ms,
23196 BUILT_IN_VA_END, BUILT_IN_NORMAL, NULL, fnattr_ms);
23197 add_builtin_function ("__builtin_ms_va_copy", fnvoid_va_copy_ms,
23198 BUILT_IN_VA_COPY, BUILT_IN_NORMAL, NULL, fnattr_ms);
23199 add_builtin_function ("__builtin_sysv_va_start", fnvoid_va_start_sysv,
23200 BUILT_IN_VA_START, BUILT_IN_NORMAL, NULL, fnattr_sysv);
23201 add_builtin_function ("__builtin_sysv_va_end", fnvoid_va_end_sysv,
23202 BUILT_IN_VA_END, BUILT_IN_NORMAL, NULL, fnattr_sysv);
23203 add_builtin_function ("__builtin_sysv_va_copy", fnvoid_va_copy_sysv,
23204 BUILT_IN_VA_COPY, BUILT_IN_NORMAL, NULL, fnattr_sysv);
23208 ix86_init_builtins (void)
23210 tree float128_type_node = make_node (REAL_TYPE);
23213 /* The __float80 type. */
23214 if (TYPE_MODE (long_double_type_node) == XFmode)
23215 (*lang_hooks.types.register_builtin_type) (long_double_type_node,
23219 /* The __float80 type. */
23220 tree float80_type_node = make_node (REAL_TYPE);
23222 TYPE_PRECISION (float80_type_node) = 80;
23223 layout_type (float80_type_node);
23224 (*lang_hooks.types.register_builtin_type) (float80_type_node,
23228 /* The __float128 type. */
23229 TYPE_PRECISION (float128_type_node) = 128;
23230 layout_type (float128_type_node);
23231 (*lang_hooks.types.register_builtin_type) (float128_type_node,
23234 /* TFmode support builtins. */
23235 ftype = build_function_type (float128_type_node, void_list_node);
23236 decl = add_builtin_function ("__builtin_infq", ftype,
23237 IX86_BUILTIN_INFQ, BUILT_IN_MD,
23239 ix86_builtins[(int) IX86_BUILTIN_INFQ] = decl;
23241 /* We will expand them to normal call if SSE2 isn't available since
23242 they are used by libgcc. */
23243 ftype = build_function_type_list (float128_type_node,
23244 float128_type_node,
23246 decl = add_builtin_function ("__builtin_fabsq", ftype,
23247 IX86_BUILTIN_FABSQ, BUILT_IN_MD,
23248 "__fabstf2", NULL_TREE);
23249 ix86_builtins[(int) IX86_BUILTIN_FABSQ] = decl;
23250 TREE_READONLY (decl) = 1;
23252 ftype = build_function_type_list (float128_type_node,
23253 float128_type_node,
23254 float128_type_node,
23256 decl = add_builtin_function ("__builtin_copysignq", ftype,
23257 IX86_BUILTIN_COPYSIGNQ, BUILT_IN_MD,
23258 "__copysigntf3", NULL_TREE);
23259 ix86_builtins[(int) IX86_BUILTIN_COPYSIGNQ] = decl;
23260 TREE_READONLY (decl) = 1;
23262 ix86_init_mmx_sse_builtins ();
23264 ix86_init_builtins_va_builtins_abi ();
23267 /* Errors in the source file can cause expand_expr to return const0_rtx
23268 where we expect a vector. To avoid crashing, use one of the vector
23269 clear instructions. */
23271 safe_vector_operand (rtx x, enum machine_mode mode)
23273 if (x == const0_rtx)
23274 x = CONST0_RTX (mode);
23278 /* Subroutine of ix86_expand_builtin to take care of binop insns. */
23281 ix86_expand_binop_builtin (enum insn_code icode, tree exp, rtx target)
23284 tree arg0 = CALL_EXPR_ARG (exp, 0);
23285 tree arg1 = CALL_EXPR_ARG (exp, 1);
23286 rtx op0 = expand_normal (arg0);
23287 rtx op1 = expand_normal (arg1);
23288 enum machine_mode tmode = insn_data[icode].operand[0].mode;
23289 enum machine_mode mode0 = insn_data[icode].operand[1].mode;
23290 enum machine_mode mode1 = insn_data[icode].operand[2].mode;
23292 if (VECTOR_MODE_P (mode0))
23293 op0 = safe_vector_operand (op0, mode0);
23294 if (VECTOR_MODE_P (mode1))
23295 op1 = safe_vector_operand (op1, mode1);
23297 if (optimize || !target
23298 || GET_MODE (target) != tmode
23299 || ! (*insn_data[icode].operand[0].predicate) (target, tmode))
23300 target = gen_reg_rtx (tmode);
23302 if (GET_MODE (op1) == SImode && mode1 == TImode)
23304 rtx x = gen_reg_rtx (V4SImode);
23305 emit_insn (gen_sse2_loadd (x, op1));
23306 op1 = gen_lowpart (TImode, x);
23309 if (!(*insn_data[icode].operand[1].predicate) (op0, mode0))
23310 op0 = copy_to_mode_reg (mode0, op0);
23311 if (!(*insn_data[icode].operand[2].predicate) (op1, mode1))
23312 op1 = copy_to_mode_reg (mode1, op1);
23314 pat = GEN_FCN (icode) (target, op0, op1);
23323 /* Subroutine of ix86_expand_builtin to take care of 2-4 argument insns. */
23326 ix86_expand_multi_arg_builtin (enum insn_code icode, tree exp, rtx target,
23327 enum multi_arg_type m_type,
23328 enum insn_code sub_code)
23333 bool comparison_p = false;
23335 bool last_arg_constant = false;
23336 int num_memory = 0;
23339 enum machine_mode mode;
23342 enum machine_mode tmode = insn_data[icode].operand[0].mode;
23346 case MULTI_ARG_3_SF:
23347 case MULTI_ARG_3_DF:
23348 case MULTI_ARG_3_DI:
23349 case MULTI_ARG_3_SI:
23350 case MULTI_ARG_3_SI_DI:
23351 case MULTI_ARG_3_HI:
23352 case MULTI_ARG_3_HI_SI:
23353 case MULTI_ARG_3_QI:
23354 case MULTI_ARG_3_PERMPS:
23355 case MULTI_ARG_3_PERMPD:
23359 case MULTI_ARG_2_SF:
23360 case MULTI_ARG_2_DF:
23361 case MULTI_ARG_2_DI:
23362 case MULTI_ARG_2_SI:
23363 case MULTI_ARG_2_HI:
23364 case MULTI_ARG_2_QI:
23368 case MULTI_ARG_2_DI_IMM:
23369 case MULTI_ARG_2_SI_IMM:
23370 case MULTI_ARG_2_HI_IMM:
23371 case MULTI_ARG_2_QI_IMM:
23373 last_arg_constant = true;
23376 case MULTI_ARG_1_SF:
23377 case MULTI_ARG_1_DF:
23378 case MULTI_ARG_1_DI:
23379 case MULTI_ARG_1_SI:
23380 case MULTI_ARG_1_HI:
23381 case MULTI_ARG_1_QI:
23382 case MULTI_ARG_1_SI_DI:
23383 case MULTI_ARG_1_HI_DI:
23384 case MULTI_ARG_1_HI_SI:
23385 case MULTI_ARG_1_QI_DI:
23386 case MULTI_ARG_1_QI_SI:
23387 case MULTI_ARG_1_QI_HI:
23388 case MULTI_ARG_1_PH2PS:
23389 case MULTI_ARG_1_PS2PH:
23393 case MULTI_ARG_2_SF_CMP:
23394 case MULTI_ARG_2_DF_CMP:
23395 case MULTI_ARG_2_DI_CMP:
23396 case MULTI_ARG_2_SI_CMP:
23397 case MULTI_ARG_2_HI_CMP:
23398 case MULTI_ARG_2_QI_CMP:
23400 comparison_p = true;
23403 case MULTI_ARG_2_SF_TF:
23404 case MULTI_ARG_2_DF_TF:
23405 case MULTI_ARG_2_DI_TF:
23406 case MULTI_ARG_2_SI_TF:
23407 case MULTI_ARG_2_HI_TF:
23408 case MULTI_ARG_2_QI_TF:
23413 case MULTI_ARG_UNKNOWN:
23415 gcc_unreachable ();
23418 if (optimize || !target
23419 || GET_MODE (target) != tmode
23420 || ! (*insn_data[icode].operand[0].predicate) (target, tmode))
23421 target = gen_reg_rtx (tmode);
23423 gcc_assert (nargs <= 4);
23425 for (i = 0; i < nargs; i++)
23427 tree arg = CALL_EXPR_ARG (exp, i);
23428 rtx op = expand_normal (arg);
23429 int adjust = (comparison_p) ? 1 : 0;
23430 enum machine_mode mode = insn_data[icode].operand[i+adjust+1].mode;
23432 if (last_arg_constant && i == nargs-1)
23434 if (GET_CODE (op) != CONST_INT)
23436 error ("last argument must be an immediate");
23437 return gen_reg_rtx (tmode);
23442 if (VECTOR_MODE_P (mode))
23443 op = safe_vector_operand (op, mode);
23445 /* If we aren't optimizing, only allow one memory operand to be
23447 if (memory_operand (op, mode))
23450 gcc_assert (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode);
23453 || ! (*insn_data[icode].operand[i+adjust+1].predicate) (op, mode)
23455 op = force_reg (mode, op);
23459 args[i].mode = mode;
23465 pat = GEN_FCN (icode) (target, args[0].op);
23470 pat = GEN_FCN (icode) (target, args[0].op, args[1].op,
23471 GEN_INT ((int)sub_code));
23472 else if (! comparison_p)
23473 pat = GEN_FCN (icode) (target, args[0].op, args[1].op);
23476 rtx cmp_op = gen_rtx_fmt_ee (sub_code, GET_MODE (target),
23480 pat = GEN_FCN (icode) (target, cmp_op, args[0].op, args[1].op);
23485 pat = GEN_FCN (icode) (target, args[0].op, args[1].op, args[2].op);
23489 gcc_unreachable ();
23499 /* Subroutine of ix86_expand_args_builtin to take care of scalar unop
23500 insns with vec_merge. */
23503 ix86_expand_unop_vec_merge_builtin (enum insn_code icode, tree exp,
23507 tree arg0 = CALL_EXPR_ARG (exp, 0);
23508 rtx op1, op0 = expand_normal (arg0);
23509 enum machine_mode tmode = insn_data[icode].operand[0].mode;
23510 enum machine_mode mode0 = insn_data[icode].operand[1].mode;
23512 if (optimize || !target
23513 || GET_MODE (target) != tmode
23514 || ! (*insn_data[icode].operand[0].predicate) (target, tmode))
23515 target = gen_reg_rtx (tmode);
23517 if (VECTOR_MODE_P (mode0))
23518 op0 = safe_vector_operand (op0, mode0);
23520 if ((optimize && !register_operand (op0, mode0))
23521 || ! (*insn_data[icode].operand[1].predicate) (op0, mode0))
23522 op0 = copy_to_mode_reg (mode0, op0);
23525 if (! (*insn_data[icode].operand[2].predicate) (op1, mode0))
23526 op1 = copy_to_mode_reg (mode0, op1);
23528 pat = GEN_FCN (icode) (target, op0, op1);
23535 /* Subroutine of ix86_expand_builtin to take care of comparison insns. */
23538 ix86_expand_sse_compare (const struct builtin_description *d,
23539 tree exp, rtx target, bool swap)
23542 tree arg0 = CALL_EXPR_ARG (exp, 0);
23543 tree arg1 = CALL_EXPR_ARG (exp, 1);
23544 rtx op0 = expand_normal (arg0);
23545 rtx op1 = expand_normal (arg1);
23547 enum machine_mode tmode = insn_data[d->icode].operand[0].mode;
23548 enum machine_mode mode0 = insn_data[d->icode].operand[1].mode;
23549 enum machine_mode mode1 = insn_data[d->icode].operand[2].mode;
23550 enum rtx_code comparison = d->comparison;
23552 if (VECTOR_MODE_P (mode0))
23553 op0 = safe_vector_operand (op0, mode0);
23554 if (VECTOR_MODE_P (mode1))
23555 op1 = safe_vector_operand (op1, mode1);
23557 /* Swap operands if we have a comparison that isn't available in
23561 rtx tmp = gen_reg_rtx (mode1);
23562 emit_move_insn (tmp, op1);
23567 if (optimize || !target
23568 || GET_MODE (target) != tmode
23569 || ! (*insn_data[d->icode].operand[0].predicate) (target, tmode))
23570 target = gen_reg_rtx (tmode);
23572 if ((optimize && !register_operand (op0, mode0))
23573 || ! (*insn_data[d->icode].operand[1].predicate) (op0, mode0))
23574 op0 = copy_to_mode_reg (mode0, op0);
23575 if ((optimize && !register_operand (op1, mode1))
23576 || ! (*insn_data[d->icode].operand[2].predicate) (op1, mode1))
23577 op1 = copy_to_mode_reg (mode1, op1);
23579 op2 = gen_rtx_fmt_ee (comparison, mode0, op0, op1);
23580 pat = GEN_FCN (d->icode) (target, op0, op1, op2);
23587 /* Subroutine of ix86_expand_builtin to take care of comi insns. */
23590 ix86_expand_sse_comi (const struct builtin_description *d, tree exp,
23594 tree arg0 = CALL_EXPR_ARG (exp, 0);
23595 tree arg1 = CALL_EXPR_ARG (exp, 1);
23596 rtx op0 = expand_normal (arg0);
23597 rtx op1 = expand_normal (arg1);
23598 enum machine_mode mode0 = insn_data[d->icode].operand[0].mode;
23599 enum machine_mode mode1 = insn_data[d->icode].operand[1].mode;
23600 enum rtx_code comparison = d->comparison;
23602 if (VECTOR_MODE_P (mode0))
23603 op0 = safe_vector_operand (op0, mode0);
23604 if (VECTOR_MODE_P (mode1))
23605 op1 = safe_vector_operand (op1, mode1);
23607 /* Swap operands if we have a comparison that isn't available in
23609 if (d->flag & BUILTIN_DESC_SWAP_OPERANDS)
23616 target = gen_reg_rtx (SImode);
23617 emit_move_insn (target, const0_rtx);
23618 target = gen_rtx_SUBREG (QImode, target, 0);
23620 if ((optimize && !register_operand (op0, mode0))
23621 || !(*insn_data[d->icode].operand[0].predicate) (op0, mode0))
23622 op0 = copy_to_mode_reg (mode0, op0);
23623 if ((optimize && !register_operand (op1, mode1))
23624 || !(*insn_data[d->icode].operand[1].predicate) (op1, mode1))
23625 op1 = copy_to_mode_reg (mode1, op1);
23627 pat = GEN_FCN (d->icode) (op0, op1);
23631 emit_insn (gen_rtx_SET (VOIDmode,
23632 gen_rtx_STRICT_LOW_PART (VOIDmode, target),
23633 gen_rtx_fmt_ee (comparison, QImode,
23637 return SUBREG_REG (target);
23640 /* Subroutine of ix86_expand_builtin to take care of ptest insns. */
23643 ix86_expand_sse_ptest (const struct builtin_description *d, tree exp,
23647 tree arg0 = CALL_EXPR_ARG (exp, 0);
23648 tree arg1 = CALL_EXPR_ARG (exp, 1);
23649 rtx op0 = expand_normal (arg0);
23650 rtx op1 = expand_normal (arg1);
23651 enum machine_mode mode0 = insn_data[d->icode].operand[0].mode;
23652 enum machine_mode mode1 = insn_data[d->icode].operand[1].mode;
23653 enum rtx_code comparison = d->comparison;
23655 if (VECTOR_MODE_P (mode0))
23656 op0 = safe_vector_operand (op0, mode0);
23657 if (VECTOR_MODE_P (mode1))
23658 op1 = safe_vector_operand (op1, mode1);
23660 target = gen_reg_rtx (SImode);
23661 emit_move_insn (target, const0_rtx);
23662 target = gen_rtx_SUBREG (QImode, target, 0);
23664 if ((optimize && !register_operand (op0, mode0))
23665 || !(*insn_data[d->icode].operand[0].predicate) (op0, mode0))
23666 op0 = copy_to_mode_reg (mode0, op0);
23667 if ((optimize && !register_operand (op1, mode1))
23668 || !(*insn_data[d->icode].operand[1].predicate) (op1, mode1))
23669 op1 = copy_to_mode_reg (mode1, op1);
23671 pat = GEN_FCN (d->icode) (op0, op1);
23675 emit_insn (gen_rtx_SET (VOIDmode,
23676 gen_rtx_STRICT_LOW_PART (VOIDmode, target),
23677 gen_rtx_fmt_ee (comparison, QImode,
23681 return SUBREG_REG (target);
23684 /* Subroutine of ix86_expand_builtin to take care of pcmpestr[im] insns. */
23687 ix86_expand_sse_pcmpestr (const struct builtin_description *d,
23688 tree exp, rtx target)
23691 tree arg0 = CALL_EXPR_ARG (exp, 0);
23692 tree arg1 = CALL_EXPR_ARG (exp, 1);
23693 tree arg2 = CALL_EXPR_ARG (exp, 2);
23694 tree arg3 = CALL_EXPR_ARG (exp, 3);
23695 tree arg4 = CALL_EXPR_ARG (exp, 4);
23696 rtx scratch0, scratch1;
23697 rtx op0 = expand_normal (arg0);
23698 rtx op1 = expand_normal (arg1);
23699 rtx op2 = expand_normal (arg2);
23700 rtx op3 = expand_normal (arg3);
23701 rtx op4 = expand_normal (arg4);
23702 enum machine_mode tmode0, tmode1, modev2, modei3, modev4, modei5, modeimm;
23704 tmode0 = insn_data[d->icode].operand[0].mode;
23705 tmode1 = insn_data[d->icode].operand[1].mode;
23706 modev2 = insn_data[d->icode].operand[2].mode;
23707 modei3 = insn_data[d->icode].operand[3].mode;
23708 modev4 = insn_data[d->icode].operand[4].mode;
23709 modei5 = insn_data[d->icode].operand[5].mode;
23710 modeimm = insn_data[d->icode].operand[6].mode;
23712 if (VECTOR_MODE_P (modev2))
23713 op0 = safe_vector_operand (op0, modev2);
23714 if (VECTOR_MODE_P (modev4))
23715 op2 = safe_vector_operand (op2, modev4);
23717 if (! (*insn_data[d->icode].operand[2].predicate) (op0, modev2))
23718 op0 = copy_to_mode_reg (modev2, op0);
23719 if (! (*insn_data[d->icode].operand[3].predicate) (op1, modei3))
23720 op1 = copy_to_mode_reg (modei3, op1);
23721 if ((optimize && !register_operand (op2, modev4))
23722 || !(*insn_data[d->icode].operand[4].predicate) (op2, modev4))
23723 op2 = copy_to_mode_reg (modev4, op2);
23724 if (! (*insn_data[d->icode].operand[5].predicate) (op3, modei5))
23725 op3 = copy_to_mode_reg (modei5, op3);
23727 if (! (*insn_data[d->icode].operand[6].predicate) (op4, modeimm))
23729 error ("the fifth argument must be a 8-bit immediate");
23733 if (d->code == IX86_BUILTIN_PCMPESTRI128)
23735 if (optimize || !target
23736 || GET_MODE (target) != tmode0
23737 || ! (*insn_data[d->icode].operand[0].predicate) (target, tmode0))
23738 target = gen_reg_rtx (tmode0);
23740 scratch1 = gen_reg_rtx (tmode1);
23742 pat = GEN_FCN (d->icode) (target, scratch1, op0, op1, op2, op3, op4);
23744 else if (d->code == IX86_BUILTIN_PCMPESTRM128)
23746 if (optimize || !target
23747 || GET_MODE (target) != tmode1
23748 || ! (*insn_data[d->icode].operand[1].predicate) (target, tmode1))
23749 target = gen_reg_rtx (tmode1);
23751 scratch0 = gen_reg_rtx (tmode0);
23753 pat = GEN_FCN (d->icode) (scratch0, target, op0, op1, op2, op3, op4);
23757 gcc_assert (d->flag);
23759 scratch0 = gen_reg_rtx (tmode0);
23760 scratch1 = gen_reg_rtx (tmode1);
23762 pat = GEN_FCN (d->icode) (scratch0, scratch1, op0, op1, op2, op3, op4);
23772 target = gen_reg_rtx (SImode);
23773 emit_move_insn (target, const0_rtx);
23774 target = gen_rtx_SUBREG (QImode, target, 0);
23777 (gen_rtx_SET (VOIDmode, gen_rtx_STRICT_LOW_PART (VOIDmode, target),
23778 gen_rtx_fmt_ee (EQ, QImode,
23779 gen_rtx_REG ((enum machine_mode) d->flag,
23782 return SUBREG_REG (target);
23789 /* Subroutine of ix86_expand_builtin to take care of pcmpistr[im] insns. */
23792 ix86_expand_sse_pcmpistr (const struct builtin_description *d,
23793 tree exp, rtx target)
23796 tree arg0 = CALL_EXPR_ARG (exp, 0);
23797 tree arg1 = CALL_EXPR_ARG (exp, 1);
23798 tree arg2 = CALL_EXPR_ARG (exp, 2);
23799 rtx scratch0, scratch1;
23800 rtx op0 = expand_normal (arg0);
23801 rtx op1 = expand_normal (arg1);
23802 rtx op2 = expand_normal (arg2);
23803 enum machine_mode tmode0, tmode1, modev2, modev3, modeimm;
23805 tmode0 = insn_data[d->icode].operand[0].mode;
23806 tmode1 = insn_data[d->icode].operand[1].mode;
23807 modev2 = insn_data[d->icode].operand[2].mode;
23808 modev3 = insn_data[d->icode].operand[3].mode;
23809 modeimm = insn_data[d->icode].operand[4].mode;
23811 if (VECTOR_MODE_P (modev2))
23812 op0 = safe_vector_operand (op0, modev2);
23813 if (VECTOR_MODE_P (modev3))
23814 op1 = safe_vector_operand (op1, modev3);
23816 if (! (*insn_data[d->icode].operand[2].predicate) (op0, modev2))
23817 op0 = copy_to_mode_reg (modev2, op0);
23818 if ((optimize && !register_operand (op1, modev3))
23819 || !(*insn_data[d->icode].operand[3].predicate) (op1, modev3))
23820 op1 = copy_to_mode_reg (modev3, op1);
23822 if (! (*insn_data[d->icode].operand[4].predicate) (op2, modeimm))
23824 error ("the third argument must be a 8-bit immediate");
23828 if (d->code == IX86_BUILTIN_PCMPISTRI128)
23830 if (optimize || !target
23831 || GET_MODE (target) != tmode0
23832 || ! (*insn_data[d->icode].operand[0].predicate) (target, tmode0))
23833 target = gen_reg_rtx (tmode0);
23835 scratch1 = gen_reg_rtx (tmode1);
23837 pat = GEN_FCN (d->icode) (target, scratch1, op0, op1, op2);
23839 else if (d->code == IX86_BUILTIN_PCMPISTRM128)
23841 if (optimize || !target
23842 || GET_MODE (target) != tmode1
23843 || ! (*insn_data[d->icode].operand[1].predicate) (target, tmode1))
23844 target = gen_reg_rtx (tmode1);
23846 scratch0 = gen_reg_rtx (tmode0);
23848 pat = GEN_FCN (d->icode) (scratch0, target, op0, op1, op2);
23852 gcc_assert (d->flag);
23854 scratch0 = gen_reg_rtx (tmode0);
23855 scratch1 = gen_reg_rtx (tmode1);
23857 pat = GEN_FCN (d->icode) (scratch0, scratch1, op0, op1, op2);
23867 target = gen_reg_rtx (SImode);
23868 emit_move_insn (target, const0_rtx);
23869 target = gen_rtx_SUBREG (QImode, target, 0);
23872 (gen_rtx_SET (VOIDmode, gen_rtx_STRICT_LOW_PART (VOIDmode, target),
23873 gen_rtx_fmt_ee (EQ, QImode,
23874 gen_rtx_REG ((enum machine_mode) d->flag,
23877 return SUBREG_REG (target);
23883 /* Subroutine of ix86_expand_builtin to take care of insns with
23884 variable number of operands. */
23887 ix86_expand_args_builtin (const struct builtin_description *d,
23888 tree exp, rtx target)
23890 rtx pat, real_target;
23891 unsigned int i, nargs;
23892 unsigned int nargs_constant = 0;
23893 int num_memory = 0;
23897 enum machine_mode mode;
23899 bool last_arg_count = false;
23900 enum insn_code icode = d->icode;
23901 const struct insn_data *insn_p = &insn_data[icode];
23902 enum machine_mode tmode = insn_p->operand[0].mode;
23903 enum machine_mode rmode = VOIDmode;
23905 enum rtx_code comparison = d->comparison;
23907 switch ((enum ix86_builtin_type) d->flag)
23909 case INT_FTYPE_V8SF_V8SF_PTEST:
23910 case INT_FTYPE_V4DI_V4DI_PTEST:
23911 case INT_FTYPE_V4DF_V4DF_PTEST:
23912 case INT_FTYPE_V4SF_V4SF_PTEST:
23913 case INT_FTYPE_V2DI_V2DI_PTEST:
23914 case INT_FTYPE_V2DF_V2DF_PTEST:
23915 return ix86_expand_sse_ptest (d, exp, target);
23916 case FLOAT128_FTYPE_FLOAT128:
23917 case FLOAT_FTYPE_FLOAT:
23918 case INT64_FTYPE_V4SF:
23919 case INT64_FTYPE_V2DF:
23920 case INT_FTYPE_V16QI:
23921 case INT_FTYPE_V8QI:
23922 case INT_FTYPE_V8SF:
23923 case INT_FTYPE_V4DF:
23924 case INT_FTYPE_V4SF:
23925 case INT_FTYPE_V2DF:
23926 case V16QI_FTYPE_V16QI:
23927 case V8SI_FTYPE_V8SF:
23928 case V8SI_FTYPE_V4SI:
23929 case V8HI_FTYPE_V8HI:
23930 case V8HI_FTYPE_V16QI:
23931 case V8QI_FTYPE_V8QI:
23932 case V8SF_FTYPE_V8SF:
23933 case V8SF_FTYPE_V8SI:
23934 case V8SF_FTYPE_V4SF:
23935 case V4SI_FTYPE_V4SI:
23936 case V4SI_FTYPE_V16QI:
23937 case V4SI_FTYPE_V4SF:
23938 case V4SI_FTYPE_V8SI:
23939 case V4SI_FTYPE_V8HI:
23940 case V4SI_FTYPE_V4DF:
23941 case V4SI_FTYPE_V2DF:
23942 case V4HI_FTYPE_V4HI:
23943 case V4DF_FTYPE_V4DF:
23944 case V4DF_FTYPE_V4SI:
23945 case V4DF_FTYPE_V4SF:
23946 case V4DF_FTYPE_V2DF:
23947 case V4SF_FTYPE_V4SF:
23948 case V4SF_FTYPE_V4SI:
23949 case V4SF_FTYPE_V8SF:
23950 case V4SF_FTYPE_V4DF:
23951 case V4SF_FTYPE_V2DF:
23952 case V2DI_FTYPE_V2DI:
23953 case V2DI_FTYPE_V16QI:
23954 case V2DI_FTYPE_V8HI:
23955 case V2DI_FTYPE_V4SI:
23956 case V2DF_FTYPE_V2DF:
23957 case V2DF_FTYPE_V4SI:
23958 case V2DF_FTYPE_V4DF:
23959 case V2DF_FTYPE_V4SF:
23960 case V2DF_FTYPE_V2SI:
23961 case V2SI_FTYPE_V2SI:
23962 case V2SI_FTYPE_V4SF:
23963 case V2SI_FTYPE_V2SF:
23964 case V2SI_FTYPE_V2DF:
23965 case V2SF_FTYPE_V2SF:
23966 case V2SF_FTYPE_V2SI:
23969 case V4SF_FTYPE_V4SF_VEC_MERGE:
23970 case V2DF_FTYPE_V2DF_VEC_MERGE:
23971 return ix86_expand_unop_vec_merge_builtin (icode, exp, target);
23972 case FLOAT128_FTYPE_FLOAT128_FLOAT128:
23973 case V16QI_FTYPE_V16QI_V16QI:
23974 case V16QI_FTYPE_V8HI_V8HI:
23975 case V8QI_FTYPE_V8QI_V8QI:
23976 case V8QI_FTYPE_V4HI_V4HI:
23977 case V8HI_FTYPE_V8HI_V8HI:
23978 case V8HI_FTYPE_V16QI_V16QI:
23979 case V8HI_FTYPE_V4SI_V4SI:
23980 case V8SF_FTYPE_V8SF_V8SF:
23981 case V8SF_FTYPE_V8SF_V8SI:
23982 case V4SI_FTYPE_V4SI_V4SI:
23983 case V4SI_FTYPE_V8HI_V8HI:
23984 case V4SI_FTYPE_V4SF_V4SF:
23985 case V4SI_FTYPE_V2DF_V2DF:
23986 case V4HI_FTYPE_V4HI_V4HI:
23987 case V4HI_FTYPE_V8QI_V8QI:
23988 case V4HI_FTYPE_V2SI_V2SI:
23989 case V4DF_FTYPE_V4DF_V4DF:
23990 case V4DF_FTYPE_V4DF_V4DI:
23991 case V4SF_FTYPE_V4SF_V4SF:
23992 case V4SF_FTYPE_V4SF_V4SI:
23993 case V4SF_FTYPE_V4SF_V2SI:
23994 case V4SF_FTYPE_V4SF_V2DF:
23995 case V4SF_FTYPE_V4SF_DI:
23996 case V4SF_FTYPE_V4SF_SI:
23997 case V2DI_FTYPE_V2DI_V2DI:
23998 case V2DI_FTYPE_V16QI_V16QI:
23999 case V2DI_FTYPE_V4SI_V4SI:
24000 case V2DI_FTYPE_V2DI_V16QI:
24001 case V2DI_FTYPE_V2DF_V2DF:
24002 case V2SI_FTYPE_V2SI_V2SI:
24003 case V2SI_FTYPE_V4HI_V4HI:
24004 case V2SI_FTYPE_V2SF_V2SF:
24005 case V2DF_FTYPE_V2DF_V2DF:
24006 case V2DF_FTYPE_V2DF_V4SF:
24007 case V2DF_FTYPE_V2DF_V2DI:
24008 case V2DF_FTYPE_V2DF_DI:
24009 case V2DF_FTYPE_V2DF_SI:
24010 case V2SF_FTYPE_V2SF_V2SF:
24011 case V1DI_FTYPE_V1DI_V1DI:
24012 case V1DI_FTYPE_V8QI_V8QI:
24013 case V1DI_FTYPE_V2SI_V2SI:
24014 if (comparison == UNKNOWN)
24015 return ix86_expand_binop_builtin (icode, exp, target);
24018 case V4SF_FTYPE_V4SF_V4SF_SWAP:
24019 case V2DF_FTYPE_V2DF_V2DF_SWAP:
24020 gcc_assert (comparison != UNKNOWN);
24024 case V8HI_FTYPE_V8HI_V8HI_COUNT:
24025 case V8HI_FTYPE_V8HI_SI_COUNT:
24026 case V4SI_FTYPE_V4SI_V4SI_COUNT:
24027 case V4SI_FTYPE_V4SI_SI_COUNT:
24028 case V4HI_FTYPE_V4HI_V4HI_COUNT:
24029 case V4HI_FTYPE_V4HI_SI_COUNT:
24030 case V2DI_FTYPE_V2DI_V2DI_COUNT:
24031 case V2DI_FTYPE_V2DI_SI_COUNT:
24032 case V2SI_FTYPE_V2SI_V2SI_COUNT:
24033 case V2SI_FTYPE_V2SI_SI_COUNT:
24034 case V1DI_FTYPE_V1DI_V1DI_COUNT:
24035 case V1DI_FTYPE_V1DI_SI_COUNT:
24037 last_arg_count = true;
24039 case UINT64_FTYPE_UINT64_UINT64:
24040 case UINT_FTYPE_UINT_UINT:
24041 case UINT_FTYPE_UINT_USHORT:
24042 case UINT_FTYPE_UINT_UCHAR:
24045 case V2DI2TI_FTYPE_V2DI_INT:
24048 nargs_constant = 1;
24050 case V8HI_FTYPE_V8HI_INT:
24051 case V8SF_FTYPE_V8SF_INT:
24052 case V4SI_FTYPE_V4SI_INT:
24053 case V4SI_FTYPE_V8SI_INT:
24054 case V4HI_FTYPE_V4HI_INT:
24055 case V4DF_FTYPE_V4DF_INT:
24056 case V4SF_FTYPE_V4SF_INT:
24057 case V4SF_FTYPE_V8SF_INT:
24058 case V2DI_FTYPE_V2DI_INT:
24059 case V2DF_FTYPE_V2DF_INT:
24060 case V2DF_FTYPE_V4DF_INT:
24062 nargs_constant = 1;
24064 case V16QI_FTYPE_V16QI_V16QI_V16QI:
24065 case V8SF_FTYPE_V8SF_V8SF_V8SF:
24066 case V4DF_FTYPE_V4DF_V4DF_V4DF:
24067 case V4SF_FTYPE_V4SF_V4SF_V4SF:
24068 case V2DF_FTYPE_V2DF_V2DF_V2DF:
24071 case V16QI_FTYPE_V16QI_V16QI_INT:
24072 case V8HI_FTYPE_V8HI_V8HI_INT:
24073 case V8SI_FTYPE_V8SI_V8SI_INT:
24074 case V8SI_FTYPE_V8SI_V4SI_INT:
24075 case V8SF_FTYPE_V8SF_V8SF_INT:
24076 case V8SF_FTYPE_V8SF_V4SF_INT:
24077 case V4SI_FTYPE_V4SI_V4SI_INT:
24078 case V4DF_FTYPE_V4DF_V4DF_INT:
24079 case V4DF_FTYPE_V4DF_V2DF_INT:
24080 case V4SF_FTYPE_V4SF_V4SF_INT:
24081 case V2DI_FTYPE_V2DI_V2DI_INT:
24082 case V2DF_FTYPE_V2DF_V2DF_INT:
24084 nargs_constant = 1;
24086 case V2DI2TI_FTYPE_V2DI_V2DI_INT:
24089 nargs_constant = 1;
24091 case V1DI2DI_FTYPE_V1DI_V1DI_INT:
24094 nargs_constant = 1;
24096 case V2DI_FTYPE_V2DI_UINT_UINT:
24098 nargs_constant = 2;
24100 case V2DI_FTYPE_V2DI_V2DI_UINT_UINT:
24102 nargs_constant = 2;
24105 gcc_unreachable ();
24108 gcc_assert (nargs <= ARRAY_SIZE (args));
24110 if (comparison != UNKNOWN)
24112 gcc_assert (nargs == 2);
24113 return ix86_expand_sse_compare (d, exp, target, swap);
24116 if (rmode == VOIDmode || rmode == tmode)
24120 || GET_MODE (target) != tmode
24121 || ! (*insn_p->operand[0].predicate) (target, tmode))
24122 target = gen_reg_rtx (tmode);
24123 real_target = target;
24127 target = gen_reg_rtx (rmode);
24128 real_target = simplify_gen_subreg (tmode, target, rmode, 0);
24131 for (i = 0; i < nargs; i++)
24133 tree arg = CALL_EXPR_ARG (exp, i);
24134 rtx op = expand_normal (arg);
24135 enum machine_mode mode = insn_p->operand[i + 1].mode;
24136 bool match = (*insn_p->operand[i + 1].predicate) (op, mode);
24138 if (last_arg_count && (i + 1) == nargs)
24140 /* SIMD shift insns take either an 8-bit immediate or
24141 register as count. But builtin functions take int as
24142 count. If count doesn't match, we put it in register. */
24145 op = simplify_gen_subreg (SImode, op, GET_MODE (op), 0);
24146 if (!(*insn_p->operand[i + 1].predicate) (op, mode))
24147 op = copy_to_reg (op);
24150 else if ((nargs - i) <= nargs_constant)
24155 case CODE_FOR_sse4_1_roundpd:
24156 case CODE_FOR_sse4_1_roundps:
24157 case CODE_FOR_sse4_1_roundsd:
24158 case CODE_FOR_sse4_1_roundss:
24159 case CODE_FOR_sse4_1_blendps:
24160 case CODE_FOR_avx_blendpd256:
24161 case CODE_FOR_avx_vpermilv4df:
24162 case CODE_FOR_avx_roundpd256:
24163 case CODE_FOR_avx_roundps256:
24164 error ("the last argument must be a 4-bit immediate");
24167 case CODE_FOR_sse4_1_blendpd:
24168 case CODE_FOR_avx_vpermilv2df:
24169 error ("the last argument must be a 2-bit immediate");
24172 case CODE_FOR_avx_vextractf128v4df:
24173 case CODE_FOR_avx_vextractf128v8sf:
24174 case CODE_FOR_avx_vextractf128v8si:
24175 case CODE_FOR_avx_vinsertf128v4df:
24176 case CODE_FOR_avx_vinsertf128v8sf:
24177 case CODE_FOR_avx_vinsertf128v8si:
24178 error ("the last argument must be a 1-bit immediate");
24181 case CODE_FOR_avx_cmpsdv2df3:
24182 case CODE_FOR_avx_cmpssv4sf3:
24183 case CODE_FOR_avx_cmppdv2df3:
24184 case CODE_FOR_avx_cmppsv4sf3:
24185 case CODE_FOR_avx_cmppdv4df3:
24186 case CODE_FOR_avx_cmppsv8sf3:
24187 error ("the last argument must be a 5-bit immediate");
24191 switch (nargs_constant)
24194 if ((nargs - i) == nargs_constant)
24196 error ("the next to last argument must be an 8-bit immediate");
24200 error ("the last argument must be an 8-bit immediate");
24203 gcc_unreachable ();
24210 if (VECTOR_MODE_P (mode))
24211 op = safe_vector_operand (op, mode);
24213 /* If we aren't optimizing, only allow one memory operand to
24215 if (memory_operand (op, mode))
24218 if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
24220 if (optimize || !match || num_memory > 1)
24221 op = copy_to_mode_reg (mode, op);
24225 op = copy_to_reg (op);
24226 op = simplify_gen_subreg (mode, op, GET_MODE (op), 0);
24231 args[i].mode = mode;
24237 pat = GEN_FCN (icode) (real_target, args[0].op);
24240 pat = GEN_FCN (icode) (real_target, args[0].op, args[1].op);
24243 pat = GEN_FCN (icode) (real_target, args[0].op, args[1].op,
24247 pat = GEN_FCN (icode) (real_target, args[0].op, args[1].op,
24248 args[2].op, args[3].op);
24251 gcc_unreachable ();
24261 /* Subroutine of ix86_expand_builtin to take care of special insns
24262 with variable number of operands. */
24265 ix86_expand_special_args_builtin (const struct builtin_description *d,
24266 tree exp, rtx target)
24270 unsigned int i, nargs, arg_adjust, memory;
24274 enum machine_mode mode;
24276 enum insn_code icode = d->icode;
24277 bool last_arg_constant = false;
24278 const struct insn_data *insn_p = &insn_data[icode];
24279 enum machine_mode tmode = insn_p->operand[0].mode;
24280 enum { load, store } klass;
24282 switch ((enum ix86_special_builtin_type) d->flag)
24284 case VOID_FTYPE_VOID:
24285 emit_insn (GEN_FCN (icode) (target));
24287 case V2DI_FTYPE_PV2DI:
24288 case V32QI_FTYPE_PCCHAR:
24289 case V16QI_FTYPE_PCCHAR:
24290 case V8SF_FTYPE_PCV4SF:
24291 case V8SF_FTYPE_PCFLOAT:
24292 case V4SF_FTYPE_PCFLOAT:
24293 case V4DF_FTYPE_PCV2DF:
24294 case V4DF_FTYPE_PCDOUBLE:
24295 case V2DF_FTYPE_PCDOUBLE:
24300 case VOID_FTYPE_PV2SF_V4SF:
24301 case VOID_FTYPE_PV4DI_V4DI:
24302 case VOID_FTYPE_PV2DI_V2DI:
24303 case VOID_FTYPE_PCHAR_V32QI:
24304 case VOID_FTYPE_PCHAR_V16QI:
24305 case VOID_FTYPE_PFLOAT_V8SF:
24306 case VOID_FTYPE_PFLOAT_V4SF:
24307 case VOID_FTYPE_PDOUBLE_V4DF:
24308 case VOID_FTYPE_PDOUBLE_V2DF:
24309 case VOID_FTYPE_PDI_DI:
24310 case VOID_FTYPE_PINT_INT:
24313 /* Reserve memory operand for target. */
24314 memory = ARRAY_SIZE (args);
24316 case V4SF_FTYPE_V4SF_PCV2SF:
24317 case V2DF_FTYPE_V2DF_PCDOUBLE:
24322 case V8SF_FTYPE_PCV8SF_V8SF:
24323 case V4DF_FTYPE_PCV4DF_V4DF:
24324 case V4SF_FTYPE_PCV4SF_V4SF:
24325 case V2DF_FTYPE_PCV2DF_V2DF:
24330 case VOID_FTYPE_PV8SF_V8SF_V8SF:
24331 case VOID_FTYPE_PV4DF_V4DF_V4DF:
24332 case VOID_FTYPE_PV4SF_V4SF_V4SF:
24333 case VOID_FTYPE_PV2DF_V2DF_V2DF:
24336 /* Reserve memory operand for target. */
24337 memory = ARRAY_SIZE (args);
24340 gcc_unreachable ();
24343 gcc_assert (nargs <= ARRAY_SIZE (args));
24345 if (klass == store)
24347 arg = CALL_EXPR_ARG (exp, 0);
24348 op = expand_normal (arg);
24349 gcc_assert (target == 0);
24350 target = gen_rtx_MEM (tmode, copy_to_mode_reg (Pmode, op));
24358 || GET_MODE (target) != tmode
24359 || ! (*insn_p->operand[0].predicate) (target, tmode))
24360 target = gen_reg_rtx (tmode);
24363 for (i = 0; i < nargs; i++)
24365 enum machine_mode mode = insn_p->operand[i + 1].mode;
24368 arg = CALL_EXPR_ARG (exp, i + arg_adjust);
24369 op = expand_normal (arg);
24370 match = (*insn_p->operand[i + 1].predicate) (op, mode);
24372 if (last_arg_constant && (i + 1) == nargs)
24378 error ("the last argument must be an 8-bit immediate");
24386 /* This must be the memory operand. */
24387 op = gen_rtx_MEM (mode, copy_to_mode_reg (Pmode, op));
24388 gcc_assert (GET_MODE (op) == mode
24389 || GET_MODE (op) == VOIDmode);
24393 /* This must be register. */
24394 if (VECTOR_MODE_P (mode))
24395 op = safe_vector_operand (op, mode);
24397 gcc_assert (GET_MODE (op) == mode
24398 || GET_MODE (op) == VOIDmode);
24399 op = copy_to_mode_reg (mode, op);
24404 args[i].mode = mode;
24410 pat = GEN_FCN (icode) (target, args[0].op);
24413 pat = GEN_FCN (icode) (target, args[0].op, args[1].op);
24416 gcc_unreachable ();
24422 return klass == store ? 0 : target;
24425 /* Return the integer constant in ARG. Constrain it to be in the range
24426 of the subparts of VEC_TYPE; issue an error if not. */
24429 get_element_number (tree vec_type, tree arg)
24431 unsigned HOST_WIDE_INT elt, max = TYPE_VECTOR_SUBPARTS (vec_type) - 1;
24433 if (!host_integerp (arg, 1)
24434 || (elt = tree_low_cst (arg, 1), elt > max))
24436 error ("selector must be an integer constant in the range 0..%wi", max);
24443 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
24444 ix86_expand_vector_init. We DO have language-level syntax for this, in
24445 the form of (type){ init-list }. Except that since we can't place emms
24446 instructions from inside the compiler, we can't allow the use of MMX
24447 registers unless the user explicitly asks for it. So we do *not* define
24448 vec_set/vec_extract/vec_init patterns for MMX modes in mmx.md. Instead
24449 we have builtins invoked by mmintrin.h that gives us license to emit
24450 these sorts of instructions. */
24453 ix86_expand_vec_init_builtin (tree type, tree exp, rtx target)
24455 enum machine_mode tmode = TYPE_MODE (type);
24456 enum machine_mode inner_mode = GET_MODE_INNER (tmode);
24457 int i, n_elt = GET_MODE_NUNITS (tmode);
24458 rtvec v = rtvec_alloc (n_elt);
24460 gcc_assert (VECTOR_MODE_P (tmode));
24461 gcc_assert (call_expr_nargs (exp) == n_elt);
24463 for (i = 0; i < n_elt; ++i)
24465 rtx x = expand_normal (CALL_EXPR_ARG (exp, i));
24466 RTVEC_ELT (v, i) = gen_lowpart (inner_mode, x);
24469 if (!target || !register_operand (target, tmode))
24470 target = gen_reg_rtx (tmode);
24472 ix86_expand_vector_init (true, target, gen_rtx_PARALLEL (tmode, v));
24476 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
24477 ix86_expand_vector_extract. They would be redundant (for non-MMX) if we
24478 had a language-level syntax for referencing vector elements. */
24481 ix86_expand_vec_ext_builtin (tree exp, rtx target)
24483 enum machine_mode tmode, mode0;
24488 arg0 = CALL_EXPR_ARG (exp, 0);
24489 arg1 = CALL_EXPR_ARG (exp, 1);
24491 op0 = expand_normal (arg0);
24492 elt = get_element_number (TREE_TYPE (arg0), arg1);
24494 tmode = TYPE_MODE (TREE_TYPE (TREE_TYPE (arg0)));
24495 mode0 = TYPE_MODE (TREE_TYPE (arg0));
24496 gcc_assert (VECTOR_MODE_P (mode0));
24498 op0 = force_reg (mode0, op0);
24500 if (optimize || !target || !register_operand (target, tmode))
24501 target = gen_reg_rtx (tmode);
24503 ix86_expand_vector_extract (true, target, op0, elt);
24508 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
24509 ix86_expand_vector_set. They would be redundant (for non-MMX) if we had
24510 a language-level syntax for referencing vector elements. */
24513 ix86_expand_vec_set_builtin (tree exp)
24515 enum machine_mode tmode, mode1;
24516 tree arg0, arg1, arg2;
24518 rtx op0, op1, target;
24520 arg0 = CALL_EXPR_ARG (exp, 0);
24521 arg1 = CALL_EXPR_ARG (exp, 1);
24522 arg2 = CALL_EXPR_ARG (exp, 2);
24524 tmode = TYPE_MODE (TREE_TYPE (arg0));
24525 mode1 = TYPE_MODE (TREE_TYPE (TREE_TYPE (arg0)));
24526 gcc_assert (VECTOR_MODE_P (tmode));
24528 op0 = expand_expr (arg0, NULL_RTX, tmode, EXPAND_NORMAL);
24529 op1 = expand_expr (arg1, NULL_RTX, mode1, EXPAND_NORMAL);
24530 elt = get_element_number (TREE_TYPE (arg0), arg2);
24532 if (GET_MODE (op1) != mode1 && GET_MODE (op1) != VOIDmode)
24533 op1 = convert_modes (mode1, GET_MODE (op1), op1, true);
24535 op0 = force_reg (tmode, op0);
24536 op1 = force_reg (mode1, op1);
24538 /* OP0 is the source of these builtin functions and shouldn't be
24539 modified. Create a copy, use it and return it as target. */
24540 target = gen_reg_rtx (tmode);
24541 emit_move_insn (target, op0);
24542 ix86_expand_vector_set (true, target, op1, elt);
24547 /* Expand an expression EXP that calls a built-in function,
24548 with result going to TARGET if that's convenient
24549 (and in mode MODE if that's convenient).
24550 SUBTARGET may be used as the target for computing one of EXP's operands.
24551 IGNORE is nonzero if the value is to be ignored. */
24554 ix86_expand_builtin (tree exp, rtx target, rtx subtarget ATTRIBUTE_UNUSED,
24555 enum machine_mode mode ATTRIBUTE_UNUSED,
24556 int ignore ATTRIBUTE_UNUSED)
24558 const struct builtin_description *d;
24560 enum insn_code icode;
24561 tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
24562 tree arg0, arg1, arg2;
24563 rtx op0, op1, op2, pat;
24564 enum machine_mode mode0, mode1, mode2;
24565 unsigned int fcode = DECL_FUNCTION_CODE (fndecl);
24567 /* Determine whether the builtin function is available under the current ISA.
24568 Originally the builtin was not created if it wasn't applicable to the
24569 current ISA based on the command line switches. With function specific
24570 options, we need to check in the context of the function making the call
24571 whether it is supported. */
24572 if (ix86_builtins_isa[fcode].isa
24573 && !(ix86_builtins_isa[fcode].isa & ix86_isa_flags))
24575 char *opts = ix86_target_string (ix86_builtins_isa[fcode].isa, 0, NULL,
24576 NULL, NULL, false);
24579 error ("%qE needs unknown isa option", fndecl);
24582 gcc_assert (opts != NULL);
24583 error ("%qE needs isa option %s", fndecl, opts);
24591 case IX86_BUILTIN_MASKMOVQ:
24592 case IX86_BUILTIN_MASKMOVDQU:
24593 icode = (fcode == IX86_BUILTIN_MASKMOVQ
24594 ? CODE_FOR_mmx_maskmovq
24595 : CODE_FOR_sse2_maskmovdqu);
24596 /* Note the arg order is different from the operand order. */
24597 arg1 = CALL_EXPR_ARG (exp, 0);
24598 arg2 = CALL_EXPR_ARG (exp, 1);
24599 arg0 = CALL_EXPR_ARG (exp, 2);
24600 op0 = expand_normal (arg0);
24601 op1 = expand_normal (arg1);
24602 op2 = expand_normal (arg2);
24603 mode0 = insn_data[icode].operand[0].mode;
24604 mode1 = insn_data[icode].operand[1].mode;
24605 mode2 = insn_data[icode].operand[2].mode;
24607 op0 = force_reg (Pmode, op0);
24608 op0 = gen_rtx_MEM (mode1, op0);
24610 if (! (*insn_data[icode].operand[0].predicate) (op0, mode0))
24611 op0 = copy_to_mode_reg (mode0, op0);
24612 if (! (*insn_data[icode].operand[1].predicate) (op1, mode1))
24613 op1 = copy_to_mode_reg (mode1, op1);
24614 if (! (*insn_data[icode].operand[2].predicate) (op2, mode2))
24615 op2 = copy_to_mode_reg (mode2, op2);
24616 pat = GEN_FCN (icode) (op0, op1, op2);
24622 case IX86_BUILTIN_LDMXCSR:
24623 op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
24624 target = assign_386_stack_local (SImode, SLOT_VIRTUAL);
24625 emit_move_insn (target, op0);
24626 emit_insn (gen_sse_ldmxcsr (target));
24629 case IX86_BUILTIN_STMXCSR:
24630 target = assign_386_stack_local (SImode, SLOT_VIRTUAL);
24631 emit_insn (gen_sse_stmxcsr (target));
24632 return copy_to_mode_reg (SImode, target);
24634 case IX86_BUILTIN_CLFLUSH:
24635 arg0 = CALL_EXPR_ARG (exp, 0);
24636 op0 = expand_normal (arg0);
24637 icode = CODE_FOR_sse2_clflush;
24638 if (! (*insn_data[icode].operand[0].predicate) (op0, Pmode))
24639 op0 = copy_to_mode_reg (Pmode, op0);
24641 emit_insn (gen_sse2_clflush (op0));
24644 case IX86_BUILTIN_MONITOR:
24645 arg0 = CALL_EXPR_ARG (exp, 0);
24646 arg1 = CALL_EXPR_ARG (exp, 1);
24647 arg2 = CALL_EXPR_ARG (exp, 2);
24648 op0 = expand_normal (arg0);
24649 op1 = expand_normal (arg1);
24650 op2 = expand_normal (arg2);
24652 op0 = copy_to_mode_reg (Pmode, op0);
24654 op1 = copy_to_mode_reg (SImode, op1);
24656 op2 = copy_to_mode_reg (SImode, op2);
24657 emit_insn ((*ix86_gen_monitor) (op0, op1, op2));
24660 case IX86_BUILTIN_MWAIT:
24661 arg0 = CALL_EXPR_ARG (exp, 0);
24662 arg1 = CALL_EXPR_ARG (exp, 1);
24663 op0 = expand_normal (arg0);
24664 op1 = expand_normal (arg1);
24666 op0 = copy_to_mode_reg (SImode, op0);
24668 op1 = copy_to_mode_reg (SImode, op1);
24669 emit_insn (gen_sse3_mwait (op0, op1));
24672 case IX86_BUILTIN_VEC_INIT_V2SI:
24673 case IX86_BUILTIN_VEC_INIT_V4HI:
24674 case IX86_BUILTIN_VEC_INIT_V8QI:
24675 return ix86_expand_vec_init_builtin (TREE_TYPE (exp), exp, target);
24677 case IX86_BUILTIN_VEC_EXT_V2DF:
24678 case IX86_BUILTIN_VEC_EXT_V2DI:
24679 case IX86_BUILTIN_VEC_EXT_V4SF:
24680 case IX86_BUILTIN_VEC_EXT_V4SI:
24681 case IX86_BUILTIN_VEC_EXT_V8HI:
24682 case IX86_BUILTIN_VEC_EXT_V2SI:
24683 case IX86_BUILTIN_VEC_EXT_V4HI:
24684 case IX86_BUILTIN_VEC_EXT_V16QI:
24685 return ix86_expand_vec_ext_builtin (exp, target);
24687 case IX86_BUILTIN_VEC_SET_V2DI:
24688 case IX86_BUILTIN_VEC_SET_V4SF:
24689 case IX86_BUILTIN_VEC_SET_V4SI:
24690 case IX86_BUILTIN_VEC_SET_V8HI:
24691 case IX86_BUILTIN_VEC_SET_V4HI:
24692 case IX86_BUILTIN_VEC_SET_V16QI:
24693 return ix86_expand_vec_set_builtin (exp);
24695 case IX86_BUILTIN_INFQ:
24697 REAL_VALUE_TYPE inf;
24701 tmp = CONST_DOUBLE_FROM_REAL_VALUE (inf, mode);
24703 tmp = validize_mem (force_const_mem (mode, tmp));
24706 target = gen_reg_rtx (mode);
24708 emit_move_insn (target, tmp);
24716 for (i = 0, d = bdesc_special_args;
24717 i < ARRAY_SIZE (bdesc_special_args);
24719 if (d->code == fcode)
24720 return ix86_expand_special_args_builtin (d, exp, target);
24722 for (i = 0, d = bdesc_args;
24723 i < ARRAY_SIZE (bdesc_args);
24725 if (d->code == fcode)
24728 case IX86_BUILTIN_FABSQ:
24729 case IX86_BUILTIN_COPYSIGNQ:
24731 /* Emit a normal call if SSE2 isn't available. */
24732 return expand_call (exp, target, ignore);
24734 return ix86_expand_args_builtin (d, exp, target);
24737 for (i = 0, d = bdesc_comi; i < ARRAY_SIZE (bdesc_comi); i++, d++)
24738 if (d->code == fcode)
24739 return ix86_expand_sse_comi (d, exp, target);
24741 for (i = 0, d = bdesc_pcmpestr;
24742 i < ARRAY_SIZE (bdesc_pcmpestr);
24744 if (d->code == fcode)
24745 return ix86_expand_sse_pcmpestr (d, exp, target);
24747 for (i = 0, d = bdesc_pcmpistr;
24748 i < ARRAY_SIZE (bdesc_pcmpistr);
24750 if (d->code == fcode)
24751 return ix86_expand_sse_pcmpistr (d, exp, target);
24753 for (i = 0, d = bdesc_multi_arg; i < ARRAY_SIZE (bdesc_multi_arg); i++, d++)
24754 if (d->code == fcode)
24755 return ix86_expand_multi_arg_builtin (d->icode, exp, target,
24756 (enum multi_arg_type)d->flag,
24759 gcc_unreachable ();
24762 /* Returns a function decl for a vectorized version of the builtin function
24763 with builtin function code FN and the result vector type TYPE, or NULL_TREE
24764 if it is not available. */
24767 ix86_builtin_vectorized_function (unsigned int fn, tree type_out,
24770 enum machine_mode in_mode, out_mode;
24773 if (TREE_CODE (type_out) != VECTOR_TYPE
24774 || TREE_CODE (type_in) != VECTOR_TYPE)
24777 out_mode = TYPE_MODE (TREE_TYPE (type_out));
24778 out_n = TYPE_VECTOR_SUBPARTS (type_out);
24779 in_mode = TYPE_MODE (TREE_TYPE (type_in));
24780 in_n = TYPE_VECTOR_SUBPARTS (type_in);
24784 case BUILT_IN_SQRT:
24785 if (out_mode == DFmode && out_n == 2
24786 && in_mode == DFmode && in_n == 2)
24787 return ix86_builtins[IX86_BUILTIN_SQRTPD];
24790 case BUILT_IN_SQRTF:
24791 if (out_mode == SFmode && out_n == 4
24792 && in_mode == SFmode && in_n == 4)
24793 return ix86_builtins[IX86_BUILTIN_SQRTPS_NR];
24796 case BUILT_IN_LRINT:
24797 if (out_mode == SImode && out_n == 4
24798 && in_mode == DFmode && in_n == 2)
24799 return ix86_builtins[IX86_BUILTIN_VEC_PACK_SFIX];
24802 case BUILT_IN_LRINTF:
24803 if (out_mode == SImode && out_n == 4
24804 && in_mode == SFmode && in_n == 4)
24805 return ix86_builtins[IX86_BUILTIN_CVTPS2DQ];
24812 /* Dispatch to a handler for a vectorization library. */
24813 if (ix86_veclib_handler)
24814 return (*ix86_veclib_handler)(fn, type_out, type_in);
24819 /* Handler for an SVML-style interface to
24820 a library with vectorized intrinsics. */
24823 ix86_veclibabi_svml (enum built_in_function fn, tree type_out, tree type_in)
24826 tree fntype, new_fndecl, args;
24829 enum machine_mode el_mode, in_mode;
24832 /* The SVML is suitable for unsafe math only. */
24833 if (!flag_unsafe_math_optimizations)
24836 el_mode = TYPE_MODE (TREE_TYPE (type_out));
24837 n = TYPE_VECTOR_SUBPARTS (type_out);
24838 in_mode = TYPE_MODE (TREE_TYPE (type_in));
24839 in_n = TYPE_VECTOR_SUBPARTS (type_in);
24840 if (el_mode != in_mode
24848 case BUILT_IN_LOG10:
24850 case BUILT_IN_TANH:
24852 case BUILT_IN_ATAN:
24853 case BUILT_IN_ATAN2:
24854 case BUILT_IN_ATANH:
24855 case BUILT_IN_CBRT:
24856 case BUILT_IN_SINH:
24858 case BUILT_IN_ASINH:
24859 case BUILT_IN_ASIN:
24860 case BUILT_IN_COSH:
24862 case BUILT_IN_ACOSH:
24863 case BUILT_IN_ACOS:
24864 if (el_mode != DFmode || n != 2)
24868 case BUILT_IN_EXPF:
24869 case BUILT_IN_LOGF:
24870 case BUILT_IN_LOG10F:
24871 case BUILT_IN_POWF:
24872 case BUILT_IN_TANHF:
24873 case BUILT_IN_TANF:
24874 case BUILT_IN_ATANF:
24875 case BUILT_IN_ATAN2F:
24876 case BUILT_IN_ATANHF:
24877 case BUILT_IN_CBRTF:
24878 case BUILT_IN_SINHF:
24879 case BUILT_IN_SINF:
24880 case BUILT_IN_ASINHF:
24881 case BUILT_IN_ASINF:
24882 case BUILT_IN_COSHF:
24883 case BUILT_IN_COSF:
24884 case BUILT_IN_ACOSHF:
24885 case BUILT_IN_ACOSF:
24886 if (el_mode != SFmode || n != 4)
24894 bname = IDENTIFIER_POINTER (DECL_NAME (implicit_built_in_decls[fn]));
24896 if (fn == BUILT_IN_LOGF)
24897 strcpy (name, "vmlsLn4");
24898 else if (fn == BUILT_IN_LOG)
24899 strcpy (name, "vmldLn2");
24902 sprintf (name, "vmls%s", bname+10);
24903 name[strlen (name)-1] = '4';
24906 sprintf (name, "vmld%s2", bname+10);
24908 /* Convert to uppercase. */
24912 for (args = DECL_ARGUMENTS (implicit_built_in_decls[fn]); args;
24913 args = TREE_CHAIN (args))
24917 fntype = build_function_type_list (type_out, type_in, NULL);
24919 fntype = build_function_type_list (type_out, type_in, type_in, NULL);
24921 /* Build a function declaration for the vectorized function. */
24922 new_fndecl = build_decl (FUNCTION_DECL, get_identifier (name), fntype);
24923 TREE_PUBLIC (new_fndecl) = 1;
24924 DECL_EXTERNAL (new_fndecl) = 1;
24925 DECL_IS_NOVOPS (new_fndecl) = 1;
24926 TREE_READONLY (new_fndecl) = 1;
24931 /* Handler for an ACML-style interface to
24932 a library with vectorized intrinsics. */
24935 ix86_veclibabi_acml (enum built_in_function fn, tree type_out, tree type_in)
24937 char name[20] = "__vr.._";
24938 tree fntype, new_fndecl, args;
24941 enum machine_mode el_mode, in_mode;
24944 /* The ACML is 64bits only and suitable for unsafe math only as
24945 it does not correctly support parts of IEEE with the required
24946 precision such as denormals. */
24948 || !flag_unsafe_math_optimizations)
24951 el_mode = TYPE_MODE (TREE_TYPE (type_out));
24952 n = TYPE_VECTOR_SUBPARTS (type_out);
24953 in_mode = TYPE_MODE (TREE_TYPE (type_in));
24954 in_n = TYPE_VECTOR_SUBPARTS (type_in);
24955 if (el_mode != in_mode
24965 case BUILT_IN_LOG2:
24966 case BUILT_IN_LOG10:
24969 if (el_mode != DFmode
24974 case BUILT_IN_SINF:
24975 case BUILT_IN_COSF:
24976 case BUILT_IN_EXPF:
24977 case BUILT_IN_POWF:
24978 case BUILT_IN_LOGF:
24979 case BUILT_IN_LOG2F:
24980 case BUILT_IN_LOG10F:
24983 if (el_mode != SFmode
24992 bname = IDENTIFIER_POINTER (DECL_NAME (implicit_built_in_decls[fn]));
24993 sprintf (name + 7, "%s", bname+10);
24996 for (args = DECL_ARGUMENTS (implicit_built_in_decls[fn]); args;
24997 args = TREE_CHAIN (args))
25001 fntype = build_function_type_list (type_out, type_in, NULL);
25003 fntype = build_function_type_list (type_out, type_in, type_in, NULL);
25005 /* Build a function declaration for the vectorized function. */
25006 new_fndecl = build_decl (FUNCTION_DECL, get_identifier (name), fntype);
25007 TREE_PUBLIC (new_fndecl) = 1;
25008 DECL_EXTERNAL (new_fndecl) = 1;
25009 DECL_IS_NOVOPS (new_fndecl) = 1;
25010 TREE_READONLY (new_fndecl) = 1;
25016 /* Returns a decl of a function that implements conversion of an integer vector
25017 into a floating-point vector, or vice-versa. TYPE is the type of the integer
25018 side of the conversion.
25019 Return NULL_TREE if it is not available. */
25022 ix86_vectorize_builtin_conversion (unsigned int code, tree type)
25024 if (TREE_CODE (type) != VECTOR_TYPE)
25030 switch (TYPE_MODE (type))
25033 return ix86_builtins[IX86_BUILTIN_CVTDQ2PS];
25038 case FIX_TRUNC_EXPR:
25039 switch (TYPE_MODE (type))
25042 return ix86_builtins[IX86_BUILTIN_CVTTPS2DQ];
25052 /* Returns a code for a target-specific builtin that implements
25053 reciprocal of the function, or NULL_TREE if not available. */
25056 ix86_builtin_reciprocal (unsigned int fn, bool md_fn,
25057 bool sqrt ATTRIBUTE_UNUSED)
25059 if (! (TARGET_SSE_MATH && TARGET_RECIP && !optimize_insn_for_size_p ()
25060 && flag_finite_math_only && !flag_trapping_math
25061 && flag_unsafe_math_optimizations))
25065 /* Machine dependent builtins. */
25068 /* Vectorized version of sqrt to rsqrt conversion. */
25069 case IX86_BUILTIN_SQRTPS_NR:
25070 return ix86_builtins[IX86_BUILTIN_RSQRTPS_NR];
25076 /* Normal builtins. */
25079 /* Sqrt to rsqrt conversion. */
25080 case BUILT_IN_SQRTF:
25081 return ix86_builtins[IX86_BUILTIN_RSQRTF];
25088 /* Store OPERAND to the memory after reload is completed. This means
25089 that we can't easily use assign_stack_local. */
25091 ix86_force_to_memory (enum machine_mode mode, rtx operand)
25095 gcc_assert (reload_completed);
25096 if (!TARGET_64BIT_MS_ABI && TARGET_RED_ZONE)
25098 result = gen_rtx_MEM (mode,
25099 gen_rtx_PLUS (Pmode,
25101 GEN_INT (-RED_ZONE_SIZE)));
25102 emit_move_insn (result, operand);
25104 else if ((TARGET_64BIT_MS_ABI || !TARGET_RED_ZONE) && TARGET_64BIT)
25110 operand = gen_lowpart (DImode, operand);
25114 gen_rtx_SET (VOIDmode,
25115 gen_rtx_MEM (DImode,
25116 gen_rtx_PRE_DEC (DImode,
25117 stack_pointer_rtx)),
25121 gcc_unreachable ();
25123 result = gen_rtx_MEM (mode, stack_pointer_rtx);
25132 split_di (&operand, 1, operands, operands + 1);
25134 gen_rtx_SET (VOIDmode,
25135 gen_rtx_MEM (SImode,
25136 gen_rtx_PRE_DEC (Pmode,
25137 stack_pointer_rtx)),
25140 gen_rtx_SET (VOIDmode,
25141 gen_rtx_MEM (SImode,
25142 gen_rtx_PRE_DEC (Pmode,
25143 stack_pointer_rtx)),
25148 /* Store HImodes as SImodes. */
25149 operand = gen_lowpart (SImode, operand);
25153 gen_rtx_SET (VOIDmode,
25154 gen_rtx_MEM (GET_MODE (operand),
25155 gen_rtx_PRE_DEC (SImode,
25156 stack_pointer_rtx)),
25160 gcc_unreachable ();
25162 result = gen_rtx_MEM (mode, stack_pointer_rtx);
25167 /* Free operand from the memory. */
25169 ix86_free_from_memory (enum machine_mode mode)
25171 if (!TARGET_RED_ZONE || TARGET_64BIT_MS_ABI)
25175 if (mode == DImode || TARGET_64BIT)
25179 /* Use LEA to deallocate stack space. In peephole2 it will be converted
25180 to pop or add instruction if registers are available. */
25181 emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx,
25182 gen_rtx_PLUS (Pmode, stack_pointer_rtx,
25187 /* Put float CONST_DOUBLE in the constant pool instead of fp regs.
25188 QImode must go into class Q_REGS.
25189 Narrow ALL_REGS to GENERAL_REGS. This supports allowing movsf and
25190 movdf to do mem-to-mem moves through integer regs. */
25192 ix86_preferred_reload_class (rtx x, enum reg_class regclass)
25194 enum machine_mode mode = GET_MODE (x);
25196 /* We're only allowed to return a subclass of CLASS. Many of the
25197 following checks fail for NO_REGS, so eliminate that early. */
25198 if (regclass == NO_REGS)
25201 /* All classes can load zeros. */
25202 if (x == CONST0_RTX (mode))
25205 /* Force constants into memory if we are loading a (nonzero) constant into
25206 an MMX or SSE register. This is because there are no MMX/SSE instructions
25207 to load from a constant. */
25209 && (MAYBE_MMX_CLASS_P (regclass) || MAYBE_SSE_CLASS_P (regclass)))
25212 /* Prefer SSE regs only, if we can use them for math. */
25213 if (TARGET_SSE_MATH && !TARGET_MIX_SSE_I387 && SSE_FLOAT_MODE_P (mode))
25214 return SSE_CLASS_P (regclass) ? regclass : NO_REGS;
25216 /* Floating-point constants need more complex checks. */
25217 if (GET_CODE (x) == CONST_DOUBLE && GET_MODE (x) != VOIDmode)
25219 /* General regs can load everything. */
25220 if (reg_class_subset_p (regclass, GENERAL_REGS))
25223 /* Floats can load 0 and 1 plus some others. Note that we eliminated
25224 zero above. We only want to wind up preferring 80387 registers if
25225 we plan on doing computation with them. */
25227 && standard_80387_constant_p (x))
25229 /* Limit class to non-sse. */
25230 if (regclass == FLOAT_SSE_REGS)
25232 if (regclass == FP_TOP_SSE_REGS)
25234 if (regclass == FP_SECOND_SSE_REGS)
25235 return FP_SECOND_REG;
25236 if (regclass == FLOAT_INT_REGS || regclass == FLOAT_REGS)
25243 /* Generally when we see PLUS here, it's the function invariant
25244 (plus soft-fp const_int). Which can only be computed into general
25246 if (GET_CODE (x) == PLUS)
25247 return reg_class_subset_p (regclass, GENERAL_REGS) ? regclass : NO_REGS;
25249 /* QImode constants are easy to load, but non-constant QImode data
25250 must go into Q_REGS. */
25251 if (GET_MODE (x) == QImode && !CONSTANT_P (x))
25253 if (reg_class_subset_p (regclass, Q_REGS))
25255 if (reg_class_subset_p (Q_REGS, regclass))
25263 /* Discourage putting floating-point values in SSE registers unless
25264 SSE math is being used, and likewise for the 387 registers. */
25266 ix86_preferred_output_reload_class (rtx x, enum reg_class regclass)
25268 enum machine_mode mode = GET_MODE (x);
25270 /* Restrict the output reload class to the register bank that we are doing
25271 math on. If we would like not to return a subset of CLASS, reject this
25272 alternative: if reload cannot do this, it will still use its choice. */
25273 mode = GET_MODE (x);
25274 if (TARGET_SSE_MATH && SSE_FLOAT_MODE_P (mode))
25275 return MAYBE_SSE_CLASS_P (regclass) ? SSE_REGS : NO_REGS;
25277 if (X87_FLOAT_MODE_P (mode))
25279 if (regclass == FP_TOP_SSE_REGS)
25281 else if (regclass == FP_SECOND_SSE_REGS)
25282 return FP_SECOND_REG;
25284 return FLOAT_CLASS_P (regclass) ? regclass : NO_REGS;
25290 static enum reg_class
25291 ix86_secondary_reload (bool in_p, rtx x, enum reg_class rclass,
25292 enum machine_mode mode,
25293 secondary_reload_info *sri ATTRIBUTE_UNUSED)
25295 /* QImode spills from non-QI registers require
25296 intermediate register on 32bit targets. */
25297 if (!in_p && mode == QImode && !TARGET_64BIT
25298 && (rclass == GENERAL_REGS
25299 || rclass == LEGACY_REGS
25300 || rclass == INDEX_REGS))
25309 if (regno >= FIRST_PSEUDO_REGISTER || GET_CODE (x) == SUBREG)
25310 regno = true_regnum (x);
25312 /* Return Q_REGS if the operand is in memory. */
25320 /* If we are copying between general and FP registers, we need a memory
25321 location. The same is true for SSE and MMX registers.
25323 To optimize register_move_cost performance, allow inline variant.
25325 The macro can't work reliably when one of the CLASSES is class containing
25326 registers from multiple units (SSE, MMX, integer). We avoid this by never
25327 combining those units in single alternative in the machine description.
25328 Ensure that this constraint holds to avoid unexpected surprises.
25330 When STRICT is false, we are being called from REGISTER_MOVE_COST, so do not
25331 enforce these sanity checks. */
25334 inline_secondary_memory_needed (enum reg_class class1, enum reg_class class2,
25335 enum machine_mode mode, int strict)
25337 if (MAYBE_FLOAT_CLASS_P (class1) != FLOAT_CLASS_P (class1)
25338 || MAYBE_FLOAT_CLASS_P (class2) != FLOAT_CLASS_P (class2)
25339 || MAYBE_SSE_CLASS_P (class1) != SSE_CLASS_P (class1)
25340 || MAYBE_SSE_CLASS_P (class2) != SSE_CLASS_P (class2)
25341 || MAYBE_MMX_CLASS_P (class1) != MMX_CLASS_P (class1)
25342 || MAYBE_MMX_CLASS_P (class2) != MMX_CLASS_P (class2))
25344 gcc_assert (!strict);
25348 if (FLOAT_CLASS_P (class1) != FLOAT_CLASS_P (class2))
25351 /* ??? This is a lie. We do have moves between mmx/general, and for
25352 mmx/sse2. But by saying we need secondary memory we discourage the
25353 register allocator from using the mmx registers unless needed. */
25354 if (MMX_CLASS_P (class1) != MMX_CLASS_P (class2))
25357 if (SSE_CLASS_P (class1) != SSE_CLASS_P (class2))
25359 /* SSE1 doesn't have any direct moves from other classes. */
25363 /* If the target says that inter-unit moves are more expensive
25364 than moving through memory, then don't generate them. */
25365 if (!TARGET_INTER_UNIT_MOVES)
25368 /* Between SSE and general, we have moves no larger than word size. */
25369 if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
25377 ix86_secondary_memory_needed (enum reg_class class1, enum reg_class class2,
25378 enum machine_mode mode, int strict)
25380 return inline_secondary_memory_needed (class1, class2, mode, strict);
25383 /* Return true if the registers in CLASS cannot represent the change from
25384 modes FROM to TO. */
25387 ix86_cannot_change_mode_class (enum machine_mode from, enum machine_mode to,
25388 enum reg_class regclass)
25393 /* x87 registers can't do subreg at all, as all values are reformatted
25394 to extended precision. */
25395 if (MAYBE_FLOAT_CLASS_P (regclass))
25398 if (MAYBE_SSE_CLASS_P (regclass) || MAYBE_MMX_CLASS_P (regclass))
25400 /* Vector registers do not support QI or HImode loads. If we don't
25401 disallow a change to these modes, reload will assume it's ok to
25402 drop the subreg from (subreg:SI (reg:HI 100) 0). This affects
25403 the vec_dupv4hi pattern. */
25404 if (GET_MODE_SIZE (from) < 4)
25407 /* Vector registers do not support subreg with nonzero offsets, which
25408 are otherwise valid for integer registers. Since we can't see
25409 whether we have a nonzero offset from here, prohibit all
25410 nonparadoxical subregs changing size. */
25411 if (GET_MODE_SIZE (to) < GET_MODE_SIZE (from))
25418 /* Return the cost of moving data of mode M between a
25419 register and memory. A value of 2 is the default; this cost is
25420 relative to those in `REGISTER_MOVE_COST'.
25422 This function is used extensively by register_move_cost that is used to
25423 build tables at startup. Make it inline in this case.
25424 When IN is 2, return maximum of in and out move cost.
25426 If moving between registers and memory is more expensive than
25427 between two registers, you should define this macro to express the
25430 Model also increased moving costs of QImode registers in non
25434 inline_memory_move_cost (enum machine_mode mode, enum reg_class regclass,
25438 if (FLOAT_CLASS_P (regclass))
25456 return MAX (ix86_cost->fp_load [index], ix86_cost->fp_store [index]);
25457 return in ? ix86_cost->fp_load [index] : ix86_cost->fp_store [index];
25459 if (SSE_CLASS_P (regclass))
25462 switch (GET_MODE_SIZE (mode))
25477 return MAX (ix86_cost->sse_load [index], ix86_cost->sse_store [index]);
25478 return in ? ix86_cost->sse_load [index] : ix86_cost->sse_store [index];
25480 if (MMX_CLASS_P (regclass))
25483 switch (GET_MODE_SIZE (mode))
25495 return MAX (ix86_cost->mmx_load [index], ix86_cost->mmx_store [index]);
25496 return in ? ix86_cost->mmx_load [index] : ix86_cost->mmx_store [index];
25498 switch (GET_MODE_SIZE (mode))
25501 if (Q_CLASS_P (regclass) || TARGET_64BIT)
25504 return ix86_cost->int_store[0];
25505 if (TARGET_PARTIAL_REG_DEPENDENCY
25506 && optimize_function_for_speed_p (cfun))
25507 cost = ix86_cost->movzbl_load;
25509 cost = ix86_cost->int_load[0];
25511 return MAX (cost, ix86_cost->int_store[0]);
25517 return MAX (ix86_cost->movzbl_load, ix86_cost->int_store[0] + 4);
25519 return ix86_cost->movzbl_load;
25521 return ix86_cost->int_store[0] + 4;
25526 return MAX (ix86_cost->int_load[1], ix86_cost->int_store[1]);
25527 return in ? ix86_cost->int_load[1] : ix86_cost->int_store[1];
25529 /* Compute number of 32bit moves needed. TFmode is moved as XFmode. */
25530 if (mode == TFmode)
25533 cost = MAX (ix86_cost->int_load[2] , ix86_cost->int_store[2]);
25535 cost = ix86_cost->int_load[2];
25537 cost = ix86_cost->int_store[2];
25538 return (cost * (((int) GET_MODE_SIZE (mode)
25539 + UNITS_PER_WORD - 1) / UNITS_PER_WORD));
25544 ix86_memory_move_cost (enum machine_mode mode, enum reg_class regclass, int in)
25546 return inline_memory_move_cost (mode, regclass, in);
25550 /* Return the cost of moving data from a register in class CLASS1 to
25551 one in class CLASS2.
25553 It is not required that the cost always equal 2 when FROM is the same as TO;
25554 on some machines it is expensive to move between registers if they are not
25555 general registers. */
25558 ix86_register_move_cost (enum machine_mode mode, enum reg_class class1,
25559 enum reg_class class2)
25561 /* In case we require secondary memory, compute cost of the store followed
25562 by load. In order to avoid bad register allocation choices, we need
25563 for this to be *at least* as high as the symmetric MEMORY_MOVE_COST. */
25565 if (inline_secondary_memory_needed (class1, class2, mode, 0))
25569 cost += inline_memory_move_cost (mode, class1, 2);
25570 cost += inline_memory_move_cost (mode, class2, 2);
25572 /* In case of copying from general_purpose_register we may emit multiple
25573 stores followed by single load causing memory size mismatch stall.
25574 Count this as arbitrarily high cost of 20. */
25575 if (CLASS_MAX_NREGS (class1, mode) > CLASS_MAX_NREGS (class2, mode))
25578 /* In the case of FP/MMX moves, the registers actually overlap, and we
25579 have to switch modes in order to treat them differently. */
25580 if ((MMX_CLASS_P (class1) && MAYBE_FLOAT_CLASS_P (class2))
25581 || (MMX_CLASS_P (class2) && MAYBE_FLOAT_CLASS_P (class1)))
25587 /* Moves between SSE/MMX and integer unit are expensive. */
25588 if (MMX_CLASS_P (class1) != MMX_CLASS_P (class2)
25589 || SSE_CLASS_P (class1) != SSE_CLASS_P (class2))
25591 /* ??? By keeping returned value relatively high, we limit the number
25592 of moves between integer and MMX/SSE registers for all targets.
25593 Additionally, high value prevents problem with x86_modes_tieable_p(),
25594 where integer modes in MMX/SSE registers are not tieable
25595 because of missing QImode and HImode moves to, from or between
25596 MMX/SSE registers. */
25597 return MAX (8, ix86_cost->mmxsse_to_integer);
25599 if (MAYBE_FLOAT_CLASS_P (class1))
25600 return ix86_cost->fp_move;
25601 if (MAYBE_SSE_CLASS_P (class1))
25602 return ix86_cost->sse_move;
25603 if (MAYBE_MMX_CLASS_P (class1))
25604 return ix86_cost->mmx_move;
25608 /* Return 1 if hard register REGNO can hold a value of machine-mode MODE. */
25611 ix86_hard_regno_mode_ok (int regno, enum machine_mode mode)
25613 /* Flags and only flags can only hold CCmode values. */
25614 if (CC_REGNO_P (regno))
25615 return GET_MODE_CLASS (mode) == MODE_CC;
25616 if (GET_MODE_CLASS (mode) == MODE_CC
25617 || GET_MODE_CLASS (mode) == MODE_RANDOM
25618 || GET_MODE_CLASS (mode) == MODE_PARTIAL_INT)
25620 if (FP_REGNO_P (regno))
25621 return VALID_FP_MODE_P (mode);
25622 if (SSE_REGNO_P (regno))
25624 /* We implement the move patterns for all vector modes into and
25625 out of SSE registers, even when no operation instructions
25626 are available. OImode move is available only when AVX is
25628 return ((TARGET_AVX && mode == OImode)
25629 || VALID_AVX256_REG_MODE (mode)
25630 || VALID_SSE_REG_MODE (mode)
25631 || VALID_SSE2_REG_MODE (mode)
25632 || VALID_MMX_REG_MODE (mode)
25633 || VALID_MMX_REG_MODE_3DNOW (mode));
25635 if (MMX_REGNO_P (regno))
25637 /* We implement the move patterns for 3DNOW modes even in MMX mode,
25638 so if the register is available at all, then we can move data of
25639 the given mode into or out of it. */
25640 return (VALID_MMX_REG_MODE (mode)
25641 || VALID_MMX_REG_MODE_3DNOW (mode));
25644 if (mode == QImode)
25646 /* Take care for QImode values - they can be in non-QI regs,
25647 but then they do cause partial register stalls. */
25648 if (regno < 4 || TARGET_64BIT)
25650 if (!TARGET_PARTIAL_REG_STALL)
25652 return reload_in_progress || reload_completed;
25654 /* We handle both integer and floats in the general purpose registers. */
25655 else if (VALID_INT_MODE_P (mode))
25657 else if (VALID_FP_MODE_P (mode))
25659 else if (VALID_DFP_MODE_P (mode))
25661 /* Lots of MMX code casts 8 byte vector modes to DImode. If we then go
25662 on to use that value in smaller contexts, this can easily force a
25663 pseudo to be allocated to GENERAL_REGS. Since this is no worse than
25664 supporting DImode, allow it. */
25665 else if (VALID_MMX_REG_MODE_3DNOW (mode) || VALID_MMX_REG_MODE (mode))
25671 /* A subroutine of ix86_modes_tieable_p. Return true if MODE is a
25672 tieable integer mode. */
25675 ix86_tieable_integer_mode_p (enum machine_mode mode)
25684 return TARGET_64BIT || !TARGET_PARTIAL_REG_STALL;
25687 return TARGET_64BIT;
25694 /* Return true if MODE1 is accessible in a register that can hold MODE2
25695 without copying. That is, all register classes that can hold MODE2
25696 can also hold MODE1. */
25699 ix86_modes_tieable_p (enum machine_mode mode1, enum machine_mode mode2)
25701 if (mode1 == mode2)
25704 if (ix86_tieable_integer_mode_p (mode1)
25705 && ix86_tieable_integer_mode_p (mode2))
25708 /* MODE2 being XFmode implies fp stack or general regs, which means we
25709 can tie any smaller floating point modes to it. Note that we do not
25710 tie this with TFmode. */
25711 if (mode2 == XFmode)
25712 return mode1 == SFmode || mode1 == DFmode;
25714 /* MODE2 being DFmode implies fp stack, general or sse regs, which means
25715 that we can tie it with SFmode. */
25716 if (mode2 == DFmode)
25717 return mode1 == SFmode;
25719 /* If MODE2 is only appropriate for an SSE register, then tie with
25720 any other mode acceptable to SSE registers. */
25721 if (GET_MODE_SIZE (mode2) == 16
25722 && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode2))
25723 return (GET_MODE_SIZE (mode1) == 16
25724 && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode1));
25726 /* If MODE2 is appropriate for an MMX register, then tie
25727 with any other mode acceptable to MMX registers. */
25728 if (GET_MODE_SIZE (mode2) == 8
25729 && ix86_hard_regno_mode_ok (FIRST_MMX_REG, mode2))
25730 return (GET_MODE_SIZE (mode1) == 8
25731 && ix86_hard_regno_mode_ok (FIRST_MMX_REG, mode1));
25736 /* Compute a (partial) cost for rtx X. Return true if the complete
25737 cost has been computed, and false if subexpressions should be
25738 scanned. In either case, *TOTAL contains the cost result. */
25741 ix86_rtx_costs (rtx x, int code, int outer_code_i, int *total, bool speed)
25743 enum rtx_code outer_code = (enum rtx_code) outer_code_i;
25744 enum machine_mode mode = GET_MODE (x);
25745 const struct processor_costs *cost = speed ? ix86_cost : &ix86_size_cost;
25753 if (TARGET_64BIT && !x86_64_immediate_operand (x, VOIDmode))
25755 else if (TARGET_64BIT && !x86_64_zext_immediate_operand (x, VOIDmode))
25757 else if (flag_pic && SYMBOLIC_CONST (x)
25759 || (!GET_CODE (x) != LABEL_REF
25760 && (GET_CODE (x) != SYMBOL_REF
25761 || !SYMBOL_REF_LOCAL_P (x)))))
25768 if (mode == VOIDmode)
25771 switch (standard_80387_constant_p (x))
25776 default: /* Other constants */
25781 /* Start with (MEM (SYMBOL_REF)), since that's where
25782 it'll probably end up. Add a penalty for size. */
25783 *total = (COSTS_N_INSNS (1)
25784 + (flag_pic != 0 && !TARGET_64BIT)
25785 + (mode == SFmode ? 0 : mode == DFmode ? 1 : 2));
25791 /* The zero extensions is often completely free on x86_64, so make
25792 it as cheap as possible. */
25793 if (TARGET_64BIT && mode == DImode
25794 && GET_MODE (XEXP (x, 0)) == SImode)
25796 else if (TARGET_ZERO_EXTEND_WITH_AND)
25797 *total = cost->add;
25799 *total = cost->movzx;
25803 *total = cost->movsx;
25807 if (CONST_INT_P (XEXP (x, 1))
25808 && (GET_MODE (XEXP (x, 0)) != DImode || TARGET_64BIT))
25810 HOST_WIDE_INT value = INTVAL (XEXP (x, 1));
25813 *total = cost->add;
25816 if ((value == 2 || value == 3)
25817 && cost->lea <= cost->shift_const)
25819 *total = cost->lea;
25829 if (!TARGET_64BIT && GET_MODE (XEXP (x, 0)) == DImode)
25831 if (CONST_INT_P (XEXP (x, 1)))
25833 if (INTVAL (XEXP (x, 1)) > 32)
25834 *total = cost->shift_const + COSTS_N_INSNS (2);
25836 *total = cost->shift_const * 2;
25840 if (GET_CODE (XEXP (x, 1)) == AND)
25841 *total = cost->shift_var * 2;
25843 *total = cost->shift_var * 6 + COSTS_N_INSNS (2);
25848 if (CONST_INT_P (XEXP (x, 1)))
25849 *total = cost->shift_const;
25851 *total = cost->shift_var;
25856 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
25858 /* ??? SSE scalar cost should be used here. */
25859 *total = cost->fmul;
25862 else if (X87_FLOAT_MODE_P (mode))
25864 *total = cost->fmul;
25867 else if (FLOAT_MODE_P (mode))
25869 /* ??? SSE vector cost should be used here. */
25870 *total = cost->fmul;
25875 rtx op0 = XEXP (x, 0);
25876 rtx op1 = XEXP (x, 1);
25878 if (CONST_INT_P (XEXP (x, 1)))
25880 unsigned HOST_WIDE_INT value = INTVAL (XEXP (x, 1));
25881 for (nbits = 0; value != 0; value &= value - 1)
25885 /* This is arbitrary. */
25888 /* Compute costs correctly for widening multiplication. */
25889 if ((GET_CODE (op0) == SIGN_EXTEND || GET_CODE (op0) == ZERO_EXTEND)
25890 && GET_MODE_SIZE (GET_MODE (XEXP (op0, 0))) * 2
25891 == GET_MODE_SIZE (mode))
25893 int is_mulwiden = 0;
25894 enum machine_mode inner_mode = GET_MODE (op0);
25896 if (GET_CODE (op0) == GET_CODE (op1))
25897 is_mulwiden = 1, op1 = XEXP (op1, 0);
25898 else if (CONST_INT_P (op1))
25900 if (GET_CODE (op0) == SIGN_EXTEND)
25901 is_mulwiden = trunc_int_for_mode (INTVAL (op1), inner_mode)
25904 is_mulwiden = !(INTVAL (op1) & ~GET_MODE_MASK (inner_mode));
25908 op0 = XEXP (op0, 0), mode = GET_MODE (op0);
25911 *total = (cost->mult_init[MODE_INDEX (mode)]
25912 + nbits * cost->mult_bit
25913 + rtx_cost (op0, outer_code, speed) + rtx_cost (op1, outer_code, speed));
25922 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
25923 /* ??? SSE cost should be used here. */
25924 *total = cost->fdiv;
25925 else if (X87_FLOAT_MODE_P (mode))
25926 *total = cost->fdiv;
25927 else if (FLOAT_MODE_P (mode))
25928 /* ??? SSE vector cost should be used here. */
25929 *total = cost->fdiv;
25931 *total = cost->divide[MODE_INDEX (mode)];
25935 if (GET_MODE_CLASS (mode) == MODE_INT
25936 && GET_MODE_BITSIZE (mode) <= GET_MODE_BITSIZE (Pmode))
25938 if (GET_CODE (XEXP (x, 0)) == PLUS
25939 && GET_CODE (XEXP (XEXP (x, 0), 0)) == MULT
25940 && CONST_INT_P (XEXP (XEXP (XEXP (x, 0), 0), 1))
25941 && CONSTANT_P (XEXP (x, 1)))
25943 HOST_WIDE_INT val = INTVAL (XEXP (XEXP (XEXP (x, 0), 0), 1));
25944 if (val == 2 || val == 4 || val == 8)
25946 *total = cost->lea;
25947 *total += rtx_cost (XEXP (XEXP (x, 0), 1), outer_code, speed);
25948 *total += rtx_cost (XEXP (XEXP (XEXP (x, 0), 0), 0),
25949 outer_code, speed);
25950 *total += rtx_cost (XEXP (x, 1), outer_code, speed);
25954 else if (GET_CODE (XEXP (x, 0)) == MULT
25955 && CONST_INT_P (XEXP (XEXP (x, 0), 1)))
25957 HOST_WIDE_INT val = INTVAL (XEXP (XEXP (x, 0), 1));
25958 if (val == 2 || val == 4 || val == 8)
25960 *total = cost->lea;
25961 *total += rtx_cost (XEXP (XEXP (x, 0), 0), outer_code, speed);
25962 *total += rtx_cost (XEXP (x, 1), outer_code, speed);
25966 else if (GET_CODE (XEXP (x, 0)) == PLUS)
25968 *total = cost->lea;
25969 *total += rtx_cost (XEXP (XEXP (x, 0), 0), outer_code, speed);
25970 *total += rtx_cost (XEXP (XEXP (x, 0), 1), outer_code, speed);
25971 *total += rtx_cost (XEXP (x, 1), outer_code, speed);
25978 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
25980 /* ??? SSE cost should be used here. */
25981 *total = cost->fadd;
25984 else if (X87_FLOAT_MODE_P (mode))
25986 *total = cost->fadd;
25989 else if (FLOAT_MODE_P (mode))
25991 /* ??? SSE vector cost should be used here. */
25992 *total = cost->fadd;
26000 if (!TARGET_64BIT && mode == DImode)
26002 *total = (cost->add * 2
26003 + (rtx_cost (XEXP (x, 0), outer_code, speed)
26004 << (GET_MODE (XEXP (x, 0)) != DImode))
26005 + (rtx_cost (XEXP (x, 1), outer_code, speed)
26006 << (GET_MODE (XEXP (x, 1)) != DImode)));
26012 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
26014 /* ??? SSE cost should be used here. */
26015 *total = cost->fchs;
26018 else if (X87_FLOAT_MODE_P (mode))
26020 *total = cost->fchs;
26023 else if (FLOAT_MODE_P (mode))
26025 /* ??? SSE vector cost should be used here. */
26026 *total = cost->fchs;
26032 if (!TARGET_64BIT && mode == DImode)
26033 *total = cost->add * 2;
26035 *total = cost->add;
26039 if (GET_CODE (XEXP (x, 0)) == ZERO_EXTRACT
26040 && XEXP (XEXP (x, 0), 1) == const1_rtx
26041 && CONST_INT_P (XEXP (XEXP (x, 0), 2))
26042 && XEXP (x, 1) == const0_rtx)
26044 /* This kind of construct is implemented using test[bwl].
26045 Treat it as if we had an AND. */
26046 *total = (cost->add
26047 + rtx_cost (XEXP (XEXP (x, 0), 0), outer_code, speed)
26048 + rtx_cost (const1_rtx, outer_code, speed));
26054 if (!(SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH))
26059 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
26060 /* ??? SSE cost should be used here. */
26061 *total = cost->fabs;
26062 else if (X87_FLOAT_MODE_P (mode))
26063 *total = cost->fabs;
26064 else if (FLOAT_MODE_P (mode))
26065 /* ??? SSE vector cost should be used here. */
26066 *total = cost->fabs;
26070 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
26071 /* ??? SSE cost should be used here. */
26072 *total = cost->fsqrt;
26073 else if (X87_FLOAT_MODE_P (mode))
26074 *total = cost->fsqrt;
26075 else if (FLOAT_MODE_P (mode))
26076 /* ??? SSE vector cost should be used here. */
26077 *total = cost->fsqrt;
26081 if (XINT (x, 1) == UNSPEC_TP)
26092 static int current_machopic_label_num;
26094 /* Given a symbol name and its associated stub, write out the
26095 definition of the stub. */
26098 machopic_output_stub (FILE *file, const char *symb, const char *stub)
26100 unsigned int length;
26101 char *binder_name, *symbol_name, lazy_ptr_name[32];
26102 int label = ++current_machopic_label_num;
26104 /* For 64-bit we shouldn't get here. */
26105 gcc_assert (!TARGET_64BIT);
26107 /* Lose our funky encoding stuff so it doesn't contaminate the stub. */
26108 symb = (*targetm.strip_name_encoding) (symb);
26110 length = strlen (stub);
26111 binder_name = XALLOCAVEC (char, length + 32);
26112 GEN_BINDER_NAME_FOR_STUB (binder_name, stub, length);
26114 length = strlen (symb);
26115 symbol_name = XALLOCAVEC (char, length + 32);
26116 GEN_SYMBOL_NAME_FOR_SYMBOL (symbol_name, symb, length);
26118 sprintf (lazy_ptr_name, "L%d$lz", label);
26121 switch_to_section (darwin_sections[machopic_picsymbol_stub_section]);
26123 switch_to_section (darwin_sections[machopic_symbol_stub_section]);
26125 fprintf (file, "%s:\n", stub);
26126 fprintf (file, "\t.indirect_symbol %s\n", symbol_name);
26130 fprintf (file, "\tcall\tLPC$%d\nLPC$%d:\tpopl\t%%eax\n", label, label);
26131 fprintf (file, "\tmovl\t%s-LPC$%d(%%eax),%%edx\n", lazy_ptr_name, label);
26132 fprintf (file, "\tjmp\t*%%edx\n");
26135 fprintf (file, "\tjmp\t*%s\n", lazy_ptr_name);
26137 fprintf (file, "%s:\n", binder_name);
26141 fprintf (file, "\tlea\t%s-LPC$%d(%%eax),%%eax\n", lazy_ptr_name, label);
26142 fprintf (file, "\tpushl\t%%eax\n");
26145 fprintf (file, "\tpushl\t$%s\n", lazy_ptr_name);
26147 fprintf (file, "\tjmp\tdyld_stub_binding_helper\n");
26149 switch_to_section (darwin_sections[machopic_lazy_symbol_ptr_section]);
26150 fprintf (file, "%s:\n", lazy_ptr_name);
26151 fprintf (file, "\t.indirect_symbol %s\n", symbol_name);
26152 fprintf (file, "\t.long %s\n", binder_name);
26156 darwin_x86_file_end (void)
26158 darwin_file_end ();
26161 #endif /* TARGET_MACHO */
26163 /* Order the registers for register allocator. */
26166 x86_order_regs_for_local_alloc (void)
26171 /* First allocate the local general purpose registers. */
26172 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
26173 if (GENERAL_REGNO_P (i) && call_used_regs[i])
26174 reg_alloc_order [pos++] = i;
26176 /* Global general purpose registers. */
26177 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
26178 if (GENERAL_REGNO_P (i) && !call_used_regs[i])
26179 reg_alloc_order [pos++] = i;
26181 /* x87 registers come first in case we are doing FP math
26183 if (!TARGET_SSE_MATH)
26184 for (i = FIRST_STACK_REG; i <= LAST_STACK_REG; i++)
26185 reg_alloc_order [pos++] = i;
26187 /* SSE registers. */
26188 for (i = FIRST_SSE_REG; i <= LAST_SSE_REG; i++)
26189 reg_alloc_order [pos++] = i;
26190 for (i = FIRST_REX_SSE_REG; i <= LAST_REX_SSE_REG; i++)
26191 reg_alloc_order [pos++] = i;
26193 /* x87 registers. */
26194 if (TARGET_SSE_MATH)
26195 for (i = FIRST_STACK_REG; i <= LAST_STACK_REG; i++)
26196 reg_alloc_order [pos++] = i;
26198 for (i = FIRST_MMX_REG; i <= LAST_MMX_REG; i++)
26199 reg_alloc_order [pos++] = i;
26201 /* Initialize the rest of array as we do not allocate some registers
26203 while (pos < FIRST_PSEUDO_REGISTER)
26204 reg_alloc_order [pos++] = 0;
26207 /* Handle a "ms_abi" or "sysv" attribute; arguments as in
26208 struct attribute_spec.handler. */
26210 ix86_handle_abi_attribute (tree *node, tree name,
26211 tree args ATTRIBUTE_UNUSED,
26212 int flags ATTRIBUTE_UNUSED, bool *no_add_attrs)
26214 if (TREE_CODE (*node) != FUNCTION_TYPE
26215 && TREE_CODE (*node) != METHOD_TYPE
26216 && TREE_CODE (*node) != FIELD_DECL
26217 && TREE_CODE (*node) != TYPE_DECL)
26219 warning (OPT_Wattributes, "%qs attribute only applies to functions",
26220 IDENTIFIER_POINTER (name));
26221 *no_add_attrs = true;
26226 warning (OPT_Wattributes, "%qs attribute only available for 64-bit",
26227 IDENTIFIER_POINTER (name));
26228 *no_add_attrs = true;
26232 /* Can combine regparm with all attributes but fastcall. */
26233 if (is_attribute_p ("ms_abi", name))
26235 if (lookup_attribute ("sysv_abi", TYPE_ATTRIBUTES (*node)))
26237 error ("ms_abi and sysv_abi attributes are not compatible");
26242 else if (is_attribute_p ("sysv_abi", name))
26244 if (lookup_attribute ("ms_abi", TYPE_ATTRIBUTES (*node)))
26246 error ("ms_abi and sysv_abi attributes are not compatible");
26255 /* Handle a "ms_struct" or "gcc_struct" attribute; arguments as in
26256 struct attribute_spec.handler. */
26258 ix86_handle_struct_attribute (tree *node, tree name,
26259 tree args ATTRIBUTE_UNUSED,
26260 int flags ATTRIBUTE_UNUSED, bool *no_add_attrs)
26263 if (DECL_P (*node))
26265 if (TREE_CODE (*node) == TYPE_DECL)
26266 type = &TREE_TYPE (*node);
26271 if (!(type && (TREE_CODE (*type) == RECORD_TYPE
26272 || TREE_CODE (*type) == UNION_TYPE)))
26274 warning (OPT_Wattributes, "%qs attribute ignored",
26275 IDENTIFIER_POINTER (name));
26276 *no_add_attrs = true;
26279 else if ((is_attribute_p ("ms_struct", name)
26280 && lookup_attribute ("gcc_struct", TYPE_ATTRIBUTES (*type)))
26281 || ((is_attribute_p ("gcc_struct", name)
26282 && lookup_attribute ("ms_struct", TYPE_ATTRIBUTES (*type)))))
26284 warning (OPT_Wattributes, "%qs incompatible attribute ignored",
26285 IDENTIFIER_POINTER (name));
26286 *no_add_attrs = true;
26293 ix86_ms_bitfield_layout_p (const_tree record_type)
26295 return (TARGET_MS_BITFIELD_LAYOUT &&
26296 !lookup_attribute ("gcc_struct", TYPE_ATTRIBUTES (record_type)))
26297 || lookup_attribute ("ms_struct", TYPE_ATTRIBUTES (record_type));
26300 /* Returns an expression indicating where the this parameter is
26301 located on entry to the FUNCTION. */
26304 x86_this_parameter (tree function)
26306 tree type = TREE_TYPE (function);
26307 bool aggr = aggregate_value_p (TREE_TYPE (type), type) != 0;
26312 const int *parm_regs;
26314 if (ix86_function_type_abi (type) == MS_ABI)
26315 parm_regs = x86_64_ms_abi_int_parameter_registers;
26317 parm_regs = x86_64_int_parameter_registers;
26318 return gen_rtx_REG (DImode, parm_regs[aggr]);
26321 nregs = ix86_function_regparm (type, function);
26323 if (nregs > 0 && !stdarg_p (type))
26327 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (type)))
26328 regno = aggr ? DX_REG : CX_REG;
26336 return gen_rtx_MEM (SImode,
26337 plus_constant (stack_pointer_rtx, 4));
26340 return gen_rtx_REG (SImode, regno);
26343 return gen_rtx_MEM (SImode, plus_constant (stack_pointer_rtx, aggr ? 8 : 4));
26346 /* Determine whether x86_output_mi_thunk can succeed. */
26349 x86_can_output_mi_thunk (const_tree thunk ATTRIBUTE_UNUSED,
26350 HOST_WIDE_INT delta ATTRIBUTE_UNUSED,
26351 HOST_WIDE_INT vcall_offset, const_tree function)
26353 /* 64-bit can handle anything. */
26357 /* For 32-bit, everything's fine if we have one free register. */
26358 if (ix86_function_regparm (TREE_TYPE (function), function) < 3)
26361 /* Need a free register for vcall_offset. */
26365 /* Need a free register for GOT references. */
26366 if (flag_pic && !(*targetm.binds_local_p) (function))
26369 /* Otherwise ok. */
26373 /* Output the assembler code for a thunk function. THUNK_DECL is the
26374 declaration for the thunk function itself, FUNCTION is the decl for
26375 the target function. DELTA is an immediate constant offset to be
26376 added to THIS. If VCALL_OFFSET is nonzero, the word at
26377 *(*this + vcall_offset) should be added to THIS. */
26380 x86_output_mi_thunk (FILE *file ATTRIBUTE_UNUSED,
26381 tree thunk ATTRIBUTE_UNUSED, HOST_WIDE_INT delta,
26382 HOST_WIDE_INT vcall_offset, tree function)
26385 rtx this_param = x86_this_parameter (function);
26388 /* If VCALL_OFFSET, we'll need THIS in a register. Might as well
26389 pull it in now and let DELTA benefit. */
26390 if (REG_P (this_param))
26391 this_reg = this_param;
26392 else if (vcall_offset)
26394 /* Put the this parameter into %eax. */
26395 xops[0] = this_param;
26396 xops[1] = this_reg = gen_rtx_REG (Pmode, AX_REG);
26397 output_asm_insn ("mov%z1\t{%0, %1|%1, %0}", xops);
26400 this_reg = NULL_RTX;
26402 /* Adjust the this parameter by a fixed constant. */
26405 xops[0] = GEN_INT (delta);
26406 xops[1] = this_reg ? this_reg : this_param;
26409 if (!x86_64_general_operand (xops[0], DImode))
26411 tmp = gen_rtx_REG (DImode, R10_REG);
26413 output_asm_insn ("mov{q}\t{%1, %0|%0, %1}", xops);
26415 xops[1] = this_param;
26417 output_asm_insn ("add{q}\t{%0, %1|%1, %0}", xops);
26420 output_asm_insn ("add{l}\t{%0, %1|%1, %0}", xops);
26423 /* Adjust the this parameter by a value stored in the vtable. */
26427 tmp = gen_rtx_REG (DImode, R10_REG);
26430 int tmp_regno = CX_REG;
26431 if (lookup_attribute ("fastcall",
26432 TYPE_ATTRIBUTES (TREE_TYPE (function))))
26433 tmp_regno = AX_REG;
26434 tmp = gen_rtx_REG (SImode, tmp_regno);
26437 xops[0] = gen_rtx_MEM (Pmode, this_reg);
26439 output_asm_insn ("mov%z1\t{%0, %1|%1, %0}", xops);
26441 /* Adjust the this parameter. */
26442 xops[0] = gen_rtx_MEM (Pmode, plus_constant (tmp, vcall_offset));
26443 if (TARGET_64BIT && !memory_operand (xops[0], Pmode))
26445 rtx tmp2 = gen_rtx_REG (DImode, R11_REG);
26446 xops[0] = GEN_INT (vcall_offset);
26448 output_asm_insn ("mov{q}\t{%0, %1|%1, %0}", xops);
26449 xops[0] = gen_rtx_MEM (Pmode, gen_rtx_PLUS (Pmode, tmp, tmp2));
26451 xops[1] = this_reg;
26452 output_asm_insn ("add%z1\t{%0, %1|%1, %0}", xops);
26455 /* If necessary, drop THIS back to its stack slot. */
26456 if (this_reg && this_reg != this_param)
26458 xops[0] = this_reg;
26459 xops[1] = this_param;
26460 output_asm_insn ("mov%z1\t{%0, %1|%1, %0}", xops);
26463 xops[0] = XEXP (DECL_RTL (function), 0);
26466 if (!flag_pic || (*targetm.binds_local_p) (function))
26467 output_asm_insn ("jmp\t%P0", xops);
26468 /* All thunks should be in the same object as their target,
26469 and thus binds_local_p should be true. */
26470 else if (TARGET_64BIT && cfun->machine->call_abi == MS_ABI)
26471 gcc_unreachable ();
26474 tmp = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, xops[0]), UNSPEC_GOTPCREL);
26475 tmp = gen_rtx_CONST (Pmode, tmp);
26476 tmp = gen_rtx_MEM (QImode, tmp);
26478 output_asm_insn ("jmp\t%A0", xops);
26483 if (!flag_pic || (*targetm.binds_local_p) (function))
26484 output_asm_insn ("jmp\t%P0", xops);
26489 rtx sym_ref = XEXP (DECL_RTL (function), 0);
26490 tmp = (gen_rtx_SYMBOL_REF
26492 machopic_indirection_name (sym_ref, /*stub_p=*/true)));
26493 tmp = gen_rtx_MEM (QImode, tmp);
26495 output_asm_insn ("jmp\t%0", xops);
26498 #endif /* TARGET_MACHO */
26500 tmp = gen_rtx_REG (SImode, CX_REG);
26501 output_set_got (tmp, NULL_RTX);
26504 output_asm_insn ("mov{l}\t{%0@GOT(%1), %1|%1, %0@GOT[%1]}", xops);
26505 output_asm_insn ("jmp\t{*}%1", xops);
26511 x86_file_start (void)
26513 default_file_start ();
26515 darwin_file_start ();
26517 if (X86_FILE_START_VERSION_DIRECTIVE)
26518 fputs ("\t.version\t\"01.01\"\n", asm_out_file);
26519 if (X86_FILE_START_FLTUSED)
26520 fputs ("\t.global\t__fltused\n", asm_out_file);
26521 if (ix86_asm_dialect == ASM_INTEL)
26522 fputs ("\t.intel_syntax noprefix\n", asm_out_file);
26526 x86_field_alignment (tree field, int computed)
26528 enum machine_mode mode;
26529 tree type = TREE_TYPE (field);
26531 if (TARGET_64BIT || TARGET_ALIGN_DOUBLE)
26533 mode = TYPE_MODE (strip_array_types (type));
26534 if (mode == DFmode || mode == DCmode
26535 || GET_MODE_CLASS (mode) == MODE_INT
26536 || GET_MODE_CLASS (mode) == MODE_COMPLEX_INT)
26537 return MIN (32, computed);
26541 /* Output assembler code to FILE to increment profiler label # LABELNO
26542 for profiling a function entry. */
26544 x86_function_profiler (FILE *file, int labelno ATTRIBUTE_UNUSED)
26548 #ifndef NO_PROFILE_COUNTERS
26549 fprintf (file, "\tleaq\t%sP%d@(%%rip),%%r11\n", LPREFIX, labelno);
26552 if (DEFAULT_ABI == SYSV_ABI && flag_pic)
26553 fprintf (file, "\tcall\t*%s@GOTPCREL(%%rip)\n", MCOUNT_NAME);
26555 fprintf (file, "\tcall\t%s\n", MCOUNT_NAME);
26559 #ifndef NO_PROFILE_COUNTERS
26560 fprintf (file, "\tleal\t%sP%d@GOTOFF(%%ebx),%%%s\n",
26561 LPREFIX, labelno, PROFILE_COUNT_REGISTER);
26563 fprintf (file, "\tcall\t*%s@GOT(%%ebx)\n", MCOUNT_NAME);
26567 #ifndef NO_PROFILE_COUNTERS
26568 fprintf (file, "\tmovl\t$%sP%d,%%%s\n", LPREFIX, labelno,
26569 PROFILE_COUNT_REGISTER);
26571 fprintf (file, "\tcall\t%s\n", MCOUNT_NAME);
26575 /* We don't have exact information about the insn sizes, but we may assume
26576 quite safely that we are informed about all 1 byte insns and memory
26577 address sizes. This is enough to eliminate unnecessary padding in
26581 min_insn_size (rtx insn)
26585 if (!INSN_P (insn) || !active_insn_p (insn))
26588 /* Discard alignments we've emit and jump instructions. */
26589 if (GET_CODE (PATTERN (insn)) == UNSPEC_VOLATILE
26590 && XINT (PATTERN (insn), 1) == UNSPECV_ALIGN)
26593 && (GET_CODE (PATTERN (insn)) == ADDR_VEC
26594 || GET_CODE (PATTERN (insn)) == ADDR_DIFF_VEC))
26597 /* Important case - calls are always 5 bytes.
26598 It is common to have many calls in the row. */
26600 && symbolic_reference_mentioned_p (PATTERN (insn))
26601 && !SIBLING_CALL_P (insn))
26603 if (get_attr_length (insn) <= 1)
26606 /* For normal instructions we may rely on the sizes of addresses
26607 and the presence of symbol to require 4 bytes of encoding.
26608 This is not the case for jumps where references are PC relative. */
26609 if (!JUMP_P (insn))
26611 l = get_attr_length_address (insn);
26612 if (l < 4 && symbolic_reference_mentioned_p (PATTERN (insn)))
26621 /* AMD K8 core mispredicts jumps when there are more than 3 jumps in 16 byte
26625 ix86_avoid_jump_misspredicts (void)
26627 rtx insn, start = get_insns ();
26628 int nbytes = 0, njumps = 0;
26631 /* Look for all minimal intervals of instructions containing 4 jumps.
26632 The intervals are bounded by START and INSN. NBYTES is the total
26633 size of instructions in the interval including INSN and not including
26634 START. When the NBYTES is smaller than 16 bytes, it is possible
26635 that the end of START and INSN ends up in the same 16byte page.
26637 The smallest offset in the page INSN can start is the case where START
26638 ends on the offset 0. Offset of INSN is then NBYTES - sizeof (INSN).
26639 We add p2align to 16byte window with maxskip 17 - NBYTES + sizeof (INSN).
26641 for (insn = get_insns (); insn; insn = NEXT_INSN (insn))
26644 nbytes += min_insn_size (insn);
26646 fprintf(dump_file, "Insn %i estimated to %i bytes\n",
26647 INSN_UID (insn), min_insn_size (insn));
26649 && GET_CODE (PATTERN (insn)) != ADDR_VEC
26650 && GET_CODE (PATTERN (insn)) != ADDR_DIFF_VEC)
26658 start = NEXT_INSN (start);
26659 if ((JUMP_P (start)
26660 && GET_CODE (PATTERN (start)) != ADDR_VEC
26661 && GET_CODE (PATTERN (start)) != ADDR_DIFF_VEC)
26663 njumps--, isjump = 1;
26666 nbytes -= min_insn_size (start);
26668 gcc_assert (njumps >= 0);
26670 fprintf (dump_file, "Interval %i to %i has %i bytes\n",
26671 INSN_UID (start), INSN_UID (insn), nbytes);
26673 if (njumps == 3 && isjump && nbytes < 16)
26675 int padsize = 15 - nbytes + min_insn_size (insn);
26678 fprintf (dump_file, "Padding insn %i by %i bytes!\n",
26679 INSN_UID (insn), padsize);
26680 emit_insn_before (gen_align (GEN_INT (padsize)), insn);
26685 /* AMD Athlon works faster
26686 when RET is not destination of conditional jump or directly preceded
26687 by other jump instruction. We avoid the penalty by inserting NOP just
26688 before the RET instructions in such cases. */
26690 ix86_pad_returns (void)
26695 FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR->preds)
26697 basic_block bb = e->src;
26698 rtx ret = BB_END (bb);
26700 bool replace = false;
26702 if (!JUMP_P (ret) || GET_CODE (PATTERN (ret)) != RETURN
26703 || optimize_bb_for_size_p (bb))
26705 for (prev = PREV_INSN (ret); prev; prev = PREV_INSN (prev))
26706 if (active_insn_p (prev) || LABEL_P (prev))
26708 if (prev && LABEL_P (prev))
26713 FOR_EACH_EDGE (e, ei, bb->preds)
26714 if (EDGE_FREQUENCY (e) && e->src->index >= 0
26715 && !(e->flags & EDGE_FALLTHRU))
26720 prev = prev_active_insn (ret);
26722 && ((JUMP_P (prev) && any_condjump_p (prev))
26725 /* Empty functions get branch mispredict even when the jump destination
26726 is not visible to us. */
26727 if (!prev && cfun->function_frequency > FUNCTION_FREQUENCY_UNLIKELY_EXECUTED)
26732 emit_insn_before (gen_return_internal_long (), ret);
26738 /* Implement machine specific optimizations. We implement padding of returns
26739 for K8 CPUs and pass to avoid 4 jumps in the single 16 byte window. */
26743 if (TARGET_PAD_RETURNS && optimize
26744 && optimize_function_for_speed_p (cfun))
26745 ix86_pad_returns ();
26746 if (TARGET_FOUR_JUMP_LIMIT && optimize
26747 && optimize_function_for_speed_p (cfun))
26748 ix86_avoid_jump_misspredicts ();
26751 /* Return nonzero when QImode register that must be represented via REX prefix
26754 x86_extended_QIreg_mentioned_p (rtx insn)
26757 extract_insn_cached (insn);
26758 for (i = 0; i < recog_data.n_operands; i++)
26759 if (REG_P (recog_data.operand[i])
26760 && REGNO (recog_data.operand[i]) >= 4)
26765 /* Return nonzero when P points to register encoded via REX prefix.
26766 Called via for_each_rtx. */
26768 extended_reg_mentioned_1 (rtx *p, void *data ATTRIBUTE_UNUSED)
26770 unsigned int regno;
26773 regno = REGNO (*p);
26774 return REX_INT_REGNO_P (regno) || REX_SSE_REGNO_P (regno);
26777 /* Return true when INSN mentions register that must be encoded using REX
26780 x86_extended_reg_mentioned_p (rtx insn)
26782 return for_each_rtx (INSN_P (insn) ? &PATTERN (insn) : &insn,
26783 extended_reg_mentioned_1, NULL);
26786 /* Generate an unsigned DImode/SImode to FP conversion. This is the same code
26787 optabs would emit if we didn't have TFmode patterns. */
26790 x86_emit_floatuns (rtx operands[2])
26792 rtx neglab, donelab, i0, i1, f0, in, out;
26793 enum machine_mode mode, inmode;
26795 inmode = GET_MODE (operands[1]);
26796 gcc_assert (inmode == SImode || inmode == DImode);
26799 in = force_reg (inmode, operands[1]);
26800 mode = GET_MODE (out);
26801 neglab = gen_label_rtx ();
26802 donelab = gen_label_rtx ();
26803 f0 = gen_reg_rtx (mode);
26805 emit_cmp_and_jump_insns (in, const0_rtx, LT, const0_rtx, inmode, 0, neglab);
26807 expand_float (out, in, 0);
26809 emit_jump_insn (gen_jump (donelab));
26812 emit_label (neglab);
26814 i0 = expand_simple_binop (inmode, LSHIFTRT, in, const1_rtx, NULL,
26816 i1 = expand_simple_binop (inmode, AND, in, const1_rtx, NULL,
26818 i0 = expand_simple_binop (inmode, IOR, i0, i1, i0, 1, OPTAB_DIRECT);
26820 expand_float (f0, i0, 0);
26822 emit_insn (gen_rtx_SET (VOIDmode, out, gen_rtx_PLUS (mode, f0, f0)));
26824 emit_label (donelab);
26827 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
26828 with all elements equal to VAR. Return true if successful. */
26831 ix86_expand_vector_init_duplicate (bool mmx_ok, enum machine_mode mode,
26832 rtx target, rtx val)
26834 enum machine_mode hmode, smode, wsmode, wvmode;
26849 val = force_reg (GET_MODE_INNER (mode), val);
26850 x = gen_rtx_VEC_DUPLICATE (mode, val);
26851 emit_insn (gen_rtx_SET (VOIDmode, target, x));
26857 if (TARGET_SSE || TARGET_3DNOW_A)
26859 val = gen_lowpart (SImode, val);
26860 x = gen_rtx_TRUNCATE (HImode, val);
26861 x = gen_rtx_VEC_DUPLICATE (mode, x);
26862 emit_insn (gen_rtx_SET (VOIDmode, target, x));
26884 /* Extend HImode to SImode using a paradoxical SUBREG. */
26885 tmp1 = gen_reg_rtx (SImode);
26886 emit_move_insn (tmp1, gen_lowpart (SImode, val));
26887 /* Insert the SImode value as low element of V4SImode vector. */
26888 tmp2 = gen_reg_rtx (V4SImode);
26889 tmp1 = gen_rtx_VEC_MERGE (V4SImode,
26890 gen_rtx_VEC_DUPLICATE (V4SImode, tmp1),
26891 CONST0_RTX (V4SImode),
26893 emit_insn (gen_rtx_SET (VOIDmode, tmp2, tmp1));
26894 /* Cast the V4SImode vector back to a V8HImode vector. */
26895 tmp1 = gen_reg_rtx (V8HImode);
26896 emit_move_insn (tmp1, gen_lowpart (V8HImode, tmp2));
26897 /* Duplicate the low short through the whole low SImode word. */
26898 emit_insn (gen_sse2_punpcklwd (tmp1, tmp1, tmp1));
26899 /* Cast the V8HImode vector back to a V4SImode vector. */
26900 tmp2 = gen_reg_rtx (V4SImode);
26901 emit_move_insn (tmp2, gen_lowpart (V4SImode, tmp1));
26902 /* Replicate the low element of the V4SImode vector. */
26903 emit_insn (gen_sse2_pshufd (tmp2, tmp2, const0_rtx));
26904 /* Cast the V2SImode back to V8HImode, and store in target. */
26905 emit_move_insn (target, gen_lowpart (V8HImode, tmp2));
26916 /* Extend QImode to SImode using a paradoxical SUBREG. */
26917 tmp1 = gen_reg_rtx (SImode);
26918 emit_move_insn (tmp1, gen_lowpart (SImode, val));
26919 /* Insert the SImode value as low element of V4SImode vector. */
26920 tmp2 = gen_reg_rtx (V4SImode);
26921 tmp1 = gen_rtx_VEC_MERGE (V4SImode,
26922 gen_rtx_VEC_DUPLICATE (V4SImode, tmp1),
26923 CONST0_RTX (V4SImode),
26925 emit_insn (gen_rtx_SET (VOIDmode, tmp2, tmp1));
26926 /* Cast the V4SImode vector back to a V16QImode vector. */
26927 tmp1 = gen_reg_rtx (V16QImode);
26928 emit_move_insn (tmp1, gen_lowpart (V16QImode, tmp2));
26929 /* Duplicate the low byte through the whole low SImode word. */
26930 emit_insn (gen_sse2_punpcklbw (tmp1, tmp1, tmp1));
26931 emit_insn (gen_sse2_punpcklbw (tmp1, tmp1, tmp1));
26932 /* Cast the V16QImode vector back to a V4SImode vector. */
26933 tmp2 = gen_reg_rtx (V4SImode);
26934 emit_move_insn (tmp2, gen_lowpart (V4SImode, tmp1));
26935 /* Replicate the low element of the V4SImode vector. */
26936 emit_insn (gen_sse2_pshufd (tmp2, tmp2, const0_rtx));
26937 /* Cast the V2SImode back to V16QImode, and store in target. */
26938 emit_move_insn (target, gen_lowpart (V16QImode, tmp2));
26946 /* Replicate the value once into the next wider mode and recurse. */
26947 val = convert_modes (wsmode, smode, val, true);
26948 x = expand_simple_binop (wsmode, ASHIFT, val,
26949 GEN_INT (GET_MODE_BITSIZE (smode)),
26950 NULL_RTX, 1, OPTAB_LIB_WIDEN);
26951 val = expand_simple_binop (wsmode, IOR, val, x, x, 1, OPTAB_LIB_WIDEN);
26953 x = gen_reg_rtx (wvmode);
26954 if (!ix86_expand_vector_init_duplicate (mmx_ok, wvmode, x, val))
26955 gcc_unreachable ();
26956 emit_move_insn (target, gen_lowpart (mode, x));
26979 rtx tmp = gen_reg_rtx (hmode);
26980 ix86_expand_vector_init_duplicate (mmx_ok, hmode, tmp, val);
26981 emit_insn (gen_rtx_SET (VOIDmode, target,
26982 gen_rtx_VEC_CONCAT (mode, tmp, tmp)));
26991 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
26992 whose ONE_VAR element is VAR, and other elements are zero. Return true
26996 ix86_expand_vector_init_one_nonzero (bool mmx_ok, enum machine_mode mode,
26997 rtx target, rtx var, int one_var)
26999 enum machine_mode vsimode;
27002 bool use_vector_set = false;
27007 /* For SSE4.1, we normally use vector set. But if the second
27008 element is zero and inter-unit moves are OK, we use movq
27010 use_vector_set = (TARGET_64BIT
27012 && !(TARGET_INTER_UNIT_MOVES
27018 use_vector_set = TARGET_SSE4_1;
27021 use_vector_set = TARGET_SSE2;
27024 use_vector_set = TARGET_SSE || TARGET_3DNOW_A;
27032 use_vector_set = TARGET_AVX;
27038 if (use_vector_set)
27040 emit_insn (gen_rtx_SET (VOIDmode, target, CONST0_RTX (mode)));
27041 var = force_reg (GET_MODE_INNER (mode), var);
27042 ix86_expand_vector_set (mmx_ok, target, var, one_var);
27058 var = force_reg (GET_MODE_INNER (mode), var);
27059 x = gen_rtx_VEC_CONCAT (mode, var, CONST0_RTX (GET_MODE_INNER (mode)));
27060 emit_insn (gen_rtx_SET (VOIDmode, target, x));
27065 if (!REG_P (target) || REGNO (target) < FIRST_PSEUDO_REGISTER)
27066 new_target = gen_reg_rtx (mode);
27068 new_target = target;
27069 var = force_reg (GET_MODE_INNER (mode), var);
27070 x = gen_rtx_VEC_DUPLICATE (mode, var);
27071 x = gen_rtx_VEC_MERGE (mode, x, CONST0_RTX (mode), const1_rtx);
27072 emit_insn (gen_rtx_SET (VOIDmode, new_target, x));
27075 /* We need to shuffle the value to the correct position, so
27076 create a new pseudo to store the intermediate result. */
27078 /* With SSE2, we can use the integer shuffle insns. */
27079 if (mode != V4SFmode && TARGET_SSE2)
27081 emit_insn (gen_sse2_pshufd_1 (new_target, new_target,
27083 GEN_INT (one_var == 1 ? 0 : 1),
27084 GEN_INT (one_var == 2 ? 0 : 1),
27085 GEN_INT (one_var == 3 ? 0 : 1)));
27086 if (target != new_target)
27087 emit_move_insn (target, new_target);
27091 /* Otherwise convert the intermediate result to V4SFmode and
27092 use the SSE1 shuffle instructions. */
27093 if (mode != V4SFmode)
27095 tmp = gen_reg_rtx (V4SFmode);
27096 emit_move_insn (tmp, gen_lowpart (V4SFmode, new_target));
27101 emit_insn (gen_sse_shufps_v4sf (tmp, tmp, tmp,
27103 GEN_INT (one_var == 1 ? 0 : 1),
27104 GEN_INT (one_var == 2 ? 0+4 : 1+4),
27105 GEN_INT (one_var == 3 ? 0+4 : 1+4)));
27107 if (mode != V4SFmode)
27108 emit_move_insn (target, gen_lowpart (V4SImode, tmp));
27109 else if (tmp != target)
27110 emit_move_insn (target, tmp);
27112 else if (target != new_target)
27113 emit_move_insn (target, new_target);
27118 vsimode = V4SImode;
27124 vsimode = V2SImode;
27130 /* Zero extend the variable element to SImode and recurse. */
27131 var = convert_modes (SImode, GET_MODE_INNER (mode), var, true);
27133 x = gen_reg_rtx (vsimode);
27134 if (!ix86_expand_vector_init_one_nonzero (mmx_ok, vsimode, x,
27136 gcc_unreachable ();
27138 emit_move_insn (target, gen_lowpart (mode, x));
27146 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
27147 consisting of the values in VALS. It is known that all elements
27148 except ONE_VAR are constants. Return true if successful. */
27151 ix86_expand_vector_init_one_var (bool mmx_ok, enum machine_mode mode,
27152 rtx target, rtx vals, int one_var)
27154 rtx var = XVECEXP (vals, 0, one_var);
27155 enum machine_mode wmode;
27158 const_vec = copy_rtx (vals);
27159 XVECEXP (const_vec, 0, one_var) = CONST0_RTX (GET_MODE_INNER (mode));
27160 const_vec = gen_rtx_CONST_VECTOR (mode, XVEC (const_vec, 0));
27168 /* For the two element vectors, it's just as easy to use
27169 the general case. */
27193 /* There's no way to set one QImode entry easily. Combine
27194 the variable value with its adjacent constant value, and
27195 promote to an HImode set. */
27196 x = XVECEXP (vals, 0, one_var ^ 1);
27199 var = convert_modes (HImode, QImode, var, true);
27200 var = expand_simple_binop (HImode, ASHIFT, var, GEN_INT (8),
27201 NULL_RTX, 1, OPTAB_LIB_WIDEN);
27202 x = GEN_INT (INTVAL (x) & 0xff);
27206 var = convert_modes (HImode, QImode, var, true);
27207 x = gen_int_mode (INTVAL (x) << 8, HImode);
27209 if (x != const0_rtx)
27210 var = expand_simple_binop (HImode, IOR, var, x, var,
27211 1, OPTAB_LIB_WIDEN);
27213 x = gen_reg_rtx (wmode);
27214 emit_move_insn (x, gen_lowpart (wmode, const_vec));
27215 ix86_expand_vector_set (mmx_ok, x, var, one_var >> 1);
27217 emit_move_insn (target, gen_lowpart (mode, x));
27224 emit_move_insn (target, const_vec);
27225 ix86_expand_vector_set (mmx_ok, target, var, one_var);
27229 /* A subroutine of ix86_expand_vector_init_general. Use vector
27230 concatenate to handle the most general case: all values variable,
27231 and none identical. */
27234 ix86_expand_vector_init_concat (enum machine_mode mode,
27235 rtx target, rtx *ops, int n)
27237 enum machine_mode cmode, hmode = VOIDmode;
27238 rtx first[8], second[4];
27278 gcc_unreachable ();
27281 if (!register_operand (ops[1], cmode))
27282 ops[1] = force_reg (cmode, ops[1]);
27283 if (!register_operand (ops[0], cmode))
27284 ops[0] = force_reg (cmode, ops[0]);
27285 emit_insn (gen_rtx_SET (VOIDmode, target,
27286 gen_rtx_VEC_CONCAT (mode, ops[0],
27306 gcc_unreachable ();
27322 gcc_unreachable ();
27327 /* FIXME: We process inputs backward to help RA. PR 36222. */
27330 for (; i > 0; i -= 2, j--)
27332 first[j] = gen_reg_rtx (cmode);
27333 v = gen_rtvec (2, ops[i - 1], ops[i]);
27334 ix86_expand_vector_init (false, first[j],
27335 gen_rtx_PARALLEL (cmode, v));
27341 gcc_assert (hmode != VOIDmode);
27342 for (i = j = 0; i < n; i += 2, j++)
27344 second[j] = gen_reg_rtx (hmode);
27345 ix86_expand_vector_init_concat (hmode, second [j],
27349 ix86_expand_vector_init_concat (mode, target, second, n);
27352 ix86_expand_vector_init_concat (mode, target, first, n);
27356 gcc_unreachable ();
27360 /* A subroutine of ix86_expand_vector_init_general. Use vector
27361 interleave to handle the most general case: all values variable,
27362 and none identical. */
27365 ix86_expand_vector_init_interleave (enum machine_mode mode,
27366 rtx target, rtx *ops, int n)
27368 enum machine_mode first_imode, second_imode, third_imode, inner_mode;
27371 rtx (*gen_load_even) (rtx, rtx, rtx);
27372 rtx (*gen_interleave_first_low) (rtx, rtx, rtx);
27373 rtx (*gen_interleave_second_low) (rtx, rtx, rtx);
27378 gen_load_even = gen_vec_setv8hi;
27379 gen_interleave_first_low = gen_vec_interleave_lowv4si;
27380 gen_interleave_second_low = gen_vec_interleave_lowv2di;
27381 inner_mode = HImode;
27382 first_imode = V4SImode;
27383 second_imode = V2DImode;
27384 third_imode = VOIDmode;
27387 gen_load_even = gen_vec_setv16qi;
27388 gen_interleave_first_low = gen_vec_interleave_lowv8hi;
27389 gen_interleave_second_low = gen_vec_interleave_lowv4si;
27390 inner_mode = QImode;
27391 first_imode = V8HImode;
27392 second_imode = V4SImode;
27393 third_imode = V2DImode;
27396 gcc_unreachable ();
27399 for (i = 0; i < n; i++)
27401 /* Extend the odd elment to SImode using a paradoxical SUBREG. */
27402 op0 = gen_reg_rtx (SImode);
27403 emit_move_insn (op0, gen_lowpart (SImode, ops [i + i]));
27405 /* Insert the SImode value as low element of V4SImode vector. */
27406 op1 = gen_reg_rtx (V4SImode);
27407 op0 = gen_rtx_VEC_MERGE (V4SImode,
27408 gen_rtx_VEC_DUPLICATE (V4SImode,
27410 CONST0_RTX (V4SImode),
27412 emit_insn (gen_rtx_SET (VOIDmode, op1, op0));
27414 /* Cast the V4SImode vector back to a vector in orignal mode. */
27415 op0 = gen_reg_rtx (mode);
27416 emit_move_insn (op0, gen_lowpart (mode, op1));
27418 /* Load even elements into the second positon. */
27419 emit_insn ((*gen_load_even) (op0,
27420 force_reg (inner_mode,
27424 /* Cast vector to FIRST_IMODE vector. */
27425 ops[i] = gen_reg_rtx (first_imode);
27426 emit_move_insn (ops[i], gen_lowpart (first_imode, op0));
27429 /* Interleave low FIRST_IMODE vectors. */
27430 for (i = j = 0; i < n; i += 2, j++)
27432 op0 = gen_reg_rtx (first_imode);
27433 emit_insn ((*gen_interleave_first_low) (op0, ops[i], ops[i + 1]));
27435 /* Cast FIRST_IMODE vector to SECOND_IMODE vector. */
27436 ops[j] = gen_reg_rtx (second_imode);
27437 emit_move_insn (ops[j], gen_lowpart (second_imode, op0));
27440 /* Interleave low SECOND_IMODE vectors. */
27441 switch (second_imode)
27444 for (i = j = 0; i < n / 2; i += 2, j++)
27446 op0 = gen_reg_rtx (second_imode);
27447 emit_insn ((*gen_interleave_second_low) (op0, ops[i],
27450 /* Cast the SECOND_IMODE vector to the THIRD_IMODE
27452 ops[j] = gen_reg_rtx (third_imode);
27453 emit_move_insn (ops[j], gen_lowpart (third_imode, op0));
27455 second_imode = V2DImode;
27456 gen_interleave_second_low = gen_vec_interleave_lowv2di;
27460 op0 = gen_reg_rtx (second_imode);
27461 emit_insn ((*gen_interleave_second_low) (op0, ops[0],
27464 /* Cast the SECOND_IMODE vector back to a vector on original
27466 emit_insn (gen_rtx_SET (VOIDmode, target,
27467 gen_lowpart (mode, op0)));
27471 gcc_unreachable ();
27475 /* A subroutine of ix86_expand_vector_init. Handle the most general case:
27476 all values variable, and none identical. */
27479 ix86_expand_vector_init_general (bool mmx_ok, enum machine_mode mode,
27480 rtx target, rtx vals)
27482 rtx ops[32], op0, op1;
27483 enum machine_mode half_mode = VOIDmode;
27490 if (!mmx_ok && !TARGET_SSE)
27502 n = GET_MODE_NUNITS (mode);
27503 for (i = 0; i < n; i++)
27504 ops[i] = XVECEXP (vals, 0, i);
27505 ix86_expand_vector_init_concat (mode, target, ops, n);
27509 half_mode = V16QImode;
27513 half_mode = V8HImode;
27517 n = GET_MODE_NUNITS (mode);
27518 for (i = 0; i < n; i++)
27519 ops[i] = XVECEXP (vals, 0, i);
27520 op0 = gen_reg_rtx (half_mode);
27521 op1 = gen_reg_rtx (half_mode);
27522 ix86_expand_vector_init_interleave (half_mode, op0, ops,
27524 ix86_expand_vector_init_interleave (half_mode, op1,
27525 &ops [n >> 1], n >> 2);
27526 emit_insn (gen_rtx_SET (VOIDmode, target,
27527 gen_rtx_VEC_CONCAT (mode, op0, op1)));
27531 if (!TARGET_SSE4_1)
27539 /* Don't use ix86_expand_vector_init_interleave if we can't
27540 move from GPR to SSE register directly. */
27541 if (!TARGET_INTER_UNIT_MOVES)
27544 n = GET_MODE_NUNITS (mode);
27545 for (i = 0; i < n; i++)
27546 ops[i] = XVECEXP (vals, 0, i);
27547 ix86_expand_vector_init_interleave (mode, target, ops, n >> 1);
27555 gcc_unreachable ();
27559 int i, j, n_elts, n_words, n_elt_per_word;
27560 enum machine_mode inner_mode;
27561 rtx words[4], shift;
27563 inner_mode = GET_MODE_INNER (mode);
27564 n_elts = GET_MODE_NUNITS (mode);
27565 n_words = GET_MODE_SIZE (mode) / UNITS_PER_WORD;
27566 n_elt_per_word = n_elts / n_words;
27567 shift = GEN_INT (GET_MODE_BITSIZE (inner_mode));
27569 for (i = 0; i < n_words; ++i)
27571 rtx word = NULL_RTX;
27573 for (j = 0; j < n_elt_per_word; ++j)
27575 rtx elt = XVECEXP (vals, 0, (i+1)*n_elt_per_word - j - 1);
27576 elt = convert_modes (word_mode, inner_mode, elt, true);
27582 word = expand_simple_binop (word_mode, ASHIFT, word, shift,
27583 word, 1, OPTAB_LIB_WIDEN);
27584 word = expand_simple_binop (word_mode, IOR, word, elt,
27585 word, 1, OPTAB_LIB_WIDEN);
27593 emit_move_insn (target, gen_lowpart (mode, words[0]));
27594 else if (n_words == 2)
27596 rtx tmp = gen_reg_rtx (mode);
27597 emit_clobber (tmp);
27598 emit_move_insn (gen_lowpart (word_mode, tmp), words[0]);
27599 emit_move_insn (gen_highpart (word_mode, tmp), words[1]);
27600 emit_move_insn (target, tmp);
27602 else if (n_words == 4)
27604 rtx tmp = gen_reg_rtx (V4SImode);
27605 gcc_assert (word_mode == SImode);
27606 vals = gen_rtx_PARALLEL (V4SImode, gen_rtvec_v (4, words));
27607 ix86_expand_vector_init_general (false, V4SImode, tmp, vals);
27608 emit_move_insn (target, gen_lowpart (mode, tmp));
27611 gcc_unreachable ();
27615 /* Initialize vector TARGET via VALS. Suppress the use of MMX
27616 instructions unless MMX_OK is true. */
27619 ix86_expand_vector_init (bool mmx_ok, rtx target, rtx vals)
27621 enum machine_mode mode = GET_MODE (target);
27622 enum machine_mode inner_mode = GET_MODE_INNER (mode);
27623 int n_elts = GET_MODE_NUNITS (mode);
27624 int n_var = 0, one_var = -1;
27625 bool all_same = true, all_const_zero = true;
27629 for (i = 0; i < n_elts; ++i)
27631 x = XVECEXP (vals, 0, i);
27632 if (!(CONST_INT_P (x)
27633 || GET_CODE (x) == CONST_DOUBLE
27634 || GET_CODE (x) == CONST_FIXED))
27635 n_var++, one_var = i;
27636 else if (x != CONST0_RTX (inner_mode))
27637 all_const_zero = false;
27638 if (i > 0 && !rtx_equal_p (x, XVECEXP (vals, 0, 0)))
27642 /* Constants are best loaded from the constant pool. */
27645 emit_move_insn (target, gen_rtx_CONST_VECTOR (mode, XVEC (vals, 0)));
27649 /* If all values are identical, broadcast the value. */
27651 && ix86_expand_vector_init_duplicate (mmx_ok, mode, target,
27652 XVECEXP (vals, 0, 0)))
27655 /* Values where only one field is non-constant are best loaded from
27656 the pool and overwritten via move later. */
27660 && ix86_expand_vector_init_one_nonzero (mmx_ok, mode, target,
27661 XVECEXP (vals, 0, one_var),
27665 if (ix86_expand_vector_init_one_var (mmx_ok, mode, target, vals, one_var))
27669 ix86_expand_vector_init_general (mmx_ok, mode, target, vals);
27673 ix86_expand_vector_set (bool mmx_ok, rtx target, rtx val, int elt)
27675 enum machine_mode mode = GET_MODE (target);
27676 enum machine_mode inner_mode = GET_MODE_INNER (mode);
27677 enum machine_mode half_mode;
27678 bool use_vec_merge = false;
27680 static rtx (*gen_extract[6][2]) (rtx, rtx)
27682 { gen_vec_extract_lo_v32qi, gen_vec_extract_hi_v32qi },
27683 { gen_vec_extract_lo_v16hi, gen_vec_extract_hi_v16hi },
27684 { gen_vec_extract_lo_v8si, gen_vec_extract_hi_v8si },
27685 { gen_vec_extract_lo_v4di, gen_vec_extract_hi_v4di },
27686 { gen_vec_extract_lo_v8sf, gen_vec_extract_hi_v8sf },
27687 { gen_vec_extract_lo_v4df, gen_vec_extract_hi_v4df }
27689 static rtx (*gen_insert[6][2]) (rtx, rtx, rtx)
27691 { gen_vec_set_lo_v32qi, gen_vec_set_hi_v32qi },
27692 { gen_vec_set_lo_v16hi, gen_vec_set_hi_v16hi },
27693 { gen_vec_set_lo_v8si, gen_vec_set_hi_v8si },
27694 { gen_vec_set_lo_v4di, gen_vec_set_hi_v4di },
27695 { gen_vec_set_lo_v8sf, gen_vec_set_hi_v8sf },
27696 { gen_vec_set_lo_v4df, gen_vec_set_hi_v4df }
27706 tmp = gen_reg_rtx (GET_MODE_INNER (mode));
27707 ix86_expand_vector_extract (true, tmp, target, 1 - elt);
27709 tmp = gen_rtx_VEC_CONCAT (mode, tmp, val);
27711 tmp = gen_rtx_VEC_CONCAT (mode, val, tmp);
27712 emit_insn (gen_rtx_SET (VOIDmode, target, tmp));
27718 use_vec_merge = TARGET_SSE4_1;
27726 /* For the two element vectors, we implement a VEC_CONCAT with
27727 the extraction of the other element. */
27729 tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (1 - elt)));
27730 tmp = gen_rtx_VEC_SELECT (inner_mode, target, tmp);
27733 op0 = val, op1 = tmp;
27735 op0 = tmp, op1 = val;
27737 tmp = gen_rtx_VEC_CONCAT (mode, op0, op1);
27738 emit_insn (gen_rtx_SET (VOIDmode, target, tmp));
27743 use_vec_merge = TARGET_SSE4_1;
27750 use_vec_merge = true;
27754 /* tmp = target = A B C D */
27755 tmp = copy_to_reg (target);
27756 /* target = A A B B */
27757 emit_insn (gen_sse_unpcklps (target, target, target));
27758 /* target = X A B B */
27759 ix86_expand_vector_set (false, target, val, 0);
27760 /* target = A X C D */
27761 emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
27762 GEN_INT (1), GEN_INT (0),
27763 GEN_INT (2+4), GEN_INT (3+4)));
27767 /* tmp = target = A B C D */
27768 tmp = copy_to_reg (target);
27769 /* tmp = X B C D */
27770 ix86_expand_vector_set (false, tmp, val, 0);
27771 /* target = A B X D */
27772 emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
27773 GEN_INT (0), GEN_INT (1),
27774 GEN_INT (0+4), GEN_INT (3+4)));
27778 /* tmp = target = A B C D */
27779 tmp = copy_to_reg (target);
27780 /* tmp = X B C D */
27781 ix86_expand_vector_set (false, tmp, val, 0);
27782 /* target = A B X D */
27783 emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
27784 GEN_INT (0), GEN_INT (1),
27785 GEN_INT (2+4), GEN_INT (0+4)));
27789 gcc_unreachable ();
27794 use_vec_merge = TARGET_SSE4_1;
27798 /* Element 0 handled by vec_merge below. */
27801 use_vec_merge = true;
27807 /* With SSE2, use integer shuffles to swap element 0 and ELT,
27808 store into element 0, then shuffle them back. */
27812 order[0] = GEN_INT (elt);
27813 order[1] = const1_rtx;
27814 order[2] = const2_rtx;
27815 order[3] = GEN_INT (3);
27816 order[elt] = const0_rtx;
27818 emit_insn (gen_sse2_pshufd_1 (target, target, order[0],
27819 order[1], order[2], order[3]));
27821 ix86_expand_vector_set (false, target, val, 0);
27823 emit_insn (gen_sse2_pshufd_1 (target, target, order[0],
27824 order[1], order[2], order[3]));
27828 /* For SSE1, we have to reuse the V4SF code. */
27829 ix86_expand_vector_set (false, gen_lowpart (V4SFmode, target),
27830 gen_lowpart (SFmode, val), elt);
27835 use_vec_merge = TARGET_SSE2;
27838 use_vec_merge = mmx_ok && (TARGET_SSE || TARGET_3DNOW_A);
27842 use_vec_merge = TARGET_SSE4_1;
27849 half_mode = V16QImode;
27855 half_mode = V8HImode;
27861 half_mode = V4SImode;
27867 half_mode = V2DImode;
27873 half_mode = V4SFmode;
27879 half_mode = V2DFmode;
27885 /* Compute offset. */
27889 gcc_assert (i <= 1);
27891 /* Extract the half. */
27892 tmp = gen_reg_rtx (half_mode);
27893 emit_insn ((*gen_extract[j][i]) (tmp, target));
27895 /* Put val in tmp at elt. */
27896 ix86_expand_vector_set (false, tmp, val, elt);
27899 emit_insn ((*gen_insert[j][i]) (target, target, tmp));
27908 tmp = gen_rtx_VEC_DUPLICATE (mode, val);
27909 tmp = gen_rtx_VEC_MERGE (mode, tmp, target, GEN_INT (1 << elt));
27910 emit_insn (gen_rtx_SET (VOIDmode, target, tmp));
27914 rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode), false);
27916 emit_move_insn (mem, target);
27918 tmp = adjust_address (mem, inner_mode, elt*GET_MODE_SIZE (inner_mode));
27919 emit_move_insn (tmp, val);
27921 emit_move_insn (target, mem);
27926 ix86_expand_vector_extract (bool mmx_ok, rtx target, rtx vec, int elt)
27928 enum machine_mode mode = GET_MODE (vec);
27929 enum machine_mode inner_mode = GET_MODE_INNER (mode);
27930 bool use_vec_extr = false;
27943 use_vec_extr = true;
27947 use_vec_extr = TARGET_SSE4_1;
27959 tmp = gen_reg_rtx (mode);
27960 emit_insn (gen_sse_shufps_v4sf (tmp, vec, vec,
27961 GEN_INT (elt), GEN_INT (elt),
27962 GEN_INT (elt+4), GEN_INT (elt+4)));
27966 tmp = gen_reg_rtx (mode);
27967 emit_insn (gen_sse_unpckhps (tmp, vec, vec));
27971 gcc_unreachable ();
27974 use_vec_extr = true;
27979 use_vec_extr = TARGET_SSE4_1;
27993 tmp = gen_reg_rtx (mode);
27994 emit_insn (gen_sse2_pshufd_1 (tmp, vec,
27995 GEN_INT (elt), GEN_INT (elt),
27996 GEN_INT (elt), GEN_INT (elt)));
28000 tmp = gen_reg_rtx (mode);
28001 emit_insn (gen_sse2_punpckhdq (tmp, vec, vec));
28005 gcc_unreachable ();
28008 use_vec_extr = true;
28013 /* For SSE1, we have to reuse the V4SF code. */
28014 ix86_expand_vector_extract (false, gen_lowpart (SFmode, target),
28015 gen_lowpart (V4SFmode, vec), elt);
28021 use_vec_extr = TARGET_SSE2;
28024 use_vec_extr = mmx_ok && (TARGET_SSE || TARGET_3DNOW_A);
28028 use_vec_extr = TARGET_SSE4_1;
28032 /* ??? Could extract the appropriate HImode element and shift. */
28039 tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (elt)));
28040 tmp = gen_rtx_VEC_SELECT (inner_mode, vec, tmp);
28042 /* Let the rtl optimizers know about the zero extension performed. */
28043 if (inner_mode == QImode || inner_mode == HImode)
28045 tmp = gen_rtx_ZERO_EXTEND (SImode, tmp);
28046 target = gen_lowpart (SImode, target);
28049 emit_insn (gen_rtx_SET (VOIDmode, target, tmp));
28053 rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode), false);
28055 emit_move_insn (mem, vec);
28057 tmp = adjust_address (mem, inner_mode, elt*GET_MODE_SIZE (inner_mode));
28058 emit_move_insn (target, tmp);
28062 /* Expand a vector reduction on V4SFmode for SSE1. FN is the binary
28063 pattern to reduce; DEST is the destination; IN is the input vector. */
28066 ix86_expand_reduc_v4sf (rtx (*fn) (rtx, rtx, rtx), rtx dest, rtx in)
28068 rtx tmp1, tmp2, tmp3;
28070 tmp1 = gen_reg_rtx (V4SFmode);
28071 tmp2 = gen_reg_rtx (V4SFmode);
28072 tmp3 = gen_reg_rtx (V4SFmode);
28074 emit_insn (gen_sse_movhlps (tmp1, in, in));
28075 emit_insn (fn (tmp2, tmp1, in));
28077 emit_insn (gen_sse_shufps_v4sf (tmp3, tmp2, tmp2,
28078 GEN_INT (1), GEN_INT (1),
28079 GEN_INT (1+4), GEN_INT (1+4)));
28080 emit_insn (fn (dest, tmp2, tmp3));
28083 /* Target hook for scalar_mode_supported_p. */
28085 ix86_scalar_mode_supported_p (enum machine_mode mode)
28087 if (DECIMAL_FLOAT_MODE_P (mode))
28089 else if (mode == TFmode)
28092 return default_scalar_mode_supported_p (mode);
28095 /* Implements target hook vector_mode_supported_p. */
28097 ix86_vector_mode_supported_p (enum machine_mode mode)
28099 if (TARGET_SSE && VALID_SSE_REG_MODE (mode))
28101 if (TARGET_SSE2 && VALID_SSE2_REG_MODE (mode))
28103 if (TARGET_AVX && VALID_AVX256_REG_MODE (mode))
28105 if (TARGET_MMX && VALID_MMX_REG_MODE (mode))
28107 if (TARGET_3DNOW && VALID_MMX_REG_MODE_3DNOW (mode))
28112 /* Target hook for c_mode_for_suffix. */
28113 static enum machine_mode
28114 ix86_c_mode_for_suffix (char suffix)
28124 /* Worker function for TARGET_MD_ASM_CLOBBERS.
28126 We do this in the new i386 backend to maintain source compatibility
28127 with the old cc0-based compiler. */
28130 ix86_md_asm_clobbers (tree outputs ATTRIBUTE_UNUSED,
28131 tree inputs ATTRIBUTE_UNUSED,
28134 clobbers = tree_cons (NULL_TREE, build_string (5, "flags"),
28136 clobbers = tree_cons (NULL_TREE, build_string (4, "fpsr"),
28141 /* Implements target vector targetm.asm.encode_section_info. This
28142 is not used by netware. */
28144 static void ATTRIBUTE_UNUSED
28145 ix86_encode_section_info (tree decl, rtx rtl, int first)
28147 default_encode_section_info (decl, rtl, first);
28149 if (TREE_CODE (decl) == VAR_DECL
28150 && (TREE_STATIC (decl) || DECL_EXTERNAL (decl))
28151 && ix86_in_large_data_p (decl))
28152 SYMBOL_REF_FLAGS (XEXP (rtl, 0)) |= SYMBOL_FLAG_FAR_ADDR;
28155 /* Worker function for REVERSE_CONDITION. */
28158 ix86_reverse_condition (enum rtx_code code, enum machine_mode mode)
28160 return (mode != CCFPmode && mode != CCFPUmode
28161 ? reverse_condition (code)
28162 : reverse_condition_maybe_unordered (code));
28165 /* Output code to perform an x87 FP register move, from OPERANDS[1]
28169 output_387_reg_move (rtx insn, rtx *operands)
28171 if (REG_P (operands[0]))
28173 if (REG_P (operands[1])
28174 && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
28176 if (REGNO (operands[0]) == FIRST_STACK_REG)
28177 return output_387_ffreep (operands, 0);
28178 return "fstp\t%y0";
28180 if (STACK_TOP_P (operands[0]))
28181 return "fld%z1\t%y1";
28184 else if (MEM_P (operands[0]))
28186 gcc_assert (REG_P (operands[1]));
28187 if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
28188 return "fstp%z0\t%y0";
28191 /* There is no non-popping store to memory for XFmode.
28192 So if we need one, follow the store with a load. */
28193 if (GET_MODE (operands[0]) == XFmode)
28194 return "fstp%z0\t%y0\n\tfld%z0\t%y0";
28196 return "fst%z0\t%y0";
28203 /* Output code to perform a conditional jump to LABEL, if C2 flag in
28204 FP status register is set. */
28207 ix86_emit_fp_unordered_jump (rtx label)
28209 rtx reg = gen_reg_rtx (HImode);
28212 emit_insn (gen_x86_fnstsw_1 (reg));
28214 if (TARGET_SAHF && (TARGET_USE_SAHF || optimize_insn_for_size_p ()))
28216 emit_insn (gen_x86_sahf_1 (reg));
28218 temp = gen_rtx_REG (CCmode, FLAGS_REG);
28219 temp = gen_rtx_UNORDERED (VOIDmode, temp, const0_rtx);
28223 emit_insn (gen_testqi_ext_ccno_0 (reg, GEN_INT (0x04)));
28225 temp = gen_rtx_REG (CCNOmode, FLAGS_REG);
28226 temp = gen_rtx_NE (VOIDmode, temp, const0_rtx);
28229 temp = gen_rtx_IF_THEN_ELSE (VOIDmode, temp,
28230 gen_rtx_LABEL_REF (VOIDmode, label),
28232 temp = gen_rtx_SET (VOIDmode, pc_rtx, temp);
28234 emit_jump_insn (temp);
28235 predict_jump (REG_BR_PROB_BASE * 10 / 100);
28238 /* Output code to perform a log1p XFmode calculation. */
28240 void ix86_emit_i387_log1p (rtx op0, rtx op1)
28242 rtx label1 = gen_label_rtx ();
28243 rtx label2 = gen_label_rtx ();
28245 rtx tmp = gen_reg_rtx (XFmode);
28246 rtx tmp2 = gen_reg_rtx (XFmode);
28248 emit_insn (gen_absxf2 (tmp, op1));
28249 emit_insn (gen_cmpxf (tmp,
28250 CONST_DOUBLE_FROM_REAL_VALUE (
28251 REAL_VALUE_ATOF ("0.29289321881345247561810596348408353", XFmode),
28253 emit_jump_insn (gen_bge (label1));
28255 emit_move_insn (tmp2, standard_80387_constant_rtx (4)); /* fldln2 */
28256 emit_insn (gen_fyl2xp1xf3_i387 (op0, op1, tmp2));
28257 emit_jump (label2);
28259 emit_label (label1);
28260 emit_move_insn (tmp, CONST1_RTX (XFmode));
28261 emit_insn (gen_addxf3 (tmp, op1, tmp));
28262 emit_move_insn (tmp2, standard_80387_constant_rtx (4)); /* fldln2 */
28263 emit_insn (gen_fyl2xxf3_i387 (op0, tmp, tmp2));
28265 emit_label (label2);
28268 /* Output code to perform a Newton-Rhapson approximation of a single precision
28269 floating point divide [http://en.wikipedia.org/wiki/N-th_root_algorithm]. */
28271 void ix86_emit_swdivsf (rtx res, rtx a, rtx b, enum machine_mode mode)
28273 rtx x0, x1, e0, e1, two;
28275 x0 = gen_reg_rtx (mode);
28276 e0 = gen_reg_rtx (mode);
28277 e1 = gen_reg_rtx (mode);
28278 x1 = gen_reg_rtx (mode);
28280 two = CONST_DOUBLE_FROM_REAL_VALUE (dconst2, SFmode);
28282 if (VECTOR_MODE_P (mode))
28283 two = ix86_build_const_vector (SFmode, true, two);
28285 two = force_reg (mode, two);
28287 /* a / b = a * rcp(b) * (2.0 - b * rcp(b)) */
28289 /* x0 = rcp(b) estimate */
28290 emit_insn (gen_rtx_SET (VOIDmode, x0,
28291 gen_rtx_UNSPEC (mode, gen_rtvec (1, b),
28294 emit_insn (gen_rtx_SET (VOIDmode, e0,
28295 gen_rtx_MULT (mode, x0, b)));
28297 emit_insn (gen_rtx_SET (VOIDmode, e1,
28298 gen_rtx_MINUS (mode, two, e0)));
28300 emit_insn (gen_rtx_SET (VOIDmode, x1,
28301 gen_rtx_MULT (mode, x0, e1)));
28303 emit_insn (gen_rtx_SET (VOIDmode, res,
28304 gen_rtx_MULT (mode, a, x1)));
28307 /* Output code to perform a Newton-Rhapson approximation of a
28308 single precision floating point [reciprocal] square root. */
28310 void ix86_emit_swsqrtsf (rtx res, rtx a, enum machine_mode mode,
28313 rtx x0, e0, e1, e2, e3, mthree, mhalf;
28316 x0 = gen_reg_rtx (mode);
28317 e0 = gen_reg_rtx (mode);
28318 e1 = gen_reg_rtx (mode);
28319 e2 = gen_reg_rtx (mode);
28320 e3 = gen_reg_rtx (mode);
28322 real_from_integer (&r, VOIDmode, -3, -1, 0);
28323 mthree = CONST_DOUBLE_FROM_REAL_VALUE (r, SFmode);
28325 real_arithmetic (&r, NEGATE_EXPR, &dconsthalf, NULL);
28326 mhalf = CONST_DOUBLE_FROM_REAL_VALUE (r, SFmode);
28328 if (VECTOR_MODE_P (mode))
28330 mthree = ix86_build_const_vector (SFmode, true, mthree);
28331 mhalf = ix86_build_const_vector (SFmode, true, mhalf);
28334 /* sqrt(a) = -0.5 * a * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0)
28335 rsqrt(a) = -0.5 * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0) */
28337 /* x0 = rsqrt(a) estimate */
28338 emit_insn (gen_rtx_SET (VOIDmode, x0,
28339 gen_rtx_UNSPEC (mode, gen_rtvec (1, a),
28342 /* If (a == 0.0) Filter out infinity to prevent NaN for sqrt(0.0). */
28347 zero = gen_reg_rtx (mode);
28348 mask = gen_reg_rtx (mode);
28350 zero = force_reg (mode, CONST0_RTX(mode));
28351 emit_insn (gen_rtx_SET (VOIDmode, mask,
28352 gen_rtx_NE (mode, zero, a)));
28354 emit_insn (gen_rtx_SET (VOIDmode, x0,
28355 gen_rtx_AND (mode, x0, mask)));
28359 emit_insn (gen_rtx_SET (VOIDmode, e0,
28360 gen_rtx_MULT (mode, x0, a)));
28362 emit_insn (gen_rtx_SET (VOIDmode, e1,
28363 gen_rtx_MULT (mode, e0, x0)));
28366 mthree = force_reg (mode, mthree);
28367 emit_insn (gen_rtx_SET (VOIDmode, e2,
28368 gen_rtx_PLUS (mode, e1, mthree)));
28370 mhalf = force_reg (mode, mhalf);
28372 /* e3 = -.5 * x0 */
28373 emit_insn (gen_rtx_SET (VOIDmode, e3,
28374 gen_rtx_MULT (mode, x0, mhalf)));
28376 /* e3 = -.5 * e0 */
28377 emit_insn (gen_rtx_SET (VOIDmode, e3,
28378 gen_rtx_MULT (mode, e0, mhalf)));
28379 /* ret = e2 * e3 */
28380 emit_insn (gen_rtx_SET (VOIDmode, res,
28381 gen_rtx_MULT (mode, e2, e3)));
28384 /* Solaris implementation of TARGET_ASM_NAMED_SECTION. */
28386 static void ATTRIBUTE_UNUSED
28387 i386_solaris_elf_named_section (const char *name, unsigned int flags,
28390 /* With Binutils 2.15, the "@unwind" marker must be specified on
28391 every occurrence of the ".eh_frame" section, not just the first
28394 && strcmp (name, ".eh_frame") == 0)
28396 fprintf (asm_out_file, "\t.section\t%s,\"%s\",@unwind\n", name,
28397 flags & SECTION_WRITE ? "aw" : "a");
28400 default_elf_asm_named_section (name, flags, decl);
28403 /* Return the mangling of TYPE if it is an extended fundamental type. */
28405 static const char *
28406 ix86_mangle_type (const_tree type)
28408 type = TYPE_MAIN_VARIANT (type);
28410 if (TREE_CODE (type) != VOID_TYPE && TREE_CODE (type) != BOOLEAN_TYPE
28411 && TREE_CODE (type) != INTEGER_TYPE && TREE_CODE (type) != REAL_TYPE)
28414 switch (TYPE_MODE (type))
28417 /* __float128 is "g". */
28420 /* "long double" or __float80 is "e". */
28427 /* For 32-bit code we can save PIC register setup by using
28428 __stack_chk_fail_local hidden function instead of calling
28429 __stack_chk_fail directly. 64-bit code doesn't need to setup any PIC
28430 register, so it is better to call __stack_chk_fail directly. */
28433 ix86_stack_protect_fail (void)
28435 return TARGET_64BIT
28436 ? default_external_stack_protect_fail ()
28437 : default_hidden_stack_protect_fail ();
28440 /* Select a format to encode pointers in exception handling data. CODE
28441 is 0 for data, 1 for code labels, 2 for function pointers. GLOBAL is
28442 true if the symbol may be affected by dynamic relocations.
28444 ??? All x86 object file formats are capable of representing this.
28445 After all, the relocation needed is the same as for the call insn.
28446 Whether or not a particular assembler allows us to enter such, I
28447 guess we'll have to see. */
28449 asm_preferred_eh_data_format (int code, int global)
28453 int type = DW_EH_PE_sdata8;
28455 || ix86_cmodel == CM_SMALL_PIC
28456 || (ix86_cmodel == CM_MEDIUM_PIC && (global || code)))
28457 type = DW_EH_PE_sdata4;
28458 return (global ? DW_EH_PE_indirect : 0) | DW_EH_PE_pcrel | type;
28460 if (ix86_cmodel == CM_SMALL
28461 || (ix86_cmodel == CM_MEDIUM && code))
28462 return DW_EH_PE_udata4;
28463 return DW_EH_PE_absptr;
28466 /* Expand copysign from SIGN to the positive value ABS_VALUE
28467 storing in RESULT. If MASK is non-null, it shall be a mask to mask out
28470 ix86_sse_copysign_to_positive (rtx result, rtx abs_value, rtx sign, rtx mask)
28472 enum machine_mode mode = GET_MODE (sign);
28473 rtx sgn = gen_reg_rtx (mode);
28474 if (mask == NULL_RTX)
28476 mask = ix86_build_signbit_mask (mode, VECTOR_MODE_P (mode), false);
28477 if (!VECTOR_MODE_P (mode))
28479 /* We need to generate a scalar mode mask in this case. */
28480 rtx tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
28481 tmp = gen_rtx_VEC_SELECT (mode, mask, tmp);
28482 mask = gen_reg_rtx (mode);
28483 emit_insn (gen_rtx_SET (VOIDmode, mask, tmp));
28487 mask = gen_rtx_NOT (mode, mask);
28488 emit_insn (gen_rtx_SET (VOIDmode, sgn,
28489 gen_rtx_AND (mode, mask, sign)));
28490 emit_insn (gen_rtx_SET (VOIDmode, result,
28491 gen_rtx_IOR (mode, abs_value, sgn)));
28494 /* Expand fabs (OP0) and return a new rtx that holds the result. The
28495 mask for masking out the sign-bit is stored in *SMASK, if that is
28498 ix86_expand_sse_fabs (rtx op0, rtx *smask)
28500 enum machine_mode mode = GET_MODE (op0);
28503 xa = gen_reg_rtx (mode);
28504 mask = ix86_build_signbit_mask (mode, VECTOR_MODE_P (mode), true);
28505 if (!VECTOR_MODE_P (mode))
28507 /* We need to generate a scalar mode mask in this case. */
28508 rtx tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
28509 tmp = gen_rtx_VEC_SELECT (mode, mask, tmp);
28510 mask = gen_reg_rtx (mode);
28511 emit_insn (gen_rtx_SET (VOIDmode, mask, tmp));
28513 emit_insn (gen_rtx_SET (VOIDmode, xa,
28514 gen_rtx_AND (mode, op0, mask)));
28522 /* Expands a comparison of OP0 with OP1 using comparison code CODE,
28523 swapping the operands if SWAP_OPERANDS is true. The expanded
28524 code is a forward jump to a newly created label in case the
28525 comparison is true. The generated label rtx is returned. */
28527 ix86_expand_sse_compare_and_jump (enum rtx_code code, rtx op0, rtx op1,
28528 bool swap_operands)
28539 label = gen_label_rtx ();
28540 tmp = gen_rtx_REG (CCFPUmode, FLAGS_REG);
28541 emit_insn (gen_rtx_SET (VOIDmode, tmp,
28542 gen_rtx_COMPARE (CCFPUmode, op0, op1)));
28543 tmp = gen_rtx_fmt_ee (code, VOIDmode, tmp, const0_rtx);
28544 tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
28545 gen_rtx_LABEL_REF (VOIDmode, label), pc_rtx);
28546 tmp = emit_jump_insn (gen_rtx_SET (VOIDmode, pc_rtx, tmp));
28547 JUMP_LABEL (tmp) = label;
28552 /* Expand a mask generating SSE comparison instruction comparing OP0 with OP1
28553 using comparison code CODE. Operands are swapped for the comparison if
28554 SWAP_OPERANDS is true. Returns a rtx for the generated mask. */
28556 ix86_expand_sse_compare_mask (enum rtx_code code, rtx op0, rtx op1,
28557 bool swap_operands)
28559 enum machine_mode mode = GET_MODE (op0);
28560 rtx mask = gen_reg_rtx (mode);
28569 if (mode == DFmode)
28570 emit_insn (gen_sse2_maskcmpdf3 (mask, op0, op1,
28571 gen_rtx_fmt_ee (code, mode, op0, op1)));
28573 emit_insn (gen_sse_maskcmpsf3 (mask, op0, op1,
28574 gen_rtx_fmt_ee (code, mode, op0, op1)));
28579 /* Generate and return a rtx of mode MODE for 2**n where n is the number
28580 of bits of the mantissa of MODE, which must be one of DFmode or SFmode. */
28582 ix86_gen_TWO52 (enum machine_mode mode)
28584 REAL_VALUE_TYPE TWO52r;
28587 real_ldexp (&TWO52r, &dconst1, mode == DFmode ? 52 : 23);
28588 TWO52 = const_double_from_real_value (TWO52r, mode);
28589 TWO52 = force_reg (mode, TWO52);
28594 /* Expand SSE sequence for computing lround from OP1 storing
28597 ix86_expand_lround (rtx op0, rtx op1)
28599 /* C code for the stuff we're doing below:
28600 tmp = op1 + copysign (nextafter (0.5, 0.0), op1)
28603 enum machine_mode mode = GET_MODE (op1);
28604 const struct real_format *fmt;
28605 REAL_VALUE_TYPE pred_half, half_minus_pred_half;
28608 /* load nextafter (0.5, 0.0) */
28609 fmt = REAL_MODE_FORMAT (mode);
28610 real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
28611 REAL_ARITHMETIC (pred_half, MINUS_EXPR, dconsthalf, half_minus_pred_half);
28613 /* adj = copysign (0.5, op1) */
28614 adj = force_reg (mode, const_double_from_real_value (pred_half, mode));
28615 ix86_sse_copysign_to_positive (adj, adj, force_reg (mode, op1), NULL_RTX);
28617 /* adj = op1 + adj */
28618 adj = expand_simple_binop (mode, PLUS, adj, op1, NULL_RTX, 0, OPTAB_DIRECT);
28620 /* op0 = (imode)adj */
28621 expand_fix (op0, adj, 0);
28624 /* Expand SSE2 sequence for computing lround from OPERAND1 storing
28627 ix86_expand_lfloorceil (rtx op0, rtx op1, bool do_floor)
28629 /* C code for the stuff we're doing below (for do_floor):
28631 xi -= (double)xi > op1 ? 1 : 0;
28634 enum machine_mode fmode = GET_MODE (op1);
28635 enum machine_mode imode = GET_MODE (op0);
28636 rtx ireg, freg, label, tmp;
28638 /* reg = (long)op1 */
28639 ireg = gen_reg_rtx (imode);
28640 expand_fix (ireg, op1, 0);
28642 /* freg = (double)reg */
28643 freg = gen_reg_rtx (fmode);
28644 expand_float (freg, ireg, 0);
28646 /* ireg = (freg > op1) ? ireg - 1 : ireg */
28647 label = ix86_expand_sse_compare_and_jump (UNLE,
28648 freg, op1, !do_floor);
28649 tmp = expand_simple_binop (imode, do_floor ? MINUS : PLUS,
28650 ireg, const1_rtx, NULL_RTX, 0, OPTAB_DIRECT);
28651 emit_move_insn (ireg, tmp);
28653 emit_label (label);
28654 LABEL_NUSES (label) = 1;
28656 emit_move_insn (op0, ireg);
28659 /* Expand rint (IEEE round to nearest) rounding OPERAND1 and storing the
28660 result in OPERAND0. */
28662 ix86_expand_rint (rtx operand0, rtx operand1)
28664 /* C code for the stuff we're doing below:
28665 xa = fabs (operand1);
28666 if (!isless (xa, 2**52))
28668 xa = xa + 2**52 - 2**52;
28669 return copysign (xa, operand1);
28671 enum machine_mode mode = GET_MODE (operand0);
28672 rtx res, xa, label, TWO52, mask;
28674 res = gen_reg_rtx (mode);
28675 emit_move_insn (res, operand1);
28677 /* xa = abs (operand1) */
28678 xa = ix86_expand_sse_fabs (res, &mask);
28680 /* if (!isless (xa, TWO52)) goto label; */
28681 TWO52 = ix86_gen_TWO52 (mode);
28682 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
28684 xa = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
28685 xa = expand_simple_binop (mode, MINUS, xa, TWO52, xa, 0, OPTAB_DIRECT);
28687 ix86_sse_copysign_to_positive (res, xa, res, mask);
28689 emit_label (label);
28690 LABEL_NUSES (label) = 1;
28692 emit_move_insn (operand0, res);
28695 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
28698 ix86_expand_floorceildf_32 (rtx operand0, rtx operand1, bool do_floor)
28700 /* C code for the stuff we expand below.
28701 double xa = fabs (x), x2;
28702 if (!isless (xa, TWO52))
28704 xa = xa + TWO52 - TWO52;
28705 x2 = copysign (xa, x);
28714 enum machine_mode mode = GET_MODE (operand0);
28715 rtx xa, TWO52, tmp, label, one, res, mask;
28717 TWO52 = ix86_gen_TWO52 (mode);
28719 /* Temporary for holding the result, initialized to the input
28720 operand to ease control flow. */
28721 res = gen_reg_rtx (mode);
28722 emit_move_insn (res, operand1);
28724 /* xa = abs (operand1) */
28725 xa = ix86_expand_sse_fabs (res, &mask);
28727 /* if (!isless (xa, TWO52)) goto label; */
28728 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
28730 /* xa = xa + TWO52 - TWO52; */
28731 xa = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
28732 xa = expand_simple_binop (mode, MINUS, xa, TWO52, xa, 0, OPTAB_DIRECT);
28734 /* xa = copysign (xa, operand1) */
28735 ix86_sse_copysign_to_positive (xa, xa, res, mask);
28737 /* generate 1.0 or -1.0 */
28738 one = force_reg (mode,
28739 const_double_from_real_value (do_floor
28740 ? dconst1 : dconstm1, mode));
28742 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
28743 tmp = ix86_expand_sse_compare_mask (UNGT, xa, res, !do_floor);
28744 emit_insn (gen_rtx_SET (VOIDmode, tmp,
28745 gen_rtx_AND (mode, one, tmp)));
28746 /* We always need to subtract here to preserve signed zero. */
28747 tmp = expand_simple_binop (mode, MINUS,
28748 xa, tmp, NULL_RTX, 0, OPTAB_DIRECT);
28749 emit_move_insn (res, tmp);
28751 emit_label (label);
28752 LABEL_NUSES (label) = 1;
28754 emit_move_insn (operand0, res);
28757 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
28760 ix86_expand_floorceil (rtx operand0, rtx operand1, bool do_floor)
28762 /* C code for the stuff we expand below.
28763 double xa = fabs (x), x2;
28764 if (!isless (xa, TWO52))
28766 x2 = (double)(long)x;
28773 if (HONOR_SIGNED_ZEROS (mode))
28774 return copysign (x2, x);
28777 enum machine_mode mode = GET_MODE (operand0);
28778 rtx xa, xi, TWO52, tmp, label, one, res, mask;
28780 TWO52 = ix86_gen_TWO52 (mode);
28782 /* Temporary for holding the result, initialized to the input
28783 operand to ease control flow. */
28784 res = gen_reg_rtx (mode);
28785 emit_move_insn (res, operand1);
28787 /* xa = abs (operand1) */
28788 xa = ix86_expand_sse_fabs (res, &mask);
28790 /* if (!isless (xa, TWO52)) goto label; */
28791 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
28793 /* xa = (double)(long)x */
28794 xi = gen_reg_rtx (mode == DFmode ? DImode : SImode);
28795 expand_fix (xi, res, 0);
28796 expand_float (xa, xi, 0);
28799 one = force_reg (mode, const_double_from_real_value (dconst1, mode));
28801 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
28802 tmp = ix86_expand_sse_compare_mask (UNGT, xa, res, !do_floor);
28803 emit_insn (gen_rtx_SET (VOIDmode, tmp,
28804 gen_rtx_AND (mode, one, tmp)));
28805 tmp = expand_simple_binop (mode, do_floor ? MINUS : PLUS,
28806 xa, tmp, NULL_RTX, 0, OPTAB_DIRECT);
28807 emit_move_insn (res, tmp);
28809 if (HONOR_SIGNED_ZEROS (mode))
28810 ix86_sse_copysign_to_positive (res, res, force_reg (mode, operand1), mask);
28812 emit_label (label);
28813 LABEL_NUSES (label) = 1;
28815 emit_move_insn (operand0, res);
28818 /* Expand SSE sequence for computing round from OPERAND1 storing
28819 into OPERAND0. Sequence that works without relying on DImode truncation
28820 via cvttsd2siq that is only available on 64bit targets. */
28822 ix86_expand_rounddf_32 (rtx operand0, rtx operand1)
28824 /* C code for the stuff we expand below.
28825 double xa = fabs (x), xa2, x2;
28826 if (!isless (xa, TWO52))
28828 Using the absolute value and copying back sign makes
28829 -0.0 -> -0.0 correct.
28830 xa2 = xa + TWO52 - TWO52;
28835 else if (dxa > 0.5)
28837 x2 = copysign (xa2, x);
28840 enum machine_mode mode = GET_MODE (operand0);
28841 rtx xa, xa2, dxa, TWO52, tmp, label, half, mhalf, one, res, mask;
28843 TWO52 = ix86_gen_TWO52 (mode);
28845 /* Temporary for holding the result, initialized to the input
28846 operand to ease control flow. */
28847 res = gen_reg_rtx (mode);
28848 emit_move_insn (res, operand1);
28850 /* xa = abs (operand1) */
28851 xa = ix86_expand_sse_fabs (res, &mask);
28853 /* if (!isless (xa, TWO52)) goto label; */
28854 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
28856 /* xa2 = xa + TWO52 - TWO52; */
28857 xa2 = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
28858 xa2 = expand_simple_binop (mode, MINUS, xa2, TWO52, xa2, 0, OPTAB_DIRECT);
28860 /* dxa = xa2 - xa; */
28861 dxa = expand_simple_binop (mode, MINUS, xa2, xa, NULL_RTX, 0, OPTAB_DIRECT);
28863 /* generate 0.5, 1.0 and -0.5 */
28864 half = force_reg (mode, const_double_from_real_value (dconsthalf, mode));
28865 one = expand_simple_binop (mode, PLUS, half, half, NULL_RTX, 0, OPTAB_DIRECT);
28866 mhalf = expand_simple_binop (mode, MINUS, half, one, NULL_RTX,
28870 tmp = gen_reg_rtx (mode);
28871 /* xa2 = xa2 - (dxa > 0.5 ? 1 : 0) */
28872 tmp = ix86_expand_sse_compare_mask (UNGT, dxa, half, false);
28873 emit_insn (gen_rtx_SET (VOIDmode, tmp,
28874 gen_rtx_AND (mode, one, tmp)));
28875 xa2 = expand_simple_binop (mode, MINUS, xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
28876 /* xa2 = xa2 + (dxa <= -0.5 ? 1 : 0) */
28877 tmp = ix86_expand_sse_compare_mask (UNGE, mhalf, dxa, false);
28878 emit_insn (gen_rtx_SET (VOIDmode, tmp,
28879 gen_rtx_AND (mode, one, tmp)));
28880 xa2 = expand_simple_binop (mode, PLUS, xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
28882 /* res = copysign (xa2, operand1) */
28883 ix86_sse_copysign_to_positive (res, xa2, force_reg (mode, operand1), mask);
28885 emit_label (label);
28886 LABEL_NUSES (label) = 1;
28888 emit_move_insn (operand0, res);
28891 /* Expand SSE sequence for computing trunc from OPERAND1 storing
28894 ix86_expand_trunc (rtx operand0, rtx operand1)
28896 /* C code for SSE variant we expand below.
28897 double xa = fabs (x), x2;
28898 if (!isless (xa, TWO52))
28900 x2 = (double)(long)x;
28901 if (HONOR_SIGNED_ZEROS (mode))
28902 return copysign (x2, x);
28905 enum machine_mode mode = GET_MODE (operand0);
28906 rtx xa, xi, TWO52, label, res, mask;
28908 TWO52 = ix86_gen_TWO52 (mode);
28910 /* Temporary for holding the result, initialized to the input
28911 operand to ease control flow. */
28912 res = gen_reg_rtx (mode);
28913 emit_move_insn (res, operand1);
28915 /* xa = abs (operand1) */
28916 xa = ix86_expand_sse_fabs (res, &mask);
28918 /* if (!isless (xa, TWO52)) goto label; */
28919 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
28921 /* x = (double)(long)x */
28922 xi = gen_reg_rtx (mode == DFmode ? DImode : SImode);
28923 expand_fix (xi, res, 0);
28924 expand_float (res, xi, 0);
28926 if (HONOR_SIGNED_ZEROS (mode))
28927 ix86_sse_copysign_to_positive (res, res, force_reg (mode, operand1), mask);
28929 emit_label (label);
28930 LABEL_NUSES (label) = 1;
28932 emit_move_insn (operand0, res);
28935 /* Expand SSE sequence for computing trunc from OPERAND1 storing
28938 ix86_expand_truncdf_32 (rtx operand0, rtx operand1)
28940 enum machine_mode mode = GET_MODE (operand0);
28941 rtx xa, mask, TWO52, label, one, res, smask, tmp;
28943 /* C code for SSE variant we expand below.
28944 double xa = fabs (x), x2;
28945 if (!isless (xa, TWO52))
28947 xa2 = xa + TWO52 - TWO52;
28951 x2 = copysign (xa2, x);
28955 TWO52 = ix86_gen_TWO52 (mode);
28957 /* Temporary for holding the result, initialized to the input
28958 operand to ease control flow. */
28959 res = gen_reg_rtx (mode);
28960 emit_move_insn (res, operand1);
28962 /* xa = abs (operand1) */
28963 xa = ix86_expand_sse_fabs (res, &smask);
28965 /* if (!isless (xa, TWO52)) goto label; */
28966 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
28968 /* res = xa + TWO52 - TWO52; */
28969 tmp = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
28970 tmp = expand_simple_binop (mode, MINUS, tmp, TWO52, tmp, 0, OPTAB_DIRECT);
28971 emit_move_insn (res, tmp);
28974 one = force_reg (mode, const_double_from_real_value (dconst1, mode));
28976 /* Compensate: res = xa2 - (res > xa ? 1 : 0) */
28977 mask = ix86_expand_sse_compare_mask (UNGT, res, xa, false);
28978 emit_insn (gen_rtx_SET (VOIDmode, mask,
28979 gen_rtx_AND (mode, mask, one)));
28980 tmp = expand_simple_binop (mode, MINUS,
28981 res, mask, NULL_RTX, 0, OPTAB_DIRECT);
28982 emit_move_insn (res, tmp);
28984 /* res = copysign (res, operand1) */
28985 ix86_sse_copysign_to_positive (res, res, force_reg (mode, operand1), smask);
28987 emit_label (label);
28988 LABEL_NUSES (label) = 1;
28990 emit_move_insn (operand0, res);
28993 /* Expand SSE sequence for computing round from OPERAND1 storing
28996 ix86_expand_round (rtx operand0, rtx operand1)
28998 /* C code for the stuff we're doing below:
28999 double xa = fabs (x);
29000 if (!isless (xa, TWO52))
29002 xa = (double)(long)(xa + nextafter (0.5, 0.0));
29003 return copysign (xa, x);
29005 enum machine_mode mode = GET_MODE (operand0);
29006 rtx res, TWO52, xa, label, xi, half, mask;
29007 const struct real_format *fmt;
29008 REAL_VALUE_TYPE pred_half, half_minus_pred_half;
29010 /* Temporary for holding the result, initialized to the input
29011 operand to ease control flow. */
29012 res = gen_reg_rtx (mode);
29013 emit_move_insn (res, operand1);
29015 TWO52 = ix86_gen_TWO52 (mode);
29016 xa = ix86_expand_sse_fabs (res, &mask);
29017 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
29019 /* load nextafter (0.5, 0.0) */
29020 fmt = REAL_MODE_FORMAT (mode);
29021 real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
29022 REAL_ARITHMETIC (pred_half, MINUS_EXPR, dconsthalf, half_minus_pred_half);
29024 /* xa = xa + 0.5 */
29025 half = force_reg (mode, const_double_from_real_value (pred_half, mode));
29026 xa = expand_simple_binop (mode, PLUS, xa, half, NULL_RTX, 0, OPTAB_DIRECT);
29028 /* xa = (double)(int64_t)xa */
29029 xi = gen_reg_rtx (mode == DFmode ? DImode : SImode);
29030 expand_fix (xi, xa, 0);
29031 expand_float (xa, xi, 0);
29033 /* res = copysign (xa, operand1) */
29034 ix86_sse_copysign_to_positive (res, xa, force_reg (mode, operand1), mask);
29036 emit_label (label);
29037 LABEL_NUSES (label) = 1;
29039 emit_move_insn (operand0, res);
29043 /* Validate whether a SSE5 instruction is valid or not.
29044 OPERANDS is the array of operands.
29045 NUM is the number of operands.
29046 USES_OC0 is true if the instruction uses OC0 and provides 4 variants.
29047 NUM_MEMORY is the maximum number of memory operands to accept.
29048 when COMMUTATIVE is set, operand 1 and 2 can be swapped. */
29051 ix86_sse5_valid_op_p (rtx operands[], rtx insn ATTRIBUTE_UNUSED, int num,
29052 bool uses_oc0, int num_memory, bool commutative)
29058 /* Count the number of memory arguments */
29061 for (i = 0; i < num; i++)
29063 enum machine_mode mode = GET_MODE (operands[i]);
29064 if (register_operand (operands[i], mode))
29067 else if (memory_operand (operands[i], mode))
29069 mem_mask |= (1 << i);
29075 rtx pattern = PATTERN (insn);
29077 /* allow 0 for pcmov */
29078 if (GET_CODE (pattern) != SET
29079 || GET_CODE (SET_SRC (pattern)) != IF_THEN_ELSE
29081 || operands[i] != CONST0_RTX (mode))
29086 /* Special case pmacsdq{l,h} where we allow the 3rd argument to be
29087 a memory operation. */
29088 if (num_memory < 0)
29090 num_memory = -num_memory;
29091 if ((mem_mask & (1 << (num-1))) != 0)
29093 mem_mask &= ~(1 << (num-1));
29098 /* If there were no memory operations, allow the insn */
29102 /* Do not allow the destination register to be a memory operand. */
29103 else if (mem_mask & (1 << 0))
29106 /* If there are too many memory operations, disallow the instruction. While
29107 the hardware only allows 1 memory reference, before register allocation
29108 for some insns, we allow two memory operations sometimes in order to allow
29109 code like the following to be optimized:
29111 float fmadd (float *a, float *b, float *c) { return (*a * *b) + *c; }
29113 or similar cases that are vectorized into using the fmaddss
29115 else if (mem_count > num_memory)
29118 /* Don't allow more than one memory operation if not optimizing. */
29119 else if (mem_count > 1 && !optimize)
29122 else if (num == 4 && mem_count == 1)
29124 /* formats (destination is the first argument), example fmaddss:
29125 xmm1, xmm1, xmm2, xmm3/mem
29126 xmm1, xmm1, xmm2/mem, xmm3
29127 xmm1, xmm2, xmm3/mem, xmm1
29128 xmm1, xmm2/mem, xmm3, xmm1 */
29130 return ((mem_mask == (1 << 1))
29131 || (mem_mask == (1 << 2))
29132 || (mem_mask == (1 << 3)));
29134 /* format, example pmacsdd:
29135 xmm1, xmm2, xmm3/mem, xmm1 */
29137 return (mem_mask == (1 << 2) || mem_mask == (1 << 1));
29139 return (mem_mask == (1 << 2));
29142 else if (num == 4 && num_memory == 2)
29144 /* If there are two memory operations, we can load one of the memory ops
29145 into the destination register. This is for optimizing the
29146 multiply/add ops, which the combiner has optimized both the multiply
29147 and the add insns to have a memory operation. We have to be careful
29148 that the destination doesn't overlap with the inputs. */
29149 rtx op0 = operands[0];
29151 if (reg_mentioned_p (op0, operands[1])
29152 || reg_mentioned_p (op0, operands[2])
29153 || reg_mentioned_p (op0, operands[3]))
29156 /* formats (destination is the first argument), example fmaddss:
29157 xmm1, xmm1, xmm2, xmm3/mem
29158 xmm1, xmm1, xmm2/mem, xmm3
29159 xmm1, xmm2, xmm3/mem, xmm1
29160 xmm1, xmm2/mem, xmm3, xmm1
29162 For the oc0 case, we will load either operands[1] or operands[3] into
29163 operands[0], so any combination of 2 memory operands is ok. */
29167 /* format, example pmacsdd:
29168 xmm1, xmm2, xmm3/mem, xmm1
29170 For the integer multiply/add instructions be more restrictive and
29171 require operands[2] and operands[3] to be the memory operands. */
29173 return (mem_mask == ((1 << 1) | (1 << 3)) || ((1 << 2) | (1 << 3)));
29175 return (mem_mask == ((1 << 2) | (1 << 3)));
29178 else if (num == 3 && num_memory == 1)
29180 /* formats, example protb:
29181 xmm1, xmm2, xmm3/mem
29182 xmm1, xmm2/mem, xmm3 */
29184 return ((mem_mask == (1 << 1)) || (mem_mask == (1 << 2)));
29186 /* format, example comeq:
29187 xmm1, xmm2, xmm3/mem */
29189 return (mem_mask == (1 << 2));
29193 gcc_unreachable ();
29199 /* Fixup an SSE5 instruction that has 2 memory input references into a form the
29200 hardware will allow by using the destination register to load one of the
29201 memory operations. Presently this is used by the multiply/add routines to
29202 allow 2 memory references. */
29205 ix86_expand_sse5_multiple_memory (rtx operands[],
29207 enum machine_mode mode)
29209 rtx op0 = operands[0];
29211 || memory_operand (op0, mode)
29212 || reg_mentioned_p (op0, operands[1])
29213 || reg_mentioned_p (op0, operands[2])
29214 || reg_mentioned_p (op0, operands[3]))
29215 gcc_unreachable ();
29217 /* For 2 memory operands, pick either operands[1] or operands[3] to move into
29218 the destination register. */
29219 if (memory_operand (operands[1], mode))
29221 emit_move_insn (op0, operands[1]);
29224 else if (memory_operand (operands[3], mode))
29226 emit_move_insn (op0, operands[3]);
29230 gcc_unreachable ();
29236 /* Table of valid machine attributes. */
29237 static const struct attribute_spec ix86_attribute_table[] =
29239 /* { name, min_len, max_len, decl_req, type_req, fn_type_req, handler } */
29240 /* Stdcall attribute says callee is responsible for popping arguments
29241 if they are not variable. */
29242 { "stdcall", 0, 0, false, true, true, ix86_handle_cconv_attribute },
29243 /* Fastcall attribute says callee is responsible for popping arguments
29244 if they are not variable. */
29245 { "fastcall", 0, 0, false, true, true, ix86_handle_cconv_attribute },
29246 /* Cdecl attribute says the callee is a normal C declaration */
29247 { "cdecl", 0, 0, false, true, true, ix86_handle_cconv_attribute },
29248 /* Regparm attribute specifies how many integer arguments are to be
29249 passed in registers. */
29250 { "regparm", 1, 1, false, true, true, ix86_handle_cconv_attribute },
29251 /* Sseregparm attribute says we are using x86_64 calling conventions
29252 for FP arguments. */
29253 { "sseregparm", 0, 0, false, true, true, ix86_handle_cconv_attribute },
29254 /* force_align_arg_pointer says this function realigns the stack at entry. */
29255 { (const char *)&ix86_force_align_arg_pointer_string, 0, 0,
29256 false, true, true, ix86_handle_cconv_attribute },
29257 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
29258 { "dllimport", 0, 0, false, false, false, handle_dll_attribute },
29259 { "dllexport", 0, 0, false, false, false, handle_dll_attribute },
29260 { "shared", 0, 0, true, false, false, ix86_handle_shared_attribute },
29262 { "ms_struct", 0, 0, false, false, false, ix86_handle_struct_attribute },
29263 { "gcc_struct", 0, 0, false, false, false, ix86_handle_struct_attribute },
29264 #ifdef SUBTARGET_ATTRIBUTE_TABLE
29265 SUBTARGET_ATTRIBUTE_TABLE,
29267 /* ms_abi and sysv_abi calling convention function attributes. */
29268 { "ms_abi", 0, 0, false, true, true, ix86_handle_abi_attribute },
29269 { "sysv_abi", 0, 0, false, true, true, ix86_handle_abi_attribute },
29271 { NULL, 0, 0, false, false, false, NULL }
29274 /* Implement targetm.vectorize.builtin_vectorization_cost. */
29276 x86_builtin_vectorization_cost (bool runtime_test)
29278 /* If the branch of the runtime test is taken - i.e. - the vectorized
29279 version is skipped - this incurs a misprediction cost (because the
29280 vectorized version is expected to be the fall-through). So we subtract
29281 the latency of a mispredicted branch from the costs that are incured
29282 when the vectorized version is executed.
29284 TODO: The values in individual target tables have to be tuned or new
29285 fields may be needed. For eg. on K8, the default branch path is the
29286 not-taken path. If the taken path is predicted correctly, the minimum
29287 penalty of going down the taken-path is 1 cycle. If the taken-path is
29288 not predicted correctly, then the minimum penalty is 10 cycles. */
29292 return (-(ix86_cost->cond_taken_branch_cost));
29298 /* This function returns the calling abi specific va_list type node.
29299 It returns the FNDECL specific va_list type. */
29302 ix86_fn_abi_va_list (tree fndecl)
29307 return va_list_type_node;
29308 gcc_assert (fndecl != NULL_TREE);
29309 abi = ix86_function_abi ((const_tree) fndecl);
29312 return ms_va_list_type_node;
29314 return sysv_va_list_type_node;
29317 /* Returns the canonical va_list type specified by TYPE. If there
29318 is no valid TYPE provided, it return NULL_TREE. */
29321 ix86_canonical_va_list_type (tree type)
29325 /* Resolve references and pointers to va_list type. */
29326 if (INDIRECT_REF_P (type))
29327 type = TREE_TYPE (type);
29328 else if (POINTER_TYPE_P (type) && POINTER_TYPE_P (TREE_TYPE(type)))
29329 type = TREE_TYPE (type);
29333 wtype = va_list_type_node;
29334 gcc_assert (wtype != NULL_TREE);
29336 if (TREE_CODE (wtype) == ARRAY_TYPE)
29338 /* If va_list is an array type, the argument may have decayed
29339 to a pointer type, e.g. by being passed to another function.
29340 In that case, unwrap both types so that we can compare the
29341 underlying records. */
29342 if (TREE_CODE (htype) == ARRAY_TYPE
29343 || POINTER_TYPE_P (htype))
29345 wtype = TREE_TYPE (wtype);
29346 htype = TREE_TYPE (htype);
29349 if (TYPE_MAIN_VARIANT (wtype) == TYPE_MAIN_VARIANT (htype))
29350 return va_list_type_node;
29351 wtype = sysv_va_list_type_node;
29352 gcc_assert (wtype != NULL_TREE);
29354 if (TREE_CODE (wtype) == ARRAY_TYPE)
29356 /* If va_list is an array type, the argument may have decayed
29357 to a pointer type, e.g. by being passed to another function.
29358 In that case, unwrap both types so that we can compare the
29359 underlying records. */
29360 if (TREE_CODE (htype) == ARRAY_TYPE
29361 || POINTER_TYPE_P (htype))
29363 wtype = TREE_TYPE (wtype);
29364 htype = TREE_TYPE (htype);
29367 if (TYPE_MAIN_VARIANT (wtype) == TYPE_MAIN_VARIANT (htype))
29368 return sysv_va_list_type_node;
29369 wtype = ms_va_list_type_node;
29370 gcc_assert (wtype != NULL_TREE);
29372 if (TREE_CODE (wtype) == ARRAY_TYPE)
29374 /* If va_list is an array type, the argument may have decayed
29375 to a pointer type, e.g. by being passed to another function.
29376 In that case, unwrap both types so that we can compare the
29377 underlying records. */
29378 if (TREE_CODE (htype) == ARRAY_TYPE
29379 || POINTER_TYPE_P (htype))
29381 wtype = TREE_TYPE (wtype);
29382 htype = TREE_TYPE (htype);
29385 if (TYPE_MAIN_VARIANT (wtype) == TYPE_MAIN_VARIANT (htype))
29386 return ms_va_list_type_node;
29389 return std_canonical_va_list_type (type);
29392 /* Iterate through the target-specific builtin types for va_list.
29393 IDX denotes the iterator, *PTREE is set to the result type of
29394 the va_list builtin, and *PNAME to its internal type.
29395 Returns zero if there is no element for this index, otherwise
29396 IDX should be increased upon the next call.
29397 Note, do not iterate a base builtin's name like __builtin_va_list.
29398 Used from c_common_nodes_and_builtins. */
29401 ix86_enum_va_list (int idx, const char **pname, tree *ptree)
29407 *ptree = ms_va_list_type_node;
29408 *pname = "__builtin_ms_va_list";
29411 *ptree = sysv_va_list_type_node;
29412 *pname = "__builtin_sysv_va_list";
29420 /* Initialize the GCC target structure. */
29421 #undef TARGET_RETURN_IN_MEMORY
29422 #define TARGET_RETURN_IN_MEMORY ix86_return_in_memory
29424 #undef TARGET_ATTRIBUTE_TABLE
29425 #define TARGET_ATTRIBUTE_TABLE ix86_attribute_table
29426 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
29427 # undef TARGET_MERGE_DECL_ATTRIBUTES
29428 # define TARGET_MERGE_DECL_ATTRIBUTES merge_dllimport_decl_attributes
29431 #undef TARGET_COMP_TYPE_ATTRIBUTES
29432 #define TARGET_COMP_TYPE_ATTRIBUTES ix86_comp_type_attributes
29434 #undef TARGET_INIT_BUILTINS
29435 #define TARGET_INIT_BUILTINS ix86_init_builtins
29436 #undef TARGET_EXPAND_BUILTIN
29437 #define TARGET_EXPAND_BUILTIN ix86_expand_builtin
29439 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
29440 #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
29441 ix86_builtin_vectorized_function
29443 #undef TARGET_VECTORIZE_BUILTIN_CONVERSION
29444 #define TARGET_VECTORIZE_BUILTIN_CONVERSION ix86_vectorize_builtin_conversion
29446 #undef TARGET_BUILTIN_RECIPROCAL
29447 #define TARGET_BUILTIN_RECIPROCAL ix86_builtin_reciprocal
29449 #undef TARGET_ASM_FUNCTION_EPILOGUE
29450 #define TARGET_ASM_FUNCTION_EPILOGUE ix86_output_function_epilogue
29452 #undef TARGET_ENCODE_SECTION_INFO
29453 #ifndef SUBTARGET_ENCODE_SECTION_INFO
29454 #define TARGET_ENCODE_SECTION_INFO ix86_encode_section_info
29456 #define TARGET_ENCODE_SECTION_INFO SUBTARGET_ENCODE_SECTION_INFO
29459 #undef TARGET_ASM_OPEN_PAREN
29460 #define TARGET_ASM_OPEN_PAREN ""
29461 #undef TARGET_ASM_CLOSE_PAREN
29462 #define TARGET_ASM_CLOSE_PAREN ""
29464 #undef TARGET_ASM_ALIGNED_HI_OP
29465 #define TARGET_ASM_ALIGNED_HI_OP ASM_SHORT
29466 #undef TARGET_ASM_ALIGNED_SI_OP
29467 #define TARGET_ASM_ALIGNED_SI_OP ASM_LONG
29469 #undef TARGET_ASM_ALIGNED_DI_OP
29470 #define TARGET_ASM_ALIGNED_DI_OP ASM_QUAD
29473 #undef TARGET_ASM_UNALIGNED_HI_OP
29474 #define TARGET_ASM_UNALIGNED_HI_OP TARGET_ASM_ALIGNED_HI_OP
29475 #undef TARGET_ASM_UNALIGNED_SI_OP
29476 #define TARGET_ASM_UNALIGNED_SI_OP TARGET_ASM_ALIGNED_SI_OP
29477 #undef TARGET_ASM_UNALIGNED_DI_OP
29478 #define TARGET_ASM_UNALIGNED_DI_OP TARGET_ASM_ALIGNED_DI_OP
29480 #undef TARGET_SCHED_ADJUST_COST
29481 #define TARGET_SCHED_ADJUST_COST ix86_adjust_cost
29482 #undef TARGET_SCHED_ISSUE_RATE
29483 #define TARGET_SCHED_ISSUE_RATE ix86_issue_rate
29484 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
29485 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
29486 ia32_multipass_dfa_lookahead
29488 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
29489 #define TARGET_FUNCTION_OK_FOR_SIBCALL ix86_function_ok_for_sibcall
29492 #undef TARGET_HAVE_TLS
29493 #define TARGET_HAVE_TLS true
29495 #undef TARGET_CANNOT_FORCE_CONST_MEM
29496 #define TARGET_CANNOT_FORCE_CONST_MEM ix86_cannot_force_const_mem
29497 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
29498 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P hook_bool_mode_const_rtx_true
29500 #undef TARGET_DELEGITIMIZE_ADDRESS
29501 #define TARGET_DELEGITIMIZE_ADDRESS ix86_delegitimize_address
29503 #undef TARGET_MS_BITFIELD_LAYOUT_P
29504 #define TARGET_MS_BITFIELD_LAYOUT_P ix86_ms_bitfield_layout_p
29507 #undef TARGET_BINDS_LOCAL_P
29508 #define TARGET_BINDS_LOCAL_P darwin_binds_local_p
29510 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
29511 #undef TARGET_BINDS_LOCAL_P
29512 #define TARGET_BINDS_LOCAL_P i386_pe_binds_local_p
29515 #undef TARGET_ASM_OUTPUT_MI_THUNK
29516 #define TARGET_ASM_OUTPUT_MI_THUNK x86_output_mi_thunk
29517 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
29518 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK x86_can_output_mi_thunk
29520 #undef TARGET_ASM_FILE_START
29521 #define TARGET_ASM_FILE_START x86_file_start
29523 #undef TARGET_DEFAULT_TARGET_FLAGS
29524 #define TARGET_DEFAULT_TARGET_FLAGS \
29526 | TARGET_SUBTARGET_DEFAULT \
29527 | TARGET_TLS_DIRECT_SEG_REFS_DEFAULT)
29529 #undef TARGET_HANDLE_OPTION
29530 #define TARGET_HANDLE_OPTION ix86_handle_option
29532 #undef TARGET_RTX_COSTS
29533 #define TARGET_RTX_COSTS ix86_rtx_costs
29534 #undef TARGET_ADDRESS_COST
29535 #define TARGET_ADDRESS_COST ix86_address_cost
29537 #undef TARGET_FIXED_CONDITION_CODE_REGS
29538 #define TARGET_FIXED_CONDITION_CODE_REGS ix86_fixed_condition_code_regs
29539 #undef TARGET_CC_MODES_COMPATIBLE
29540 #define TARGET_CC_MODES_COMPATIBLE ix86_cc_modes_compatible
29542 #undef TARGET_MACHINE_DEPENDENT_REORG
29543 #define TARGET_MACHINE_DEPENDENT_REORG ix86_reorg
29545 #undef TARGET_BUILD_BUILTIN_VA_LIST
29546 #define TARGET_BUILD_BUILTIN_VA_LIST ix86_build_builtin_va_list
29548 #undef TARGET_FN_ABI_VA_LIST
29549 #define TARGET_FN_ABI_VA_LIST ix86_fn_abi_va_list
29551 #undef TARGET_CANONICAL_VA_LIST_TYPE
29552 #define TARGET_CANONICAL_VA_LIST_TYPE ix86_canonical_va_list_type
29554 #undef TARGET_EXPAND_BUILTIN_VA_START
29555 #define TARGET_EXPAND_BUILTIN_VA_START ix86_va_start
29557 #undef TARGET_MD_ASM_CLOBBERS
29558 #define TARGET_MD_ASM_CLOBBERS ix86_md_asm_clobbers
29560 #undef TARGET_PROMOTE_PROTOTYPES
29561 #define TARGET_PROMOTE_PROTOTYPES hook_bool_const_tree_true
29562 #undef TARGET_STRUCT_VALUE_RTX
29563 #define TARGET_STRUCT_VALUE_RTX ix86_struct_value_rtx
29564 #undef TARGET_SETUP_INCOMING_VARARGS
29565 #define TARGET_SETUP_INCOMING_VARARGS ix86_setup_incoming_varargs
29566 #undef TARGET_MUST_PASS_IN_STACK
29567 #define TARGET_MUST_PASS_IN_STACK ix86_must_pass_in_stack
29568 #undef TARGET_PASS_BY_REFERENCE
29569 #define TARGET_PASS_BY_REFERENCE ix86_pass_by_reference
29570 #undef TARGET_INTERNAL_ARG_POINTER
29571 #define TARGET_INTERNAL_ARG_POINTER ix86_internal_arg_pointer
29572 #undef TARGET_UPDATE_STACK_BOUNDARY
29573 #define TARGET_UPDATE_STACK_BOUNDARY ix86_update_stack_boundary
29574 #undef TARGET_GET_DRAP_RTX
29575 #define TARGET_GET_DRAP_RTX ix86_get_drap_rtx
29576 #undef TARGET_DWARF_HANDLE_FRAME_UNSPEC
29577 #define TARGET_DWARF_HANDLE_FRAME_UNSPEC ix86_dwarf_handle_frame_unspec
29578 #undef TARGET_STRICT_ARGUMENT_NAMING
29579 #define TARGET_STRICT_ARGUMENT_NAMING hook_bool_CUMULATIVE_ARGS_true
29581 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
29582 #define TARGET_GIMPLIFY_VA_ARG_EXPR ix86_gimplify_va_arg
29584 #undef TARGET_SCALAR_MODE_SUPPORTED_P
29585 #define TARGET_SCALAR_MODE_SUPPORTED_P ix86_scalar_mode_supported_p
29587 #undef TARGET_VECTOR_MODE_SUPPORTED_P
29588 #define TARGET_VECTOR_MODE_SUPPORTED_P ix86_vector_mode_supported_p
29590 #undef TARGET_C_MODE_FOR_SUFFIX
29591 #define TARGET_C_MODE_FOR_SUFFIX ix86_c_mode_for_suffix
29594 #undef TARGET_ASM_OUTPUT_DWARF_DTPREL
29595 #define TARGET_ASM_OUTPUT_DWARF_DTPREL i386_output_dwarf_dtprel
29598 #ifdef SUBTARGET_INSERT_ATTRIBUTES
29599 #undef TARGET_INSERT_ATTRIBUTES
29600 #define TARGET_INSERT_ATTRIBUTES SUBTARGET_INSERT_ATTRIBUTES
29603 #undef TARGET_MANGLE_TYPE
29604 #define TARGET_MANGLE_TYPE ix86_mangle_type
29606 #undef TARGET_STACK_PROTECT_FAIL
29607 #define TARGET_STACK_PROTECT_FAIL ix86_stack_protect_fail
29609 #undef TARGET_FUNCTION_VALUE
29610 #define TARGET_FUNCTION_VALUE ix86_function_value
29612 #undef TARGET_SECONDARY_RELOAD
29613 #define TARGET_SECONDARY_RELOAD ix86_secondary_reload
29615 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
29616 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST x86_builtin_vectorization_cost
29618 #undef TARGET_SET_CURRENT_FUNCTION
29619 #define TARGET_SET_CURRENT_FUNCTION ix86_set_current_function
29621 #undef TARGET_OPTION_VALID_ATTRIBUTE_P
29622 #define TARGET_OPTION_VALID_ATTRIBUTE_P ix86_valid_target_attribute_p
29624 #undef TARGET_OPTION_SAVE
29625 #define TARGET_OPTION_SAVE ix86_function_specific_save
29627 #undef TARGET_OPTION_RESTORE
29628 #define TARGET_OPTION_RESTORE ix86_function_specific_restore
29630 #undef TARGET_OPTION_PRINT
29631 #define TARGET_OPTION_PRINT ix86_function_specific_print
29633 #undef TARGET_OPTION_CAN_INLINE_P
29634 #define TARGET_OPTION_CAN_INLINE_P ix86_can_inline_p
29636 #undef TARGET_EXPAND_TO_RTL_HOOK
29637 #define TARGET_EXPAND_TO_RTL_HOOK ix86_maybe_switch_abi
29639 struct gcc_target targetm = TARGET_INITIALIZER;
29641 #include "gt-i386.h"
OSDN Git Service
