1 /* Subroutines used for code generation on IA-32.
2 Copyright (C) 1988, 1992, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001,
3 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009
4 Free Software Foundation, Inc.
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation; either version 3, or (at your option)
13 GCC is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
30 #include "hard-reg-set.h"
32 #include "insn-config.h"
33 #include "conditions.h"
35 #include "insn-codes.h"
36 #include "insn-attr.h"
44 #include "basic-block.h"
47 #include "target-def.h"
48 #include "langhooks.h"
53 #include "tm-constrs.h"
57 static int x86_builtin_vectorization_cost (bool);
58 static rtx legitimize_dllimport_symbol (rtx, bool);
60 #ifndef CHECK_STACK_LIMIT
61 #define CHECK_STACK_LIMIT (-1)
64 /* Return index of given mode in mult and division cost tables. */
65 #define MODE_INDEX(mode) \
66 ((mode) == QImode ? 0 \
67 : (mode) == HImode ? 1 \
68 : (mode) == SImode ? 2 \
69 : (mode) == DImode ? 3 \
72 /* Processor costs (relative to an add) */
73 /* We assume COSTS_N_INSNS is defined as (N)*4 and an addition is 2 bytes. */
74 #define COSTS_N_BYTES(N) ((N) * 2)
76 #define DUMMY_STRINGOP_ALGS {libcall, {{-1, libcall}}}
79 struct processor_costs ix86_size_cost = {/* costs for tuning for size */
80 COSTS_N_BYTES (2), /* cost of an add instruction */
81 COSTS_N_BYTES (3), /* cost of a lea instruction */
82 COSTS_N_BYTES (2), /* variable shift costs */
83 COSTS_N_BYTES (3), /* constant shift costs */
84 {COSTS_N_BYTES (3), /* cost of starting multiply for QI */
85 COSTS_N_BYTES (3), /* HI */
86 COSTS_N_BYTES (3), /* SI */
87 COSTS_N_BYTES (3), /* DI */
88 COSTS_N_BYTES (5)}, /* other */
89 0, /* cost of multiply per each bit set */
90 {COSTS_N_BYTES (3), /* cost of a divide/mod for QI */
91 COSTS_N_BYTES (3), /* HI */
92 COSTS_N_BYTES (3), /* SI */
93 COSTS_N_BYTES (3), /* DI */
94 COSTS_N_BYTES (5)}, /* other */
95 COSTS_N_BYTES (3), /* cost of movsx */
96 COSTS_N_BYTES (3), /* cost of movzx */
99 2, /* cost for loading QImode using movzbl */
100 {2, 2, 2}, /* cost of loading integer registers
101 in QImode, HImode and SImode.
102 Relative to reg-reg move (2). */
103 {2, 2, 2}, /* cost of storing integer registers */
104 2, /* cost of reg,reg fld/fst */
105 {2, 2, 2}, /* cost of loading fp registers
106 in SFmode, DFmode and XFmode */
107 {2, 2, 2}, /* cost of storing fp registers
108 in SFmode, DFmode and XFmode */
109 3, /* cost of moving MMX register */
110 {3, 3}, /* cost of loading MMX registers
111 in SImode and DImode */
112 {3, 3}, /* cost of storing MMX registers
113 in SImode and DImode */
114 3, /* cost of moving SSE register */
115 {3, 3, 3}, /* cost of loading SSE registers
116 in SImode, DImode and TImode */
117 {3, 3, 3}, /* cost of storing SSE registers
118 in SImode, DImode and TImode */
119 3, /* MMX or SSE register to integer */
120 0, /* size of l1 cache */
121 0, /* size of l2 cache */
122 0, /* size of prefetch block */
123 0, /* number of parallel prefetches */
125 COSTS_N_BYTES (2), /* cost of FADD and FSUB insns. */
126 COSTS_N_BYTES (2), /* cost of FMUL instruction. */
127 COSTS_N_BYTES (2), /* cost of FDIV instruction. */
128 COSTS_N_BYTES (2), /* cost of FABS instruction. */
129 COSTS_N_BYTES (2), /* cost of FCHS instruction. */
130 COSTS_N_BYTES (2), /* cost of FSQRT instruction. */
131 {{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
132 {rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}}},
133 {{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
134 {rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}}},
135 1, /* scalar_stmt_cost. */
136 1, /* scalar load_cost. */
137 1, /* scalar_store_cost. */
138 1, /* vec_stmt_cost. */
139 1, /* vec_to_scalar_cost. */
140 1, /* scalar_to_vec_cost. */
141 1, /* vec_align_load_cost. */
142 1, /* vec_unalign_load_cost. */
143 1, /* vec_store_cost. */
144 1, /* cond_taken_branch_cost. */
145 1, /* cond_not_taken_branch_cost. */
148 /* Processor costs (relative to an add) */
150 struct processor_costs i386_cost = { /* 386 specific costs */
151 COSTS_N_INSNS (1), /* cost of an add instruction */
152 COSTS_N_INSNS (1), /* cost of a lea instruction */
153 COSTS_N_INSNS (3), /* variable shift costs */
154 COSTS_N_INSNS (2), /* constant shift costs */
155 {COSTS_N_INSNS (6), /* cost of starting multiply for QI */
156 COSTS_N_INSNS (6), /* HI */
157 COSTS_N_INSNS (6), /* SI */
158 COSTS_N_INSNS (6), /* DI */
159 COSTS_N_INSNS (6)}, /* other */
160 COSTS_N_INSNS (1), /* cost of multiply per each bit set */
161 {COSTS_N_INSNS (23), /* cost of a divide/mod for QI */
162 COSTS_N_INSNS (23), /* HI */
163 COSTS_N_INSNS (23), /* SI */
164 COSTS_N_INSNS (23), /* DI */
165 COSTS_N_INSNS (23)}, /* other */
166 COSTS_N_INSNS (3), /* cost of movsx */
167 COSTS_N_INSNS (2), /* cost of movzx */
168 15, /* "large" insn */
170 4, /* cost for loading QImode using movzbl */
171 {2, 4, 2}, /* cost of loading integer registers
172 in QImode, HImode and SImode.
173 Relative to reg-reg move (2). */
174 {2, 4, 2}, /* cost of storing integer registers */
175 2, /* cost of reg,reg fld/fst */
176 {8, 8, 8}, /* cost of loading fp registers
177 in SFmode, DFmode and XFmode */
178 {8, 8, 8}, /* cost of storing fp registers
179 in SFmode, DFmode and XFmode */
180 2, /* cost of moving MMX register */
181 {4, 8}, /* cost of loading MMX registers
182 in SImode and DImode */
183 {4, 8}, /* cost of storing MMX registers
184 in SImode and DImode */
185 2, /* cost of moving SSE register */
186 {4, 8, 16}, /* cost of loading SSE registers
187 in SImode, DImode and TImode */
188 {4, 8, 16}, /* cost of storing SSE registers
189 in SImode, DImode and TImode */
190 3, /* MMX or SSE register to integer */
191 0, /* size of l1 cache */
192 0, /* size of l2 cache */
193 0, /* size of prefetch block */
194 0, /* number of parallel prefetches */
196 COSTS_N_INSNS (23), /* cost of FADD and FSUB insns. */
197 COSTS_N_INSNS (27), /* cost of FMUL instruction. */
198 COSTS_N_INSNS (88), /* cost of FDIV instruction. */
199 COSTS_N_INSNS (22), /* cost of FABS instruction. */
200 COSTS_N_INSNS (24), /* cost of FCHS instruction. */
201 COSTS_N_INSNS (122), /* cost of FSQRT instruction. */
202 {{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
203 DUMMY_STRINGOP_ALGS},
204 {{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}},
205 DUMMY_STRINGOP_ALGS},
206 1, /* scalar_stmt_cost. */
207 1, /* scalar load_cost. */
208 1, /* scalar_store_cost. */
209 1, /* vec_stmt_cost. */
210 1, /* vec_to_scalar_cost. */
211 1, /* scalar_to_vec_cost. */
212 1, /* vec_align_load_cost. */
213 2, /* vec_unalign_load_cost. */
214 1, /* vec_store_cost. */
215 3, /* cond_taken_branch_cost. */
216 1, /* cond_not_taken_branch_cost. */
220 struct processor_costs i486_cost = { /* 486 specific costs */
221 COSTS_N_INSNS (1), /* cost of an add instruction */
222 COSTS_N_INSNS (1), /* cost of a lea instruction */
223 COSTS_N_INSNS (3), /* variable shift costs */
224 COSTS_N_INSNS (2), /* constant shift costs */
225 {COSTS_N_INSNS (12), /* cost of starting multiply for QI */
226 COSTS_N_INSNS (12), /* HI */
227 COSTS_N_INSNS (12), /* SI */
228 COSTS_N_INSNS (12), /* DI */
229 COSTS_N_INSNS (12)}, /* other */
230 1, /* cost of multiply per each bit set */
231 {COSTS_N_INSNS (40), /* cost of a divide/mod for QI */
232 COSTS_N_INSNS (40), /* HI */
233 COSTS_N_INSNS (40), /* SI */
234 COSTS_N_INSNS (40), /* DI */
235 COSTS_N_INSNS (40)}, /* other */
236 COSTS_N_INSNS (3), /* cost of movsx */
237 COSTS_N_INSNS (2), /* cost of movzx */
238 15, /* "large" insn */
240 4, /* cost for loading QImode using movzbl */
241 {2, 4, 2}, /* cost of loading integer registers
242 in QImode, HImode and SImode.
243 Relative to reg-reg move (2). */
244 {2, 4, 2}, /* cost of storing integer registers */
245 2, /* cost of reg,reg fld/fst */
246 {8, 8, 8}, /* cost of loading fp registers
247 in SFmode, DFmode and XFmode */
248 {8, 8, 8}, /* cost of storing fp registers
249 in SFmode, DFmode and XFmode */
250 2, /* cost of moving MMX register */
251 {4, 8}, /* cost of loading MMX registers
252 in SImode and DImode */
253 {4, 8}, /* cost of storing MMX registers
254 in SImode and DImode */
255 2, /* cost of moving SSE register */
256 {4, 8, 16}, /* cost of loading SSE registers
257 in SImode, DImode and TImode */
258 {4, 8, 16}, /* cost of storing SSE registers
259 in SImode, DImode and TImode */
260 3, /* MMX or SSE register to integer */
261 4, /* size of l1 cache. 486 has 8kB cache
262 shared for code and data, so 4kB is
263 not really precise. */
264 4, /* size of l2 cache */
265 0, /* size of prefetch block */
266 0, /* number of parallel prefetches */
268 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
269 COSTS_N_INSNS (16), /* cost of FMUL instruction. */
270 COSTS_N_INSNS (73), /* cost of FDIV instruction. */
271 COSTS_N_INSNS (3), /* cost of FABS instruction. */
272 COSTS_N_INSNS (3), /* cost of FCHS instruction. */
273 COSTS_N_INSNS (83), /* cost of FSQRT instruction. */
274 {{rep_prefix_4_byte, {{-1, rep_prefix_4_byte}}},
275 DUMMY_STRINGOP_ALGS},
276 {{rep_prefix_4_byte, {{-1, rep_prefix_4_byte}}},
277 DUMMY_STRINGOP_ALGS},
278 1, /* scalar_stmt_cost. */
279 1, /* scalar load_cost. */
280 1, /* scalar_store_cost. */
281 1, /* vec_stmt_cost. */
282 1, /* vec_to_scalar_cost. */
283 1, /* scalar_to_vec_cost. */
284 1, /* vec_align_load_cost. */
285 2, /* vec_unalign_load_cost. */
286 1, /* vec_store_cost. */
287 3, /* cond_taken_branch_cost. */
288 1, /* cond_not_taken_branch_cost. */
292 struct processor_costs pentium_cost = {
293 COSTS_N_INSNS (1), /* cost of an add instruction */
294 COSTS_N_INSNS (1), /* cost of a lea instruction */
295 COSTS_N_INSNS (4), /* variable shift costs */
296 COSTS_N_INSNS (1), /* constant shift costs */
297 {COSTS_N_INSNS (11), /* cost of starting multiply for QI */
298 COSTS_N_INSNS (11), /* HI */
299 COSTS_N_INSNS (11), /* SI */
300 COSTS_N_INSNS (11), /* DI */
301 COSTS_N_INSNS (11)}, /* other */
302 0, /* cost of multiply per each bit set */
303 {COSTS_N_INSNS (25), /* cost of a divide/mod for QI */
304 COSTS_N_INSNS (25), /* HI */
305 COSTS_N_INSNS (25), /* SI */
306 COSTS_N_INSNS (25), /* DI */
307 COSTS_N_INSNS (25)}, /* other */
308 COSTS_N_INSNS (3), /* cost of movsx */
309 COSTS_N_INSNS (2), /* cost of movzx */
310 8, /* "large" insn */
312 6, /* cost for loading QImode using movzbl */
313 {2, 4, 2}, /* cost of loading integer registers
314 in QImode, HImode and SImode.
315 Relative to reg-reg move (2). */
316 {2, 4, 2}, /* cost of storing integer registers */
317 2, /* cost of reg,reg fld/fst */
318 {2, 2, 6}, /* cost of loading fp registers
319 in SFmode, DFmode and XFmode */
320 {4, 4, 6}, /* cost of storing fp registers
321 in SFmode, DFmode and XFmode */
322 8, /* cost of moving MMX register */
323 {8, 8}, /* cost of loading MMX registers
324 in SImode and DImode */
325 {8, 8}, /* cost of storing MMX registers
326 in SImode and DImode */
327 2, /* cost of moving SSE register */
328 {4, 8, 16}, /* cost of loading SSE registers
329 in SImode, DImode and TImode */
330 {4, 8, 16}, /* cost of storing SSE registers
331 in SImode, DImode and TImode */
332 3, /* MMX or SSE register to integer */
333 8, /* size of l1 cache. */
334 8, /* size of l2 cache */
335 0, /* size of prefetch block */
336 0, /* number of parallel prefetches */
338 COSTS_N_INSNS (3), /* cost of FADD and FSUB insns. */
339 COSTS_N_INSNS (3), /* cost of FMUL instruction. */
340 COSTS_N_INSNS (39), /* cost of FDIV instruction. */
341 COSTS_N_INSNS (1), /* cost of FABS instruction. */
342 COSTS_N_INSNS (1), /* cost of FCHS instruction. */
343 COSTS_N_INSNS (70), /* cost of FSQRT instruction. */
344 {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
345 DUMMY_STRINGOP_ALGS},
346 {{libcall, {{-1, rep_prefix_4_byte}}},
347 DUMMY_STRINGOP_ALGS},
348 1, /* scalar_stmt_cost. */
349 1, /* scalar load_cost. */
350 1, /* scalar_store_cost. */
351 1, /* vec_stmt_cost. */
352 1, /* vec_to_scalar_cost. */
353 1, /* scalar_to_vec_cost. */
354 1, /* vec_align_load_cost. */
355 2, /* vec_unalign_load_cost. */
356 1, /* vec_store_cost. */
357 3, /* cond_taken_branch_cost. */
358 1, /* cond_not_taken_branch_cost. */
362 struct processor_costs pentiumpro_cost = {
363 COSTS_N_INSNS (1), /* cost of an add instruction */
364 COSTS_N_INSNS (1), /* cost of a lea instruction */
365 COSTS_N_INSNS (1), /* variable shift costs */
366 COSTS_N_INSNS (1), /* constant shift costs */
367 {COSTS_N_INSNS (4), /* cost of starting multiply for QI */
368 COSTS_N_INSNS (4), /* HI */
369 COSTS_N_INSNS (4), /* SI */
370 COSTS_N_INSNS (4), /* DI */
371 COSTS_N_INSNS (4)}, /* other */
372 0, /* cost of multiply per each bit set */
373 {COSTS_N_INSNS (17), /* cost of a divide/mod for QI */
374 COSTS_N_INSNS (17), /* HI */
375 COSTS_N_INSNS (17), /* SI */
376 COSTS_N_INSNS (17), /* DI */
377 COSTS_N_INSNS (17)}, /* other */
378 COSTS_N_INSNS (1), /* cost of movsx */
379 COSTS_N_INSNS (1), /* cost of movzx */
380 8, /* "large" insn */
382 2, /* cost for loading QImode using movzbl */
383 {4, 4, 4}, /* cost of loading integer registers
384 in QImode, HImode and SImode.
385 Relative to reg-reg move (2). */
386 {2, 2, 2}, /* cost of storing integer registers */
387 2, /* cost of reg,reg fld/fst */
388 {2, 2, 6}, /* cost of loading fp registers
389 in SFmode, DFmode and XFmode */
390 {4, 4, 6}, /* cost of storing fp registers
391 in SFmode, DFmode and XFmode */
392 2, /* cost of moving MMX register */
393 {2, 2}, /* cost of loading MMX registers
394 in SImode and DImode */
395 {2, 2}, /* cost of storing MMX registers
396 in SImode and DImode */
397 2, /* cost of moving SSE register */
398 {2, 2, 8}, /* cost of loading SSE registers
399 in SImode, DImode and TImode */
400 {2, 2, 8}, /* cost of storing SSE registers
401 in SImode, DImode and TImode */
402 3, /* MMX or SSE register to integer */
403 8, /* size of l1 cache. */
404 256, /* size of l2 cache */
405 32, /* size of prefetch block */
406 6, /* number of parallel prefetches */
408 COSTS_N_INSNS (3), /* cost of FADD and FSUB insns. */
409 COSTS_N_INSNS (5), /* cost of FMUL instruction. */
410 COSTS_N_INSNS (56), /* cost of FDIV instruction. */
411 COSTS_N_INSNS (2), /* cost of FABS instruction. */
412 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
413 COSTS_N_INSNS (56), /* cost of FSQRT instruction. */
414 /* PentiumPro has optimized rep instructions for blocks aligned by 8 bytes (we ensure
415 the alignment). For small blocks inline loop is still a noticeable win, for bigger
416 blocks either rep movsl or rep movsb is way to go. Rep movsb has apparently
417 more expensive startup time in CPU, but after 4K the difference is down in the noise.
419 {{rep_prefix_4_byte, {{128, loop}, {1024, unrolled_loop},
420 {8192, rep_prefix_4_byte}, {-1, rep_prefix_1_byte}}},
421 DUMMY_STRINGOP_ALGS},
422 {{rep_prefix_4_byte, {{1024, unrolled_loop},
423 {8192, rep_prefix_4_byte}, {-1, libcall}}},
424 DUMMY_STRINGOP_ALGS},
425 1, /* scalar_stmt_cost. */
426 1, /* scalar load_cost. */
427 1, /* scalar_store_cost. */
428 1, /* vec_stmt_cost. */
429 1, /* vec_to_scalar_cost. */
430 1, /* scalar_to_vec_cost. */
431 1, /* vec_align_load_cost. */
432 2, /* vec_unalign_load_cost. */
433 1, /* vec_store_cost. */
434 3, /* cond_taken_branch_cost. */
435 1, /* cond_not_taken_branch_cost. */
439 struct processor_costs geode_cost = {
440 COSTS_N_INSNS (1), /* cost of an add instruction */
441 COSTS_N_INSNS (1), /* cost of a lea instruction */
442 COSTS_N_INSNS (2), /* variable shift costs */
443 COSTS_N_INSNS (1), /* constant shift costs */
444 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
445 COSTS_N_INSNS (4), /* HI */
446 COSTS_N_INSNS (7), /* SI */
447 COSTS_N_INSNS (7), /* DI */
448 COSTS_N_INSNS (7)}, /* other */
449 0, /* cost of multiply per each bit set */
450 {COSTS_N_INSNS (15), /* cost of a divide/mod for QI */
451 COSTS_N_INSNS (23), /* HI */
452 COSTS_N_INSNS (39), /* SI */
453 COSTS_N_INSNS (39), /* DI */
454 COSTS_N_INSNS (39)}, /* other */
455 COSTS_N_INSNS (1), /* cost of movsx */
456 COSTS_N_INSNS (1), /* cost of movzx */
457 8, /* "large" insn */
459 1, /* cost for loading QImode using movzbl */
460 {1, 1, 1}, /* cost of loading integer registers
461 in QImode, HImode and SImode.
462 Relative to reg-reg move (2). */
463 {1, 1, 1}, /* cost of storing integer registers */
464 1, /* cost of reg,reg fld/fst */
465 {1, 1, 1}, /* cost of loading fp registers
466 in SFmode, DFmode and XFmode */
467 {4, 6, 6}, /* cost of storing fp registers
468 in SFmode, DFmode and XFmode */
470 1, /* cost of moving MMX register */
471 {1, 1}, /* cost of loading MMX registers
472 in SImode and DImode */
473 {1, 1}, /* cost of storing MMX registers
474 in SImode and DImode */
475 1, /* cost of moving SSE register */
476 {1, 1, 1}, /* cost of loading SSE registers
477 in SImode, DImode and TImode */
478 {1, 1, 1}, /* cost of storing SSE registers
479 in SImode, DImode and TImode */
480 1, /* MMX or SSE register to integer */
481 64, /* size of l1 cache. */
482 128, /* size of l2 cache. */
483 32, /* size of prefetch block */
484 1, /* number of parallel prefetches */
486 COSTS_N_INSNS (6), /* cost of FADD and FSUB insns. */
487 COSTS_N_INSNS (11), /* cost of FMUL instruction. */
488 COSTS_N_INSNS (47), /* cost of FDIV instruction. */
489 COSTS_N_INSNS (1), /* cost of FABS instruction. */
490 COSTS_N_INSNS (1), /* cost of FCHS instruction. */
491 COSTS_N_INSNS (54), /* cost of FSQRT instruction. */
492 {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
493 DUMMY_STRINGOP_ALGS},
494 {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
495 DUMMY_STRINGOP_ALGS},
496 1, /* scalar_stmt_cost. */
497 1, /* scalar load_cost. */
498 1, /* scalar_store_cost. */
499 1, /* vec_stmt_cost. */
500 1, /* vec_to_scalar_cost. */
501 1, /* scalar_to_vec_cost. */
502 1, /* vec_align_load_cost. */
503 2, /* vec_unalign_load_cost. */
504 1, /* vec_store_cost. */
505 3, /* cond_taken_branch_cost. */
506 1, /* cond_not_taken_branch_cost. */
510 struct processor_costs k6_cost = {
511 COSTS_N_INSNS (1), /* cost of an add instruction */
512 COSTS_N_INSNS (2), /* cost of a lea instruction */
513 COSTS_N_INSNS (1), /* variable shift costs */
514 COSTS_N_INSNS (1), /* constant shift costs */
515 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
516 COSTS_N_INSNS (3), /* HI */
517 COSTS_N_INSNS (3), /* SI */
518 COSTS_N_INSNS (3), /* DI */
519 COSTS_N_INSNS (3)}, /* other */
520 0, /* cost of multiply per each bit set */
521 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
522 COSTS_N_INSNS (18), /* HI */
523 COSTS_N_INSNS (18), /* SI */
524 COSTS_N_INSNS (18), /* DI */
525 COSTS_N_INSNS (18)}, /* other */
526 COSTS_N_INSNS (2), /* cost of movsx */
527 COSTS_N_INSNS (2), /* cost of movzx */
528 8, /* "large" insn */
530 3, /* cost for loading QImode using movzbl */
531 {4, 5, 4}, /* cost of loading integer registers
532 in QImode, HImode and SImode.
533 Relative to reg-reg move (2). */
534 {2, 3, 2}, /* cost of storing integer registers */
535 4, /* cost of reg,reg fld/fst */
536 {6, 6, 6}, /* cost of loading fp registers
537 in SFmode, DFmode and XFmode */
538 {4, 4, 4}, /* cost of storing fp registers
539 in SFmode, DFmode and XFmode */
540 2, /* cost of moving MMX register */
541 {2, 2}, /* cost of loading MMX registers
542 in SImode and DImode */
543 {2, 2}, /* cost of storing MMX registers
544 in SImode and DImode */
545 2, /* cost of moving SSE register */
546 {2, 2, 8}, /* cost of loading SSE registers
547 in SImode, DImode and TImode */
548 {2, 2, 8}, /* cost of storing SSE registers
549 in SImode, DImode and TImode */
550 6, /* MMX or SSE register to integer */
551 32, /* size of l1 cache. */
552 32, /* size of l2 cache. Some models
553 have integrated l2 cache, but
554 optimizing for k6 is not important
555 enough to worry about that. */
556 32, /* size of prefetch block */
557 1, /* number of parallel prefetches */
559 COSTS_N_INSNS (2), /* cost of FADD and FSUB insns. */
560 COSTS_N_INSNS (2), /* cost of FMUL instruction. */
561 COSTS_N_INSNS (56), /* cost of FDIV instruction. */
562 COSTS_N_INSNS (2), /* cost of FABS instruction. */
563 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
564 COSTS_N_INSNS (56), /* cost of FSQRT instruction. */
565 {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
566 DUMMY_STRINGOP_ALGS},
567 {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}},
568 DUMMY_STRINGOP_ALGS},
569 1, /* scalar_stmt_cost. */
570 1, /* scalar load_cost. */
571 1, /* scalar_store_cost. */
572 1, /* vec_stmt_cost. */
573 1, /* vec_to_scalar_cost. */
574 1, /* scalar_to_vec_cost. */
575 1, /* vec_align_load_cost. */
576 2, /* vec_unalign_load_cost. */
577 1, /* vec_store_cost. */
578 3, /* cond_taken_branch_cost. */
579 1, /* cond_not_taken_branch_cost. */
583 struct processor_costs athlon_cost = {
584 COSTS_N_INSNS (1), /* cost of an add instruction */
585 COSTS_N_INSNS (2), /* cost of a lea instruction */
586 COSTS_N_INSNS (1), /* variable shift costs */
587 COSTS_N_INSNS (1), /* constant shift costs */
588 {COSTS_N_INSNS (5), /* cost of starting multiply for QI */
589 COSTS_N_INSNS (5), /* HI */
590 COSTS_N_INSNS (5), /* SI */
591 COSTS_N_INSNS (5), /* DI */
592 COSTS_N_INSNS (5)}, /* other */
593 0, /* cost of multiply per each bit set */
594 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
595 COSTS_N_INSNS (26), /* HI */
596 COSTS_N_INSNS (42), /* SI */
597 COSTS_N_INSNS (74), /* DI */
598 COSTS_N_INSNS (74)}, /* other */
599 COSTS_N_INSNS (1), /* cost of movsx */
600 COSTS_N_INSNS (1), /* cost of movzx */
601 8, /* "large" insn */
603 4, /* cost for loading QImode using movzbl */
604 {3, 4, 3}, /* cost of loading integer registers
605 in QImode, HImode and SImode.
606 Relative to reg-reg move (2). */
607 {3, 4, 3}, /* cost of storing integer registers */
608 4, /* cost of reg,reg fld/fst */
609 {4, 4, 12}, /* cost of loading fp registers
610 in SFmode, DFmode and XFmode */
611 {6, 6, 8}, /* cost of storing fp registers
612 in SFmode, DFmode and XFmode */
613 2, /* cost of moving MMX register */
614 {4, 4}, /* cost of loading MMX registers
615 in SImode and DImode */
616 {4, 4}, /* cost of storing MMX registers
617 in SImode and DImode */
618 2, /* cost of moving SSE register */
619 {4, 4, 6}, /* cost of loading SSE registers
620 in SImode, DImode and TImode */
621 {4, 4, 5}, /* cost of storing SSE registers
622 in SImode, DImode and TImode */
623 5, /* MMX or SSE register to integer */
624 64, /* size of l1 cache. */
625 256, /* size of l2 cache. */
626 64, /* size of prefetch block */
627 6, /* number of parallel prefetches */
629 COSTS_N_INSNS (4), /* cost of FADD and FSUB insns. */
630 COSTS_N_INSNS (4), /* cost of FMUL instruction. */
631 COSTS_N_INSNS (24), /* cost of FDIV instruction. */
632 COSTS_N_INSNS (2), /* cost of FABS instruction. */
633 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
634 COSTS_N_INSNS (35), /* cost of FSQRT instruction. */
635 /* For some reason, Athlon deals better with REP prefix (relative to loops)
636 compared to K8. Alignment becomes important after 8 bytes for memcpy and
637 128 bytes for memset. */
638 {{libcall, {{2048, rep_prefix_4_byte}, {-1, libcall}}},
639 DUMMY_STRINGOP_ALGS},
640 {{libcall, {{2048, rep_prefix_4_byte}, {-1, libcall}}},
641 DUMMY_STRINGOP_ALGS},
642 1, /* scalar_stmt_cost. */
643 1, /* scalar load_cost. */
644 1, /* scalar_store_cost. */
645 1, /* vec_stmt_cost. */
646 1, /* vec_to_scalar_cost. */
647 1, /* scalar_to_vec_cost. */
648 1, /* vec_align_load_cost. */
649 2, /* vec_unalign_load_cost. */
650 1, /* vec_store_cost. */
651 3, /* cond_taken_branch_cost. */
652 1, /* cond_not_taken_branch_cost. */
656 struct processor_costs k8_cost = {
657 COSTS_N_INSNS (1), /* cost of an add instruction */
658 COSTS_N_INSNS (2), /* cost of a lea instruction */
659 COSTS_N_INSNS (1), /* variable shift costs */
660 COSTS_N_INSNS (1), /* constant shift costs */
661 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
662 COSTS_N_INSNS (4), /* HI */
663 COSTS_N_INSNS (3), /* SI */
664 COSTS_N_INSNS (4), /* DI */
665 COSTS_N_INSNS (5)}, /* other */
666 0, /* cost of multiply per each bit set */
667 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
668 COSTS_N_INSNS (26), /* HI */
669 COSTS_N_INSNS (42), /* SI */
670 COSTS_N_INSNS (74), /* DI */
671 COSTS_N_INSNS (74)}, /* other */
672 COSTS_N_INSNS (1), /* cost of movsx */
673 COSTS_N_INSNS (1), /* cost of movzx */
674 8, /* "large" insn */
676 4, /* cost for loading QImode using movzbl */
677 {3, 4, 3}, /* cost of loading integer registers
678 in QImode, HImode and SImode.
679 Relative to reg-reg move (2). */
680 {3, 4, 3}, /* cost of storing integer registers */
681 4, /* cost of reg,reg fld/fst */
682 {4, 4, 12}, /* cost of loading fp registers
683 in SFmode, DFmode and XFmode */
684 {6, 6, 8}, /* cost of storing fp registers
685 in SFmode, DFmode and XFmode */
686 2, /* cost of moving MMX register */
687 {3, 3}, /* cost of loading MMX registers
688 in SImode and DImode */
689 {4, 4}, /* cost of storing MMX registers
690 in SImode and DImode */
691 2, /* cost of moving SSE register */
692 {4, 3, 6}, /* cost of loading SSE registers
693 in SImode, DImode and TImode */
694 {4, 4, 5}, /* cost of storing SSE registers
695 in SImode, DImode and TImode */
696 5, /* MMX or SSE register to integer */
697 64, /* size of l1 cache. */
698 512, /* size of l2 cache. */
699 64, /* size of prefetch block */
700 /* New AMD processors never drop prefetches; if they cannot be performed
701 immediately, they are queued. We set number of simultaneous prefetches
702 to a large constant to reflect this (it probably is not a good idea not
703 to limit number of prefetches at all, as their execution also takes some
705 100, /* number of parallel prefetches */
707 COSTS_N_INSNS (4), /* cost of FADD and FSUB insns. */
708 COSTS_N_INSNS (4), /* cost of FMUL instruction. */
709 COSTS_N_INSNS (19), /* cost of FDIV instruction. */
710 COSTS_N_INSNS (2), /* cost of FABS instruction. */
711 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
712 COSTS_N_INSNS (35), /* cost of FSQRT instruction. */
713 /* K8 has optimized REP instruction for medium sized blocks, but for very small
714 blocks it is better to use loop. For large blocks, libcall can do
715 nontemporary accesses and beat inline considerably. */
716 {{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
717 {libcall, {{16, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
718 {{libcall, {{8, loop}, {24, unrolled_loop},
719 {2048, rep_prefix_4_byte}, {-1, libcall}}},
720 {libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
721 4, /* scalar_stmt_cost. */
722 2, /* scalar load_cost. */
723 2, /* scalar_store_cost. */
724 5, /* vec_stmt_cost. */
725 0, /* vec_to_scalar_cost. */
726 2, /* scalar_to_vec_cost. */
727 2, /* vec_align_load_cost. */
728 3, /* vec_unalign_load_cost. */
729 3, /* vec_store_cost. */
730 3, /* cond_taken_branch_cost. */
731 2, /* cond_not_taken_branch_cost. */
734 struct processor_costs amdfam10_cost = {
735 COSTS_N_INSNS (1), /* cost of an add instruction */
736 COSTS_N_INSNS (2), /* cost of a lea instruction */
737 COSTS_N_INSNS (1), /* variable shift costs */
738 COSTS_N_INSNS (1), /* constant shift costs */
739 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
740 COSTS_N_INSNS (4), /* HI */
741 COSTS_N_INSNS (3), /* SI */
742 COSTS_N_INSNS (4), /* DI */
743 COSTS_N_INSNS (5)}, /* other */
744 0, /* cost of multiply per each bit set */
745 {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */
746 COSTS_N_INSNS (35), /* HI */
747 COSTS_N_INSNS (51), /* SI */
748 COSTS_N_INSNS (83), /* DI */
749 COSTS_N_INSNS (83)}, /* other */
750 COSTS_N_INSNS (1), /* cost of movsx */
751 COSTS_N_INSNS (1), /* cost of movzx */
752 8, /* "large" insn */
754 4, /* cost for loading QImode using movzbl */
755 {3, 4, 3}, /* cost of loading integer registers
756 in QImode, HImode and SImode.
757 Relative to reg-reg move (2). */
758 {3, 4, 3}, /* cost of storing integer registers */
759 4, /* cost of reg,reg fld/fst */
760 {4, 4, 12}, /* cost of loading fp registers
761 in SFmode, DFmode and XFmode */
762 {6, 6, 8}, /* cost of storing fp registers
763 in SFmode, DFmode and XFmode */
764 2, /* cost of moving MMX register */
765 {3, 3}, /* cost of loading MMX registers
766 in SImode and DImode */
767 {4, 4}, /* cost of storing MMX registers
768 in SImode and DImode */
769 2, /* cost of moving SSE register */
770 {4, 4, 3}, /* cost of loading SSE registers
771 in SImode, DImode and TImode */
772 {4, 4, 5}, /* cost of storing SSE registers
773 in SImode, DImode and TImode */
774 3, /* MMX or SSE register to integer */
776 MOVD reg64, xmmreg Double FSTORE 4
777 MOVD reg32, xmmreg Double FSTORE 4
779 MOVD reg64, xmmreg Double FADD 3
781 MOVD reg32, xmmreg Double FADD 3
783 64, /* size of l1 cache. */
784 512, /* size of l2 cache. */
785 64, /* size of prefetch block */
786 /* New AMD processors never drop prefetches; if they cannot be performed
787 immediately, they are queued. We set number of simultaneous prefetches
788 to a large constant to reflect this (it probably is not a good idea not
789 to limit number of prefetches at all, as their execution also takes some
791 100, /* number of parallel prefetches */
793 COSTS_N_INSNS (4), /* cost of FADD and FSUB insns. */
794 COSTS_N_INSNS (4), /* cost of FMUL instruction. */
795 COSTS_N_INSNS (19), /* cost of FDIV instruction. */
796 COSTS_N_INSNS (2), /* cost of FABS instruction. */
797 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
798 COSTS_N_INSNS (35), /* cost of FSQRT instruction. */
800 /* AMDFAM10 has optimized REP instruction for medium sized blocks, but for
801 very small blocks it is better to use loop. For large blocks, libcall can
802 do nontemporary accesses and beat inline considerably. */
803 {{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}},
804 {libcall, {{16, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
805 {{libcall, {{8, loop}, {24, unrolled_loop},
806 {2048, rep_prefix_4_byte}, {-1, libcall}}},
807 {libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
808 4, /* scalar_stmt_cost. */
809 2, /* scalar load_cost. */
810 2, /* scalar_store_cost. */
811 6, /* vec_stmt_cost. */
812 0, /* vec_to_scalar_cost. */
813 2, /* scalar_to_vec_cost. */
814 2, /* vec_align_load_cost. */
815 2, /* vec_unalign_load_cost. */
816 2, /* vec_store_cost. */
817 2, /* cond_taken_branch_cost. */
818 1, /* cond_not_taken_branch_cost. */
822 struct processor_costs pentium4_cost = {
823 COSTS_N_INSNS (1), /* cost of an add instruction */
824 COSTS_N_INSNS (3), /* cost of a lea instruction */
825 COSTS_N_INSNS (4), /* variable shift costs */
826 COSTS_N_INSNS (4), /* constant shift costs */
827 {COSTS_N_INSNS (15), /* cost of starting multiply for QI */
828 COSTS_N_INSNS (15), /* HI */
829 COSTS_N_INSNS (15), /* SI */
830 COSTS_N_INSNS (15), /* DI */
831 COSTS_N_INSNS (15)}, /* other */
832 0, /* cost of multiply per each bit set */
833 {COSTS_N_INSNS (56), /* cost of a divide/mod for QI */
834 COSTS_N_INSNS (56), /* HI */
835 COSTS_N_INSNS (56), /* SI */
836 COSTS_N_INSNS (56), /* DI */
837 COSTS_N_INSNS (56)}, /* other */
838 COSTS_N_INSNS (1), /* cost of movsx */
839 COSTS_N_INSNS (1), /* cost of movzx */
840 16, /* "large" insn */
842 2, /* cost for loading QImode using movzbl */
843 {4, 5, 4}, /* cost of loading integer registers
844 in QImode, HImode and SImode.
845 Relative to reg-reg move (2). */
846 {2, 3, 2}, /* cost of storing integer registers */
847 2, /* cost of reg,reg fld/fst */
848 {2, 2, 6}, /* cost of loading fp registers
849 in SFmode, DFmode and XFmode */
850 {4, 4, 6}, /* cost of storing fp registers
851 in SFmode, DFmode and XFmode */
852 2, /* cost of moving MMX register */
853 {2, 2}, /* cost of loading MMX registers
854 in SImode and DImode */
855 {2, 2}, /* cost of storing MMX registers
856 in SImode and DImode */
857 12, /* cost of moving SSE register */
858 {12, 12, 12}, /* cost of loading SSE registers
859 in SImode, DImode and TImode */
860 {2, 2, 8}, /* cost of storing SSE registers
861 in SImode, DImode and TImode */
862 10, /* MMX or SSE register to integer */
863 8, /* size of l1 cache. */
864 256, /* size of l2 cache. */
865 64, /* size of prefetch block */
866 6, /* number of parallel prefetches */
868 COSTS_N_INSNS (5), /* cost of FADD and FSUB insns. */
869 COSTS_N_INSNS (7), /* cost of FMUL instruction. */
870 COSTS_N_INSNS (43), /* cost of FDIV instruction. */
871 COSTS_N_INSNS (2), /* cost of FABS instruction. */
872 COSTS_N_INSNS (2), /* cost of FCHS instruction. */
873 COSTS_N_INSNS (43), /* cost of FSQRT instruction. */
874 {{libcall, {{12, loop_1_byte}, {-1, rep_prefix_4_byte}}},
875 DUMMY_STRINGOP_ALGS},
876 {{libcall, {{6, loop_1_byte}, {48, loop}, {20480, rep_prefix_4_byte},
878 DUMMY_STRINGOP_ALGS},
879 1, /* scalar_stmt_cost. */
880 1, /* scalar load_cost. */
881 1, /* scalar_store_cost. */
882 1, /* vec_stmt_cost. */
883 1, /* vec_to_scalar_cost. */
884 1, /* scalar_to_vec_cost. */
885 1, /* vec_align_load_cost. */
886 2, /* vec_unalign_load_cost. */
887 1, /* vec_store_cost. */
888 3, /* cond_taken_branch_cost. */
889 1, /* cond_not_taken_branch_cost. */
893 struct processor_costs nocona_cost = {
894 COSTS_N_INSNS (1), /* cost of an add instruction */
895 COSTS_N_INSNS (1), /* cost of a lea instruction */
896 COSTS_N_INSNS (1), /* variable shift costs */
897 COSTS_N_INSNS (1), /* constant shift costs */
898 {COSTS_N_INSNS (10), /* cost of starting multiply for QI */
899 COSTS_N_INSNS (10), /* HI */
900 COSTS_N_INSNS (10), /* SI */
901 COSTS_N_INSNS (10), /* DI */
902 COSTS_N_INSNS (10)}, /* other */
903 0, /* cost of multiply per each bit set */
904 {COSTS_N_INSNS (66), /* cost of a divide/mod for QI */
905 COSTS_N_INSNS (66), /* HI */
906 COSTS_N_INSNS (66), /* SI */
907 COSTS_N_INSNS (66), /* DI */
908 COSTS_N_INSNS (66)}, /* other */
909 COSTS_N_INSNS (1), /* cost of movsx */
910 COSTS_N_INSNS (1), /* cost of movzx */
911 16, /* "large" insn */
913 4, /* cost for loading QImode using movzbl */
914 {4, 4, 4}, /* cost of loading integer registers
915 in QImode, HImode and SImode.
916 Relative to reg-reg move (2). */
917 {4, 4, 4}, /* cost of storing integer registers */
918 3, /* cost of reg,reg fld/fst */
919 {12, 12, 12}, /* cost of loading fp registers
920 in SFmode, DFmode and XFmode */
921 {4, 4, 4}, /* cost of storing fp registers
922 in SFmode, DFmode and XFmode */
923 6, /* cost of moving MMX register */
924 {12, 12}, /* cost of loading MMX registers
925 in SImode and DImode */
926 {12, 12}, /* cost of storing MMX registers
927 in SImode and DImode */
928 6, /* cost of moving SSE register */
929 {12, 12, 12}, /* cost of loading SSE registers
930 in SImode, DImode and TImode */
931 {12, 12, 12}, /* cost of storing SSE registers
932 in SImode, DImode and TImode */
933 8, /* MMX or SSE register to integer */
934 8, /* size of l1 cache. */
935 1024, /* size of l2 cache. */
936 128, /* size of prefetch block */
937 8, /* number of parallel prefetches */
939 COSTS_N_INSNS (6), /* cost of FADD and FSUB insns. */
940 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
941 COSTS_N_INSNS (40), /* cost of FDIV instruction. */
942 COSTS_N_INSNS (3), /* cost of FABS instruction. */
943 COSTS_N_INSNS (3), /* cost of FCHS instruction. */
944 COSTS_N_INSNS (44), /* cost of FSQRT instruction. */
945 {{libcall, {{12, loop_1_byte}, {-1, rep_prefix_4_byte}}},
946 {libcall, {{32, loop}, {20000, rep_prefix_8_byte},
947 {100000, unrolled_loop}, {-1, libcall}}}},
948 {{libcall, {{6, loop_1_byte}, {48, loop}, {20480, rep_prefix_4_byte},
950 {libcall, {{24, loop}, {64, unrolled_loop},
951 {8192, rep_prefix_8_byte}, {-1, libcall}}}},
952 1, /* scalar_stmt_cost. */
953 1, /* scalar load_cost. */
954 1, /* scalar_store_cost. */
955 1, /* vec_stmt_cost. */
956 1, /* vec_to_scalar_cost. */
957 1, /* scalar_to_vec_cost. */
958 1, /* vec_align_load_cost. */
959 2, /* vec_unalign_load_cost. */
960 1, /* vec_store_cost. */
961 3, /* cond_taken_branch_cost. */
962 1, /* cond_not_taken_branch_cost. */
966 struct processor_costs core2_cost = {
967 COSTS_N_INSNS (1), /* cost of an add instruction */
968 COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */
969 COSTS_N_INSNS (1), /* variable shift costs */
970 COSTS_N_INSNS (1), /* constant shift costs */
971 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
972 COSTS_N_INSNS (3), /* HI */
973 COSTS_N_INSNS (3), /* SI */
974 COSTS_N_INSNS (3), /* DI */
975 COSTS_N_INSNS (3)}, /* other */
976 0, /* cost of multiply per each bit set */
977 {COSTS_N_INSNS (22), /* cost of a divide/mod for QI */
978 COSTS_N_INSNS (22), /* HI */
979 COSTS_N_INSNS (22), /* SI */
980 COSTS_N_INSNS (22), /* DI */
981 COSTS_N_INSNS (22)}, /* other */
982 COSTS_N_INSNS (1), /* cost of movsx */
983 COSTS_N_INSNS (1), /* cost of movzx */
984 8, /* "large" insn */
986 2, /* cost for loading QImode using movzbl */
987 {6, 6, 6}, /* cost of loading integer registers
988 in QImode, HImode and SImode.
989 Relative to reg-reg move (2). */
990 {4, 4, 4}, /* cost of storing integer registers */
991 2, /* cost of reg,reg fld/fst */
992 {6, 6, 6}, /* cost of loading fp registers
993 in SFmode, DFmode and XFmode */
994 {4, 4, 4}, /* cost of storing fp registers
995 in SFmode, DFmode and XFmode */
996 2, /* cost of moving MMX register */
997 {6, 6}, /* cost of loading MMX registers
998 in SImode and DImode */
999 {4, 4}, /* cost of storing MMX registers
1000 in SImode and DImode */
1001 2, /* cost of moving SSE register */
1002 {6, 6, 6}, /* cost of loading SSE registers
1003 in SImode, DImode and TImode */
1004 {4, 4, 4}, /* cost of storing SSE registers
1005 in SImode, DImode and TImode */
1006 2, /* MMX or SSE register to integer */
1007 32, /* size of l1 cache. */
1008 2048, /* size of l2 cache. */
1009 128, /* size of prefetch block */
1010 8, /* number of parallel prefetches */
1011 3, /* Branch cost */
1012 COSTS_N_INSNS (3), /* cost of FADD and FSUB insns. */
1013 COSTS_N_INSNS (5), /* cost of FMUL instruction. */
1014 COSTS_N_INSNS (32), /* cost of FDIV instruction. */
1015 COSTS_N_INSNS (1), /* cost of FABS instruction. */
1016 COSTS_N_INSNS (1), /* cost of FCHS instruction. */
1017 COSTS_N_INSNS (58), /* cost of FSQRT instruction. */
1018 {{libcall, {{11, loop}, {-1, rep_prefix_4_byte}}},
1019 {libcall, {{32, loop}, {64, rep_prefix_4_byte},
1020 {8192, rep_prefix_8_byte}, {-1, libcall}}}},
1021 {{libcall, {{8, loop}, {15, unrolled_loop},
1022 {2048, rep_prefix_4_byte}, {-1, libcall}}},
1023 {libcall, {{24, loop}, {32, unrolled_loop},
1024 {8192, rep_prefix_8_byte}, {-1, libcall}}}},
1025 1, /* scalar_stmt_cost. */
1026 1, /* scalar load_cost. */
1027 1, /* scalar_store_cost. */
1028 1, /* vec_stmt_cost. */
1029 1, /* vec_to_scalar_cost. */
1030 1, /* scalar_to_vec_cost. */
1031 1, /* vec_align_load_cost. */
1032 2, /* vec_unalign_load_cost. */
1033 1, /* vec_store_cost. */
1034 3, /* cond_taken_branch_cost. */
1035 1, /* cond_not_taken_branch_cost. */
1039 struct processor_costs atom_cost = {
1040 COSTS_N_INSNS (1), /* cost of an add instruction */
1041 COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */
1042 COSTS_N_INSNS (1), /* variable shift costs */
1043 COSTS_N_INSNS (1), /* constant shift costs */
1044 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1045 COSTS_N_INSNS (4), /* HI */
1046 COSTS_N_INSNS (3), /* SI */
1047 COSTS_N_INSNS (4), /* DI */
1048 COSTS_N_INSNS (2)}, /* other */
1049 0, /* cost of multiply per each bit set */
1050 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
1051 COSTS_N_INSNS (26), /* HI */
1052 COSTS_N_INSNS (42), /* SI */
1053 COSTS_N_INSNS (74), /* DI */
1054 COSTS_N_INSNS (74)}, /* other */
1055 COSTS_N_INSNS (1), /* cost of movsx */
1056 COSTS_N_INSNS (1), /* cost of movzx */
1057 8, /* "large" insn */
1058 17, /* MOVE_RATIO */
1059 2, /* cost for loading QImode using movzbl */
1060 {4, 4, 4}, /* cost of loading integer registers
1061 in QImode, HImode and SImode.
1062 Relative to reg-reg move (2). */
1063 {4, 4, 4}, /* cost of storing integer registers */
1064 4, /* cost of reg,reg fld/fst */
1065 {12, 12, 12}, /* cost of loading fp registers
1066 in SFmode, DFmode and XFmode */
1067 {6, 6, 8}, /* cost of storing fp registers
1068 in SFmode, DFmode and XFmode */
1069 2, /* cost of moving MMX register */
1070 {8, 8}, /* cost of loading MMX registers
1071 in SImode and DImode */
1072 {8, 8}, /* cost of storing MMX registers
1073 in SImode and DImode */
1074 2, /* cost of moving SSE register */
1075 {8, 8, 8}, /* cost of loading SSE registers
1076 in SImode, DImode and TImode */
1077 {8, 8, 8}, /* cost of storing SSE registers
1078 in SImode, DImode and TImode */
1079 5, /* MMX or SSE register to integer */
1080 32, /* size of l1 cache. */
1081 256, /* size of l2 cache. */
1082 64, /* size of prefetch block */
1083 6, /* number of parallel prefetches */
1084 3, /* Branch cost */
1085 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
1086 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
1087 COSTS_N_INSNS (20), /* cost of FDIV instruction. */
1088 COSTS_N_INSNS (8), /* cost of FABS instruction. */
1089 COSTS_N_INSNS (8), /* cost of FCHS instruction. */
1090 COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
1091 {{libcall, {{11, loop}, {-1, rep_prefix_4_byte}}},
1092 {libcall, {{32, loop}, {64, rep_prefix_4_byte},
1093 {8192, rep_prefix_8_byte}, {-1, libcall}}}},
1094 {{libcall, {{8, loop}, {15, unrolled_loop},
1095 {2048, rep_prefix_4_byte}, {-1, libcall}}},
1096 {libcall, {{24, loop}, {32, unrolled_loop},
1097 {8192, rep_prefix_8_byte}, {-1, libcall}}}},
1098 1, /* scalar_stmt_cost. */
1099 1, /* scalar load_cost. */
1100 1, /* scalar_store_cost. */
1101 1, /* vec_stmt_cost. */
1102 1, /* vec_to_scalar_cost. */
1103 1, /* scalar_to_vec_cost. */
1104 1, /* vec_align_load_cost. */
1105 2, /* vec_unalign_load_cost. */
1106 1, /* vec_store_cost. */
1107 3, /* cond_taken_branch_cost. */
1108 1, /* cond_not_taken_branch_cost. */
1111 /* Generic64 should produce code tuned for Nocona and K8. */
1113 struct processor_costs generic64_cost = {
1114 COSTS_N_INSNS (1), /* cost of an add instruction */
1115 /* On all chips taken into consideration lea is 2 cycles and more. With
1116 this cost however our current implementation of synth_mult results in
1117 use of unnecessary temporary registers causing regression on several
1118 SPECfp benchmarks. */
1119 COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */
1120 COSTS_N_INSNS (1), /* variable shift costs */
1121 COSTS_N_INSNS (1), /* constant shift costs */
1122 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1123 COSTS_N_INSNS (4), /* HI */
1124 COSTS_N_INSNS (3), /* SI */
1125 COSTS_N_INSNS (4), /* DI */
1126 COSTS_N_INSNS (2)}, /* other */
1127 0, /* cost of multiply per each bit set */
1128 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
1129 COSTS_N_INSNS (26), /* HI */
1130 COSTS_N_INSNS (42), /* SI */
1131 COSTS_N_INSNS (74), /* DI */
1132 COSTS_N_INSNS (74)}, /* other */
1133 COSTS_N_INSNS (1), /* cost of movsx */
1134 COSTS_N_INSNS (1), /* cost of movzx */
1135 8, /* "large" insn */
1136 17, /* MOVE_RATIO */
1137 4, /* cost for loading QImode using movzbl */
1138 {4, 4, 4}, /* cost of loading integer registers
1139 in QImode, HImode and SImode.
1140 Relative to reg-reg move (2). */
1141 {4, 4, 4}, /* cost of storing integer registers */
1142 4, /* cost of reg,reg fld/fst */
1143 {12, 12, 12}, /* cost of loading fp registers
1144 in SFmode, DFmode and XFmode */
1145 {6, 6, 8}, /* cost of storing fp registers
1146 in SFmode, DFmode and XFmode */
1147 2, /* cost of moving MMX register */
1148 {8, 8}, /* cost of loading MMX registers
1149 in SImode and DImode */
1150 {8, 8}, /* cost of storing MMX registers
1151 in SImode and DImode */
1152 2, /* cost of moving SSE register */
1153 {8, 8, 8}, /* cost of loading SSE registers
1154 in SImode, DImode and TImode */
1155 {8, 8, 8}, /* cost of storing SSE registers
1156 in SImode, DImode and TImode */
1157 5, /* MMX or SSE register to integer */
1158 32, /* size of l1 cache. */
1159 512, /* size of l2 cache. */
1160 64, /* size of prefetch block */
1161 6, /* number of parallel prefetches */
1162 /* Benchmarks shows large regressions on K8 sixtrack benchmark when this value
1163 is increased to perhaps more appropriate value of 5. */
1164 3, /* Branch cost */
1165 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
1166 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
1167 COSTS_N_INSNS (20), /* cost of FDIV instruction. */
1168 COSTS_N_INSNS (8), /* cost of FABS instruction. */
1169 COSTS_N_INSNS (8), /* cost of FCHS instruction. */
1170 COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
1171 {DUMMY_STRINGOP_ALGS,
1172 {libcall, {{32, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
1173 {DUMMY_STRINGOP_ALGS,
1174 {libcall, {{32, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}},
1175 1, /* scalar_stmt_cost. */
1176 1, /* scalar load_cost. */
1177 1, /* scalar_store_cost. */
1178 1, /* vec_stmt_cost. */
1179 1, /* vec_to_scalar_cost. */
1180 1, /* scalar_to_vec_cost. */
1181 1, /* vec_align_load_cost. */
1182 2, /* vec_unalign_load_cost. */
1183 1, /* vec_store_cost. */
1184 3, /* cond_taken_branch_cost. */
1185 1, /* cond_not_taken_branch_cost. */
1188 /* Generic32 should produce code tuned for Athlon, PPro, Pentium4, Nocona and K8. */
1190 struct processor_costs generic32_cost = {
1191 COSTS_N_INSNS (1), /* cost of an add instruction */
1192 COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */
1193 COSTS_N_INSNS (1), /* variable shift costs */
1194 COSTS_N_INSNS (1), /* constant shift costs */
1195 {COSTS_N_INSNS (3), /* cost of starting multiply for QI */
1196 COSTS_N_INSNS (4), /* HI */
1197 COSTS_N_INSNS (3), /* SI */
1198 COSTS_N_INSNS (4), /* DI */
1199 COSTS_N_INSNS (2)}, /* other */
1200 0, /* cost of multiply per each bit set */
1201 {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */
1202 COSTS_N_INSNS (26), /* HI */
1203 COSTS_N_INSNS (42), /* SI */
1204 COSTS_N_INSNS (74), /* DI */
1205 COSTS_N_INSNS (74)}, /* other */
1206 COSTS_N_INSNS (1), /* cost of movsx */
1207 COSTS_N_INSNS (1), /* cost of movzx */
1208 8, /* "large" insn */
1209 17, /* MOVE_RATIO */
1210 4, /* cost for loading QImode using movzbl */
1211 {4, 4, 4}, /* cost of loading integer registers
1212 in QImode, HImode and SImode.
1213 Relative to reg-reg move (2). */
1214 {4, 4, 4}, /* cost of storing integer registers */
1215 4, /* cost of reg,reg fld/fst */
1216 {12, 12, 12}, /* cost of loading fp registers
1217 in SFmode, DFmode and XFmode */
1218 {6, 6, 8}, /* cost of storing fp registers
1219 in SFmode, DFmode and XFmode */
1220 2, /* cost of moving MMX register */
1221 {8, 8}, /* cost of loading MMX registers
1222 in SImode and DImode */
1223 {8, 8}, /* cost of storing MMX registers
1224 in SImode and DImode */
1225 2, /* cost of moving SSE register */
1226 {8, 8, 8}, /* cost of loading SSE registers
1227 in SImode, DImode and TImode */
1228 {8, 8, 8}, /* cost of storing SSE registers
1229 in SImode, DImode and TImode */
1230 5, /* MMX or SSE register to integer */
1231 32, /* size of l1 cache. */
1232 256, /* size of l2 cache. */
1233 64, /* size of prefetch block */
1234 6, /* number of parallel prefetches */
1235 3, /* Branch cost */
1236 COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */
1237 COSTS_N_INSNS (8), /* cost of FMUL instruction. */
1238 COSTS_N_INSNS (20), /* cost of FDIV instruction. */
1239 COSTS_N_INSNS (8), /* cost of FABS instruction. */
1240 COSTS_N_INSNS (8), /* cost of FCHS instruction. */
1241 COSTS_N_INSNS (40), /* cost of FSQRT instruction. */
1242 {{libcall, {{32, loop}, {8192, rep_prefix_4_byte}, {-1, libcall}}},
1243 DUMMY_STRINGOP_ALGS},
1244 {{libcall, {{32, loop}, {8192, rep_prefix_4_byte}, {-1, libcall}}},
1245 DUMMY_STRINGOP_ALGS},
1246 1, /* scalar_stmt_cost. */
1247 1, /* scalar load_cost. */
1248 1, /* scalar_store_cost. */
1249 1, /* vec_stmt_cost. */
1250 1, /* vec_to_scalar_cost. */
1251 1, /* scalar_to_vec_cost. */
1252 1, /* vec_align_load_cost. */
1253 2, /* vec_unalign_load_cost. */
1254 1, /* vec_store_cost. */
1255 3, /* cond_taken_branch_cost. */
1256 1, /* cond_not_taken_branch_cost. */
1259 const struct processor_costs *ix86_cost = &pentium_cost;
1261 /* Processor feature/optimization bitmasks. */
1262 #define m_386 (1<<PROCESSOR_I386)
1263 #define m_486 (1<<PROCESSOR_I486)
1264 #define m_PENT (1<<PROCESSOR_PENTIUM)
1265 #define m_PPRO (1<<PROCESSOR_PENTIUMPRO)
1266 #define m_PENT4 (1<<PROCESSOR_PENTIUM4)
1267 #define m_NOCONA (1<<PROCESSOR_NOCONA)
1268 #define m_CORE2 (1<<PROCESSOR_CORE2)
1269 #define m_ATOM (1<<PROCESSOR_ATOM)
1271 #define m_GEODE (1<<PROCESSOR_GEODE)
1272 #define m_K6 (1<<PROCESSOR_K6)
1273 #define m_K6_GEODE (m_K6 | m_GEODE)
1274 #define m_K8 (1<<PROCESSOR_K8)
1275 #define m_ATHLON (1<<PROCESSOR_ATHLON)
1276 #define m_ATHLON_K8 (m_K8 | m_ATHLON)
1277 #define m_AMDFAM10 (1<<PROCESSOR_AMDFAM10)
1278 #define m_AMD_MULTIPLE (m_K8 | m_ATHLON | m_AMDFAM10)
1280 #define m_GENERIC32 (1<<PROCESSOR_GENERIC32)
1281 #define m_GENERIC64 (1<<PROCESSOR_GENERIC64)
1283 /* Generic instruction choice should be common subset of supported CPUs
1284 (PPro/PENT4/NOCONA/CORE2/Athlon/K8). */
1285 #define m_GENERIC (m_GENERIC32 | m_GENERIC64)
1287 /* Feature tests against the various tunings. */
1288 unsigned char ix86_tune_features[X86_TUNE_LAST];
1290 /* Feature tests against the various tunings used to create ix86_tune_features
1291 based on the processor mask. */
1292 static unsigned int initial_ix86_tune_features[X86_TUNE_LAST] = {
1293 /* X86_TUNE_USE_LEAVE: Leave does not affect Nocona SPEC2000 results
1294 negatively, so enabling for Generic64 seems like good code size
1295 tradeoff. We can't enable it for 32bit generic because it does not
1296 work well with PPro base chips. */
1297 m_386 | m_K6_GEODE | m_AMD_MULTIPLE | m_CORE2 | m_GENERIC64,
1299 /* X86_TUNE_PUSH_MEMORY */
1300 m_386 | m_K6_GEODE | m_AMD_MULTIPLE | m_PENT4
1301 | m_NOCONA | m_CORE2 | m_GENERIC,
1303 /* X86_TUNE_ZERO_EXTEND_WITH_AND */
1306 /* X86_TUNE_UNROLL_STRLEN */
1307 m_486 | m_PENT | m_ATOM | m_PPRO | m_AMD_MULTIPLE | m_K6
1308 | m_CORE2 | m_GENERIC,
1310 /* X86_TUNE_DEEP_BRANCH_PREDICTION */
1311 m_ATOM | m_PPRO | m_K6_GEODE | m_AMD_MULTIPLE | m_PENT4 | m_GENERIC,
1313 /* X86_TUNE_BRANCH_PREDICTION_HINTS: Branch hints were put in P4 based
1314 on simulation result. But after P4 was made, no performance benefit
1315 was observed with branch hints. It also increases the code size.
1316 As a result, icc never generates branch hints. */
1319 /* X86_TUNE_DOUBLE_WITH_ADD */
1322 /* X86_TUNE_USE_SAHF */
1323 m_ATOM | m_PPRO | m_K6_GEODE | m_K8 | m_AMDFAM10 | m_PENT4
1324 | m_NOCONA | m_CORE2 | m_GENERIC,
1326 /* X86_TUNE_MOVX: Enable to zero extend integer registers to avoid
1327 partial dependencies. */
1328 m_AMD_MULTIPLE | m_ATOM | m_PPRO | m_PENT4 | m_NOCONA
1329 | m_CORE2 | m_GENERIC | m_GEODE /* m_386 | m_K6 */,
1331 /* X86_TUNE_PARTIAL_REG_STALL: We probably ought to watch for partial
1332 register stalls on Generic32 compilation setting as well. However
1333 in current implementation the partial register stalls are not eliminated
1334 very well - they can be introduced via subregs synthesized by combine
1335 and can happen in caller/callee saving sequences. Because this option
1336 pays back little on PPro based chips and is in conflict with partial reg
1337 dependencies used by Athlon/P4 based chips, it is better to leave it off
1338 for generic32 for now. */
1341 /* X86_TUNE_PARTIAL_FLAG_REG_STALL */
1342 m_CORE2 | m_GENERIC,
1344 /* X86_TUNE_USE_HIMODE_FIOP */
1345 m_386 | m_486 | m_K6_GEODE,
1347 /* X86_TUNE_USE_SIMODE_FIOP */
1348 ~(m_PPRO | m_AMD_MULTIPLE | m_PENT | m_ATOM | m_CORE2 | m_GENERIC),
1350 /* X86_TUNE_USE_MOV0 */
1353 /* X86_TUNE_USE_CLTD */
1354 ~(m_PENT | m_ATOM | m_K6 | m_CORE2 | m_GENERIC),
1356 /* X86_TUNE_USE_XCHGB: Use xchgb %rh,%rl instead of rolw/rorw $8,rx. */
1359 /* X86_TUNE_SPLIT_LONG_MOVES */
1362 /* X86_TUNE_READ_MODIFY_WRITE */
1365 /* X86_TUNE_READ_MODIFY */
1368 /* X86_TUNE_PROMOTE_QIMODE */
1369 m_K6_GEODE | m_PENT | m_ATOM | m_386 | m_486 | m_AMD_MULTIPLE
1370 | m_CORE2 | m_GENERIC /* | m_PENT4 ? */,
1372 /* X86_TUNE_FAST_PREFIX */
1373 ~(m_PENT | m_486 | m_386),
1375 /* X86_TUNE_SINGLE_STRINGOP */
1376 m_386 | m_PENT4 | m_NOCONA,
1378 /* X86_TUNE_QIMODE_MATH */
1381 /* X86_TUNE_HIMODE_MATH: On PPro this flag is meant to avoid partial
1382 register stalls. Just like X86_TUNE_PARTIAL_REG_STALL this option
1383 might be considered for Generic32 if our scheme for avoiding partial
1384 stalls was more effective. */
1387 /* X86_TUNE_PROMOTE_QI_REGS */
1390 /* X86_TUNE_PROMOTE_HI_REGS */
1393 /* X86_TUNE_ADD_ESP_4: Enable if add/sub is preferred over 1/2 push/pop. */
1394 m_ATOM | m_AMD_MULTIPLE | m_K6_GEODE | m_PENT4 | m_NOCONA
1395 | m_CORE2 | m_GENERIC,
1397 /* X86_TUNE_ADD_ESP_8 */
1398 m_AMD_MULTIPLE | m_ATOM | m_PPRO | m_K6_GEODE | m_386
1399 | m_486 | m_PENT4 | m_NOCONA | m_CORE2 | m_GENERIC,
1401 /* X86_TUNE_SUB_ESP_4 */
1402 m_AMD_MULTIPLE | m_ATOM | m_PPRO | m_PENT4 | m_NOCONA | m_CORE2
1405 /* X86_TUNE_SUB_ESP_8 */
1406 m_AMD_MULTIPLE | m_ATOM | m_PPRO | m_386 | m_486
1407 | m_PENT4 | m_NOCONA | m_CORE2 | m_GENERIC,
1409 /* X86_TUNE_INTEGER_DFMODE_MOVES: Enable if integer moves are preferred
1410 for DFmode copies */
1411 ~(m_AMD_MULTIPLE | m_ATOM | m_PENT4 | m_NOCONA | m_PPRO | m_CORE2
1412 | m_GENERIC | m_GEODE),
1414 /* X86_TUNE_PARTIAL_REG_DEPENDENCY */
1415 m_AMD_MULTIPLE | m_ATOM | m_PENT4 | m_NOCONA | m_CORE2 | m_GENERIC,
1417 /* X86_TUNE_SSE_PARTIAL_REG_DEPENDENCY: In the Generic model we have a
1418 conflict here in between PPro/Pentium4 based chips that thread 128bit
1419 SSE registers as single units versus K8 based chips that divide SSE
1420 registers to two 64bit halves. This knob promotes all store destinations
1421 to be 128bit to allow register renaming on 128bit SSE units, but usually
1422 results in one extra microop on 64bit SSE units. Experimental results
1423 shows that disabling this option on P4 brings over 20% SPECfp regression,
1424 while enabling it on K8 brings roughly 2.4% regression that can be partly
1425 masked by careful scheduling of moves. */
1426 m_ATOM | m_PENT4 | m_NOCONA | m_PPRO | m_CORE2 | m_GENERIC
1429 /* X86_TUNE_SSE_UNALIGNED_MOVE_OPTIMAL */
1432 /* X86_TUNE_SSE_SPLIT_REGS: Set for machines where the type and dependencies
1433 are resolved on SSE register parts instead of whole registers, so we may
1434 maintain just lower part of scalar values in proper format leaving the
1435 upper part undefined. */
1438 /* X86_TUNE_SSE_TYPELESS_STORES */
1441 /* X86_TUNE_SSE_LOAD0_BY_PXOR */
1442 m_PPRO | m_PENT4 | m_NOCONA,
1444 /* X86_TUNE_MEMORY_MISMATCH_STALL */
1445 m_AMD_MULTIPLE | m_ATOM | m_PENT4 | m_NOCONA | m_CORE2 | m_GENERIC,
1447 /* X86_TUNE_PROLOGUE_USING_MOVE */
1448 m_ATHLON_K8 | m_ATOM | m_PPRO | m_CORE2 | m_GENERIC,
1450 /* X86_TUNE_EPILOGUE_USING_MOVE */
1451 m_ATHLON_K8 | m_ATOM | m_PPRO | m_CORE2 | m_GENERIC,
1453 /* X86_TUNE_SHIFT1 */
1456 /* X86_TUNE_USE_FFREEP */
1459 /* X86_TUNE_INTER_UNIT_MOVES */
1460 ~(m_AMD_MULTIPLE | m_ATOM | m_GENERIC),
1462 /* X86_TUNE_INTER_UNIT_CONVERSIONS */
1465 /* X86_TUNE_FOUR_JUMP_LIMIT: Some CPU cores are not able to predict more
1466 than 4 branch instructions in the 16 byte window. */
1467 m_ATOM | m_PPRO | m_AMD_MULTIPLE | m_PENT4 | m_NOCONA | m_CORE2
1470 /* X86_TUNE_SCHEDULE */
1471 m_PPRO | m_AMD_MULTIPLE | m_K6_GEODE | m_PENT | m_ATOM | m_CORE2
1474 /* X86_TUNE_USE_BT */
1475 m_AMD_MULTIPLE | m_ATOM | m_CORE2 | m_GENERIC,
1477 /* X86_TUNE_USE_INCDEC */
1478 ~(m_PENT4 | m_NOCONA | m_GENERIC | m_ATOM),
1480 /* X86_TUNE_PAD_RETURNS */
1481 m_AMD_MULTIPLE | m_CORE2 | m_GENERIC,
1483 /* X86_TUNE_EXT_80387_CONSTANTS */
1484 m_K6_GEODE | m_ATHLON_K8 | m_ATOM | m_PENT4 | m_NOCONA | m_PPRO
1485 | m_CORE2 | m_GENERIC,
1487 /* X86_TUNE_SHORTEN_X87_SSE */
1490 /* X86_TUNE_AVOID_VECTOR_DECODE */
1493 /* X86_TUNE_PROMOTE_HIMODE_IMUL: Modern CPUs have same latency for HImode
1494 and SImode multiply, but 386 and 486 do HImode multiply faster. */
1497 /* X86_TUNE_SLOW_IMUL_IMM32_MEM: Imul of 32-bit constant and memory is
1498 vector path on AMD machines. */
1499 m_K8 | m_GENERIC64 | m_AMDFAM10,
1501 /* X86_TUNE_SLOW_IMUL_IMM8: Imul of 8-bit constant is vector path on AMD
1503 m_K8 | m_GENERIC64 | m_AMDFAM10,
1505 /* X86_TUNE_MOVE_M1_VIA_OR: On pentiums, it is faster to load -1 via OR
1509 /* X86_TUNE_NOT_UNPAIRABLE: NOT is not pairable on Pentium, while XOR is,
1510 but one byte longer. */
1513 /* X86_TUNE_NOT_VECTORMODE: On AMD K6, NOT is vector decoded with memory
1514 operand that cannot be represented using a modRM byte. The XOR
1515 replacement is long decoded, so this split helps here as well. */
1518 /* X86_TUNE_USE_VECTOR_FP_CONVERTS: Prefer vector packed SSE conversion
1520 m_AMDFAM10 | m_GENERIC,
1522 /* X86_TUNE_USE_VECTOR_CONVERTS: Prefer vector packed SSE conversion
1523 from integer to FP. */
1526 /* X86_TUNE_FUSE_CMP_AND_BRANCH: Fuse a compare or test instruction
1527 with a subsequent conditional jump instruction into a single
1528 compare-and-branch uop. */
1531 /* X86_TUNE_OPT_AGU: Optimize for Address Generation Unit. This flag
1532 will impact LEA instruction selection. */
1536 /* Feature tests against the various architecture variations. */
1537 unsigned char ix86_arch_features[X86_ARCH_LAST];
1539 /* Feature tests against the various architecture variations, used to create
1540 ix86_arch_features based on the processor mask. */
1541 static unsigned int initial_ix86_arch_features[X86_ARCH_LAST] = {
1542 /* X86_ARCH_CMOVE: Conditional move was added for pentiumpro. */
1543 ~(m_386 | m_486 | m_PENT | m_K6),
1545 /* X86_ARCH_CMPXCHG: Compare and exchange was added for 80486. */
1548 /* X86_ARCH_CMPXCHG8B: Compare and exchange 8 bytes was added for pentium. */
1551 /* X86_ARCH_XADD: Exchange and add was added for 80486. */
1554 /* X86_ARCH_BSWAP: Byteswap was added for 80486. */
1558 static const unsigned int x86_accumulate_outgoing_args
1559 = m_AMD_MULTIPLE | m_ATOM | m_PENT4 | m_NOCONA | m_PPRO | m_CORE2
1562 static const unsigned int x86_arch_always_fancy_math_387
1563 = m_PENT | m_ATOM | m_PPRO | m_AMD_MULTIPLE | m_PENT4
1564 | m_NOCONA | m_CORE2 | m_GENERIC;
1566 static enum stringop_alg stringop_alg = no_stringop;
1568 /* In case the average insn count for single function invocation is
1569 lower than this constant, emit fast (but longer) prologue and
1571 #define FAST_PROLOGUE_INSN_COUNT 20
1573 /* Names for 8 (low), 8 (high), and 16-bit registers, respectively. */
1574 static const char *const qi_reg_name[] = QI_REGISTER_NAMES;
1575 static const char *const qi_high_reg_name[] = QI_HIGH_REGISTER_NAMES;
1576 static const char *const hi_reg_name[] = HI_REGISTER_NAMES;
1578 /* Array of the smallest class containing reg number REGNO, indexed by
1579 REGNO. Used by REGNO_REG_CLASS in i386.h. */
1581 enum reg_class const regclass_map[FIRST_PSEUDO_REGISTER] =
1583 /* ax, dx, cx, bx */
1584 AREG, DREG, CREG, BREG,
1585 /* si, di, bp, sp */
1586 SIREG, DIREG, NON_Q_REGS, NON_Q_REGS,
1588 FP_TOP_REG, FP_SECOND_REG, FLOAT_REGS, FLOAT_REGS,
1589 FLOAT_REGS, FLOAT_REGS, FLOAT_REGS, FLOAT_REGS,
1592 /* flags, fpsr, fpcr, frame */
1593 NO_REGS, NO_REGS, NO_REGS, NON_Q_REGS,
1595 SSE_FIRST_REG, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
1598 MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS,
1601 NON_Q_REGS, NON_Q_REGS, NON_Q_REGS, NON_Q_REGS,
1602 NON_Q_REGS, NON_Q_REGS, NON_Q_REGS, NON_Q_REGS,
1603 /* SSE REX registers */
1604 SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS,
1608 /* The "default" register map used in 32bit mode. */
1610 int const dbx_register_map[FIRST_PSEUDO_REGISTER] =
1612 0, 2, 1, 3, 6, 7, 4, 5, /* general regs */
1613 12, 13, 14, 15, 16, 17, 18, 19, /* fp regs */
1614 -1, -1, -1, -1, -1, /* arg, flags, fpsr, fpcr, frame */
1615 21, 22, 23, 24, 25, 26, 27, 28, /* SSE */
1616 29, 30, 31, 32, 33, 34, 35, 36, /* MMX */
1617 -1, -1, -1, -1, -1, -1, -1, -1, /* extended integer registers */
1618 -1, -1, -1, -1, -1, -1, -1, -1, /* extended SSE registers */
1621 /* The "default" register map used in 64bit mode. */
1623 int const dbx64_register_map[FIRST_PSEUDO_REGISTER] =
1625 0, 1, 2, 3, 4, 5, 6, 7, /* general regs */
1626 33, 34, 35, 36, 37, 38, 39, 40, /* fp regs */
1627 -1, -1, -1, -1, -1, /* arg, flags, fpsr, fpcr, frame */
1628 17, 18, 19, 20, 21, 22, 23, 24, /* SSE */
1629 41, 42, 43, 44, 45, 46, 47, 48, /* MMX */
1630 8,9,10,11,12,13,14,15, /* extended integer registers */
1631 25, 26, 27, 28, 29, 30, 31, 32, /* extended SSE registers */
1634 /* Define the register numbers to be used in Dwarf debugging information.
1635 The SVR4 reference port C compiler uses the following register numbers
1636 in its Dwarf output code:
1637 0 for %eax (gcc regno = 0)
1638 1 for %ecx (gcc regno = 2)
1639 2 for %edx (gcc regno = 1)
1640 3 for %ebx (gcc regno = 3)
1641 4 for %esp (gcc regno = 7)
1642 5 for %ebp (gcc regno = 6)
1643 6 for %esi (gcc regno = 4)
1644 7 for %edi (gcc regno = 5)
1645 The following three DWARF register numbers are never generated by
1646 the SVR4 C compiler or by the GNU compilers, but SDB on x86/svr4
1647 believes these numbers have these meanings.
1648 8 for %eip (no gcc equivalent)
1649 9 for %eflags (gcc regno = 17)
1650 10 for %trapno (no gcc equivalent)
1651 It is not at all clear how we should number the FP stack registers
1652 for the x86 architecture. If the version of SDB on x86/svr4 were
1653 a bit less brain dead with respect to floating-point then we would
1654 have a precedent to follow with respect to DWARF register numbers
1655 for x86 FP registers, but the SDB on x86/svr4 is so completely
1656 broken with respect to FP registers that it is hardly worth thinking
1657 of it as something to strive for compatibility with.
1658 The version of x86/svr4 SDB I have at the moment does (partially)
1659 seem to believe that DWARF register number 11 is associated with
1660 the x86 register %st(0), but that's about all. Higher DWARF
1661 register numbers don't seem to be associated with anything in
1662 particular, and even for DWARF regno 11, SDB only seems to under-
1663 stand that it should say that a variable lives in %st(0) (when
1664 asked via an `=' command) if we said it was in DWARF regno 11,
1665 but SDB still prints garbage when asked for the value of the
1666 variable in question (via a `/' command).
1667 (Also note that the labels SDB prints for various FP stack regs
1668 when doing an `x' command are all wrong.)
1669 Note that these problems generally don't affect the native SVR4
1670 C compiler because it doesn't allow the use of -O with -g and
1671 because when it is *not* optimizing, it allocates a memory
1672 location for each floating-point variable, and the memory
1673 location is what gets described in the DWARF AT_location
1674 attribute for the variable in question.
1675 Regardless of the severe mental illness of the x86/svr4 SDB, we
1676 do something sensible here and we use the following DWARF
1677 register numbers. Note that these are all stack-top-relative
1679 11 for %st(0) (gcc regno = 8)
1680 12 for %st(1) (gcc regno = 9)
1681 13 for %st(2) (gcc regno = 10)
1682 14 for %st(3) (gcc regno = 11)
1683 15 for %st(4) (gcc regno = 12)
1684 16 for %st(5) (gcc regno = 13)
1685 17 for %st(6) (gcc regno = 14)
1686 18 for %st(7) (gcc regno = 15)
1688 int const svr4_dbx_register_map[FIRST_PSEUDO_REGISTER] =
1690 0, 2, 1, 3, 6, 7, 5, 4, /* general regs */
1691 11, 12, 13, 14, 15, 16, 17, 18, /* fp regs */
1692 -1, 9, -1, -1, -1, /* arg, flags, fpsr, fpcr, frame */
1693 21, 22, 23, 24, 25, 26, 27, 28, /* SSE registers */
1694 29, 30, 31, 32, 33, 34, 35, 36, /* MMX registers */
1695 -1, -1, -1, -1, -1, -1, -1, -1, /* extended integer registers */
1696 -1, -1, -1, -1, -1, -1, -1, -1, /* extended SSE registers */
1699 /* Test and compare insns in i386.md store the information needed to
1700 generate branch and scc insns here. */
1702 rtx ix86_compare_op0 = NULL_RTX;
1703 rtx ix86_compare_op1 = NULL_RTX;
1705 /* Define parameter passing and return registers. */
1707 static int const x86_64_int_parameter_registers[6] =
1709 DI_REG, SI_REG, DX_REG, CX_REG, R8_REG, R9_REG
1712 static int const x86_64_ms_abi_int_parameter_registers[4] =
1714 CX_REG, DX_REG, R8_REG, R9_REG
1717 static int const x86_64_int_return_registers[4] =
1719 AX_REG, DX_REG, DI_REG, SI_REG
1722 /* Define the structure for the machine field in struct function. */
1724 struct GTY(()) stack_local_entry {
1725 unsigned short mode;
1728 struct stack_local_entry *next;
1731 /* Structure describing stack frame layout.
1732 Stack grows downward:
1738 saved frame pointer if frame_pointer_needed
1739 <- HARD_FRAME_POINTER
1748 [va_arg registers] (
1749 > to_allocate <- FRAME_POINTER
1761 HOST_WIDE_INT frame;
1763 int outgoing_arguments_size;
1766 HOST_WIDE_INT to_allocate;
1767 /* The offsets relative to ARG_POINTER. */
1768 HOST_WIDE_INT frame_pointer_offset;
1769 HOST_WIDE_INT hard_frame_pointer_offset;
1770 HOST_WIDE_INT stack_pointer_offset;
1772 /* When save_regs_using_mov is set, emit prologue using
1773 move instead of push instructions. */
1774 bool save_regs_using_mov;
1777 /* Code model option. */
1778 enum cmodel ix86_cmodel;
1780 enum asm_dialect ix86_asm_dialect = ASM_ATT;
1782 enum tls_dialect ix86_tls_dialect = TLS_DIALECT_GNU;
1784 /* Which unit we are generating floating point math for. */
1785 enum fpmath_unit ix86_fpmath;
1787 /* Which cpu are we scheduling for. */
1788 enum attr_cpu ix86_schedule;
1790 /* Which cpu are we optimizing for. */
1791 enum processor_type ix86_tune;
1793 /* Which instruction set architecture to use. */
1794 enum processor_type ix86_arch;
1796 /* true if sse prefetch instruction is not NOOP. */
1797 int x86_prefetch_sse;
1799 /* ix86_regparm_string as a number */
1800 static int ix86_regparm;
1802 /* -mstackrealign option */
1803 extern int ix86_force_align_arg_pointer;
1804 static const char ix86_force_align_arg_pointer_string[]
1805 = "force_align_arg_pointer";
1807 static rtx (*ix86_gen_leave) (void);
1808 static rtx (*ix86_gen_pop1) (rtx);
1809 static rtx (*ix86_gen_add3) (rtx, rtx, rtx);
1810 static rtx (*ix86_gen_sub3) (rtx, rtx, rtx);
1811 static rtx (*ix86_gen_sub3_carry) (rtx, rtx, rtx, rtx);
1812 static rtx (*ix86_gen_one_cmpl2) (rtx, rtx);
1813 static rtx (*ix86_gen_monitor) (rtx, rtx, rtx);
1814 static rtx (*ix86_gen_andsp) (rtx, rtx, rtx);
1816 /* Preferred alignment for stack boundary in bits. */
1817 unsigned int ix86_preferred_stack_boundary;
1819 /* Alignment for incoming stack boundary in bits specified at
1821 static unsigned int ix86_user_incoming_stack_boundary;
1823 /* Default alignment for incoming stack boundary in bits. */
1824 static unsigned int ix86_default_incoming_stack_boundary;
1826 /* Alignment for incoming stack boundary in bits. */
1827 unsigned int ix86_incoming_stack_boundary;
1829 /* The abi used by target. */
1830 enum calling_abi ix86_abi;
1832 /* Values 1-5: see jump.c */
1833 int ix86_branch_cost;
1835 /* Calling abi specific va_list type nodes. */
1836 static GTY(()) tree sysv_va_list_type_node;
1837 static GTY(()) tree ms_va_list_type_node;
1839 /* Variables which are this size or smaller are put in the data/bss
1840 or ldata/lbss sections. */
1842 int ix86_section_threshold = 65536;
1844 /* Prefix built by ASM_GENERATE_INTERNAL_LABEL. */
1845 char internal_label_prefix[16];
1846 int internal_label_prefix_len;
1848 /* Fence to use after loop using movnt. */
1851 /* Register class used for passing given 64bit part of the argument.
1852 These represent classes as documented by the PS ABI, with the exception
1853 of SSESF, SSEDF classes, that are basically SSE class, just gcc will
1854 use SF or DFmode move instead of DImode to avoid reformatting penalties.
1856 Similarly we play games with INTEGERSI_CLASS to use cheaper SImode moves
1857 whenever possible (upper half does contain padding). */
1858 enum x86_64_reg_class
1861 X86_64_INTEGER_CLASS,
1862 X86_64_INTEGERSI_CLASS,
1869 X86_64_COMPLEX_X87_CLASS,
1873 #define MAX_CLASSES 4
1875 /* Table of constants used by fldpi, fldln2, etc.... */
1876 static REAL_VALUE_TYPE ext_80387_constants_table [5];
1877 static bool ext_80387_constants_init = 0;
1880 static struct machine_function * ix86_init_machine_status (void);
1881 static rtx ix86_function_value (const_tree, const_tree, bool);
1882 static int ix86_function_regparm (const_tree, const_tree);
1883 static void ix86_compute_frame_layout (struct ix86_frame *);
1884 static bool ix86_expand_vector_init_one_nonzero (bool, enum machine_mode,
1886 static void ix86_add_new_builtins (int);
1888 enum ix86_function_specific_strings
1890 IX86_FUNCTION_SPECIFIC_ARCH,
1891 IX86_FUNCTION_SPECIFIC_TUNE,
1892 IX86_FUNCTION_SPECIFIC_FPMATH,
1893 IX86_FUNCTION_SPECIFIC_MAX
1896 static char *ix86_target_string (int, int, const char *, const char *,
1897 const char *, bool);
1898 static void ix86_debug_options (void) ATTRIBUTE_UNUSED;
1899 static void ix86_function_specific_save (struct cl_target_option *);
1900 static void ix86_function_specific_restore (struct cl_target_option *);
1901 static void ix86_function_specific_print (FILE *, int,
1902 struct cl_target_option *);
1903 static bool ix86_valid_target_attribute_p (tree, tree, tree, int);
1904 static bool ix86_valid_target_attribute_inner_p (tree, char *[]);
1905 static bool ix86_can_inline_p (tree, tree);
1906 static void ix86_set_current_function (tree);
1908 static enum calling_abi ix86_function_abi (const_tree);
1911 /* The svr4 ABI for the i386 says that records and unions are returned
1913 #ifndef DEFAULT_PCC_STRUCT_RETURN
1914 #define DEFAULT_PCC_STRUCT_RETURN 1
1917 /* Whether -mtune= or -march= were specified */
1918 static int ix86_tune_defaulted;
1919 static int ix86_arch_specified;
1921 /* Bit flags that specify the ISA we are compiling for. */
1922 int ix86_isa_flags = TARGET_64BIT_DEFAULT | TARGET_SUBTARGET_ISA_DEFAULT;
1924 /* A mask of ix86_isa_flags that includes bit X if X
1925 was set or cleared on the command line. */
1926 static int ix86_isa_flags_explicit;
1928 /* Define a set of ISAs which are available when a given ISA is
1929 enabled. MMX and SSE ISAs are handled separately. */
1931 #define OPTION_MASK_ISA_MMX_SET OPTION_MASK_ISA_MMX
1932 #define OPTION_MASK_ISA_3DNOW_SET \
1933 (OPTION_MASK_ISA_3DNOW | OPTION_MASK_ISA_MMX_SET)
1935 #define OPTION_MASK_ISA_SSE_SET OPTION_MASK_ISA_SSE
1936 #define OPTION_MASK_ISA_SSE2_SET \
1937 (OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_SSE_SET)
1938 #define OPTION_MASK_ISA_SSE3_SET \
1939 (OPTION_MASK_ISA_SSE3 | OPTION_MASK_ISA_SSE2_SET)
1940 #define OPTION_MASK_ISA_SSSE3_SET \
1941 (OPTION_MASK_ISA_SSSE3 | OPTION_MASK_ISA_SSE3_SET)
1942 #define OPTION_MASK_ISA_SSE4_1_SET \
1943 (OPTION_MASK_ISA_SSE4_1 | OPTION_MASK_ISA_SSSE3_SET)
1944 #define OPTION_MASK_ISA_SSE4_2_SET \
1945 (OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_SSE4_1_SET)
1946 #define OPTION_MASK_ISA_AVX_SET \
1947 (OPTION_MASK_ISA_AVX | OPTION_MASK_ISA_SSE4_2_SET)
1948 #define OPTION_MASK_ISA_FMA_SET \
1949 (OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_AVX_SET)
1951 /* SSE4 includes both SSE4.1 and SSE4.2. -msse4 should be the same
1953 #define OPTION_MASK_ISA_SSE4_SET OPTION_MASK_ISA_SSE4_2_SET
1955 #define OPTION_MASK_ISA_SSE4A_SET \
1956 (OPTION_MASK_ISA_SSE4A | OPTION_MASK_ISA_SSE3_SET)
1957 #define OPTION_MASK_ISA_SSE5_SET \
1958 (OPTION_MASK_ISA_SSE5 | OPTION_MASK_ISA_SSE4A_SET)
1960 /* AES and PCLMUL need SSE2 because they use xmm registers */
1961 #define OPTION_MASK_ISA_AES_SET \
1962 (OPTION_MASK_ISA_AES | OPTION_MASK_ISA_SSE2_SET)
1963 #define OPTION_MASK_ISA_PCLMUL_SET \
1964 (OPTION_MASK_ISA_PCLMUL | OPTION_MASK_ISA_SSE2_SET)
1966 #define OPTION_MASK_ISA_ABM_SET \
1967 (OPTION_MASK_ISA_ABM | OPTION_MASK_ISA_POPCNT)
1968 #define OPTION_MASK_ISA_POPCNT_SET OPTION_MASK_ISA_POPCNT
1969 #define OPTION_MASK_ISA_CX16_SET OPTION_MASK_ISA_CX16
1970 #define OPTION_MASK_ISA_SAHF_SET OPTION_MASK_ISA_SAHF
1972 /* Define a set of ISAs which aren't available when a given ISA is
1973 disabled. MMX and SSE ISAs are handled separately. */
1975 #define OPTION_MASK_ISA_MMX_UNSET \
1976 (OPTION_MASK_ISA_MMX | OPTION_MASK_ISA_3DNOW_UNSET)
1977 #define OPTION_MASK_ISA_3DNOW_UNSET \
1978 (OPTION_MASK_ISA_3DNOW | OPTION_MASK_ISA_3DNOW_A_UNSET)
1979 #define OPTION_MASK_ISA_3DNOW_A_UNSET OPTION_MASK_ISA_3DNOW_A
1981 #define OPTION_MASK_ISA_SSE_UNSET \
1982 (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_SSE2_UNSET)
1983 #define OPTION_MASK_ISA_SSE2_UNSET \
1984 (OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_SSE3_UNSET)
1985 #define OPTION_MASK_ISA_SSE3_UNSET \
1986 (OPTION_MASK_ISA_SSE3 \
1987 | OPTION_MASK_ISA_SSSE3_UNSET \
1988 | OPTION_MASK_ISA_SSE4A_UNSET )
1989 #define OPTION_MASK_ISA_SSSE3_UNSET \
1990 (OPTION_MASK_ISA_SSSE3 | OPTION_MASK_ISA_SSE4_1_UNSET)
1991 #define OPTION_MASK_ISA_SSE4_1_UNSET \
1992 (OPTION_MASK_ISA_SSE4_1 | OPTION_MASK_ISA_SSE4_2_UNSET)
1993 #define OPTION_MASK_ISA_SSE4_2_UNSET \
1994 (OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_AVX_UNSET )
1995 #define OPTION_MASK_ISA_AVX_UNSET \
1996 (OPTION_MASK_ISA_AVX | OPTION_MASK_ISA_FMA_UNSET)
1997 #define OPTION_MASK_ISA_FMA_UNSET OPTION_MASK_ISA_FMA
1999 /* SSE4 includes both SSE4.1 and SSE4.2. -mno-sse4 should the same
2001 #define OPTION_MASK_ISA_SSE4_UNSET OPTION_MASK_ISA_SSE4_1_UNSET
2003 #define OPTION_MASK_ISA_SSE4A_UNSET \
2004 (OPTION_MASK_ISA_SSE4A | OPTION_MASK_ISA_SSE5_UNSET)
2005 #define OPTION_MASK_ISA_SSE5_UNSET OPTION_MASK_ISA_SSE5
2006 #define OPTION_MASK_ISA_AES_UNSET OPTION_MASK_ISA_AES
2007 #define OPTION_MASK_ISA_PCLMUL_UNSET OPTION_MASK_ISA_PCLMUL
2008 #define OPTION_MASK_ISA_ABM_UNSET OPTION_MASK_ISA_ABM
2009 #define OPTION_MASK_ISA_POPCNT_UNSET OPTION_MASK_ISA_POPCNT
2010 #define OPTION_MASK_ISA_CX16_UNSET OPTION_MASK_ISA_CX16
2011 #define OPTION_MASK_ISA_SAHF_UNSET OPTION_MASK_ISA_SAHF
2013 /* Vectorization library interface and handlers. */
2014 tree (*ix86_veclib_handler)(enum built_in_function, tree, tree) = NULL;
2015 static tree ix86_veclibabi_svml (enum built_in_function, tree, tree);
2016 static tree ix86_veclibabi_acml (enum built_in_function, tree, tree);
2018 /* Processor target table, indexed by processor number */
2021 const struct processor_costs *cost; /* Processor costs */
2022 const int align_loop; /* Default alignments. */
2023 const int align_loop_max_skip;
2024 const int align_jump;
2025 const int align_jump_max_skip;
2026 const int align_func;
2029 static const struct ptt processor_target_table[PROCESSOR_max] =
2031 {&i386_cost, 4, 3, 4, 3, 4},
2032 {&i486_cost, 16, 15, 16, 15, 16},
2033 {&pentium_cost, 16, 7, 16, 7, 16},
2034 {&pentiumpro_cost, 16, 15, 16, 10, 16},
2035 {&geode_cost, 0, 0, 0, 0, 0},
2036 {&k6_cost, 32, 7, 32, 7, 32},
2037 {&athlon_cost, 16, 7, 16, 7, 16},
2038 {&pentium4_cost, 0, 0, 0, 0, 0},
2039 {&k8_cost, 16, 7, 16, 7, 16},
2040 {&nocona_cost, 0, 0, 0, 0, 0},
2041 {&core2_cost, 16, 10, 16, 10, 16},
2042 {&generic32_cost, 16, 7, 16, 7, 16},
2043 {&generic64_cost, 16, 10, 16, 10, 16},
2044 {&amdfam10_cost, 32, 24, 32, 7, 32},
2045 {&atom_cost, 16, 7, 16, 7, 16}
2048 static const char *const cpu_names[TARGET_CPU_DEFAULT_max] =
2074 /* Implement TARGET_HANDLE_OPTION. */
2077 ix86_handle_option (size_t code, const char *arg ATTRIBUTE_UNUSED, int value)
2084 ix86_isa_flags |= OPTION_MASK_ISA_MMX_SET;
2085 ix86_isa_flags_explicit |= OPTION_MASK_ISA_MMX_SET;
2089 ix86_isa_flags &= ~OPTION_MASK_ISA_MMX_UNSET;
2090 ix86_isa_flags_explicit |= OPTION_MASK_ISA_MMX_UNSET;
2097 ix86_isa_flags |= OPTION_MASK_ISA_3DNOW_SET;
2098 ix86_isa_flags_explicit |= OPTION_MASK_ISA_3DNOW_SET;
2102 ix86_isa_flags &= ~OPTION_MASK_ISA_3DNOW_UNSET;
2103 ix86_isa_flags_explicit |= OPTION_MASK_ISA_3DNOW_UNSET;
2113 ix86_isa_flags |= OPTION_MASK_ISA_SSE_SET;
2114 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE_SET;
2118 ix86_isa_flags &= ~OPTION_MASK_ISA_SSE_UNSET;
2119 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE_UNSET;
2126 ix86_isa_flags |= OPTION_MASK_ISA_SSE2_SET;
2127 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE2_SET;
2131 ix86_isa_flags &= ~OPTION_MASK_ISA_SSE2_UNSET;
2132 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE2_UNSET;
2139 ix86_isa_flags |= OPTION_MASK_ISA_SSE3_SET;
2140 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE3_SET;
2144 ix86_isa_flags &= ~OPTION_MASK_ISA_SSE3_UNSET;
2145 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE3_UNSET;
2152 ix86_isa_flags |= OPTION_MASK_ISA_SSSE3_SET;
2153 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSSE3_SET;
2157 ix86_isa_flags &= ~OPTION_MASK_ISA_SSSE3_UNSET;
2158 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSSE3_UNSET;
2165 ix86_isa_flags |= OPTION_MASK_ISA_SSE4_1_SET;
2166 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE4_1_SET;
2170 ix86_isa_flags &= ~OPTION_MASK_ISA_SSE4_1_UNSET;
2171 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE4_1_UNSET;
2178 ix86_isa_flags |= OPTION_MASK_ISA_SSE4_2_SET;
2179 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE4_2_SET;
2183 ix86_isa_flags &= ~OPTION_MASK_ISA_SSE4_2_UNSET;
2184 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE4_2_UNSET;
2191 ix86_isa_flags |= OPTION_MASK_ISA_AVX_SET;
2192 ix86_isa_flags_explicit |= OPTION_MASK_ISA_AVX_SET;
2196 ix86_isa_flags &= ~OPTION_MASK_ISA_AVX_UNSET;
2197 ix86_isa_flags_explicit |= OPTION_MASK_ISA_AVX_UNSET;
2204 ix86_isa_flags |= OPTION_MASK_ISA_FMA_SET;
2205 ix86_isa_flags_explicit |= OPTION_MASK_ISA_FMA_SET;
2209 ix86_isa_flags &= ~OPTION_MASK_ISA_FMA_UNSET;
2210 ix86_isa_flags_explicit |= OPTION_MASK_ISA_FMA_UNSET;
2215 ix86_isa_flags |= OPTION_MASK_ISA_SSE4_SET;
2216 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE4_SET;
2220 ix86_isa_flags &= ~OPTION_MASK_ISA_SSE4_UNSET;
2221 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE4_UNSET;
2227 ix86_isa_flags |= OPTION_MASK_ISA_SSE4A_SET;
2228 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE4A_SET;
2232 ix86_isa_flags &= ~OPTION_MASK_ISA_SSE4A_UNSET;
2233 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE4A_UNSET;
2240 ix86_isa_flags |= OPTION_MASK_ISA_SSE5_SET;
2241 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE5_SET;
2245 ix86_isa_flags &= ~OPTION_MASK_ISA_SSE5_UNSET;
2246 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SSE5_UNSET;
2253 ix86_isa_flags |= OPTION_MASK_ISA_ABM_SET;
2254 ix86_isa_flags_explicit |= OPTION_MASK_ISA_ABM_SET;
2258 ix86_isa_flags &= ~OPTION_MASK_ISA_ABM_UNSET;
2259 ix86_isa_flags_explicit |= OPTION_MASK_ISA_ABM_UNSET;
2266 ix86_isa_flags |= OPTION_MASK_ISA_POPCNT_SET;
2267 ix86_isa_flags_explicit |= OPTION_MASK_ISA_POPCNT_SET;
2271 ix86_isa_flags &= ~OPTION_MASK_ISA_POPCNT_UNSET;
2272 ix86_isa_flags_explicit |= OPTION_MASK_ISA_POPCNT_UNSET;
2279 ix86_isa_flags |= OPTION_MASK_ISA_SAHF_SET;
2280 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SAHF_SET;
2284 ix86_isa_flags &= ~OPTION_MASK_ISA_SAHF_UNSET;
2285 ix86_isa_flags_explicit |= OPTION_MASK_ISA_SAHF_UNSET;
2292 ix86_isa_flags |= OPTION_MASK_ISA_CX16_SET;
2293 ix86_isa_flags_explicit |= OPTION_MASK_ISA_CX16_SET;
2297 ix86_isa_flags &= ~OPTION_MASK_ISA_CX16_UNSET;
2298 ix86_isa_flags_explicit |= OPTION_MASK_ISA_CX16_UNSET;
2305 ix86_isa_flags |= OPTION_MASK_ISA_AES_SET;
2306 ix86_isa_flags_explicit |= OPTION_MASK_ISA_AES_SET;
2310 ix86_isa_flags &= ~OPTION_MASK_ISA_AES_UNSET;
2311 ix86_isa_flags_explicit |= OPTION_MASK_ISA_AES_UNSET;
2318 ix86_isa_flags |= OPTION_MASK_ISA_PCLMUL_SET;
2319 ix86_isa_flags_explicit |= OPTION_MASK_ISA_PCLMUL_SET;
2323 ix86_isa_flags &= ~OPTION_MASK_ISA_PCLMUL_UNSET;
2324 ix86_isa_flags_explicit |= OPTION_MASK_ISA_PCLMUL_UNSET;
2333 /* Return a string the documents the current -m options. The caller is
2334 responsible for freeing the string. */
2337 ix86_target_string (int isa, int flags, const char *arch, const char *tune,
2338 const char *fpmath, bool add_nl_p)
2340 struct ix86_target_opts
2342 const char *option; /* option string */
2343 int mask; /* isa mask options */
2346 /* This table is ordered so that options like -msse5 or -msse4.2 that imply
2347 preceding options while match those first. */
2348 static struct ix86_target_opts isa_opts[] =
2350 { "-m64", OPTION_MASK_ISA_64BIT },
2351 { "-msse5", OPTION_MASK_ISA_SSE5 },
2352 { "-msse4a", OPTION_MASK_ISA_SSE4A },
2353 { "-msse4.2", OPTION_MASK_ISA_SSE4_2 },
2354 { "-msse4.1", OPTION_MASK_ISA_SSE4_1 },
2355 { "-mssse3", OPTION_MASK_ISA_SSSE3 },
2356 { "-msse3", OPTION_MASK_ISA_SSE3 },
2357 { "-msse2", OPTION_MASK_ISA_SSE2 },
2358 { "-msse", OPTION_MASK_ISA_SSE },
2359 { "-m3dnow", OPTION_MASK_ISA_3DNOW },
2360 { "-m3dnowa", OPTION_MASK_ISA_3DNOW_A },
2361 { "-mmmx", OPTION_MASK_ISA_MMX },
2362 { "-mabm", OPTION_MASK_ISA_ABM },
2363 { "-mpopcnt", OPTION_MASK_ISA_POPCNT },
2364 { "-maes", OPTION_MASK_ISA_AES },
2365 { "-mpclmul", OPTION_MASK_ISA_PCLMUL },
2369 static struct ix86_target_opts flag_opts[] =
2371 { "-m128bit-long-double", MASK_128BIT_LONG_DOUBLE },
2372 { "-m80387", MASK_80387 },
2373 { "-maccumulate-outgoing-args", MASK_ACCUMULATE_OUTGOING_ARGS },
2374 { "-malign-double", MASK_ALIGN_DOUBLE },
2375 { "-mcld", MASK_CLD },
2376 { "-mfp-ret-in-387", MASK_FLOAT_RETURNS },
2377 { "-mieee-fp", MASK_IEEE_FP },
2378 { "-minline-all-stringops", MASK_INLINE_ALL_STRINGOPS },
2379 { "-minline-stringops-dynamically", MASK_INLINE_STRINGOPS_DYNAMICALLY },
2380 { "-mms-bitfields", MASK_MS_BITFIELD_LAYOUT },
2381 { "-mno-align-stringops", MASK_NO_ALIGN_STRINGOPS },
2382 { "-mno-fancy-math-387", MASK_NO_FANCY_MATH_387 },
2383 { "-mno-fused-madd", MASK_NO_FUSED_MADD },
2384 { "-mno-push-args", MASK_NO_PUSH_ARGS },
2385 { "-mno-red-zone", MASK_NO_RED_ZONE },
2386 { "-momit-leaf-frame-pointer", MASK_OMIT_LEAF_FRAME_POINTER },
2387 { "-mrecip", MASK_RECIP },
2388 { "-mrtd", MASK_RTD },
2389 { "-msseregparm", MASK_SSEREGPARM },
2390 { "-mstack-arg-probe", MASK_STACK_PROBE },
2391 { "-mtls-direct-seg-refs", MASK_TLS_DIRECT_SEG_REFS },
2394 const char *opts[ARRAY_SIZE (isa_opts) + ARRAY_SIZE (flag_opts) + 6][2];
2397 char target_other[40];
2406 memset (opts, '\0', sizeof (opts));
2408 /* Add -march= option. */
2411 opts[num][0] = "-march=";
2412 opts[num++][1] = arch;
2415 /* Add -mtune= option. */
2418 opts[num][0] = "-mtune=";
2419 opts[num++][1] = tune;
2422 /* Pick out the options in isa options. */
2423 for (i = 0; i < ARRAY_SIZE (isa_opts); i++)
2425 if ((isa & isa_opts[i].mask) != 0)
2427 opts[num++][0] = isa_opts[i].option;
2428 isa &= ~ isa_opts[i].mask;
2432 if (isa && add_nl_p)
2434 opts[num++][0] = isa_other;
2435 sprintf (isa_other, "(other isa: 0x%x)", isa);
2438 /* Add flag options. */
2439 for (i = 0; i < ARRAY_SIZE (flag_opts); i++)
2441 if ((flags & flag_opts[i].mask) != 0)
2443 opts[num++][0] = flag_opts[i].option;
2444 flags &= ~ flag_opts[i].mask;
2448 if (flags && add_nl_p)
2450 opts[num++][0] = target_other;
2451 sprintf (target_other, "(other flags: 0x%x)", isa);
2454 /* Add -fpmath= option. */
2457 opts[num][0] = "-mfpmath=";
2458 opts[num++][1] = fpmath;
2465 gcc_assert (num < ARRAY_SIZE (opts));
2467 /* Size the string. */
2469 sep_len = (add_nl_p) ? 3 : 1;
2470 for (i = 0; i < num; i++)
2473 for (j = 0; j < 2; j++)
2475 len += strlen (opts[i][j]);
2478 /* Build the string. */
2479 ret = ptr = (char *) xmalloc (len);
2482 for (i = 0; i < num; i++)
2486 for (j = 0; j < 2; j++)
2487 len2[j] = (opts[i][j]) ? strlen (opts[i][j]) : 0;
2494 if (add_nl_p && line_len + len2[0] + len2[1] > 70)
2502 for (j = 0; j < 2; j++)
2505 memcpy (ptr, opts[i][j], len2[j]);
2507 line_len += len2[j];
2512 gcc_assert (ret + len >= ptr);
2517 /* Function that is callable from the debugger to print the current
2520 ix86_debug_options (void)
2522 char *opts = ix86_target_string (ix86_isa_flags, target_flags,
2523 ix86_arch_string, ix86_tune_string,
2524 ix86_fpmath_string, true);
2528 fprintf (stderr, "%s\n\n", opts);
2532 fprintf (stderr, "<no options>\n\n");
2537 /* Sometimes certain combinations of command options do not make
2538 sense on a particular target machine. You can define a macro
2539 `OVERRIDE_OPTIONS' to take account of this. This macro, if
2540 defined, is executed once just after all the command options have
2543 Don't use this macro to turn on various extra optimizations for
2544 `-O'. That is what `OPTIMIZATION_OPTIONS' is for. */
2547 override_options (bool main_args_p)
2550 unsigned int ix86_arch_mask, ix86_tune_mask;
2555 /* Comes from final.c -- no real reason to change it. */
2556 #define MAX_CODE_ALIGN 16
2564 PTA_PREFETCH_SSE = 1 << 4,
2566 PTA_3DNOW_A = 1 << 6,
2570 PTA_POPCNT = 1 << 10,
2572 PTA_SSE4A = 1 << 12,
2573 PTA_NO_SAHF = 1 << 13,
2574 PTA_SSE4_1 = 1 << 14,
2575 PTA_SSE4_2 = 1 << 15,
2578 PTA_PCLMUL = 1 << 18,
2585 const char *const name; /* processor name or nickname. */
2586 const enum processor_type processor;
2587 const enum attr_cpu schedule;
2588 const unsigned /*enum pta_flags*/ flags;
2590 const processor_alias_table[] =
2592 {"i386", PROCESSOR_I386, CPU_NONE, 0},
2593 {"i486", PROCESSOR_I486, CPU_NONE, 0},
2594 {"i586", PROCESSOR_PENTIUM, CPU_PENTIUM, 0},
2595 {"pentium", PROCESSOR_PENTIUM, CPU_PENTIUM, 0},
2596 {"pentium-mmx", PROCESSOR_PENTIUM, CPU_PENTIUM, PTA_MMX},
2597 {"winchip-c6", PROCESSOR_I486, CPU_NONE, PTA_MMX},
2598 {"winchip2", PROCESSOR_I486, CPU_NONE, PTA_MMX | PTA_3DNOW},
2599 {"c3", PROCESSOR_I486, CPU_NONE, PTA_MMX | PTA_3DNOW},
2600 {"c3-2", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, PTA_MMX | PTA_SSE},
2601 {"i686", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, 0},
2602 {"pentiumpro", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, 0},
2603 {"pentium2", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, PTA_MMX},
2604 {"pentium3", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
2606 {"pentium3m", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
2608 {"pentium-m", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO,
2609 PTA_MMX | PTA_SSE | PTA_SSE2},
2610 {"pentium4", PROCESSOR_PENTIUM4, CPU_NONE,
2611 PTA_MMX |PTA_SSE | PTA_SSE2},
2612 {"pentium4m", PROCESSOR_PENTIUM4, CPU_NONE,
2613 PTA_MMX | PTA_SSE | PTA_SSE2},
2614 {"prescott", PROCESSOR_NOCONA, CPU_NONE,
2615 PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3},
2616 {"nocona", PROCESSOR_NOCONA, CPU_NONE,
2617 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
2618 | PTA_CX16 | PTA_NO_SAHF},
2619 {"core2", PROCESSOR_CORE2, CPU_CORE2,
2620 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
2621 | PTA_SSSE3 | PTA_CX16},
2622 {"atom", PROCESSOR_ATOM, CPU_ATOM,
2623 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3
2624 | PTA_SSSE3 | PTA_CX16},
2625 {"geode", PROCESSOR_GEODE, CPU_GEODE,
2626 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A |PTA_PREFETCH_SSE},
2627 {"k6", PROCESSOR_K6, CPU_K6, PTA_MMX},
2628 {"k6-2", PROCESSOR_K6, CPU_K6, PTA_MMX | PTA_3DNOW},
2629 {"k6-3", PROCESSOR_K6, CPU_K6, PTA_MMX | PTA_3DNOW},
2630 {"athlon", PROCESSOR_ATHLON, CPU_ATHLON,
2631 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_PREFETCH_SSE},
2632 {"athlon-tbird", PROCESSOR_ATHLON, CPU_ATHLON,
2633 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_PREFETCH_SSE},
2634 {"athlon-4", PROCESSOR_ATHLON, CPU_ATHLON,
2635 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE},
2636 {"athlon-xp", PROCESSOR_ATHLON, CPU_ATHLON,
2637 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE},
2638 {"athlon-mp", PROCESSOR_ATHLON, CPU_ATHLON,
2639 PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE},
2640 {"x86-64", PROCESSOR_K8, CPU_K8,
2641 PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_NO_SAHF},
2642 {"k8", PROCESSOR_K8, CPU_K8,
2643 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
2644 | PTA_SSE2 | PTA_NO_SAHF},
2645 {"k8-sse3", PROCESSOR_K8, CPU_K8,
2646 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
2647 | PTA_SSE2 | PTA_SSE3 | PTA_NO_SAHF},
2648 {"opteron", PROCESSOR_K8, CPU_K8,
2649 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
2650 | PTA_SSE2 | PTA_NO_SAHF},
2651 {"opteron-sse3", PROCESSOR_K8, CPU_K8,
2652 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
2653 | PTA_SSE2 | PTA_SSE3 | PTA_NO_SAHF},
2654 {"athlon64", PROCESSOR_K8, CPU_K8,
2655 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
2656 | PTA_SSE2 | PTA_NO_SAHF},
2657 {"athlon64-sse3", PROCESSOR_K8, CPU_K8,
2658 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
2659 | PTA_SSE2 | PTA_SSE3 | PTA_NO_SAHF},
2660 {"athlon-fx", PROCESSOR_K8, CPU_K8,
2661 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
2662 | PTA_SSE2 | PTA_NO_SAHF},
2663 {"amdfam10", PROCESSOR_AMDFAM10, CPU_AMDFAM10,
2664 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
2665 | PTA_SSE2 | PTA_SSE3 | PTA_SSE4A | PTA_CX16 | PTA_ABM},
2666 {"barcelona", PROCESSOR_AMDFAM10, CPU_AMDFAM10,
2667 PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE
2668 | PTA_SSE2 | PTA_SSE3 | PTA_SSE4A | PTA_CX16 | PTA_ABM},
2669 {"generic32", PROCESSOR_GENERIC32, CPU_PENTIUMPRO,
2670 0 /* flags are only used for -march switch. */ },
2671 {"generic64", PROCESSOR_GENERIC64, CPU_GENERIC64,
2672 PTA_64BIT /* flags are only used for -march switch. */ },
2675 int const pta_size = ARRAY_SIZE (processor_alias_table);
2677 /* Set up prefix/suffix so the error messages refer to either the command
2678 line argument, or the attribute(target). */
2687 prefix = "option(\"";
2692 #ifdef SUBTARGET_OVERRIDE_OPTIONS
2693 SUBTARGET_OVERRIDE_OPTIONS;
2696 #ifdef SUBSUBTARGET_OVERRIDE_OPTIONS
2697 SUBSUBTARGET_OVERRIDE_OPTIONS;
2700 /* -fPIC is the default for x86_64. */
2701 if (TARGET_MACHO && TARGET_64BIT)
2704 /* Set the default values for switches whose default depends on TARGET_64BIT
2705 in case they weren't overwritten by command line options. */
2708 /* Mach-O doesn't support omitting the frame pointer for now. */
2709 if (flag_omit_frame_pointer == 2)
2710 flag_omit_frame_pointer = (TARGET_MACHO ? 0 : 1);
2711 if (flag_asynchronous_unwind_tables == 2)
2712 flag_asynchronous_unwind_tables = 1;
2713 if (flag_pcc_struct_return == 2)
2714 flag_pcc_struct_return = 0;
2718 if (flag_omit_frame_pointer == 2)
2719 flag_omit_frame_pointer = 0;
2720 if (flag_asynchronous_unwind_tables == 2)
2721 flag_asynchronous_unwind_tables = 0;
2722 if (flag_pcc_struct_return == 2)
2723 flag_pcc_struct_return = DEFAULT_PCC_STRUCT_RETURN;
2726 /* Need to check -mtune=generic first. */
2727 if (ix86_tune_string)
2729 if (!strcmp (ix86_tune_string, "generic")
2730 || !strcmp (ix86_tune_string, "i686")
2731 /* As special support for cross compilers we read -mtune=native
2732 as -mtune=generic. With native compilers we won't see the
2733 -mtune=native, as it was changed by the driver. */
2734 || !strcmp (ix86_tune_string, "native"))
2737 ix86_tune_string = "generic64";
2739 ix86_tune_string = "generic32";
2741 /* If this call is for setting the option attribute, allow the
2742 generic32/generic64 that was previously set. */
2743 else if (!main_args_p
2744 && (!strcmp (ix86_tune_string, "generic32")
2745 || !strcmp (ix86_tune_string, "generic64")))
2747 else if (!strncmp (ix86_tune_string, "generic", 7))
2748 error ("bad value (%s) for %stune=%s %s",
2749 ix86_tune_string, prefix, suffix, sw);
2753 if (ix86_arch_string)
2754 ix86_tune_string = ix86_arch_string;
2755 if (!ix86_tune_string)
2757 ix86_tune_string = cpu_names[TARGET_CPU_DEFAULT];
2758 ix86_tune_defaulted = 1;
2761 /* ix86_tune_string is set to ix86_arch_string or defaulted. We
2762 need to use a sensible tune option. */
2763 if (!strcmp (ix86_tune_string, "generic")
2764 || !strcmp (ix86_tune_string, "x86-64")
2765 || !strcmp (ix86_tune_string, "i686"))
2768 ix86_tune_string = "generic64";
2770 ix86_tune_string = "generic32";
2773 if (ix86_stringop_string)
2775 if (!strcmp (ix86_stringop_string, "rep_byte"))
2776 stringop_alg = rep_prefix_1_byte;
2777 else if (!strcmp (ix86_stringop_string, "libcall"))
2778 stringop_alg = libcall;
2779 else if (!strcmp (ix86_stringop_string, "rep_4byte"))
2780 stringop_alg = rep_prefix_4_byte;
2781 else if (!strcmp (ix86_stringop_string, "rep_8byte")
2783 /* rep; movq isn't available in 32-bit code. */
2784 stringop_alg = rep_prefix_8_byte;
2785 else if (!strcmp (ix86_stringop_string, "byte_loop"))
2786 stringop_alg = loop_1_byte;
2787 else if (!strcmp (ix86_stringop_string, "loop"))
2788 stringop_alg = loop;
2789 else if (!strcmp (ix86_stringop_string, "unrolled_loop"))
2790 stringop_alg = unrolled_loop;
2792 error ("bad value (%s) for %sstringop-strategy=%s %s",
2793 ix86_stringop_string, prefix, suffix, sw);
2795 if (!strcmp (ix86_tune_string, "x86-64"))
2796 warning (OPT_Wdeprecated, "%stune=x86-64%s is deprecated. Use "
2797 "%stune=k8%s or %stune=generic%s instead as appropriate.",
2798 prefix, suffix, prefix, suffix, prefix, suffix);
2800 if (!ix86_arch_string)
2801 ix86_arch_string = TARGET_64BIT ? "x86-64" : "i386";
2803 ix86_arch_specified = 1;
2805 if (!strcmp (ix86_arch_string, "generic"))
2806 error ("generic CPU can be used only for %stune=%s %s",
2807 prefix, suffix, sw);
2808 if (!strncmp (ix86_arch_string, "generic", 7))
2809 error ("bad value (%s) for %sarch=%s %s",
2810 ix86_arch_string, prefix, suffix, sw);
2812 /* Validate -mabi= value. */
2813 if (ix86_abi_string)
2815 if (strcmp (ix86_abi_string, "sysv") == 0)
2816 ix86_abi = SYSV_ABI;
2817 else if (strcmp (ix86_abi_string, "ms") == 0)
2820 error ("unknown ABI (%s) for %sabi=%s %s",
2821 ix86_abi_string, prefix, suffix, sw);
2824 ix86_abi = DEFAULT_ABI;
2826 if (ix86_cmodel_string != 0)
2828 if (!strcmp (ix86_cmodel_string, "small"))
2829 ix86_cmodel = flag_pic ? CM_SMALL_PIC : CM_SMALL;
2830 else if (!strcmp (ix86_cmodel_string, "medium"))
2831 ix86_cmodel = flag_pic ? CM_MEDIUM_PIC : CM_MEDIUM;
2832 else if (!strcmp (ix86_cmodel_string, "large"))
2833 ix86_cmodel = flag_pic ? CM_LARGE_PIC : CM_LARGE;
2835 error ("code model %s does not support PIC mode", ix86_cmodel_string);
2836 else if (!strcmp (ix86_cmodel_string, "32"))
2837 ix86_cmodel = CM_32;
2838 else if (!strcmp (ix86_cmodel_string, "kernel") && !flag_pic)
2839 ix86_cmodel = CM_KERNEL;
2841 error ("bad value (%s) for %scmodel=%s %s",
2842 ix86_cmodel_string, prefix, suffix, sw);
2846 /* For TARGET_64BIT and MS_ABI, force pic on, in order to enable the
2847 use of rip-relative addressing. This eliminates fixups that
2848 would otherwise be needed if this object is to be placed in a
2849 DLL, and is essentially just as efficient as direct addressing. */
2850 if (TARGET_64BIT && DEFAULT_ABI == MS_ABI)
2851 ix86_cmodel = CM_SMALL_PIC, flag_pic = 1;
2852 else if (TARGET_64BIT)
2853 ix86_cmodel = flag_pic ? CM_SMALL_PIC : CM_SMALL;
2855 ix86_cmodel = CM_32;
2857 if (ix86_asm_string != 0)
2860 && !strcmp (ix86_asm_string, "intel"))
2861 ix86_asm_dialect = ASM_INTEL;
2862 else if (!strcmp (ix86_asm_string, "att"))
2863 ix86_asm_dialect = ASM_ATT;
2865 error ("bad value (%s) for %sasm=%s %s",
2866 ix86_asm_string, prefix, suffix, sw);
2868 if ((TARGET_64BIT == 0) != (ix86_cmodel == CM_32))
2869 error ("code model %qs not supported in the %s bit mode",
2870 ix86_cmodel_string, TARGET_64BIT ? "64" : "32");
2871 if ((TARGET_64BIT != 0) != ((ix86_isa_flags & OPTION_MASK_ISA_64BIT) != 0))
2872 sorry ("%i-bit mode not compiled in",
2873 (ix86_isa_flags & OPTION_MASK_ISA_64BIT) ? 64 : 32);
2875 for (i = 0; i < pta_size; i++)
2876 if (! strcmp (ix86_arch_string, processor_alias_table[i].name))
2878 ix86_schedule = processor_alias_table[i].schedule;
2879 ix86_arch = processor_alias_table[i].processor;
2880 /* Default cpu tuning to the architecture. */
2881 ix86_tune = ix86_arch;
2883 if (TARGET_64BIT && !(processor_alias_table[i].flags & PTA_64BIT))
2884 error ("CPU you selected does not support x86-64 "
2887 if (processor_alias_table[i].flags & PTA_MMX
2888 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_MMX))
2889 ix86_isa_flags |= OPTION_MASK_ISA_MMX;
2890 if (processor_alias_table[i].flags & PTA_3DNOW
2891 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_3DNOW))
2892 ix86_isa_flags |= OPTION_MASK_ISA_3DNOW;
2893 if (processor_alias_table[i].flags & PTA_3DNOW_A
2894 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_3DNOW_A))
2895 ix86_isa_flags |= OPTION_MASK_ISA_3DNOW_A;
2896 if (processor_alias_table[i].flags & PTA_SSE
2897 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE))
2898 ix86_isa_flags |= OPTION_MASK_ISA_SSE;
2899 if (processor_alias_table[i].flags & PTA_SSE2
2900 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE2))
2901 ix86_isa_flags |= OPTION_MASK_ISA_SSE2;
2902 if (processor_alias_table[i].flags & PTA_SSE3
2903 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE3))
2904 ix86_isa_flags |= OPTION_MASK_ISA_SSE3;
2905 if (processor_alias_table[i].flags & PTA_SSSE3
2906 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSSE3))
2907 ix86_isa_flags |= OPTION_MASK_ISA_SSSE3;
2908 if (processor_alias_table[i].flags & PTA_SSE4_1
2909 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4_1))
2910 ix86_isa_flags |= OPTION_MASK_ISA_SSE4_1;
2911 if (processor_alias_table[i].flags & PTA_SSE4_2
2912 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4_2))
2913 ix86_isa_flags |= OPTION_MASK_ISA_SSE4_2;
2914 if (processor_alias_table[i].flags & PTA_AVX
2915 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX))
2916 ix86_isa_flags |= OPTION_MASK_ISA_AVX;
2917 if (processor_alias_table[i].flags & PTA_FMA
2918 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_FMA))
2919 ix86_isa_flags |= OPTION_MASK_ISA_FMA;
2920 if (processor_alias_table[i].flags & PTA_SSE4A
2921 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4A))
2922 ix86_isa_flags |= OPTION_MASK_ISA_SSE4A;
2923 if (processor_alias_table[i].flags & PTA_SSE5
2924 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE5))
2925 ix86_isa_flags |= OPTION_MASK_ISA_SSE5;
2926 if (processor_alias_table[i].flags & PTA_ABM
2927 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_ABM))
2928 ix86_isa_flags |= OPTION_MASK_ISA_ABM;
2929 if (processor_alias_table[i].flags & PTA_CX16
2930 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_CX16))
2931 ix86_isa_flags |= OPTION_MASK_ISA_CX16;
2932 if (processor_alias_table[i].flags & (PTA_POPCNT | PTA_ABM)
2933 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_POPCNT))
2934 ix86_isa_flags |= OPTION_MASK_ISA_POPCNT;
2935 if (!(TARGET_64BIT && (processor_alias_table[i].flags & PTA_NO_SAHF))
2936 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SAHF))
2937 ix86_isa_flags |= OPTION_MASK_ISA_SAHF;
2938 if (processor_alias_table[i].flags & PTA_AES
2939 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_AES))
2940 ix86_isa_flags |= OPTION_MASK_ISA_AES;
2941 if (processor_alias_table[i].flags & PTA_PCLMUL
2942 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_PCLMUL))
2943 ix86_isa_flags |= OPTION_MASK_ISA_PCLMUL;
2944 if (processor_alias_table[i].flags & (PTA_PREFETCH_SSE | PTA_SSE))
2945 x86_prefetch_sse = true;
2951 error ("bad value (%s) for %sarch=%s %s",
2952 ix86_arch_string, prefix, suffix, sw);
2954 ix86_arch_mask = 1u << ix86_arch;
2955 for (i = 0; i < X86_ARCH_LAST; ++i)
2956 ix86_arch_features[i] = !!(initial_ix86_arch_features[i] & ix86_arch_mask);
2958 for (i = 0; i < pta_size; i++)
2959 if (! strcmp (ix86_tune_string, processor_alias_table[i].name))
2961 ix86_schedule = processor_alias_table[i].schedule;
2962 ix86_tune = processor_alias_table[i].processor;
2963 if (TARGET_64BIT && !(processor_alias_table[i].flags & PTA_64BIT))
2965 if (ix86_tune_defaulted)
2967 ix86_tune_string = "x86-64";
2968 for (i = 0; i < pta_size; i++)
2969 if (! strcmp (ix86_tune_string,
2970 processor_alias_table[i].name))
2972 ix86_schedule = processor_alias_table[i].schedule;
2973 ix86_tune = processor_alias_table[i].processor;
2976 error ("CPU you selected does not support x86-64 "
2979 /* Intel CPUs have always interpreted SSE prefetch instructions as
2980 NOPs; so, we can enable SSE prefetch instructions even when
2981 -mtune (rather than -march) points us to a processor that has them.
2982 However, the VIA C3 gives a SIGILL, so we only do that for i686 and
2983 higher processors. */
2985 && (processor_alias_table[i].flags & (PTA_PREFETCH_SSE | PTA_SSE)))
2986 x86_prefetch_sse = true;
2990 error ("bad value (%s) for %stune=%s %s",
2991 ix86_tune_string, prefix, suffix, sw);
2993 ix86_tune_mask = 1u << ix86_tune;
2994 for (i = 0; i < X86_TUNE_LAST; ++i)
2995 ix86_tune_features[i] = !!(initial_ix86_tune_features[i] & ix86_tune_mask);
2998 ix86_cost = &ix86_size_cost;
3000 ix86_cost = processor_target_table[ix86_tune].cost;
3002 /* Arrange to set up i386_stack_locals for all functions. */
3003 init_machine_status = ix86_init_machine_status;
3005 /* Validate -mregparm= value. */
3006 if (ix86_regparm_string)
3009 warning (0, "%sregparm%s is ignored in 64-bit mode", prefix, suffix);
3010 i = atoi (ix86_regparm_string);
3011 if (i < 0 || i > REGPARM_MAX)
3012 error ("%sregparm=%d%s is not between 0 and %d",
3013 prefix, i, suffix, REGPARM_MAX);
3018 ix86_regparm = REGPARM_MAX;
3020 /* If the user has provided any of the -malign-* options,
3021 warn and use that value only if -falign-* is not set.
3022 Remove this code in GCC 3.2 or later. */
3023 if (ix86_align_loops_string)
3025 warning (0, "%salign-loops%s is obsolete, use -falign-loops%s",
3026 prefix, suffix, suffix);
3027 if (align_loops == 0)
3029 i = atoi (ix86_align_loops_string);
3030 if (i < 0 || i > MAX_CODE_ALIGN)
3031 error ("%salign-loops=%d%s is not between 0 and %d",
3032 prefix, i, suffix, MAX_CODE_ALIGN);
3034 align_loops = 1 << i;
3038 if (ix86_align_jumps_string)
3040 warning (0, "%salign-jumps%s is obsolete, use -falign-jumps%s",
3041 prefix, suffix, suffix);
3042 if (align_jumps == 0)
3044 i = atoi (ix86_align_jumps_string);
3045 if (i < 0 || i > MAX_CODE_ALIGN)
3046 error ("%salign-loops=%d%s is not between 0 and %d",
3047 prefix, i, suffix, MAX_CODE_ALIGN);
3049 align_jumps = 1 << i;
3053 if (ix86_align_funcs_string)
3055 warning (0, "%salign-functions%s is obsolete, use -falign-functions%s",
3056 prefix, suffix, suffix);
3057 if (align_functions == 0)
3059 i = atoi (ix86_align_funcs_string);
3060 if (i < 0 || i > MAX_CODE_ALIGN)
3061 error ("%salign-loops=%d%s is not between 0 and %d",
3062 prefix, i, suffix, MAX_CODE_ALIGN);
3064 align_functions = 1 << i;
3068 /* Default align_* from the processor table. */
3069 if (align_loops == 0)
3071 align_loops = processor_target_table[ix86_tune].align_loop;
3072 align_loops_max_skip = processor_target_table[ix86_tune].align_loop_max_skip;
3074 if (align_jumps == 0)
3076 align_jumps = processor_target_table[ix86_tune].align_jump;
3077 align_jumps_max_skip = processor_target_table[ix86_tune].align_jump_max_skip;
3079 if (align_functions == 0)
3081 align_functions = processor_target_table[ix86_tune].align_func;
3084 /* Validate -mbranch-cost= value, or provide default. */
3085 ix86_branch_cost = ix86_cost->branch_cost;
3086 if (ix86_branch_cost_string)
3088 i = atoi (ix86_branch_cost_string);
3090 error ("%sbranch-cost=%d%s is not between 0 and 5", prefix, i, suffix);
3092 ix86_branch_cost = i;
3094 if (ix86_section_threshold_string)
3096 i = atoi (ix86_section_threshold_string);
3098 error ("%slarge-data-threshold=%d%s is negative", prefix, i, suffix);
3100 ix86_section_threshold = i;
3103 if (ix86_tls_dialect_string)
3105 if (strcmp (ix86_tls_dialect_string, "gnu") == 0)
3106 ix86_tls_dialect = TLS_DIALECT_GNU;
3107 else if (strcmp (ix86_tls_dialect_string, "gnu2") == 0)
3108 ix86_tls_dialect = TLS_DIALECT_GNU2;
3109 else if (strcmp (ix86_tls_dialect_string, "sun") == 0)
3110 ix86_tls_dialect = TLS_DIALECT_SUN;
3112 error ("bad value (%s) for %stls-dialect=%s %s",
3113 ix86_tls_dialect_string, prefix, suffix, sw);
3116 if (ix87_precision_string)
3118 i = atoi (ix87_precision_string);
3119 if (i != 32 && i != 64 && i != 80)
3120 error ("pc%d is not valid precision setting (32, 64 or 80)", i);
3125 target_flags |= TARGET_SUBTARGET64_DEFAULT & ~target_flags_explicit;
3127 /* Enable by default the SSE and MMX builtins. Do allow the user to
3128 explicitly disable any of these. In particular, disabling SSE and
3129 MMX for kernel code is extremely useful. */
3130 if (!ix86_arch_specified)
3132 |= ((OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_MMX
3133 | TARGET_SUBTARGET64_ISA_DEFAULT) & ~ix86_isa_flags_explicit);
3136 warning (0, "%srtd%s is ignored in 64bit mode", prefix, suffix);
3140 target_flags |= TARGET_SUBTARGET32_DEFAULT & ~target_flags_explicit;
3142 if (!ix86_arch_specified)
3144 |= TARGET_SUBTARGET32_ISA_DEFAULT & ~ix86_isa_flags_explicit;
3146 /* i386 ABI does not specify red zone. It still makes sense to use it
3147 when programmer takes care to stack from being destroyed. */
3148 if (!(target_flags_explicit & MASK_NO_RED_ZONE))
3149 target_flags |= MASK_NO_RED_ZONE;
3152 /* Keep nonleaf frame pointers. */
3153 if (flag_omit_frame_pointer)
3154 target_flags &= ~MASK_OMIT_LEAF_FRAME_POINTER;
3155 else if (TARGET_OMIT_LEAF_FRAME_POINTER)
3156 flag_omit_frame_pointer = 1;
3158 /* If we're doing fast math, we don't care about comparison order
3159 wrt NaNs. This lets us use a shorter comparison sequence. */
3160 if (flag_finite_math_only)
3161 target_flags &= ~MASK_IEEE_FP;
3163 /* If the architecture always has an FPU, turn off NO_FANCY_MATH_387,
3164 since the insns won't need emulation. */
3165 if (x86_arch_always_fancy_math_387 & ix86_arch_mask)
3166 target_flags &= ~MASK_NO_FANCY_MATH_387;
3168 /* Likewise, if the target doesn't have a 387, or we've specified
3169 software floating point, don't use 387 inline intrinsics. */
3171 target_flags |= MASK_NO_FANCY_MATH_387;
3173 /* Turn on MMX builtins for -msse. */
3176 ix86_isa_flags |= OPTION_MASK_ISA_MMX & ~ix86_isa_flags_explicit;
3177 x86_prefetch_sse = true;
3180 /* Turn on popcnt instruction for -msse4.2 or -mabm. */
3181 if (TARGET_SSE4_2 || TARGET_ABM)
3182 ix86_isa_flags |= OPTION_MASK_ISA_POPCNT & ~ix86_isa_flags_explicit;
3184 /* Validate -mpreferred-stack-boundary= value or default it to
3185 PREFERRED_STACK_BOUNDARY_DEFAULT. */
3186 ix86_preferred_stack_boundary = PREFERRED_STACK_BOUNDARY_DEFAULT;
3187 if (ix86_preferred_stack_boundary_string)
3189 i = atoi (ix86_preferred_stack_boundary_string);
3190 if (i < (TARGET_64BIT ? 4 : 2) || i > 12)
3191 error ("%spreferred-stack-boundary=%d%s is not between %d and 12",
3192 prefix, i, suffix, TARGET_64BIT ? 4 : 2);
3194 ix86_preferred_stack_boundary = (1 << i) * BITS_PER_UNIT;
3197 /* Set the default value for -mstackrealign. */
3198 if (ix86_force_align_arg_pointer == -1)
3199 ix86_force_align_arg_pointer = STACK_REALIGN_DEFAULT;
3201 /* Validate -mincoming-stack-boundary= value or default it to
3202 MIN_STACK_BOUNDARY/PREFERRED_STACK_BOUNDARY. */
3203 if (ix86_force_align_arg_pointer)
3204 ix86_default_incoming_stack_boundary = MIN_STACK_BOUNDARY;
3206 ix86_default_incoming_stack_boundary = PREFERRED_STACK_BOUNDARY;
3207 ix86_incoming_stack_boundary = ix86_default_incoming_stack_boundary;
3208 if (ix86_incoming_stack_boundary_string)
3210 i = atoi (ix86_incoming_stack_boundary_string);
3211 if (i < (TARGET_64BIT ? 4 : 2) || i > 12)
3212 error ("-mincoming-stack-boundary=%d is not between %d and 12",
3213 i, TARGET_64BIT ? 4 : 2);
3216 ix86_user_incoming_stack_boundary = (1 << i) * BITS_PER_UNIT;
3217 ix86_incoming_stack_boundary
3218 = ix86_user_incoming_stack_boundary;
3222 /* Accept -msseregparm only if at least SSE support is enabled. */
3223 if (TARGET_SSEREGPARM
3225 error ("%ssseregparm%s used without SSE enabled", prefix, suffix);
3227 ix86_fpmath = TARGET_FPMATH_DEFAULT;
3228 if (ix86_fpmath_string != 0)
3230 if (! strcmp (ix86_fpmath_string, "387"))
3231 ix86_fpmath = FPMATH_387;
3232 else if (! strcmp (ix86_fpmath_string, "sse"))
3236 warning (0, "SSE instruction set disabled, using 387 arithmetics");
3237 ix86_fpmath = FPMATH_387;
3240 ix86_fpmath = FPMATH_SSE;
3242 else if (! strcmp (ix86_fpmath_string, "387,sse")
3243 || ! strcmp (ix86_fpmath_string, "387+sse")
3244 || ! strcmp (ix86_fpmath_string, "sse,387")
3245 || ! strcmp (ix86_fpmath_string, "sse+387")
3246 || ! strcmp (ix86_fpmath_string, "both"))
3250 warning (0, "SSE instruction set disabled, using 387 arithmetics");
3251 ix86_fpmath = FPMATH_387;
3253 else if (!TARGET_80387)
3255 warning (0, "387 instruction set disabled, using SSE arithmetics");
3256 ix86_fpmath = FPMATH_SSE;
3259 ix86_fpmath = (enum fpmath_unit) (FPMATH_SSE | FPMATH_387);
3262 error ("bad value (%s) for %sfpmath=%s %s",
3263 ix86_fpmath_string, prefix, suffix, sw);
3266 /* If the i387 is disabled, then do not return values in it. */
3268 target_flags &= ~MASK_FLOAT_RETURNS;
3270 /* Use external vectorized library in vectorizing intrinsics. */
3271 if (ix86_veclibabi_string)
3273 if (strcmp (ix86_veclibabi_string, "svml") == 0)
3274 ix86_veclib_handler = ix86_veclibabi_svml;
3275 else if (strcmp (ix86_veclibabi_string, "acml") == 0)
3276 ix86_veclib_handler = ix86_veclibabi_acml;
3278 error ("unknown vectorization library ABI type (%s) for "
3279 "%sveclibabi=%s %s", ix86_veclibabi_string,
3280 prefix, suffix, sw);
3283 if ((x86_accumulate_outgoing_args & ix86_tune_mask)
3284 && !(target_flags_explicit & MASK_ACCUMULATE_OUTGOING_ARGS)
3286 target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;
3288 /* ??? Unwind info is not correct around the CFG unless either a frame
3289 pointer is present or M_A_O_A is set. Fixing this requires rewriting
3290 unwind info generation to be aware of the CFG and propagating states
3292 if ((flag_unwind_tables || flag_asynchronous_unwind_tables
3293 || flag_exceptions || flag_non_call_exceptions)
3294 && flag_omit_frame_pointer
3295 && !(target_flags & MASK_ACCUMULATE_OUTGOING_ARGS))
3297 if (target_flags_explicit & MASK_ACCUMULATE_OUTGOING_ARGS)
3298 warning (0, "unwind tables currently require either a frame pointer "
3299 "or %saccumulate-outgoing-args%s for correctness",
3301 target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;
3304 /* If stack probes are required, the space used for large function
3305 arguments on the stack must also be probed, so enable
3306 -maccumulate-outgoing-args so this happens in the prologue. */
3307 if (TARGET_STACK_PROBE
3308 && !(target_flags & MASK_ACCUMULATE_OUTGOING_ARGS))
3310 if (target_flags_explicit & MASK_ACCUMULATE_OUTGOING_ARGS)
3311 warning (0, "stack probing requires %saccumulate-outgoing-args%s "
3312 "for correctness", prefix, suffix);
3313 target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS;
3316 /* For sane SSE instruction set generation we need fcomi instruction.
3317 It is safe to enable all CMOVE instructions. */
3321 /* Figure out what ASM_GENERATE_INTERNAL_LABEL builds as a prefix. */
3324 ASM_GENERATE_INTERNAL_LABEL (internal_label_prefix, "LX", 0);
3325 p = strchr (internal_label_prefix, 'X');
3326 internal_label_prefix_len = p - internal_label_prefix;
3330 /* When scheduling description is not available, disable scheduler pass
3331 so it won't slow down the compilation and make x87 code slower. */
3332 if (!TARGET_SCHEDULE)
3333 flag_schedule_insns_after_reload = flag_schedule_insns = 0;
3335 if (!PARAM_SET_P (PARAM_SIMULTANEOUS_PREFETCHES))
3336 set_param_value ("simultaneous-prefetches",
3337 ix86_cost->simultaneous_prefetches);
3338 if (!PARAM_SET_P (PARAM_L1_CACHE_LINE_SIZE))
3339 set_param_value ("l1-cache-line-size", ix86_cost->prefetch_block);
3340 if (!PARAM_SET_P (PARAM_L1_CACHE_SIZE))
3341 set_param_value ("l1-cache-size", ix86_cost->l1_cache_size);
3342 if (!PARAM_SET_P (PARAM_L2_CACHE_SIZE))
3343 set_param_value ("l2-cache-size", ix86_cost->l2_cache_size);
3345 /* If using typedef char *va_list, signal that __builtin_va_start (&ap, 0)
3346 can be optimized to ap = __builtin_next_arg (0). */
3348 targetm.expand_builtin_va_start = NULL;
3352 ix86_gen_leave = gen_leave_rex64;
3353 ix86_gen_pop1 = gen_popdi1;
3354 ix86_gen_add3 = gen_adddi3;
3355 ix86_gen_sub3 = gen_subdi3;
3356 ix86_gen_sub3_carry = gen_subdi3_carry_rex64;
3357 ix86_gen_one_cmpl2 = gen_one_cmpldi2;
3358 ix86_gen_monitor = gen_sse3_monitor64;
3359 ix86_gen_andsp = gen_anddi3;
3363 ix86_gen_leave = gen_leave;
3364 ix86_gen_pop1 = gen_popsi1;
3365 ix86_gen_add3 = gen_addsi3;
3366 ix86_gen_sub3 = gen_subsi3;
3367 ix86_gen_sub3_carry = gen_subsi3_carry;
3368 ix86_gen_one_cmpl2 = gen_one_cmplsi2;
3369 ix86_gen_monitor = gen_sse3_monitor;
3370 ix86_gen_andsp = gen_andsi3;
3374 /* Use -mcld by default for 32-bit code if configured with --enable-cld. */
3376 target_flags |= MASK_CLD & ~target_flags_explicit;
3379 /* Save the initial options in case the user does function specific options */
3381 target_option_default_node = target_option_current_node
3382 = build_target_option_node ();
3385 /* Save the current options */
3388 ix86_function_specific_save (struct cl_target_option *ptr)
3390 gcc_assert (IN_RANGE (ix86_arch, 0, 255));
3391 gcc_assert (IN_RANGE (ix86_schedule, 0, 255));
3392 gcc_assert (IN_RANGE (ix86_tune, 0, 255));
3393 gcc_assert (IN_RANGE (ix86_fpmath, 0, 255));
3394 gcc_assert (IN_RANGE (ix86_branch_cost, 0, 255));
3396 ptr->arch = ix86_arch;
3397 ptr->schedule = ix86_schedule;
3398 ptr->tune = ix86_tune;
3399 ptr->fpmath = ix86_fpmath;
3400 ptr->branch_cost = ix86_branch_cost;
3401 ptr->tune_defaulted = ix86_tune_defaulted;
3402 ptr->arch_specified = ix86_arch_specified;
3403 ptr->ix86_isa_flags_explicit = ix86_isa_flags_explicit;
3404 ptr->target_flags_explicit = target_flags_explicit;
3407 /* Restore the current options */
3410 ix86_function_specific_restore (struct cl_target_option *ptr)
3412 enum processor_type old_tune = ix86_tune;
3413 enum processor_type old_arch = ix86_arch;
3414 unsigned int ix86_arch_mask, ix86_tune_mask;
3417 ix86_arch = (enum processor_type) ptr->arch;
3418 ix86_schedule = (enum attr_cpu) ptr->schedule;
3419 ix86_tune = (enum processor_type) ptr->tune;
3420 ix86_fpmath = (enum fpmath_unit) ptr->fpmath;
3421 ix86_branch_cost = ptr->branch_cost;
3422 ix86_tune_defaulted = ptr->tune_defaulted;
3423 ix86_arch_specified = ptr->arch_specified;
3424 ix86_isa_flags_explicit = ptr->ix86_isa_flags_explicit;
3425 target_flags_explicit = ptr->target_flags_explicit;
3427 /* Recreate the arch feature tests if the arch changed */
3428 if (old_arch != ix86_arch)
3430 ix86_arch_mask = 1u << ix86_arch;
3431 for (i = 0; i < X86_ARCH_LAST; ++i)
3432 ix86_arch_features[i]
3433 = !!(initial_ix86_arch_features[i] & ix86_arch_mask);
3436 /* Recreate the tune optimization tests */
3437 if (old_tune != ix86_tune)
3439 ix86_tune_mask = 1u << ix86_tune;
3440 for (i = 0; i < X86_TUNE_LAST; ++i)
3441 ix86_tune_features[i]
3442 = !!(initial_ix86_tune_features[i] & ix86_tune_mask);
3446 /* Print the current options */
3449 ix86_function_specific_print (FILE *file, int indent,
3450 struct cl_target_option *ptr)
3453 = ix86_target_string (ptr->ix86_isa_flags, ptr->target_flags,
3454 NULL, NULL, NULL, false);
3456 fprintf (file, "%*sarch = %d (%s)\n",
3459 ((ptr->arch < TARGET_CPU_DEFAULT_max)
3460 ? cpu_names[ptr->arch]
3463 fprintf (file, "%*stune = %d (%s)\n",
3466 ((ptr->tune < TARGET_CPU_DEFAULT_max)
3467 ? cpu_names[ptr->tune]
3470 fprintf (file, "%*sfpmath = %d%s%s\n", indent, "", ptr->fpmath,
3471 (ptr->fpmath & FPMATH_387) ? ", 387" : "",
3472 (ptr->fpmath & FPMATH_SSE) ? ", sse" : "");
3473 fprintf (file, "%*sbranch_cost = %d\n", indent, "", ptr->branch_cost);
3477 fprintf (file, "%*s%s\n", indent, "", target_string);
3478 free (target_string);
3483 /* Inner function to process the attribute((target(...))), take an argument and
3484 set the current options from the argument. If we have a list, recursively go
3488 ix86_valid_target_attribute_inner_p (tree args, char *p_strings[])
3493 #define IX86_ATTR_ISA(S,O) { S, sizeof (S)-1, ix86_opt_isa, O, 0 }
3494 #define IX86_ATTR_STR(S,O) { S, sizeof (S)-1, ix86_opt_str, O, 0 }
3495 #define IX86_ATTR_YES(S,O,M) { S, sizeof (S)-1, ix86_opt_yes, O, M }
3496 #define IX86_ATTR_NO(S,O,M) { S, sizeof (S)-1, ix86_opt_no, O, M }
3511 enum ix86_opt_type type;
3516 IX86_ATTR_ISA ("3dnow", OPT_m3dnow),
3517 IX86_ATTR_ISA ("abm", OPT_mabm),
3518 IX86_ATTR_ISA ("aes", OPT_maes),
3519 IX86_ATTR_ISA ("avx", OPT_mavx),
3520 IX86_ATTR_ISA ("mmx", OPT_mmmx),
3521 IX86_ATTR_ISA ("pclmul", OPT_mpclmul),
3522 IX86_ATTR_ISA ("popcnt", OPT_mpopcnt),
3523 IX86_ATTR_ISA ("sse", OPT_msse),
3524 IX86_ATTR_ISA ("sse2", OPT_msse2),
3525 IX86_ATTR_ISA ("sse3", OPT_msse3),
3526 IX86_ATTR_ISA ("sse4", OPT_msse4),
3527 IX86_ATTR_ISA ("sse4.1", OPT_msse4_1),
3528 IX86_ATTR_ISA ("sse4.2", OPT_msse4_2),
3529 IX86_ATTR_ISA ("sse4a", OPT_msse4a),
3530 IX86_ATTR_ISA ("sse5", OPT_msse5),
3531 IX86_ATTR_ISA ("ssse3", OPT_mssse3),
3533 /* string options */
3534 IX86_ATTR_STR ("arch=", IX86_FUNCTION_SPECIFIC_ARCH),
3535 IX86_ATTR_STR ("fpmath=", IX86_FUNCTION_SPECIFIC_FPMATH),
3536 IX86_ATTR_STR ("tune=", IX86_FUNCTION_SPECIFIC_TUNE),
3539 IX86_ATTR_YES ("cld",
3543 IX86_ATTR_NO ("fancy-math-387",
3544 OPT_mfancy_math_387,
3545 MASK_NO_FANCY_MATH_387),
3547 IX86_ATTR_NO ("fused-madd",
3549 MASK_NO_FUSED_MADD),
3551 IX86_ATTR_YES ("ieee-fp",
3555 IX86_ATTR_YES ("inline-all-stringops",
3556 OPT_minline_all_stringops,
3557 MASK_INLINE_ALL_STRINGOPS),
3559 IX86_ATTR_YES ("inline-stringops-dynamically",
3560 OPT_minline_stringops_dynamically,
3561 MASK_INLINE_STRINGOPS_DYNAMICALLY),
3563 IX86_ATTR_NO ("align-stringops",
3564 OPT_mno_align_stringops,
3565 MASK_NO_ALIGN_STRINGOPS),
3567 IX86_ATTR_YES ("recip",
3573 /* If this is a list, recurse to get the options. */
3574 if (TREE_CODE (args) == TREE_LIST)
3578 for (; args; args = TREE_CHAIN (args))
3579 if (TREE_VALUE (args)
3580 && !ix86_valid_target_attribute_inner_p (TREE_VALUE (args), p_strings))
3586 else if (TREE_CODE (args) != STRING_CST)
3589 /* Handle multiple arguments separated by commas. */
3590 next_optstr = ASTRDUP (TREE_STRING_POINTER (args));
3592 while (next_optstr && *next_optstr != '\0')
3594 char *p = next_optstr;
3596 char *comma = strchr (next_optstr, ',');
3597 const char *opt_string;
3598 size_t len, opt_len;
3603 enum ix86_opt_type type = ix86_opt_unknown;
3609 len = comma - next_optstr;
3610 next_optstr = comma + 1;
3618 /* Recognize no-xxx. */
3619 if (len > 3 && p[0] == 'n' && p[1] == 'o' && p[2] == '-')
3628 /* Find the option. */
3631 for (i = 0; i < ARRAY_SIZE (attrs); i++)
3633 type = attrs[i].type;
3634 opt_len = attrs[i].len;
3635 if (ch == attrs[i].string[0]
3636 && ((type != ix86_opt_str) ? len == opt_len : len > opt_len)
3637 && memcmp (p, attrs[i].string, opt_len) == 0)
3640 mask = attrs[i].mask;
3641 opt_string = attrs[i].string;
3646 /* Process the option. */
3649 error ("attribute(target(\"%s\")) is unknown", orig_p);
3653 else if (type == ix86_opt_isa)
3654 ix86_handle_option (opt, p, opt_set_p);
3656 else if (type == ix86_opt_yes || type == ix86_opt_no)
3658 if (type == ix86_opt_no)
3659 opt_set_p = !opt_set_p;
3662 target_flags |= mask;
3664 target_flags &= ~mask;
3667 else if (type == ix86_opt_str)
3671 error ("option(\"%s\") was already specified", opt_string);
3675 p_strings[opt] = xstrdup (p + opt_len);
3685 /* Return a TARGET_OPTION_NODE tree of the target options listed or NULL. */
3688 ix86_valid_target_attribute_tree (tree args)
3690 const char *orig_arch_string = ix86_arch_string;
3691 const char *orig_tune_string = ix86_tune_string;
3692 const char *orig_fpmath_string = ix86_fpmath_string;
3693 int orig_tune_defaulted = ix86_tune_defaulted;
3694 int orig_arch_specified = ix86_arch_specified;
3695 char *option_strings[IX86_FUNCTION_SPECIFIC_MAX] = { NULL, NULL, NULL };
3698 struct cl_target_option *def
3699 = TREE_TARGET_OPTION (target_option_default_node);
3701 /* Process each of the options on the chain. */
3702 if (! ix86_valid_target_attribute_inner_p (args, option_strings))
3705 /* If the changed options are different from the default, rerun override_options,
3706 and then save the options away. The string options are are attribute options,
3707 and will be undone when we copy the save structure. */
3708 if (ix86_isa_flags != def->ix86_isa_flags
3709 || target_flags != def->target_flags
3710 || option_strings[IX86_FUNCTION_SPECIFIC_ARCH]
3711 || option_strings[IX86_FUNCTION_SPECIFIC_TUNE]
3712 || option_strings[IX86_FUNCTION_SPECIFIC_FPMATH])
3714 /* If we are using the default tune= or arch=, undo the string assigned,
3715 and use the default. */
3716 if (option_strings[IX86_FUNCTION_SPECIFIC_ARCH])
3717 ix86_arch_string = option_strings[IX86_FUNCTION_SPECIFIC_ARCH];
3718 else if (!orig_arch_specified)
3719 ix86_arch_string = NULL;
3721 if (option_strings[IX86_FUNCTION_SPECIFIC_TUNE])
3722 ix86_tune_string = option_strings[IX86_FUNCTION_SPECIFIC_TUNE];
3723 else if (orig_tune_defaulted)
3724 ix86_tune_string = NULL;
3726 /* If fpmath= is not set, and we now have sse2 on 32-bit, use it. */
3727 if (option_strings[IX86_FUNCTION_SPECIFIC_FPMATH])
3728 ix86_fpmath_string = option_strings[IX86_FUNCTION_SPECIFIC_FPMATH];
3729 else if (!TARGET_64BIT && TARGET_SSE)
3730 ix86_fpmath_string = "sse,387";
3732 /* Do any overrides, such as arch=xxx, or tune=xxx support. */
3733 override_options (false);
3735 /* Add any builtin functions with the new isa if any. */
3736 ix86_add_new_builtins (ix86_isa_flags);
3738 /* Save the current options unless we are validating options for
3740 t = build_target_option_node ();
3742 ix86_arch_string = orig_arch_string;
3743 ix86_tune_string = orig_tune_string;
3744 ix86_fpmath_string = orig_fpmath_string;
3746 /* Free up memory allocated to hold the strings */
3747 for (i = 0; i < IX86_FUNCTION_SPECIFIC_MAX; i++)
3748 if (option_strings[i])
3749 free (option_strings[i]);
3755 /* Hook to validate attribute((target("string"))). */
3758 ix86_valid_target_attribute_p (tree fndecl,
3759 tree ARG_UNUSED (name),
3761 int ARG_UNUSED (flags))
3763 struct cl_target_option cur_target;
3765 tree old_optimize = build_optimization_node ();
3766 tree new_target, new_optimize;
3767 tree func_optimize = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (fndecl);
3769 /* If the function changed the optimization levels as well as setting target
3770 options, start with the optimizations specified. */
3771 if (func_optimize && func_optimize != old_optimize)
3772 cl_optimization_restore (TREE_OPTIMIZATION (func_optimize));
3774 /* The target attributes may also change some optimization flags, so update
3775 the optimization options if necessary. */
3776 cl_target_option_save (&cur_target);
3777 new_target = ix86_valid_target_attribute_tree (args);
3778 new_optimize = build_optimization_node ();
3785 DECL_FUNCTION_SPECIFIC_TARGET (fndecl) = new_target;
3787 if (old_optimize != new_optimize)
3788 DECL_FUNCTION_SPECIFIC_OPTIMIZATION (fndecl) = new_optimize;
3791 cl_target_option_restore (&cur_target);
3793 if (old_optimize != new_optimize)
3794 cl_optimization_restore (TREE_OPTIMIZATION (old_optimize));
3800 /* Hook to determine if one function can safely inline another. */
3803 ix86_can_inline_p (tree caller, tree callee)
3806 tree caller_tree = DECL_FUNCTION_SPECIFIC_TARGET (caller);
3807 tree callee_tree = DECL_FUNCTION_SPECIFIC_TARGET (callee);
3809 /* If callee has no option attributes, then it is ok to inline. */
3813 /* If caller has no option attributes, but callee does then it is not ok to
3815 else if (!caller_tree)
3820 struct cl_target_option *caller_opts = TREE_TARGET_OPTION (caller_tree);
3821 struct cl_target_option *callee_opts = TREE_TARGET_OPTION (callee_tree);
3823 /* Callee's isa options should a subset of the caller's, i.e. a SSE5 function
3824 can inline a SSE2 function but a SSE2 function can't inline a SSE5
3826 if ((caller_opts->ix86_isa_flags & callee_opts->ix86_isa_flags)
3827 != callee_opts->ix86_isa_flags)
3830 /* See if we have the same non-isa options. */
3831 else if (caller_opts->target_flags != callee_opts->target_flags)
3834 /* See if arch, tune, etc. are the same. */
3835 else if (caller_opts->arch != callee_opts->arch)
3838 else if (caller_opts->tune != callee_opts->tune)
3841 else if (caller_opts->fpmath != callee_opts->fpmath)
3844 else if (caller_opts->branch_cost != callee_opts->branch_cost)
3855 /* Remember the last target of ix86_set_current_function. */
3856 static GTY(()) tree ix86_previous_fndecl;
3858 /* Establish appropriate back-end context for processing the function
3859 FNDECL. The argument might be NULL to indicate processing at top
3860 level, outside of any function scope. */
3862 ix86_set_current_function (tree fndecl)
3864 /* Only change the context if the function changes. This hook is called
3865 several times in the course of compiling a function, and we don't want to
3866 slow things down too much or call target_reinit when it isn't safe. */
3867 if (fndecl && fndecl != ix86_previous_fndecl)
3869 tree old_tree = (ix86_previous_fndecl
3870 ? DECL_FUNCTION_SPECIFIC_TARGET (ix86_previous_fndecl)
3873 tree new_tree = (fndecl
3874 ? DECL_FUNCTION_SPECIFIC_TARGET (fndecl)
3877 ix86_previous_fndecl = fndecl;
3878 if (old_tree == new_tree)
3883 cl_target_option_restore (TREE_TARGET_OPTION (new_tree));
3889 struct cl_target_option *def
3890 = TREE_TARGET_OPTION (target_option_current_node);
3892 cl_target_option_restore (def);
3899 /* Return true if this goes in large data/bss. */
3902 ix86_in_large_data_p (tree exp)
3904 if (ix86_cmodel != CM_MEDIUM && ix86_cmodel != CM_MEDIUM_PIC)
3907 /* Functions are never large data. */
3908 if (TREE_CODE (exp) == FUNCTION_DECL)
3911 if (TREE_CODE (exp) == VAR_DECL && DECL_SECTION_NAME (exp))
3913 const char *section = TREE_STRING_POINTER (DECL_SECTION_NAME (exp));
3914 if (strcmp (section, ".ldata") == 0
3915 || strcmp (section, ".lbss") == 0)
3921 HOST_WIDE_INT size = int_size_in_bytes (TREE_TYPE (exp));
3923 /* If this is an incomplete type with size 0, then we can't put it
3924 in data because it might be too big when completed. */
3925 if (!size || size > ix86_section_threshold)
3932 /* Switch to the appropriate section for output of DECL.
3933 DECL is either a `VAR_DECL' node or a constant of some sort.
3934 RELOC indicates whether forming the initial value of DECL requires
3935 link-time relocations. */
3937 static section * x86_64_elf_select_section (tree, int, unsigned HOST_WIDE_INT)
3941 x86_64_elf_select_section (tree decl, int reloc,
3942 unsigned HOST_WIDE_INT align)
3944 if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC)
3945 && ix86_in_large_data_p (decl))
3947 const char *sname = NULL;
3948 unsigned int flags = SECTION_WRITE;
3949 switch (categorize_decl_for_section (decl, reloc))
3954 case SECCAT_DATA_REL:
3955 sname = ".ldata.rel";
3957 case SECCAT_DATA_REL_LOCAL:
3958 sname = ".ldata.rel.local";
3960 case SECCAT_DATA_REL_RO:
3961 sname = ".ldata.rel.ro";
3963 case SECCAT_DATA_REL_RO_LOCAL:
3964 sname = ".ldata.rel.ro.local";
3968 flags |= SECTION_BSS;
3971 case SECCAT_RODATA_MERGE_STR:
3972 case SECCAT_RODATA_MERGE_STR_INIT:
3973 case SECCAT_RODATA_MERGE_CONST:
3977 case SECCAT_SRODATA:
3984 /* We don't split these for medium model. Place them into
3985 default sections and hope for best. */
3987 case SECCAT_EMUTLS_VAR:
3988 case SECCAT_EMUTLS_TMPL:
3993 /* We might get called with string constants, but get_named_section
3994 doesn't like them as they are not DECLs. Also, we need to set
3995 flags in that case. */
3997 return get_section (sname, flags, NULL);
3998 return get_named_section (decl, sname, reloc);
4001 return default_elf_select_section (decl, reloc, align);
4004 /* Build up a unique section name, expressed as a
4005 STRING_CST node, and assign it to DECL_SECTION_NAME (decl).
4006 RELOC indicates whether the initial value of EXP requires
4007 link-time relocations. */
4009 static void ATTRIBUTE_UNUSED
4010 x86_64_elf_unique_section (tree decl, int reloc)
4012 if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC)
4013 && ix86_in_large_data_p (decl))
4015 const char *prefix = NULL;
4016 /* We only need to use .gnu.linkonce if we don't have COMDAT groups. */
4017 bool one_only = DECL_ONE_ONLY (decl) && !HAVE_COMDAT_GROUP;
4019 switch (categorize_decl_for_section (decl, reloc))
4022 case SECCAT_DATA_REL:
4023 case SECCAT_DATA_REL_LOCAL:
4024 case SECCAT_DATA_REL_RO:
4025 case SECCAT_DATA_REL_RO_LOCAL:
4026 prefix = one_only ? ".ld" : ".ldata";
4029 prefix = one_only ? ".lb" : ".lbss";
4032 case SECCAT_RODATA_MERGE_STR:
4033 case SECCAT_RODATA_MERGE_STR_INIT:
4034 case SECCAT_RODATA_MERGE_CONST:
4035 prefix = one_only ? ".lr" : ".lrodata";
4037 case SECCAT_SRODATA:
4044 /* We don't split these for medium model. Place them into
4045 default sections and hope for best. */
4047 case SECCAT_EMUTLS_VAR:
4048 prefix = targetm.emutls.var_section;
4050 case SECCAT_EMUTLS_TMPL:
4051 prefix = targetm.emutls.tmpl_section;
4056 const char *name, *linkonce;
4059 name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
4060 name = targetm.strip_name_encoding (name);
4062 /* If we're using one_only, then there needs to be a .gnu.linkonce
4063 prefix to the section name. */
4064 linkonce = one_only ? ".gnu.linkonce" : "";
4066 string = ACONCAT ((linkonce, prefix, ".", name, NULL));
4068 DECL_SECTION_NAME (decl) = build_string (strlen (string), string);
4072 default_unique_section (decl, reloc);
4075 #ifdef COMMON_ASM_OP
4076 /* This says how to output assembler code to declare an
4077 uninitialized external linkage data object.
4079 For medium model x86-64 we need to use .largecomm opcode for
4082 x86_elf_aligned_common (FILE *file,
4083 const char *name, unsigned HOST_WIDE_INT size,
4086 if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC)
4087 && size > (unsigned int)ix86_section_threshold)
4088 fprintf (file, ".largecomm\t");
4090 fprintf (file, "%s", COMMON_ASM_OP);
4091 assemble_name (file, name);
4092 fprintf (file, ","HOST_WIDE_INT_PRINT_UNSIGNED",%u\n",
4093 size, align / BITS_PER_UNIT);
4097 /* Utility function for targets to use in implementing
4098 ASM_OUTPUT_ALIGNED_BSS. */
4101 x86_output_aligned_bss (FILE *file, tree decl ATTRIBUTE_UNUSED,
4102 const char *name, unsigned HOST_WIDE_INT size,
4105 if ((ix86_cmodel == CM_MEDIUM || ix86_cmodel == CM_MEDIUM_PIC)
4106 && size > (unsigned int)ix86_section_threshold)
4107 switch_to_section (get_named_section (decl, ".lbss", 0));
4109 switch_to_section (bss_section);
4110 ASM_OUTPUT_ALIGN (file, floor_log2 (align / BITS_PER_UNIT));
4111 #ifdef ASM_DECLARE_OBJECT_NAME
4112 last_assemble_variable_decl = decl;
4113 ASM_DECLARE_OBJECT_NAME (file, name, decl);
4115 /* Standard thing is just output label for the object. */
4116 ASM_OUTPUT_LABEL (file, name);
4117 #endif /* ASM_DECLARE_OBJECT_NAME */
4118 ASM_OUTPUT_SKIP (file, size ? size : 1);
4122 optimization_options (int level, int size ATTRIBUTE_UNUSED)
4124 /* For -O2 and beyond, turn off -fschedule-insns by default. It tends to
4125 make the problem with not enough registers even worse. */
4126 #ifdef INSN_SCHEDULING
4128 flag_schedule_insns = 0;
4132 /* The Darwin libraries never set errno, so we might as well
4133 avoid calling them when that's the only reason we would. */
4134 flag_errno_math = 0;
4136 /* The default values of these switches depend on the TARGET_64BIT
4137 that is not known at this moment. Mark these values with 2 and
4138 let user the to override these. In case there is no command line option
4139 specifying them, we will set the defaults in override_options. */
4141 flag_omit_frame_pointer = 2;
4142 flag_pcc_struct_return = 2;
4143 flag_asynchronous_unwind_tables = 2;
4144 flag_vect_cost_model = 1;
4145 #ifdef SUBTARGET_OPTIMIZATION_OPTIONS
4146 SUBTARGET_OPTIMIZATION_OPTIONS;
4150 /* Decide whether we can make a sibling call to a function. DECL is the
4151 declaration of the function being targeted by the call and EXP is the
4152 CALL_EXPR representing the call. */
4155 ix86_function_ok_for_sibcall (tree decl, tree exp)
4160 /* If we are generating position-independent code, we cannot sibcall
4161 optimize any indirect call, or a direct call to a global function,
4162 as the PLT requires %ebx be live. */
4163 if (!TARGET_64BIT && flag_pic && (!decl || !targetm.binds_local_p (decl)))
4170 func = TREE_TYPE (CALL_EXPR_FN (exp));
4171 if (POINTER_TYPE_P (func))
4172 func = TREE_TYPE (func);
4175 /* Check that the return value locations are the same. Like
4176 if we are returning floats on the 80387 register stack, we cannot
4177 make a sibcall from a function that doesn't return a float to a
4178 function that does or, conversely, from a function that does return
4179 a float to a function that doesn't; the necessary stack adjustment
4180 would not be executed. This is also the place we notice
4181 differences in the return value ABI. Note that it is ok for one
4182 of the functions to have void return type as long as the return
4183 value of the other is passed in a register. */
4184 a = ix86_function_value (TREE_TYPE (exp), func, false);
4185 b = ix86_function_value (TREE_TYPE (DECL_RESULT (cfun->decl)),
4187 if (STACK_REG_P (a) || STACK_REG_P (b))
4189 if (!rtx_equal_p (a, b))
4192 else if (VOID_TYPE_P (TREE_TYPE (DECL_RESULT (cfun->decl))))
4194 else if (!rtx_equal_p (a, b))
4197 /* If this call is indirect, we'll need to be able to use a call-clobbered
4198 register for the address of the target function. Make sure that all
4199 such registers are not used for passing parameters. */
4200 if (!decl && !TARGET_64BIT)
4204 /* We're looking at the CALL_EXPR, we need the type of the function. */
4205 type = CALL_EXPR_FN (exp); /* pointer expression */
4206 type = TREE_TYPE (type); /* pointer type */
4207 type = TREE_TYPE (type); /* function type */
4209 if (ix86_function_regparm (type, NULL) >= 3)
4211 /* ??? Need to count the actual number of registers to be used,
4212 not the possible number of registers. Fix later. */
4217 /* Dllimport'd functions are also called indirectly. */
4218 if (TARGET_DLLIMPORT_DECL_ATTRIBUTES
4220 && decl && DECL_DLLIMPORT_P (decl)
4221 && ix86_function_regparm (TREE_TYPE (decl), NULL) >= 3)
4224 /* If we need to align the outgoing stack, then sibcalling would
4225 unalign the stack, which may break the called function. */
4226 if (ix86_incoming_stack_boundary < PREFERRED_STACK_BOUNDARY)
4229 /* Otherwise okay. That also includes certain types of indirect calls. */
4233 /* Handle "cdecl", "stdcall", "fastcall", "regparm" and "sseregparm"
4234 calling convention attributes;
4235 arguments as in struct attribute_spec.handler. */
4238 ix86_handle_cconv_attribute (tree *node, tree name,
4240 int flags ATTRIBUTE_UNUSED,
4243 if (TREE_CODE (*node) != FUNCTION_TYPE
4244 && TREE_CODE (*node) != METHOD_TYPE
4245 && TREE_CODE (*node) != FIELD_DECL
4246 && TREE_CODE (*node) != TYPE_DECL)
4248 warning (OPT_Wattributes, "%qE attribute only applies to functions",
4250 *no_add_attrs = true;
4254 /* Can combine regparm with all attributes but fastcall. */
4255 if (is_attribute_p ("regparm", name))
4259 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node)))
4261 error ("fastcall and regparm attributes are not compatible");
4264 cst = TREE_VALUE (args);
4265 if (TREE_CODE (cst) != INTEGER_CST)
4267 warning (OPT_Wattributes,
4268 "%qE attribute requires an integer constant argument",
4270 *no_add_attrs = true;
4272 else if (compare_tree_int (cst, REGPARM_MAX) > 0)
4274 warning (OPT_Wattributes, "argument to %qE attribute larger than %d",
4276 *no_add_attrs = true;
4284 /* Do not warn when emulating the MS ABI. */
4285 if (TREE_CODE (*node) != FUNCTION_TYPE || ix86_function_type_abi (*node)!=MS_ABI)
4286 warning (OPT_Wattributes, "%qE attribute ignored",
4288 *no_add_attrs = true;
4292 /* Can combine fastcall with stdcall (redundant) and sseregparm. */
4293 if (is_attribute_p ("fastcall", name))
4295 if (lookup_attribute ("cdecl", TYPE_ATTRIBUTES (*node)))
4297 error ("fastcall and cdecl attributes are not compatible");
4299 if (lookup_attribute ("stdcall", TYPE_ATTRIBUTES (*node)))
4301 error ("fastcall and stdcall attributes are not compatible");
4303 if (lookup_attribute ("regparm", TYPE_ATTRIBUTES (*node)))
4305 error ("fastcall and regparm attributes are not compatible");
4309 /* Can combine stdcall with fastcall (redundant), regparm and
4311 else if (is_attribute_p ("stdcall", name))
4313 if (lookup_attribute ("cdecl", TYPE_ATTRIBUTES (*node)))
4315 error ("stdcall and cdecl attributes are not compatible");
4317 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node)))
4319 error ("stdcall and fastcall attributes are not compatible");
4323 /* Can combine cdecl with regparm and sseregparm. */
4324 else if (is_attribute_p ("cdecl", name))
4326 if (lookup_attribute ("stdcall", TYPE_ATTRIBUTES (*node)))
4328 error ("stdcall and cdecl attributes are not compatible");
4330 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node)))
4332 error ("fastcall and cdecl attributes are not compatible");
4336 /* Can combine sseregparm with all attributes. */
4341 /* Return 0 if the attributes for two types are incompatible, 1 if they
4342 are compatible, and 2 if they are nearly compatible (which causes a
4343 warning to be generated). */
4346 ix86_comp_type_attributes (const_tree type1, const_tree type2)
4348 /* Check for mismatch of non-default calling convention. */
4349 const char *const rtdstr = TARGET_RTD ? "cdecl" : "stdcall";
4351 if (TREE_CODE (type1) != FUNCTION_TYPE
4352 && TREE_CODE (type1) != METHOD_TYPE)
4355 /* Check for mismatched fastcall/regparm types. */
4356 if ((!lookup_attribute ("fastcall", TYPE_ATTRIBUTES (type1))
4357 != !lookup_attribute ("fastcall", TYPE_ATTRIBUTES (type2)))
4358 || (ix86_function_regparm (type1, NULL)
4359 != ix86_function_regparm (type2, NULL)))
4362 /* Check for mismatched sseregparm types. */
4363 if (!lookup_attribute ("sseregparm", TYPE_ATTRIBUTES (type1))
4364 != !lookup_attribute ("sseregparm", TYPE_ATTRIBUTES (type2)))
4367 /* Check for mismatched return types (cdecl vs stdcall). */
4368 if (!lookup_attribute (rtdstr, TYPE_ATTRIBUTES (type1))
4369 != !lookup_attribute (rtdstr, TYPE_ATTRIBUTES (type2)))
4375 /* Return the regparm value for a function with the indicated TYPE and DECL.
4376 DECL may be NULL when calling function indirectly
4377 or considering a libcall. */
4380 ix86_function_regparm (const_tree type, const_tree decl)
4385 static bool error_issued;
4388 return (ix86_function_type_abi (type) == SYSV_ABI
4389 ? X86_64_REGPARM_MAX : X64_REGPARM_MAX);
4391 regparm = ix86_regparm;
4392 attr = lookup_attribute ("regparm", TYPE_ATTRIBUTES (type));
4396 = TREE_INT_CST_LOW (TREE_VALUE (TREE_VALUE (attr)));
4398 if (decl && TREE_CODE (decl) == FUNCTION_DECL)
4400 /* We can't use regparm(3) for nested functions because
4401 these pass static chain pointer in %ecx register. */
4402 if (!error_issued && regparm == 3
4403 && decl_function_context (decl)
4404 && !DECL_NO_STATIC_CHAIN (decl))
4406 error ("nested functions are limited to 2 register parameters");
4407 error_issued = true;
4415 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (type)))
4418 /* Use register calling convention for local functions when possible. */
4420 && TREE_CODE (decl) == FUNCTION_DECL
4424 /* FIXME: remove this CONST_CAST when cgraph.[ch] is constified. */
4425 struct cgraph_local_info *i = cgraph_local_info (CONST_CAST_TREE(decl));
4428 int local_regparm, globals = 0, regno;
4431 /* Make sure no regparm register is taken by a
4432 fixed register variable. */
4433 for (local_regparm = 0; local_regparm < REGPARM_MAX; local_regparm++)
4434 if (fixed_regs[local_regparm])
4437 /* We can't use regparm(3) for nested functions as these use
4438 static chain pointer in third argument. */
4439 if (local_regparm == 3
4440 && decl_function_context (decl)
4441 && !DECL_NO_STATIC_CHAIN (decl))
4444 /* If the function realigns its stackpointer, the prologue will
4445 clobber %ecx. If we've already generated code for the callee,
4446 the callee DECL_STRUCT_FUNCTION is gone, so we fall back to
4447 scanning the attributes for the self-realigning property. */
4448 f = DECL_STRUCT_FUNCTION (decl);
4449 /* Since current internal arg pointer won't conflict with
4450 parameter passing regs, so no need to change stack
4451 realignment and adjust regparm number.
4453 Each fixed register usage increases register pressure,
4454 so less registers should be used for argument passing.
4455 This functionality can be overriden by an explicit
4457 for (regno = 0; regno <= DI_REG; regno++)
4458 if (fixed_regs[regno])
4462 = globals < local_regparm ? local_regparm - globals : 0;
4464 if (local_regparm > regparm)
4465 regparm = local_regparm;
4472 /* Return 1 or 2, if we can pass up to SSE_REGPARM_MAX SFmode (1) and
4473 DFmode (2) arguments in SSE registers for a function with the
4474 indicated TYPE and DECL. DECL may be NULL when calling function
4475 indirectly or considering a libcall. Otherwise return 0. */
4478 ix86_function_sseregparm (const_tree type, const_tree decl, bool warn)
4480 gcc_assert (!TARGET_64BIT);
4482 /* Use SSE registers to pass SFmode and DFmode arguments if requested
4483 by the sseregparm attribute. */
4484 if (TARGET_SSEREGPARM
4485 || (type && lookup_attribute ("sseregparm", TYPE_ATTRIBUTES (type))))
4492 error ("Calling %qD with attribute sseregparm without "
4493 "SSE/SSE2 enabled", decl);
4495 error ("Calling %qT with attribute sseregparm without "
4496 "SSE/SSE2 enabled", type);
4504 /* For local functions, pass up to SSE_REGPARM_MAX SFmode
4505 (and DFmode for SSE2) arguments in SSE registers. */
4506 if (decl && TARGET_SSE_MATH && optimize && !profile_flag)
4508 /* FIXME: remove this CONST_CAST when cgraph.[ch] is constified. */
4509 struct cgraph_local_info *i = cgraph_local_info (CONST_CAST_TREE(decl));
4511 return TARGET_SSE2 ? 2 : 1;
4517 /* Return true if EAX is live at the start of the function. Used by
4518 ix86_expand_prologue to determine if we need special help before
4519 calling allocate_stack_worker. */
4522 ix86_eax_live_at_start_p (void)
4524 /* Cheat. Don't bother working forward from ix86_function_regparm
4525 to the function type to whether an actual argument is located in
4526 eax. Instead just look at cfg info, which is still close enough
4527 to correct at this point. This gives false positives for broken
4528 functions that might use uninitialized data that happens to be
4529 allocated in eax, but who cares? */
4530 return REGNO_REG_SET_P (df_get_live_out (ENTRY_BLOCK_PTR), 0);
4533 /* Value is the number of bytes of arguments automatically
4534 popped when returning from a subroutine call.
4535 FUNDECL is the declaration node of the function (as a tree),
4536 FUNTYPE is the data type of the function (as a tree),
4537 or for a library call it is an identifier node for the subroutine name.
4538 SIZE is the number of bytes of arguments passed on the stack.
4540 On the 80386, the RTD insn may be used to pop them if the number
4541 of args is fixed, but if the number is variable then the caller
4542 must pop them all. RTD can't be used for library calls now
4543 because the library is compiled with the Unix compiler.
4544 Use of RTD is a selectable option, since it is incompatible with
4545 standard Unix calling sequences. If the option is not selected,
4546 the caller must always pop the args.
4548 The attribute stdcall is equivalent to RTD on a per module basis. */
4551 ix86_return_pops_args (tree fundecl, tree funtype, int size)
4555 /* None of the 64-bit ABIs pop arguments. */
4559 rtd = TARGET_RTD && (!fundecl || TREE_CODE (fundecl) != IDENTIFIER_NODE);
4561 /* Cdecl functions override -mrtd, and never pop the stack. */
4562 if (! lookup_attribute ("cdecl", TYPE_ATTRIBUTES (funtype)))
4564 /* Stdcall and fastcall functions will pop the stack if not
4566 if (lookup_attribute ("stdcall", TYPE_ATTRIBUTES (funtype))
4567 || lookup_attribute ("fastcall", TYPE_ATTRIBUTES (funtype)))
4570 if (rtd && ! stdarg_p (funtype))
4574 /* Lose any fake structure return argument if it is passed on the stack. */
4575 if (aggregate_value_p (TREE_TYPE (funtype), fundecl)
4576 && !KEEP_AGGREGATE_RETURN_POINTER)
4578 int nregs = ix86_function_regparm (funtype, fundecl);
4580 return GET_MODE_SIZE (Pmode);
4586 /* Argument support functions. */
4588 /* Return true when register may be used to pass function parameters. */
4590 ix86_function_arg_regno_p (int regno)
4593 const int *parm_regs;
4598 return (regno < REGPARM_MAX
4599 || (TARGET_SSE && SSE_REGNO_P (regno) && !fixed_regs[regno]));
4601 return (regno < REGPARM_MAX
4602 || (TARGET_MMX && MMX_REGNO_P (regno)
4603 && (regno < FIRST_MMX_REG + MMX_REGPARM_MAX))
4604 || (TARGET_SSE && SSE_REGNO_P (regno)
4605 && (regno < FIRST_SSE_REG + SSE_REGPARM_MAX)));
4610 if (SSE_REGNO_P (regno) && TARGET_SSE)
4615 if (TARGET_SSE && SSE_REGNO_P (regno)
4616 && (regno < FIRST_SSE_REG + SSE_REGPARM_MAX))
4620 /* TODO: The function should depend on current function ABI but
4621 builtins.c would need updating then. Therefore we use the
4624 /* RAX is used as hidden argument to va_arg functions. */
4625 if (ix86_abi == SYSV_ABI && regno == AX_REG)
4628 if (ix86_abi == MS_ABI)
4629 parm_regs = x86_64_ms_abi_int_parameter_registers;
4631 parm_regs = x86_64_int_parameter_registers;
4632 for (i = 0; i < (ix86_abi == MS_ABI ? X64_REGPARM_MAX
4633 : X86_64_REGPARM_MAX); i++)
4634 if (regno == parm_regs[i])
4639 /* Return if we do not know how to pass TYPE solely in registers. */
4642 ix86_must_pass_in_stack (enum machine_mode mode, const_tree type)
4644 if (must_pass_in_stack_var_size_or_pad (mode, type))
4647 /* For 32-bit, we want TImode aggregates to go on the stack. But watch out!
4648 The layout_type routine is crafty and tries to trick us into passing
4649 currently unsupported vector types on the stack by using TImode. */
4650 return (!TARGET_64BIT && mode == TImode
4651 && type && TREE_CODE (type) != VECTOR_TYPE);
4654 /* It returns the size, in bytes, of the area reserved for arguments passed
4655 in registers for the function represented by fndecl dependent to the used
4658 ix86_reg_parm_stack_space (const_tree fndecl)
4660 enum calling_abi call_abi = SYSV_ABI;
4661 if (fndecl != NULL_TREE && TREE_CODE (fndecl) == FUNCTION_DECL)
4662 call_abi = ix86_function_abi (fndecl);
4664 call_abi = ix86_function_type_abi (fndecl);
4665 if (call_abi == MS_ABI)
4670 /* Returns value SYSV_ABI, MS_ABI dependent on fntype, specifying the
4673 ix86_function_type_abi (const_tree fntype)
4675 if (TARGET_64BIT && fntype != NULL)
4677 enum calling_abi abi = ix86_abi;
4678 if (abi == SYSV_ABI)
4680 if (lookup_attribute ("ms_abi", TYPE_ATTRIBUTES (fntype)))
4683 else if (lookup_attribute ("sysv_abi", TYPE_ATTRIBUTES (fntype)))
4690 static enum calling_abi
4691 ix86_function_abi (const_tree fndecl)
4695 return ix86_function_type_abi (TREE_TYPE (fndecl));
4698 /* Returns value SYSV_ABI, MS_ABI dependent on cfun, specifying the
4701 ix86_cfun_abi (void)
4703 if (! cfun || ! TARGET_64BIT)
4705 return cfun->machine->call_abi;
4709 extern void init_regs (void);
4711 /* Implementation of call abi switching target hook. Specific to FNDECL
4712 the specific call register sets are set. See also CONDITIONAL_REGISTER_USAGE
4713 for more details. */
4715 ix86_call_abi_override (const_tree fndecl)
4717 if (fndecl == NULL_TREE)
4718 cfun->machine->call_abi = ix86_abi;
4720 cfun->machine->call_abi = ix86_function_type_abi (TREE_TYPE (fndecl));
4723 /* MS and SYSV ABI have different set of call used registers. Avoid expensive
4724 re-initialization of init_regs each time we switch function context since
4725 this is needed only during RTL expansion. */
4727 ix86_maybe_switch_abi (void)
4730 call_used_regs[SI_REG] == (cfun->machine->call_abi == MS_ABI))
4734 /* Initialize a variable CUM of type CUMULATIVE_ARGS
4735 for a call to a function whose data type is FNTYPE.
4736 For a library call, FNTYPE is 0. */
4739 init_cumulative_args (CUMULATIVE_ARGS *cum, /* Argument info to initialize */
4740 tree fntype, /* tree ptr for function decl */
4741 rtx libname, /* SYMBOL_REF of library name or 0 */
4744 struct cgraph_local_info *i = fndecl ? cgraph_local_info (fndecl) : NULL;
4745 memset (cum, 0, sizeof (*cum));
4748 cum->call_abi = ix86_function_abi (fndecl);
4750 cum->call_abi = ix86_function_type_abi (fntype);
4751 /* Set up the number of registers to use for passing arguments. */
4753 if (cum->call_abi == MS_ABI && !ACCUMULATE_OUTGOING_ARGS)
4754 sorry ("ms_abi attribute require -maccumulate-outgoing-args or subtarget optimization implying it");
4755 cum->nregs = ix86_regparm;
4758 if (cum->call_abi != ix86_abi)
4759 cum->nregs = ix86_abi != SYSV_ABI ? X86_64_REGPARM_MAX
4764 cum->sse_nregs = SSE_REGPARM_MAX;
4767 if (cum->call_abi != ix86_abi)
4768 cum->sse_nregs = ix86_abi != SYSV_ABI ? X86_64_SSE_REGPARM_MAX
4769 : X64_SSE_REGPARM_MAX;
4773 cum->mmx_nregs = MMX_REGPARM_MAX;
4774 cum->warn_avx = true;
4775 cum->warn_sse = true;
4776 cum->warn_mmx = true;
4778 /* Because type might mismatch in between caller and callee, we need to
4779 use actual type of function for local calls.
4780 FIXME: cgraph_analyze can be told to actually record if function uses
4781 va_start so for local functions maybe_vaarg can be made aggressive
4783 FIXME: once typesytem is fixed, we won't need this code anymore. */
4785 fntype = TREE_TYPE (fndecl);
4786 cum->maybe_vaarg = (fntype
4787 ? (!prototype_p (fntype) || stdarg_p (fntype))
4792 /* If there are variable arguments, then we won't pass anything
4793 in registers in 32-bit mode. */
4794 if (stdarg_p (fntype))
4805 /* Use ecx and edx registers if function has fastcall attribute,
4806 else look for regparm information. */
4809 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (fntype)))
4815 cum->nregs = ix86_function_regparm (fntype, fndecl);
4818 /* Set up the number of SSE registers used for passing SFmode
4819 and DFmode arguments. Warn for mismatching ABI. */
4820 cum->float_in_sse = ix86_function_sseregparm (fntype, fndecl, true);
4824 /* Return the "natural" mode for TYPE. In most cases, this is just TYPE_MODE.
4825 But in the case of vector types, it is some vector mode.
4827 When we have only some of our vector isa extensions enabled, then there
4828 are some modes for which vector_mode_supported_p is false. For these
4829 modes, the generic vector support in gcc will choose some non-vector mode
4830 in order to implement the type. By computing the natural mode, we'll
4831 select the proper ABI location for the operand and not depend on whatever
4832 the middle-end decides to do with these vector types.
4834 The midde-end can't deal with the vector types > 16 bytes. In this
4835 case, we return the original mode and warn ABI change if CUM isn't
4838 static enum machine_mode
4839 type_natural_mode (const_tree type, CUMULATIVE_ARGS *cum)
4841 enum machine_mode mode = TYPE_MODE (type);
4843 if (TREE_CODE (type) == VECTOR_TYPE && !VECTOR_MODE_P (mode))
4845 HOST_WIDE_INT size = int_size_in_bytes (type);
4846 if ((size == 8 || size == 16 || size == 32)
4847 /* ??? Generic code allows us to create width 1 vectors. Ignore. */
4848 && TYPE_VECTOR_SUBPARTS (type) > 1)
4850 enum machine_mode innermode = TYPE_MODE (TREE_TYPE (type));
4852 if (TREE_CODE (TREE_TYPE (type)) == REAL_TYPE)
4853 mode = MIN_MODE_VECTOR_FLOAT;
4855 mode = MIN_MODE_VECTOR_INT;
4857 /* Get the mode which has this inner mode and number of units. */
4858 for (; mode != VOIDmode; mode = GET_MODE_WIDER_MODE (mode))
4859 if (GET_MODE_NUNITS (mode) == TYPE_VECTOR_SUBPARTS (type)
4860 && GET_MODE_INNER (mode) == innermode)
4862 if (size == 32 && !TARGET_AVX)
4864 static bool warnedavx;
4871 warning (0, "AVX vector argument without AVX "
4872 "enabled changes the ABI");
4874 return TYPE_MODE (type);
4887 /* We want to pass a value in REGNO whose "natural" mode is MODE. However,
4888 this may not agree with the mode that the type system has chosen for the
4889 register, which is ORIG_MODE. If ORIG_MODE is not BLKmode, then we can
4890 go ahead and use it. Otherwise we have to build a PARALLEL instead. */
4893 gen_reg_or_parallel (enum machine_mode mode, enum machine_mode orig_mode,
4898 if (orig_mode != BLKmode)
4899 tmp = gen_rtx_REG (orig_mode, regno);
4902 tmp = gen_rtx_REG (mode, regno);
4903 tmp = gen_rtx_EXPR_LIST (VOIDmode, tmp, const0_rtx);
4904 tmp = gen_rtx_PARALLEL (orig_mode, gen_rtvec (1, tmp));
4910 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
4911 of this code is to classify each 8bytes of incoming argument by the register
4912 class and assign registers accordingly. */
4914 /* Return the union class of CLASS1 and CLASS2.
4915 See the x86-64 PS ABI for details. */
4917 static enum x86_64_reg_class
4918 merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
4920 /* Rule #1: If both classes are equal, this is the resulting class. */
4921 if (class1 == class2)
4924 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
4926 if (class1 == X86_64_NO_CLASS)
4928 if (class2 == X86_64_NO_CLASS)
4931 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
4932 if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
4933 return X86_64_MEMORY_CLASS;
4935 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
4936 if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
4937 || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
4938 return X86_64_INTEGERSI_CLASS;
4939 if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
4940 || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
4941 return X86_64_INTEGER_CLASS;
4943 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
4945 if (class1 == X86_64_X87_CLASS
4946 || class1 == X86_64_X87UP_CLASS
4947 || class1 == X86_64_COMPLEX_X87_CLASS
4948 || class2 == X86_64_X87_CLASS
4949 || class2 == X86_64_X87UP_CLASS
4950 || class2 == X86_64_COMPLEX_X87_CLASS)
4951 return X86_64_MEMORY_CLASS;
4953 /* Rule #6: Otherwise class SSE is used. */
4954 return X86_64_SSE_CLASS;
4957 /* Classify the argument of type TYPE and mode MODE.
4958 CLASSES will be filled by the register class used to pass each word
4959 of the operand. The number of words is returned. In case the parameter
4960 should be passed in memory, 0 is returned. As a special case for zero
4961 sized containers, classes[0] will be NO_CLASS and 1 is returned.
4963 BIT_OFFSET is used internally for handling records and specifies offset
4964 of the offset in bits modulo 256 to avoid overflow cases.
4966 See the x86-64 PS ABI for details.
4970 classify_argument (enum machine_mode mode, const_tree type,
4971 enum x86_64_reg_class classes[MAX_CLASSES], int bit_offset)
4973 HOST_WIDE_INT bytes =
4974 (mode == BLKmode) ? int_size_in_bytes (type) : (int) GET_MODE_SIZE (mode);
4975 int words = (bytes + (bit_offset % 64) / 8 + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
4977 /* Variable sized entities are always passed/returned in memory. */
4981 if (mode != VOIDmode
4982 && targetm.calls.must_pass_in_stack (mode, type))
4985 if (type && AGGREGATE_TYPE_P (type))
4989 enum x86_64_reg_class subclasses[MAX_CLASSES];
4991 /* On x86-64 we pass structures larger than 32 bytes on the stack. */
4995 for (i = 0; i < words; i++)
4996 classes[i] = X86_64_NO_CLASS;
4998 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
4999 signalize memory class, so handle it as special case. */
5002 classes[0] = X86_64_NO_CLASS;
5006 /* Classify each field of record and merge classes. */
5007 switch (TREE_CODE (type))
5010 /* And now merge the fields of structure. */
5011 for (field = TYPE_FIELDS (type); field; field = TREE_CHAIN (field))
5013 if (TREE_CODE (field) == FIELD_DECL)
5017 if (TREE_TYPE (field) == error_mark_node)
5020 /* Bitfields are always classified as integer. Handle them
5021 early, since later code would consider them to be
5022 misaligned integers. */
5023 if (DECL_BIT_FIELD (field))
5025 for (i = (int_bit_position (field) + (bit_offset % 64)) / 8 / 8;
5026 i < ((int_bit_position (field) + (bit_offset % 64))
5027 + tree_low_cst (DECL_SIZE (field), 0)
5030 merge_classes (X86_64_INTEGER_CLASS,
5037 type = TREE_TYPE (field);
5039 /* Flexible array member is ignored. */
5040 if (TYPE_MODE (type) == BLKmode
5041 && TREE_CODE (type) == ARRAY_TYPE
5042 && TYPE_SIZE (type) == NULL_TREE
5043 && TYPE_DOMAIN (type) != NULL_TREE
5044 && (TYPE_MAX_VALUE (TYPE_DOMAIN (type))
5049 if (!warned && warn_psabi)
5052 inform (input_location,
5053 "The ABI of passing struct with"
5054 " a flexible array member has"
5055 " changed in GCC 4.4");
5059 num = classify_argument (TYPE_MODE (type), type,
5061 (int_bit_position (field)
5062 + bit_offset) % 256);
5065 pos = (int_bit_position (field) + (bit_offset % 64)) / 8 / 8;
5066 for (i = 0; i < num && (i + pos) < words; i++)
5068 merge_classes (subclasses[i], classes[i + pos]);
5075 /* Arrays are handled as small records. */
5078 num = classify_argument (TYPE_MODE (TREE_TYPE (type)),
5079 TREE_TYPE (type), subclasses, bit_offset);
5083 /* The partial classes are now full classes. */
5084 if (subclasses[0] == X86_64_SSESF_CLASS && bytes != 4)
5085 subclasses[0] = X86_64_SSE_CLASS;
5086 if (subclasses[0] == X86_64_INTEGERSI_CLASS
5087 && !((bit_offset % 64) == 0 && bytes == 4))
5088 subclasses[0] = X86_64_INTEGER_CLASS;
5090 for (i = 0; i < words; i++)
5091 classes[i] = subclasses[i % num];
5096 case QUAL_UNION_TYPE:
5097 /* Unions are similar to RECORD_TYPE but offset is always 0.
5099 for (field = TYPE_FIELDS (type); field; field = TREE_CHAIN (field))
5101 if (TREE_CODE (field) == FIELD_DECL)
5105 if (TREE_TYPE (field) == error_mark_node)
5108 num = classify_argument (TYPE_MODE (TREE_TYPE (field)),
5109 TREE_TYPE (field), subclasses,
5113 for (i = 0; i < num; i++)
5114 classes[i] = merge_classes (subclasses[i], classes[i]);
5125 /* When size > 16 bytes, if the first one isn't
5126 X86_64_SSE_CLASS or any other ones aren't
5127 X86_64_SSEUP_CLASS, everything should be passed in
5129 if (classes[0] != X86_64_SSE_CLASS)
5132 for (i = 1; i < words; i++)
5133 if (classes[i] != X86_64_SSEUP_CLASS)
5137 /* Final merger cleanup. */
5138 for (i = 0; i < words; i++)
5140 /* If one class is MEMORY, everything should be passed in
5142 if (classes[i] == X86_64_MEMORY_CLASS)
5145 /* The X86_64_SSEUP_CLASS should be always preceded by
5146 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
5147 if (classes[i] == X86_64_SSEUP_CLASS
5148 && classes[i - 1] != X86_64_SSE_CLASS
5149 && classes[i - 1] != X86_64_SSEUP_CLASS)
5151 /* The first one should never be X86_64_SSEUP_CLASS. */
5152 gcc_assert (i != 0);
5153 classes[i] = X86_64_SSE_CLASS;
5156 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
5157 everything should be passed in memory. */
5158 if (classes[i] == X86_64_X87UP_CLASS
5159 && (classes[i - 1] != X86_64_X87_CLASS))
5163 /* The first one should never be X86_64_X87UP_CLASS. */
5164 gcc_assert (i != 0);
5165 if (!warned && warn_psabi)
5168 inform (input_location,
5169 "The ABI of passing union with long double"
5170 " has changed in GCC 4.4");
5178 /* Compute alignment needed. We align all types to natural boundaries with
5179 exception of XFmode that is aligned to 64bits. */
5180 if (mode != VOIDmode && mode != BLKmode)
5182 int mode_alignment = GET_MODE_BITSIZE (mode);
5185 mode_alignment = 128;
5186 else if (mode == XCmode)
5187 mode_alignment = 256;
5188 if (COMPLEX_MODE_P (mode))
5189 mode_alignment /= 2;
5190 /* Misaligned fields are always returned in memory. */
5191 if (bit_offset % mode_alignment)
5195 /* for V1xx modes, just use the base mode */
5196 if (VECTOR_MODE_P (mode) && mode != V1DImode
5197 && GET_MODE_SIZE (GET_MODE_INNER (mode)) == bytes)
5198 mode = GET_MODE_INNER (mode);
5200 /* Classification of atomic types. */
5205 classes[0] = X86_64_SSE_CLASS;
5208 classes[0] = X86_64_SSE_CLASS;
5209 classes[1] = X86_64_SSEUP_CLASS;
5219 int size = (bit_offset % 64)+ (int) GET_MODE_BITSIZE (mode);
5223 classes[0] = X86_64_INTEGERSI_CLASS;
5226 else if (size <= 64)
5228 classes[0] = X86_64_INTEGER_CLASS;
5231 else if (size <= 64+32)
5233 classes[0] = X86_64_INTEGER_CLASS;
5234 classes[1] = X86_64_INTEGERSI_CLASS;
5237 else if (size <= 64+64)
5239 classes[0] = classes[1] = X86_64_INTEGER_CLASS;
5247 classes[0] = classes[1] = X86_64_INTEGER_CLASS;
5251 /* OImode shouldn't be used directly. */
5256 if (!(bit_offset % 64))
5257 classes[0] = X86_64_SSESF_CLASS;
5259 classes[0] = X86_64_SSE_CLASS;
5262 classes[0] = X86_64_SSEDF_CLASS;
5265 classes[0] = X86_64_X87_CLASS;
5266 classes[1] = X86_64_X87UP_CLASS;
5269 classes[0] = X86_64_SSE_CLASS;
5270 classes[1] = X86_64_SSEUP_CLASS;
5273 classes[0] = X86_64_SSE_CLASS;
5274 if (!(bit_offset % 64))
5280 if (!warned && warn_psabi)
5283 inform (input_location,
5284 "The ABI of passing structure with complex float"
5285 " member has changed in GCC 4.4");
5287 classes[1] = X86_64_SSESF_CLASS;
5291 classes[0] = X86_64_SSEDF_CLASS;
5292 classes[1] = X86_64_SSEDF_CLASS;
5295 classes[0] = X86_64_COMPLEX_X87_CLASS;
5298 /* This modes is larger than 16 bytes. */
5306 classes[0] = X86_64_SSE_CLASS;
5307 classes[1] = X86_64_SSEUP_CLASS;
5308 classes[2] = X86_64_SSEUP_CLASS;
5309 classes[3] = X86_64_SSEUP_CLASS;
5317 classes[0] = X86_64_SSE_CLASS;
5318 classes[1] = X86_64_SSEUP_CLASS;
5325 classes[0] = X86_64_SSE_CLASS;
5331 gcc_assert (VECTOR_MODE_P (mode));
5336 gcc_assert (GET_MODE_CLASS (GET_MODE_INNER (mode)) == MODE_INT);
5338 if (bit_offset + GET_MODE_BITSIZE (mode) <= 32)
5339 classes[0] = X86_64_INTEGERSI_CLASS;
5341 classes[0] = X86_64_INTEGER_CLASS;
5342 classes[1] = X86_64_INTEGER_CLASS;
5343 return 1 + (bytes > 8);
5347 /* Examine the argument and return set number of register required in each
5348 class. Return 0 iff parameter should be passed in memory. */
5350 examine_argument (enum machine_mode mode, const_tree type, int in_return,
5351 int *int_nregs, int *sse_nregs)
5353 enum x86_64_reg_class regclass[MAX_CLASSES];
5354 int n = classify_argument (mode, type, regclass, 0);
5360 for (n--; n >= 0; n--)
5361 switch (regclass[n])
5363 case X86_64_INTEGER_CLASS:
5364 case X86_64_INTEGERSI_CLASS:
5367 case X86_64_SSE_CLASS:
5368 case X86_64_SSESF_CLASS:
5369 case X86_64_SSEDF_CLASS:
5372 case X86_64_NO_CLASS:
5373 case X86_64_SSEUP_CLASS:
5375 case X86_64_X87_CLASS:
5376 case X86_64_X87UP_CLASS:
5380 case X86_64_COMPLEX_X87_CLASS:
5381 return in_return ? 2 : 0;
5382 case X86_64_MEMORY_CLASS:
5388 /* Construct container for the argument used by GCC interface. See
5389 FUNCTION_ARG for the detailed description. */
5392 construct_container (enum machine_mode mode, enum machine_mode orig_mode,
5393 const_tree type, int in_return, int nintregs, int nsseregs,
5394 const int *intreg, int sse_regno)
5396 /* The following variables hold the static issued_error state. */
5397 static bool issued_sse_arg_error;
5398 static bool issued_sse_ret_error;
5399 static bool issued_x87_ret_error;
5401 enum machine_mode tmpmode;
5403 (mode == BLKmode) ? int_size_in_bytes (type) : (int) GET_MODE_SIZE (mode);
5404 enum x86_64_reg_class regclass[MAX_CLASSES];
5408 int needed_sseregs, needed_intregs;
5409 rtx exp[MAX_CLASSES];
5412 n = classify_argument (mode, type, regclass, 0);
5415 if (!examine_argument (mode, type, in_return, &needed_intregs,
5418 if (needed_intregs > nintregs || needed_sseregs > nsseregs)
5421 /* We allowed the user to turn off SSE for kernel mode. Don't crash if
5422 some less clueful developer tries to use floating-point anyway. */
5423 if (needed_sseregs && !TARGET_SSE)
5427 if (!issued_sse_ret_error)
5429 error ("SSE register return with SSE disabled");
5430 issued_sse_ret_error = true;
5433 else if (!issued_sse_arg_error)
5435 error ("SSE register argument with SSE disabled");
5436 issued_sse_arg_error = true;
5441 /* Likewise, error if the ABI requires us to return values in the
5442 x87 registers and the user specified -mno-80387. */
5443 if (!TARGET_80387 && in_return)
5444 for (i = 0; i < n; i++)
5445 if (regclass[i] == X86_64_X87_CLASS
5446 || regclass[i] == X86_64_X87UP_CLASS
5447 || regclass[i] == X86_64_COMPLEX_X87_CLASS)
5449 if (!issued_x87_ret_error)
5451 error ("x87 register return with x87 disabled");
5452 issued_x87_ret_error = true;
5457 /* First construct simple cases. Avoid SCmode, since we want to use
5458 single register to pass this type. */
5459 if (n == 1 && mode != SCmode)
5460 switch (regclass[0])
5462 case X86_64_INTEGER_CLASS:
5463 case X86_64_INTEGERSI_CLASS:
5464 return gen_rtx_REG (mode, intreg[0]);
5465 case X86_64_SSE_CLASS:
5466 case X86_64_SSESF_CLASS:
5467 case X86_64_SSEDF_CLASS:
5468 if (mode != BLKmode)
5469 return gen_reg_or_parallel (mode, orig_mode,
5470 SSE_REGNO (sse_regno));
5472 case X86_64_X87_CLASS:
5473 case X86_64_COMPLEX_X87_CLASS:
5474 return gen_rtx_REG (mode, FIRST_STACK_REG);
5475 case X86_64_NO_CLASS:
5476 /* Zero sized array, struct or class. */
5481 if (n == 2 && regclass[0] == X86_64_SSE_CLASS
5482 && regclass[1] == X86_64_SSEUP_CLASS && mode != BLKmode)
5483 return gen_rtx_REG (mode, SSE_REGNO (sse_regno));
5485 && regclass[0] == X86_64_SSE_CLASS
5486 && regclass[1] == X86_64_SSEUP_CLASS
5487 && regclass[2] == X86_64_SSEUP_CLASS
5488 && regclass[3] == X86_64_SSEUP_CLASS
5490 return gen_rtx_REG (mode, SSE_REGNO (sse_regno));
5493 && regclass[0] == X86_64_X87_CLASS && regclass[1] == X86_64_X87UP_CLASS)
5494 return gen_rtx_REG (XFmode, FIRST_STACK_REG);
5495 if (n == 2 && regclass[0] == X86_64_INTEGER_CLASS
5496 && regclass[1] == X86_64_INTEGER_CLASS
5497 && (mode == CDImode || mode == TImode || mode == TFmode)
5498 && intreg[0] + 1 == intreg[1])
5499 return gen_rtx_REG (mode, intreg[0]);
5501 /* Otherwise figure out the entries of the PARALLEL. */
5502 for (i = 0; i < n; i++)
5506 switch (regclass[i])
5508 case X86_64_NO_CLASS:
5510 case X86_64_INTEGER_CLASS:
5511 case X86_64_INTEGERSI_CLASS:
5512 /* Merge TImodes on aligned occasions here too. */
5513 if (i * 8 + 8 > bytes)
5514 tmpmode = mode_for_size ((bytes - i * 8) * BITS_PER_UNIT, MODE_INT, 0);
5515 else if (regclass[i] == X86_64_INTEGERSI_CLASS)
5519 /* We've requested 24 bytes we don't have mode for. Use DImode. */
5520 if (tmpmode == BLKmode)
5522 exp [nexps++] = gen_rtx_EXPR_LIST (VOIDmode,
5523 gen_rtx_REG (tmpmode, *intreg),
5527 case X86_64_SSESF_CLASS:
5528 exp [nexps++] = gen_rtx_EXPR_LIST (VOIDmode,
5529 gen_rtx_REG (SFmode,
5530 SSE_REGNO (sse_regno)),
5534 case X86_64_SSEDF_CLASS:
5535 exp [nexps++] = gen_rtx_EXPR_LIST (VOIDmode,
5536 gen_rtx_REG (DFmode,
5537 SSE_REGNO (sse_regno)),
5541 case X86_64_SSE_CLASS:
5549 if (i == 0 && regclass[1] == X86_64_SSEUP_CLASS)
5559 && regclass[1] == X86_64_SSEUP_CLASS
5560 && regclass[2] == X86_64_SSEUP_CLASS
5561 && regclass[3] == X86_64_SSEUP_CLASS);
5568 exp [nexps++] = gen_rtx_EXPR_LIST (VOIDmode,
5569 gen_rtx_REG (tmpmode,
5570 SSE_REGNO (sse_regno)),
5579 /* Empty aligned struct, union or class. */
5583 ret = gen_rtx_PARALLEL (mode, rtvec_alloc (nexps));
5584 for (i = 0; i < nexps; i++)
5585 XVECEXP (ret, 0, i) = exp [i];
5589 /* Update the data in CUM to advance over an argument of mode MODE
5590 and data type TYPE. (TYPE is null for libcalls where that information
5591 may not be available.) */
5594 function_arg_advance_32 (CUMULATIVE_ARGS *cum, enum machine_mode mode,
5595 tree type, HOST_WIDE_INT bytes, HOST_WIDE_INT words)
5611 cum->words += words;
5612 cum->nregs -= words;
5613 cum->regno += words;
5615 if (cum->nregs <= 0)
5623 /* OImode shouldn't be used directly. */
5627 if (cum->float_in_sse < 2)
5630 if (cum->float_in_sse < 1)
5647 if (!type || !AGGREGATE_TYPE_P (type))
5649 cum->sse_words += words;
5650 cum->sse_nregs -= 1;
5651 cum->sse_regno += 1;
5652 if (cum->sse_nregs <= 0)
5665 if (!type || !AGGREGATE_TYPE_P (type))
5667 cum->mmx_words += words;
5668 cum->mmx_nregs -= 1;
5669 cum->mmx_regno += 1;
5670 if (cum->mmx_nregs <= 0)
5681 function_arg_advance_64 (CUMULATIVE_ARGS *cum, enum machine_mode mode,
5682 tree type, HOST_WIDE_INT words, int named)
5684 int int_nregs, sse_nregs;
5686 /* Unnamed 256bit vector mode parameters are passed on stack. */
5687 if (!named && VALID_AVX256_REG_MODE (mode))
5690 if (!examine_argument (mode, type, 0, &int_nregs, &sse_nregs))
5691 cum->words += words;
5692 else if (sse_nregs <= cum->sse_nregs && int_nregs <= cum->nregs)
5694 cum->nregs -= int_nregs;
5695 cum->sse_nregs -= sse_nregs;
5696 cum->regno += int_nregs;
5697 cum->sse_regno += sse_nregs;
5700 cum->words += words;
5704 function_arg_advance_ms_64 (CUMULATIVE_ARGS *cum, HOST_WIDE_INT bytes,
5705 HOST_WIDE_INT words)
5707 /* Otherwise, this should be passed indirect. */
5708 gcc_assert (bytes == 1 || bytes == 2 || bytes == 4 || bytes == 8);
5710 cum->words += words;
5719 function_arg_advance (CUMULATIVE_ARGS *cum, enum machine_mode mode,
5720 tree type, int named)
5722 HOST_WIDE_INT bytes, words;
5724 if (mode == BLKmode)
5725 bytes = int_size_in_bytes (type);
5727 bytes = GET_MODE_SIZE (mode);
5728 words = (bytes + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
5731 mode = type_natural_mode (type, NULL);
5733 if (TARGET_64BIT && (cum ? cum->call_abi : ix86_abi) == MS_ABI)
5734 function_arg_advance_ms_64 (cum, bytes, words);
5735 else if (TARGET_64BIT)
5736 function_arg_advance_64 (cum, mode, type, words, named);
5738 function_arg_advance_32 (cum, mode, type, bytes, words);
5741 /* Define where to put the arguments to a function.
5742 Value is zero to push the argument on the stack,
5743 or a hard register in which to store the argument.
5745 MODE is the argument's machine mode.
5746 TYPE is the data type of the argument (as a tree).
5747 This is null for libcalls where that information may
5749 CUM is a variable of type CUMULATIVE_ARGS which gives info about
5750 the preceding args and about the function being called.
5751 NAMED is nonzero if this argument is a named parameter
5752 (otherwise it is an extra parameter matching an ellipsis). */
5755 function_arg_32 (CUMULATIVE_ARGS *cum, enum machine_mode mode,
5756 enum machine_mode orig_mode, tree type,
5757 HOST_WIDE_INT bytes, HOST_WIDE_INT words)
5759 static bool warnedsse, warnedmmx;
5761 /* Avoid the AL settings for the Unix64 ABI. */
5762 if (mode == VOIDmode)
5778 if (words <= cum->nregs)
5780 int regno = cum->regno;
5782 /* Fastcall allocates the first two DWORD (SImode) or
5783 smaller arguments to ECX and EDX if it isn't an
5789 || (type && AGGREGATE_TYPE_P (type)))
5792 /* ECX not EAX is the first allocated register. */
5793 if (regno == AX_REG)
5796 return gen_rtx_REG (mode, regno);
5801 if (cum->float_in_sse < 2)
5804 if (cum->float_in_sse < 1)
5808 /* In 32bit, we pass TImode in xmm registers. */
5815 if (!type || !AGGREGATE_TYPE_P (type))
5817 if (!TARGET_SSE && !warnedsse && cum->warn_sse)
5820 warning (0, "SSE vector argument without SSE enabled "
5824 return gen_reg_or_parallel (mode, orig_mode,
5825 cum->sse_regno + FIRST_SSE_REG);
5830 /* OImode shouldn't be used directly. */
5839 if (!type || !AGGREGATE_TYPE_P (type))
5842 return gen_reg_or_parallel (mode, orig_mode,
5843 cum->sse_regno + FIRST_SSE_REG);
5852 if (!type || !AGGREGATE_TYPE_P (type))
5854 if (!TARGET_MMX && !warnedmmx && cum->warn_mmx)
5857 warning (0, "MMX vector argument without MMX enabled "
5861 return gen_reg_or_parallel (mode, orig_mode,
5862 cum->mmx_regno + FIRST_MMX_REG);
5871 function_arg_64 (CUMULATIVE_ARGS *cum, enum machine_mode mode,
5872 enum machine_mode orig_mode, tree type, int named)
5874 /* Handle a hidden AL argument containing number of registers
5875 for varargs x86-64 functions. */
5876 if (mode == VOIDmode)
5877 return GEN_INT (cum->maybe_vaarg
5878 ? (cum->sse_nregs < 0
5879 ? (cum->call_abi == ix86_abi
5881 : (ix86_abi != SYSV_ABI ? X86_64_SSE_REGPARM_MAX
5882 : X64_SSE_REGPARM_MAX))
5897 /* Unnamed 256bit vector mode parameters are passed on stack. */
5903 return construct_container (mode, orig_mode, type, 0, cum->nregs,
5905 &x86_64_int_parameter_registers [cum->regno],
5910 function_arg_ms_64 (CUMULATIVE_ARGS *cum, enum machine_mode mode,
5911 enum machine_mode orig_mode, int named,
5912 HOST_WIDE_INT bytes)
5916 /* We need to add clobber for MS_ABI->SYSV ABI calls in expand_call.
5917 We use value of -2 to specify that current function call is MSABI. */
5918 if (mode == VOIDmode)
5919 return GEN_INT (-2);
5921 /* If we've run out of registers, it goes on the stack. */
5922 if (cum->nregs == 0)
5925 regno = x86_64_ms_abi_int_parameter_registers[cum->regno];
5927 /* Only floating point modes are passed in anything but integer regs. */
5928 if (TARGET_SSE && (mode == SFmode || mode == DFmode))
5931 regno = cum->regno + FIRST_SSE_REG;
5936 /* Unnamed floating parameters are passed in both the
5937 SSE and integer registers. */
5938 t1 = gen_rtx_REG (mode, cum->regno + FIRST_SSE_REG);
5939 t2 = gen_rtx_REG (mode, regno);
5940 t1 = gen_rtx_EXPR_LIST (VOIDmode, t1, const0_rtx);
5941 t2 = gen_rtx_EXPR_LIST (VOIDmode, t2, const0_rtx);
5942 return gen_rtx_PARALLEL (mode, gen_rtvec (2, t1, t2));
5945 /* Handle aggregated types passed in register. */
5946 if (orig_mode == BLKmode)
5948 if (bytes > 0 && bytes <= 8)
5949 mode = (bytes > 4 ? DImode : SImode);
5950 if (mode == BLKmode)
5954 return gen_reg_or_parallel (mode, orig_mode, regno);
5958 function_arg (CUMULATIVE_ARGS *cum, enum machine_mode omode,
5959 tree type, int named)
5961 enum machine_mode mode = omode;
5962 HOST_WIDE_INT bytes, words;
5964 if (mode == BLKmode)
5965 bytes = int_size_in_bytes (type);
5967 bytes = GET_MODE_SIZE (mode);
5968 words = (bytes + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
5970 /* To simplify the code below, represent vector types with a vector mode
5971 even if MMX/SSE are not active. */
5972 if (type && TREE_CODE (type) == VECTOR_TYPE)
5973 mode = type_natural_mode (type, cum);
5975 if (TARGET_64BIT && (cum ? cum->call_abi : ix86_abi) == MS_ABI)
5976 return function_arg_ms_64 (cum, mode, omode, named, bytes);
5977 else if (TARGET_64BIT)
5978 return function_arg_64 (cum, mode, omode, type, named);
5980 return function_arg_32 (cum, mode, omode, type, bytes, words);
5983 /* A C expression that indicates when an argument must be passed by
5984 reference. If nonzero for an argument, a copy of that argument is
5985 made in memory and a pointer to the argument is passed instead of
5986 the argument itself. The pointer is passed in whatever way is
5987 appropriate for passing a pointer to that type. */
5990 ix86_pass_by_reference (CUMULATIVE_ARGS *cum ATTRIBUTE_UNUSED,
5991 enum machine_mode mode ATTRIBUTE_UNUSED,
5992 const_tree type, bool named ATTRIBUTE_UNUSED)
5994 /* See Windows x64 Software Convention. */
5995 if (TARGET_64BIT && (cum ? cum->call_abi : ix86_abi) == MS_ABI)
5997 int msize = (int) GET_MODE_SIZE (mode);
6000 /* Arrays are passed by reference. */
6001 if (TREE_CODE (type) == ARRAY_TYPE)
6004 if (AGGREGATE_TYPE_P (type))
6006 /* Structs/unions of sizes other than 8, 16, 32, or 64 bits
6007 are passed by reference. */
6008 msize = int_size_in_bytes (type);
6012 /* __m128 is passed by reference. */
6014 case 1: case 2: case 4: case 8:
6020 else if (TARGET_64BIT && type && int_size_in_bytes (type) == -1)
6026 /* Return true when TYPE should be 128bit aligned for 32bit argument passing
6029 contains_aligned_value_p (tree type)
6031 enum machine_mode mode = TYPE_MODE (type);
6032 if (((TARGET_SSE && SSE_REG_MODE_P (mode))
6036 && (!TYPE_USER_ALIGN (type) || TYPE_ALIGN (type) > 128))
6038 if (TYPE_ALIGN (type) < 128)
6041 if (AGGREGATE_TYPE_P (type))
6043 /* Walk the aggregates recursively. */
6044 switch (TREE_CODE (type))
6048 case QUAL_UNION_TYPE:
6052 /* Walk all the structure fields. */
6053 for (field = TYPE_FIELDS (type); field; field = TREE_CHAIN (field))
6055 if (TREE_CODE (field) == FIELD_DECL
6056 && contains_aligned_value_p (TREE_TYPE (field)))
6063 /* Just for use if some languages passes arrays by value. */
6064 if (contains_aligned_value_p (TREE_TYPE (type)))
6075 /* Gives the alignment boundary, in bits, of an argument with the
6076 specified mode and type. */
6079 ix86_function_arg_boundary (enum machine_mode mode, tree type)
6084 /* Since canonical type is used for call, we convert it to
6085 canonical type if needed. */
6086 if (!TYPE_STRUCTURAL_EQUALITY_P (type))
6087 type = TYPE_CANONICAL (type);
6088 align = TYPE_ALIGN (type);
6091 align = GET_MODE_ALIGNMENT (mode);
6092 if (align < PARM_BOUNDARY)
6093 align = PARM_BOUNDARY;
6094 /* In 32bit, only _Decimal128 and __float128 are aligned to their
6095 natural boundaries. */
6096 if (!TARGET_64BIT && mode != TDmode && mode != TFmode)
6098 /* i386 ABI defines all arguments to be 4 byte aligned. We have to
6099 make an exception for SSE modes since these require 128bit
6102 The handling here differs from field_alignment. ICC aligns MMX
6103 arguments to 4 byte boundaries, while structure fields are aligned
6104 to 8 byte boundaries. */
6107 if (!(TARGET_SSE && SSE_REG_MODE_P (mode)))
6108 align = PARM_BOUNDARY;
6112 if (!contains_aligned_value_p (type))
6113 align = PARM_BOUNDARY;
6116 if (align > BIGGEST_ALIGNMENT)
6117 align = BIGGEST_ALIGNMENT;
6121 /* Return true if N is a possible register number of function value. */
6124 ix86_function_value_regno_p (int regno)
6131 case FIRST_FLOAT_REG:
6132 /* TODO: The function should depend on current function ABI but
6133 builtins.c would need updating then. Therefore we use the
6135 if (TARGET_64BIT && ix86_abi == MS_ABI)
6137 return TARGET_FLOAT_RETURNS_IN_80387;
6143 if (TARGET_MACHO || TARGET_64BIT)
6151 /* Define how to find the value returned by a function.
6152 VALTYPE is the data type of the value (as a tree).
6153 If the precise function being called is known, FUNC is its FUNCTION_DECL;
6154 otherwise, FUNC is 0. */
6157 function_value_32 (enum machine_mode orig_mode, enum machine_mode mode,
6158 const_tree fntype, const_tree fn)
6162 /* 8-byte vector modes in %mm0. See ix86_return_in_memory for where
6163 we normally prevent this case when mmx is not available. However
6164 some ABIs may require the result to be returned like DImode. */
6165 if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 8)
6166 regno = TARGET_MMX ? FIRST_MMX_REG : 0;
6168 /* 16-byte vector modes in %xmm0. See ix86_return_in_memory for where
6169 we prevent this case when sse is not available. However some ABIs
6170 may require the result to be returned like integer TImode. */
6171 else if (mode == TImode
6172 || (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 16))
6173 regno = TARGET_SSE ? FIRST_SSE_REG : 0;
6175 /* 32-byte vector modes in %ymm0. */
6176 else if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 32)
6177 regno = TARGET_AVX ? FIRST_SSE_REG : 0;
6179 /* Floating point return values in %st(0) (unless -mno-fp-ret-in-387). */
6180 else if (X87_FLOAT_MODE_P (mode) && TARGET_FLOAT_RETURNS_IN_80387)
6181 regno = FIRST_FLOAT_REG;
6183 /* Most things go in %eax. */
6186 /* Override FP return register with %xmm0 for local functions when
6187 SSE math is enabled or for functions with sseregparm attribute. */
6188 if ((fn || fntype) && (mode == SFmode || mode == DFmode))
6190 int sse_level = ix86_function_sseregparm (fntype, fn, false);
6191 if ((sse_level >= 1 && mode == SFmode)
6192 || (sse_level == 2 && mode == DFmode))
6193 regno = FIRST_SSE_REG;
6196 /* OImode shouldn't be used directly. */
6197 gcc_assert (mode != OImode);
6199 return gen_rtx_REG (orig_mode, regno);
6203 function_value_64 (enum machine_mode orig_mode, enum machine_mode mode,
6208 /* Handle libcalls, which don't provide a type node. */
6209 if (valtype == NULL)
6221 return gen_rtx_REG (mode, FIRST_SSE_REG);
6224 return gen_rtx_REG (mode, FIRST_FLOAT_REG);
6228 return gen_rtx_REG (mode, AX_REG);
6232 ret = construct_container (mode, orig_mode, valtype, 1,
6233 X86_64_REGPARM_MAX, X86_64_SSE_REGPARM_MAX,
6234 x86_64_int_return_registers, 0);
6236 /* For zero sized structures, construct_container returns NULL, but we
6237 need to keep rest of compiler happy by returning meaningful value. */
6239 ret = gen_rtx_REG (orig_mode, AX_REG);
6245 function_value_ms_64 (enum machine_mode orig_mode, enum machine_mode mode)
6247 unsigned int regno = AX_REG;
6251 switch (GET_MODE_SIZE (mode))
6254 if((SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
6255 && !COMPLEX_MODE_P (mode))
6256 regno = FIRST_SSE_REG;
6260 if (mode == SFmode || mode == DFmode)
6261 regno = FIRST_SSE_REG;
6267 return gen_rtx_REG (orig_mode, regno);
6271 ix86_function_value_1 (const_tree valtype, const_tree fntype_or_decl,
6272 enum machine_mode orig_mode, enum machine_mode mode)
6274 const_tree fn, fntype;
6277 if (fntype_or_decl && DECL_P (fntype_or_decl))
6278 fn = fntype_or_decl;
6279 fntype = fn ? TREE_TYPE (fn) : fntype_or_decl;
6281 if (TARGET_64BIT && ix86_function_type_abi (fntype) == MS_ABI)
6282 return function_value_ms_64 (orig_mode, mode);
6283 else if (TARGET_64BIT)
6284 return function_value_64 (orig_mode, mode, valtype);
6286 return function_value_32 (orig_mode, mode, fntype, fn);
6290 ix86_function_value (const_tree valtype, const_tree fntype_or_decl,
6291 bool outgoing ATTRIBUTE_UNUSED)
6293 enum machine_mode mode, orig_mode;
6295 orig_mode = TYPE_MODE (valtype);
6296 mode = type_natural_mode (valtype, NULL);
6297 return ix86_function_value_1 (valtype, fntype_or_decl, orig_mode, mode);
6301 ix86_libcall_value (enum machine_mode mode)
6303 return ix86_function_value_1 (NULL, NULL, mode, mode);
6306 /* Return true iff type is returned in memory. */
6308 static int ATTRIBUTE_UNUSED
6309 return_in_memory_32 (const_tree type, enum machine_mode mode)
6313 if (mode == BLKmode)
6316 size = int_size_in_bytes (type);
6318 if (MS_AGGREGATE_RETURN && AGGREGATE_TYPE_P (type) && size <= 8)
6321 if (VECTOR_MODE_P (mode) || mode == TImode)
6323 /* User-created vectors small enough to fit in EAX. */
6327 /* MMX/3dNow values are returned in MM0,
6328 except when it doesn't exits. */
6330 return (TARGET_MMX ? 0 : 1);
6332 /* SSE values are returned in XMM0, except when it doesn't exist. */
6334 return (TARGET_SSE ? 0 : 1);
6336 /* AVX values are returned in YMM0, except when it doesn't exist. */
6338 return TARGET_AVX ? 0 : 1;
6347 /* OImode shouldn't be used directly. */
6348 gcc_assert (mode != OImode);
6353 static int ATTRIBUTE_UNUSED
6354 return_in_memory_64 (const_tree type, enum machine_mode mode)
6356 int needed_intregs, needed_sseregs;
6357 return !examine_argument (mode, type, 1, &needed_intregs, &needed_sseregs);
6360 static int ATTRIBUTE_UNUSED
6361 return_in_memory_ms_64 (const_tree type, enum machine_mode mode)
6363 HOST_WIDE_INT size = int_size_in_bytes (type);
6365 /* __m128 is returned in xmm0. */
6366 if ((SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode))
6367 && !COMPLEX_MODE_P (mode) && (GET_MODE_SIZE (mode) == 16 || size == 16))
6370 /* Otherwise, the size must be exactly in [1248]. */
6371 return (size != 1 && size != 2 && size != 4 && size != 8);
6375 ix86_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
6377 #ifdef SUBTARGET_RETURN_IN_MEMORY
6378 return SUBTARGET_RETURN_IN_MEMORY (type, fntype);
6380 const enum machine_mode mode = type_natural_mode (type, NULL);
6384 if (ix86_function_type_abi (fntype) == MS_ABI)
6385 return return_in_memory_ms_64 (type, mode);
6387 return return_in_memory_64 (type, mode);
6390 return return_in_memory_32 (type, mode);
6394 /* Return false iff TYPE is returned in memory. This version is used
6395 on Solaris 10. It is similar to the generic ix86_return_in_memory,
6396 but differs notably in that when MMX is available, 8-byte vectors
6397 are returned in memory, rather than in MMX registers. */
6400 ix86_sol10_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED)
6403 enum machine_mode mode = type_natural_mode (type, NULL);
6406 return return_in_memory_64 (type, mode);
6408 if (mode == BLKmode)
6411 size = int_size_in_bytes (type);
6413 if (VECTOR_MODE_P (mode))
6415 /* Return in memory only if MMX registers *are* available. This
6416 seems backwards, but it is consistent with the existing
6423 else if (mode == TImode)
6425 else if (mode == XFmode)
6431 /* When returning SSE vector types, we have a choice of either
6432 (1) being abi incompatible with a -march switch, or
6433 (2) generating an error.
6434 Given no good solution, I think the safest thing is one warning.
6435 The user won't be able to use -Werror, but....
6437 Choose the STRUCT_VALUE_RTX hook because that's (at present) only
6438 called in response to actually generating a caller or callee that
6439 uses such a type. As opposed to TARGET_RETURN_IN_MEMORY, which is called
6440 via aggregate_value_p for general type probing from tree-ssa. */
6443 ix86_struct_value_rtx (tree type, int incoming ATTRIBUTE_UNUSED)
6445 static bool warnedsse, warnedmmx;
6447 if (!TARGET_64BIT && type)
6449 /* Look at the return type of the function, not the function type. */
6450 enum machine_mode mode = TYPE_MODE (TREE_TYPE (type));
6452 if (!TARGET_SSE && !warnedsse)
6455 || (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 16))
6458 warning (0, "SSE vector return without SSE enabled "
6463 if (!TARGET_MMX && !warnedmmx)
6465 if (VECTOR_MODE_P (mode) && GET_MODE_SIZE (mode) == 8)
6468 warning (0, "MMX vector return without MMX enabled "
6478 /* Create the va_list data type. */
6480 /* Returns the calling convention specific va_list date type.
6481 The argument ABI can be DEFAULT_ABI, MS_ABI, or SYSV_ABI. */
6484 ix86_build_builtin_va_list_abi (enum calling_abi abi)
6486 tree f_gpr, f_fpr, f_ovf, f_sav, record, type_decl;
6488 /* For i386 we use plain pointer to argument area. */
6489 if (!TARGET_64BIT || abi == MS_ABI)
6490 return build_pointer_type (char_type_node);
6492 record = (*lang_hooks.types.make_type) (RECORD_TYPE);
6493 type_decl = build_decl (TYPE_DECL, get_identifier ("__va_list_tag"), record);
6495 f_gpr = build_decl (FIELD_DECL, get_identifier ("gp_offset"),
6496 unsigned_type_node);
6497 f_fpr = build_decl (FIELD_DECL, get_identifier ("fp_offset"),
6498 unsigned_type_node);
6499 f_ovf = build_decl (FIELD_DECL, get_identifier ("overflow_arg_area"),
6501 f_sav = build_decl (FIELD_DECL, get_identifier ("reg_save_area"),
6504 va_list_gpr_counter_field = f_gpr;
6505 va_list_fpr_counter_field = f_fpr;
6507 DECL_FIELD_CONTEXT (f_gpr) = record;
6508 DECL_FIELD_CONTEXT (f_fpr) = record;
6509 DECL_FIELD_CONTEXT (f_ovf) = record;
6510 DECL_FIELD_CONTEXT (f_sav) = record;
6512 TREE_CHAIN (record) = type_decl;
6513 TYPE_NAME (record) = type_decl;
6514 TYPE_FIELDS (record) = f_gpr;
6515 TREE_CHAIN (f_gpr) = f_fpr;
6516 TREE_CHAIN (f_fpr) = f_ovf;
6517 TREE_CHAIN (f_ovf) = f_sav;
6519 layout_type (record);
6521 /* The correct type is an array type of one element. */
6522 return build_array_type (record, build_index_type (size_zero_node));
6525 /* Setup the builtin va_list data type and for 64-bit the additional
6526 calling convention specific va_list data types. */
6529 ix86_build_builtin_va_list (void)
6531 tree ret = ix86_build_builtin_va_list_abi (ix86_abi);
6533 /* Initialize abi specific va_list builtin types. */
6537 if (ix86_abi == MS_ABI)
6539 t = ix86_build_builtin_va_list_abi (SYSV_ABI);
6540 if (TREE_CODE (t) != RECORD_TYPE)
6541 t = build_variant_type_copy (t);
6542 sysv_va_list_type_node = t;
6547 if (TREE_CODE (t) != RECORD_TYPE)
6548 t = build_variant_type_copy (t);
6549 sysv_va_list_type_node = t;
6551 if (ix86_abi != MS_ABI)
6553 t = ix86_build_builtin_va_list_abi (MS_ABI);
6554 if (TREE_CODE (t) != RECORD_TYPE)
6555 t = build_variant_type_copy (t);
6556 ms_va_list_type_node = t;
6561 if (TREE_CODE (t) != RECORD_TYPE)
6562 t = build_variant_type_copy (t);
6563 ms_va_list_type_node = t;
6570 /* Worker function for TARGET_SETUP_INCOMING_VARARGS. */
6573 setup_incoming_varargs_64 (CUMULATIVE_ARGS *cum)
6582 int regparm = ix86_regparm;
6584 if (cum->call_abi != ix86_abi)
6585 regparm = ix86_abi != SYSV_ABI ? X86_64_REGPARM_MAX : X64_REGPARM_MAX;
6587 /* GPR size of varargs save area. */
6588 if (cfun->va_list_gpr_size)
6589 ix86_varargs_gpr_size = X86_64_REGPARM_MAX * UNITS_PER_WORD;
6591 ix86_varargs_gpr_size = 0;
6593 /* FPR size of varargs save area. We don't need it if we don't pass
6594 anything in SSE registers. */
6595 if (cum->sse_nregs && cfun->va_list_fpr_size)
6596 ix86_varargs_fpr_size = X86_64_SSE_REGPARM_MAX * 16;
6598 ix86_varargs_fpr_size = 0;
6600 if (! ix86_varargs_gpr_size && ! ix86_varargs_fpr_size)
6603 save_area = frame_pointer_rtx;
6604 set = get_varargs_alias_set ();
6606 for (i = cum->regno;
6608 && i < cum->regno + cfun->va_list_gpr_size / UNITS_PER_WORD;
6611 mem = gen_rtx_MEM (Pmode,
6612 plus_constant (save_area, i * UNITS_PER_WORD));
6613 MEM_NOTRAP_P (mem) = 1;
6614 set_mem_alias_set (mem, set);
6615 emit_move_insn (mem, gen_rtx_REG (Pmode,
6616 x86_64_int_parameter_registers[i]));
6619 if (ix86_varargs_fpr_size)
6621 /* Now emit code to save SSE registers. The AX parameter contains number
6622 of SSE parameter registers used to call this function. We use
6623 sse_prologue_save insn template that produces computed jump across
6624 SSE saves. We need some preparation work to get this working. */
6626 label = gen_label_rtx ();
6627 label_ref = gen_rtx_LABEL_REF (Pmode, label);
6629 /* Compute address to jump to :
6630 label - eax*4 + nnamed_sse_arguments*4 Or
6631 label - eax*5 + nnamed_sse_arguments*5 for AVX. */
6632 tmp_reg = gen_reg_rtx (Pmode);
6633 nsse_reg = gen_reg_rtx (Pmode);
6634 emit_insn (gen_zero_extendqidi2 (nsse_reg, gen_rtx_REG (QImode, AX_REG)));
6635 emit_insn (gen_rtx_SET (VOIDmode, tmp_reg,
6636 gen_rtx_MULT (Pmode, nsse_reg,
6639 /* vmovaps is one byte longer than movaps. */
6641 emit_insn (gen_rtx_SET (VOIDmode, tmp_reg,
6642 gen_rtx_PLUS (Pmode, tmp_reg,
6648 gen_rtx_CONST (DImode,
6649 gen_rtx_PLUS (DImode,
6651 GEN_INT (cum->sse_regno
6652 * (TARGET_AVX ? 5 : 4)))));
6654 emit_move_insn (nsse_reg, label_ref);
6655 emit_insn (gen_subdi3 (nsse_reg, nsse_reg, tmp_reg));
6657 /* Compute address of memory block we save into. We always use pointer
6658 pointing 127 bytes after first byte to store - this is needed to keep
6659 instruction size limited by 4 bytes (5 bytes for AVX) with one
6660 byte displacement. */
6661 tmp_reg = gen_reg_rtx (Pmode);
6662 emit_insn (gen_rtx_SET (VOIDmode, tmp_reg,
6663 plus_constant (save_area,
6664 ix86_varargs_gpr_size + 127)));
6665 mem = gen_rtx_MEM (BLKmode, plus_constant (tmp_reg, -127));
6666 MEM_NOTRAP_P (mem) = 1;
6667 set_mem_alias_set (mem, set);
6668 set_mem_align (mem, BITS_PER_WORD);
6670 /* And finally do the dirty job! */
6671 emit_insn (gen_sse_prologue_save (mem, nsse_reg,
6672 GEN_INT (cum->sse_regno), label));
6677 setup_incoming_varargs_ms_64 (CUMULATIVE_ARGS *cum)
6679 alias_set_type set = get_varargs_alias_set ();
6682 for (i = cum->regno; i < X64_REGPARM_MAX; i++)
6686 mem = gen_rtx_MEM (Pmode,
6687 plus_constant (virtual_incoming_args_rtx,
6688 i * UNITS_PER_WORD));
6689 MEM_NOTRAP_P (mem) = 1;
6690 set_mem_alias_set (mem, set);
6692 reg = gen_rtx_REG (Pmode, x86_64_ms_abi_int_parameter_registers[i]);
6693 emit_move_insn (mem, reg);
6698 ix86_setup_incoming_varargs (CUMULATIVE_ARGS *cum, enum machine_mode mode,
6699 tree type, int *pretend_size ATTRIBUTE_UNUSED,
6702 CUMULATIVE_ARGS next_cum;
6705 /* This argument doesn't appear to be used anymore. Which is good,
6706 because the old code here didn't suppress rtl generation. */
6707 gcc_assert (!no_rtl);
6712 fntype = TREE_TYPE (current_function_decl);
6714 /* For varargs, we do not want to skip the dummy va_dcl argument.
6715 For stdargs, we do want to skip the last named argument. */
6717 if (stdarg_p (fntype))
6718 function_arg_advance (&next_cum, mode, type, 1);
6720 if (cum->call_abi == MS_ABI)
6721 setup_incoming_varargs_ms_64 (&next_cum);
6723 setup_incoming_varargs_64 (&next_cum);
6726 /* Checks if TYPE is of kind va_list char *. */
6729 is_va_list_char_pointer (tree type)
6733 /* For 32-bit it is always true. */
6736 canonic = ix86_canonical_va_list_type (type);
6737 return (canonic == ms_va_list_type_node
6738 || (ix86_abi == MS_ABI && canonic == va_list_type_node));
6741 /* Implement va_start. */
6744 ix86_va_start (tree valist, rtx nextarg)
6746 HOST_WIDE_INT words, n_gpr, n_fpr;
6747 tree f_gpr, f_fpr, f_ovf, f_sav;
6748 tree gpr, fpr, ovf, sav, t;
6751 /* Only 64bit target needs something special. */
6752 if (!TARGET_64BIT || is_va_list_char_pointer (TREE_TYPE (valist)))
6754 std_expand_builtin_va_start (valist, nextarg);
6758 f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node));
6759 f_fpr = TREE_CHAIN (f_gpr);
6760 f_ovf = TREE_CHAIN (f_fpr);
6761 f_sav = TREE_CHAIN (f_ovf);
6763 valist = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (valist)), valist);
6764 gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr), valist, f_gpr, NULL_TREE);
6765 fpr = build3 (COMPONENT_REF, TREE_TYPE (f_fpr), valist, f_fpr, NULL_TREE);
6766 ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), valist, f_ovf, NULL_TREE);
6767 sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), valist, f_sav, NULL_TREE);
6769 /* Count number of gp and fp argument registers used. */
6770 words = crtl->args.info.words;
6771 n_gpr = crtl->args.info.regno;
6772 n_fpr = crtl->args.info.sse_regno;
6774 if (cfun->va_list_gpr_size)
6776 type = TREE_TYPE (gpr);
6777 t = build2 (MODIFY_EXPR, type,
6778 gpr, build_int_cst (type, n_gpr * 8));
6779 TREE_SIDE_EFFECTS (t) = 1;
6780 expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
6783 if (TARGET_SSE && cfun->va_list_fpr_size)
6785 type = TREE_TYPE (fpr);
6786 t = build2 (MODIFY_EXPR, type, fpr,
6787 build_int_cst (type, n_fpr * 16 + 8*X86_64_REGPARM_MAX));
6788 TREE_SIDE_EFFECTS (t) = 1;
6789 expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
6792 /* Find the overflow area. */
6793 type = TREE_TYPE (ovf);
6794 t = make_tree (type, crtl->args.internal_arg_pointer);
6796 t = build2 (POINTER_PLUS_EXPR, type, t,
6797 size_int (words * UNITS_PER_WORD));
6798 t = build2 (MODIFY_EXPR, type, ovf, t);
6799 TREE_SIDE_EFFECTS (t) = 1;
6800 expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
6802 if (ix86_varargs_gpr_size || ix86_varargs_fpr_size)
6804 /* Find the register save area.
6805 Prologue of the function save it right above stack frame. */
6806 type = TREE_TYPE (sav);
6807 t = make_tree (type, frame_pointer_rtx);
6808 if (!ix86_varargs_gpr_size)
6809 t = build2 (POINTER_PLUS_EXPR, type, t,
6810 size_int (-8 * X86_64_REGPARM_MAX));
6811 t = build2 (MODIFY_EXPR, type, sav, t);
6812 TREE_SIDE_EFFECTS (t) = 1;
6813 expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
6817 /* Implement va_arg. */
6820 ix86_gimplify_va_arg (tree valist, tree type, gimple_seq *pre_p,
6823 static const int intreg[6] = { 0, 1, 2, 3, 4, 5 };
6824 tree f_gpr, f_fpr, f_ovf, f_sav;
6825 tree gpr, fpr, ovf, sav, t;
6827 tree lab_false, lab_over = NULL_TREE;
6832 enum machine_mode nat_mode;
6835 /* Only 64bit target needs something special. */
6836 if (!TARGET_64BIT || is_va_list_char_pointer (TREE_TYPE (valist)))
6837 return std_gimplify_va_arg_expr (valist, type, pre_p, post_p);
6839 f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node));
6840 f_fpr = TREE_CHAIN (f_gpr);
6841 f_ovf = TREE_CHAIN (f_fpr);
6842 f_sav = TREE_CHAIN (f_ovf);
6844 gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr),
6845 build_va_arg_indirect_ref (valist), f_gpr, NULL_TREE);
6846 valist = build_va_arg_indirect_ref (valist);
6847 fpr = build3 (COMPONENT_REF, TREE_TYPE (f_fpr), valist, f_fpr, NULL_TREE);
6848 ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), valist, f_ovf, NULL_TREE);
6849 sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), valist, f_sav, NULL_TREE);
6851 indirect_p = pass_by_reference (NULL, TYPE_MODE (type), type, false);
6853 type = build_pointer_type (type);
6854 size = int_size_in_bytes (type);
6855 rsize = (size + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
6857 nat_mode = type_natural_mode (type, NULL);
6866 /* Unnamed 256bit vector mode parameters are passed on stack. */
6867 if (ix86_cfun_abi () == SYSV_ABI)
6874 container = construct_container (nat_mode, TYPE_MODE (type),
6875 type, 0, X86_64_REGPARM_MAX,
6876 X86_64_SSE_REGPARM_MAX, intreg,
6881 /* Pull the value out of the saved registers. */
6883 addr = create_tmp_var (ptr_type_node, "addr");
6884 DECL_POINTER_ALIAS_SET (addr) = get_varargs_alias_set ();
6888 int needed_intregs, needed_sseregs;
6890 tree int_addr, sse_addr;
6892 lab_false = create_artificial_label ();
6893 lab_over = create_artificial_label ();
6895 examine_argument (nat_mode, type, 0, &needed_intregs, &needed_sseregs);
6897 need_temp = (!REG_P (container)
6898 && ((needed_intregs && TYPE_ALIGN (type) > 64)
6899 || TYPE_ALIGN (type) > 128));
6901 /* In case we are passing structure, verify that it is consecutive block
6902 on the register save area. If not we need to do moves. */
6903 if (!need_temp && !REG_P (container))
6905 /* Verify that all registers are strictly consecutive */
6906 if (SSE_REGNO_P (REGNO (XEXP (XVECEXP (container, 0, 0), 0))))
6910 for (i = 0; i < XVECLEN (container, 0) && !need_temp; i++)
6912 rtx slot = XVECEXP (container, 0, i);
6913 if (REGNO (XEXP (slot, 0)) != FIRST_SSE_REG + (unsigned int) i
6914 || INTVAL (XEXP (slot, 1)) != i * 16)
6922 for (i = 0; i < XVECLEN (container, 0) && !need_temp; i++)
6924 rtx slot = XVECEXP (container, 0, i);
6925 if (REGNO (XEXP (slot, 0)) != (unsigned int) i
6926 || INTVAL (XEXP (slot, 1)) != i * 8)
6938 int_addr = create_tmp_var (ptr_type_node, "int_addr");
6939 DECL_POINTER_ALIAS_SET (int_addr) = get_varargs_alias_set ();
6940 sse_addr = create_tmp_var (ptr_type_node, "sse_addr");
6941 DECL_POINTER_ALIAS_SET (sse_addr) = get_varargs_alias_set ();
6944 /* First ensure that we fit completely in registers. */
6947 t = build_int_cst (TREE_TYPE (gpr),
6948 (X86_64_REGPARM_MAX - needed_intregs + 1) * 8);
6949 t = build2 (GE_EXPR, boolean_type_node, gpr, t);
6950 t2 = build1 (GOTO_EXPR, void_type_node, lab_false);
6951 t = build3 (COND_EXPR, void_type_node, t, t2, NULL_TREE);
6952 gimplify_and_add (t, pre_p);
6956 t = build_int_cst (TREE_TYPE (fpr),
6957 (X86_64_SSE_REGPARM_MAX - needed_sseregs + 1) * 16
6958 + X86_64_REGPARM_MAX * 8);
6959 t = build2 (GE_EXPR, boolean_type_node, fpr, t);
6960 t2 = build1 (GOTO_EXPR, void_type_node, lab_false);
6961 t = build3 (COND_EXPR, void_type_node, t, t2, NULL_TREE);
6962 gimplify_and_add (t, pre_p);
6965 /* Compute index to start of area used for integer regs. */
6968 /* int_addr = gpr + sav; */
6969 t = fold_convert (sizetype, gpr);
6970 t = build2 (POINTER_PLUS_EXPR, ptr_type_node, sav, t);
6971 gimplify_assign (int_addr, t, pre_p);
6975 /* sse_addr = fpr + sav; */
6976 t = fold_convert (sizetype, fpr);
6977 t = build2 (POINTER_PLUS_EXPR, ptr_type_node, sav, t);
6978 gimplify_assign (sse_addr, t, pre_p);
6983 tree temp = create_tmp_var (type, "va_arg_tmp");
6986 t = build1 (ADDR_EXPR, build_pointer_type (type), temp);
6987 gimplify_assign (addr, t, pre_p);
6989 for (i = 0; i < XVECLEN (container, 0); i++)
6991 rtx slot = XVECEXP (container, 0, i);
6992 rtx reg = XEXP (slot, 0);
6993 enum machine_mode mode = GET_MODE (reg);
6994 tree piece_type = lang_hooks.types.type_for_mode (mode, 1);
6995 tree addr_type = build_pointer_type (piece_type);
6996 tree daddr_type = build_pointer_type_for_mode (piece_type,
7000 tree dest_addr, dest;
7002 if (SSE_REGNO_P (REGNO (reg)))
7004 src_addr = sse_addr;
7005 src_offset = (REGNO (reg) - FIRST_SSE_REG) * 16;
7009 src_addr = int_addr;
7010 src_offset = REGNO (reg) * 8;
7012 src_addr = fold_convert (addr_type, src_addr);
7013 src_addr = fold_build2 (POINTER_PLUS_EXPR, addr_type, src_addr,
7014 size_int (src_offset));
7015 src = build_va_arg_indirect_ref (src_addr);
7017 dest_addr = fold_convert (daddr_type, addr);
7018 dest_addr = fold_build2 (POINTER_PLUS_EXPR, daddr_type, dest_addr,
7019 size_int (INTVAL (XEXP (slot, 1))));
7020 dest = build_va_arg_indirect_ref (dest_addr);
7022 gimplify_assign (dest, src, pre_p);
7028 t = build2 (PLUS_EXPR, TREE_TYPE (gpr), gpr,
7029 build_int_cst (TREE_TYPE (gpr), needed_intregs * 8));
7030 gimplify_assign (gpr, t, pre_p);
7035 t = build2 (PLUS_EXPR, TREE_TYPE (fpr), fpr,
7036 build_int_cst (TREE_TYPE (fpr), needed_sseregs * 16));
7037 gimplify_assign (fpr, t, pre_p);
7040 gimple_seq_add_stmt (pre_p, gimple_build_goto (lab_over));
7042 gimple_seq_add_stmt (pre_p, gimple_build_label (lab_false));
7045 /* ... otherwise out of the overflow area. */
7047 /* When we align parameter on stack for caller, if the parameter
7048 alignment is beyond MAX_SUPPORTED_STACK_ALIGNMENT, it will be
7049 aligned at MAX_SUPPORTED_STACK_ALIGNMENT. We will match callee
7050 here with caller. */
7051 arg_boundary = FUNCTION_ARG_BOUNDARY (VOIDmode, type);
7052 if ((unsigned int) arg_boundary > MAX_SUPPORTED_STACK_ALIGNMENT)
7053 arg_boundary = MAX_SUPPORTED_STACK_ALIGNMENT;
7055 /* Care for on-stack alignment if needed. */
7056 if (arg_boundary <= 64
7057 || integer_zerop (TYPE_SIZE (type)))
7061 HOST_WIDE_INT align = arg_boundary / 8;
7062 t = build2 (POINTER_PLUS_EXPR, TREE_TYPE (ovf), ovf,
7063 size_int (align - 1));
7064 t = fold_convert (sizetype, t);
7065 t = build2 (BIT_AND_EXPR, TREE_TYPE (t), t,
7067 t = fold_convert (TREE_TYPE (ovf), t);
7069 gimplify_expr (&t, pre_p, NULL, is_gimple_val, fb_rvalue);
7070 gimplify_assign (addr, t, pre_p);
7072 t = build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t,
7073 size_int (rsize * UNITS_PER_WORD));
7074 gimplify_assign (unshare_expr (ovf), t, pre_p);
7077 gimple_seq_add_stmt (pre_p, gimple_build_label (lab_over));
7079 ptrtype = build_pointer_type (type);
7080 addr = fold_convert (ptrtype, addr);
7083 addr = build_va_arg_indirect_ref (addr);
7084 return build_va_arg_indirect_ref (addr);
7087 /* Return nonzero if OPNUM's MEM should be matched
7088 in movabs* patterns. */
7091 ix86_check_movabs (rtx insn, int opnum)
7095 set = PATTERN (insn);
7096 if (GET_CODE (set) == PARALLEL)
7097 set = XVECEXP (set, 0, 0);
7098 gcc_assert (GET_CODE (set) == SET);
7099 mem = XEXP (set, opnum);
7100 while (GET_CODE (mem) == SUBREG)
7101 mem = SUBREG_REG (mem);
7102 gcc_assert (MEM_P (mem));
7103 return (volatile_ok || !MEM_VOLATILE_P (mem));
7106 /* Initialize the table of extra 80387 mathematical constants. */
7109 init_ext_80387_constants (void)
7111 static const char * cst[5] =
7113 "0.3010299956639811952256464283594894482", /* 0: fldlg2 */
7114 "0.6931471805599453094286904741849753009", /* 1: fldln2 */
7115 "1.4426950408889634073876517827983434472", /* 2: fldl2e */
7116 "3.3219280948873623478083405569094566090", /* 3: fldl2t */
7117 "3.1415926535897932385128089594061862044", /* 4: fldpi */
7121 for (i = 0; i < 5; i++)
7123 real_from_string (&ext_80387_constants_table[i], cst[i]);
7124 /* Ensure each constant is rounded to XFmode precision. */
7125 real_convert (&ext_80387_constants_table[i],
7126 XFmode, &ext_80387_constants_table[i]);
7129 ext_80387_constants_init = 1;
7132 /* Return true if the constant is something that can be loaded with
7133 a special instruction. */
7136 standard_80387_constant_p (rtx x)
7138 enum machine_mode mode = GET_MODE (x);
7142 if (!(X87_FLOAT_MODE_P (mode) && (GET_CODE (x) == CONST_DOUBLE)))
7145 if (x == CONST0_RTX (mode))
7147 if (x == CONST1_RTX (mode))
7150 REAL_VALUE_FROM_CONST_DOUBLE (r, x);
7152 /* For XFmode constants, try to find a special 80387 instruction when
7153 optimizing for size or on those CPUs that benefit from them. */
7155 && (optimize_function_for_size_p (cfun) || TARGET_EXT_80387_CONSTANTS))
7159 if (! ext_80387_constants_init)
7160 init_ext_80387_constants ();
7162 for (i = 0; i < 5; i++)
7163 if (real_identical (&r, &ext_80387_constants_table[i]))
7167 /* Load of the constant -0.0 or -1.0 will be split as
7168 fldz;fchs or fld1;fchs sequence. */
7169 if (real_isnegzero (&r))
7171 if (real_identical (&r, &dconstm1))
7177 /* Return the opcode of the special instruction to be used to load
7181 standard_80387_constant_opcode (rtx x)
7183 switch (standard_80387_constant_p (x))
7207 /* Return the CONST_DOUBLE representing the 80387 constant that is
7208 loaded by the specified special instruction. The argument IDX
7209 matches the return value from standard_80387_constant_p. */
7212 standard_80387_constant_rtx (int idx)
7216 if (! ext_80387_constants_init)
7217 init_ext_80387_constants ();
7233 return CONST_DOUBLE_FROM_REAL_VALUE (ext_80387_constants_table[i],
7237 /* Return 1 if mode is a valid mode for sse. */
7239 standard_sse_mode_p (enum machine_mode mode)
7256 /* Return 1 if X is all 0s. For all 1s, return 2 if X is in 128bit
7257 SSE modes and SSE2 is enabled, return 3 if X is in 256bit AVX
7258 modes and AVX is enabled. */
7261 standard_sse_constant_p (rtx x)
7263 enum machine_mode mode = GET_MODE (x);
7265 if (x == const0_rtx || x == CONST0_RTX (GET_MODE (x)))
7267 if (vector_all_ones_operand (x, mode))
7269 if (standard_sse_mode_p (mode))
7270 return TARGET_SSE2 ? 2 : -2;
7271 else if (VALID_AVX256_REG_MODE (mode))
7272 return TARGET_AVX ? 3 : -3;
7278 /* Return the opcode of the special instruction to be used to load
7282 standard_sse_constant_opcode (rtx insn, rtx x)
7284 switch (standard_sse_constant_p (x))
7287 switch (get_attr_mode (insn))
7290 return TARGET_AVX ? "vxorps\t%0, %0, %0" : "xorps\t%0, %0";
7292 return TARGET_AVX ? "vxorpd\t%0, %0, %0" : "xorpd\t%0, %0";
7294 return TARGET_AVX ? "vpxor\t%0, %0, %0" : "pxor\t%0, %0";
7296 return "vxorps\t%x0, %x0, %x0";
7298 return "vxorpd\t%x0, %x0, %x0";
7300 return "vpxor\t%x0, %x0, %x0";
7306 switch (get_attr_mode (insn))
7311 return "vpcmpeqd\t%0, %0, %0";
7317 return "pcmpeqd\t%0, %0";
7322 /* Returns 1 if OP contains a symbol reference */
7325 symbolic_reference_mentioned_p (rtx op)
7330 if (GET_CODE (op) == SYMBOL_REF || GET_CODE (op) == LABEL_REF)
7333 fmt = GET_RTX_FORMAT (GET_CODE (op));
7334 for (i = GET_RTX_LENGTH (GET_CODE (op)) - 1; i >= 0; i--)
7340 for (j = XVECLEN (op, i) - 1; j >= 0; j--)
7341 if (symbolic_reference_mentioned_p (XVECEXP (op, i, j)))
7345 else if (fmt[i] == 'e' && symbolic_reference_mentioned_p (XEXP (op, i)))
7352 /* Return 1 if it is appropriate to emit `ret' instructions in the
7353 body of a function. Do this only if the epilogue is simple, needing a
7354 couple of insns. Prior to reloading, we can't tell how many registers
7355 must be saved, so return 0 then. Return 0 if there is no frame
7356 marker to de-allocate. */
7359 ix86_can_use_return_insn_p (void)
7361 struct ix86_frame frame;
7363 if (! reload_completed || frame_pointer_needed)
7366 /* Don't allow more than 32 pop, since that's all we can do
7367 with one instruction. */
7368 if (crtl->args.pops_args
7369 && crtl->args.size >= 32768)
7372 ix86_compute_frame_layout (&frame);
7373 return frame.to_allocate == 0 && (frame.nregs + frame.nsseregs) == 0;
7376 /* Value should be nonzero if functions must have frame pointers.
7377 Zero means the frame pointer need not be set up (and parms may
7378 be accessed via the stack pointer) in functions that seem suitable. */
7381 ix86_frame_pointer_required (void)
7383 /* If we accessed previous frames, then the generated code expects
7384 to be able to access the saved ebp value in our frame. */
7385 if (cfun->machine->accesses_prev_frame)
7388 /* Several x86 os'es need a frame pointer for other reasons,
7389 usually pertaining to setjmp. */
7390 if (SUBTARGET_FRAME_POINTER_REQUIRED)
7393 /* In override_options, TARGET_OMIT_LEAF_FRAME_POINTER turns off
7394 the frame pointer by default. Turn it back on now if we've not
7395 got a leaf function. */
7396 if (TARGET_OMIT_LEAF_FRAME_POINTER
7397 && (!current_function_is_leaf
7398 || ix86_current_function_calls_tls_descriptor))
7407 /* Record that the current function accesses previous call frames. */
7410 ix86_setup_frame_addresses (void)
7412 cfun->machine->accesses_prev_frame = 1;
7415 #if (defined(HAVE_GAS_HIDDEN) && (SUPPORTS_ONE_ONLY - 0)) || TARGET_MACHO
7416 # define USE_HIDDEN_LINKONCE 1
7418 # define USE_HIDDEN_LINKONCE 0
7421 static int pic_labels_used;
7423 /* Fills in the label name that should be used for a pc thunk for
7424 the given register. */
7427 get_pc_thunk_name (char name[32], unsigned int regno)
7429 gcc_assert (!TARGET_64BIT);
7431 if (USE_HIDDEN_LINKONCE)
7432 sprintf (name, "__i686.get_pc_thunk.%s", reg_names[regno]);
7434 ASM_GENERATE_INTERNAL_LABEL (name, "LPR", regno);
7438 /* This function generates code for -fpic that loads %ebx with
7439 the return address of the caller and then returns. */
7442 ix86_file_end (void)
7447 for (regno = 0; regno < 8; ++regno)
7451 if (! ((pic_labels_used >> regno) & 1))
7454 get_pc_thunk_name (name, regno);
7459 switch_to_section (darwin_sections[text_coal_section]);
7460 fputs ("\t.weak_definition\t", asm_out_file);
7461 assemble_name (asm_out_file, name);
7462 fputs ("\n\t.private_extern\t", asm_out_file);
7463 assemble_name (asm_out_file, name);
7464 fputs ("\n", asm_out_file);
7465 ASM_OUTPUT_LABEL (asm_out_file, name);
7469 if (USE_HIDDEN_LINKONCE)
7473 decl = build_decl (FUNCTION_DECL, get_identifier (name),
7475 TREE_PUBLIC (decl) = 1;
7476 TREE_STATIC (decl) = 1;
7477 DECL_ONE_ONLY (decl) = 1;
7479 (*targetm.asm_out.unique_section) (decl, 0);
7480 switch_to_section (get_named_section (decl, NULL, 0));
7482 (*targetm.asm_out.globalize_label) (asm_out_file, name);
7483 fputs ("\t.hidden\t", asm_out_file);
7484 assemble_name (asm_out_file, name);
7485 fputc ('\n', asm_out_file);
7486 ASM_DECLARE_FUNCTION_NAME (asm_out_file, name, decl);
7490 switch_to_section (text_section);
7491 ASM_OUTPUT_LABEL (asm_out_file, name);
7494 xops[0] = gen_rtx_REG (Pmode, regno);
7495 xops[1] = gen_rtx_MEM (Pmode, stack_pointer_rtx);
7496 output_asm_insn ("mov%z0\t{%1, %0|%0, %1}", xops);
7497 output_asm_insn ("ret", xops);
7500 if (NEED_INDICATE_EXEC_STACK)
7501 file_end_indicate_exec_stack ();
7504 /* Emit code for the SET_GOT patterns. */
7507 output_set_got (rtx dest, rtx label ATTRIBUTE_UNUSED)
7513 if (TARGET_VXWORKS_RTP && flag_pic)
7515 /* Load (*VXWORKS_GOTT_BASE) into the PIC register. */
7516 xops[2] = gen_rtx_MEM (Pmode,
7517 gen_rtx_SYMBOL_REF (Pmode, VXWORKS_GOTT_BASE));
7518 output_asm_insn ("mov{l}\t{%2, %0|%0, %2}", xops);
7520 /* Load (*VXWORKS_GOTT_BASE)[VXWORKS_GOTT_INDEX] into the PIC register.
7521 Use %P and a local symbol in order to print VXWORKS_GOTT_INDEX as
7522 an unadorned address. */
7523 xops[2] = gen_rtx_SYMBOL_REF (Pmode, VXWORKS_GOTT_INDEX);
7524 SYMBOL_REF_FLAGS (xops[2]) |= SYMBOL_FLAG_LOCAL;
7525 output_asm_insn ("mov{l}\t{%P2(%0), %0|%0, DWORD PTR %P2[%0]}", xops);
7529 xops[1] = gen_rtx_SYMBOL_REF (Pmode, GOT_SYMBOL_NAME);
7531 if (! TARGET_DEEP_BRANCH_PREDICTION || !flag_pic)
7533 xops[2] = gen_rtx_LABEL_REF (Pmode, label ? label : gen_label_rtx ());
7536 output_asm_insn ("mov%z0\t{%2, %0|%0, %2}", xops);
7538 output_asm_insn ("call\t%a2", xops);
7541 /* Output the Mach-O "canonical" label name ("Lxx$pb") here too. This
7542 is what will be referenced by the Mach-O PIC subsystem. */
7544 ASM_OUTPUT_LABEL (asm_out_file, MACHOPIC_FUNCTION_BASE_NAME);
7547 (*targetm.asm_out.internal_label) (asm_out_file, "L",
7548 CODE_LABEL_NUMBER (XEXP (xops[2], 0)));
7551 output_asm_insn ("pop%z0\t%0", xops);
7556 get_pc_thunk_name (name, REGNO (dest));
7557 pic_labels_used |= 1 << REGNO (dest);
7559 xops[2] = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (name));
7560 xops[2] = gen_rtx_MEM (QImode, xops[2]);
7561 output_asm_insn ("call\t%X2", xops);
7562 /* Output the Mach-O "canonical" label name ("Lxx$pb") here too. This
7563 is what will be referenced by the Mach-O PIC subsystem. */
7566 ASM_OUTPUT_LABEL (asm_out_file, MACHOPIC_FUNCTION_BASE_NAME);
7568 targetm.asm_out.internal_label (asm_out_file, "L",
7569 CODE_LABEL_NUMBER (label));
7576 if (!flag_pic || TARGET_DEEP_BRANCH_PREDICTION)
7577 output_asm_insn ("add%z0\t{%1, %0|%0, %1}", xops);
7579 output_asm_insn ("add%z0\t{%1+[.-%a2], %0|%0, %1+(.-%a2)}", xops);
7584 /* Generate an "push" pattern for input ARG. */
7589 return gen_rtx_SET (VOIDmode,
7591 gen_rtx_PRE_DEC (Pmode,
7592 stack_pointer_rtx)),
7596 /* Return >= 0 if there is an unused call-clobbered register available
7597 for the entire function. */
7600 ix86_select_alt_pic_regnum (void)
7602 if (current_function_is_leaf && !crtl->profile
7603 && !ix86_current_function_calls_tls_descriptor)
7606 /* Can't use the same register for both PIC and DRAP. */
7608 drap = REGNO (crtl->drap_reg);
7611 for (i = 2; i >= 0; --i)
7612 if (i != drap && !df_regs_ever_live_p (i))
7616 return INVALID_REGNUM;
7619 /* Return 1 if we need to save REGNO. */
7621 ix86_save_reg (unsigned int regno, int maybe_eh_return)
7623 if (pic_offset_table_rtx
7624 && regno == REAL_PIC_OFFSET_TABLE_REGNUM
7625 && (df_regs_ever_live_p (REAL_PIC_OFFSET_TABLE_REGNUM)
7627 || crtl->calls_eh_return
7628 || crtl->uses_const_pool))
7630 if (ix86_select_alt_pic_regnum () != INVALID_REGNUM)
7635 if (crtl->calls_eh_return && maybe_eh_return)
7640 unsigned test = EH_RETURN_DATA_REGNO (i);
7641 if (test == INVALID_REGNUM)
7649 && regno == REGNO (crtl->drap_reg))
7652 return (df_regs_ever_live_p (regno)
7653 && !call_used_regs[regno]
7654 && !fixed_regs[regno]
7655 && (regno != HARD_FRAME_POINTER_REGNUM || !frame_pointer_needed));
7658 /* Return number of saved general prupose registers. */
7661 ix86_nsaved_regs (void)
7666 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
7667 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
7672 /* Return number of saved SSE registrers. */
7675 ix86_nsaved_sseregs (void)
7680 if (ix86_cfun_abi () != MS_ABI)
7682 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
7683 if (SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
7688 /* Given FROM and TO register numbers, say whether this elimination is
7689 allowed. If stack alignment is needed, we can only replace argument
7690 pointer with hard frame pointer, or replace frame pointer with stack
7691 pointer. Otherwise, frame pointer elimination is automatically
7692 handled and all other eliminations are valid. */
7695 ix86_can_eliminate (int from, int to)
7697 if (stack_realign_fp)
7698 return ((from == ARG_POINTER_REGNUM
7699 && to == HARD_FRAME_POINTER_REGNUM)
7700 || (from == FRAME_POINTER_REGNUM
7701 && to == STACK_POINTER_REGNUM));
7703 return to == STACK_POINTER_REGNUM ? !frame_pointer_needed : 1;
7706 /* Return the offset between two registers, one to be eliminated, and the other
7707 its replacement, at the start of a routine. */
7710 ix86_initial_elimination_offset (int from, int to)
7712 struct ix86_frame frame;
7713 ix86_compute_frame_layout (&frame);
7715 if (from == ARG_POINTER_REGNUM && to == HARD_FRAME_POINTER_REGNUM)
7716 return frame.hard_frame_pointer_offset;
7717 else if (from == FRAME_POINTER_REGNUM
7718 && to == HARD_FRAME_POINTER_REGNUM)
7719 return frame.hard_frame_pointer_offset - frame.frame_pointer_offset;
7722 gcc_assert (to == STACK_POINTER_REGNUM);
7724 if (from == ARG_POINTER_REGNUM)
7725 return frame.stack_pointer_offset;
7727 gcc_assert (from == FRAME_POINTER_REGNUM);
7728 return frame.stack_pointer_offset - frame.frame_pointer_offset;
7732 /* In a dynamically-aligned function, we can't know the offset from
7733 stack pointer to frame pointer, so we must ensure that setjmp
7734 eliminates fp against the hard fp (%ebp) rather than trying to
7735 index from %esp up to the top of the frame across a gap that is
7736 of unknown (at compile-time) size. */
7738 ix86_builtin_setjmp_frame_value (void)
7740 return stack_realign_fp ? hard_frame_pointer_rtx : virtual_stack_vars_rtx;
7743 /* Fill structure ix86_frame about frame of currently computed function. */
7746 ix86_compute_frame_layout (struct ix86_frame *frame)
7748 HOST_WIDE_INT total_size;
7749 unsigned int stack_alignment_needed;
7750 HOST_WIDE_INT offset;
7751 unsigned int preferred_alignment;
7752 HOST_WIDE_INT size = get_frame_size ();
7754 frame->nregs = ix86_nsaved_regs ();
7755 frame->nsseregs = ix86_nsaved_sseregs ();
7758 stack_alignment_needed = crtl->stack_alignment_needed / BITS_PER_UNIT;
7759 preferred_alignment = crtl->preferred_stack_boundary / BITS_PER_UNIT;
7761 /* MS ABI seem to require stack alignment to be always 16 except for function
7763 if (ix86_cfun_abi () == MS_ABI && preferred_alignment < 16)
7765 preferred_alignment = 16;
7766 stack_alignment_needed = 16;
7767 crtl->preferred_stack_boundary = 128;
7768 crtl->stack_alignment_needed = 128;
7771 gcc_assert (!size || stack_alignment_needed);
7772 gcc_assert (preferred_alignment >= STACK_BOUNDARY / BITS_PER_UNIT);
7773 gcc_assert (preferred_alignment <= stack_alignment_needed);
7775 /* During reload iteration the amount of registers saved can change.
7776 Recompute the value as needed. Do not recompute when amount of registers
7777 didn't change as reload does multiple calls to the function and does not
7778 expect the decision to change within single iteration. */
7779 if (!optimize_function_for_size_p (cfun)
7780 && cfun->machine->use_fast_prologue_epilogue_nregs != frame->nregs)
7782 int count = frame->nregs;
7784 cfun->machine->use_fast_prologue_epilogue_nregs = count;
7785 /* The fast prologue uses move instead of push to save registers. This
7786 is significantly longer, but also executes faster as modern hardware
7787 can execute the moves in parallel, but can't do that for push/pop.
7789 Be careful about choosing what prologue to emit: When function takes
7790 many instructions to execute we may use slow version as well as in
7791 case function is known to be outside hot spot (this is known with
7792 feedback only). Weight the size of function by number of registers
7793 to save as it is cheap to use one or two push instructions but very
7794 slow to use many of them. */
7796 count = (count - 1) * FAST_PROLOGUE_INSN_COUNT;
7797 if (cfun->function_frequency < FUNCTION_FREQUENCY_NORMAL
7798 || (flag_branch_probabilities
7799 && cfun->function_frequency < FUNCTION_FREQUENCY_HOT))
7800 cfun->machine->use_fast_prologue_epilogue = false;
7802 cfun->machine->use_fast_prologue_epilogue
7803 = !expensive_function_p (count);
7805 if (TARGET_PROLOGUE_USING_MOVE
7806 && cfun->machine->use_fast_prologue_epilogue)
7807 frame->save_regs_using_mov = true;
7809 frame->save_regs_using_mov = false;
7812 /* Skip return address and saved base pointer. */
7813 offset = frame_pointer_needed ? UNITS_PER_WORD * 2 : UNITS_PER_WORD;
7815 frame->hard_frame_pointer_offset = offset;
7817 /* Set offset to aligned because the realigned frame starts from
7819 if (stack_realign_fp)
7820 offset = (offset + stack_alignment_needed -1) & -stack_alignment_needed;
7822 /* Register save area */
7823 offset += frame->nregs * UNITS_PER_WORD;
7825 /* Align SSE reg save area. */
7826 if (frame->nsseregs)
7827 frame->padding0 = ((offset + 16 - 1) & -16) - offset;
7829 frame->padding0 = 0;
7831 /* SSE register save area. */
7832 offset += frame->padding0 + frame->nsseregs * 16;
7835 frame->va_arg_size = ix86_varargs_gpr_size + ix86_varargs_fpr_size;
7836 offset += frame->va_arg_size;
7838 /* Align start of frame for local function. */
7839 frame->padding1 = ((offset + stack_alignment_needed - 1)
7840 & -stack_alignment_needed) - offset;
7842 offset += frame->padding1;
7844 /* Frame pointer points here. */
7845 frame->frame_pointer_offset = offset;
7849 /* Add outgoing arguments area. Can be skipped if we eliminated
7850 all the function calls as dead code.
7851 Skipping is however impossible when function calls alloca. Alloca
7852 expander assumes that last crtl->outgoing_args_size
7853 of stack frame are unused. */
7854 if (ACCUMULATE_OUTGOING_ARGS
7855 && (!current_function_is_leaf || cfun->calls_alloca
7856 || ix86_current_function_calls_tls_descriptor))
7858 offset += crtl->outgoing_args_size;
7859 frame->outgoing_arguments_size = crtl->outgoing_args_size;
7862 frame->outgoing_arguments_size = 0;
7864 /* Align stack boundary. Only needed if we're calling another function
7866 if (!current_function_is_leaf || cfun->calls_alloca
7867 || ix86_current_function_calls_tls_descriptor)
7868 frame->padding2 = ((offset + preferred_alignment - 1)
7869 & -preferred_alignment) - offset;
7871 frame->padding2 = 0;
7873 offset += frame->padding2;
7875 /* We've reached end of stack frame. */
7876 frame->stack_pointer_offset = offset;
7878 /* Size prologue needs to allocate. */
7879 frame->to_allocate =
7880 (size + frame->padding1 + frame->padding2
7881 + frame->outgoing_arguments_size + frame->va_arg_size);
7883 if ((!frame->to_allocate && frame->nregs <= 1)
7884 || (TARGET_64BIT && frame->to_allocate >= (HOST_WIDE_INT) 0x80000000))
7885 frame->save_regs_using_mov = false;
7887 if (!TARGET_64BIT_MS_ABI && TARGET_RED_ZONE && current_function_sp_is_unchanging
7888 && current_function_is_leaf
7889 && !ix86_current_function_calls_tls_descriptor)
7891 frame->red_zone_size = frame->to_allocate;
7892 if (frame->save_regs_using_mov)
7893 frame->red_zone_size += frame->nregs * UNITS_PER_WORD;
7894 if (frame->red_zone_size > RED_ZONE_SIZE - RED_ZONE_RESERVE)
7895 frame->red_zone_size = RED_ZONE_SIZE - RED_ZONE_RESERVE;
7898 frame->red_zone_size = 0;
7899 frame->to_allocate -= frame->red_zone_size;
7900 frame->stack_pointer_offset -= frame->red_zone_size;
7902 fprintf (stderr, "\n");
7903 fprintf (stderr, "size: %ld\n", (long)size);
7904 fprintf (stderr, "nregs: %ld\n", (long)frame->nregs);
7905 fprintf (stderr, "nsseregs: %ld\n", (long)frame->nsseregs);
7906 fprintf (stderr, "padding0: %ld\n", (long)frame->padding0);
7907 fprintf (stderr, "alignment1: %ld\n", (long)stack_alignment_needed);
7908 fprintf (stderr, "padding1: %ld\n", (long)frame->padding1);
7909 fprintf (stderr, "va_arg: %ld\n", (long)frame->va_arg_size);
7910 fprintf (stderr, "padding2: %ld\n", (long)frame->padding2);
7911 fprintf (stderr, "to_allocate: %ld\n", (long)frame->to_allocate);
7912 fprintf (stderr, "red_zone_size: %ld\n", (long)frame->red_zone_size);
7913 fprintf (stderr, "frame_pointer_offset: %ld\n", (long)frame->frame_pointer_offset);
7914 fprintf (stderr, "hard_frame_pointer_offset: %ld\n",
7915 (long)frame->hard_frame_pointer_offset);
7916 fprintf (stderr, "stack_pointer_offset: %ld\n", (long)frame->stack_pointer_offset);
7917 fprintf (stderr, "current_function_is_leaf: %ld\n", (long)current_function_is_leaf);
7918 fprintf (stderr, "cfun->calls_alloca: %ld\n", (long)cfun->calls_alloca);
7919 fprintf (stderr, "x86_current_function_calls_tls_descriptor: %ld\n", (long)ix86_current_function_calls_tls_descriptor);
7923 /* Emit code to save registers in the prologue. */
7926 ix86_emit_save_regs (void)
7931 for (regno = FIRST_PSEUDO_REGISTER - 1; regno-- > 0; )
7932 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
7934 insn = emit_insn (gen_push (gen_rtx_REG (Pmode, regno)));
7935 RTX_FRAME_RELATED_P (insn) = 1;
7939 /* Emit code to save registers using MOV insns. First register
7940 is restored from POINTER + OFFSET. */
7942 ix86_emit_save_regs_using_mov (rtx pointer, HOST_WIDE_INT offset)
7947 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
7948 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
7950 insn = emit_move_insn (adjust_address (gen_rtx_MEM (Pmode, pointer),
7952 gen_rtx_REG (Pmode, regno));
7953 RTX_FRAME_RELATED_P (insn) = 1;
7954 offset += UNITS_PER_WORD;
7958 /* Emit code to save registers using MOV insns. First register
7959 is restored from POINTER + OFFSET. */
7961 ix86_emit_save_sse_regs_using_mov (rtx pointer, HOST_WIDE_INT offset)
7967 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
7968 if (SSE_REGNO_P (regno) && ix86_save_reg (regno, true))
7970 mem = adjust_address (gen_rtx_MEM (TImode, pointer), TImode, offset);
7971 set_mem_align (mem, 128);
7972 insn = emit_move_insn (mem, gen_rtx_REG (TImode, regno));
7973 RTX_FRAME_RELATED_P (insn) = 1;
7978 /* Expand prologue or epilogue stack adjustment.
7979 The pattern exist to put a dependency on all ebp-based memory accesses.
7980 STYLE should be negative if instructions should be marked as frame related,
7981 zero if %r11 register is live and cannot be freely used and positive
7985 pro_epilogue_adjust_stack (rtx dest, rtx src, rtx offset, int style)
7990 insn = emit_insn (gen_pro_epilogue_adjust_stack_1 (dest, src, offset));
7991 else if (x86_64_immediate_operand (offset, DImode))
7992 insn = emit_insn (gen_pro_epilogue_adjust_stack_rex64 (dest, src, offset));
7996 /* r11 is used by indirect sibcall return as well, set before the
7997 epilogue and used after the epilogue. ATM indirect sibcall
7998 shouldn't be used together with huge frame sizes in one
7999 function because of the frame_size check in sibcall.c. */
8001 r11 = gen_rtx_REG (DImode, R11_REG);
8002 insn = emit_insn (gen_rtx_SET (DImode, r11, offset));
8004 RTX_FRAME_RELATED_P (insn) = 1;
8005 insn = emit_insn (gen_pro_epilogue_adjust_stack_rex64_2 (dest, src, r11,
8009 RTX_FRAME_RELATED_P (insn) = 1;
8012 /* Find an available register to be used as dynamic realign argument
8013 pointer regsiter. Such a register will be written in prologue and
8014 used in begin of body, so it must not be
8015 1. parameter passing register.
8017 We reuse static-chain register if it is available. Otherwise, we
8018 use DI for i386 and R13 for x86-64. We chose R13 since it has
8021 Return: the regno of chosen register. */
8024 find_drap_reg (void)
8026 tree decl = cfun->decl;
8030 /* Use R13 for nested function or function need static chain.
8031 Since function with tail call may use any caller-saved
8032 registers in epilogue, DRAP must not use caller-saved
8033 register in such case. */
8034 if ((decl_function_context (decl)
8035 && !DECL_NO_STATIC_CHAIN (decl))
8036 || crtl->tail_call_emit)
8043 /* Use DI for nested function or function need static chain.
8044 Since function with tail call may use any caller-saved
8045 registers in epilogue, DRAP must not use caller-saved
8046 register in such case. */
8047 if ((decl_function_context (decl)
8048 && !DECL_NO_STATIC_CHAIN (decl))
8049 || crtl->tail_call_emit)
8052 /* Reuse static chain register if it isn't used for parameter
8054 if (ix86_function_regparm (TREE_TYPE (decl), decl) <= 2
8055 && !lookup_attribute ("fastcall",
8056 TYPE_ATTRIBUTES (TREE_TYPE (decl))))
8063 /* Update incoming stack boundary and estimated stack alignment. */
8066 ix86_update_stack_boundary (void)
8068 /* Prefer the one specified at command line. */
8069 ix86_incoming_stack_boundary
8070 = (ix86_user_incoming_stack_boundary
8071 ? ix86_user_incoming_stack_boundary
8072 : ix86_default_incoming_stack_boundary);
8074 /* Incoming stack alignment can be changed on individual functions
8075 via force_align_arg_pointer attribute. We use the smallest
8076 incoming stack boundary. */
8077 if (ix86_incoming_stack_boundary > MIN_STACK_BOUNDARY
8078 && lookup_attribute (ix86_force_align_arg_pointer_string,
8079 TYPE_ATTRIBUTES (TREE_TYPE (current_function_decl))))
8080 ix86_incoming_stack_boundary = MIN_STACK_BOUNDARY;
8082 /* The incoming stack frame has to be aligned at least at
8083 parm_stack_boundary. */
8084 if (ix86_incoming_stack_boundary < crtl->parm_stack_boundary)
8085 ix86_incoming_stack_boundary = crtl->parm_stack_boundary;
8087 /* Stack at entrance of main is aligned by runtime. We use the
8088 smallest incoming stack boundary. */
8089 if (ix86_incoming_stack_boundary > MAIN_STACK_BOUNDARY
8090 && DECL_NAME (current_function_decl)
8091 && MAIN_NAME_P (DECL_NAME (current_function_decl))
8092 && DECL_FILE_SCOPE_P (current_function_decl))
8093 ix86_incoming_stack_boundary = MAIN_STACK_BOUNDARY;
8095 /* x86_64 vararg needs 16byte stack alignment for register save
8099 && crtl->stack_alignment_estimated < 128)
8100 crtl->stack_alignment_estimated = 128;
8103 /* Handle the TARGET_GET_DRAP_RTX hook. Return NULL if no DRAP is
8104 needed or an rtx for DRAP otherwise. */
8107 ix86_get_drap_rtx (void)
8109 if (ix86_force_drap || !ACCUMULATE_OUTGOING_ARGS)
8110 crtl->need_drap = true;
8112 if (stack_realign_drap)
8114 /* Assign DRAP to vDRAP and returns vDRAP */
8115 unsigned int regno = find_drap_reg ();
8120 arg_ptr = gen_rtx_REG (Pmode, regno);
8121 crtl->drap_reg = arg_ptr;
8124 drap_vreg = copy_to_reg (arg_ptr);
8128 insn = emit_insn_before (seq, NEXT_INSN (entry_of_function ()));
8129 RTX_FRAME_RELATED_P (insn) = 1;
8136 /* Handle the TARGET_INTERNAL_ARG_POINTER hook. */
8139 ix86_internal_arg_pointer (void)
8141 return virtual_incoming_args_rtx;
8144 /* Handle the TARGET_DWARF_HANDLE_FRAME_UNSPEC hook.
8145 This is called from dwarf2out.c to emit call frame instructions
8146 for frame-related insns containing UNSPECs and UNSPEC_VOLATILEs. */
8148 ix86_dwarf_handle_frame_unspec (const char *label, rtx pattern, int index)
8150 rtx unspec = SET_SRC (pattern);
8151 gcc_assert (GET_CODE (unspec) == UNSPEC);
8155 case UNSPEC_REG_SAVE:
8156 dwarf2out_reg_save_reg (label, XVECEXP (unspec, 0, 0),
8157 SET_DEST (pattern));
8159 case UNSPEC_DEF_CFA:
8160 dwarf2out_def_cfa (label, REGNO (SET_DEST (pattern)),
8161 INTVAL (XVECEXP (unspec, 0, 0)));
8168 /* Finalize stack_realign_needed flag, which will guide prologue/epilogue
8169 to be generated in correct form. */
8171 ix86_finalize_stack_realign_flags (void)
8173 /* Check if stack realign is really needed after reload, and
8174 stores result in cfun */
8175 unsigned int incoming_stack_boundary
8176 = (crtl->parm_stack_boundary > ix86_incoming_stack_boundary
8177 ? crtl->parm_stack_boundary : ix86_incoming_stack_boundary);
8178 unsigned int stack_realign = (incoming_stack_boundary
8179 < (current_function_is_leaf
8180 ? crtl->max_used_stack_slot_alignment
8181 : crtl->stack_alignment_needed));
8183 if (crtl->stack_realign_finalized)
8185 /* After stack_realign_needed is finalized, we can't no longer
8187 gcc_assert (crtl->stack_realign_needed == stack_realign);
8191 crtl->stack_realign_needed = stack_realign;
8192 crtl->stack_realign_finalized = true;
8196 /* Expand the prologue into a bunch of separate insns. */
8199 ix86_expand_prologue (void)
8203 struct ix86_frame frame;
8204 HOST_WIDE_INT allocate;
8206 ix86_finalize_stack_realign_flags ();
8208 /* DRAP should not coexist with stack_realign_fp */
8209 gcc_assert (!(crtl->drap_reg && stack_realign_fp));
8211 ix86_compute_frame_layout (&frame);
8213 /* Emit prologue code to adjust stack alignment and setup DRAP, in case
8214 of DRAP is needed and stack realignment is really needed after reload */
8215 if (crtl->drap_reg && crtl->stack_realign_needed)
8218 int align_bytes = crtl->stack_alignment_needed / BITS_PER_UNIT;
8219 int param_ptr_offset = (call_used_regs[REGNO (crtl->drap_reg)]
8220 ? 0 : UNITS_PER_WORD);
8222 gcc_assert (stack_realign_drap);
8224 /* Grab the argument pointer. */
8225 x = plus_constant (stack_pointer_rtx,
8226 (UNITS_PER_WORD + param_ptr_offset));
8229 /* Only need to push parameter pointer reg if it is caller
8231 if (!call_used_regs[REGNO (crtl->drap_reg)])
8233 /* Push arg pointer reg */
8234 insn = emit_insn (gen_push (y));
8235 RTX_FRAME_RELATED_P (insn) = 1;
8238 insn = emit_insn (gen_rtx_SET (VOIDmode, y, x));
8239 RTX_FRAME_RELATED_P (insn) = 1;
8241 /* Align the stack. */
8242 insn = emit_insn ((*ix86_gen_andsp) (stack_pointer_rtx,
8244 GEN_INT (-align_bytes)));
8245 RTX_FRAME_RELATED_P (insn) = 1;
8247 /* Replicate the return address on the stack so that return
8248 address can be reached via (argp - 1) slot. This is needed
8249 to implement macro RETURN_ADDR_RTX and intrinsic function
8250 expand_builtin_return_addr etc. */
8252 x = gen_frame_mem (Pmode,
8253 plus_constant (x, -UNITS_PER_WORD));
8254 insn = emit_insn (gen_push (x));
8255 RTX_FRAME_RELATED_P (insn) = 1;
8258 /* Note: AT&T enter does NOT have reversed args. Enter is probably
8259 slower on all targets. Also sdb doesn't like it. */
8261 if (frame_pointer_needed)
8263 insn = emit_insn (gen_push (hard_frame_pointer_rtx));
8264 RTX_FRAME_RELATED_P (insn) = 1;
8266 insn = emit_move_insn (hard_frame_pointer_rtx, stack_pointer_rtx);
8267 RTX_FRAME_RELATED_P (insn) = 1;
8270 if (stack_realign_fp)
8272 int align_bytes = crtl->stack_alignment_needed / BITS_PER_UNIT;
8273 gcc_assert (align_bytes > MIN_STACK_BOUNDARY / BITS_PER_UNIT);
8275 /* Align the stack. */
8276 insn = emit_insn ((*ix86_gen_andsp) (stack_pointer_rtx,
8278 GEN_INT (-align_bytes)));
8279 RTX_FRAME_RELATED_P (insn) = 1;
8282 allocate = frame.to_allocate + frame.nsseregs * 16 + frame.padding0;
8284 if (!frame.save_regs_using_mov)
8285 ix86_emit_save_regs ();
8287 allocate += frame.nregs * UNITS_PER_WORD;
8289 /* When using red zone we may start register saving before allocating
8290 the stack frame saving one cycle of the prologue. However I will
8291 avoid doing this if I am going to have to probe the stack since
8292 at least on x86_64 the stack probe can turn into a call that clobbers
8293 a red zone location */
8294 if (!TARGET_64BIT_MS_ABI && TARGET_RED_ZONE && frame.save_regs_using_mov
8295 && (! TARGET_STACK_PROBE || allocate < CHECK_STACK_LIMIT))
8296 ix86_emit_save_regs_using_mov ((frame_pointer_needed
8297 && !crtl->stack_realign_needed)
8298 ? hard_frame_pointer_rtx
8299 : stack_pointer_rtx,
8300 -frame.nregs * UNITS_PER_WORD);
8304 else if (! TARGET_STACK_PROBE || allocate < CHECK_STACK_LIMIT)
8305 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
8306 GEN_INT (-allocate), -1);
8309 /* Only valid for Win32. */
8310 rtx eax = gen_rtx_REG (Pmode, AX_REG);
8314 gcc_assert (!TARGET_64BIT || cfun->machine->call_abi == MS_ABI);
8316 if (cfun->machine->call_abi == MS_ABI)
8319 eax_live = ix86_eax_live_at_start_p ();
8323 emit_insn (gen_push (eax));
8324 allocate -= UNITS_PER_WORD;
8327 emit_move_insn (eax, GEN_INT (allocate));
8330 insn = gen_allocate_stack_worker_64 (eax, eax);
8332 insn = gen_allocate_stack_worker_32 (eax, eax);
8333 insn = emit_insn (insn);
8334 RTX_FRAME_RELATED_P (insn) = 1;
8335 t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, GEN_INT (-allocate));
8336 t = gen_rtx_SET (VOIDmode, stack_pointer_rtx, t);
8337 add_reg_note (insn, REG_FRAME_RELATED_EXPR, t);
8341 if (frame_pointer_needed)
8342 t = plus_constant (hard_frame_pointer_rtx,
8345 - frame.nregs * UNITS_PER_WORD);
8347 t = plus_constant (stack_pointer_rtx, allocate);
8348 emit_move_insn (eax, gen_rtx_MEM (Pmode, t));
8352 if (frame.save_regs_using_mov
8353 && !(!TARGET_64BIT_MS_ABI && TARGET_RED_ZONE
8354 && (! TARGET_STACK_PROBE || allocate < CHECK_STACK_LIMIT)))
8356 if (!frame_pointer_needed
8357 || !frame.to_allocate
8358 || crtl->stack_realign_needed)
8359 ix86_emit_save_regs_using_mov (stack_pointer_rtx,
8361 + frame.nsseregs * 16 + frame.padding0);
8363 ix86_emit_save_regs_using_mov (hard_frame_pointer_rtx,
8364 -frame.nregs * UNITS_PER_WORD);
8366 if (!frame_pointer_needed
8367 || !frame.to_allocate
8368 || crtl->stack_realign_needed)
8369 ix86_emit_save_sse_regs_using_mov (stack_pointer_rtx,
8372 ix86_emit_save_sse_regs_using_mov (hard_frame_pointer_rtx,
8373 - frame.nregs * UNITS_PER_WORD
8374 - frame.nsseregs * 16
8377 pic_reg_used = false;
8378 if (pic_offset_table_rtx
8379 && (df_regs_ever_live_p (REAL_PIC_OFFSET_TABLE_REGNUM)
8382 unsigned int alt_pic_reg_used = ix86_select_alt_pic_regnum ();
8384 if (alt_pic_reg_used != INVALID_REGNUM)
8385 SET_REGNO (pic_offset_table_rtx, alt_pic_reg_used);
8387 pic_reg_used = true;
8394 if (ix86_cmodel == CM_LARGE_PIC)
8396 rtx tmp_reg = gen_rtx_REG (DImode, R11_REG);
8397 rtx label = gen_label_rtx ();
8399 LABEL_PRESERVE_P (label) = 1;
8400 gcc_assert (REGNO (pic_offset_table_rtx) != REGNO (tmp_reg));
8401 insn = emit_insn (gen_set_rip_rex64 (pic_offset_table_rtx, label));
8402 insn = emit_insn (gen_set_got_offset_rex64 (tmp_reg, label));
8403 insn = emit_insn (gen_adddi3 (pic_offset_table_rtx,
8404 pic_offset_table_rtx, tmp_reg));
8407 insn = emit_insn (gen_set_got_rex64 (pic_offset_table_rtx));
8410 insn = emit_insn (gen_set_got (pic_offset_table_rtx));
8413 /* In the pic_reg_used case, make sure that the got load isn't deleted
8414 when mcount needs it. Blockage to avoid call movement across mcount
8415 call is emitted in generic code after the NOTE_INSN_PROLOGUE_END
8417 if (crtl->profile && pic_reg_used)
8418 emit_insn (gen_prologue_use (pic_offset_table_rtx));
8420 if (crtl->drap_reg && !crtl->stack_realign_needed)
8422 /* vDRAP is setup but after reload it turns out stack realign
8423 isn't necessary, here we will emit prologue to setup DRAP
8424 without stack realign adjustment */
8425 int drap_bp_offset = UNITS_PER_WORD * 2;
8426 rtx x = plus_constant (hard_frame_pointer_rtx, drap_bp_offset);
8427 insn = emit_insn (gen_rtx_SET (VOIDmode, crtl->drap_reg, x));
8430 /* Prevent instructions from being scheduled into register save push
8431 sequence when access to the redzone area is done through frame pointer.
8432 The offset betweeh the frame pointer and the stack pointer is calculated
8433 relative to the value of the stack pointer at the end of the function
8434 prologue, and moving instructions that access redzone area via frame
8435 pointer inside push sequence violates this assumption. */
8436 if (frame_pointer_needed && frame.red_zone_size)
8437 emit_insn (gen_memory_blockage ());
8439 /* Emit cld instruction if stringops are used in the function. */
8440 if (TARGET_CLD && ix86_current_function_needs_cld)
8441 emit_insn (gen_cld ());
8444 /* Emit code to restore saved registers using MOV insns. First register
8445 is restored from POINTER + OFFSET. */
8447 ix86_emit_restore_regs_using_mov (rtx pointer, HOST_WIDE_INT offset,
8448 int maybe_eh_return)
8451 rtx base_address = gen_rtx_MEM (Pmode, pointer);
8453 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
8454 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, maybe_eh_return))
8456 /* Ensure that adjust_address won't be forced to produce pointer
8457 out of range allowed by x86-64 instruction set. */
8458 if (TARGET_64BIT && offset != trunc_int_for_mode (offset, SImode))
8462 r11 = gen_rtx_REG (DImode, R11_REG);
8463 emit_move_insn (r11, GEN_INT (offset));
8464 emit_insn (gen_adddi3 (r11, r11, pointer));
8465 base_address = gen_rtx_MEM (Pmode, r11);
8468 emit_move_insn (gen_rtx_REG (Pmode, regno),
8469 adjust_address (base_address, Pmode, offset));
8470 offset += UNITS_PER_WORD;
8474 /* Emit code to restore saved registers using MOV insns. First register
8475 is restored from POINTER + OFFSET. */
8477 ix86_emit_restore_sse_regs_using_mov (rtx pointer, HOST_WIDE_INT offset,
8478 int maybe_eh_return)
8481 rtx base_address = gen_rtx_MEM (TImode, pointer);
8484 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
8485 if (SSE_REGNO_P (regno) && ix86_save_reg (regno, maybe_eh_return))
8487 /* Ensure that adjust_address won't be forced to produce pointer
8488 out of range allowed by x86-64 instruction set. */
8489 if (TARGET_64BIT && offset != trunc_int_for_mode (offset, SImode))
8493 r11 = gen_rtx_REG (DImode, R11_REG);
8494 emit_move_insn (r11, GEN_INT (offset));
8495 emit_insn (gen_adddi3 (r11, r11, pointer));
8496 base_address = gen_rtx_MEM (TImode, r11);
8499 mem = adjust_address (base_address, TImode, offset);
8500 set_mem_align (mem, 128);
8501 emit_move_insn (gen_rtx_REG (TImode, regno), mem);
8506 /* Restore function stack, frame, and registers. */
8509 ix86_expand_epilogue (int style)
8513 struct ix86_frame frame;
8514 HOST_WIDE_INT offset;
8516 ix86_finalize_stack_realign_flags ();
8518 /* When stack is realigned, SP must be valid. */
8519 sp_valid = (!frame_pointer_needed
8520 || current_function_sp_is_unchanging
8521 || stack_realign_fp);
8523 ix86_compute_frame_layout (&frame);
8525 /* See the comment about red zone and frame
8526 pointer usage in ix86_expand_prologue. */
8527 if (frame_pointer_needed && frame.red_zone_size)
8528 emit_insn (gen_memory_blockage ());
8530 /* Calculate start of saved registers relative to ebp. Special care
8531 must be taken for the normal return case of a function using
8532 eh_return: the eax and edx registers are marked as saved, but not
8533 restored along this path. */
8534 offset = frame.nregs;
8535 if (crtl->calls_eh_return && style != 2)
8537 offset *= -UNITS_PER_WORD;
8538 offset -= frame.nsseregs * 16 + frame.padding0;
8540 /* If we're only restoring one register and sp is not valid then
8541 using a move instruction to restore the register since it's
8542 less work than reloading sp and popping the register.
8544 The default code result in stack adjustment using add/lea instruction,
8545 while this code results in LEAVE instruction (or discrete equivalent),
8546 so it is profitable in some other cases as well. Especially when there
8547 are no registers to restore. We also use this code when TARGET_USE_LEAVE
8548 and there is exactly one register to pop. This heuristic may need some
8549 tuning in future. */
8550 if ((!sp_valid && (frame.nregs + frame.nsseregs) <= 1)
8551 || (TARGET_EPILOGUE_USING_MOVE
8552 && cfun->machine->use_fast_prologue_epilogue
8553 && ((frame.nregs + frame.nsseregs) > 1 || frame.to_allocate))
8554 || (frame_pointer_needed && !(frame.nregs + frame.nsseregs) && frame.to_allocate)
8555 || (frame_pointer_needed && TARGET_USE_LEAVE
8556 && cfun->machine->use_fast_prologue_epilogue
8557 && (frame.nregs + frame.nsseregs) == 1)
8558 || crtl->calls_eh_return)
8560 /* Restore registers. We can use ebp or esp to address the memory
8561 locations. If both are available, default to ebp, since offsets
8562 are known to be small. Only exception is esp pointing directly
8563 to the end of block of saved registers, where we may simplify
8566 If we are realigning stack with bp and sp, regs restore can't
8567 be addressed by bp. sp must be used instead. */
8569 if (!frame_pointer_needed
8570 || (sp_valid && !frame.to_allocate)
8571 || stack_realign_fp)
8573 ix86_emit_restore_sse_regs_using_mov (stack_pointer_rtx,
8574 frame.to_allocate, style == 2);
8575 ix86_emit_restore_regs_using_mov (stack_pointer_rtx,
8577 + frame.nsseregs * 16
8578 + frame.padding0, style == 2);
8582 ix86_emit_restore_sse_regs_using_mov (hard_frame_pointer_rtx,
8583 offset, style == 2);
8584 ix86_emit_restore_regs_using_mov (hard_frame_pointer_rtx,
8586 + frame.nsseregs * 16
8587 + frame.padding0, style == 2);
8590 /* eh_return epilogues need %ecx added to the stack pointer. */
8593 rtx tmp, sa = EH_RETURN_STACKADJ_RTX;
8595 /* Stack align doesn't work with eh_return. */
8596 gcc_assert (!crtl->stack_realign_needed);
8598 if (frame_pointer_needed)
8600 tmp = gen_rtx_PLUS (Pmode, hard_frame_pointer_rtx, sa);
8601 tmp = plus_constant (tmp, UNITS_PER_WORD);
8602 emit_insn (gen_rtx_SET (VOIDmode, sa, tmp));
8604 tmp = gen_rtx_MEM (Pmode, hard_frame_pointer_rtx);
8605 emit_move_insn (hard_frame_pointer_rtx, tmp);
8607 pro_epilogue_adjust_stack (stack_pointer_rtx, sa,
8612 tmp = gen_rtx_PLUS (Pmode, stack_pointer_rtx, sa);
8613 tmp = plus_constant (tmp, (frame.to_allocate
8614 + frame.nregs * UNITS_PER_WORD
8615 + frame.nsseregs * 16
8617 emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx, tmp));
8620 else if (!frame_pointer_needed)
8621 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
8622 GEN_INT (frame.to_allocate
8623 + frame.nregs * UNITS_PER_WORD
8624 + frame.nsseregs * 16
8627 /* If not an i386, mov & pop is faster than "leave". */
8628 else if (TARGET_USE_LEAVE || optimize_function_for_size_p (cfun)
8629 || !cfun->machine->use_fast_prologue_epilogue)
8630 emit_insn ((*ix86_gen_leave) ());
8633 pro_epilogue_adjust_stack (stack_pointer_rtx,
8634 hard_frame_pointer_rtx,
8637 emit_insn ((*ix86_gen_pop1) (hard_frame_pointer_rtx));
8642 /* First step is to deallocate the stack frame so that we can
8645 If we realign stack with frame pointer, then stack pointer
8646 won't be able to recover via lea $offset(%bp), %sp, because
8647 there is a padding area between bp and sp for realign.
8648 "add $to_allocate, %sp" must be used instead. */
8651 gcc_assert (frame_pointer_needed);
8652 gcc_assert (!stack_realign_fp);
8653 pro_epilogue_adjust_stack (stack_pointer_rtx,
8654 hard_frame_pointer_rtx,
8655 GEN_INT (offset), style);
8656 ix86_emit_restore_sse_regs_using_mov (stack_pointer_rtx,
8657 frame.to_allocate, style == 2);
8658 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
8659 GEN_INT (frame.nsseregs * 16), style);
8661 else if (frame.to_allocate || frame.nsseregs)
8663 ix86_emit_restore_sse_regs_using_mov (stack_pointer_rtx,
8666 pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx,
8667 GEN_INT (frame.to_allocate
8668 + frame.nsseregs * 16
8669 + frame.padding0), style);
8672 for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
8673 if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, false))
8674 emit_insn ((*ix86_gen_pop1) (gen_rtx_REG (Pmode, regno)));
8675 if (frame_pointer_needed)
8677 /* Leave results in shorter dependency chains on CPUs that are
8678 able to grok it fast. */
8679 if (TARGET_USE_LEAVE)
8680 emit_insn ((*ix86_gen_leave) ());
8683 /* For stack realigned really happens, recover stack
8684 pointer to hard frame pointer is a must, if not using
8686 if (stack_realign_fp)
8687 pro_epilogue_adjust_stack (stack_pointer_rtx,
8688 hard_frame_pointer_rtx,
8690 emit_insn ((*ix86_gen_pop1) (hard_frame_pointer_rtx));
8695 if (crtl->drap_reg && crtl->stack_realign_needed)
8697 int param_ptr_offset = (call_used_regs[REGNO (crtl->drap_reg)]
8698 ? 0 : UNITS_PER_WORD);
8699 gcc_assert (stack_realign_drap);
8700 emit_insn ((*ix86_gen_add3) (stack_pointer_rtx,
8702 GEN_INT (-(UNITS_PER_WORD
8703 + param_ptr_offset))));
8704 if (!call_used_regs[REGNO (crtl->drap_reg)])
8705 emit_insn ((*ix86_gen_pop1) (crtl->drap_reg));
8709 /* Sibcall epilogues don't want a return instruction. */
8713 if (crtl->args.pops_args && crtl->args.size)
8715 rtx popc = GEN_INT (crtl->args.pops_args);
8717 /* i386 can only pop 64K bytes. If asked to pop more, pop
8718 return address, do explicit add, and jump indirectly to the
8721 if (crtl->args.pops_args >= 65536)
8723 rtx ecx = gen_rtx_REG (SImode, CX_REG);
8725 /* There is no "pascal" calling convention in any 64bit ABI. */
8726 gcc_assert (!TARGET_64BIT);
8728 emit_insn (gen_popsi1 (ecx));
8729 emit_insn (gen_addsi3 (stack_pointer_rtx, stack_pointer_rtx, popc));
8730 emit_jump_insn (gen_return_indirect_internal (ecx));
8733 emit_jump_insn (gen_return_pop_internal (popc));
8736 emit_jump_insn (gen_return_internal ());
8739 /* Reset from the function's potential modifications. */
8742 ix86_output_function_epilogue (FILE *file ATTRIBUTE_UNUSED,
8743 HOST_WIDE_INT size ATTRIBUTE_UNUSED)
8745 if (pic_offset_table_rtx)
8746 SET_REGNO (pic_offset_table_rtx, REAL_PIC_OFFSET_TABLE_REGNUM);
8748 /* Mach-O doesn't support labels at the end of objects, so if
8749 it looks like we might want one, insert a NOP. */
8751 rtx insn = get_last_insn ();
8754 && NOTE_KIND (insn) != NOTE_INSN_DELETED_LABEL)
8755 insn = PREV_INSN (insn);
8759 && NOTE_KIND (insn) == NOTE_INSN_DELETED_LABEL)))
8760 fputs ("\tnop\n", file);
8766 /* Extract the parts of an RTL expression that is a valid memory address
8767 for an instruction. Return 0 if the structure of the address is
8768 grossly off. Return -1 if the address contains ASHIFT, so it is not
8769 strictly valid, but still used for computing length of lea instruction. */
8772 ix86_decompose_address (rtx addr, struct ix86_address *out)
8774 rtx base = NULL_RTX, index = NULL_RTX, disp = NULL_RTX;
8775 rtx base_reg, index_reg;
8776 HOST_WIDE_INT scale = 1;
8777 rtx scale_rtx = NULL_RTX;
8779 enum ix86_address_seg seg = SEG_DEFAULT;
8781 if (REG_P (addr) || GET_CODE (addr) == SUBREG)
8783 else if (GET_CODE (addr) == PLUS)
8793 addends[n++] = XEXP (op, 1);
8796 while (GET_CODE (op) == PLUS);
8801 for (i = n; i >= 0; --i)
8804 switch (GET_CODE (op))
8809 index = XEXP (op, 0);
8810 scale_rtx = XEXP (op, 1);
8814 if (XINT (op, 1) == UNSPEC_TP
8815 && TARGET_TLS_DIRECT_SEG_REFS
8816 && seg == SEG_DEFAULT)
8817 seg = TARGET_64BIT ? SEG_FS : SEG_GS;
8846 else if (GET_CODE (addr) == MULT)
8848 index = XEXP (addr, 0); /* index*scale */
8849 scale_rtx = XEXP (addr, 1);
8851 else if (GET_CODE (addr) == ASHIFT)
8855 /* We're called for lea too, which implements ashift on occasion. */
8856 index = XEXP (addr, 0);
8857 tmp = XEXP (addr, 1);
8858 if (!CONST_INT_P (tmp))
8860 scale = INTVAL (tmp);
8861 if ((unsigned HOST_WIDE_INT) scale > 3)
8867 disp = addr; /* displacement */
8869 /* Extract the integral value of scale. */
8872 if (!CONST_INT_P (scale_rtx))
8874 scale = INTVAL (scale_rtx);
8877 base_reg = base && GET_CODE (base) == SUBREG ? SUBREG_REG (base) : base;
8878 index_reg = index && GET_CODE (index) == SUBREG ? SUBREG_REG (index) : index;
8880 /* Allow arg pointer and stack pointer as index if there is not scaling. */
8881 if (base_reg && index_reg && scale == 1
8882 && (index_reg == arg_pointer_rtx
8883 || index_reg == frame_pointer_rtx
8884 || (REG_P (index_reg) && REGNO (index_reg) == STACK_POINTER_REGNUM)))
8887 tmp = base, base = index, index = tmp;
8888 tmp = base_reg, base_reg = index_reg, index_reg = tmp;
8891 /* Special case: %ebp cannot be encoded as a base without a displacement. */
8892 if ((base_reg == hard_frame_pointer_rtx
8893 || base_reg == frame_pointer_rtx
8894 || base_reg == arg_pointer_rtx) && !disp)
8897 /* Special case: on K6, [%esi] makes the instruction vector decoded.
8898 Avoid this by transforming to [%esi+0].
8899 Reload calls address legitimization without cfun defined, so we need
8900 to test cfun for being non-NULL. */
8901 if (TARGET_K6 && cfun && optimize_function_for_speed_p (cfun)
8902 && base_reg && !index_reg && !disp
8904 && REGNO_REG_CLASS (REGNO (base_reg)) == SIREG)
8907 /* Special case: encode reg+reg instead of reg*2. */
8908 if (!base && index && scale && scale == 2)
8909 base = index, base_reg = index_reg, scale = 1;
8911 /* Special case: scaling cannot be encoded without base or displacement. */
8912 if (!base && !disp && index && scale != 1)
8924 /* Return cost of the memory address x.
8925 For i386, it is better to use a complex address than let gcc copy
8926 the address into a reg and make a new pseudo. But not if the address
8927 requires to two regs - that would mean more pseudos with longer
8930 ix86_address_cost (rtx x, bool speed ATTRIBUTE_UNUSED)
8932 struct ix86_address parts;
8934 int ok = ix86_decompose_address (x, &parts);
8938 if (parts.base && GET_CODE (parts.base) == SUBREG)
8939 parts.base = SUBREG_REG (parts.base);
8940 if (parts.index && GET_CODE (parts.index) == SUBREG)
8941 parts.index = SUBREG_REG (parts.index);
8943 /* Attempt to minimize number of registers in the address. */
8945 && (!REG_P (parts.base) || REGNO (parts.base) >= FIRST_PSEUDO_REGISTER))
8947 && (!REG_P (parts.index)
8948 || REGNO (parts.index) >= FIRST_PSEUDO_REGISTER)))
8952 && (!REG_P (parts.base) || REGNO (parts.base) >= FIRST_PSEUDO_REGISTER)
8954 && (!REG_P (parts.index) || REGNO (parts.index) >= FIRST_PSEUDO_REGISTER)
8955 && parts.base != parts.index)
8958 /* AMD-K6 don't like addresses with ModR/M set to 00_xxx_100b,
8959 since it's predecode logic can't detect the length of instructions
8960 and it degenerates to vector decoded. Increase cost of such
8961 addresses here. The penalty is minimally 2 cycles. It may be worthwhile
8962 to split such addresses or even refuse such addresses at all.
8964 Following addressing modes are affected:
8969 The first and last case may be avoidable by explicitly coding the zero in
8970 memory address, but I don't have AMD-K6 machine handy to check this
8974 && ((!parts.disp && parts.base && parts.index && parts.scale != 1)
8975 || (parts.disp && !parts.base && parts.index && parts.scale != 1)
8976 || (!parts.disp && parts.base && parts.index && parts.scale == 1)))
8982 /* Allow {LABEL | SYMBOL}_REF - SYMBOL_REF-FOR-PICBASE for Mach-O as
8983 this is used for to form addresses to local data when -fPIC is in
8987 darwin_local_data_pic (rtx disp)
8989 return (GET_CODE (disp) == UNSPEC
8990 && XINT (disp, 1) == UNSPEC_MACHOPIC_OFFSET);
8993 /* Determine if a given RTX is a valid constant. We already know this
8994 satisfies CONSTANT_P. */
8997 legitimate_constant_p (rtx x)
8999 switch (GET_CODE (x))
9004 if (GET_CODE (x) == PLUS)
9006 if (!CONST_INT_P (XEXP (x, 1)))
9011 if (TARGET_MACHO && darwin_local_data_pic (x))
9014 /* Only some unspecs are valid as "constants". */
9015 if (GET_CODE (x) == UNSPEC)
9016 switch (XINT (x, 1))
9021 return TARGET_64BIT;
9024 x = XVECEXP (x, 0, 0);
9025 return (GET_CODE (x) == SYMBOL_REF
9026 && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
9028 x = XVECEXP (x, 0, 0);
9029 return (GET_CODE (x) == SYMBOL_REF
9030 && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_DYNAMIC);
9035 /* We must have drilled down to a symbol. */
9036 if (GET_CODE (x) == LABEL_REF)
9038 if (GET_CODE (x) != SYMBOL_REF)
9043 /* TLS symbols are never valid. */
9044 if (SYMBOL_REF_TLS_MODEL (x))
9047 /* DLLIMPORT symbols are never valid. */
9048 if (TARGET_DLLIMPORT_DECL_ATTRIBUTES
9049 && SYMBOL_REF_DLLIMPORT_P (x))
9054 if (GET_MODE (x) == TImode
9055 && x != CONST0_RTX (TImode)
9061 if (!standard_sse_constant_p (x))
9068 /* Otherwise we handle everything else in the move patterns. */
9072 /* Determine if it's legal to put X into the constant pool. This
9073 is not possible for the address of thread-local symbols, which
9074 is checked above. */
9077 ix86_cannot_force_const_mem (rtx x)
9079 /* We can always put integral constants and vectors in memory. */
9080 switch (GET_CODE (x))
9090 return !legitimate_constant_p (x);
9094 /* Nonzero if the constant value X is a legitimate general operand
9095 when generating PIC code. It is given that flag_pic is on and
9096 that X satisfies CONSTANT_P or is a CONST_DOUBLE. */
9099 legitimate_pic_operand_p (rtx x)
9103 switch (GET_CODE (x))
9106 inner = XEXP (x, 0);
9107 if (GET_CODE (inner) == PLUS
9108 && CONST_INT_P (XEXP (inner, 1)))
9109 inner = XEXP (inner, 0);
9111 /* Only some unspecs are valid as "constants". */
9112 if (GET_CODE (inner) == UNSPEC)
9113 switch (XINT (inner, 1))
9118 return TARGET_64BIT;
9120 x = XVECEXP (inner, 0, 0);
9121 return (GET_CODE (x) == SYMBOL_REF
9122 && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_EXEC);
9123 case UNSPEC_MACHOPIC_OFFSET:
9124 return legitimate_pic_address_disp_p (x);
9132 return legitimate_pic_address_disp_p (x);
9139 /* Determine if a given CONST RTX is a valid memory displacement
9143 legitimate_pic_address_disp_p (rtx disp)
9147 /* In 64bit mode we can allow direct addresses of symbols and labels
9148 when they are not dynamic symbols. */
9151 rtx op0 = disp, op1;
9153 switch (GET_CODE (disp))
9159 if (GET_CODE (XEXP (disp, 0)) != PLUS)
9161 op0 = XEXP (XEXP (disp, 0), 0);
9162 op1 = XEXP (XEXP (disp, 0), 1);
9163 if (!CONST_INT_P (op1)
9164 || INTVAL (op1) >= 16*1024*1024
9165 || INTVAL (op1) < -16*1024*1024)
9167 if (GET_CODE (op0) == LABEL_REF)
9169 if (GET_CODE (op0) != SYMBOL_REF)
9174 /* TLS references should always be enclosed in UNSPEC. */
9175 if (SYMBOL_REF_TLS_MODEL (op0))
9177 if (!SYMBOL_REF_FAR_ADDR_P (op0) && SYMBOL_REF_LOCAL_P (op0)
9178 && ix86_cmodel != CM_LARGE_PIC)
9186 if (GET_CODE (disp) != CONST)
9188 disp = XEXP (disp, 0);
9192 /* We are unsafe to allow PLUS expressions. This limit allowed distance
9193 of GOT tables. We should not need these anyway. */
9194 if (GET_CODE (disp) != UNSPEC
9195 || (XINT (disp, 1) != UNSPEC_GOTPCREL
9196 && XINT (disp, 1) != UNSPEC_GOTOFF
9197 && XINT (disp, 1) != UNSPEC_PLTOFF))
9200 if (GET_CODE (XVECEXP (disp, 0, 0)) != SYMBOL_REF
9201 && GET_CODE (XVECEXP (disp, 0, 0)) != LABEL_REF)
9207 if (GET_CODE (disp) == PLUS)
9209 if (!CONST_INT_P (XEXP (disp, 1)))
9211 disp = XEXP (disp, 0);
9215 if (TARGET_MACHO && darwin_local_data_pic (disp))
9218 if (GET_CODE (disp) != UNSPEC)
9221 switch (XINT (disp, 1))
9226 /* We need to check for both symbols and labels because VxWorks loads
9227 text labels with @GOT rather than @GOTOFF. See gotoff_operand for
9229 return (GET_CODE (XVECEXP (disp, 0, 0)) == SYMBOL_REF
9230 || GET_CODE (XVECEXP (disp, 0, 0)) == LABEL_REF);
9232 /* Refuse GOTOFF in 64bit mode since it is always 64bit when used.
9233 While ABI specify also 32bit relocation but we don't produce it in
9234 small PIC model at all. */
9235 if ((GET_CODE (XVECEXP (disp, 0, 0)) == SYMBOL_REF
9236 || GET_CODE (XVECEXP (disp, 0, 0)) == LABEL_REF)
9238 return gotoff_operand (XVECEXP (disp, 0, 0), Pmode);
9240 case UNSPEC_GOTTPOFF:
9241 case UNSPEC_GOTNTPOFF:
9242 case UNSPEC_INDNTPOFF:
9245 disp = XVECEXP (disp, 0, 0);
9246 return (GET_CODE (disp) == SYMBOL_REF
9247 && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_INITIAL_EXEC);
9249 disp = XVECEXP (disp, 0, 0);
9250 return (GET_CODE (disp) == SYMBOL_REF
9251 && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_EXEC);
9253 disp = XVECEXP (disp, 0, 0);
9254 return (GET_CODE (disp) == SYMBOL_REF
9255 && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_DYNAMIC);
9261 /* Recognizes RTL expressions that are valid memory addresses for an
9262 instruction. The MODE argument is the machine mode for the MEM
9263 expression that wants to use this address.
9265 It only recognizes address in canonical form. LEGITIMIZE_ADDRESS should
9266 convert common non-canonical forms to canonical form so that they will
9270 ix86_legitimate_address_p (enum machine_mode mode ATTRIBUTE_UNUSED,
9271 rtx addr, bool strict)
9273 struct ix86_address parts;
9274 rtx base, index, disp;
9275 HOST_WIDE_INT scale;
9276 const char *reason = NULL;
9277 rtx reason_rtx = NULL_RTX;
9279 if (ix86_decompose_address (addr, &parts) <= 0)
9281 reason = "decomposition failed";
9286 index = parts.index;
9288 scale = parts.scale;
9290 /* Validate base register.
9292 Don't allow SUBREG's that span more than a word here. It can lead to spill
9293 failures when the base is one word out of a two word structure, which is
9294 represented internally as a DImode int. */
9303 else if (GET_CODE (base) == SUBREG
9304 && REG_P (SUBREG_REG (base))
9305 && GET_MODE_SIZE (GET_MODE (SUBREG_REG (base)))
9307 reg = SUBREG_REG (base);
9310 reason = "base is not a register";
9314 if (GET_MODE (base) != Pmode)
9316 reason = "base is not in Pmode";
9320 if ((strict && ! REG_OK_FOR_BASE_STRICT_P (reg))
9321 || (! strict && ! REG_OK_FOR_BASE_NONSTRICT_P (reg)))
9323 reason = "base is not valid";
9328 /* Validate index register.
9330 Don't allow SUBREG's that span more than a word here -- same as above. */
9339 else if (GET_CODE (index) == SUBREG
9340 && REG_P (SUBREG_REG (index))
9341 && GET_MODE_SIZE (GET_MODE (SUBREG_REG (index)))
9343 reg = SUBREG_REG (index);
9346 reason = "index is not a register";
9350 if (GET_MODE (index) != Pmode)
9352 reason = "index is not in Pmode";
9356 if ((strict && ! REG_OK_FOR_INDEX_STRICT_P (reg))
9357 || (! strict && ! REG_OK_FOR_INDEX_NONSTRICT_P (reg)))
9359 reason = "index is not valid";
9364 /* Validate scale factor. */
9367 reason_rtx = GEN_INT (scale);
9370 reason = "scale without index";
9374 if (scale != 2 && scale != 4 && scale != 8)
9376 reason = "scale is not a valid multiplier";
9381 /* Validate displacement. */
9386 if (GET_CODE (disp) == CONST
9387 && GET_CODE (XEXP (disp, 0)) == UNSPEC
9388 && XINT (XEXP (disp, 0), 1) != UNSPEC_MACHOPIC_OFFSET)
9389 switch (XINT (XEXP (disp, 0), 1))
9391 /* Refuse GOTOFF and GOT in 64bit mode since it is always 64bit when
9392 used. While ABI specify also 32bit relocations, we don't produce
9393 them at all and use IP relative instead. */
9396 gcc_assert (flag_pic);
9398 goto is_legitimate_pic;
9399 reason = "64bit address unspec";
9402 case UNSPEC_GOTPCREL:
9403 gcc_assert (flag_pic);
9404 goto is_legitimate_pic;
9406 case UNSPEC_GOTTPOFF:
9407 case UNSPEC_GOTNTPOFF:
9408 case UNSPEC_INDNTPOFF:
9414 reason = "invalid address unspec";
9418 else if (SYMBOLIC_CONST (disp)
9422 && MACHOPIC_INDIRECT
9423 && !machopic_operand_p (disp)
9429 if (TARGET_64BIT && (index || base))
9431 /* foo@dtpoff(%rX) is ok. */
9432 if (GET_CODE (disp) != CONST
9433 || GET_CODE (XEXP (disp, 0)) != PLUS
9434 || GET_CODE (XEXP (XEXP (disp, 0), 0)) != UNSPEC
9435 || !CONST_INT_P (XEXP (XEXP (disp, 0), 1))
9436 || (XINT (XEXP (XEXP (disp, 0), 0), 1) != UNSPEC_DTPOFF
9437 && XINT (XEXP (XEXP (disp, 0), 0), 1) != UNSPEC_NTPOFF))
9439 reason = "non-constant pic memory reference";
9443 else if (! legitimate_pic_address_disp_p (disp))
9445 reason = "displacement is an invalid pic construct";
9449 /* This code used to verify that a symbolic pic displacement
9450 includes the pic_offset_table_rtx register.
9452 While this is good idea, unfortunately these constructs may
9453 be created by "adds using lea" optimization for incorrect
9462 This code is nonsensical, but results in addressing
9463 GOT table with pic_offset_table_rtx base. We can't
9464 just refuse it easily, since it gets matched by
9465 "addsi3" pattern, that later gets split to lea in the
9466 case output register differs from input. While this
9467 can be handled by separate addsi pattern for this case
9468 that never results in lea, this seems to be easier and
9469 correct fix for crash to disable this test. */
9471 else if (GET_CODE (disp) != LABEL_REF
9472 && !CONST_INT_P (disp)
9473 && (GET_CODE (disp) != CONST
9474 || !legitimate_constant_p (disp))
9475 && (GET_CODE (disp) != SYMBOL_REF
9476 || !legitimate_constant_p (disp)))
9478 reason = "displacement is not constant";
9481 else if (TARGET_64BIT
9482 && !x86_64_immediate_operand (disp, VOIDmode))
9484 reason = "displacement is out of range";
9489 /* Everything looks valid. */
9496 /* Determine if a given RTX is a valid constant address. */
9499 constant_address_p (rtx x)
9501 return CONSTANT_P (x) && ix86_legitimate_address_p (Pmode, x, 1);
9504 /* Return a unique alias set for the GOT. */
9506 static alias_set_type
9507 ix86_GOT_alias_set (void)
9509 static alias_set_type set = -1;
9511 set = new_alias_set ();
9515 /* Return a legitimate reference for ORIG (an address) using the
9516 register REG. If REG is 0, a new pseudo is generated.
9518 There are two types of references that must be handled:
9520 1. Global data references must load the address from the GOT, via
9521 the PIC reg. An insn is emitted to do this load, and the reg is
9524 2. Static data references, constant pool addresses, and code labels
9525 compute the address as an offset from the GOT, whose base is in
9526 the PIC reg. Static data objects have SYMBOL_FLAG_LOCAL set to
9527 differentiate them from global data objects. The returned
9528 address is the PIC reg + an unspec constant.
9530 TARGET_LEGITIMATE_ADDRESS_P rejects symbolic references unless the PIC
9531 reg also appears in the address. */
9534 legitimize_pic_address (rtx orig, rtx reg)
9541 if (TARGET_MACHO && !TARGET_64BIT)
9544 reg = gen_reg_rtx (Pmode);
9545 /* Use the generic Mach-O PIC machinery. */
9546 return machopic_legitimize_pic_address (orig, GET_MODE (orig), reg);
9550 if (TARGET_64BIT && legitimate_pic_address_disp_p (addr))
9552 else if (TARGET_64BIT
9553 && ix86_cmodel != CM_SMALL_PIC
9554 && gotoff_operand (addr, Pmode))
9557 /* This symbol may be referenced via a displacement from the PIC
9558 base address (@GOTOFF). */
9560 if (reload_in_progress)
9561 df_set_regs_ever_live (PIC_OFFSET_TABLE_REGNUM, true);
9562 if (GET_CODE (addr) == CONST)
9563 addr = XEXP (addr, 0);
9564 if (GET_CODE (addr) == PLUS)
9566 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, XEXP (addr, 0)),
9568 new_rtx = gen_rtx_PLUS (Pmode, new_rtx, XEXP (addr, 1));
9571 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOTOFF);
9572 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
9574 tmpreg = gen_reg_rtx (Pmode);
9577 emit_move_insn (tmpreg, new_rtx);
9581 new_rtx = expand_simple_binop (Pmode, PLUS, reg, pic_offset_table_rtx,
9582 tmpreg, 1, OPTAB_DIRECT);
9585 else new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, tmpreg);
9587 else if (!TARGET_64BIT && gotoff_operand (addr, Pmode))
9589 /* This symbol may be referenced via a displacement from the PIC
9590 base address (@GOTOFF). */
9592 if (reload_in_progress)
9593 df_set_regs_ever_live (PIC_OFFSET_TABLE_REGNUM, true);
9594 if (GET_CODE (addr) == CONST)
9595 addr = XEXP (addr, 0);
9596 if (GET_CODE (addr) == PLUS)
9598 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, XEXP (addr, 0)),
9600 new_rtx = gen_rtx_PLUS (Pmode, new_rtx, XEXP (addr, 1));
9603 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOTOFF);
9604 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
9605 new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
9609 emit_move_insn (reg, new_rtx);
9613 else if ((GET_CODE (addr) == SYMBOL_REF && SYMBOL_REF_TLS_MODEL (addr) == 0)
9614 /* We can't use @GOTOFF for text labels on VxWorks;
9615 see gotoff_operand. */
9616 || (TARGET_VXWORKS_RTP && GET_CODE (addr) == LABEL_REF))
9618 if (TARGET_DLLIMPORT_DECL_ATTRIBUTES)
9620 if (GET_CODE (addr) == SYMBOL_REF && SYMBOL_REF_DLLIMPORT_P (addr))
9621 return legitimize_dllimport_symbol (addr, true);
9622 if (GET_CODE (addr) == CONST && GET_CODE (XEXP (addr, 0)) == PLUS
9623 && GET_CODE (XEXP (XEXP (addr, 0), 0)) == SYMBOL_REF
9624 && SYMBOL_REF_DLLIMPORT_P (XEXP (XEXP (addr, 0), 0)))
9626 rtx t = legitimize_dllimport_symbol (XEXP (XEXP (addr, 0), 0), true);
9627 return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (addr, 0), 1));
9631 if (TARGET_64BIT && ix86_cmodel != CM_LARGE_PIC)
9633 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOTPCREL);
9634 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
9635 new_rtx = gen_const_mem (Pmode, new_rtx);
9636 set_mem_alias_set (new_rtx, ix86_GOT_alias_set ());
9639 reg = gen_reg_rtx (Pmode);
9640 /* Use directly gen_movsi, otherwise the address is loaded
9641 into register for CSE. We don't want to CSE this addresses,
9642 instead we CSE addresses from the GOT table, so skip this. */
9643 emit_insn (gen_movsi (reg, new_rtx));
9648 /* This symbol must be referenced via a load from the
9649 Global Offset Table (@GOT). */
9651 if (reload_in_progress)
9652 df_set_regs_ever_live (PIC_OFFSET_TABLE_REGNUM, true);
9653 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOT);
9654 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
9656 new_rtx = force_reg (Pmode, new_rtx);
9657 new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
9658 new_rtx = gen_const_mem (Pmode, new_rtx);
9659 set_mem_alias_set (new_rtx, ix86_GOT_alias_set ());
9662 reg = gen_reg_rtx (Pmode);
9663 emit_move_insn (reg, new_rtx);
9669 if (CONST_INT_P (addr)
9670 && !x86_64_immediate_operand (addr, VOIDmode))
9674 emit_move_insn (reg, addr);
9678 new_rtx = force_reg (Pmode, addr);
9680 else if (GET_CODE (addr) == CONST)
9682 addr = XEXP (addr, 0);
9684 /* We must match stuff we generate before. Assume the only
9685 unspecs that can get here are ours. Not that we could do
9686 anything with them anyway.... */
9687 if (GET_CODE (addr) == UNSPEC
9688 || (GET_CODE (addr) == PLUS
9689 && GET_CODE (XEXP (addr, 0)) == UNSPEC))
9691 gcc_assert (GET_CODE (addr) == PLUS);
9693 if (GET_CODE (addr) == PLUS)
9695 rtx op0 = XEXP (addr, 0), op1 = XEXP (addr, 1);
9697 /* Check first to see if this is a constant offset from a @GOTOFF
9698 symbol reference. */
9699 if (gotoff_operand (op0, Pmode)
9700 && CONST_INT_P (op1))
9704 if (reload_in_progress)
9705 df_set_regs_ever_live (PIC_OFFSET_TABLE_REGNUM, true);
9706 new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, op0),
9708 new_rtx = gen_rtx_PLUS (Pmode, new_rtx, op1);
9709 new_rtx = gen_rtx_CONST (Pmode, new_rtx);
9710 new_rtx = gen_rtx_PLUS (Pmode, pic_offset_table_rtx, new_rtx);
9714 emit_move_insn (reg, new_rtx);
9720 if (INTVAL (op1) < -16*1024*1024
9721 || INTVAL (op1) >= 16*1024*1024)
9723 if (!x86_64_immediate_operand (op1, Pmode))
9724 op1 = force_reg (Pmode, op1);
9725 new_rtx = gen_rtx_PLUS (Pmode, force_reg (Pmode, op0), op1);
9731 base = legitimize_pic_address (XEXP (addr, 0), reg);
9732 new_rtx = legitimize_pic_address (XEXP (addr, 1),
9733 base == reg ? NULL_RTX : reg);
9735 if (CONST_INT_P (new_rtx))
9736 new_rtx = plus_constant (base, INTVAL (new_rtx));
9739 if (GET_CODE (new_rtx) == PLUS && CONSTANT_P (XEXP (new_rtx, 1)))
9741 base = gen_rtx_PLUS (Pmode, base, XEXP (new_rtx, 0));
9742 new_rtx = XEXP (new_rtx, 1);
9744 new_rtx = gen_rtx_PLUS (Pmode, base, new_rtx);
9752 /* Load the thread pointer. If TO_REG is true, force it into a register. */
9755 get_thread_pointer (int to_reg)
9759 tp = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx), UNSPEC_TP);
9763 reg = gen_reg_rtx (Pmode);
9764 insn = gen_rtx_SET (VOIDmode, reg, tp);
9765 insn = emit_insn (insn);
9770 /* A subroutine of ix86_legitimize_address and ix86_expand_move. FOR_MOV is
9771 false if we expect this to be used for a memory address and true if
9772 we expect to load the address into a register. */
9775 legitimize_tls_address (rtx x, enum tls_model model, int for_mov)
9777 rtx dest, base, off, pic, tp;
9782 case TLS_MODEL_GLOBAL_DYNAMIC:
9783 dest = gen_reg_rtx (Pmode);
9784 tp = TARGET_GNU2_TLS ? get_thread_pointer (1) : 0;
9786 if (TARGET_64BIT && ! TARGET_GNU2_TLS)
9788 rtx rax = gen_rtx_REG (Pmode, AX_REG), insns;
9791 emit_call_insn (gen_tls_global_dynamic_64 (rax, x));
9792 insns = get_insns ();
9795 RTL_CONST_CALL_P (insns) = 1;
9796 emit_libcall_block (insns, dest, rax, x);
9798 else if (TARGET_64BIT && TARGET_GNU2_TLS)
9799 emit_insn (gen_tls_global_dynamic_64 (dest, x));
9801 emit_insn (gen_tls_global_dynamic_32 (dest, x));
9803 if (TARGET_GNU2_TLS)
9805 dest = force_reg (Pmode, gen_rtx_PLUS (Pmode, tp, dest));
9807 set_unique_reg_note (get_last_insn (), REG_EQUIV, x);
9811 case TLS_MODEL_LOCAL_DYNAMIC:
9812 base = gen_reg_rtx (Pmode);
9813 tp = TARGET_GNU2_TLS ? get_thread_pointer (1) : 0;
9815 if (TARGET_64BIT && ! TARGET_GNU2_TLS)
9817 rtx rax = gen_rtx_REG (Pmode, AX_REG), insns, note;
9820 emit_call_insn (gen_tls_local_dynamic_base_64 (rax));
9821 insns = get_insns ();
9824 note = gen_rtx_EXPR_LIST (VOIDmode, const0_rtx, NULL);
9825 note = gen_rtx_EXPR_LIST (VOIDmode, ix86_tls_get_addr (), note);
9826 RTL_CONST_CALL_P (insns) = 1;
9827 emit_libcall_block (insns, base, rax, note);
9829 else if (TARGET_64BIT && TARGET_GNU2_TLS)
9830 emit_insn (gen_tls_local_dynamic_base_64 (base));
9832 emit_insn (gen_tls_local_dynamic_base_32 (base));
9834 if (TARGET_GNU2_TLS)
9836 rtx x = ix86_tls_module_base ();
9838 set_unique_reg_note (get_last_insn (), REG_EQUIV,
9839 gen_rtx_MINUS (Pmode, x, tp));
9842 off = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x), UNSPEC_DTPOFF);
9843 off = gen_rtx_CONST (Pmode, off);
9845 dest = force_reg (Pmode, gen_rtx_PLUS (Pmode, base, off));
9847 if (TARGET_GNU2_TLS)
9849 dest = force_reg (Pmode, gen_rtx_PLUS (Pmode, dest, tp));
9851 set_unique_reg_note (get_last_insn (), REG_EQUIV, x);
9856 case TLS_MODEL_INITIAL_EXEC:
9860 type = UNSPEC_GOTNTPOFF;
9864 if (reload_in_progress)
9865 df_set_regs_ever_live (PIC_OFFSET_TABLE_REGNUM, true);
9866 pic = pic_offset_table_rtx;
9867 type = TARGET_ANY_GNU_TLS ? UNSPEC_GOTNTPOFF : UNSPEC_GOTTPOFF;
9869 else if (!TARGET_ANY_GNU_TLS)
9871 pic = gen_reg_rtx (Pmode);
9872 emit_insn (gen_set_got (pic));
9873 type = UNSPEC_GOTTPOFF;
9878 type = UNSPEC_INDNTPOFF;
9881 off = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x), type);
9882 off = gen_rtx_CONST (Pmode, off);
9884 off = gen_rtx_PLUS (Pmode, pic, off);
9885 off = gen_const_mem (Pmode, off);
9886 set_mem_alias_set (off, ix86_GOT_alias_set ());
9888 if (TARGET_64BIT || TARGET_ANY_GNU_TLS)
9890 base = get_thread_pointer (for_mov || !TARGET_TLS_DIRECT_SEG_REFS);
9891 off = force_reg (Pmode, off);
9892 return gen_rtx_PLUS (Pmode, base, off);
9896 base = get_thread_pointer (true);
9897 dest = gen_reg_rtx (Pmode);
9898 emit_insn (gen_subsi3 (dest, base, off));
9902 case TLS_MODEL_LOCAL_EXEC:
9903 off = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, x),
9904 (TARGET_64BIT || TARGET_ANY_GNU_TLS)
9905 ? UNSPEC_NTPOFF : UNSPEC_TPOFF);
9906 off = gen_rtx_CONST (Pmode, off);
9908 if (TARGET_64BIT || TARGET_ANY_GNU_TLS)
9910 base = get_thread_pointer (for_mov || !TARGET_TLS_DIRECT_SEG_REFS);
9911 return gen_rtx_PLUS (Pmode, base, off);
9915 base = get_thread_pointer (true);
9916 dest = gen_reg_rtx (Pmode);
9917 emit_insn (gen_subsi3 (dest, base, off));
9928 /* Create or return the unique __imp_DECL dllimport symbol corresponding
9931 static GTY((if_marked ("tree_map_marked_p"), param_is (struct tree_map)))
9932 htab_t dllimport_map;
9935 get_dllimport_decl (tree decl)
9937 struct tree_map *h, in;
9941 size_t namelen, prefixlen;
9947 dllimport_map = htab_create_ggc (512, tree_map_hash, tree_map_eq, 0);
9949 in.hash = htab_hash_pointer (decl);
9950 in.base.from = decl;
9951 loc = htab_find_slot_with_hash (dllimport_map, &in, in.hash, INSERT);
9952 h = (struct tree_map *) *loc;
9956 *loc = h = GGC_NEW (struct tree_map);
9958 h->base.from = decl;
9959 h->to = to = build_decl (VAR_DECL, NULL, ptr_type_node);
9960 DECL_ARTIFICIAL (to) = 1;
9961 DECL_IGNORED_P (to) = 1;
9962 DECL_EXTERNAL (to) = 1;
9963 TREE_READONLY (to) = 1;
9965 name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl));
9966 name = targetm.strip_name_encoding (name);
9967 prefix = name[0] == FASTCALL_PREFIX || user_label_prefix[0] == 0
9968 ? "*__imp_" : "*__imp__";
9969 namelen = strlen (name);
9970 prefixlen = strlen (prefix);
9971 imp_name = (char *) alloca (namelen + prefixlen + 1);
9972 memcpy (imp_name, prefix, prefixlen);
9973 memcpy (imp_name + prefixlen, name, namelen + 1);
9975 name = ggc_alloc_string (imp_name, namelen + prefixlen);
9976 rtl = gen_rtx_SYMBOL_REF (Pmode, name);
9977 SET_SYMBOL_REF_DECL (rtl, to);
9978 SYMBOL_REF_FLAGS (rtl) = SYMBOL_FLAG_LOCAL;
9980 rtl = gen_const_mem (Pmode, rtl);
9981 set_mem_alias_set (rtl, ix86_GOT_alias_set ());
9983 SET_DECL_RTL (to, rtl);
9984 SET_DECL_ASSEMBLER_NAME (to, get_identifier (name));
9989 /* Expand SYMBOL into its corresponding dllimport symbol. WANT_REG is
9990 true if we require the result be a register. */
9993 legitimize_dllimport_symbol (rtx symbol, bool want_reg)
9998 gcc_assert (SYMBOL_REF_DECL (symbol));
9999 imp_decl = get_dllimport_decl (SYMBOL_REF_DECL (symbol));
10001 x = DECL_RTL (imp_decl);
10003 x = force_reg (Pmode, x);
10007 /* Try machine-dependent ways of modifying an illegitimate address
10008 to be legitimate. If we find one, return the new, valid address.
10009 This macro is used in only one place: `memory_address' in explow.c.
10011 OLDX is the address as it was before break_out_memory_refs was called.
10012 In some cases it is useful to look at this to decide what needs to be done.
10014 It is always safe for this macro to do nothing. It exists to recognize
10015 opportunities to optimize the output.
10017 For the 80386, we handle X+REG by loading X into a register R and
10018 using R+REG. R will go in a general reg and indexing will be used.
10019 However, if REG is a broken-out memory address or multiplication,
10020 nothing needs to be done because REG can certainly go in a general reg.
10022 When -fpic is used, special handling is needed for symbolic references.
10023 See comments by legitimize_pic_address in i386.c for details. */
10026 ix86_legitimize_address (rtx x, rtx oldx ATTRIBUTE_UNUSED,
10027 enum machine_mode mode)
10032 log = GET_CODE (x) == SYMBOL_REF ? SYMBOL_REF_TLS_MODEL (x) : 0;
10034 return legitimize_tls_address (x, (enum tls_model) log, false);
10035 if (GET_CODE (x) == CONST
10036 && GET_CODE (XEXP (x, 0)) == PLUS
10037 && GET_CODE (XEXP (XEXP (x, 0), 0)) == SYMBOL_REF
10038 && (log = SYMBOL_REF_TLS_MODEL (XEXP (XEXP (x, 0), 0))))
10040 rtx t = legitimize_tls_address (XEXP (XEXP (x, 0), 0),
10041 (enum tls_model) log, false);
10042 return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (x, 0), 1));
10045 if (TARGET_DLLIMPORT_DECL_ATTRIBUTES)
10047 if (GET_CODE (x) == SYMBOL_REF && SYMBOL_REF_DLLIMPORT_P (x))
10048 return legitimize_dllimport_symbol (x, true);
10049 if (GET_CODE (x) == CONST
10050 && GET_CODE (XEXP (x, 0)) == PLUS
10051 && GET_CODE (XEXP (XEXP (x, 0), 0)) == SYMBOL_REF
10052 && SYMBOL_REF_DLLIMPORT_P (XEXP (XEXP (x, 0), 0)))
10054 rtx t = legitimize_dllimport_symbol (XEXP (XEXP (x, 0), 0), true);
10055 return gen_rtx_PLUS (Pmode, t, XEXP (XEXP (x, 0), 1));
10059 if (flag_pic && SYMBOLIC_CONST (x))
10060 return legitimize_pic_address (x, 0);
10062 /* Canonicalize shifts by 0, 1, 2, 3 into multiply */
10063 if (GET_CODE (x) == ASHIFT
10064 && CONST_INT_P (XEXP (x, 1))
10065 && (unsigned HOST_WIDE_INT) INTVAL (XEXP (x, 1)) < 4)
10068 log = INTVAL (XEXP (x, 1));
10069 x = gen_rtx_MULT (Pmode, force_reg (Pmode, XEXP (x, 0)),
10070 GEN_INT (1 << log));
10073 if (GET_CODE (x) == PLUS)
10075 /* Canonicalize shifts by 0, 1, 2, 3 into multiply. */
10077 if (GET_CODE (XEXP (x, 0)) == ASHIFT
10078 && CONST_INT_P (XEXP (XEXP (x, 0), 1))
10079 && (unsigned HOST_WIDE_INT) INTVAL (XEXP (XEXP (x, 0), 1)) < 4)
10082 log = INTVAL (XEXP (XEXP (x, 0), 1));
10083 XEXP (x, 0) = gen_rtx_MULT (Pmode,
10084 force_reg (Pmode, XEXP (XEXP (x, 0), 0)),
10085 GEN_INT (1 << log));
10088 if (GET_CODE (XEXP (x, 1)) == ASHIFT
10089 && CONST_INT_P (XEXP (XEXP (x, 1), 1))
10090 && (unsigned HOST_WIDE_INT) INTVAL (XEXP (XEXP (x, 1), 1)) < 4)
10093 log = INTVAL (XEXP (XEXP (x, 1), 1));
10094 XEXP (x, 1) = gen_rtx_MULT (Pmode,
10095 force_reg (Pmode, XEXP (XEXP (x, 1), 0)),
10096 GEN_INT (1 << log));
10099 /* Put multiply first if it isn't already. */
10100 if (GET_CODE (XEXP (x, 1)) == MULT)
10102 rtx tmp = XEXP (x, 0);
10103 XEXP (x, 0) = XEXP (x, 1);
10108 /* Canonicalize (plus (mult (reg) (const)) (plus (reg) (const)))
10109 into (plus (plus (mult (reg) (const)) (reg)) (const)). This can be
10110 created by virtual register instantiation, register elimination, and
10111 similar optimizations. */
10112 if (GET_CODE (XEXP (x, 0)) == MULT && GET_CODE (XEXP (x, 1)) == PLUS)
10115 x = gen_rtx_PLUS (Pmode,
10116 gen_rtx_PLUS (Pmode, XEXP (x, 0),
10117 XEXP (XEXP (x, 1), 0)),
10118 XEXP (XEXP (x, 1), 1));
10122 (plus (plus (mult (reg) (const)) (plus (reg) (const))) const)
10123 into (plus (plus (mult (reg) (const)) (reg)) (const)). */
10124 else if (GET_CODE (x) == PLUS && GET_CODE (XEXP (x, 0)) == PLUS
10125 && GET_CODE (XEXP (XEXP (x, 0), 0)) == MULT
10126 && GET_CODE (XEXP (XEXP (x, 0), 1)) == PLUS
10127 && CONSTANT_P (XEXP (x, 1)))
10130 rtx other = NULL_RTX;
10132 if (CONST_INT_P (XEXP (x, 1)))
10134 constant = XEXP (x, 1);
10135 other = XEXP (XEXP (XEXP (x, 0), 1), 1);
10137 else if (CONST_INT_P (XEXP (XEXP (XEXP (x, 0), 1), 1)))
10139 constant = XEXP (XEXP (XEXP (x, 0), 1), 1);
10140 other = XEXP (x, 1);
10148 x = gen_rtx_PLUS (Pmode,
10149 gen_rtx_PLUS (Pmode, XEXP (XEXP (x, 0), 0),
10150 XEXP (XEXP (XEXP (x, 0), 1), 0)),
10151 plus_constant (other, INTVAL (constant)));
10155 if (changed && ix86_legitimate_address_p (mode, x, FALSE))
10158 if (GET_CODE (XEXP (x, 0)) == MULT)
10161 XEXP (x, 0) = force_operand (XEXP (x, 0), 0);
10164 if (GET_CODE (XEXP (x, 1)) == MULT)
10167 XEXP (x, 1) = force_operand (XEXP (x, 1), 0);
10171 && REG_P (XEXP (x, 1))
10172 && REG_P (XEXP (x, 0)))
10175 if (flag_pic && SYMBOLIC_CONST (XEXP (x, 1)))
10178 x = legitimize_pic_address (x, 0);
10181 if (changed && ix86_legitimate_address_p (mode, x, FALSE))
10184 if (REG_P (XEXP (x, 0)))
10186 rtx temp = gen_reg_rtx (Pmode);
10187 rtx val = force_operand (XEXP (x, 1), temp);
10189 emit_move_insn (temp, val);
10191 XEXP (x, 1) = temp;
10195 else if (REG_P (XEXP (x, 1)))
10197 rtx temp = gen_reg_rtx (Pmode);
10198 rtx val = force_operand (XEXP (x, 0), temp);
10200 emit_move_insn (temp, val);
10202 XEXP (x, 0) = temp;
10210 /* Print an integer constant expression in assembler syntax. Addition
10211 and subtraction are the only arithmetic that may appear in these
10212 expressions. FILE is the stdio stream to write to, X is the rtx, and
10213 CODE is the operand print code from the output string. */
10216 output_pic_addr_const (FILE *file, rtx x, int code)
10220 switch (GET_CODE (x))
10223 gcc_assert (flag_pic);
10228 if (! TARGET_MACHO || TARGET_64BIT)
10229 output_addr_const (file, x);
10232 const char *name = XSTR (x, 0);
10234 /* Mark the decl as referenced so that cgraph will
10235 output the function. */
10236 if (SYMBOL_REF_DECL (x))
10237 mark_decl_referenced (SYMBOL_REF_DECL (x));
10240 if (MACHOPIC_INDIRECT
10241 && machopic_classify_symbol (x) == MACHOPIC_UNDEFINED_FUNCTION)
10242 name = machopic_indirection_name (x, /*stub_p=*/true);
10244 assemble_name (file, name);
10246 if (!TARGET_MACHO && !(TARGET_64BIT && DEFAULT_ABI == MS_ABI)
10247 && code == 'P' && ! SYMBOL_REF_LOCAL_P (x))
10248 fputs ("@PLT", file);
10255 ASM_GENERATE_INTERNAL_LABEL (buf, "L", CODE_LABEL_NUMBER (x));
10256 assemble_name (asm_out_file, buf);
10260 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (x));
10264 /* This used to output parentheses around the expression,
10265 but that does not work on the 386 (either ATT or BSD assembler). */
10266 output_pic_addr_const (file, XEXP (x, 0), code);
10270 if (GET_MODE (x) == VOIDmode)
10272 /* We can use %d if the number is <32 bits and positive. */
10273 if (CONST_DOUBLE_HIGH (x) || CONST_DOUBLE_LOW (x) < 0)
10274 fprintf (file, "0x%lx%08lx",
10275 (unsigned long) CONST_DOUBLE_HIGH (x),
10276 (unsigned long) CONST_DOUBLE_LOW (x));
10278 fprintf (file, HOST_WIDE_INT_PRINT_DEC, CONST_DOUBLE_LOW (x));
10281 /* We can't handle floating point constants;
10282 PRINT_OPERAND must handle them. */
10283 output_operand_lossage ("floating constant misused");
10287 /* Some assemblers need integer constants to appear first. */
10288 if (CONST_INT_P (XEXP (x, 0)))
10290 output_pic_addr_const (file, XEXP (x, 0), code);
10292 output_pic_addr_const (file, XEXP (x, 1), code);
10296 gcc_assert (CONST_INT_P (XEXP (x, 1)));
10297 output_pic_addr_const (file, XEXP (x, 1), code);
10299 output_pic_addr_const (file, XEXP (x, 0), code);
10305 putc (ASSEMBLER_DIALECT == ASM_INTEL ? '(' : '[', file);
10306 output_pic_addr_const (file, XEXP (x, 0), code);
10308 output_pic_addr_const (file, XEXP (x, 1), code);
10310 putc (ASSEMBLER_DIALECT == ASM_INTEL ? ')' : ']', file);
10314 gcc_assert (XVECLEN (x, 0) == 1);
10315 output_pic_addr_const (file, XVECEXP (x, 0, 0), code);
10316 switch (XINT (x, 1))
10319 fputs ("@GOT", file);
10321 case UNSPEC_GOTOFF:
10322 fputs ("@GOTOFF", file);
10324 case UNSPEC_PLTOFF:
10325 fputs ("@PLTOFF", file);
10327 case UNSPEC_GOTPCREL:
10328 fputs (ASSEMBLER_DIALECT == ASM_ATT ?
10329 "@GOTPCREL(%rip)" : "@GOTPCREL[rip]", file);
10331 case UNSPEC_GOTTPOFF:
10332 /* FIXME: This might be @TPOFF in Sun ld too. */
10333 fputs ("@GOTTPOFF", file);
10336 fputs ("@TPOFF", file);
10338 case UNSPEC_NTPOFF:
10340 fputs ("@TPOFF", file);
10342 fputs ("@NTPOFF", file);
10344 case UNSPEC_DTPOFF:
10345 fputs ("@DTPOFF", file);
10347 case UNSPEC_GOTNTPOFF:
10349 fputs (ASSEMBLER_DIALECT == ASM_ATT ?
10350 "@GOTTPOFF(%rip)": "@GOTTPOFF[rip]", file);
10352 fputs ("@GOTNTPOFF", file);
10354 case UNSPEC_INDNTPOFF:
10355 fputs ("@INDNTPOFF", file);
10358 case UNSPEC_MACHOPIC_OFFSET:
10360 machopic_output_function_base_name (file);
10364 output_operand_lossage ("invalid UNSPEC as operand");
10370 output_operand_lossage ("invalid expression as operand");
10374 /* This is called from dwarf2out.c via TARGET_ASM_OUTPUT_DWARF_DTPREL.
10375 We need to emit DTP-relative relocations. */
10377 static void ATTRIBUTE_UNUSED
10378 i386_output_dwarf_dtprel (FILE *file, int size, rtx x)
10380 fputs (ASM_LONG, file);
10381 output_addr_const (file, x);
10382 fputs ("@DTPOFF", file);
10388 fputs (", 0", file);
10391 gcc_unreachable ();
10395 /* Return true if X is a representation of the PIC register. This copes
10396 with calls from ix86_find_base_term, where the register might have
10397 been replaced by a cselib value. */
10400 ix86_pic_register_p (rtx x)
10402 if (GET_CODE (x) == VALUE)
10403 return (pic_offset_table_rtx
10404 && rtx_equal_for_cselib_p (x, pic_offset_table_rtx));
10406 return REG_P (x) && REGNO (x) == PIC_OFFSET_TABLE_REGNUM;
10409 /* In the name of slightly smaller debug output, and to cater to
10410 general assembler lossage, recognize PIC+GOTOFF and turn it back
10411 into a direct symbol reference.
10413 On Darwin, this is necessary to avoid a crash, because Darwin
10414 has a different PIC label for each routine but the DWARF debugging
10415 information is not associated with any particular routine, so it's
10416 necessary to remove references to the PIC label from RTL stored by
10417 the DWARF output code. */
10420 ix86_delegitimize_address (rtx orig_x)
10423 /* reg_addend is NULL or a multiple of some register. */
10424 rtx reg_addend = NULL_RTX;
10425 /* const_addend is NULL or a const_int. */
10426 rtx const_addend = NULL_RTX;
10427 /* This is the result, or NULL. */
10428 rtx result = NULL_RTX;
10435 if (GET_CODE (x) != CONST
10436 || GET_CODE (XEXP (x, 0)) != UNSPEC
10437 || XINT (XEXP (x, 0), 1) != UNSPEC_GOTPCREL
10438 || !MEM_P (orig_x))
10440 return XVECEXP (XEXP (x, 0), 0, 0);
10443 if (GET_CODE (x) != PLUS
10444 || GET_CODE (XEXP (x, 1)) != CONST)
10447 if (ix86_pic_register_p (XEXP (x, 0)))
10448 /* %ebx + GOT/GOTOFF */
10450 else if (GET_CODE (XEXP (x, 0)) == PLUS)
10452 /* %ebx + %reg * scale + GOT/GOTOFF */
10453 reg_addend = XEXP (x, 0);
10454 if (ix86_pic_register_p (XEXP (reg_addend, 0)))
10455 reg_addend = XEXP (reg_addend, 1);
10456 else if (ix86_pic_register_p (XEXP (reg_addend, 1)))
10457 reg_addend = XEXP (reg_addend, 0);
10460 if (!REG_P (reg_addend)
10461 && GET_CODE (reg_addend) != MULT
10462 && GET_CODE (reg_addend) != ASHIFT)
10468 x = XEXP (XEXP (x, 1), 0);
10469 if (GET_CODE (x) == PLUS
10470 && CONST_INT_P (XEXP (x, 1)))
10472 const_addend = XEXP (x, 1);
10476 if (GET_CODE (x) == UNSPEC
10477 && ((XINT (x, 1) == UNSPEC_GOT && MEM_P (orig_x))
10478 || (XINT (x, 1) == UNSPEC_GOTOFF && !MEM_P (orig_x))))
10479 result = XVECEXP (x, 0, 0);
10481 if (TARGET_MACHO && darwin_local_data_pic (x)
10482 && !MEM_P (orig_x))
10483 result = XVECEXP (x, 0, 0);
10489 result = gen_rtx_CONST (Pmode, gen_rtx_PLUS (Pmode, result, const_addend));
10491 result = gen_rtx_PLUS (Pmode, reg_addend, result);
10495 /* If X is a machine specific address (i.e. a symbol or label being
10496 referenced as a displacement from the GOT implemented using an
10497 UNSPEC), then return the base term. Otherwise return X. */
10500 ix86_find_base_term (rtx x)
10506 if (GET_CODE (x) != CONST)
10508 term = XEXP (x, 0);
10509 if (GET_CODE (term) == PLUS
10510 && (CONST_INT_P (XEXP (term, 1))
10511 || GET_CODE (XEXP (term, 1)) == CONST_DOUBLE))
10512 term = XEXP (term, 0);
10513 if (GET_CODE (term) != UNSPEC
10514 || XINT (term, 1) != UNSPEC_GOTPCREL)
10517 return XVECEXP (term, 0, 0);
10520 return ix86_delegitimize_address (x);
10524 put_condition_code (enum rtx_code code, enum machine_mode mode, int reverse,
10525 int fp, FILE *file)
10527 const char *suffix;
10529 if (mode == CCFPmode || mode == CCFPUmode)
10531 enum rtx_code second_code, bypass_code;
10532 ix86_fp_comparison_codes (code, &bypass_code, &code, &second_code);
10533 gcc_assert (bypass_code == UNKNOWN && second_code == UNKNOWN);
10534 code = ix86_fp_compare_code_to_integer (code);
10538 code = reverse_condition (code);
10589 gcc_assert (mode == CCmode || mode == CCNOmode || mode == CCGCmode);
10593 /* ??? Use "nbe" instead of "a" for fcmov lossage on some assemblers.
10594 Those same assemblers have the same but opposite lossage on cmov. */
10595 if (mode == CCmode)
10596 suffix = fp ? "nbe" : "a";
10597 else if (mode == CCCmode)
10600 gcc_unreachable ();
10616 gcc_unreachable ();
10620 gcc_assert (mode == CCmode || mode == CCCmode);
10637 gcc_unreachable ();
10641 /* ??? As above. */
10642 gcc_assert (mode == CCmode || mode == CCCmode);
10643 suffix = fp ? "nb" : "ae";
10646 gcc_assert (mode == CCmode || mode == CCGCmode || mode == CCNOmode);
10650 /* ??? As above. */
10651 if (mode == CCmode)
10653 else if (mode == CCCmode)
10654 suffix = fp ? "nb" : "ae";
10656 gcc_unreachable ();
10659 suffix = fp ? "u" : "p";
10662 suffix = fp ? "nu" : "np";
10665 gcc_unreachable ();
10667 fputs (suffix, file);
10670 /* Print the name of register X to FILE based on its machine mode and number.
10671 If CODE is 'w', pretend the mode is HImode.
10672 If CODE is 'b', pretend the mode is QImode.
10673 If CODE is 'k', pretend the mode is SImode.
10674 If CODE is 'q', pretend the mode is DImode.
10675 If CODE is 'x', pretend the mode is V4SFmode.
10676 If CODE is 't', pretend the mode is V8SFmode.
10677 If CODE is 'h', pretend the reg is the 'high' byte register.
10678 If CODE is 'y', print "st(0)" instead of "st", if the reg is stack op.
10679 If CODE is 'd', duplicate the operand for AVX instruction.
10683 print_reg (rtx x, int code, FILE *file)
10686 bool duplicated = code == 'd' && TARGET_AVX;
10688 gcc_assert (x == pc_rtx
10689 || (REGNO (x) != ARG_POINTER_REGNUM
10690 && REGNO (x) != FRAME_POINTER_REGNUM
10691 && REGNO (x) != FLAGS_REG
10692 && REGNO (x) != FPSR_REG
10693 && REGNO (x) != FPCR_REG));
10695 if (ASSEMBLER_DIALECT == ASM_ATT)
10700 gcc_assert (TARGET_64BIT);
10701 fputs ("rip", file);
10705 if (code == 'w' || MMX_REG_P (x))
10707 else if (code == 'b')
10709 else if (code == 'k')
10711 else if (code == 'q')
10713 else if (code == 'y')
10715 else if (code == 'h')
10717 else if (code == 'x')
10719 else if (code == 't')
10722 code = GET_MODE_SIZE (GET_MODE (x));
10724 /* Irritatingly, AMD extended registers use different naming convention
10725 from the normal registers. */
10726 if (REX_INT_REG_P (x))
10728 gcc_assert (TARGET_64BIT);
10732 error ("extended registers have no high halves");
10735 fprintf (file, "r%ib", REGNO (x) - FIRST_REX_INT_REG + 8);
10738 fprintf (file, "r%iw", REGNO (x) - FIRST_REX_INT_REG + 8);
10741 fprintf (file, "r%id", REGNO (x) - FIRST_REX_INT_REG + 8);
10744 fprintf (file, "r%i", REGNO (x) - FIRST_REX_INT_REG + 8);
10747 error ("unsupported operand size for extended register");
10757 if (STACK_TOP_P (x))
10766 if (! ANY_FP_REG_P (x))
10767 putc (code == 8 && TARGET_64BIT ? 'r' : 'e', file);
10772 reg = hi_reg_name[REGNO (x)];
10775 if (REGNO (x) >= ARRAY_SIZE (qi_reg_name))
10777 reg = qi_reg_name[REGNO (x)];
10780 if (REGNO (x) >= ARRAY_SIZE (qi_high_reg_name))
10782 reg = qi_high_reg_name[REGNO (x)];
10787 gcc_assert (!duplicated);
10789 fputs (hi_reg_name[REGNO (x)] + 1, file);
10794 gcc_unreachable ();
10800 if (ASSEMBLER_DIALECT == ASM_ATT)
10801 fprintf (file, ", %%%s", reg);
10803 fprintf (file, ", %s", reg);
10807 /* Locate some local-dynamic symbol still in use by this function
10808 so that we can print its name in some tls_local_dynamic_base
10812 get_some_local_dynamic_name_1 (rtx *px, void *data ATTRIBUTE_UNUSED)
10816 if (GET_CODE (x) == SYMBOL_REF
10817 && SYMBOL_REF_TLS_MODEL (x) == TLS_MODEL_LOCAL_DYNAMIC)
10819 cfun->machine->some_ld_name = XSTR (x, 0);
10826 static const char *
10827 get_some_local_dynamic_name (void)
10831 if (cfun->machine->some_ld_name)
10832 return cfun->machine->some_ld_name;
10834 for (insn = get_insns (); insn ; insn = NEXT_INSN (insn))
10836 && for_each_rtx (&PATTERN (insn), get_some_local_dynamic_name_1, 0))
10837 return cfun->machine->some_ld_name;
10839 gcc_unreachable ();
10842 /* Meaning of CODE:
10843 L,W,B,Q,S,T -- print the opcode suffix for specified size of operand.
10844 C -- print opcode suffix for set/cmov insn.
10845 c -- like C, but print reversed condition
10846 E,e -- likewise, but for compare-and-branch fused insn.
10847 F,f -- likewise, but for floating-point.
10848 O -- if HAVE_AS_IX86_CMOV_SUN_SYNTAX, expand to "w.", "l." or "q.",
10850 R -- print the prefix for register names.
10851 z -- print the opcode suffix for the size of the current operand.
10852 Z -- likewise, with special suffixes for x87 instructions.
10853 * -- print a star (in certain assembler syntax)
10854 A -- print an absolute memory reference.
10855 w -- print the operand as if it's a "word" (HImode) even if it isn't.
10856 s -- print a shift double count, followed by the assemblers argument
10858 b -- print the QImode name of the register for the indicated operand.
10859 %b0 would print %al if operands[0] is reg 0.
10860 w -- likewise, print the HImode name of the register.
10861 k -- likewise, print the SImode name of the register.
10862 q -- likewise, print the DImode name of the register.
10863 x -- likewise, print the V4SFmode name of the register.
10864 t -- likewise, print the V8SFmode name of the register.
10865 h -- print the QImode name for a "high" register, either ah, bh, ch or dh.
10866 y -- print "st(0)" instead of "st" as a register.
10867 d -- print duplicated register operand for AVX instruction.
10868 D -- print condition for SSE cmp instruction.
10869 P -- if PIC, print an @PLT suffix.
10870 X -- don't print any sort of PIC '@' suffix for a symbol.
10871 & -- print some in-use local-dynamic symbol name.
10872 H -- print a memory address offset by 8; used for sse high-parts
10873 Y -- print condition for SSE5 com* instruction.
10874 + -- print a branch hint as 'cs' or 'ds' prefix
10875 ; -- print a semicolon (after prefixes due to bug in older gas).
10879 print_operand (FILE *file, rtx x, int code)
10886 if (ASSEMBLER_DIALECT == ASM_ATT)
10891 assemble_name (file, get_some_local_dynamic_name ());
10895 switch (ASSEMBLER_DIALECT)
10902 /* Intel syntax. For absolute addresses, registers should not
10903 be surrounded by braces. */
10907 PRINT_OPERAND (file, x, 0);
10914 gcc_unreachable ();
10917 PRINT_OPERAND (file, x, 0);
10922 if (ASSEMBLER_DIALECT == ASM_ATT)
10927 if (ASSEMBLER_DIALECT == ASM_ATT)
10932 if (ASSEMBLER_DIALECT == ASM_ATT)
10937 if (ASSEMBLER_DIALECT == ASM_ATT)
10942 if (ASSEMBLER_DIALECT == ASM_ATT)
10947 if (ASSEMBLER_DIALECT == ASM_ATT)
10952 if (GET_MODE_CLASS (GET_MODE (x)) == MODE_INT)
10954 /* Opcodes don't get size suffixes if using Intel opcodes. */
10955 if (ASSEMBLER_DIALECT == ASM_INTEL)
10958 switch (GET_MODE_SIZE (GET_MODE (x)))
10977 output_operand_lossage
10978 ("invalid operand size for operand code '%c'", code);
10983 if (GET_MODE_CLASS (GET_MODE (x)) == MODE_FLOAT)
10985 (0, "non-integer operand used with operand code '%c'", code);
10989 /* 387 opcodes don't get size suffixes if using Intel opcodes. */
10990 if (ASSEMBLER_DIALECT == ASM_INTEL)
10993 if (GET_MODE_CLASS (GET_MODE (x)) == MODE_INT)
10995 switch (GET_MODE_SIZE (GET_MODE (x)))
10998 #ifdef HAVE_AS_IX86_FILDS
11008 #ifdef HAVE_AS_IX86_FILDQ
11011 fputs ("ll", file);
11019 else if (GET_MODE_CLASS (GET_MODE (x)) == MODE_FLOAT)
11021 /* 387 opcodes don't get size suffixes
11022 if the operands are registers. */
11023 if (STACK_REG_P (x))
11026 switch (GET_MODE_SIZE (GET_MODE (x)))
11047 output_operand_lossage
11048 ("invalid operand type used with operand code '%c'", code);
11052 output_operand_lossage
11053 ("invalid operand size for operand code '%c'", code);
11070 if (CONST_INT_P (x) || ! SHIFT_DOUBLE_OMITS_COUNT)
11072 PRINT_OPERAND (file, x, 0);
11073 fputs (", ", file);
11078 /* Little bit of braindamage here. The SSE compare instructions
11079 does use completely different names for the comparisons that the
11080 fp conditional moves. */
11083 switch (GET_CODE (x))
11086 fputs ("eq", file);
11089 fputs ("eq_us", file);
11092 fputs ("lt", file);
11095 fputs ("nge", file);
11098 fputs ("le", file);
11101 fputs ("ngt", file);
11104 fputs ("unord", file);
11107 fputs ("neq", file);
11110 fputs ("neq_oq", file);
11113 fputs ("ge", file);
11116 fputs ("nlt", file);
11119 fputs ("gt", file);
11122 fputs ("nle", file);
11125 fputs ("ord", file);
11128 output_operand_lossage ("operand is not a condition code, invalid operand code 'D'");
11134 switch (GET_CODE (x))
11138 fputs ("eq", file);
11142 fputs ("lt", file);
11146 fputs ("le", file);
11149 fputs ("unord", file);
11153 fputs ("neq", file);
11157 fputs ("nlt", file);
11161 fputs ("nle", file);
11164 fputs ("ord", file);
11167 output_operand_lossage ("operand is not a condition code, invalid operand code 'D'");
11173 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
11174 if (ASSEMBLER_DIALECT == ASM_ATT)
11176 switch (GET_MODE (x))
11178 case HImode: putc ('w', file); break;
11180 case SFmode: putc ('l', file); break;
11182 case DFmode: putc ('q', file); break;
11183 default: gcc_unreachable ();
11190 if (!COMPARISON_P (x))
11192 output_operand_lossage ("operand is neither a constant nor a "
11193 "condition code, invalid operand code "
11197 put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)), 0, 0, file);
11200 if (!COMPARISON_P (x))
11202 output_operand_lossage ("operand is neither a constant nor a "
11203 "condition code, invalid operand code "
11207 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
11208 if (ASSEMBLER_DIALECT == ASM_ATT)
11211 put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)), 0, 1, file);
11214 /* Like above, but reverse condition */
11216 /* Check to see if argument to %c is really a constant
11217 and not a condition code which needs to be reversed. */
11218 if (!COMPARISON_P (x))
11220 output_operand_lossage ("operand is neither a constant nor a "
11221 "condition code, invalid operand "
11225 put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)), 1, 0, file);
11228 if (!COMPARISON_P (x))
11230 output_operand_lossage ("operand is neither a constant nor a "
11231 "condition code, invalid operand "
11235 #ifdef HAVE_AS_IX86_CMOV_SUN_SYNTAX
11236 if (ASSEMBLER_DIALECT == ASM_ATT)
11239 put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)), 1, 1, file);
11243 put_condition_code (GET_CODE (x), CCmode, 0, 0, file);
11247 put_condition_code (GET_CODE (x), CCmode, 1, 0, file);
11251 /* It doesn't actually matter what mode we use here, as we're
11252 only going to use this for printing. */
11253 x = adjust_address_nv (x, DImode, 8);
11261 || optimize_function_for_size_p (cfun) || !TARGET_BRANCH_PREDICTION_HINTS)
11264 x = find_reg_note (current_output_insn, REG_BR_PROB, 0);
11267 int pred_val = INTVAL (XEXP (x, 0));
11269 if (pred_val < REG_BR_PROB_BASE * 45 / 100
11270 || pred_val > REG_BR_PROB_BASE * 55 / 100)
11272 int taken = pred_val > REG_BR_PROB_BASE / 2;
11273 int cputaken = final_forward_branch_p (current_output_insn) == 0;
11275 /* Emit hints only in the case default branch prediction
11276 heuristics would fail. */
11277 if (taken != cputaken)
11279 /* We use 3e (DS) prefix for taken branches and
11280 2e (CS) prefix for not taken branches. */
11282 fputs ("ds ; ", file);
11284 fputs ("cs ; ", file);
11292 switch (GET_CODE (x))
11295 fputs ("neq", file);
11298 fputs ("eq", file);
11302 fputs (INTEGRAL_MODE_P (GET_MODE (x)) ? "ge" : "unlt", file);
11306 fputs (INTEGRAL_MODE_P (GET_MODE (x)) ? "gt" : "unle", file);
11310 fputs ("le", file);
11314 fputs ("lt", file);
11317 fputs ("unord", file);
11320 fputs ("ord", file);
11323 fputs ("ueq", file);
11326 fputs ("nlt", file);
11329 fputs ("nle", file);
11332 fputs ("ule", file);
11335 fputs ("ult", file);
11338 fputs ("une", file);
11341 output_operand_lossage ("operand is not a condition code, invalid operand code 'D'");
11348 fputs (" ; ", file);
11355 output_operand_lossage ("invalid operand code '%c'", code);
11360 print_reg (x, code, file);
11362 else if (MEM_P (x))
11364 /* No `byte ptr' prefix for call instructions or BLKmode operands. */
11365 if (ASSEMBLER_DIALECT == ASM_INTEL && code != 'X' && code != 'P'
11366 && GET_MODE (x) != BLKmode)
11369 switch (GET_MODE_SIZE (GET_MODE (x)))
11371 case 1: size = "BYTE"; break;
11372 case 2: size = "WORD"; break;
11373 case 4: size = "DWORD"; break;
11374 case 8: size = "QWORD"; break;
11375 case 12: size = "XWORD"; break;
11377 if (GET_MODE (x) == XFmode)
11383 gcc_unreachable ();
11386 /* Check for explicit size override (codes 'b', 'w' and 'k') */
11389 else if (code == 'w')
11391 else if (code == 'k')
11394 fputs (size, file);
11395 fputs (" PTR ", file);
11399 /* Avoid (%rip) for call operands. */
11400 if (CONSTANT_ADDRESS_P (x) && code == 'P'
11401 && !CONST_INT_P (x))
11402 output_addr_const (file, x);
11403 else if (this_is_asm_operands && ! address_operand (x, VOIDmode))
11404 output_operand_lossage ("invalid constraints for operand");
11406 output_address (x);
11409 else if (GET_CODE (x) == CONST_DOUBLE && GET_MODE (x) == SFmode)
11414 REAL_VALUE_FROM_CONST_DOUBLE (r, x);
11415 REAL_VALUE_TO_TARGET_SINGLE (r, l);
11417 if (ASSEMBLER_DIALECT == ASM_ATT)
11419 fprintf (file, "0x%08lx", (long unsigned int) l);
11422 /* These float cases don't actually occur as immediate operands. */
11423 else if (GET_CODE (x) == CONST_DOUBLE && GET_MODE (x) == DFmode)
11427 real_to_decimal (dstr, CONST_DOUBLE_REAL_VALUE (x), sizeof (dstr), 0, 1);
11428 fprintf (file, "%s", dstr);
11431 else if (GET_CODE (x) == CONST_DOUBLE
11432 && GET_MODE (x) == XFmode)
11436 real_to_decimal (dstr, CONST_DOUBLE_REAL_VALUE (x), sizeof (dstr), 0, 1);
11437 fprintf (file, "%s", dstr);
11442 /* We have patterns that allow zero sets of memory, for instance.
11443 In 64-bit mode, we should probably support all 8-byte vectors,
11444 since we can in fact encode that into an immediate. */
11445 if (GET_CODE (x) == CONST_VECTOR)
11447 gcc_assert (x == CONST0_RTX (GET_MODE (x)));
11453 if (CONST_INT_P (x) || GET_CODE (x) == CONST_DOUBLE)
11455 if (ASSEMBLER_DIALECT == ASM_ATT)
11458 else if (GET_CODE (x) == CONST || GET_CODE (x) == SYMBOL_REF
11459 || GET_CODE (x) == LABEL_REF)
11461 if (ASSEMBLER_DIALECT == ASM_ATT)
11464 fputs ("OFFSET FLAT:", file);
11467 if (CONST_INT_P (x))
11468 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (x));
11470 output_pic_addr_const (file, x, code);
11472 output_addr_const (file, x);
11476 /* Print a memory operand whose address is ADDR. */
11479 print_operand_address (FILE *file, rtx addr)
11481 struct ix86_address parts;
11482 rtx base, index, disp;
11484 int ok = ix86_decompose_address (addr, &parts);
11489 index = parts.index;
11491 scale = parts.scale;
11499 if (ASSEMBLER_DIALECT == ASM_ATT)
11501 fputs ((parts.seg == SEG_FS ? "fs:" : "gs:"), file);
11504 gcc_unreachable ();
11507 /* Use one byte shorter RIP relative addressing for 64bit mode. */
11508 if (TARGET_64BIT && !base && !index)
11512 if (GET_CODE (disp) == CONST
11513 && GET_CODE (XEXP (disp, 0)) == PLUS
11514 && CONST_INT_P (XEXP (XEXP (disp, 0), 1)))
11515 symbol = XEXP (XEXP (disp, 0), 0);
11517 if (GET_CODE (symbol) == LABEL_REF
11518 || (GET_CODE (symbol) == SYMBOL_REF
11519 && SYMBOL_REF_TLS_MODEL (symbol) == 0))
11522 if (!base && !index)
11524 /* Displacement only requires special attention. */
11526 if (CONST_INT_P (disp))
11528 if (ASSEMBLER_DIALECT == ASM_INTEL && parts.seg == SEG_DEFAULT)
11529 fputs ("ds:", file);
11530 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (disp));
11533 output_pic_addr_const (file, disp, 0);
11535 output_addr_const (file, disp);
11539 if (ASSEMBLER_DIALECT == ASM_ATT)
11544 output_pic_addr_const (file, disp, 0);
11545 else if (GET_CODE (disp) == LABEL_REF)
11546 output_asm_label (disp);
11548 output_addr_const (file, disp);
11553 print_reg (base, 0, file);
11557 print_reg (index, 0, file);
11559 fprintf (file, ",%d", scale);
11565 rtx offset = NULL_RTX;
11569 /* Pull out the offset of a symbol; print any symbol itself. */
11570 if (GET_CODE (disp) == CONST
11571 && GET_CODE (XEXP (disp, 0)) == PLUS
11572 && CONST_INT_P (XEXP (XEXP (disp, 0), 1)))
11574 offset = XEXP (XEXP (disp, 0), 1);
11575 disp = gen_rtx_CONST (VOIDmode,
11576 XEXP (XEXP (disp, 0), 0));
11580 output_pic_addr_const (file, disp, 0);
11581 else if (GET_CODE (disp) == LABEL_REF)
11582 output_asm_label (disp);
11583 else if (CONST_INT_P (disp))
11586 output_addr_const (file, disp);
11592 print_reg (base, 0, file);
11595 if (INTVAL (offset) >= 0)
11597 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (offset));
11601 fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (offset));
11608 print_reg (index, 0, file);
11610 fprintf (file, "*%d", scale);
11618 output_addr_const_extra (FILE *file, rtx x)
11622 if (GET_CODE (x) != UNSPEC)
11625 op = XVECEXP (x, 0, 0);
11626 switch (XINT (x, 1))
11628 case UNSPEC_GOTTPOFF:
11629 output_addr_const (file, op);
11630 /* FIXME: This might be @TPOFF in Sun ld. */
11631 fputs ("@GOTTPOFF", file);
11634 output_addr_const (file, op);
11635 fputs ("@TPOFF", file);
11637 case UNSPEC_NTPOFF:
11638 output_addr_const (file, op);
11640 fputs ("@TPOFF", file);
11642 fputs ("@NTPOFF", file);
11644 case UNSPEC_DTPOFF:
11645 output_addr_const (file, op);
11646 fputs ("@DTPOFF", file);
11648 case UNSPEC_GOTNTPOFF:
11649 output_addr_const (file, op);
11651 fputs (ASSEMBLER_DIALECT == ASM_ATT ?
11652 "@GOTTPOFF(%rip)" : "@GOTTPOFF[rip]", file);
11654 fputs ("@GOTNTPOFF", file);
11656 case UNSPEC_INDNTPOFF:
11657 output_addr_const (file, op);
11658 fputs ("@INDNTPOFF", file);
11661 case UNSPEC_MACHOPIC_OFFSET:
11662 output_addr_const (file, op);
11664 machopic_output_function_base_name (file);
11675 /* Split one or more DImode RTL references into pairs of SImode
11676 references. The RTL can be REG, offsettable MEM, integer constant, or
11677 CONST_DOUBLE. "operands" is a pointer to an array of DImode RTL to
11678 split and "num" is its length. lo_half and hi_half are output arrays
11679 that parallel "operands". */
11682 split_di (rtx operands[], int num, rtx lo_half[], rtx hi_half[])
11686 rtx op = operands[num];
11688 /* simplify_subreg refuse to split volatile memory addresses,
11689 but we still have to handle it. */
11692 lo_half[num] = adjust_address (op, SImode, 0);
11693 hi_half[num] = adjust_address (op, SImode, 4);
11697 lo_half[num] = simplify_gen_subreg (SImode, op,
11698 GET_MODE (op) == VOIDmode
11699 ? DImode : GET_MODE (op), 0);
11700 hi_half[num] = simplify_gen_subreg (SImode, op,
11701 GET_MODE (op) == VOIDmode
11702 ? DImode : GET_MODE (op), 4);
11706 /* Split one or more TImode RTL references into pairs of DImode
11707 references. The RTL can be REG, offsettable MEM, integer constant, or
11708 CONST_DOUBLE. "operands" is a pointer to an array of DImode RTL to
11709 split and "num" is its length. lo_half and hi_half are output arrays
11710 that parallel "operands". */
11713 split_ti (rtx operands[], int num, rtx lo_half[], rtx hi_half[])
11717 rtx op = operands[num];
11719 /* simplify_subreg refuse to split volatile memory addresses, but we
11720 still have to handle it. */
11723 lo_half[num] = adjust_address (op, DImode, 0);
11724 hi_half[num] = adjust_address (op, DImode, 8);
11728 lo_half[num] = simplify_gen_subreg (DImode, op, TImode, 0);
11729 hi_half[num] = simplify_gen_subreg (DImode, op, TImode, 8);
11734 /* Output code to perform a 387 binary operation in INSN, one of PLUS,
11735 MINUS, MULT or DIV. OPERANDS are the insn operands, where operands[3]
11736 is the expression of the binary operation. The output may either be
11737 emitted here, or returned to the caller, like all output_* functions.
11739 There is no guarantee that the operands are the same mode, as they
11740 might be within FLOAT or FLOAT_EXTEND expressions. */
11742 #ifndef SYSV386_COMPAT
11743 /* Set to 1 for compatibility with brain-damaged assemblers. No-one
11744 wants to fix the assemblers because that causes incompatibility
11745 with gcc. No-one wants to fix gcc because that causes
11746 incompatibility with assemblers... You can use the option of
11747 -DSYSV386_COMPAT=0 if you recompile both gcc and gas this way. */
11748 #define SYSV386_COMPAT 1
11752 output_387_binary_op (rtx insn, rtx *operands)
11754 static char buf[40];
11757 int is_sse = SSE_REG_P (operands[0]) || SSE_REG_P (operands[1]) || SSE_REG_P (operands[2]);
11759 #ifdef ENABLE_CHECKING
11760 /* Even if we do not want to check the inputs, this documents input
11761 constraints. Which helps in understanding the following code. */
11762 if (STACK_REG_P (operands[0])
11763 && ((REG_P (operands[1])
11764 && REGNO (operands[0]) == REGNO (operands[1])
11765 && (STACK_REG_P (operands[2]) || MEM_P (operands[2])))
11766 || (REG_P (operands[2])
11767 && REGNO (operands[0]) == REGNO (operands[2])
11768 && (STACK_REG_P (operands[1]) || MEM_P (operands[1]))))
11769 && (STACK_TOP_P (operands[1]) || STACK_TOP_P (operands[2])))
11772 gcc_assert (is_sse);
11775 switch (GET_CODE (operands[3]))
11778 if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
11779 || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
11787 if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
11788 || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
11796 if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
11797 || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
11805 if (GET_MODE_CLASS (GET_MODE (operands[1])) == MODE_INT
11806 || GET_MODE_CLASS (GET_MODE (operands[2])) == MODE_INT)
11814 gcc_unreachable ();
11821 strcpy (buf, ssep);
11822 if (GET_MODE (operands[0]) == SFmode)
11823 strcat (buf, "ss\t{%2, %1, %0|%0, %1, %2}");
11825 strcat (buf, "sd\t{%2, %1, %0|%0, %1, %2}");
11829 strcpy (buf, ssep + 1);
11830 if (GET_MODE (operands[0]) == SFmode)
11831 strcat (buf, "ss\t{%2, %0|%0, %2}");
11833 strcat (buf, "sd\t{%2, %0|%0, %2}");
11839 switch (GET_CODE (operands[3]))
11843 if (REG_P (operands[2]) && REGNO (operands[0]) == REGNO (operands[2]))
11845 rtx temp = operands[2];
11846 operands[2] = operands[1];
11847 operands[1] = temp;
11850 /* know operands[0] == operands[1]. */
11852 if (MEM_P (operands[2]))
11858 if (find_regno_note (insn, REG_DEAD, REGNO (operands[2])))
11860 if (STACK_TOP_P (operands[0]))
11861 /* How is it that we are storing to a dead operand[2]?
11862 Well, presumably operands[1] is dead too. We can't
11863 store the result to st(0) as st(0) gets popped on this
11864 instruction. Instead store to operands[2] (which I
11865 think has to be st(1)). st(1) will be popped later.
11866 gcc <= 2.8.1 didn't have this check and generated
11867 assembly code that the Unixware assembler rejected. */
11868 p = "p\t{%0, %2|%2, %0}"; /* st(1) = st(0) op st(1); pop */
11870 p = "p\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0); pop */
11874 if (STACK_TOP_P (operands[0]))
11875 p = "\t{%y2, %0|%0, %y2}"; /* st(0) = st(0) op st(r2) */
11877 p = "\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0) */
11882 if (MEM_P (operands[1]))
11888 if (MEM_P (operands[2]))
11894 if (find_regno_note (insn, REG_DEAD, REGNO (operands[2])))
11897 /* The SystemV/386 SVR3.2 assembler, and probably all AT&T
11898 derived assemblers, confusingly reverse the direction of
11899 the operation for fsub{r} and fdiv{r} when the
11900 destination register is not st(0). The Intel assembler
11901 doesn't have this brain damage. Read !SYSV386_COMPAT to
11902 figure out what the hardware really does. */
11903 if (STACK_TOP_P (operands[0]))
11904 p = "{p\t%0, %2|rp\t%2, %0}";
11906 p = "{rp\t%2, %0|p\t%0, %2}";
11908 if (STACK_TOP_P (operands[0]))
11909 /* As above for fmul/fadd, we can't store to st(0). */
11910 p = "rp\t{%0, %2|%2, %0}"; /* st(1) = st(0) op st(1); pop */
11912 p = "p\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0); pop */
11917 if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
11920 if (STACK_TOP_P (operands[0]))
11921 p = "{rp\t%0, %1|p\t%1, %0}";
11923 p = "{p\t%1, %0|rp\t%0, %1}";
11925 if (STACK_TOP_P (operands[0]))
11926 p = "p\t{%0, %1|%1, %0}"; /* st(1) = st(1) op st(0); pop */
11928 p = "rp\t{%1, %0|%0, %1}"; /* st(r2) = st(0) op st(r2); pop */
11933 if (STACK_TOP_P (operands[0]))
11935 if (STACK_TOP_P (operands[1]))
11936 p = "\t{%y2, %0|%0, %y2}"; /* st(0) = st(0) op st(r2) */
11938 p = "r\t{%y1, %0|%0, %y1}"; /* st(0) = st(r1) op st(0) */
11941 else if (STACK_TOP_P (operands[1]))
11944 p = "{\t%1, %0|r\t%0, %1}";
11946 p = "r\t{%1, %0|%0, %1}"; /* st(r2) = st(0) op st(r2) */
11952 p = "{r\t%2, %0|\t%0, %2}";
11954 p = "\t{%2, %0|%0, %2}"; /* st(r1) = st(r1) op st(0) */
11960 gcc_unreachable ();
11967 /* Return needed mode for entity in optimize_mode_switching pass. */
11970 ix86_mode_needed (int entity, rtx insn)
11972 enum attr_i387_cw mode;
11974 /* The mode UNINITIALIZED is used to store control word after a
11975 function call or ASM pattern. The mode ANY specify that function
11976 has no requirements on the control word and make no changes in the
11977 bits we are interested in. */
11980 || (NONJUMP_INSN_P (insn)
11981 && (asm_noperands (PATTERN (insn)) >= 0
11982 || GET_CODE (PATTERN (insn)) == ASM_INPUT)))
11983 return I387_CW_UNINITIALIZED;
11985 if (recog_memoized (insn) < 0)
11986 return I387_CW_ANY;
11988 mode = get_attr_i387_cw (insn);
11993 if (mode == I387_CW_TRUNC)
11998 if (mode == I387_CW_FLOOR)
12003 if (mode == I387_CW_CEIL)
12008 if (mode == I387_CW_MASK_PM)
12013 gcc_unreachable ();
12016 return I387_CW_ANY;
12019 /* Output code to initialize control word copies used by trunc?f?i and
12020 rounding patterns. CURRENT_MODE is set to current control word,
12021 while NEW_MODE is set to new control word. */
12024 emit_i387_cw_initialization (int mode)
12026 rtx stored_mode = assign_386_stack_local (HImode, SLOT_CW_STORED);
12029 enum ix86_stack_slot slot;
12031 rtx reg = gen_reg_rtx (HImode);
12033 emit_insn (gen_x86_fnstcw_1 (stored_mode));
12034 emit_move_insn (reg, copy_rtx (stored_mode));
12036 if (TARGET_64BIT || TARGET_PARTIAL_REG_STALL
12037 || optimize_function_for_size_p (cfun))
12041 case I387_CW_TRUNC:
12042 /* round toward zero (truncate) */
12043 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0c00)));
12044 slot = SLOT_CW_TRUNC;
12047 case I387_CW_FLOOR:
12048 /* round down toward -oo */
12049 emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
12050 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0400)));
12051 slot = SLOT_CW_FLOOR;
12055 /* round up toward +oo */
12056 emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00)));
12057 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0800)));
12058 slot = SLOT_CW_CEIL;
12061 case I387_CW_MASK_PM:
12062 /* mask precision exception for nearbyint() */
12063 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0020)));
12064 slot = SLOT_CW_MASK_PM;
12068 gcc_unreachable ();
12075 case I387_CW_TRUNC:
12076 /* round toward zero (truncate) */
12077 emit_insn (gen_movsi_insv_1 (reg, GEN_INT (0xc)));
12078 slot = SLOT_CW_TRUNC;
12081 case I387_CW_FLOOR:
12082 /* round down toward -oo */
12083 emit_insn (gen_movsi_insv_1 (reg, GEN_INT (0x4)));
12084 slot = SLOT_CW_FLOOR;
12088 /* round up toward +oo */
12089 emit_insn (gen_movsi_insv_1 (reg, GEN_INT (0x8)));
12090 slot = SLOT_CW_CEIL;
12093 case I387_CW_MASK_PM:
12094 /* mask precision exception for nearbyint() */
12095 emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0020)));
12096 slot = SLOT_CW_MASK_PM;
12100 gcc_unreachable ();
12104 gcc_assert (slot < MAX_386_STACK_LOCALS);
12106 new_mode = assign_386_stack_local (HImode, slot);
12107 emit_move_insn (new_mode, reg);
12110 /* Output code for INSN to convert a float to a signed int. OPERANDS
12111 are the insn operands. The output may be [HSD]Imode and the input
12112 operand may be [SDX]Fmode. */
12115 output_fix_trunc (rtx insn, rtx *operands, int fisttp)
12117 int stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG) != 0;
12118 int dimode_p = GET_MODE (operands[0]) == DImode;
12119 int round_mode = get_attr_i387_cw (insn);
12121 /* Jump through a hoop or two for DImode, since the hardware has no
12122 non-popping instruction. We used to do this a different way, but
12123 that was somewhat fragile and broke with post-reload splitters. */
12124 if ((dimode_p || fisttp) && !stack_top_dies)
12125 output_asm_insn ("fld\t%y1", operands);
12127 gcc_assert (STACK_TOP_P (operands[1]));
12128 gcc_assert (MEM_P (operands[0]));
12129 gcc_assert (GET_MODE (operands[1]) != TFmode);
12132 output_asm_insn ("fisttp%Z0\t%0", operands);
12135 if (round_mode != I387_CW_ANY)
12136 output_asm_insn ("fldcw\t%3", operands);
12137 if (stack_top_dies || dimode_p)
12138 output_asm_insn ("fistp%Z0\t%0", operands);
12140 output_asm_insn ("fist%Z0\t%0", operands);
12141 if (round_mode != I387_CW_ANY)
12142 output_asm_insn ("fldcw\t%2", operands);
12148 /* Output code for x87 ffreep insn. The OPNO argument, which may only
12149 have the values zero or one, indicates the ffreep insn's operand
12150 from the OPERANDS array. */
12152 static const char *
12153 output_387_ffreep (rtx *operands ATTRIBUTE_UNUSED, int opno)
12155 if (TARGET_USE_FFREEP)
12156 #if HAVE_AS_IX86_FFREEP
12157 return opno ? "ffreep\t%y1" : "ffreep\t%y0";
12160 static char retval[] = ".word\t0xc_df";
12161 int regno = REGNO (operands[opno]);
12163 gcc_assert (FP_REGNO_P (regno));
12165 retval[9] = '0' + (regno - FIRST_STACK_REG);
12170 return opno ? "fstp\t%y1" : "fstp\t%y0";
12174 /* Output code for INSN to compare OPERANDS. EFLAGS_P is 1 when fcomi
12175 should be used. UNORDERED_P is true when fucom should be used. */
12178 output_fp_compare (rtx insn, rtx *operands, int eflags_p, int unordered_p)
12180 int stack_top_dies;
12181 rtx cmp_op0, cmp_op1;
12182 int is_sse = SSE_REG_P (operands[0]) || SSE_REG_P (operands[1]);
12186 cmp_op0 = operands[0];
12187 cmp_op1 = operands[1];
12191 cmp_op0 = operands[1];
12192 cmp_op1 = operands[2];
12197 static const char ucomiss[] = "vucomiss\t{%1, %0|%0, %1}";
12198 static const char ucomisd[] = "vucomisd\t{%1, %0|%0, %1}";
12199 static const char comiss[] = "vcomiss\t{%1, %0|%0, %1}";
12200 static const char comisd[] = "vcomisd\t{%1, %0|%0, %1}";
12202 if (GET_MODE (operands[0]) == SFmode)
12204 return &ucomiss[TARGET_AVX ? 0 : 1];
12206 return &comiss[TARGET_AVX ? 0 : 1];
12209 return &ucomisd[TARGET_AVX ? 0 : 1];
12211 return &comisd[TARGET_AVX ? 0 : 1];
12214 gcc_assert (STACK_TOP_P (cmp_op0));
12216 stack_top_dies = find_regno_note (insn, REG_DEAD, FIRST_STACK_REG) != 0;
12218 if (cmp_op1 == CONST0_RTX (GET_MODE (cmp_op1)))
12220 if (stack_top_dies)
12222 output_asm_insn ("ftst\n\tfnstsw\t%0", operands);
12223 return output_387_ffreep (operands, 1);
12226 return "ftst\n\tfnstsw\t%0";
12229 if (STACK_REG_P (cmp_op1)
12231 && find_regno_note (insn, REG_DEAD, REGNO (cmp_op1))
12232 && REGNO (cmp_op1) != FIRST_STACK_REG)
12234 /* If both the top of the 387 stack dies, and the other operand
12235 is also a stack register that dies, then this must be a
12236 `fcompp' float compare */
12240 /* There is no double popping fcomi variant. Fortunately,
12241 eflags is immune from the fstp's cc clobbering. */
12243 output_asm_insn ("fucomip\t{%y1, %0|%0, %y1}", operands);
12245 output_asm_insn ("fcomip\t{%y1, %0|%0, %y1}", operands);
12246 return output_387_ffreep (operands, 0);
12251 return "fucompp\n\tfnstsw\t%0";
12253 return "fcompp\n\tfnstsw\t%0";
12258 /* Encoded here as eflags_p | intmode | unordered_p | stack_top_dies. */
12260 static const char * const alt[16] =
12262 "fcom%Z2\t%y2\n\tfnstsw\t%0",
12263 "fcomp%Z2\t%y2\n\tfnstsw\t%0",
12264 "fucom%Z2\t%y2\n\tfnstsw\t%0",
12265 "fucomp%Z2\t%y2\n\tfnstsw\t%0",
12267 "ficom%Z2\t%y2\n\tfnstsw\t%0",
12268 "ficomp%Z2\t%y2\n\tfnstsw\t%0",
12272 "fcomi\t{%y1, %0|%0, %y1}",
12273 "fcomip\t{%y1, %0|%0, %y1}",
12274 "fucomi\t{%y1, %0|%0, %y1}",
12275 "fucomip\t{%y1, %0|%0, %y1}",
12286 mask = eflags_p << 3;
12287 mask |= (GET_MODE_CLASS (GET_MODE (cmp_op1)) == MODE_INT) << 2;
12288 mask |= unordered_p << 1;
12289 mask |= stack_top_dies;
12291 gcc_assert (mask < 16);
12300 ix86_output_addr_vec_elt (FILE *file, int value)
12302 const char *directive = ASM_LONG;
12306 directive = ASM_QUAD;
12308 gcc_assert (!TARGET_64BIT);
12311 fprintf (file, "%s%s%d\n", directive, LPREFIX, value);
12315 ix86_output_addr_diff_elt (FILE *file, int value, int rel)
12317 const char *directive = ASM_LONG;
12320 if (TARGET_64BIT && CASE_VECTOR_MODE == DImode)
12321 directive = ASM_QUAD;
12323 gcc_assert (!TARGET_64BIT);
12325 /* We can't use @GOTOFF for text labels on VxWorks; see gotoff_operand. */
12326 if (TARGET_64BIT || TARGET_VXWORKS_RTP)
12327 fprintf (file, "%s%s%d-%s%d\n",
12328 directive, LPREFIX, value, LPREFIX, rel);
12329 else if (HAVE_AS_GOTOFF_IN_DATA)
12330 fprintf (file, "%s%s%d@GOTOFF\n", ASM_LONG, LPREFIX, value);
12332 else if (TARGET_MACHO)
12334 fprintf (file, "%s%s%d-", ASM_LONG, LPREFIX, value);
12335 machopic_output_function_base_name (file);
12336 fprintf(file, "\n");
12340 asm_fprintf (file, "%s%U%s+[.-%s%d]\n",
12341 ASM_LONG, GOT_SYMBOL_NAME, LPREFIX, value);
12344 /* Generate either "mov $0, reg" or "xor reg, reg", as appropriate
12348 ix86_expand_clear (rtx dest)
12352 /* We play register width games, which are only valid after reload. */
12353 gcc_assert (reload_completed);
12355 /* Avoid HImode and its attendant prefix byte. */
12356 if (GET_MODE_SIZE (GET_MODE (dest)) < 4)
12357 dest = gen_rtx_REG (SImode, REGNO (dest));
12358 tmp = gen_rtx_SET (VOIDmode, dest, const0_rtx);
12360 /* This predicate should match that for movsi_xor and movdi_xor_rex64. */
12361 if (reload_completed && (!TARGET_USE_MOV0 || optimize_insn_for_speed_p ()))
12363 rtx clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
12364 tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, clob));
12370 /* X is an unchanging MEM. If it is a constant pool reference, return
12371 the constant pool rtx, else NULL. */
12374 maybe_get_pool_constant (rtx x)
12376 x = ix86_delegitimize_address (XEXP (x, 0));
12378 if (GET_CODE (x) == SYMBOL_REF && CONSTANT_POOL_ADDRESS_P (x))
12379 return get_pool_constant (x);
12385 ix86_expand_move (enum machine_mode mode, rtx operands[])
12388 enum tls_model model;
12393 if (GET_CODE (op1) == SYMBOL_REF)
12395 model = SYMBOL_REF_TLS_MODEL (op1);
12398 op1 = legitimize_tls_address (op1, model, true);
12399 op1 = force_operand (op1, op0);
12403 else if (TARGET_DLLIMPORT_DECL_ATTRIBUTES
12404 && SYMBOL_REF_DLLIMPORT_P (op1))
12405 op1 = legitimize_dllimport_symbol (op1, false);
12407 else if (GET_CODE (op1) == CONST
12408 && GET_CODE (XEXP (op1, 0)) == PLUS
12409 && GET_CODE (XEXP (XEXP (op1, 0), 0)) == SYMBOL_REF)
12411 rtx addend = XEXP (XEXP (op1, 0), 1);
12412 rtx symbol = XEXP (XEXP (op1, 0), 0);
12415 model = SYMBOL_REF_TLS_MODEL (symbol);
12417 tmp = legitimize_tls_address (symbol, model, true);
12418 else if (TARGET_DLLIMPORT_DECL_ATTRIBUTES
12419 && SYMBOL_REF_DLLIMPORT_P (symbol))
12420 tmp = legitimize_dllimport_symbol (symbol, true);
12424 tmp = force_operand (tmp, NULL);
12425 tmp = expand_simple_binop (Pmode, PLUS, tmp, addend,
12426 op0, 1, OPTAB_DIRECT);
12432 if (flag_pic && mode == Pmode && symbolic_operand (op1, Pmode))
12434 if (TARGET_MACHO && !TARGET_64BIT)
12439 rtx temp = ((reload_in_progress
12440 || ((op0 && REG_P (op0))
12442 ? op0 : gen_reg_rtx (Pmode));
12443 op1 = machopic_indirect_data_reference (op1, temp);
12444 op1 = machopic_legitimize_pic_address (op1, mode,
12445 temp == op1 ? 0 : temp);
12447 else if (MACHOPIC_INDIRECT)
12448 op1 = machopic_indirect_data_reference (op1, 0);
12456 op1 = force_reg (Pmode, op1);
12457 else if (!TARGET_64BIT || !x86_64_movabs_operand (op1, Pmode))
12459 rtx reg = !can_create_pseudo_p () ? op0 : NULL_RTX;
12460 op1 = legitimize_pic_address (op1, reg);
12469 && (PUSH_ROUNDING (GET_MODE_SIZE (mode)) != GET_MODE_SIZE (mode)
12470 || !push_operand (op0, mode))
12472 op1 = force_reg (mode, op1);
12474 if (push_operand (op0, mode)
12475 && ! general_no_elim_operand (op1, mode))
12476 op1 = copy_to_mode_reg (mode, op1);
12478 /* Force large constants in 64bit compilation into register
12479 to get them CSEed. */
12480 if (can_create_pseudo_p ()
12481 && (mode == DImode) && TARGET_64BIT
12482 && immediate_operand (op1, mode)
12483 && !x86_64_zext_immediate_operand (op1, VOIDmode)
12484 && !register_operand (op0, mode)
12486 op1 = copy_to_mode_reg (mode, op1);
12488 if (can_create_pseudo_p ()
12489 && FLOAT_MODE_P (mode)
12490 && GET_CODE (op1) == CONST_DOUBLE)
12492 /* If we are loading a floating point constant to a register,
12493 force the value to memory now, since we'll get better code
12494 out the back end. */
12496 op1 = validize_mem (force_const_mem (mode, op1));
12497 if (!register_operand (op0, mode))
12499 rtx temp = gen_reg_rtx (mode);
12500 emit_insn (gen_rtx_SET (VOIDmode, temp, op1));
12501 emit_move_insn (op0, temp);
12507 emit_insn (gen_rtx_SET (VOIDmode, op0, op1));
12511 ix86_expand_vector_move (enum machine_mode mode, rtx operands[])
12513 rtx op0 = operands[0], op1 = operands[1];
12514 unsigned int align = GET_MODE_ALIGNMENT (mode);
12516 /* Force constants other than zero into memory. We do not know how
12517 the instructions used to build constants modify the upper 64 bits
12518 of the register, once we have that information we may be able
12519 to handle some of them more efficiently. */
12520 if (can_create_pseudo_p ()
12521 && register_operand (op0, mode)
12522 && (CONSTANT_P (op1)
12523 || (GET_CODE (op1) == SUBREG
12524 && CONSTANT_P (SUBREG_REG (op1))))
12525 && standard_sse_constant_p (op1) <= 0)
12526 op1 = validize_mem (force_const_mem (mode, op1));
12528 /* We need to check memory alignment for SSE mode since attribute
12529 can make operands unaligned. */
12530 if (can_create_pseudo_p ()
12531 && SSE_REG_MODE_P (mode)
12532 && ((MEM_P (op0) && (MEM_ALIGN (op0) < align))
12533 || (MEM_P (op1) && (MEM_ALIGN (op1) < align))))
12537 /* ix86_expand_vector_move_misalign() does not like constants ... */
12538 if (CONSTANT_P (op1)
12539 || (GET_CODE (op1) == SUBREG
12540 && CONSTANT_P (SUBREG_REG (op1))))
12541 op1 = validize_mem (force_const_mem (mode, op1));
12543 /* ... nor both arguments in memory. */
12544 if (!register_operand (op0, mode)
12545 && !register_operand (op1, mode))
12546 op1 = force_reg (mode, op1);
12548 tmp[0] = op0; tmp[1] = op1;
12549 ix86_expand_vector_move_misalign (mode, tmp);
12553 /* Make operand1 a register if it isn't already. */
12554 if (can_create_pseudo_p ()
12555 && !register_operand (op0, mode)
12556 && !register_operand (op1, mode))
12558 emit_move_insn (op0, force_reg (GET_MODE (op0), op1));
12562 emit_insn (gen_rtx_SET (VOIDmode, op0, op1));
12565 /* Implement the movmisalign patterns for SSE. Non-SSE modes go
12566 straight to ix86_expand_vector_move. */
12567 /* Code generation for scalar reg-reg moves of single and double precision data:
12568 if (x86_sse_partial_reg_dependency == true | x86_sse_split_regs == true)
12572 if (x86_sse_partial_reg_dependency == true)
12577 Code generation for scalar loads of double precision data:
12578 if (x86_sse_split_regs == true)
12579 movlpd mem, reg (gas syntax)
12583 Code generation for unaligned packed loads of single precision data
12584 (x86_sse_unaligned_move_optimal overrides x86_sse_partial_reg_dependency):
12585 if (x86_sse_unaligned_move_optimal)
12588 if (x86_sse_partial_reg_dependency == true)
12600 Code generation for unaligned packed loads of double precision data
12601 (x86_sse_unaligned_move_optimal overrides x86_sse_split_regs):
12602 if (x86_sse_unaligned_move_optimal)
12605 if (x86_sse_split_regs == true)
12618 ix86_expand_vector_move_misalign (enum machine_mode mode, rtx operands[])
12627 switch (GET_MODE_CLASS (mode))
12629 case MODE_VECTOR_INT:
12631 switch (GET_MODE_SIZE (mode))
12634 op0 = gen_lowpart (V16QImode, op0);
12635 op1 = gen_lowpart (V16QImode, op1);
12636 emit_insn (gen_avx_movdqu (op0, op1));
12639 op0 = gen_lowpart (V32QImode, op0);
12640 op1 = gen_lowpart (V32QImode, op1);
12641 emit_insn (gen_avx_movdqu256 (op0, op1));
12644 gcc_unreachable ();
12647 case MODE_VECTOR_FLOAT:
12648 op0 = gen_lowpart (mode, op0);
12649 op1 = gen_lowpart (mode, op1);
12654 emit_insn (gen_avx_movups (op0, op1));
12657 emit_insn (gen_avx_movups256 (op0, op1));
12660 emit_insn (gen_avx_movupd (op0, op1));
12663 emit_insn (gen_avx_movupd256 (op0, op1));
12666 gcc_unreachable ();
12671 gcc_unreachable ();
12679 /* If we're optimizing for size, movups is the smallest. */
12680 if (optimize_insn_for_size_p ())
12682 op0 = gen_lowpart (V4SFmode, op0);
12683 op1 = gen_lowpart (V4SFmode, op1);
12684 emit_insn (gen_sse_movups (op0, op1));
12688 /* ??? If we have typed data, then it would appear that using
12689 movdqu is the only way to get unaligned data loaded with
12691 if (TARGET_SSE2 && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
12693 op0 = gen_lowpart (V16QImode, op0);
12694 op1 = gen_lowpart (V16QImode, op1);
12695 emit_insn (gen_sse2_movdqu (op0, op1));
12699 if (TARGET_SSE2 && mode == V2DFmode)
12703 if (TARGET_SSE_UNALIGNED_MOVE_OPTIMAL)
12705 op0 = gen_lowpart (V2DFmode, op0);
12706 op1 = gen_lowpart (V2DFmode, op1);
12707 emit_insn (gen_sse2_movupd (op0, op1));
12711 /* When SSE registers are split into halves, we can avoid
12712 writing to the top half twice. */
12713 if (TARGET_SSE_SPLIT_REGS)
12715 emit_clobber (op0);
12720 /* ??? Not sure about the best option for the Intel chips.
12721 The following would seem to satisfy; the register is
12722 entirely cleared, breaking the dependency chain. We
12723 then store to the upper half, with a dependency depth
12724 of one. A rumor has it that Intel recommends two movsd
12725 followed by an unpacklpd, but this is unconfirmed. And
12726 given that the dependency depth of the unpacklpd would
12727 still be one, I'm not sure why this would be better. */
12728 zero = CONST0_RTX (V2DFmode);
12731 m = adjust_address (op1, DFmode, 0);
12732 emit_insn (gen_sse2_loadlpd (op0, zero, m));
12733 m = adjust_address (op1, DFmode, 8);
12734 emit_insn (gen_sse2_loadhpd (op0, op0, m));
12738 if (TARGET_SSE_UNALIGNED_MOVE_OPTIMAL)
12740 op0 = gen_lowpart (V4SFmode, op0);
12741 op1 = gen_lowpart (V4SFmode, op1);
12742 emit_insn (gen_sse_movups (op0, op1));
12746 if (TARGET_SSE_PARTIAL_REG_DEPENDENCY)
12747 emit_move_insn (op0, CONST0_RTX (mode));
12749 emit_clobber (op0);
12751 if (mode != V4SFmode)
12752 op0 = gen_lowpart (V4SFmode, op0);
12753 m = adjust_address (op1, V2SFmode, 0);
12754 emit_insn (gen_sse_loadlps (op0, op0, m));
12755 m = adjust_address (op1, V2SFmode, 8);
12756 emit_insn (gen_sse_loadhps (op0, op0, m));
12759 else if (MEM_P (op0))
12761 /* If we're optimizing for size, movups is the smallest. */
12762 if (optimize_insn_for_size_p ())
12764 op0 = gen_lowpart (V4SFmode, op0);
12765 op1 = gen_lowpart (V4SFmode, op1);
12766 emit_insn (gen_sse_movups (op0, op1));
12770 /* ??? Similar to above, only less clear because of quote
12771 typeless stores unquote. */
12772 if (TARGET_SSE2 && !TARGET_SSE_TYPELESS_STORES
12773 && GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
12775 op0 = gen_lowpart (V16QImode, op0);
12776 op1 = gen_lowpart (V16QImode, op1);
12777 emit_insn (gen_sse2_movdqu (op0, op1));
12781 if (TARGET_SSE2 && mode == V2DFmode)
12783 m = adjust_address (op0, DFmode, 0);
12784 emit_insn (gen_sse2_storelpd (m, op1));
12785 m = adjust_address (op0, DFmode, 8);
12786 emit_insn (gen_sse2_storehpd (m, op1));
12790 if (mode != V4SFmode)
12791 op1 = gen_lowpart (V4SFmode, op1);
12792 m = adjust_address (op0, V2SFmode, 0);
12793 emit_insn (gen_sse_storelps (m, op1));
12794 m = adjust_address (op0, V2SFmode, 8);
12795 emit_insn (gen_sse_storehps (m, op1));
12799 gcc_unreachable ();
12802 /* Expand a push in MODE. This is some mode for which we do not support
12803 proper push instructions, at least from the registers that we expect
12804 the value to live in. */
12807 ix86_expand_push (enum machine_mode mode, rtx x)
12811 tmp = expand_simple_binop (Pmode, PLUS, stack_pointer_rtx,
12812 GEN_INT (-GET_MODE_SIZE (mode)),
12813 stack_pointer_rtx, 1, OPTAB_DIRECT);
12814 if (tmp != stack_pointer_rtx)
12815 emit_move_insn (stack_pointer_rtx, tmp);
12817 tmp = gen_rtx_MEM (mode, stack_pointer_rtx);
12819 /* When we push an operand onto stack, it has to be aligned at least
12820 at the function argument boundary. However since we don't have
12821 the argument type, we can't determine the actual argument
12823 emit_move_insn (tmp, x);
12826 /* Helper function of ix86_fixup_binary_operands to canonicalize
12827 operand order. Returns true if the operands should be swapped. */
12830 ix86_swap_binary_operands_p (enum rtx_code code, enum machine_mode mode,
12833 rtx dst = operands[0];
12834 rtx src1 = operands[1];
12835 rtx src2 = operands[2];
12837 /* If the operation is not commutative, we can't do anything. */
12838 if (GET_RTX_CLASS (code) != RTX_COMM_ARITH)
12841 /* Highest priority is that src1 should match dst. */
12842 if (rtx_equal_p (dst, src1))
12844 if (rtx_equal_p (dst, src2))
12847 /* Next highest priority is that immediate constants come second. */
12848 if (immediate_operand (src2, mode))
12850 if (immediate_operand (src1, mode))
12853 /* Lowest priority is that memory references should come second. */
12863 /* Fix up OPERANDS to satisfy ix86_binary_operator_ok. Return the
12864 destination to use for the operation. If different from the true
12865 destination in operands[0], a copy operation will be required. */
12868 ix86_fixup_binary_operands (enum rtx_code code, enum machine_mode mode,
12871 rtx dst = operands[0];
12872 rtx src1 = operands[1];
12873 rtx src2 = operands[2];
12875 /* Canonicalize operand order. */
12876 if (ix86_swap_binary_operands_p (code, mode, operands))
12880 /* It is invalid to swap operands of different modes. */
12881 gcc_assert (GET_MODE (src1) == GET_MODE (src2));
12888 /* Both source operands cannot be in memory. */
12889 if (MEM_P (src1) && MEM_P (src2))
12891 /* Optimization: Only read from memory once. */
12892 if (rtx_equal_p (src1, src2))
12894 src2 = force_reg (mode, src2);
12898 src2 = force_reg (mode, src2);
12901 /* If the destination is memory, and we do not have matching source
12902 operands, do things in registers. */
12903 if (MEM_P (dst) && !rtx_equal_p (dst, src1))
12904 dst = gen_reg_rtx (mode);
12906 /* Source 1 cannot be a constant. */
12907 if (CONSTANT_P (src1))
12908 src1 = force_reg (mode, src1);
12910 /* Source 1 cannot be a non-matching memory. */
12911 if (MEM_P (src1) && !rtx_equal_p (dst, src1))
12912 src1 = force_reg (mode, src1);
12914 operands[1] = src1;
12915 operands[2] = src2;
12919 /* Similarly, but assume that the destination has already been
12920 set up properly. */
12923 ix86_fixup_binary_operands_no_copy (enum rtx_code code,
12924 enum machine_mode mode, rtx operands[])
12926 rtx dst = ix86_fixup_binary_operands (code, mode, operands);
12927 gcc_assert (dst == operands[0]);
12930 /* Attempt to expand a binary operator. Make the expansion closer to the
12931 actual machine, then just general_operand, which will allow 3 separate
12932 memory references (one output, two input) in a single insn. */
12935 ix86_expand_binary_operator (enum rtx_code code, enum machine_mode mode,
12938 rtx src1, src2, dst, op, clob;
12940 dst = ix86_fixup_binary_operands (code, mode, operands);
12941 src1 = operands[1];
12942 src2 = operands[2];
12944 /* Emit the instruction. */
12946 op = gen_rtx_SET (VOIDmode, dst, gen_rtx_fmt_ee (code, mode, src1, src2));
12947 if (reload_in_progress)
12949 /* Reload doesn't know about the flags register, and doesn't know that
12950 it doesn't want to clobber it. We can only do this with PLUS. */
12951 gcc_assert (code == PLUS);
12956 clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
12957 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
12960 /* Fix up the destination if needed. */
12961 if (dst != operands[0])
12962 emit_move_insn (operands[0], dst);
12965 /* Return TRUE or FALSE depending on whether the binary operator meets the
12966 appropriate constraints. */
12969 ix86_binary_operator_ok (enum rtx_code code, enum machine_mode mode,
12972 rtx dst = operands[0];
12973 rtx src1 = operands[1];
12974 rtx src2 = operands[2];
12976 /* Both source operands cannot be in memory. */
12977 if (MEM_P (src1) && MEM_P (src2))
12980 /* Canonicalize operand order for commutative operators. */
12981 if (ix86_swap_binary_operands_p (code, mode, operands))
12988 /* If the destination is memory, we must have a matching source operand. */
12989 if (MEM_P (dst) && !rtx_equal_p (dst, src1))
12992 /* Source 1 cannot be a constant. */
12993 if (CONSTANT_P (src1))
12996 /* Source 1 cannot be a non-matching memory. */
12997 if (MEM_P (src1) && !rtx_equal_p (dst, src1))
13003 /* Attempt to expand a unary operator. Make the expansion closer to the
13004 actual machine, then just general_operand, which will allow 2 separate
13005 memory references (one output, one input) in a single insn. */
13008 ix86_expand_unary_operator (enum rtx_code code, enum machine_mode mode,
13011 int matching_memory;
13012 rtx src, dst, op, clob;
13017 /* If the destination is memory, and we do not have matching source
13018 operands, do things in registers. */
13019 matching_memory = 0;
13022 if (rtx_equal_p (dst, src))
13023 matching_memory = 1;
13025 dst = gen_reg_rtx (mode);
13028 /* When source operand is memory, destination must match. */
13029 if (MEM_P (src) && !matching_memory)
13030 src = force_reg (mode, src);
13032 /* Emit the instruction. */
13034 op = gen_rtx_SET (VOIDmode, dst, gen_rtx_fmt_e (code, mode, src));
13035 if (reload_in_progress || code == NOT)
13037 /* Reload doesn't know about the flags register, and doesn't know that
13038 it doesn't want to clobber it. */
13039 gcc_assert (code == NOT);
13044 clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
13045 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, op, clob)));
13048 /* Fix up the destination if needed. */
13049 if (dst != operands[0])
13050 emit_move_insn (operands[0], dst);
13053 #define LEA_SEARCH_THRESHOLD 12
13055 /* Search backward for non-agu definition of register number REGNO1
13056 or register number REGNO2 in INSN's basic block until
13057 1. Pass LEA_SEARCH_THRESHOLD instructions, or
13058 2. Reach BB boundary, or
13059 3. Reach agu definition.
13060 Returns the distance between the non-agu definition point and INSN.
13061 If no definition point, returns -1. */
13064 distance_non_agu_define (unsigned int regno1, unsigned int regno2,
13067 basic_block bb = BLOCK_FOR_INSN (insn);
13070 enum attr_type insn_type;
13072 if (insn != BB_HEAD (bb))
13074 rtx prev = PREV_INSN (insn);
13075 while (prev && distance < LEA_SEARCH_THRESHOLD)
13080 for (def_rec = DF_INSN_DEFS (prev); *def_rec; def_rec++)
13081 if (DF_REF_TYPE (*def_rec) == DF_REF_REG_DEF
13082 && !DF_REF_IS_ARTIFICIAL (*def_rec)
13083 && (regno1 == DF_REF_REGNO (*def_rec)
13084 || regno2 == DF_REF_REGNO (*def_rec)))
13086 insn_type = get_attr_type (prev);
13087 if (insn_type != TYPE_LEA)
13091 if (prev == BB_HEAD (bb))
13093 prev = PREV_INSN (prev);
13097 if (distance < LEA_SEARCH_THRESHOLD)
13101 bool simple_loop = false;
13103 FOR_EACH_EDGE (e, ei, bb->preds)
13106 simple_loop = true;
13112 rtx prev = BB_END (bb);
13115 && distance < LEA_SEARCH_THRESHOLD)
13120 for (def_rec = DF_INSN_DEFS (prev); *def_rec; def_rec++)
13121 if (DF_REF_TYPE (*def_rec) == DF_REF_REG_DEF
13122 && !DF_REF_IS_ARTIFICIAL (*def_rec)
13123 && (regno1 == DF_REF_REGNO (*def_rec)
13124 || regno2 == DF_REF_REGNO (*def_rec)))
13126 insn_type = get_attr_type (prev);
13127 if (insn_type != TYPE_LEA)
13131 prev = PREV_INSN (prev);
13139 /* get_attr_type may modify recog data. We want to make sure
13140 that recog data is valid for instruction INSN, on which
13141 distance_non_agu_define is called. INSN is unchanged here. */
13142 extract_insn_cached (insn);
13146 /* Return the distance between INSN and the next insn that uses
13147 register number REGNO0 in memory address. Return -1 if no such
13148 a use is found within LEA_SEARCH_THRESHOLD or REGNO0 is set. */
13151 distance_agu_use (unsigned int regno0, rtx insn)
13153 basic_block bb = BLOCK_FOR_INSN (insn);
13158 if (insn != BB_END (bb))
13160 rtx next = NEXT_INSN (insn);
13161 while (next && distance < LEA_SEARCH_THRESHOLD)
13167 for (use_rec = DF_INSN_USES (next); *use_rec; use_rec++)
13168 if ((DF_REF_TYPE (*use_rec) == DF_REF_REG_MEM_LOAD
13169 || DF_REF_TYPE (*use_rec) == DF_REF_REG_MEM_STORE)
13170 && regno0 == DF_REF_REGNO (*use_rec))
13172 /* Return DISTANCE if OP0 is used in memory
13173 address in NEXT. */
13177 for (def_rec = DF_INSN_DEFS (next); *def_rec; def_rec++)
13178 if (DF_REF_TYPE (*def_rec) == DF_REF_REG_DEF
13179 && !DF_REF_IS_ARTIFICIAL (*def_rec)
13180 && regno0 == DF_REF_REGNO (*def_rec))
13182 /* Return -1 if OP0 is set in NEXT. */
13186 if (next == BB_END (bb))
13188 next = NEXT_INSN (next);
13192 if (distance < LEA_SEARCH_THRESHOLD)
13196 bool simple_loop = false;
13198 FOR_EACH_EDGE (e, ei, bb->succs)
13201 simple_loop = true;
13207 rtx next = BB_HEAD (bb);
13210 && distance < LEA_SEARCH_THRESHOLD)
13216 for (use_rec = DF_INSN_USES (next); *use_rec; use_rec++)
13217 if ((DF_REF_TYPE (*use_rec) == DF_REF_REG_MEM_LOAD
13218 || DF_REF_TYPE (*use_rec) == DF_REF_REG_MEM_STORE)
13219 && regno0 == DF_REF_REGNO (*use_rec))
13221 /* Return DISTANCE if OP0 is used in memory
13222 address in NEXT. */
13226 for (def_rec = DF_INSN_DEFS (next); *def_rec; def_rec++)
13227 if (DF_REF_TYPE (*def_rec) == DF_REF_REG_DEF
13228 && !DF_REF_IS_ARTIFICIAL (*def_rec)
13229 && regno0 == DF_REF_REGNO (*def_rec))
13231 /* Return -1 if OP0 is set in NEXT. */
13236 next = NEXT_INSN (next);
13244 /* Define this macro to tune LEA priority vs ADD, it take effect when
13245 there is a dilemma of choicing LEA or ADD
13246 Negative value: ADD is more preferred than LEA
13248 Positive value: LEA is more preferred than ADD*/
13249 #define IX86_LEA_PRIORITY 2
13251 /* Return true if it is ok to optimize an ADD operation to LEA
13252 operation to avoid flag register consumation. For the processors
13253 like ATOM, if the destination register of LEA holds an actual
13254 address which will be used soon, LEA is better and otherwise ADD
13258 ix86_lea_for_add_ok (enum rtx_code code ATTRIBUTE_UNUSED,
13259 rtx insn, rtx operands[])
13261 unsigned int regno0 = true_regnum (operands[0]);
13262 unsigned int regno1 = true_regnum (operands[1]);
13263 unsigned int regno2;
13265 if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun))
13266 return regno0 != regno1;
13268 regno2 = true_regnum (operands[2]);
13270 /* If a = b + c, (a!=b && a!=c), must use lea form. */
13271 if (regno0 != regno1 && regno0 != regno2)
13275 int dist_define, dist_use;
13276 dist_define = distance_non_agu_define (regno1, regno2, insn);
13277 if (dist_define <= 0)
13280 /* If this insn has both backward non-agu dependence and forward
13281 agu dependence, the one with short distance take effect. */
13282 dist_use = distance_agu_use (regno0, insn);
13284 || (dist_define + IX86_LEA_PRIORITY) < dist_use)
13291 /* Return true if destination reg of SET_BODY is shift count of
13295 ix86_dep_by_shift_count_body (const_rtx set_body, const_rtx use_body)
13301 /* Retrieve destination of SET_BODY. */
13302 switch (GET_CODE (set_body))
13305 set_dest = SET_DEST (set_body);
13306 if (!set_dest || !REG_P (set_dest))
13310 for (i = XVECLEN (set_body, 0) - 1; i >= 0; i--)
13311 if (ix86_dep_by_shift_count_body (XVECEXP (set_body, 0, i),
13319 /* Retrieve shift count of USE_BODY. */
13320 switch (GET_CODE (use_body))
13323 shift_rtx = XEXP (use_body, 1);
13326 for (i = XVECLEN (use_body, 0) - 1; i >= 0; i--)
13327 if (ix86_dep_by_shift_count_body (set_body,
13328 XVECEXP (use_body, 0, i)))
13336 && (GET_CODE (shift_rtx) == ASHIFT
13337 || GET_CODE (shift_rtx) == LSHIFTRT
13338 || GET_CODE (shift_rtx) == ASHIFTRT
13339 || GET_CODE (shift_rtx) == ROTATE
13340 || GET_CODE (shift_rtx) == ROTATERT))
13342 rtx shift_count = XEXP (shift_rtx, 1);
13344 /* Return true if shift count is dest of SET_BODY. */
13345 if (REG_P (shift_count)
13346 && true_regnum (set_dest) == true_regnum (shift_count))
13353 /* Return true if destination reg of SET_INSN is shift count of
13357 ix86_dep_by_shift_count (const_rtx set_insn, const_rtx use_insn)
13359 return ix86_dep_by_shift_count_body (PATTERN (set_insn),
13360 PATTERN (use_insn));
13363 /* Return TRUE or FALSE depending on whether the unary operator meets the
13364 appropriate constraints. */
13367 ix86_unary_operator_ok (enum rtx_code code ATTRIBUTE_UNUSED,
13368 enum machine_mode mode ATTRIBUTE_UNUSED,
13369 rtx operands[2] ATTRIBUTE_UNUSED)
13371 /* If one of operands is memory, source and destination must match. */
13372 if ((MEM_P (operands[0])
13373 || MEM_P (operands[1]))
13374 && ! rtx_equal_p (operands[0], operands[1]))
13379 /* Post-reload splitter for converting an SF or DFmode value in an
13380 SSE register into an unsigned SImode. */
13383 ix86_split_convert_uns_si_sse (rtx operands[])
13385 enum machine_mode vecmode;
13386 rtx value, large, zero_or_two31, input, two31, x;
13388 large = operands[1];
13389 zero_or_two31 = operands[2];
13390 input = operands[3];
13391 two31 = operands[4];
13392 vecmode = GET_MODE (large);
13393 value = gen_rtx_REG (vecmode, REGNO (operands[0]));
13395 /* Load up the value into the low element. We must ensure that the other
13396 elements are valid floats -- zero is the easiest such value. */
13399 if (vecmode == V4SFmode)
13400 emit_insn (gen_vec_setv4sf_0 (value, CONST0_RTX (V4SFmode), input));
13402 emit_insn (gen_sse2_loadlpd (value, CONST0_RTX (V2DFmode), input));
13406 input = gen_rtx_REG (vecmode, REGNO (input));
13407 emit_move_insn (value, CONST0_RTX (vecmode));
13408 if (vecmode == V4SFmode)
13409 emit_insn (gen_sse_movss (value, value, input));
13411 emit_insn (gen_sse2_movsd (value, value, input));
13414 emit_move_insn (large, two31);
13415 emit_move_insn (zero_or_two31, MEM_P (two31) ? large : two31);
13417 x = gen_rtx_fmt_ee (LE, vecmode, large, value);
13418 emit_insn (gen_rtx_SET (VOIDmode, large, x));
13420 x = gen_rtx_AND (vecmode, zero_or_two31, large);
13421 emit_insn (gen_rtx_SET (VOIDmode, zero_or_two31, x));
13423 x = gen_rtx_MINUS (vecmode, value, zero_or_two31);
13424 emit_insn (gen_rtx_SET (VOIDmode, value, x));
13426 large = gen_rtx_REG (V4SImode, REGNO (large));
13427 emit_insn (gen_ashlv4si3 (large, large, GEN_INT (31)));
13429 x = gen_rtx_REG (V4SImode, REGNO (value));
13430 if (vecmode == V4SFmode)
13431 emit_insn (gen_sse2_cvttps2dq (x, value));
13433 emit_insn (gen_sse2_cvttpd2dq (x, value));
13436 emit_insn (gen_xorv4si3 (value, value, large));
13439 /* Convert an unsigned DImode value into a DFmode, using only SSE.
13440 Expects the 64-bit DImode to be supplied in a pair of integral
13441 registers. Requires SSE2; will use SSE3 if available. For x86_32,
13442 -mfpmath=sse, !optimize_size only. */
13445 ix86_expand_convert_uns_didf_sse (rtx target, rtx input)
13447 REAL_VALUE_TYPE bias_lo_rvt, bias_hi_rvt;
13448 rtx int_xmm, fp_xmm;
13449 rtx biases, exponents;
13452 int_xmm = gen_reg_rtx (V4SImode);
13453 if (TARGET_INTER_UNIT_MOVES)
13454 emit_insn (gen_movdi_to_sse (int_xmm, input));
13455 else if (TARGET_SSE_SPLIT_REGS)
13457 emit_clobber (int_xmm);
13458 emit_move_insn (gen_lowpart (DImode, int_xmm), input);
13462 x = gen_reg_rtx (V2DImode);
13463 ix86_expand_vector_init_one_nonzero (false, V2DImode, x, input, 0);
13464 emit_move_insn (int_xmm, gen_lowpart (V4SImode, x));
13467 x = gen_rtx_CONST_VECTOR (V4SImode,
13468 gen_rtvec (4, GEN_INT (0x43300000UL),
13469 GEN_INT (0x45300000UL),
13470 const0_rtx, const0_rtx));
13471 exponents = validize_mem (force_const_mem (V4SImode, x));
13473 /* int_xmm = {0x45300000UL, fp_xmm/hi, 0x43300000, fp_xmm/lo } */
13474 emit_insn (gen_sse2_punpckldq (int_xmm, int_xmm, exponents));
13476 /* Concatenating (juxtaposing) (0x43300000UL ## fp_value_low_xmm)
13477 yields a valid DF value equal to (0x1.0p52 + double(fp_value_lo_xmm)).
13478 Similarly (0x45300000UL ## fp_value_hi_xmm) yields
13479 (0x1.0p84 + double(fp_value_hi_xmm)).
13480 Note these exponents differ by 32. */
13482 fp_xmm = copy_to_mode_reg (V2DFmode, gen_lowpart (V2DFmode, int_xmm));
13484 /* Subtract off those 0x1.0p52 and 0x1.0p84 biases, to produce values
13485 in [0,2**32-1] and [0]+[2**32,2**64-1] respectively. */
13486 real_ldexp (&bias_lo_rvt, &dconst1, 52);
13487 real_ldexp (&bias_hi_rvt, &dconst1, 84);
13488 biases = const_double_from_real_value (bias_lo_rvt, DFmode);
13489 x = const_double_from_real_value (bias_hi_rvt, DFmode);
13490 biases = gen_rtx_CONST_VECTOR (V2DFmode, gen_rtvec (2, biases, x));
13491 biases = validize_mem (force_const_mem (V2DFmode, biases));
13492 emit_insn (gen_subv2df3 (fp_xmm, fp_xmm, biases));
13494 /* Add the upper and lower DFmode values together. */
13496 emit_insn (gen_sse3_haddv2df3 (fp_xmm, fp_xmm, fp_xmm));
13499 x = copy_to_mode_reg (V2DFmode, fp_xmm);
13500 emit_insn (gen_sse2_unpckhpd (fp_xmm, fp_xmm, fp_xmm));
13501 emit_insn (gen_addv2df3 (fp_xmm, fp_xmm, x));
13504 ix86_expand_vector_extract (false, target, fp_xmm, 0);
13507 /* Not used, but eases macroization of patterns. */
13509 ix86_expand_convert_uns_sixf_sse (rtx target ATTRIBUTE_UNUSED,
13510 rtx input ATTRIBUTE_UNUSED)
13512 gcc_unreachable ();
13515 /* Convert an unsigned SImode value into a DFmode. Only currently used
13516 for SSE, but applicable anywhere. */
13519 ix86_expand_convert_uns_sidf_sse (rtx target, rtx input)
13521 REAL_VALUE_TYPE TWO31r;
13524 x = expand_simple_binop (SImode, PLUS, input, GEN_INT (-2147483647 - 1),
13525 NULL, 1, OPTAB_DIRECT);
13527 fp = gen_reg_rtx (DFmode);
13528 emit_insn (gen_floatsidf2 (fp, x));
13530 real_ldexp (&TWO31r, &dconst1, 31);
13531 x = const_double_from_real_value (TWO31r, DFmode);
13533 x = expand_simple_binop (DFmode, PLUS, fp, x, target, 0, OPTAB_DIRECT);
13535 emit_move_insn (target, x);
13538 /* Convert a signed DImode value into a DFmode. Only used for SSE in
13539 32-bit mode; otherwise we have a direct convert instruction. */
13542 ix86_expand_convert_sign_didf_sse (rtx target, rtx input)
13544 REAL_VALUE_TYPE TWO32r;
13545 rtx fp_lo, fp_hi, x;
13547 fp_lo = gen_reg_rtx (DFmode);
13548 fp_hi = gen_reg_rtx (DFmode);
13550 emit_insn (gen_floatsidf2 (fp_hi, gen_highpart (SImode, input)));
13552 real_ldexp (&TWO32r, &dconst1, 32);
13553 x = const_double_from_real_value (TWO32r, DFmode);
13554 fp_hi = expand_simple_binop (DFmode, MULT, fp_hi, x, fp_hi, 0, OPTAB_DIRECT);
13556 ix86_expand_convert_uns_sidf_sse (fp_lo, gen_lowpart (SImode, input));
13558 x = expand_simple_binop (DFmode, PLUS, fp_hi, fp_lo, target,
13561 emit_move_insn (target, x);
13564 /* Convert an unsigned SImode value into a SFmode, using only SSE.
13565 For x86_32, -mfpmath=sse, !optimize_size only. */
13567 ix86_expand_convert_uns_sisf_sse (rtx target, rtx input)
13569 REAL_VALUE_TYPE ONE16r;
13570 rtx fp_hi, fp_lo, int_hi, int_lo, x;
13572 real_ldexp (&ONE16r, &dconst1, 16);
13573 x = const_double_from_real_value (ONE16r, SFmode);
13574 int_lo = expand_simple_binop (SImode, AND, input, GEN_INT(0xffff),
13575 NULL, 0, OPTAB_DIRECT);
13576 int_hi = expand_simple_binop (SImode, LSHIFTRT, input, GEN_INT(16),
13577 NULL, 0, OPTAB_DIRECT);
13578 fp_hi = gen_reg_rtx (SFmode);
13579 fp_lo = gen_reg_rtx (SFmode);
13580 emit_insn (gen_floatsisf2 (fp_hi, int_hi));
13581 emit_insn (gen_floatsisf2 (fp_lo, int_lo));
13582 fp_hi = expand_simple_binop (SFmode, MULT, fp_hi, x, fp_hi,
13584 fp_hi = expand_simple_binop (SFmode, PLUS, fp_hi, fp_lo, target,
13586 if (!rtx_equal_p (target, fp_hi))
13587 emit_move_insn (target, fp_hi);
13590 /* A subroutine of ix86_build_signbit_mask. If VECT is true,
13591 then replicate the value for all elements of the vector
13595 ix86_build_const_vector (enum machine_mode mode, bool vect, rtx value)
13602 v = gen_rtvec (4, value, value, value, value);
13603 return gen_rtx_CONST_VECTOR (V4SImode, v);
13607 v = gen_rtvec (2, value, value);
13608 return gen_rtx_CONST_VECTOR (V2DImode, v);
13612 v = gen_rtvec (4, value, value, value, value);
13614 v = gen_rtvec (4, value, CONST0_RTX (SFmode),
13615 CONST0_RTX (SFmode), CONST0_RTX (SFmode));
13616 return gen_rtx_CONST_VECTOR (V4SFmode, v);
13620 v = gen_rtvec (2, value, value);
13622 v = gen_rtvec (2, value, CONST0_RTX (DFmode));
13623 return gen_rtx_CONST_VECTOR (V2DFmode, v);
13626 gcc_unreachable ();
13630 /* A subroutine of ix86_expand_fp_absneg_operator, copysign expanders
13631 and ix86_expand_int_vcond. Create a mask for the sign bit in MODE
13632 for an SSE register. If VECT is true, then replicate the mask for
13633 all elements of the vector register. If INVERT is true, then create
13634 a mask excluding the sign bit. */
13637 ix86_build_signbit_mask (enum machine_mode mode, bool vect, bool invert)
13639 enum machine_mode vec_mode, imode;
13640 HOST_WIDE_INT hi, lo;
13645 /* Find the sign bit, sign extended to 2*HWI. */
13651 vec_mode = (mode == SImode) ? V4SImode : V4SFmode;
13652 lo = 0x80000000, hi = lo < 0;
13658 vec_mode = (mode == DImode) ? V2DImode : V2DFmode;
13659 if (HOST_BITS_PER_WIDE_INT >= 64)
13660 lo = (HOST_WIDE_INT)1 << shift, hi = -1;
13662 lo = 0, hi = (HOST_WIDE_INT)1 << (shift - HOST_BITS_PER_WIDE_INT);
13667 vec_mode = VOIDmode;
13668 if (HOST_BITS_PER_WIDE_INT >= 64)
13671 lo = 0, hi = (HOST_WIDE_INT)1 << shift;
13678 lo = 0, hi = (HOST_WIDE_INT)1 << (shift - HOST_BITS_PER_WIDE_INT);
13682 lo = ~lo, hi = ~hi;
13688 mask = immed_double_const (lo, hi, imode);
13690 vec = gen_rtvec (2, v, mask);
13691 v = gen_rtx_CONST_VECTOR (V2DImode, vec);
13692 v = copy_to_mode_reg (mode, gen_lowpart (mode, v));
13699 gcc_unreachable ();
13703 lo = ~lo, hi = ~hi;
13705 /* Force this value into the low part of a fp vector constant. */
13706 mask = immed_double_const (lo, hi, imode);
13707 mask = gen_lowpart (mode, mask);
13709 if (vec_mode == VOIDmode)
13710 return force_reg (mode, mask);
13712 v = ix86_build_const_vector (mode, vect, mask);
13713 return force_reg (vec_mode, v);
13716 /* Generate code for floating point ABS or NEG. */
13719 ix86_expand_fp_absneg_operator (enum rtx_code code, enum machine_mode mode,
13722 rtx mask, set, use, clob, dst, src;
13723 bool use_sse = false;
13724 bool vector_mode = VECTOR_MODE_P (mode);
13725 enum machine_mode elt_mode = mode;
13729 elt_mode = GET_MODE_INNER (mode);
13732 else if (mode == TFmode)
13734 else if (TARGET_SSE_MATH)
13735 use_sse = SSE_FLOAT_MODE_P (mode);
13737 /* NEG and ABS performed with SSE use bitwise mask operations.
13738 Create the appropriate mask now. */
13740 mask = ix86_build_signbit_mask (elt_mode, vector_mode, code == ABS);
13749 set = gen_rtx_fmt_ee (code == NEG ? XOR : AND, mode, src, mask);
13750 set = gen_rtx_SET (VOIDmode, dst, set);
13755 set = gen_rtx_fmt_e (code, mode, src);
13756 set = gen_rtx_SET (VOIDmode, dst, set);
13759 use = gen_rtx_USE (VOIDmode, mask);
13760 clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG));
13761 emit_insn (gen_rtx_PARALLEL (VOIDmode,
13762 gen_rtvec (3, set, use, clob)));
13769 /* Expand a copysign operation. Special case operand 0 being a constant. */
13772 ix86_expand_copysign (rtx operands[])
13774 enum machine_mode mode;
13775 rtx dest, op0, op1, mask, nmask;
13777 dest = operands[0];
13781 mode = GET_MODE (dest);
13783 if (GET_CODE (op0) == CONST_DOUBLE)
13785 rtx (*copysign_insn)(rtx, rtx, rtx, rtx);
13787 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
13788 op0 = simplify_unary_operation (ABS, mode, op0, mode);
13790 if (mode == SFmode || mode == DFmode)
13792 enum machine_mode vmode;
13794 vmode = mode == SFmode ? V4SFmode : V2DFmode;
13796 if (op0 == CONST0_RTX (mode))
13797 op0 = CONST0_RTX (vmode);
13802 if (mode == SFmode)
13803 v = gen_rtvec (4, op0, CONST0_RTX (SFmode),
13804 CONST0_RTX (SFmode), CONST0_RTX (SFmode));
13806 v = gen_rtvec (2, op0, CONST0_RTX (DFmode));
13808 op0 = force_reg (vmode, gen_rtx_CONST_VECTOR (vmode, v));
13811 else if (op0 != CONST0_RTX (mode))
13812 op0 = force_reg (mode, op0);
13814 mask = ix86_build_signbit_mask (mode, 0, 0);
13816 if (mode == SFmode)
13817 copysign_insn = gen_copysignsf3_const;
13818 else if (mode == DFmode)
13819 copysign_insn = gen_copysigndf3_const;
13821 copysign_insn = gen_copysigntf3_const;
13823 emit_insn (copysign_insn (dest, op0, op1, mask));
13827 rtx (*copysign_insn)(rtx, rtx, rtx, rtx, rtx, rtx);
13829 nmask = ix86_build_signbit_mask (mode, 0, 1);
13830 mask = ix86_build_signbit_mask (mode, 0, 0);
13832 if (mode == SFmode)
13833 copysign_insn = gen_copysignsf3_var;
13834 else if (mode == DFmode)
13835 copysign_insn = gen_copysigndf3_var;
13837 copysign_insn = gen_copysigntf3_var;
13839 emit_insn (copysign_insn (dest, NULL_RTX, op0, op1, nmask, mask));
13843 /* Deconstruct a copysign operation into bit masks. Operand 0 is known to
13844 be a constant, and so has already been expanded into a vector constant. */
13847 ix86_split_copysign_const (rtx operands[])
13849 enum machine_mode mode, vmode;
13850 rtx dest, op0, op1, mask, x;
13852 dest = operands[0];
13855 mask = operands[3];
13857 mode = GET_MODE (dest);
13858 vmode = GET_MODE (mask);
13860 dest = simplify_gen_subreg (vmode, dest, mode, 0);
13861 x = gen_rtx_AND (vmode, dest, mask);
13862 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
13864 if (op0 != CONST0_RTX (vmode))
13866 x = gen_rtx_IOR (vmode, dest, op0);
13867 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
13871 /* Deconstruct a copysign operation into bit masks. Operand 0 is variable,
13872 so we have to do two masks. */
13875 ix86_split_copysign_var (rtx operands[])
13877 enum machine_mode mode, vmode;
13878 rtx dest, scratch, op0, op1, mask, nmask, x;
13880 dest = operands[0];
13881 scratch = operands[1];
13884 nmask = operands[4];
13885 mask = operands[5];
13887 mode = GET_MODE (dest);
13888 vmode = GET_MODE (mask);
13890 if (rtx_equal_p (op0, op1))
13892 /* Shouldn't happen often (it's useless, obviously), but when it does
13893 we'd generate incorrect code if we continue below. */
13894 emit_move_insn (dest, op0);
13898 if (REG_P (mask) && REGNO (dest) == REGNO (mask)) /* alternative 0 */
13900 gcc_assert (REGNO (op1) == REGNO (scratch));
13902 x = gen_rtx_AND (vmode, scratch, mask);
13903 emit_insn (gen_rtx_SET (VOIDmode, scratch, x));
13906 op0 = simplify_gen_subreg (vmode, op0, mode, 0);
13907 x = gen_rtx_NOT (vmode, dest);
13908 x = gen_rtx_AND (vmode, x, op0);
13909 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
13913 if (REGNO (op1) == REGNO (scratch)) /* alternative 1,3 */
13915 x = gen_rtx_AND (vmode, scratch, mask);
13917 else /* alternative 2,4 */
13919 gcc_assert (REGNO (mask) == REGNO (scratch));
13920 op1 = simplify_gen_subreg (vmode, op1, mode, 0);
13921 x = gen_rtx_AND (vmode, scratch, op1);
13923 emit_insn (gen_rtx_SET (VOIDmode, scratch, x));
13925 if (REGNO (op0) == REGNO (dest)) /* alternative 1,2 */
13927 dest = simplify_gen_subreg (vmode, op0, mode, 0);
13928 x = gen_rtx_AND (vmode, dest, nmask);
13930 else /* alternative 3,4 */
13932 gcc_assert (REGNO (nmask) == REGNO (dest));
13934 op0 = simplify_gen_subreg (vmode, op0, mode, 0);
13935 x = gen_rtx_AND (vmode, dest, op0);
13937 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
13940 x = gen_rtx_IOR (vmode, dest, scratch);
13941 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
13944 /* Return TRUE or FALSE depending on whether the first SET in INSN
13945 has source and destination with matching CC modes, and that the
13946 CC mode is at least as constrained as REQ_MODE. */
13949 ix86_match_ccmode (rtx insn, enum machine_mode req_mode)
13952 enum machine_mode set_mode;
13954 set = PATTERN (insn);
13955 if (GET_CODE (set) == PARALLEL)
13956 set = XVECEXP (set, 0, 0);
13957 gcc_assert (GET_CODE (set) == SET);
13958 gcc_assert (GET_CODE (SET_SRC (set)) == COMPARE);
13960 set_mode = GET_MODE (SET_DEST (set));
13964 if (req_mode != CCNOmode
13965 && (req_mode != CCmode
13966 || XEXP (SET_SRC (set), 1) != const0_rtx))
13970 if (req_mode == CCGCmode)
13974 if (req_mode == CCGOCmode || req_mode == CCNOmode)
13978 if (req_mode == CCZmode)
13989 gcc_unreachable ();
13992 return (GET_MODE (SET_SRC (set)) == set_mode);
13995 /* Generate insn patterns to do an integer compare of OPERANDS. */
13998 ix86_expand_int_compare (enum rtx_code code, rtx op0, rtx op1)
14000 enum machine_mode cmpmode;
14003 cmpmode = SELECT_CC_MODE (code, op0, op1);
14004 flags = gen_rtx_REG (cmpmode, FLAGS_REG);
14006 /* This is very simple, but making the interface the same as in the
14007 FP case makes the rest of the code easier. */
14008 tmp = gen_rtx_COMPARE (cmpmode, op0, op1);
14009 emit_insn (gen_rtx_SET (VOIDmode, flags, tmp));
14011 /* Return the test that should be put into the flags user, i.e.
14012 the bcc, scc, or cmov instruction. */
14013 return gen_rtx_fmt_ee (code, VOIDmode, flags, const0_rtx);
14016 /* Figure out whether to use ordered or unordered fp comparisons.
14017 Return the appropriate mode to use. */
14020 ix86_fp_compare_mode (enum rtx_code code ATTRIBUTE_UNUSED)
14022 /* ??? In order to make all comparisons reversible, we do all comparisons
14023 non-trapping when compiling for IEEE. Once gcc is able to distinguish
14024 all forms trapping and nontrapping comparisons, we can make inequality
14025 comparisons trapping again, since it results in better code when using
14026 FCOM based compares. */
14027 return TARGET_IEEE_FP ? CCFPUmode : CCFPmode;
14031 ix86_cc_mode (enum rtx_code code, rtx op0, rtx op1)
14033 enum machine_mode mode = GET_MODE (op0);
14035 if (SCALAR_FLOAT_MODE_P (mode))
14037 gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));
14038 return ix86_fp_compare_mode (code);
14043 /* Only zero flag is needed. */
14044 case EQ: /* ZF=0 */
14045 case NE: /* ZF!=0 */
14047 /* Codes needing carry flag. */
14048 case GEU: /* CF=0 */
14049 case LTU: /* CF=1 */
14050 /* Detect overflow checks. They need just the carry flag. */
14051 if (GET_CODE (op0) == PLUS
14052 && rtx_equal_p (op1, XEXP (op0, 0)))
14056 case GTU: /* CF=0 & ZF=0 */
14057 case LEU: /* CF=1 | ZF=1 */
14058 /* Detect overflow checks. They need just the carry flag. */
14059 if (GET_CODE (op0) == MINUS
14060 && rtx_equal_p (op1, XEXP (op0, 0)))
14064 /* Codes possibly doable only with sign flag when
14065 comparing against zero. */
14066 case GE: /* SF=OF or SF=0 */
14067 case LT: /* SF<>OF or SF=1 */
14068 if (op1 == const0_rtx)
14071 /* For other cases Carry flag is not required. */
14073 /* Codes doable only with sign flag when comparing
14074 against zero, but we miss jump instruction for it
14075 so we need to use relational tests against overflow
14076 that thus needs to be zero. */
14077 case GT: /* ZF=0 & SF=OF */
14078 case LE: /* ZF=1 | SF<>OF */
14079 if (op1 == const0_rtx)
14083 /* strcmp pattern do (use flags) and combine may ask us for proper
14088 gcc_unreachable ();
14092 /* Return the fixed registers used for condition codes. */
14095 ix86_fixed_condition_code_regs (unsigned int *p1, unsigned int *p2)
14102 /* If two condition code modes are compatible, return a condition code
14103 mode which is compatible with both. Otherwise, return
14106 static enum machine_mode
14107 ix86_cc_modes_compatible (enum machine_mode m1, enum machine_mode m2)
14112 if (GET_MODE_CLASS (m1) != MODE_CC || GET_MODE_CLASS (m2) != MODE_CC)
14115 if ((m1 == CCGCmode && m2 == CCGOCmode)
14116 || (m1 == CCGOCmode && m2 == CCGCmode))
14122 gcc_unreachable ();
14152 /* These are only compatible with themselves, which we already
14158 /* Split comparison code CODE into comparisons we can do using branch
14159 instructions. BYPASS_CODE is comparison code for branch that will
14160 branch around FIRST_CODE and SECOND_CODE. If some of branches
14161 is not required, set value to UNKNOWN.
14162 We never require more than two branches. */
14165 ix86_fp_comparison_codes (enum rtx_code code, enum rtx_code *bypass_code,
14166 enum rtx_code *first_code,
14167 enum rtx_code *second_code)
14169 *first_code = code;
14170 *bypass_code = UNKNOWN;
14171 *second_code = UNKNOWN;
14173 /* The fcomi comparison sets flags as follows:
14183 case GT: /* GTU - CF=0 & ZF=0 */
14184 case GE: /* GEU - CF=0 */
14185 case ORDERED: /* PF=0 */
14186 case UNORDERED: /* PF=1 */
14187 case UNEQ: /* EQ - ZF=1 */
14188 case UNLT: /* LTU - CF=1 */
14189 case UNLE: /* LEU - CF=1 | ZF=1 */
14190 case LTGT: /* EQ - ZF=0 */
14192 case LT: /* LTU - CF=1 - fails on unordered */
14193 *first_code = UNLT;
14194 *bypass_code = UNORDERED;
14196 case LE: /* LEU - CF=1 | ZF=1 - fails on unordered */
14197 *first_code = UNLE;
14198 *bypass_code = UNORDERED;
14200 case EQ: /* EQ - ZF=1 - fails on unordered */
14201 *first_code = UNEQ;
14202 *bypass_code = UNORDERED;
14204 case NE: /* NE - ZF=0 - fails on unordered */
14205 *first_code = LTGT;
14206 *second_code = UNORDERED;
14208 case UNGE: /* GEU - CF=0 - fails on unordered */
14210 *second_code = UNORDERED;
14212 case UNGT: /* GTU - CF=0 & ZF=0 - fails on unordered */
14214 *second_code = UNORDERED;
14217 gcc_unreachable ();
14219 if (!TARGET_IEEE_FP)
14221 *second_code = UNKNOWN;
14222 *bypass_code = UNKNOWN;
14226 /* Return cost of comparison done fcom + arithmetics operations on AX.
14227 All following functions do use number of instructions as a cost metrics.
14228 In future this should be tweaked to compute bytes for optimize_size and
14229 take into account performance of various instructions on various CPUs. */
14231 ix86_fp_comparison_arithmetics_cost (enum rtx_code code)
14233 if (!TARGET_IEEE_FP)
14235 /* The cost of code output by ix86_expand_fp_compare. */
14259 gcc_unreachable ();
14263 /* Return cost of comparison done using fcomi operation.
14264 See ix86_fp_comparison_arithmetics_cost for the metrics. */
14266 ix86_fp_comparison_fcomi_cost (enum rtx_code code)
14268 enum rtx_code bypass_code, first_code, second_code;
14269 /* Return arbitrarily high cost when instruction is not supported - this
14270 prevents gcc from using it. */
14273 ix86_fp_comparison_codes (code, &bypass_code, &first_code, &second_code);
14274 return (bypass_code != UNKNOWN || second_code != UNKNOWN) + 2;
14277 /* Return cost of comparison done using sahf operation.
14278 See ix86_fp_comparison_arithmetics_cost for the metrics. */
14280 ix86_fp_comparison_sahf_cost (enum rtx_code code)
14282 enum rtx_code bypass_code, first_code, second_code;
14283 /* Return arbitrarily high cost when instruction is not preferred - this
14284 avoids gcc from using it. */
14285 if (!(TARGET_SAHF && (TARGET_USE_SAHF || optimize_insn_for_size_p ())))
14287 ix86_fp_comparison_codes (code, &bypass_code, &first_code, &second_code);
14288 return (bypass_code != UNKNOWN || second_code != UNKNOWN) + 3;
14291 /* Compute cost of the comparison done using any method.
14292 See ix86_fp_comparison_arithmetics_cost for the metrics. */
14294 ix86_fp_comparison_cost (enum rtx_code code)
14296 int fcomi_cost, sahf_cost, arithmetics_cost = 1024;
14299 fcomi_cost = ix86_fp_comparison_fcomi_cost (code);
14300 sahf_cost = ix86_fp_comparison_sahf_cost (code);
14302 min = arithmetics_cost = ix86_fp_comparison_arithmetics_cost (code);
14303 if (min > sahf_cost)
14305 if (min > fcomi_cost)
14310 /* Return true if we should use an FCOMI instruction for this
14314 ix86_use_fcomi_compare (enum rtx_code code ATTRIBUTE_UNUSED)
14316 enum rtx_code swapped_code = swap_condition (code);
14318 return ((ix86_fp_comparison_cost (code)
14319 == ix86_fp_comparison_fcomi_cost (code))
14320 || (ix86_fp_comparison_cost (swapped_code)
14321 == ix86_fp_comparison_fcomi_cost (swapped_code)));
14324 /* Swap, force into registers, or otherwise massage the two operands
14325 to a fp comparison. The operands are updated in place; the new
14326 comparison code is returned. */
14328 static enum rtx_code
14329 ix86_prepare_fp_compare_args (enum rtx_code code, rtx *pop0, rtx *pop1)
14331 enum machine_mode fpcmp_mode = ix86_fp_compare_mode (code);
14332 rtx op0 = *pop0, op1 = *pop1;
14333 enum machine_mode op_mode = GET_MODE (op0);
14334 int is_sse = TARGET_SSE_MATH && SSE_FLOAT_MODE_P (op_mode);
14336 /* All of the unordered compare instructions only work on registers.
14337 The same is true of the fcomi compare instructions. The XFmode
14338 compare instructions require registers except when comparing
14339 against zero or when converting operand 1 from fixed point to
14343 && (fpcmp_mode == CCFPUmode
14344 || (op_mode == XFmode
14345 && ! (standard_80387_constant_p (op0) == 1
14346 || standard_80387_constant_p (op1) == 1)
14347 && GET_CODE (op1) != FLOAT)
14348 || ix86_use_fcomi_compare (code)))
14350 op0 = force_reg (op_mode, op0);
14351 op1 = force_reg (op_mode, op1);
14355 /* %%% We only allow op1 in memory; op0 must be st(0). So swap
14356 things around if they appear profitable, otherwise force op0
14357 into a register. */
14359 if (standard_80387_constant_p (op0) == 0
14361 && ! (standard_80387_constant_p (op1) == 0
14365 tmp = op0, op0 = op1, op1 = tmp;
14366 code = swap_condition (code);
14370 op0 = force_reg (op_mode, op0);
14372 if (CONSTANT_P (op1))
14374 int tmp = standard_80387_constant_p (op1);
14376 op1 = validize_mem (force_const_mem (op_mode, op1));
14380 op1 = force_reg (op_mode, op1);
14383 op1 = force_reg (op_mode, op1);
14387 /* Try to rearrange the comparison to make it cheaper. */
14388 if (ix86_fp_comparison_cost (code)
14389 > ix86_fp_comparison_cost (swap_condition (code))
14390 && (REG_P (op1) || can_create_pseudo_p ()))
14393 tmp = op0, op0 = op1, op1 = tmp;
14394 code = swap_condition (code);
14396 op0 = force_reg (op_mode, op0);
14404 /* Convert comparison codes we use to represent FP comparison to integer
14405 code that will result in proper branch. Return UNKNOWN if no such code
14409 ix86_fp_compare_code_to_integer (enum rtx_code code)
14438 /* Generate insn patterns to do a floating point compare of OPERANDS. */
14441 ix86_expand_fp_compare (enum rtx_code code, rtx op0, rtx op1, rtx scratch,
14442 rtx *second_test, rtx *bypass_test)
14444 enum machine_mode fpcmp_mode, intcmp_mode;
14446 int cost = ix86_fp_comparison_cost (code);
14447 enum rtx_code bypass_code, first_code, second_code;
14449 fpcmp_mode = ix86_fp_compare_mode (code);
14450 code = ix86_prepare_fp_compare_args (code, &op0, &op1);
14453 *second_test = NULL_RTX;
14455 *bypass_test = NULL_RTX;
14457 ix86_fp_comparison_codes (code, &bypass_code, &first_code, &second_code);
14459 /* Do fcomi/sahf based test when profitable. */
14460 if (ix86_fp_comparison_arithmetics_cost (code) > cost
14461 && (bypass_code == UNKNOWN || bypass_test)
14462 && (second_code == UNKNOWN || second_test))
14464 tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1);
14465 tmp = gen_rtx_SET (VOIDmode, gen_rtx_REG (fpcmp_mode, FLAGS_REG),
14471 gcc_assert (TARGET_SAHF);
14474 scratch = gen_reg_rtx (HImode);
14475 tmp2 = gen_rtx_CLOBBER (VOIDmode, scratch);
14477 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, tmp2)));
14480 /* The FP codes work out to act like unsigned. */
14481 intcmp_mode = fpcmp_mode;
14483 if (bypass_code != UNKNOWN)
14484 *bypass_test = gen_rtx_fmt_ee (bypass_code, VOIDmode,
14485 gen_rtx_REG (intcmp_mode, FLAGS_REG),
14487 if (second_code != UNKNOWN)
14488 *second_test = gen_rtx_fmt_ee (second_code, VOIDmode,
14489 gen_rtx_REG (intcmp_mode, FLAGS_REG),
14494 /* Sadness wrt reg-stack pops killing fpsr -- gotta get fnstsw first. */
14495 tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1);
14496 tmp2 = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW);
14498 scratch = gen_reg_rtx (HImode);
14499 emit_insn (gen_rtx_SET (VOIDmode, scratch, tmp2));
14501 /* In the unordered case, we have to check C2 for NaN's, which
14502 doesn't happen to work out to anything nice combination-wise.
14503 So do some bit twiddling on the value we've got in AH to come
14504 up with an appropriate set of condition codes. */
14506 intcmp_mode = CCNOmode;
14511 if (code == GT || !TARGET_IEEE_FP)
14513 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x45)));
14518 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
14519 emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
14520 emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x44)));
14521 intcmp_mode = CCmode;
14527 if (code == LT && TARGET_IEEE_FP)
14529 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
14530 emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x01)));
14531 intcmp_mode = CCmode;
14536 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x01)));
14542 if (code == GE || !TARGET_IEEE_FP)
14544 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x05)));
14549 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
14550 emit_insn (gen_xorqi_cc_ext_1 (scratch, scratch,
14557 if (code == LE && TARGET_IEEE_FP)
14559 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
14560 emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx));
14561 emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
14562 intcmp_mode = CCmode;
14567 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x45)));
14573 if (code == EQ && TARGET_IEEE_FP)
14575 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
14576 emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40)));
14577 intcmp_mode = CCmode;
14582 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x40)));
14589 if (code == NE && TARGET_IEEE_FP)
14591 emit_insn (gen_andqi_ext_0 (scratch, scratch, GEN_INT (0x45)));
14592 emit_insn (gen_xorqi_cc_ext_1 (scratch, scratch,
14598 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x40)));
14604 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x04)));
14608 emit_insn (gen_testqi_ext_ccno_0 (scratch, GEN_INT (0x04)));
14613 gcc_unreachable ();
14617 /* Return the test that should be put into the flags user, i.e.
14618 the bcc, scc, or cmov instruction. */
14619 return gen_rtx_fmt_ee (code, VOIDmode,
14620 gen_rtx_REG (intcmp_mode, FLAGS_REG),
14625 ix86_expand_compare (enum rtx_code code, rtx *second_test, rtx *bypass_test)
14628 op0 = ix86_compare_op0;
14629 op1 = ix86_compare_op1;
14632 *second_test = NULL_RTX;
14634 *bypass_test = NULL_RTX;
14636 if (GET_MODE_CLASS (GET_MODE (ix86_compare_op0)) == MODE_CC)
14637 ret = gen_rtx_fmt_ee (code, VOIDmode, ix86_compare_op0, ix86_compare_op1);
14639 else if (SCALAR_FLOAT_MODE_P (GET_MODE (op0)))
14641 gcc_assert (!DECIMAL_FLOAT_MODE_P (GET_MODE (op0)));
14642 ret = ix86_expand_fp_compare (code, op0, op1, NULL_RTX,
14643 second_test, bypass_test);
14646 ret = ix86_expand_int_compare (code, op0, op1);
14651 /* Return true if the CODE will result in nontrivial jump sequence. */
14653 ix86_fp_jump_nontrivial_p (enum rtx_code code)
14655 enum rtx_code bypass_code, first_code, second_code;
14658 ix86_fp_comparison_codes (code, &bypass_code, &first_code, &second_code);
14659 return bypass_code != UNKNOWN || second_code != UNKNOWN;
14663 ix86_expand_branch (enum rtx_code code, rtx label)
14667 switch (GET_MODE (ix86_compare_op0))
14673 tmp = ix86_expand_compare (code, NULL, NULL);
14674 tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
14675 gen_rtx_LABEL_REF (VOIDmode, label),
14677 emit_jump_insn (gen_rtx_SET (VOIDmode, pc_rtx, tmp));
14686 enum rtx_code bypass_code, first_code, second_code;
14688 code = ix86_prepare_fp_compare_args (code, &ix86_compare_op0,
14689 &ix86_compare_op1);
14691 ix86_fp_comparison_codes (code, &bypass_code, &first_code, &second_code);
14693 /* Check whether we will use the natural sequence with one jump. If
14694 so, we can expand jump early. Otherwise delay expansion by
14695 creating compound insn to not confuse optimizers. */
14696 if (bypass_code == UNKNOWN && second_code == UNKNOWN)
14698 ix86_split_fp_branch (code, ix86_compare_op0, ix86_compare_op1,
14699 gen_rtx_LABEL_REF (VOIDmode, label),
14700 pc_rtx, NULL_RTX, NULL_RTX);
14704 tmp = gen_rtx_fmt_ee (code, VOIDmode,
14705 ix86_compare_op0, ix86_compare_op1);
14706 tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
14707 gen_rtx_LABEL_REF (VOIDmode, label),
14709 tmp = gen_rtx_SET (VOIDmode, pc_rtx, tmp);
14711 use_fcomi = ix86_use_fcomi_compare (code);
14712 vec = rtvec_alloc (3 + !use_fcomi);
14713 RTVEC_ELT (vec, 0) = tmp;
14715 = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCFPmode, FPSR_REG));
14717 = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCFPmode, FLAGS_REG));
14720 = gen_rtx_CLOBBER (VOIDmode, gen_rtx_SCRATCH (HImode));
14722 emit_jump_insn (gen_rtx_PARALLEL (VOIDmode, vec));
14731 /* Expand DImode branch into multiple compare+branch. */
14733 rtx lo[2], hi[2], label2;
14734 enum rtx_code code1, code2, code3;
14735 enum machine_mode submode;
14737 if (CONSTANT_P (ix86_compare_op0) && ! CONSTANT_P (ix86_compare_op1))
14739 tmp = ix86_compare_op0;
14740 ix86_compare_op0 = ix86_compare_op1;
14741 ix86_compare_op1 = tmp;
14742 code = swap_condition (code);
14744 if (GET_MODE (ix86_compare_op0) == DImode)
14746 split_di (&ix86_compare_op0, 1, lo+0, hi+0);
14747 split_di (&ix86_compare_op1, 1, lo+1, hi+1);
14752 split_ti (&ix86_compare_op0, 1, lo+0, hi+0);
14753 split_ti (&ix86_compare_op1, 1, lo+1, hi+1);
14757 /* When comparing for equality, we can use (hi0^hi1)|(lo0^lo1) to
14758 avoid two branches. This costs one extra insn, so disable when
14759 optimizing for size. */
14761 if ((code == EQ || code == NE)
14762 && (!optimize_insn_for_size_p ()
14763 || hi[1] == const0_rtx || lo[1] == const0_rtx))
14768 if (hi[1] != const0_rtx)
14769 xor1 = expand_binop (submode, xor_optab, xor1, hi[1],
14770 NULL_RTX, 0, OPTAB_WIDEN);
14773 if (lo[1] != const0_rtx)
14774 xor0 = expand_binop (submode, xor_optab, xor0, lo[1],
14775 NULL_RTX, 0, OPTAB_WIDEN);
14777 tmp = expand_binop (submode, ior_optab, xor1, xor0,
14778 NULL_RTX, 0, OPTAB_WIDEN);
14780 ix86_compare_op0 = tmp;
14781 ix86_compare_op1 = const0_rtx;
14782 ix86_expand_branch (code, label);
14786 /* Otherwise, if we are doing less-than or greater-or-equal-than,
14787 op1 is a constant and the low word is zero, then we can just
14788 examine the high word. Similarly for low word -1 and
14789 less-or-equal-than or greater-than. */
14791 if (CONST_INT_P (hi[1]))
14794 case LT: case LTU: case GE: case GEU:
14795 if (lo[1] == const0_rtx)
14797 ix86_compare_op0 = hi[0];
14798 ix86_compare_op1 = hi[1];
14799 ix86_expand_branch (code, label);
14803 case LE: case LEU: case GT: case GTU:
14804 if (lo[1] == constm1_rtx)
14806 ix86_compare_op0 = hi[0];
14807 ix86_compare_op1 = hi[1];
14808 ix86_expand_branch (code, label);
14816 /* Otherwise, we need two or three jumps. */
14818 label2 = gen_label_rtx ();
14821 code2 = swap_condition (code);
14822 code3 = unsigned_condition (code);
14826 case LT: case GT: case LTU: case GTU:
14829 case LE: code1 = LT; code2 = GT; break;
14830 case GE: code1 = GT; code2 = LT; break;
14831 case LEU: code1 = LTU; code2 = GTU; break;
14832 case GEU: code1 = GTU; code2 = LTU; break;
14834 case EQ: code1 = UNKNOWN; code2 = NE; break;
14835 case NE: code2 = UNKNOWN; break;
14838 gcc_unreachable ();
14843 * if (hi(a) < hi(b)) goto true;
14844 * if (hi(a) > hi(b)) goto false;
14845 * if (lo(a) < lo(b)) goto true;
14849 ix86_compare_op0 = hi[0];
14850 ix86_compare_op1 = hi[1];
14852 if (code1 != UNKNOWN)
14853 ix86_expand_branch (code1, label);
14854 if (code2 != UNKNOWN)
14855 ix86_expand_branch (code2, label2);
14857 ix86_compare_op0 = lo[0];
14858 ix86_compare_op1 = lo[1];
14859 ix86_expand_branch (code3, label);
14861 if (code2 != UNKNOWN)
14862 emit_label (label2);
14867 /* If we have already emitted a compare insn, go straight to simple.
14868 ix86_expand_compare won't emit anything if ix86_compare_emitted
14870 gcc_assert (GET_MODE_CLASS (GET_MODE (ix86_compare_op0)) == MODE_CC);
14875 /* Split branch based on floating point condition. */
14877 ix86_split_fp_branch (enum rtx_code code, rtx op1, rtx op2,
14878 rtx target1, rtx target2, rtx tmp, rtx pushed)
14880 rtx second, bypass;
14881 rtx label = NULL_RTX;
14883 int bypass_probability = -1, second_probability = -1, probability = -1;
14886 if (target2 != pc_rtx)
14889 code = reverse_condition_maybe_unordered (code);
14894 condition = ix86_expand_fp_compare (code, op1, op2,
14895 tmp, &second, &bypass);
14897 /* Remove pushed operand from stack. */
14899 ix86_free_from_memory (GET_MODE (pushed));
14901 if (split_branch_probability >= 0)
14903 /* Distribute the probabilities across the jumps.
14904 Assume the BYPASS and SECOND to be always test
14906 probability = split_branch_probability;
14908 /* Value of 1 is low enough to make no need for probability
14909 to be updated. Later we may run some experiments and see
14910 if unordered values are more frequent in practice. */
14912 bypass_probability = 1;
14914 second_probability = 1;
14916 if (bypass != NULL_RTX)
14918 label = gen_label_rtx ();
14919 i = emit_jump_insn (gen_rtx_SET
14921 gen_rtx_IF_THEN_ELSE (VOIDmode,
14923 gen_rtx_LABEL_REF (VOIDmode,
14926 if (bypass_probability >= 0)
14927 add_reg_note (i, REG_BR_PROB, GEN_INT (bypass_probability));
14929 i = emit_jump_insn (gen_rtx_SET
14931 gen_rtx_IF_THEN_ELSE (VOIDmode,
14932 condition, target1, target2)));
14933 if (probability >= 0)
14934 add_reg_note (i, REG_BR_PROB, GEN_INT (probability));
14935 if (second != NULL_RTX)
14937 i = emit_jump_insn (gen_rtx_SET
14939 gen_rtx_IF_THEN_ELSE (VOIDmode, second, target1,
14941 if (second_probability >= 0)
14942 add_reg_note (i, REG_BR_PROB, GEN_INT (second_probability));
14944 if (label != NULL_RTX)
14945 emit_label (label);
14949 ix86_expand_setcc (enum rtx_code code, rtx dest)
14951 rtx ret, tmp, tmpreg, equiv;
14952 rtx second_test, bypass_test;
14954 gcc_assert (GET_MODE (dest) == QImode);
14956 ret = ix86_expand_compare (code, &second_test, &bypass_test);
14957 PUT_MODE (ret, QImode);
14962 emit_insn (gen_rtx_SET (VOIDmode, tmp, ret));
14963 if (bypass_test || second_test)
14965 rtx test = second_test;
14967 rtx tmp2 = gen_reg_rtx (QImode);
14970 gcc_assert (!second_test);
14971 test = bypass_test;
14973 PUT_CODE (test, reverse_condition_maybe_unordered (GET_CODE (test)));
14975 PUT_MODE (test, QImode);
14976 emit_insn (gen_rtx_SET (VOIDmode, tmp2, test));
14979 emit_insn (gen_andqi3 (tmp, tmpreg, tmp2));
14981 emit_insn (gen_iorqi3 (tmp, tmpreg, tmp2));
14984 /* Attach a REG_EQUAL note describing the comparison result. */
14985 if (ix86_compare_op0 && ix86_compare_op1)
14987 equiv = simplify_gen_relational (code, QImode,
14988 GET_MODE (ix86_compare_op0),
14989 ix86_compare_op0, ix86_compare_op1);
14990 set_unique_reg_note (get_last_insn (), REG_EQUAL, equiv);
14994 /* Expand comparison setting or clearing carry flag. Return true when
14995 successful and set pop for the operation. */
14997 ix86_expand_carry_flag_compare (enum rtx_code code, rtx op0, rtx op1, rtx *pop)
14999 enum machine_mode mode =
15000 GET_MODE (op0) != VOIDmode ? GET_MODE (op0) : GET_MODE (op1);
15002 /* Do not handle DImode compares that go through special path. */
15003 if (mode == (TARGET_64BIT ? TImode : DImode))
15006 if (SCALAR_FLOAT_MODE_P (mode))
15008 rtx second_test = NULL, bypass_test = NULL;
15009 rtx compare_op, compare_seq;
15011 gcc_assert (!DECIMAL_FLOAT_MODE_P (mode));
15013 /* Shortcut: following common codes never translate
15014 into carry flag compares. */
15015 if (code == EQ || code == NE || code == UNEQ || code == LTGT
15016 || code == ORDERED || code == UNORDERED)
15019 /* These comparisons require zero flag; swap operands so they won't. */
15020 if ((code == GT || code == UNLE || code == LE || code == UNGT)
15021 && !TARGET_IEEE_FP)
15026 code = swap_condition (code);
15029 /* Try to expand the comparison and verify that we end up with
15030 carry flag based comparison. This fails to be true only when
15031 we decide to expand comparison using arithmetic that is not
15032 too common scenario. */
15034 compare_op = ix86_expand_fp_compare (code, op0, op1, NULL_RTX,
15035 &second_test, &bypass_test);
15036 compare_seq = get_insns ();
15039 if (second_test || bypass_test)
15042 if (GET_MODE (XEXP (compare_op, 0)) == CCFPmode
15043 || GET_MODE (XEXP (compare_op, 0)) == CCFPUmode)
15044 code = ix86_fp_compare_code_to_integer (GET_CODE (compare_op));
15046 code = GET_CODE (compare_op);
15048 if (code != LTU && code != GEU)
15051 emit_insn (compare_seq);
15056 if (!INTEGRAL_MODE_P (mode))
15065 /* Convert a==0 into (unsigned)a<1. */
15068 if (op1 != const0_rtx)
15071 code = (code == EQ ? LTU : GEU);
15074 /* Convert a>b into b<a or a>=b-1. */
15077 if (CONST_INT_P (op1))
15079 op1 = gen_int_mode (INTVAL (op1) + 1, GET_MODE (op0));
15080 /* Bail out on overflow. We still can swap operands but that
15081 would force loading of the constant into register. */
15082 if (op1 == const0_rtx
15083 || !x86_64_immediate_operand (op1, GET_MODE (op1)))
15085 code = (code == GTU ? GEU : LTU);
15092 code = (code == GTU ? LTU : GEU);
15096 /* Convert a>=0 into (unsigned)a<0x80000000. */
15099 if (mode == DImode || op1 != const0_rtx)
15101 op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
15102 code = (code == LT ? GEU : LTU);
15106 if (mode == DImode || op1 != constm1_rtx)
15108 op1 = gen_int_mode (1 << (GET_MODE_BITSIZE (mode) - 1), mode);
15109 code = (code == LE ? GEU : LTU);
15115 /* Swapping operands may cause constant to appear as first operand. */
15116 if (!nonimmediate_operand (op0, VOIDmode))
15118 if (!can_create_pseudo_p ())
15120 op0 = force_reg (mode, op0);
15122 ix86_compare_op0 = op0;
15123 ix86_compare_op1 = op1;
15124 *pop = ix86_expand_compare (code, NULL, NULL);
15125 gcc_assert (GET_CODE (*pop) == LTU || GET_CODE (*pop) == GEU);
15130 ix86_expand_int_movcc (rtx operands[])
15132 enum rtx_code code = GET_CODE (operands[1]), compare_code;
15133 rtx compare_seq, compare_op;
15134 rtx second_test, bypass_test;
15135 enum machine_mode mode = GET_MODE (operands[0]);
15136 bool sign_bit_compare_p = false;;
15139 ix86_compare_op0 = XEXP (operands[1], 0);
15140 ix86_compare_op1 = XEXP (operands[1], 1);
15141 compare_op = ix86_expand_compare (code, &second_test, &bypass_test);
15142 compare_seq = get_insns ();
15145 compare_code = GET_CODE (compare_op);
15147 if ((ix86_compare_op1 == const0_rtx && (code == GE || code == LT))
15148 || (ix86_compare_op1 == constm1_rtx && (code == GT || code == LE)))
15149 sign_bit_compare_p = true;
15151 /* Don't attempt mode expansion here -- if we had to expand 5 or 6
15152 HImode insns, we'd be swallowed in word prefix ops. */
15154 if ((mode != HImode || TARGET_FAST_PREFIX)
15155 && (mode != (TARGET_64BIT ? TImode : DImode))
15156 && CONST_INT_P (operands[2])
15157 && CONST_INT_P (operands[3]))
15159 rtx out = operands[0];
15160 HOST_WIDE_INT ct = INTVAL (operands[2]);
15161 HOST_WIDE_INT cf = INTVAL (operands[3]);
15162 HOST_WIDE_INT diff;
15165 /* Sign bit compares are better done using shifts than we do by using
15167 if (sign_bit_compare_p
15168 || ix86_expand_carry_flag_compare (code, ix86_compare_op0,
15169 ix86_compare_op1, &compare_op))
15171 /* Detect overlap between destination and compare sources. */
15174 if (!sign_bit_compare_p)
15176 bool fpcmp = false;
15178 compare_code = GET_CODE (compare_op);
15180 if (GET_MODE (XEXP (compare_op, 0)) == CCFPmode
15181 || GET_MODE (XEXP (compare_op, 0)) == CCFPUmode)
15184 compare_code = ix86_fp_compare_code_to_integer (compare_code);
15187 /* To simplify rest of code, restrict to the GEU case. */
15188 if (compare_code == LTU)
15190 HOST_WIDE_INT tmp = ct;
15193 compare_code = reverse_condition (compare_code);
15194 code = reverse_condition (code);
15199 PUT_CODE (compare_op,
15200 reverse_condition_maybe_unordered
15201 (GET_CODE (compare_op)));
15203 PUT_CODE (compare_op, reverse_condition (GET_CODE (compare_op)));
15207 if (reg_overlap_mentioned_p (out, ix86_compare_op0)
15208 || reg_overlap_mentioned_p (out, ix86_compare_op1))
15209 tmp = gen_reg_rtx (mode);
15211 if (mode == DImode)
15212 emit_insn (gen_x86_movdicc_0_m1_rex64 (tmp, compare_op));
15214 emit_insn (gen_x86_movsicc_0_m1 (gen_lowpart (SImode, tmp), compare_op));
15218 if (code == GT || code == GE)
15219 code = reverse_condition (code);
15222 HOST_WIDE_INT tmp = ct;
15227 tmp = emit_store_flag (tmp, code, ix86_compare_op0,
15228 ix86_compare_op1, VOIDmode, 0, -1);
15241 tmp = expand_simple_binop (mode, PLUS,
15243 copy_rtx (tmp), 1, OPTAB_DIRECT);
15254 tmp = expand_simple_binop (mode, IOR,
15256 copy_rtx (tmp), 1, OPTAB_DIRECT);
15258 else if (diff == -1 && ct)
15268 tmp = expand_simple_unop (mode, NOT, tmp, copy_rtx (tmp), 1);
15270 tmp = expand_simple_binop (mode, PLUS,
15271 copy_rtx (tmp), GEN_INT (cf),
15272 copy_rtx (tmp), 1, OPTAB_DIRECT);
15280 * andl cf - ct, dest
15290 tmp = expand_simple_unop (mode, NOT, tmp, copy_rtx (tmp), 1);
15293 tmp = expand_simple_binop (mode, AND,
15295 gen_int_mode (cf - ct, mode),
15296 copy_rtx (tmp), 1, OPTAB_DIRECT);
15298 tmp = expand_simple_binop (mode, PLUS,
15299 copy_rtx (tmp), GEN_INT (ct),
15300 copy_rtx (tmp), 1, OPTAB_DIRECT);
15303 if (!rtx_equal_p (tmp, out))
15304 emit_move_insn (copy_rtx (out), copy_rtx (tmp));
15306 return 1; /* DONE */
15311 enum machine_mode cmp_mode = GET_MODE (ix86_compare_op0);
15314 tmp = ct, ct = cf, cf = tmp;
15317 if (SCALAR_FLOAT_MODE_P (cmp_mode))
15319 gcc_assert (!DECIMAL_FLOAT_MODE_P (cmp_mode));
15321 /* We may be reversing unordered compare to normal compare, that
15322 is not valid in general (we may convert non-trapping condition
15323 to trapping one), however on i386 we currently emit all
15324 comparisons unordered. */
15325 compare_code = reverse_condition_maybe_unordered (compare_code);
15326 code = reverse_condition_maybe_unordered (code);
15330 compare_code = reverse_condition (compare_code);
15331 code = reverse_condition (code);
15335 compare_code = UNKNOWN;
15336 if (GET_MODE_CLASS (GET_MODE (ix86_compare_op0)) == MODE_INT
15337 && CONST_INT_P (ix86_compare_op1))
15339 if (ix86_compare_op1 == const0_rtx
15340 && (code == LT || code == GE))
15341 compare_code = code;
15342 else if (ix86_compare_op1 == constm1_rtx)
15346 else if (code == GT)
15351 /* Optimize dest = (op0 < 0) ? -1 : cf. */
15352 if (compare_code != UNKNOWN
15353 && GET_MODE (ix86_compare_op0) == GET_MODE (out)
15354 && (cf == -1 || ct == -1))
15356 /* If lea code below could be used, only optimize
15357 if it results in a 2 insn sequence. */
15359 if (! (diff == 1 || diff == 2 || diff == 4 || diff == 8
15360 || diff == 3 || diff == 5 || diff == 9)
15361 || (compare_code == LT && ct == -1)
15362 || (compare_code == GE && cf == -1))
15365 * notl op1 (if necessary)
15373 code = reverse_condition (code);
15376 out = emit_store_flag (out, code, ix86_compare_op0,
15377 ix86_compare_op1, VOIDmode, 0, -1);
15379 out = expand_simple_binop (mode, IOR,
15381 out, 1, OPTAB_DIRECT);
15382 if (out != operands[0])
15383 emit_move_insn (operands[0], out);
15385 return 1; /* DONE */
15390 if ((diff == 1 || diff == 2 || diff == 4 || diff == 8
15391 || diff == 3 || diff == 5 || diff == 9)
15392 && ((mode != QImode && mode != HImode) || !TARGET_PARTIAL_REG_STALL)
15394 || x86_64_immediate_operand (GEN_INT (cf), VOIDmode)))
15400 * lea cf(dest*(ct-cf)),dest
15404 * This also catches the degenerate setcc-only case.
15410 out = emit_store_flag (out, code, ix86_compare_op0,
15411 ix86_compare_op1, VOIDmode, 0, 1);
15414 /* On x86_64 the lea instruction operates on Pmode, so we need
15415 to get arithmetics done in proper mode to match. */
15417 tmp = copy_rtx (out);
15421 out1 = copy_rtx (out);
15422 tmp = gen_rtx_MULT (mode, out1, GEN_INT (diff & ~1));
15426 tmp = gen_rtx_PLUS (mode, tmp, out1);
15432 tmp = gen_rtx_PLUS (mode, tmp, GEN_INT (cf));
15435 if (!rtx_equal_p (tmp, out))
15438 out = force_operand (tmp, copy_rtx (out));
15440 emit_insn (gen_rtx_SET (VOIDmode, copy_rtx (out), copy_rtx (tmp)));
15442 if (!rtx_equal_p (out, operands[0]))
15443 emit_move_insn (operands[0], copy_rtx (out));
15445 return 1; /* DONE */
15449 * General case: Jumpful:
15450 * xorl dest,dest cmpl op1, op2
15451 * cmpl op1, op2 movl ct, dest
15452 * setcc dest jcc 1f
15453 * decl dest movl cf, dest
15454 * andl (cf-ct),dest 1:
15457 * Size 20. Size 14.
15459 * This is reasonably steep, but branch mispredict costs are
15460 * high on modern cpus, so consider failing only if optimizing
15464 if ((!TARGET_CMOVE || (mode == QImode && TARGET_PARTIAL_REG_STALL))
15465 && BRANCH_COST (optimize_insn_for_speed_p (),
15470 enum machine_mode cmp_mode = GET_MODE (ix86_compare_op0);
15475 if (SCALAR_FLOAT_MODE_P (cmp_mode))
15477 gcc_assert (!DECIMAL_FLOAT_MODE_P (cmp_mode));
15479 /* We may be reversing unordered compare to normal compare,
15480 that is not valid in general (we may convert non-trapping
15481 condition to trapping one), however on i386 we currently
15482 emit all comparisons unordered. */
15483 code = reverse_condition_maybe_unordered (code);
15487 code = reverse_condition (code);
15488 if (compare_code != UNKNOWN)
15489 compare_code = reverse_condition (compare_code);
15493 if (compare_code != UNKNOWN)
15495 /* notl op1 (if needed)
15500 For x < 0 (resp. x <= -1) there will be no notl,
15501 so if possible swap the constants to get rid of the
15503 True/false will be -1/0 while code below (store flag
15504 followed by decrement) is 0/-1, so the constants need
15505 to be exchanged once more. */
15507 if (compare_code == GE || !cf)
15509 code = reverse_condition (code);
15514 HOST_WIDE_INT tmp = cf;
15519 out = emit_store_flag (out, code, ix86_compare_op0,
15520 ix86_compare_op1, VOIDmode, 0, -1);
15524 out = emit_store_flag (out, code, ix86_compare_op0,
15525 ix86_compare_op1, VOIDmode, 0, 1);
15527 out = expand_simple_binop (mode, PLUS, copy_rtx (out), constm1_rtx,
15528 copy_rtx (out), 1, OPTAB_DIRECT);
15531 out = expand_simple_binop (mode, AND, copy_rtx (out),
15532 gen_int_mode (cf - ct, mode),
15533 copy_rtx (out), 1, OPTAB_DIRECT);
15535 out = expand_simple_binop (mode, PLUS, copy_rtx (out), GEN_INT (ct),
15536 copy_rtx (out), 1, OPTAB_DIRECT);
15537 if (!rtx_equal_p (out, operands[0]))
15538 emit_move_insn (operands[0], copy_rtx (out));
15540 return 1; /* DONE */
15544 if (!TARGET_CMOVE || (mode == QImode && TARGET_PARTIAL_REG_STALL))
15546 /* Try a few things more with specific constants and a variable. */
15549 rtx var, orig_out, out, tmp;
15551 if (BRANCH_COST (optimize_insn_for_speed_p (), false) <= 2)
15552 return 0; /* FAIL */
15554 /* If one of the two operands is an interesting constant, load a
15555 constant with the above and mask it in with a logical operation. */
15557 if (CONST_INT_P (operands[2]))
15560 if (INTVAL (operands[2]) == 0 && operands[3] != constm1_rtx)
15561 operands[3] = constm1_rtx, op = and_optab;
15562 else if (INTVAL (operands[2]) == -1 && operands[3] != const0_rtx)
15563 operands[3] = const0_rtx, op = ior_optab;
15565 return 0; /* FAIL */
15567 else if (CONST_INT_P (operands[3]))
15570 if (INTVAL (operands[3]) == 0 && operands[2] != constm1_rtx)
15571 operands[2] = constm1_rtx, op = and_optab;
15572 else if (INTVAL (operands[3]) == -1 && operands[3] != const0_rtx)
15573 operands[2] = const0_rtx, op = ior_optab;
15575 return 0; /* FAIL */
15578 return 0; /* FAIL */
15580 orig_out = operands[0];
15581 tmp = gen_reg_rtx (mode);
15584 /* Recurse to get the constant loaded. */
15585 if (ix86_expand_int_movcc (operands) == 0)
15586 return 0; /* FAIL */
15588 /* Mask in the interesting variable. */
15589 out = expand_binop (mode, op, var, tmp, orig_out, 0,
15591 if (!rtx_equal_p (out, orig_out))
15592 emit_move_insn (copy_rtx (orig_out), copy_rtx (out));
15594 return 1; /* DONE */
15598 * For comparison with above,
15608 if (! nonimmediate_operand (operands[2], mode))
15609 operands[2] = force_reg (mode, operands[2]);
15610 if (! nonimmediate_operand (operands[3], mode))
15611 operands[3] = force_reg (mode, operands[3]);
15613 if (bypass_test && reg_overlap_mentioned_p (operands[0], operands[3]))
15615 rtx tmp = gen_reg_rtx (mode);
15616 emit_move_insn (tmp, operands[3]);
15619 if (second_test && reg_overlap_mentioned_p (operands[0], operands[2]))
15621 rtx tmp = gen_reg_rtx (mode);
15622 emit_move_insn (tmp, operands[2]);
15626 if (! register_operand (operands[2], VOIDmode)
15628 || ! register_operand (operands[3], VOIDmode)))
15629 operands[2] = force_reg (mode, operands[2]);
15632 && ! register_operand (operands[3], VOIDmode))
15633 operands[3] = force_reg (mode, operands[3]);
15635 emit_insn (compare_seq);
15636 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
15637 gen_rtx_IF_THEN_ELSE (mode,
15638 compare_op, operands[2],
15641 emit_insn (gen_rtx_SET (VOIDmode, copy_rtx (operands[0]),
15642 gen_rtx_IF_THEN_ELSE (mode,
15644 copy_rtx (operands[3]),
15645 copy_rtx (operands[0]))));
15647 emit_insn (gen_rtx_SET (VOIDmode, copy_rtx (operands[0]),
15648 gen_rtx_IF_THEN_ELSE (mode,
15650 copy_rtx (operands[2]),
15651 copy_rtx (operands[0]))));
15653 return 1; /* DONE */
15656 /* Swap, force into registers, or otherwise massage the two operands
15657 to an sse comparison with a mask result. Thus we differ a bit from
15658 ix86_prepare_fp_compare_args which expects to produce a flags result.
15660 The DEST operand exists to help determine whether to commute commutative
15661 operators. The POP0/POP1 operands are updated in place. The new
15662 comparison code is returned, or UNKNOWN if not implementable. */
15664 static enum rtx_code
15665 ix86_prepare_sse_fp_compare_args (rtx dest, enum rtx_code code,
15666 rtx *pop0, rtx *pop1)
15674 /* We have no LTGT as an operator. We could implement it with
15675 NE & ORDERED, but this requires an extra temporary. It's
15676 not clear that it's worth it. */
15683 /* These are supported directly. */
15690 /* For commutative operators, try to canonicalize the destination
15691 operand to be first in the comparison - this helps reload to
15692 avoid extra moves. */
15693 if (!dest || !rtx_equal_p (dest, *pop1))
15701 /* These are not supported directly. Swap the comparison operands
15702 to transform into something that is supported. */
15706 code = swap_condition (code);
15710 gcc_unreachable ();
15716 /* Detect conditional moves that exactly match min/max operational
15717 semantics. Note that this is IEEE safe, as long as we don't
15718 interchange the operands.
15720 Returns FALSE if this conditional move doesn't match a MIN/MAX,
15721 and TRUE if the operation is successful and instructions are emitted. */
15724 ix86_expand_sse_fp_minmax (rtx dest, enum rtx_code code, rtx cmp_op0,
15725 rtx cmp_op1, rtx if_true, rtx if_false)
15727 enum machine_mode mode;
15733 else if (code == UNGE)
15736 if_true = if_false;
15742 if (rtx_equal_p (cmp_op0, if_true) && rtx_equal_p (cmp_op1, if_false))
15744 else if (rtx_equal_p (cmp_op1, if_true) && rtx_equal_p (cmp_op0, if_false))
15749 mode = GET_MODE (dest);
15751 /* We want to check HONOR_NANS and HONOR_SIGNED_ZEROS here,
15752 but MODE may be a vector mode and thus not appropriate. */
15753 if (!flag_finite_math_only || !flag_unsafe_math_optimizations)
15755 int u = is_min ? UNSPEC_IEEE_MIN : UNSPEC_IEEE_MAX;
15758 if_true = force_reg (mode, if_true);
15759 v = gen_rtvec (2, if_true, if_false);
15760 tmp = gen_rtx_UNSPEC (mode, v, u);
15764 code = is_min ? SMIN : SMAX;
15765 tmp = gen_rtx_fmt_ee (code, mode, if_true, if_false);
15768 emit_insn (gen_rtx_SET (VOIDmode, dest, tmp));
15772 /* Expand an sse vector comparison. Return the register with the result. */
15775 ix86_expand_sse_cmp (rtx dest, enum rtx_code code, rtx cmp_op0, rtx cmp_op1,
15776 rtx op_true, rtx op_false)
15778 enum machine_mode mode = GET_MODE (dest);
15781 cmp_op0 = force_reg (mode, cmp_op0);
15782 if (!nonimmediate_operand (cmp_op1, mode))
15783 cmp_op1 = force_reg (mode, cmp_op1);
15786 || reg_overlap_mentioned_p (dest, op_true)
15787 || reg_overlap_mentioned_p (dest, op_false))
15788 dest = gen_reg_rtx (mode);
15790 x = gen_rtx_fmt_ee (code, mode, cmp_op0, cmp_op1);
15791 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
15796 /* Expand DEST = CMP ? OP_TRUE : OP_FALSE into a sequence of logical
15797 operations. This is used for both scalar and vector conditional moves. */
15800 ix86_expand_sse_movcc (rtx dest, rtx cmp, rtx op_true, rtx op_false)
15802 enum machine_mode mode = GET_MODE (dest);
15805 if (op_false == CONST0_RTX (mode))
15807 op_true = force_reg (mode, op_true);
15808 x = gen_rtx_AND (mode, cmp, op_true);
15809 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
15811 else if (op_true == CONST0_RTX (mode))
15813 op_false = force_reg (mode, op_false);
15814 x = gen_rtx_NOT (mode, cmp);
15815 x = gen_rtx_AND (mode, x, op_false);
15816 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
15818 else if (TARGET_SSE5)
15820 rtx pcmov = gen_rtx_SET (mode, dest,
15821 gen_rtx_IF_THEN_ELSE (mode, cmp,
15828 op_true = force_reg (mode, op_true);
15829 op_false = force_reg (mode, op_false);
15831 t2 = gen_reg_rtx (mode);
15833 t3 = gen_reg_rtx (mode);
15837 x = gen_rtx_AND (mode, op_true, cmp);
15838 emit_insn (gen_rtx_SET (VOIDmode, t2, x));
15840 x = gen_rtx_NOT (mode, cmp);
15841 x = gen_rtx_AND (mode, x, op_false);
15842 emit_insn (gen_rtx_SET (VOIDmode, t3, x));
15844 x = gen_rtx_IOR (mode, t3, t2);
15845 emit_insn (gen_rtx_SET (VOIDmode, dest, x));
15849 /* Expand a floating-point conditional move. Return true if successful. */
15852 ix86_expand_fp_movcc (rtx operands[])
15854 enum machine_mode mode = GET_MODE (operands[0]);
15855 enum rtx_code code = GET_CODE (operands[1]);
15856 rtx tmp, compare_op, second_test, bypass_test;
15858 ix86_compare_op0 = XEXP (operands[1], 0);
15859 ix86_compare_op1 = XEXP (operands[1], 1);
15860 if (TARGET_SSE_MATH && SSE_FLOAT_MODE_P (mode))
15862 enum machine_mode cmode;
15864 /* Since we've no cmove for sse registers, don't force bad register
15865 allocation just to gain access to it. Deny movcc when the
15866 comparison mode doesn't match the move mode. */
15867 cmode = GET_MODE (ix86_compare_op0);
15868 if (cmode == VOIDmode)
15869 cmode = GET_MODE (ix86_compare_op1);
15873 code = ix86_prepare_sse_fp_compare_args (operands[0], code,
15875 &ix86_compare_op1);
15876 if (code == UNKNOWN)
15879 if (ix86_expand_sse_fp_minmax (operands[0], code, ix86_compare_op0,
15880 ix86_compare_op1, operands[2],
15884 tmp = ix86_expand_sse_cmp (operands[0], code, ix86_compare_op0,
15885 ix86_compare_op1, operands[2], operands[3]);
15886 ix86_expand_sse_movcc (operands[0], tmp, operands[2], operands[3]);
15890 /* The floating point conditional move instructions don't directly
15891 support conditions resulting from a signed integer comparison. */
15893 compare_op = ix86_expand_compare (code, &second_test, &bypass_test);
15895 /* The floating point conditional move instructions don't directly
15896 support signed integer comparisons. */
15898 if (!fcmov_comparison_operator (compare_op, VOIDmode))
15900 gcc_assert (!second_test && !bypass_test);
15901 tmp = gen_reg_rtx (QImode);
15902 ix86_expand_setcc (code, tmp);
15904 ix86_compare_op0 = tmp;
15905 ix86_compare_op1 = const0_rtx;
15906 compare_op = ix86_expand_compare (code, &second_test, &bypass_test);
15908 if (bypass_test && reg_overlap_mentioned_p (operands[0], operands[3]))
15910 tmp = gen_reg_rtx (mode);
15911 emit_move_insn (tmp, operands[3]);
15914 if (second_test && reg_overlap_mentioned_p (operands[0], operands[2]))
15916 tmp = gen_reg_rtx (mode);
15917 emit_move_insn (tmp, operands[2]);
15921 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
15922 gen_rtx_IF_THEN_ELSE (mode, compare_op,
15923 operands[2], operands[3])));
15925 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
15926 gen_rtx_IF_THEN_ELSE (mode, bypass_test,
15927 operands[3], operands[0])));
15929 emit_insn (gen_rtx_SET (VOIDmode, operands[0],
15930 gen_rtx_IF_THEN_ELSE (mode, second_test,
15931 operands[2], operands[0])));
15936 /* Expand a floating-point vector conditional move; a vcond operation
15937 rather than a movcc operation. */
15940 ix86_expand_fp_vcond (rtx operands[])
15942 enum rtx_code code = GET_CODE (operands[3]);
15945 code = ix86_prepare_sse_fp_compare_args (operands[0], code,
15946 &operands[4], &operands[5]);
15947 if (code == UNKNOWN)
15950 if (ix86_expand_sse_fp_minmax (operands[0], code, operands[4],
15951 operands[5], operands[1], operands[2]))
15954 cmp = ix86_expand_sse_cmp (operands[0], code, operands[4], operands[5],
15955 operands[1], operands[2]);
15956 ix86_expand_sse_movcc (operands[0], cmp, operands[1], operands[2]);
15960 /* Expand a signed/unsigned integral vector conditional move. */
15963 ix86_expand_int_vcond (rtx operands[])
15965 enum machine_mode mode = GET_MODE (operands[0]);
15966 enum rtx_code code = GET_CODE (operands[3]);
15967 bool negate = false;
15970 cop0 = operands[4];
15971 cop1 = operands[5];
15973 /* SSE5 supports all of the comparisons on all vector int types. */
15976 /* Canonicalize the comparison to EQ, GT, GTU. */
15987 code = reverse_condition (code);
15993 code = reverse_condition (code);
15999 code = swap_condition (code);
16000 x = cop0, cop0 = cop1, cop1 = x;
16004 gcc_unreachable ();
16007 /* Only SSE4.1/SSE4.2 supports V2DImode. */
16008 if (mode == V2DImode)
16013 /* SSE4.1 supports EQ. */
16014 if (!TARGET_SSE4_1)
16020 /* SSE4.2 supports GT/GTU. */
16021 if (!TARGET_SSE4_2)
16026 gcc_unreachable ();
16030 /* Unsigned parallel compare is not supported by the hardware. Play some
16031 tricks to turn this into a signed comparison against 0. */
16034 cop0 = force_reg (mode, cop0);
16043 /* Perform a parallel modulo subtraction. */
16044 t1 = gen_reg_rtx (mode);
16045 emit_insn ((mode == V4SImode
16047 : gen_subv2di3) (t1, cop0, cop1));
16049 /* Extract the original sign bit of op0. */
16050 mask = ix86_build_signbit_mask (GET_MODE_INNER (mode),
16052 t2 = gen_reg_rtx (mode);
16053 emit_insn ((mode == V4SImode
16055 : gen_andv2di3) (t2, cop0, mask));
16057 /* XOR it back into the result of the subtraction. This results
16058 in the sign bit set iff we saw unsigned underflow. */
16059 x = gen_reg_rtx (mode);
16060 emit_insn ((mode == V4SImode
16062 : gen_xorv2di3) (x, t1, t2));
16070 /* Perform a parallel unsigned saturating subtraction. */
16071 x = gen_reg_rtx (mode);
16072 emit_insn (gen_rtx_SET (VOIDmode, x,
16073 gen_rtx_US_MINUS (mode, cop0, cop1)));
16080 gcc_unreachable ();
16084 cop1 = CONST0_RTX (mode);
16088 x = ix86_expand_sse_cmp (operands[0], code, cop0, cop1,
16089 operands[1+negate], operands[2-negate]);
16091 ix86_expand_sse_movcc (operands[0], x, operands[1+negate],
16092 operands[2-negate]);
16096 /* Unpack OP[1] into the next wider integer vector type. UNSIGNED_P is
16097 true if we should do zero extension, else sign extension. HIGH_P is
16098 true if we want the N/2 high elements, else the low elements. */
16101 ix86_expand_sse_unpack (rtx operands[2], bool unsigned_p, bool high_p)
16103 enum machine_mode imode = GET_MODE (operands[1]);
16104 rtx (*unpack)(rtx, rtx, rtx);
16111 unpack = gen_vec_interleave_highv16qi;
16113 unpack = gen_vec_interleave_lowv16qi;
16117 unpack = gen_vec_interleave_highv8hi;
16119 unpack = gen_vec_interleave_lowv8hi;
16123 unpack = gen_vec_interleave_highv4si;
16125 unpack = gen_vec_interleave_lowv4si;
16128 gcc_unreachable ();
16131 dest = gen_lowpart (imode, operands[0]);
16134 se = force_reg (imode, CONST0_RTX (imode));
16136 se = ix86_expand_sse_cmp (gen_reg_rtx (imode), GT, CONST0_RTX (imode),
16137 operands[1], pc_rtx, pc_rtx);
16139 emit_insn (unpack (dest, operands[1], se));
16142 /* This function performs the same task as ix86_expand_sse_unpack,
16143 but with SSE4.1 instructions. */
16146 ix86_expand_sse4_unpack (rtx operands[2], bool unsigned_p, bool high_p)
16148 enum machine_mode imode = GET_MODE (operands[1]);
16149 rtx (*unpack)(rtx, rtx);
16156 unpack = gen_sse4_1_zero_extendv8qiv8hi2;
16158 unpack = gen_sse4_1_extendv8qiv8hi2;
16162 unpack = gen_sse4_1_zero_extendv4hiv4si2;
16164 unpack = gen_sse4_1_extendv4hiv4si2;
16168 unpack = gen_sse4_1_zero_extendv2siv2di2;
16170 unpack = gen_sse4_1_extendv2siv2di2;
16173 gcc_unreachable ();
16176 dest = operands[0];
16179 /* Shift higher 8 bytes to lower 8 bytes. */
16180 src = gen_reg_rtx (imode);
16181 emit_insn (gen_sse2_lshrti3 (gen_lowpart (TImode, src),
16182 gen_lowpart (TImode, operands[1]),
16188 emit_insn (unpack (dest, src));
16191 /* This function performs the same task as ix86_expand_sse_unpack,
16192 but with sse5 instructions. */
16195 ix86_expand_sse5_unpack (rtx operands[2], bool unsigned_p, bool high_p)
16197 enum machine_mode imode = GET_MODE (operands[1]);
16198 int pperm_bytes[16];
16200 int h = (high_p) ? 8 : 0;
16203 rtvec v = rtvec_alloc (16);
16206 rtx op0 = operands[0], op1 = operands[1];
16211 vs = rtvec_alloc (8);
16212 h2 = (high_p) ? 8 : 0;
16213 for (i = 0; i < 8; i++)
16215 pperm_bytes[2*i+0] = PPERM_SRC | PPERM_SRC2 | i | h;
16216 pperm_bytes[2*i+1] = ((unsigned_p)
16218 : PPERM_SIGN | PPERM_SRC2 | i | h);
16221 for (i = 0; i < 16; i++)
16222 RTVEC_ELT (v, i) = GEN_INT (pperm_bytes[i]);
16224 for (i = 0; i < 8; i++)
16225 RTVEC_ELT (vs, i) = GEN_INT (i + h2);
16227 p = gen_rtx_PARALLEL (VOIDmode, vs);
16228 x = force_reg (V16QImode, gen_rtx_CONST_VECTOR (V16QImode, v));
16230 emit_insn (gen_sse5_pperm_zero_v16qi_v8hi (op0, op1, p, x));
16232 emit_insn (gen_sse5_pperm_sign_v16qi_v8hi (op0, op1, p, x));
16236 vs = rtvec_alloc (4);
16237 h2 = (high_p) ? 4 : 0;
16238 for (i = 0; i < 4; i++)
16240 sign_extend = ((unsigned_p)
16242 : PPERM_SIGN | PPERM_SRC2 | ((2*i) + 1 + h));
16243 pperm_bytes[4*i+0] = PPERM_SRC | PPERM_SRC2 | ((2*i) + 0 + h);
16244 pperm_bytes[4*i+1] = PPERM_SRC | PPERM_SRC2 | ((2*i) + 1 + h);
16245 pperm_bytes[4*i+2] = sign_extend;
16246 pperm_bytes[4*i+3] = sign_extend;
16249 for (i = 0; i < 16; i++)
16250 RTVEC_ELT (v, i) = GEN_INT (pperm_bytes[i]);
16252 for (i = 0; i < 4; i++)
16253 RTVEC_ELT (vs, i) = GEN_INT (i + h2);
16255 p = gen_rtx_PARALLEL (VOIDmode, vs);
16256 x = force_reg (V16QImode, gen_rtx_CONST_VECTOR (V16QImode, v));
16258 emit_insn (gen_sse5_pperm_zero_v8hi_v4si (op0, op1, p, x));
16260 emit_insn (gen_sse5_pperm_sign_v8hi_v4si (op0, op1, p, x));
16264 vs = rtvec_alloc (2);
16265 h2 = (high_p) ? 2 : 0;
16266 for (i = 0; i < 2; i++)
16268 sign_extend = ((unsigned_p)
16270 : PPERM_SIGN | PPERM_SRC2 | ((4*i) + 3 + h));
16271 pperm_bytes[8*i+0] = PPERM_SRC | PPERM_SRC2 | ((4*i) + 0 + h);
16272 pperm_bytes[8*i+1] = PPERM_SRC | PPERM_SRC2 | ((4*i) + 1 + h);
16273 pperm_bytes[8*i+2] = PPERM_SRC | PPERM_SRC2 | ((4*i) + 2 + h);
16274 pperm_bytes[8*i+3] = PPERM_SRC | PPERM_SRC2 | ((4*i) + 3 + h);
16275 pperm_bytes[8*i+4] = sign_extend;
16276 pperm_bytes[8*i+5] = sign_extend;
16277 pperm_bytes[8*i+6] = sign_extend;
16278 pperm_bytes[8*i+7] = sign_extend;
16281 for (i = 0; i < 16; i++)
16282 RTVEC_ELT (v, i) = GEN_INT (pperm_bytes[i]);
16284 for (i = 0; i < 2; i++)
16285 RTVEC_ELT (vs, i) = GEN_INT (i + h2);
16287 p = gen_rtx_PARALLEL (VOIDmode, vs);
16288 x = force_reg (V16QImode, gen_rtx_CONST_VECTOR (V16QImode, v));
16290 emit_insn (gen_sse5_pperm_zero_v4si_v2di (op0, op1, p, x));
16292 emit_insn (gen_sse5_pperm_sign_v4si_v2di (op0, op1, p, x));
16296 gcc_unreachable ();
16302 /* Pack the high bits from OPERANDS[1] and low bits from OPERANDS[2] into the
16303 next narrower integer vector type */
16305 ix86_expand_sse5_pack (rtx operands[3])
16307 enum machine_mode imode = GET_MODE (operands[0]);
16308 int pperm_bytes[16];
16310 rtvec v = rtvec_alloc (16);
16312 rtx op0 = operands[0];
16313 rtx op1 = operands[1];
16314 rtx op2 = operands[2];
16319 for (i = 0; i < 8; i++)
16321 pperm_bytes[i+0] = PPERM_SRC | PPERM_SRC1 | (i*2);
16322 pperm_bytes[i+8] = PPERM_SRC | PPERM_SRC2 | (i*2);
16325 for (i = 0; i < 16; i++)
16326 RTVEC_ELT (v, i) = GEN_INT (pperm_bytes[i]);
16328 x = force_reg (V16QImode, gen_rtx_CONST_VECTOR (V16QImode, v));
16329 emit_insn (gen_sse5_pperm_pack_v8hi_v16qi (op0, op1, op2, x));
16333 for (i = 0; i < 4; i++)
16335 pperm_bytes[(2*i)+0] = PPERM_SRC | PPERM_SRC1 | ((i*4) + 0);
16336 pperm_bytes[(2*i)+1] = PPERM_SRC | PPERM_SRC1 | ((i*4) + 1);
16337 pperm_bytes[(2*i)+8] = PPERM_SRC | PPERM_SRC2 | ((i*4) + 0);
16338 pperm_bytes[(2*i)+9] = PPERM_SRC | PPERM_SRC2 | ((i*4) + 1);
16341 for (i = 0; i < 16; i++)
16342 RTVEC_ELT (v, i) = GEN_INT (pperm_bytes[i]);
16344 x = force_reg (V16QImode, gen_rtx_CONST_VECTOR (V16QImode, v));
16345 emit_insn (gen_sse5_pperm_pack_v4si_v8hi (op0, op1, op2, x));
16349 for (i = 0; i < 2; i++)
16351 pperm_bytes[(4*i)+0] = PPERM_SRC | PPERM_SRC1 | ((i*8) + 0);
16352 pperm_bytes[(4*i)+1] = PPERM_SRC | PPERM_SRC1 | ((i*8) + 1);
16353 pperm_bytes[(4*i)+2] = PPERM_SRC | PPERM_SRC1 | ((i*8) + 2);
16354 pperm_bytes[(4*i)+3] = PPERM_SRC | PPERM_SRC1 | ((i*8) + 3);
16355 pperm_bytes[(4*i)+8] = PPERM_SRC | PPERM_SRC2 | ((i*8) + 0);
16356 pperm_bytes[(4*i)+9] = PPERM_SRC | PPERM_SRC2 | ((i*8) + 1);
16357 pperm_bytes[(4*i)+10] = PPERM_SRC | PPERM_SRC2 | ((i*8) + 2);
16358 pperm_bytes[(4*i)+11] = PPERM_SRC | PPERM_SRC2 | ((i*8) + 3);
16361 for (i = 0; i < 16; i++)
16362 RTVEC_ELT (v, i) = GEN_INT (pperm_bytes[i]);
16364 x = force_reg (V16QImode, gen_rtx_CONST_VECTOR (V16QImode, v));
16365 emit_insn (gen_sse5_pperm_pack_v2di_v4si (op0, op1, op2, x));
16369 gcc_unreachable ();
16375 /* Expand conditional increment or decrement using adb/sbb instructions.
16376 The default case using setcc followed by the conditional move can be
16377 done by generic code. */
16379 ix86_expand_int_addcc (rtx operands[])
16381 enum rtx_code code = GET_CODE (operands[1]);
16383 rtx val = const0_rtx;
16384 bool fpcmp = false;
16385 enum machine_mode mode = GET_MODE (operands[0]);
16387 ix86_compare_op0 = XEXP (operands[1], 0);
16388 ix86_compare_op1 = XEXP (operands[1], 1);
16389 if (operands[3] != const1_rtx
16390 && operands[3] != constm1_rtx)
16392 if (!ix86_expand_carry_flag_compare (code, ix86_compare_op0,
16393 ix86_compare_op1, &compare_op))
16395 code = GET_CODE (compare_op);
16397 if (GET_MODE (XEXP (compare_op, 0)) == CCFPmode
16398 || GET_MODE (XEXP (compare_op, 0)) == CCFPUmode)
16401 code = ix86_fp_compare_code_to_integer (code);
16408 PUT_CODE (compare_op,
16409 reverse_condition_maybe_unordered
16410 (GET_CODE (compare_op)));
16412 PUT_CODE (compare_op, reverse_condition (GET_CODE (compare_op)));
16414 PUT_MODE (compare_op, mode);
16416 /* Construct either adc or sbb insn. */
16417 if ((code == LTU) == (operands[3] == constm1_rtx))
16419 switch (GET_MODE (operands[0]))
16422 emit_insn (gen_subqi3_carry (operands[0], operands[2], val, compare_op));
16425 emit_insn (gen_subhi3_carry (operands[0], operands[2], val, compare_op));
16428 emit_insn (gen_subsi3_carry (operands[0], operands[2], val, compare_op));
16431 emit_insn (gen_subdi3_carry_rex64 (operands[0], operands[2], val, compare_op));
16434 gcc_unreachable ();
16439 switch (GET_MODE (operands[0]))
16442 emit_insn (gen_addqi3_carry (operands[0], operands[2], val, compare_op));
16445 emit_insn (gen_addhi3_carry (operands[0], operands[2], val, compare_op));
16448 emit_insn (gen_addsi3_carry (operands[0], operands[2], val, compare_op));
16451 emit_insn (gen_adddi3_carry_rex64 (operands[0], operands[2], val, compare_op));
16454 gcc_unreachable ();
16457 return 1; /* DONE */
16461 /* Split operands 0 and 1 into SImode parts. Similar to split_di, but
16462 works for floating pointer parameters and nonoffsetable memories.
16463 For pushes, it returns just stack offsets; the values will be saved
16464 in the right order. Maximally three parts are generated. */
16467 ix86_split_to_parts (rtx operand, rtx *parts, enum machine_mode mode)
16472 size = mode==XFmode ? 3 : GET_MODE_SIZE (mode) / 4;
16474 size = (GET_MODE_SIZE (mode) + 4) / 8;
16476 gcc_assert (!REG_P (operand) || !MMX_REGNO_P (REGNO (operand)));
16477 gcc_assert (size >= 2 && size <= 4);
16479 /* Optimize constant pool reference to immediates. This is used by fp
16480 moves, that force all constants to memory to allow combining. */
16481 if (MEM_P (operand) && MEM_READONLY_P (operand))
16483 rtx tmp = maybe_get_pool_constant (operand);
16488 if (MEM_P (operand) && !offsettable_memref_p (operand))
16490 /* The only non-offsetable memories we handle are pushes. */
16491 int ok = push_operand (operand, VOIDmode);
16495 operand = copy_rtx (operand);
16496 PUT_MODE (operand, Pmode);
16497 parts[0] = parts[1] = parts[2] = parts[3] = operand;
16501 if (GET_CODE (operand) == CONST_VECTOR)
16503 enum machine_mode imode = int_mode_for_mode (mode);
16504 /* Caution: if we looked through a constant pool memory above,
16505 the operand may actually have a different mode now. That's
16506 ok, since we want to pun this all the way back to an integer. */
16507 operand = simplify_subreg (imode, operand, GET_MODE (operand), 0);
16508 gcc_assert (operand != NULL);
16514 if (mode == DImode)
16515 split_di (&operand, 1, &parts[0], &parts[1]);
16520 if (REG_P (operand))
16522 gcc_assert (reload_completed);
16523 for (i = 0; i < size; i++)
16524 parts[i] = gen_rtx_REG (SImode, REGNO (operand) + i);
16526 else if (offsettable_memref_p (operand))
16528 operand = adjust_address (operand, SImode, 0);
16529 parts[0] = operand;
16530 for (i = 1; i < size; i++)
16531 parts[i] = adjust_address (operand, SImode, 4 * i);
16533 else if (GET_CODE (operand) == CONST_DOUBLE)
16538 REAL_VALUE_FROM_CONST_DOUBLE (r, operand);
16542 real_to_target (l, &r, mode);
16543 parts[3] = gen_int_mode (l[3], SImode);
16544 parts[2] = gen_int_mode (l[2], SImode);
16547 REAL_VALUE_TO_TARGET_LONG_DOUBLE (r, l);
16548 parts[2] = gen_int_mode (l[2], SImode);
16551 REAL_VALUE_TO_TARGET_DOUBLE (r, l);
16554 gcc_unreachable ();
16556 parts[1] = gen_int_mode (l[1], SImode);
16557 parts[0] = gen_int_mode (l[0], SImode);
16560 gcc_unreachable ();
16565 if (mode == TImode)
16566 split_ti (&operand, 1, &parts[0], &parts[1]);
16567 if (mode == XFmode || mode == TFmode)
16569 enum machine_mode upper_mode = mode==XFmode ? SImode : DImode;
16570 if (REG_P (operand))
16572 gcc_assert (reload_completed);
16573 parts[0] = gen_rtx_REG (DImode, REGNO (operand) + 0);
16574 parts[1] = gen_rtx_REG (upper_mode, REGNO (operand) + 1);
16576 else if (offsettable_memref_p (operand))
16578 operand = adjust_address (operand, DImode, 0);
16579 parts[0] = operand;
16580 parts[1] = adjust_address (operand, upper_mode, 8);
16582 else if (GET_CODE (operand) == CONST_DOUBLE)
16587 REAL_VALUE_FROM_CONST_DOUBLE (r, operand);
16588 real_to_target (l, &r, mode);
16590 /* Do not use shift by 32 to avoid warning on 32bit systems. */
16591 if (HOST_BITS_PER_WIDE_INT >= 64)
16594 ((l[0] & (((HOST_WIDE_INT) 2 << 31) - 1))
16595 + ((((HOST_WIDE_INT) l[1]) << 31) << 1),
16598 parts[0] = immed_double_const (l[0], l[1], DImode);
16600 if (upper_mode == SImode)
16601 parts[1] = gen_int_mode (l[2], SImode);
16602 else if (HOST_BITS_PER_WIDE_INT >= 64)
16605 ((l[2] & (((HOST_WIDE_INT) 2 << 31) - 1))
16606 + ((((HOST_WIDE_INT) l[3]) << 31) << 1),
16609 parts[1] = immed_double_const (l[2], l[3], DImode);
16612 gcc_unreachable ();
16619 /* Emit insns to perform a move or push of DI, DF, XF, and TF values.
16620 Return false when normal moves are needed; true when all required
16621 insns have been emitted. Operands 2-4 contain the input values
16622 int the correct order; operands 5-7 contain the output values. */
16625 ix86_split_long_move (rtx operands[])
16630 int collisions = 0;
16631 enum machine_mode mode = GET_MODE (operands[0]);
16632 bool collisionparts[4];
16634 /* The DFmode expanders may ask us to move double.
16635 For 64bit target this is single move. By hiding the fact
16636 here we simplify i386.md splitters. */
16637 if (GET_MODE_SIZE (GET_MODE (operands[0])) == 8 && TARGET_64BIT)
16639 /* Optimize constant pool reference to immediates. This is used by
16640 fp moves, that force all constants to memory to allow combining. */
16642 if (MEM_P (operands[1])
16643 && GET_CODE (XEXP (operands[1], 0)) == SYMBOL_REF
16644 && CONSTANT_POOL_ADDRESS_P (XEXP (operands[1], 0)))
16645 operands[1] = get_pool_constant (XEXP (operands[1], 0));
16646 if (push_operand (operands[0], VOIDmode))
16648 operands[0] = copy_rtx (operands[0]);
16649 PUT_MODE (operands[0], Pmode);
16652 operands[0] = gen_lowpart (DImode, operands[0]);
16653 operands[1] = gen_lowpart (DImode, operands[1]);
16654 emit_move_insn (operands[0], operands[1]);
16658 /* The only non-offsettable memory we handle is push. */
16659 if (push_operand (operands[0], VOIDmode))
16662 gcc_assert (!MEM_P (operands[0])
16663 || offsettable_memref_p (operands[0]));
16665 nparts = ix86_split_to_parts (operands[1], part[1], GET_MODE (operands[0]));
16666 ix86_split_to_parts (operands[0], part[0], GET_MODE (operands[0]));
16668 /* When emitting push, take care for source operands on the stack. */
16669 if (push && MEM_P (operands[1])
16670 && reg_overlap_mentioned_p (stack_pointer_rtx, operands[1]))
16671 for (i = 0; i < nparts - 1; i++)
16672 part[1][i] = change_address (part[1][i],
16673 GET_MODE (part[1][i]),
16674 XEXP (part[1][i + 1], 0));
16676 /* We need to do copy in the right order in case an address register
16677 of the source overlaps the destination. */
16678 if (REG_P (part[0][0]) && MEM_P (part[1][0]))
16682 for (i = 0; i < nparts; i++)
16685 = reg_overlap_mentioned_p (part[0][i], XEXP (part[1][0], 0));
16686 if (collisionparts[i])
16690 /* Collision in the middle part can be handled by reordering. */
16691 if (collisions == 1 && nparts == 3 && collisionparts [1])
16693 tmp = part[0][1]; part[0][1] = part[0][2]; part[0][2] = tmp;
16694 tmp = part[1][1]; part[1][1] = part[1][2]; part[1][2] = tmp;
16696 else if (collisions == 1
16698 && (collisionparts [1] || collisionparts [2]))
16700 if (collisionparts [1])
16702 tmp = part[0][1]; part[0][1] = part[0][2]; part[0][2] = tmp;
16703 tmp = part[1][1]; part[1][1] = part[1][2]; part[1][2] = tmp;
16707 tmp = part[0][2]; part[0][2] = part[0][3]; part[0][3] = tmp;
16708 tmp = part[1][2]; part[1][2] = part[1][3]; part[1][3] = tmp;
16712 /* If there are more collisions, we can't handle it by reordering.
16713 Do an lea to the last part and use only one colliding move. */
16714 else if (collisions > 1)
16720 base = part[0][nparts - 1];
16722 /* Handle the case when the last part isn't valid for lea.
16723 Happens in 64-bit mode storing the 12-byte XFmode. */
16724 if (GET_MODE (base) != Pmode)
16725 base = gen_rtx_REG (Pmode, REGNO (base));
16727 emit_insn (gen_rtx_SET (VOIDmode, base, XEXP (part[1][0], 0)));
16728 part[1][0] = replace_equiv_address (part[1][0], base);
16729 for (i = 1; i < nparts; i++)
16731 tmp = plus_constant (base, UNITS_PER_WORD * i);
16732 part[1][i] = replace_equiv_address (part[1][i], tmp);
16743 if (TARGET_128BIT_LONG_DOUBLE && mode == XFmode)
16744 emit_insn (gen_addsi3 (stack_pointer_rtx, stack_pointer_rtx, GEN_INT (-4)));
16745 emit_move_insn (part[0][2], part[1][2]);
16747 else if (nparts == 4)
16749 emit_move_insn (part[0][3], part[1][3]);
16750 emit_move_insn (part[0][2], part[1][2]);
16755 /* In 64bit mode we don't have 32bit push available. In case this is
16756 register, it is OK - we will just use larger counterpart. We also
16757 retype memory - these comes from attempt to avoid REX prefix on
16758 moving of second half of TFmode value. */
16759 if (GET_MODE (part[1][1]) == SImode)
16761 switch (GET_CODE (part[1][1]))
16764 part[1][1] = adjust_address (part[1][1], DImode, 0);
16768 part[1][1] = gen_rtx_REG (DImode, REGNO (part[1][1]));
16772 gcc_unreachable ();
16775 if (GET_MODE (part[1][0]) == SImode)
16776 part[1][0] = part[1][1];
16779 emit_move_insn (part[0][1], part[1][1]);
16780 emit_move_insn (part[0][0], part[1][0]);
16784 /* Choose correct order to not overwrite the source before it is copied. */
16785 if ((REG_P (part[0][0])
16786 && REG_P (part[1][1])
16787 && (REGNO (part[0][0]) == REGNO (part[1][1])
16789 && REGNO (part[0][0]) == REGNO (part[1][2]))
16791 && REGNO (part[0][0]) == REGNO (part[1][3]))))
16793 && reg_overlap_mentioned_p (part[0][0], XEXP (part[1][0], 0))))
16795 for (i = 0, j = nparts - 1; i < nparts; i++, j--)
16797 operands[2 + i] = part[0][j];
16798 operands[6 + i] = part[1][j];
16803 for (i = 0; i < nparts; i++)
16805 operands[2 + i] = part[0][i];
16806 operands[6 + i] = part[1][i];
16810 /* If optimizing for size, attempt to locally unCSE nonzero constants. */
16811 if (optimize_insn_for_size_p ())
16813 for (j = 0; j < nparts - 1; j++)
16814 if (CONST_INT_P (operands[6 + j])
16815 && operands[6 + j] != const0_rtx
16816 && REG_P (operands[2 + j]))
16817 for (i = j; i < nparts - 1; i++)
16818 if (CONST_INT_P (operands[7 + i])
16819 && INTVAL (operands[7 + i]) == INTVAL (operands[6 + j]))
16820 operands[7 + i] = operands[2 + j];
16823 for (i = 0; i < nparts; i++)
16824 emit_move_insn (operands[2 + i], operands[6 + i]);
16829 /* Helper function of ix86_split_ashl used to generate an SImode/DImode
16830 left shift by a constant, either using a single shift or
16831 a sequence of add instructions. */
16834 ix86_expand_ashl_const (rtx operand, int count, enum machine_mode mode)
16838 emit_insn ((mode == DImode
16840 : gen_adddi3) (operand, operand, operand));
16842 else if (!optimize_insn_for_size_p ()
16843 && count * ix86_cost->add <= ix86_cost->shift_const)
16846 for (i=0; i<count; i++)
16848 emit_insn ((mode == DImode
16850 : gen_adddi3) (operand, operand, operand));
16854 emit_insn ((mode == DImode
16856 : gen_ashldi3) (operand, operand, GEN_INT (count)));
16860 ix86_split_ashl (rtx *operands, rtx scratch, enum machine_mode mode)
16862 rtx low[2], high[2];
16864 const int single_width = mode == DImode ? 32 : 64;
16866 if (CONST_INT_P (operands[2]))
16868 (mode == DImode ? split_di : split_ti) (operands, 2, low, high);
16869 count = INTVAL (operands[2]) & (single_width * 2 - 1);
16871 if (count >= single_width)
16873 emit_move_insn (high[0], low[1]);
16874 emit_move_insn (low[0], const0_rtx);
16876 if (count > single_width)
16877 ix86_expand_ashl_const (high[0], count - single_width, mode);
16881 if (!rtx_equal_p (operands[0], operands[1]))
16882 emit_move_insn (operands[0], operands[1]);
16883 emit_insn ((mode == DImode
16885 : gen_x86_64_shld) (high[0], low[0], GEN_INT (count)));
16886 ix86_expand_ashl_const (low[0], count, mode);
16891 (mode == DImode ? split_di : split_ti) (operands, 1, low, high);
16893 if (operands[1] == const1_rtx)
16895 /* Assuming we've chosen a QImode capable registers, then 1 << N
16896 can be done with two 32/64-bit shifts, no branches, no cmoves. */
16897 if (ANY_QI_REG_P (low[0]) && ANY_QI_REG_P (high[0]))
16899 rtx s, d, flags = gen_rtx_REG (CCZmode, FLAGS_REG);
16901 ix86_expand_clear (low[0]);
16902 ix86_expand_clear (high[0]);
16903 emit_insn (gen_testqi_ccz_1 (operands[2], GEN_INT (single_width)));
16905 d = gen_lowpart (QImode, low[0]);
16906 d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
16907 s = gen_rtx_EQ (QImode, flags, const0_rtx);
16908 emit_insn (gen_rtx_SET (VOIDmode, d, s));
16910 d = gen_lowpart (QImode, high[0]);
16911 d = gen_rtx_STRICT_LOW_PART (VOIDmode, d);
16912 s = gen_rtx_NE (QImode, flags, const0_rtx);
16913 emit_insn (gen_rtx_SET (VOIDmode, d, s));
16916 /* Otherwise, we can get the same results by manually performing
16917 a bit extract operation on bit 5/6, and then performing the two
16918 shifts. The two methods of getting 0/1 into low/high are exactly
16919 the same size. Avoiding the shift in the bit extract case helps
16920 pentium4 a bit; no one else seems to care much either way. */
16925 if (TARGET_PARTIAL_REG_STALL && !optimize_insn_for_size_p ())
16926 x = gen_rtx_ZERO_EXTEND (mode == DImode ? SImode : DImode, operands[2]);
16928 x = gen_lowpart (mode == DImode ? SImode : DImode, operands[2]);
16929 emit_insn (gen_rtx_SET (VOIDmode, high[0], x));
16931 emit_insn ((mode == DImode
16933 : gen_lshrdi3) (high[0], high[0], GEN_INT (mode == DImode ? 5 : 6)));
16934 emit_insn ((mode == DImode
16936 : gen_anddi3) (high[0], high[0], GEN_INT (1)));
16937 emit_move_insn (low[0], high[0]);
16938 emit_insn ((mode == DImode
16940 : gen_xordi3) (low[0], low[0], GEN_INT (1)));
16943 emit_insn ((mode == DImode
16945 : gen_ashldi3) (low[0], low[0], operands[2]));
16946 emit_insn ((mode == DImode
16948 : gen_ashldi3) (high[0], high[0], operands[2]));
16952 if (operands[1] == constm1_rtx)
16954 /* For -1 << N, we can avoid the shld instruction, because we
16955 know that we're shifting 0...31/63 ones into a -1. */
16956 emit_move_insn (low[0], constm1_rtx);
16957 if (optimize_insn_for_size_p ())
16958 emit_move_insn (high[0], low[0]);
16960 emit_move_insn (high[0], constm1_rtx);
16964 if (!rtx_equal_p (operands[0], operands[1]))
16965 emit_move_insn (operands[0], operands[1]);
16967 (mode == DImode ? split_di : split_ti) (operands, 1, low, high);
16968 emit_insn ((mode == DImode
16970 : gen_x86_64_shld) (high[0], low[0], operands[2]));
16973 emit_insn ((mode == DImode ? gen_ashlsi3 : gen_ashldi3) (low[0], low[0], operands[2]));
16975 if (TARGET_CMOVE && scratch)
16977 ix86_expand_clear (scratch);
16978 emit_insn ((mode == DImode
16979 ? gen_x86_shift_adj_1
16980 : gen_x86_64_shift_adj_1) (high[0], low[0], operands[2],
16984 emit_insn ((mode == DImode
16985 ? gen_x86_shift_adj_2
16986 : gen_x86_64_shift_adj_2) (high[0], low[0], operands[2]));
16990 ix86_split_ashr (rtx *operands, rtx scratch, enum machine_mode mode)
16992 rtx low[2], high[2];
16994 const int single_width = mode == DImode ? 32 : 64;
16996 if (CONST_INT_P (operands[2]))
16998 (mode == DImode ? split_di : split_ti) (operands, 2, low, high);
16999 count = INTVAL (operands[2]) & (single_width * 2 - 1);
17001 if (count == single_width * 2 - 1)
17003 emit_move_insn (high[0], high[1]);
17004 emit_insn ((mode == DImode
17006 : gen_ashrdi3) (high[0], high[0],
17007 GEN_INT (single_width - 1)));
17008 emit_move_insn (low[0], high[0]);
17011 else if (count >= single_width)
17013 emit_move_insn (low[0], high[1]);
17014 emit_move_insn (high[0], low[0]);
17015 emit_insn ((mode == DImode
17017 : gen_ashrdi3) (high[0], high[0],
17018 GEN_INT (single_width - 1)));
17019 if (count > single_width)
17020 emit_insn ((mode == DImode
17022 : gen_ashrdi3) (low[0], low[0],
17023 GEN_INT (count - single_width)));
17027 if (!rtx_equal_p (operands[0], operands[1]))
17028 emit_move_insn (operands[0], operands[1]);
17029 emit_insn ((mode == DImode
17031 : gen_x86_64_shrd) (low[0], high[0], GEN_INT (count)));
17032 emit_insn ((mode == DImode
17034 : gen_ashrdi3) (high[0], high[0], GEN_INT (count)));
17039 if (!rtx_equal_p (operands[0], operands[1]))
17040 emit_move_insn (operands[0], operands[1]);
17042 (mode == DImode ? split_di : split_ti) (operands, 1, low, high);
17044 emit_insn ((mode == DImode
17046 : gen_x86_64_shrd) (low[0], high[0], operands[2]));
17047 emit_insn ((mode == DImode
17049 : gen_ashrdi3) (high[0], high[0], operands[2]));
17051 if (TARGET_CMOVE && scratch)
17053 emit_move_insn (scratch, high[0]);
17054 emit_insn ((mode == DImode
17056 : gen_ashrdi3) (scratch, scratch,
17057 GEN_INT (single_width - 1)));
17058 emit_insn ((mode == DImode
17059 ? gen_x86_shift_adj_1
17060 : gen_x86_64_shift_adj_1) (low[0], high[0], operands[2],
17064 emit_insn ((mode == DImode
17065 ? gen_x86_shift_adj_3
17066 : gen_x86_64_shift_adj_3) (low[0], high[0], operands[2]));
17071 ix86_split_lshr (rtx *operands, rtx scratch, enum machine_mode mode)
17073 rtx low[2], high[2];
17075 const int single_width = mode == DImode ? 32 : 64;
17077 if (CONST_INT_P (operands[2]))
17079 (mode == DImode ? split_di : split_ti) (operands, 2, low, high);
17080 count = INTVAL (operands[2]) & (single_width * 2 - 1);
17082 if (count >= single_width)
17084 emit_move_insn (low[0], high[1]);
17085 ix86_expand_clear (high[0]);
17087 if (count > single_width)
17088 emit_insn ((mode == DImode
17090 : gen_lshrdi3) (low[0], low[0],
17091 GEN_INT (count - single_width)));
17095 if (!rtx_equal_p (operands[0], operands[1]))
17096 emit_move_insn (operands[0], operands[1]);
17097 emit_insn ((mode == DImode
17099 : gen_x86_64_shrd) (low[0], high[0], GEN_INT (count)));
17100 emit_insn ((mode == DImode
17102 : gen_lshrdi3) (high[0], high[0], GEN_INT (count)));
17107 if (!rtx_equal_p (operands[0], operands[1]))
17108 emit_move_insn (operands[0], operands[1]);
17110 (mode == DImode ? split_di : split_ti) (operands, 1, low, high);
17112 emit_insn ((mode == DImode
17114 : gen_x86_64_shrd) (low[0], high[0], operands[2]));
17115 emit_insn ((mode == DImode
17117 : gen_lshrdi3) (high[0], high[0], operands[2]));
17119 /* Heh. By reversing the arguments, we can reuse this pattern. */
17120 if (TARGET_CMOVE && scratch)
17122 ix86_expand_clear (scratch);
17123 emit_insn ((mode == DImode
17124 ? gen_x86_shift_adj_1
17125 : gen_x86_64_shift_adj_1) (low[0], high[0], operands[2],
17129 emit_insn ((mode == DImode
17130 ? gen_x86_shift_adj_2
17131 : gen_x86_64_shift_adj_2) (low[0], high[0], operands[2]));
17135 /* Predict just emitted jump instruction to be taken with probability PROB. */
17137 predict_jump (int prob)
17139 rtx insn = get_last_insn ();
17140 gcc_assert (JUMP_P (insn));
17141 add_reg_note (insn, REG_BR_PROB, GEN_INT (prob));
17144 /* Helper function for the string operations below. Dest VARIABLE whether
17145 it is aligned to VALUE bytes. If true, jump to the label. */
17147 ix86_expand_aligntest (rtx variable, int value, bool epilogue)
17149 rtx label = gen_label_rtx ();
17150 rtx tmpcount = gen_reg_rtx (GET_MODE (variable));
17151 if (GET_MODE (variable) == DImode)
17152 emit_insn (gen_anddi3 (tmpcount, variable, GEN_INT (value)));
17154 emit_insn (gen_andsi3 (tmpcount, variable, GEN_INT (value)));
17155 emit_cmp_and_jump_insns (tmpcount, const0_rtx, EQ, 0, GET_MODE (variable),
17158 predict_jump (REG_BR_PROB_BASE * 50 / 100);
17160 predict_jump (REG_BR_PROB_BASE * 90 / 100);
17164 /* Adjust COUNTER by the VALUE. */
17166 ix86_adjust_counter (rtx countreg, HOST_WIDE_INT value)
17168 if (GET_MODE (countreg) == DImode)
17169 emit_insn (gen_adddi3 (countreg, countreg, GEN_INT (-value)));
17171 emit_insn (gen_addsi3 (countreg, countreg, GEN_INT (-value)));
17174 /* Zero extend possibly SImode EXP to Pmode register. */
17176 ix86_zero_extend_to_Pmode (rtx exp)
17179 if (GET_MODE (exp) == VOIDmode)
17180 return force_reg (Pmode, exp);
17181 if (GET_MODE (exp) == Pmode)
17182 return copy_to_mode_reg (Pmode, exp);
17183 r = gen_reg_rtx (Pmode);
17184 emit_insn (gen_zero_extendsidi2 (r, exp));
17188 /* Divide COUNTREG by SCALE. */
17190 scale_counter (rtx countreg, int scale)
17193 rtx piece_size_mask;
17197 if (CONST_INT_P (countreg))
17198 return GEN_INT (INTVAL (countreg) / scale);
17199 gcc_assert (REG_P (countreg));
17201 piece_size_mask = GEN_INT (scale - 1);
17202 sc = expand_simple_binop (GET_MODE (countreg), LSHIFTRT, countreg,
17203 GEN_INT (exact_log2 (scale)),
17204 NULL, 1, OPTAB_DIRECT);
17208 /* Return mode for the memcpy/memset loop counter. Prefer SImode over
17209 DImode for constant loop counts. */
17211 static enum machine_mode
17212 counter_mode (rtx count_exp)
17214 if (GET_MODE (count_exp) != VOIDmode)
17215 return GET_MODE (count_exp);
17216 if (GET_CODE (count_exp) != CONST_INT)
17218 if (TARGET_64BIT && (INTVAL (count_exp) & ~0xffffffff))
17223 /* When SRCPTR is non-NULL, output simple loop to move memory
17224 pointer to SRCPTR to DESTPTR via chunks of MODE unrolled UNROLL times,
17225 overall size is COUNT specified in bytes. When SRCPTR is NULL, output the
17226 equivalent loop to set memory by VALUE (supposed to be in MODE).
17228 The size is rounded down to whole number of chunk size moved at once.
17229 SRCMEM and DESTMEM provide MEMrtx to feed proper aliasing info. */
17233 expand_set_or_movmem_via_loop (rtx destmem, rtx srcmem,
17234 rtx destptr, rtx srcptr, rtx value,
17235 rtx count, enum machine_mode mode, int unroll,
17238 rtx out_label, top_label, iter, tmp;
17239 enum machine_mode iter_mode = counter_mode (count);
17240 rtx piece_size = GEN_INT (GET_MODE_SIZE (mode) * unroll);
17241 rtx piece_size_mask = GEN_INT (~((GET_MODE_SIZE (mode) * unroll) - 1));
17247 top_label = gen_label_rtx ();
17248 out_label = gen_label_rtx ();
17249 iter = gen_reg_rtx (iter_mode);
17251 size = expand_simple_binop (iter_mode, AND, count, piece_size_mask,
17252 NULL, 1, OPTAB_DIRECT);
17253 /* Those two should combine. */
17254 if (piece_size == const1_rtx)
17256 emit_cmp_and_jump_insns (size, const0_rtx, EQ, NULL_RTX, iter_mode,
17258 predict_jump (REG_BR_PROB_BASE * 10 / 100);
17260 emit_move_insn (iter, const0_rtx);
17262 emit_label (top_label);
17264 tmp = convert_modes (Pmode, iter_mode, iter, true);
17265 x_addr = gen_rtx_PLUS (Pmode, destptr, tmp);
17266 destmem = change_address (destmem, mode, x_addr);
17270 y_addr = gen_rtx_PLUS (Pmode, srcptr, copy_rtx (tmp));
17271 srcmem = change_address (srcmem, mode, y_addr);
17273 /* When unrolling for chips that reorder memory reads and writes,
17274 we can save registers by using single temporary.
17275 Also using 4 temporaries is overkill in 32bit mode. */
17276 if (!TARGET_64BIT && 0)
17278 for (i = 0; i < unroll; i++)
17283 adjust_address (copy_rtx (destmem), mode, GET_MODE_SIZE (mode));
17285 adjust_address (copy_rtx (srcmem), mode, GET_MODE_SIZE (mode));
17287 emit_move_insn (destmem, srcmem);
17293 gcc_assert (unroll <= 4);
17294 for (i = 0; i < unroll; i++)
17296 tmpreg[i] = gen_reg_rtx (mode);
17300 adjust_address (copy_rtx (srcmem), mode, GET_MODE_SIZE (mode));
17302 emit_move_insn (tmpreg[i], srcmem);
17304 for (i = 0; i < unroll; i++)
17309 adjust_address (copy_rtx (destmem), mode, GET_MODE_SIZE (mode));
17311 emit_move_insn (destmem, tmpreg[i]);
17316 for (i = 0; i < unroll; i++)
17320 adjust_address (copy_rtx (destmem), mode, GET_MODE_SIZE (mode));
17321 emit_move_insn (destmem, value);
17324 tmp = expand_simple_binop (iter_mode, PLUS, iter, piece_size, iter,
17325 true, OPTAB_LIB_WIDEN);
17327 emit_move_insn (iter, tmp);
17329 emit_cmp_and_jump_insns (iter, size, LT, NULL_RTX, iter_mode,
17331 if (expected_size != -1)
17333 expected_size /= GET_MODE_SIZE (mode) * unroll;
17334 if (expected_size == 0)
17336 else if (expected_size > REG_BR_PROB_BASE)
17337 predict_jump (REG_BR_PROB_BASE - 1);
17339 predict_jump (REG_BR_PROB_BASE - (REG_BR_PROB_BASE + expected_size / 2) / expected_size);
17342 predict_jump (REG_BR_PROB_BASE * 80 / 100);
17343 iter = ix86_zero_extend_to_Pmode (iter);
17344 tmp = expand_simple_binop (Pmode, PLUS, destptr, iter, destptr,
17345 true, OPTAB_LIB_WIDEN);
17346 if (tmp != destptr)
17347 emit_move_insn (destptr, tmp);
17350 tmp = expand_simple_binop (Pmode, PLUS, srcptr, iter, srcptr,
17351 true, OPTAB_LIB_WIDEN);
17353 emit_move_insn (srcptr, tmp);
17355 emit_label (out_label);
17358 /* Output "rep; mov" instruction.
17359 Arguments have same meaning as for previous function */
17361 expand_movmem_via_rep_mov (rtx destmem, rtx srcmem,
17362 rtx destptr, rtx srcptr,
17364 enum machine_mode mode)
17370 /* If the size is known, it is shorter to use rep movs. */
17371 if (mode == QImode && CONST_INT_P (count)
17372 && !(INTVAL (count) & 3))
17375 if (destptr != XEXP (destmem, 0) || GET_MODE (destmem) != BLKmode)
17376 destmem = adjust_automodify_address_nv (destmem, BLKmode, destptr, 0);
17377 if (srcptr != XEXP (srcmem, 0) || GET_MODE (srcmem) != BLKmode)
17378 srcmem = adjust_automodify_address_nv (srcmem, BLKmode, srcptr, 0);
17379 countreg = ix86_zero_extend_to_Pmode (scale_counter (count, GET_MODE_SIZE (mode)));
17380 if (mode != QImode)
17382 destexp = gen_rtx_ASHIFT (Pmode, countreg,
17383 GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
17384 destexp = gen_rtx_PLUS (Pmode, destexp, destptr);
17385 srcexp = gen_rtx_ASHIFT (Pmode, countreg,
17386 GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
17387 srcexp = gen_rtx_PLUS (Pmode, srcexp, srcptr);
17391 destexp = gen_rtx_PLUS (Pmode, destptr, countreg);
17392 srcexp = gen_rtx_PLUS (Pmode, srcptr, countreg);
17394 if (CONST_INT_P (count))
17396 count = GEN_INT (INTVAL (count)
17397 & ~((HOST_WIDE_INT) GET_MODE_SIZE (mode) - 1));
17398 destmem = shallow_copy_rtx (destmem);
17399 srcmem = shallow_copy_rtx (srcmem);
17400 set_mem_size (destmem, count);
17401 set_mem_size (srcmem, count);
17405 if (MEM_SIZE (destmem))
17406 set_mem_size (destmem, NULL_RTX);
17407 if (MEM_SIZE (srcmem))
17408 set_mem_size (srcmem, NULL_RTX);
17410 emit_insn (gen_rep_mov (destptr, destmem, srcptr, srcmem, countreg,
17414 /* Output "rep; stos" instruction.
17415 Arguments have same meaning as for previous function */
17417 expand_setmem_via_rep_stos (rtx destmem, rtx destptr, rtx value,
17418 rtx count, enum machine_mode mode,
17424 if (destptr != XEXP (destmem, 0) || GET_MODE (destmem) != BLKmode)
17425 destmem = adjust_automodify_address_nv (destmem, BLKmode, destptr, 0);
17426 value = force_reg (mode, gen_lowpart (mode, value));
17427 countreg = ix86_zero_extend_to_Pmode (scale_counter (count, GET_MODE_SIZE (mode)));
17428 if (mode != QImode)
17430 destexp = gen_rtx_ASHIFT (Pmode, countreg,
17431 GEN_INT (exact_log2 (GET_MODE_SIZE (mode))));
17432 destexp = gen_rtx_PLUS (Pmode, destexp, destptr);
17435 destexp = gen_rtx_PLUS (Pmode, destptr, countreg);
17436 if (orig_value == const0_rtx && CONST_INT_P (count))
17438 count = GEN_INT (INTVAL (count)
17439 & ~((HOST_WIDE_INT) GET_MODE_SIZE (mode) - 1));
17440 destmem = shallow_copy_rtx (destmem);
17441 set_mem_size (destmem, count);
17443 else if (MEM_SIZE (destmem))
17444 set_mem_size (destmem, NULL_RTX);
17445 emit_insn (gen_rep_stos (destptr, countreg, destmem, value, destexp));
17449 emit_strmov (rtx destmem, rtx srcmem,
17450 rtx destptr, rtx srcptr, enum machine_mode mode, int offset)
17452 rtx src = adjust_automodify_address_nv (srcmem, mode, srcptr, offset);
17453 rtx dest = adjust_automodify_address_nv (destmem, mode, destptr, offset);
17454 emit_insn (gen_strmov (destptr, dest, srcptr, src));
17457 /* Output code to copy at most count & (max_size - 1) bytes from SRC to DEST. */
17459 expand_movmem_epilogue (rtx destmem, rtx srcmem,
17460 rtx destptr, rtx srcptr, rtx count, int max_size)
17463 if (CONST_INT_P (count))
17465 HOST_WIDE_INT countval = INTVAL (count);
17468 if ((countval & 0x10) && max_size > 16)
17472 emit_strmov (destmem, srcmem, destptr, srcptr, DImode, offset);
17473 emit_strmov (destmem, srcmem, destptr, srcptr, DImode, offset + 8);
17476 gcc_unreachable ();
17479 if ((countval & 0x08) && max_size > 8)
17482 emit_strmov (destmem, srcmem, destptr, srcptr, DImode, offset);
17485 emit_strmov (destmem, srcmem, destptr, srcptr, SImode, offset);
17486 emit_strmov (destmem, srcmem, destptr, srcptr, SImode, offset + 4);
17490 if ((countval & 0x04) && max_size > 4)
17492 emit_strmov (destmem, srcmem, destptr, srcptr, SImode, offset);
17495 if ((countval & 0x02) && max_size > 2)
17497 emit_strmov (destmem, srcmem, destptr, srcptr, HImode, offset);
17500 if ((countval & 0x01) && max_size > 1)
17502 emit_strmov (destmem, srcmem, destptr, srcptr, QImode, offset);
17509 count = expand_simple_binop (GET_MODE (count), AND, count, GEN_INT (max_size - 1),
17510 count, 1, OPTAB_DIRECT);
17511 expand_set_or_movmem_via_loop (destmem, srcmem, destptr, srcptr, NULL,
17512 count, QImode, 1, 4);
17516 /* When there are stringops, we can cheaply increase dest and src pointers.
17517 Otherwise we save code size by maintaining offset (zero is readily
17518 available from preceding rep operation) and using x86 addressing modes.
17520 if (TARGET_SINGLE_STRINGOP)
17524 rtx label = ix86_expand_aligntest (count, 4, true);
17525 src = change_address (srcmem, SImode, srcptr);
17526 dest = change_address (destmem, SImode, destptr);
17527 emit_insn (gen_strmov (destptr, dest, srcptr, src));
17528 emit_label (label);
17529 LABEL_NUSES (label) = 1;
17533 rtx label = ix86_expand_aligntest (count, 2, true);
17534 src = change_address (srcmem, HImode, srcptr);
17535 dest = change_address (destmem, HImode, destptr);
17536 emit_insn (gen_strmov (destptr, dest, srcptr, src));
17537 emit_label (label);
17538 LABEL_NUSES (label) = 1;
17542 rtx label = ix86_expand_aligntest (count, 1, true);
17543 src = change_address (srcmem, QImode, srcptr);
17544 dest = change_address (destmem, QImode, destptr);
17545 emit_insn (gen_strmov (destptr, dest, srcptr, src));
17546 emit_label (label);
17547 LABEL_NUSES (label) = 1;
17552 rtx offset = force_reg (Pmode, const0_rtx);
17557 rtx label = ix86_expand_aligntest (count, 4, true);
17558 src = change_address (srcmem, SImode, srcptr);
17559 dest = change_address (destmem, SImode, destptr);
17560 emit_move_insn (dest, src);
17561 tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (4), NULL,
17562 true, OPTAB_LIB_WIDEN);
17564 emit_move_insn (offset, tmp);
17565 emit_label (label);
17566 LABEL_NUSES (label) = 1;
17570 rtx label = ix86_expand_aligntest (count, 2, true);
17571 tmp = gen_rtx_PLUS (Pmode, srcptr, offset);
17572 src = change_address (srcmem, HImode, tmp);
17573 tmp = gen_rtx_PLUS (Pmode, destptr, offset);
17574 dest = change_address (destmem, HImode, tmp);
17575 emit_move_insn (dest, src);
17576 tmp = expand_simple_binop (Pmode, PLUS, offset, GEN_INT (2), tmp,
17577 true, OPTAB_LIB_WIDEN);
17579 emit_move_insn (offset, tmp);
17580 emit_label (label);
17581 LABEL_NUSES (label) = 1;
17585 rtx label = ix86_expand_aligntest (count, 1, true);
17586 tmp = gen_rtx_PLUS (Pmode, srcptr, offset);
17587 src = change_address (srcmem, QImode, tmp);
17588 tmp = gen_rtx_PLUS (Pmode, destptr, offset);
17589 dest = change_address (destmem, QImode, tmp);
17590 emit_move_insn (dest, src);
17591 emit_label (label);
17592 LABEL_NUSES (label) = 1;
17597 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
17599 expand_setmem_epilogue_via_loop (rtx destmem, rtx destptr, rtx value,
17600 rtx count, int max_size)
17603 expand_simple_binop (counter_mode (count), AND, count,
17604 GEN_INT (max_size - 1), count, 1, OPTAB_DIRECT);
17605 expand_set_or_movmem_via_loop (destmem, NULL, destptr, NULL,
17606 gen_lowpart (QImode, value), count, QImode,
17610 /* Output code to set at most count & (max_size - 1) bytes starting by DEST. */
17612 expand_setmem_epilogue (rtx destmem, rtx destptr, rtx value, rtx count, int max_size)
17616 if (CONST_INT_P (count))
17618 HOST_WIDE_INT countval = INTVAL (count);
17621 if ((countval & 0x10) && max_size > 16)
17625 dest = adjust_automodify_address_nv (destmem, DImode, destptr, offset);
17626 emit_insn (gen_strset (destptr, dest, value));
17627 dest = adjust_automodify_address_nv (destmem, DImode, destptr, offset + 8);
17628 emit_insn (gen_strset (destptr, dest, value));
17631 gcc_unreachable ();
17634 if ((countval & 0x08) && max_size > 8)
17638 dest = adjust_automodify_address_nv (destmem, DImode, destptr, offset);
17639 emit_insn (gen_strset (destptr, dest, value));
17643 dest = adjust_automodify_address_nv (destmem, SImode, destptr, offset);
17644 emit_insn (gen_strset (destptr, dest, value));
17645 dest = adjust_automodify_address_nv (destmem, SImode, destptr, offset + 4);
17646 emit_insn (gen_strset (destptr, dest, value));
17650 if ((countval & 0x04) && max_size > 4)
17652 dest = adjust_automodify_address_nv (destmem, SImode, destptr, offset);
17653 emit_insn (gen_strset (destptr, dest, gen_lowpart (SImode, value)));
17656 if ((countval & 0x02) && max_size > 2)
17658 dest = adjust_automodify_address_nv (destmem, HImode, destptr, offset);
17659 emit_insn (gen_strset (destptr, dest, gen_lowpart (HImode, value)));
17662 if ((countval & 0x01) && max_size > 1)
17664 dest = adjust_automodify_address_nv (destmem, QImode, destptr, offset);
17665 emit_insn (gen_strset (destptr, dest, gen_lowpart (QImode, value)));
17672 expand_setmem_epilogue_via_loop (destmem, destptr, value, count, max_size);
17677 rtx label = ix86_expand_aligntest (count, 16, true);
17680 dest = change_address (destmem, DImode, destptr);
17681 emit_insn (gen_strset (destptr, dest, value));
17682 emit_insn (gen_strset (destptr, dest, value));
17686 dest = change_address (destmem, SImode, destptr);
17687 emit_insn (gen_strset (destptr, dest, value));
17688 emit_insn (gen_strset (destptr, dest, value));
17689 emit_insn (gen_strset (destptr, dest, value));
17690 emit_insn (gen_strset (destptr, dest, value));
17692 emit_label (label);
17693 LABEL_NUSES (label) = 1;
17697 rtx label = ix86_expand_aligntest (count, 8, true);
17700 dest = change_address (destmem, DImode, destptr);
17701 emit_insn (gen_strset (destptr, dest, value));
17705 dest = change_address (destmem, SImode, destptr);
17706 emit_insn (gen_strset (destptr, dest, value));
17707 emit_insn (gen_strset (destptr, dest, value));
17709 emit_label (label);
17710 LABEL_NUSES (label) = 1;
17714 rtx label = ix86_expand_aligntest (count, 4, true);
17715 dest = change_address (destmem, SImode, destptr);
17716 emit_insn (gen_strset (destptr, dest, gen_lowpart (SImode, value)));
17717 emit_label (label);
17718 LABEL_NUSES (label) = 1;
17722 rtx label = ix86_expand_aligntest (count, 2, true);
17723 dest = change_address (destmem, HImode, destptr);
17724 emit_insn (gen_strset (destptr, dest, gen_lowpart (HImode, value)));
17725 emit_label (label);
17726 LABEL_NUSES (label) = 1;
17730 rtx label = ix86_expand_aligntest (count, 1, true);
17731 dest = change_address (destmem, QImode, destptr);
17732 emit_insn (gen_strset (destptr, dest, gen_lowpart (QImode, value)));
17733 emit_label (label);
17734 LABEL_NUSES (label) = 1;
17738 /* Copy enough from DEST to SRC to align DEST known to by aligned by ALIGN to
17739 DESIRED_ALIGNMENT. */
17741 expand_movmem_prologue (rtx destmem, rtx srcmem,
17742 rtx destptr, rtx srcptr, rtx count,
17743 int align, int desired_alignment)
17745 if (align <= 1 && desired_alignment > 1)
17747 rtx label = ix86_expand_aligntest (destptr, 1, false);
17748 srcmem = change_address (srcmem, QImode, srcptr);
17749 destmem = change_address (destmem, QImode, destptr);
17750 emit_insn (gen_strmov (destptr, destmem, srcptr, srcmem));
17751 ix86_adjust_counter (count, 1);
17752 emit_label (label);
17753 LABEL_NUSES (label) = 1;
17755 if (align <= 2 && desired_alignment > 2)
17757 rtx label = ix86_expand_aligntest (destptr, 2, false);
17758 srcmem = change_address (srcmem, HImode, srcptr);
17759 destmem = change_address (destmem, HImode, destptr);
17760 emit_insn (gen_strmov (destptr, destmem, srcptr, srcmem));
17761 ix86_adjust_counter (count, 2);
17762 emit_label (label);
17763 LABEL_NUSES (label) = 1;
17765 if (align <= 4 && desired_alignment > 4)
17767 rtx label = ix86_expand_aligntest (destptr, 4, false);
17768 srcmem = change_address (srcmem, SImode, srcptr);
17769 destmem = change_address (destmem, SImode, destptr);
17770 emit_insn (gen_strmov (destptr, destmem, srcptr, srcmem));
17771 ix86_adjust_counter (count, 4);
17772 emit_label (label);
17773 LABEL_NUSES (label) = 1;
17775 gcc_assert (desired_alignment <= 8);
17778 /* Copy enough from DST to SRC to align DST known to DESIRED_ALIGN.
17779 ALIGN_BYTES is how many bytes need to be copied. */
17781 expand_constant_movmem_prologue (rtx dst, rtx *srcp, rtx destreg, rtx srcreg,
17782 int desired_align, int align_bytes)
17785 rtx src_size, dst_size;
17787 int src_align_bytes = get_mem_align_offset (src, desired_align * BITS_PER_UNIT);
17788 if (src_align_bytes >= 0)
17789 src_align_bytes = desired_align - src_align_bytes;
17790 src_size = MEM_SIZE (src);
17791 dst_size = MEM_SIZE (dst);
17792 if (align_bytes & 1)
17794 dst = adjust_automodify_address_nv (dst, QImode, destreg, 0);
17795 src = adjust_automodify_address_nv (src, QImode, srcreg, 0);
17797 emit_insn (gen_strmov (destreg, dst, srcreg, src));
17799 if (align_bytes & 2)
17801 dst = adjust_automodify_address_nv (dst, HImode, destreg, off);
17802 src = adjust_automodify_address_nv (src, HImode, srcreg, off);
17803 if (MEM_ALIGN (dst) < 2 * BITS_PER_UNIT)
17804 set_mem_align (dst, 2 * BITS_PER_UNIT);
17805 if (src_align_bytes >= 0
17806 && (src_align_bytes & 1) == (align_bytes & 1)
17807 && MEM_ALIGN (src) < 2 * BITS_PER_UNIT)
17808 set_mem_align (src, 2 * BITS_PER_UNIT);
17810 emit_insn (gen_strmov (destreg, dst, srcreg, src));
17812 if (align_bytes & 4)
17814 dst = adjust_automodify_address_nv (dst, SImode, destreg, off);
17815 src = adjust_automodify_address_nv (src, SImode, srcreg, off);
17816 if (MEM_ALIGN (dst) < 4 * BITS_PER_UNIT)
17817 set_mem_align (dst, 4 * BITS_PER_UNIT);
17818 if (src_align_bytes >= 0)
17820 unsigned int src_align = 0;
17821 if ((src_align_bytes & 3) == (align_bytes & 3))
17823 else if ((src_align_bytes & 1) == (align_bytes & 1))
17825 if (MEM_ALIGN (src) < src_align * BITS_PER_UNIT)
17826 set_mem_align (src, src_align * BITS_PER_UNIT);
17829 emit_insn (gen_strmov (destreg, dst, srcreg, src));
17831 dst = adjust_automodify_address_nv (dst, BLKmode, destreg, off);
17832 src = adjust_automodify_address_nv (src, BLKmode, srcreg, off);
17833 if (MEM_ALIGN (dst) < (unsigned int) desired_align * BITS_PER_UNIT)
17834 set_mem_align (dst, desired_align * BITS_PER_UNIT);
17835 if (src_align_bytes >= 0)
17837 unsigned int src_align = 0;
17838 if ((src_align_bytes & 7) == (align_bytes & 7))
17840 else if ((src_align_bytes & 3) == (align_bytes & 3))
17842 else if ((src_align_bytes & 1) == (align_bytes & 1))
17844 if (src_align > (unsigned int) desired_align)
17845 src_align = desired_align;
17846 if (MEM_ALIGN (src) < src_align * BITS_PER_UNIT)
17847 set_mem_align (src, src_align * BITS_PER_UNIT);
17850 set_mem_size (dst, GEN_INT (INTVAL (dst_size) - align_bytes));
17852 set_mem_size (dst, GEN_INT (INTVAL (src_size) - align_bytes));
17857 /* Set enough from DEST to align DEST known to by aligned by ALIGN to
17858 DESIRED_ALIGNMENT. */
17860 expand_setmem_prologue (rtx destmem, rtx destptr, rtx value, rtx count,
17861 int align, int desired_alignment)
17863 if (align <= 1 && desired_alignment > 1)
17865 rtx label = ix86_expand_aligntest (destptr, 1, false);
17866 destmem = change_address (destmem, QImode, destptr);
17867 emit_insn (gen_strset (destptr, destmem, gen_lowpart (QImode, value)));
17868 ix86_adjust_counter (count, 1);
17869 emit_label (label);
17870 LABEL_NUSES (label) = 1;
17872 if (align <= 2 && desired_alignment > 2)
17874 rtx label = ix86_expand_aligntest (destptr, 2, false);
17875 destmem = change_address (destmem, HImode, destptr);
17876 emit_insn (gen_strset (destptr, destmem, gen_lowpart (HImode, value)));
17877 ix86_adjust_counter (count, 2);
17878 emit_label (label);
17879 LABEL_NUSES (label) = 1;
17881 if (align <= 4 && desired_alignment > 4)
17883 rtx label = ix86_expand_aligntest (destptr, 4, false);
17884 destmem = change_address (destmem, SImode, destptr);
17885 emit_insn (gen_strset (destptr, destmem, gen_lowpart (SImode, value)));
17886 ix86_adjust_counter (count, 4);
17887 emit_label (label);
17888 LABEL_NUSES (label) = 1;
17890 gcc_assert (desired_alignment <= 8);
17893 /* Set enough from DST to align DST known to by aligned by ALIGN to
17894 DESIRED_ALIGN. ALIGN_BYTES is how many bytes need to be stored. */
17896 expand_constant_setmem_prologue (rtx dst, rtx destreg, rtx value,
17897 int desired_align, int align_bytes)
17900 rtx dst_size = MEM_SIZE (dst);
17901 if (align_bytes & 1)
17903 dst = adjust_automodify_address_nv (dst, QImode, destreg, 0);
17905 emit_insn (gen_strset (destreg, dst,
17906 gen_lowpart (QImode, value)));
17908 if (align_bytes & 2)
17910 dst = adjust_automodify_address_nv (dst, HImode, destreg, off);
17911 if (MEM_ALIGN (dst) < 2 * BITS_PER_UNIT)
17912 set_mem_align (dst, 2 * BITS_PER_UNIT);
17914 emit_insn (gen_strset (destreg, dst,
17915 gen_lowpart (HImode, value)));
17917 if (align_bytes & 4)
17919 dst = adjust_automodify_address_nv (dst, SImode, destreg, off);
17920 if (MEM_ALIGN (dst) < 4 * BITS_PER_UNIT)
17921 set_mem_align (dst, 4 * BITS_PER_UNIT);
17923 emit_insn (gen_strset (destreg, dst,
17924 gen_lowpart (SImode, value)));
17926 dst = adjust_automodify_address_nv (dst, BLKmode, destreg, off);
17927 if (MEM_ALIGN (dst) < (unsigned int) desired_align * BITS_PER_UNIT)
17928 set_mem_align (dst, desired_align * BITS_PER_UNIT);
17930 set_mem_size (dst, GEN_INT (INTVAL (dst_size) - align_bytes));
17934 /* Given COUNT and EXPECTED_SIZE, decide on codegen of string operation. */
17935 static enum stringop_alg
17936 decide_alg (HOST_WIDE_INT count, HOST_WIDE_INT expected_size, bool memset,
17937 int *dynamic_check)
17939 const struct stringop_algs * algs;
17940 bool optimize_for_speed;
17941 /* Algorithms using the rep prefix want at least edi and ecx;
17942 additionally, memset wants eax and memcpy wants esi. Don't
17943 consider such algorithms if the user has appropriated those
17944 registers for their own purposes. */
17945 bool rep_prefix_usable = !(fixed_regs[CX_REG] || fixed_regs[DI_REG]
17947 ? fixed_regs[AX_REG] : fixed_regs[SI_REG]));
17949 #define ALG_USABLE_P(alg) (rep_prefix_usable \
17950 || (alg != rep_prefix_1_byte \
17951 && alg != rep_prefix_4_byte \
17952 && alg != rep_prefix_8_byte))
17953 const struct processor_costs *cost;
17955 /* Even if the string operation call is cold, we still might spend a lot
17956 of time processing large blocks. */
17957 if (optimize_function_for_size_p (cfun)
17958 || (optimize_insn_for_size_p ()
17959 && expected_size != -1 && expected_size < 256))
17960 optimize_for_speed = false;
17962 optimize_for_speed = true;
17964 cost = optimize_for_speed ? ix86_cost : &ix86_size_cost;
17966 *dynamic_check = -1;
17968 algs = &cost->memset[TARGET_64BIT != 0];
17970 algs = &cost->memcpy[TARGET_64BIT != 0];
17971 if (stringop_alg != no_stringop && ALG_USABLE_P (stringop_alg))
17972 return stringop_alg;
17973 /* rep; movq or rep; movl is the smallest variant. */
17974 else if (!optimize_for_speed)
17976 if (!count || (count & 3))
17977 return rep_prefix_usable ? rep_prefix_1_byte : loop_1_byte;
17979 return rep_prefix_usable ? rep_prefix_4_byte : loop;
17981 /* Very tiny blocks are best handled via the loop, REP is expensive to setup.
17983 else if (expected_size != -1 && expected_size < 4)
17984 return loop_1_byte;
17985 else if (expected_size != -1)
17988 enum stringop_alg alg = libcall;
17989 for (i = 0; i < NAX_STRINGOP_ALGS; i++)
17991 /* We get here if the algorithms that were not libcall-based
17992 were rep-prefix based and we are unable to use rep prefixes
17993 based on global register usage. Break out of the loop and
17994 use the heuristic below. */
17995 if (algs->size[i].max == 0)
17997 if (algs->size[i].max >= expected_size || algs->size[i].max == -1)
17999 enum stringop_alg candidate = algs->size[i].alg;
18001 if (candidate != libcall && ALG_USABLE_P (candidate))
18003 /* Honor TARGET_INLINE_ALL_STRINGOPS by picking
18004 last non-libcall inline algorithm. */
18005 if (TARGET_INLINE_ALL_STRINGOPS)
18007 /* When the current size is best to be copied by a libcall,
18008 but we are still forced to inline, run the heuristic below
18009 that will pick code for medium sized blocks. */
18010 if (alg != libcall)
18014 else if (ALG_USABLE_P (candidate))
18018 gcc_assert (TARGET_INLINE_ALL_STRINGOPS || !rep_prefix_usable);
18020 /* When asked to inline the call anyway, try to pick meaningful choice.
18021 We look for maximal size of block that is faster to copy by hand and
18022 take blocks of at most of that size guessing that average size will
18023 be roughly half of the block.
18025 If this turns out to be bad, we might simply specify the preferred
18026 choice in ix86_costs. */
18027 if ((TARGET_INLINE_ALL_STRINGOPS || TARGET_INLINE_STRINGOPS_DYNAMICALLY)
18028 && (algs->unknown_size == libcall || !ALG_USABLE_P (algs->unknown_size)))
18031 enum stringop_alg alg;
18033 bool any_alg_usable_p = true;
18035 for (i = 0; i < NAX_STRINGOP_ALGS; i++)
18037 enum stringop_alg candidate = algs->size[i].alg;
18038 any_alg_usable_p = any_alg_usable_p && ALG_USABLE_P (candidate);
18040 if (candidate != libcall && candidate
18041 && ALG_USABLE_P (candidate))
18042 max = algs->size[i].max;
18044 /* If there aren't any usable algorithms, then recursing on
18045 smaller sizes isn't going to find anything. Just return the
18046 simple byte-at-a-time copy loop. */
18047 if (!any_alg_usable_p)
18049 /* Pick something reasonable. */
18050 if (TARGET_INLINE_STRINGOPS_DYNAMICALLY)
18051 *dynamic_check = 128;
18052 return loop_1_byte;
18056 alg = decide_alg (count, max / 2, memset, dynamic_check);
18057 gcc_assert (*dynamic_check == -1);
18058 gcc_assert (alg != libcall);
18059 if (TARGET_INLINE_STRINGOPS_DYNAMICALLY)
18060 *dynamic_check = max;
18063 return ALG_USABLE_P (algs->unknown_size) ? algs->unknown_size : libcall;
18064 #undef ALG_USABLE_P
18067 /* Decide on alignment. We know that the operand is already aligned to ALIGN
18068 (ALIGN can be based on profile feedback and thus it is not 100% guaranteed). */
18070 decide_alignment (int align,
18071 enum stringop_alg alg,
18074 int desired_align = 0;
18078 gcc_unreachable ();
18080 case unrolled_loop:
18081 desired_align = GET_MODE_SIZE (Pmode);
18083 case rep_prefix_8_byte:
18086 case rep_prefix_4_byte:
18087 /* PentiumPro has special logic triggering for 8 byte aligned blocks.
18088 copying whole cacheline at once. */
18089 if (TARGET_PENTIUMPRO)
18094 case rep_prefix_1_byte:
18095 /* PentiumPro has special logic triggering for 8 byte aligned blocks.
18096 copying whole cacheline at once. */
18097 if (TARGET_PENTIUMPRO)
18111 if (desired_align < align)
18112 desired_align = align;
18113 if (expected_size != -1 && expected_size < 4)
18114 desired_align = align;
18115 return desired_align;
18118 /* Return the smallest power of 2 greater than VAL. */
18120 smallest_pow2_greater_than (int val)
18128 /* Expand string move (memcpy) operation. Use i386 string operations when
18129 profitable. expand_setmem contains similar code. The code depends upon
18130 architecture, block size and alignment, but always has the same
18133 1) Prologue guard: Conditional that jumps up to epilogues for small
18134 blocks that can be handled by epilogue alone. This is faster but
18135 also needed for correctness, since prologue assume the block is larger
18136 than the desired alignment.
18138 Optional dynamic check for size and libcall for large
18139 blocks is emitted here too, with -minline-stringops-dynamically.
18141 2) Prologue: copy first few bytes in order to get destination aligned
18142 to DESIRED_ALIGN. It is emitted only when ALIGN is less than
18143 DESIRED_ALIGN and and up to DESIRED_ALIGN - ALIGN bytes can be copied.
18144 We emit either a jump tree on power of two sized blocks, or a byte loop.
18146 3) Main body: the copying loop itself, copying in SIZE_NEEDED chunks
18147 with specified algorithm.
18149 4) Epilogue: code copying tail of the block that is too small to be
18150 handled by main body (or up to size guarded by prologue guard). */
18153 ix86_expand_movmem (rtx dst, rtx src, rtx count_exp, rtx align_exp,
18154 rtx expected_align_exp, rtx expected_size_exp)
18160 rtx jump_around_label = NULL;
18161 HOST_WIDE_INT align = 1;
18162 unsigned HOST_WIDE_INT count = 0;
18163 HOST_WIDE_INT expected_size = -1;
18164 int size_needed = 0, epilogue_size_needed;
18165 int desired_align = 0, align_bytes = 0;
18166 enum stringop_alg alg;
18168 bool need_zero_guard = false;
18170 if (CONST_INT_P (align_exp))
18171 align = INTVAL (align_exp);
18172 /* i386 can do misaligned access on reasonably increased cost. */
18173 if (CONST_INT_P (expected_align_exp)
18174 && INTVAL (expected_align_exp) > align)
18175 align = INTVAL (expected_align_exp);
18176 /* ALIGN is the minimum of destination and source alignment, but we care here
18177 just about destination alignment. */
18178 else if (MEM_ALIGN (dst) > (unsigned HOST_WIDE_INT) align * BITS_PER_UNIT)
18179 align = MEM_ALIGN (dst) / BITS_PER_UNIT;
18181 if (CONST_INT_P (count_exp))
18182 count = expected_size = INTVAL (count_exp);
18183 if (CONST_INT_P (expected_size_exp) && count == 0)
18184 expected_size = INTVAL (expected_size_exp);
18186 /* Make sure we don't need to care about overflow later on. */
18187 if (count > ((unsigned HOST_WIDE_INT) 1 << 30))
18190 /* Step 0: Decide on preferred algorithm, desired alignment and
18191 size of chunks to be copied by main loop. */
18193 alg = decide_alg (count, expected_size, false, &dynamic_check);
18194 desired_align = decide_alignment (align, alg, expected_size);
18196 if (!TARGET_ALIGN_STRINGOPS)
18197 align = desired_align;
18199 if (alg == libcall)
18201 gcc_assert (alg != no_stringop);
18203 count_exp = copy_to_mode_reg (GET_MODE (count_exp), count_exp);
18204 destreg = copy_to_mode_reg (Pmode, XEXP (dst, 0));
18205 srcreg = copy_to_mode_reg (Pmode, XEXP (src, 0));
18210 gcc_unreachable ();
18212 need_zero_guard = true;
18213 size_needed = GET_MODE_SIZE (Pmode);
18215 case unrolled_loop:
18216 need_zero_guard = true;
18217 size_needed = GET_MODE_SIZE (Pmode) * (TARGET_64BIT ? 4 : 2);
18219 case rep_prefix_8_byte:
18222 case rep_prefix_4_byte:
18225 case rep_prefix_1_byte:
18229 need_zero_guard = true;
18234 epilogue_size_needed = size_needed;
18236 /* Step 1: Prologue guard. */
18238 /* Alignment code needs count to be in register. */
18239 if (CONST_INT_P (count_exp) && desired_align > align)
18241 if (INTVAL (count_exp) > desired_align
18242 && INTVAL (count_exp) > size_needed)
18245 = get_mem_align_offset (dst, desired_align * BITS_PER_UNIT);
18246 if (align_bytes <= 0)
18249 align_bytes = desired_align - align_bytes;
18251 if (align_bytes == 0)
18252 count_exp = force_reg (counter_mode (count_exp), count_exp);
18254 gcc_assert (desired_align >= 1 && align >= 1);
18256 /* Ensure that alignment prologue won't copy past end of block. */
18257 if (size_needed > 1 || (desired_align > 1 && desired_align > align))
18259 epilogue_size_needed = MAX (size_needed - 1, desired_align - align);
18260 /* Epilogue always copies COUNT_EXP & EPILOGUE_SIZE_NEEDED bytes.
18261 Make sure it is power of 2. */
18262 epilogue_size_needed = smallest_pow2_greater_than (epilogue_size_needed);
18266 if (count < (unsigned HOST_WIDE_INT)epilogue_size_needed)
18268 /* If main algorithm works on QImode, no epilogue is needed.
18269 For small sizes just don't align anything. */
18270 if (size_needed == 1)
18271 desired_align = align;
18278 label = gen_label_rtx ();
18279 emit_cmp_and_jump_insns (count_exp,
18280 GEN_INT (epilogue_size_needed),
18281 LTU, 0, counter_mode (count_exp), 1, label);
18282 if (expected_size == -1 || expected_size < epilogue_size_needed)
18283 predict_jump (REG_BR_PROB_BASE * 60 / 100);
18285 predict_jump (REG_BR_PROB_BASE * 20 / 100);
18289 /* Emit code to decide on runtime whether library call or inline should be
18291 if (dynamic_check != -1)
18293 if (CONST_INT_P (count_exp))
18295 if (UINTVAL (count_exp) >= (unsigned HOST_WIDE_INT)dynamic_check)
18297 emit_block_move_via_libcall (dst, src, count_exp, false);
18298 count_exp = const0_rtx;
18304 rtx hot_label = gen_label_rtx ();
18305 jump_around_label = gen_label_rtx ();
18306 emit_cmp_and_jump_insns (count_exp, GEN_INT (dynamic_check - 1),
18307 LEU, 0, GET_MODE (count_exp), 1, hot_label);
18308 predict_jump (REG_BR_PROB_BASE * 90 / 100);
18309 emit_block_move_via_libcall (dst, src, count_exp, false);
18310 emit_jump (jump_around_label);
18311 emit_label (hot_label);
18315 /* Step 2: Alignment prologue. */
18317 if (desired_align > align)
18319 if (align_bytes == 0)
18321 /* Except for the first move in epilogue, we no longer know
18322 constant offset in aliasing info. It don't seems to worth
18323 the pain to maintain it for the first move, so throw away
18325 src = change_address (src, BLKmode, srcreg);
18326 dst = change_address (dst, BLKmode, destreg);
18327 expand_movmem_prologue (dst, src, destreg, srcreg, count_exp, align,
18332 /* If we know how many bytes need to be stored before dst is
18333 sufficiently aligned, maintain aliasing info accurately. */
18334 dst = expand_constant_movmem_prologue (dst, &src, destreg, srcreg,
18335 desired_align, align_bytes);
18336 count_exp = plus_constant (count_exp, -align_bytes);
18337 count -= align_bytes;
18339 if (need_zero_guard
18340 && (count < (unsigned HOST_WIDE_INT) size_needed
18341 || (align_bytes == 0
18342 && count < ((unsigned HOST_WIDE_INT) size_needed
18343 + desired_align - align))))
18345 /* It is possible that we copied enough so the main loop will not
18347 gcc_assert (size_needed > 1);
18348 if (label == NULL_RTX)
18349 label = gen_label_rtx ();
18350 emit_cmp_and_jump_insns (count_exp,
18351 GEN_INT (size_needed),
18352 LTU, 0, counter_mode (count_exp), 1, label);
18353 if (expected_size == -1
18354 || expected_size < (desired_align - align) / 2 + size_needed)
18355 predict_jump (REG_BR_PROB_BASE * 20 / 100);
18357 predict_jump (REG_BR_PROB_BASE * 60 / 100);
18360 if (label && size_needed == 1)
18362 emit_label (label);
18363 LABEL_NUSES (label) = 1;
18365 epilogue_size_needed = 1;
18367 else if (label == NULL_RTX)
18368 epilogue_size_needed = size_needed;
18370 /* Step 3: Main loop. */
18376 gcc_unreachable ();
18378 expand_set_or_movmem_via_loop (dst, src, destreg, srcreg, NULL,
18379 count_exp, QImode, 1, expected_size);
18382 expand_set_or_movmem_via_loop (dst, src, destreg, srcreg, NULL,
18383 count_exp, Pmode, 1, expected_size);
18385 case unrolled_loop:
18386 /* Unroll only by factor of 2 in 32bit mode, since we don't have enough
18387 registers for 4 temporaries anyway. */
18388 expand_set_or_movmem_via_loop (dst, src, destreg, srcreg, NULL,
18389 count_exp, Pmode, TARGET_64BIT ? 4 : 2,
18392 case rep_prefix_8_byte:
18393 expand_movmem_via_rep_mov (dst, src, destreg, srcreg, count_exp,
18396 case rep_prefix_4_byte:
18397 expand_movmem_via_rep_mov (dst, src, destreg, srcreg, count_exp,
18400 case rep_prefix_1_byte:
18401 expand_movmem_via_rep_mov (dst, src, destreg, srcreg, count_exp,
18405 /* Adjust properly the offset of src and dest memory for aliasing. */
18406 if (CONST_INT_P (count_exp))
18408 src = adjust_automodify_address_nv (src, BLKmode, srcreg,
18409 (count / size_needed) * size_needed);
18410 dst = adjust_automodify_address_nv (dst, BLKmode, destreg,
18411 (count / size_needed) * size_needed);
18415 src = change_address (src, BLKmode, srcreg);
18416 dst = change_address (dst, BLKmode, destreg);
18419 /* Step 4: Epilogue to copy the remaining bytes. */
18423 /* When the main loop is done, COUNT_EXP might hold original count,
18424 while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
18425 Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
18426 bytes. Compensate if needed. */
18428 if (size_needed < epilogue_size_needed)
18431 expand_simple_binop (counter_mode (count_exp), AND, count_exp,
18432 GEN_INT (size_needed - 1), count_exp, 1,
18434 if (tmp != count_exp)
18435 emit_move_insn (count_exp, tmp);
18437 emit_label (label);
18438 LABEL_NUSES (label) = 1;
18441 if (count_exp != const0_rtx && epilogue_size_needed > 1)
18442 expand_movmem_epilogue (dst, src, destreg, srcreg, count_exp,
18443 epilogue_size_needed);
18444 if (jump_around_label)
18445 emit_label (jump_around_label);
18449 /* Helper function for memcpy. For QImode value 0xXY produce
18450 0xXYXYXYXY of wide specified by MODE. This is essentially
18451 a * 0x10101010, but we can do slightly better than
18452 synth_mult by unwinding the sequence by hand on CPUs with
18455 promote_duplicated_reg (enum machine_mode mode, rtx val)
18457 enum machine_mode valmode = GET_MODE (val);
18459 int nops = mode == DImode ? 3 : 2;
18461 gcc_assert (mode == SImode || mode == DImode);
18462 if (val == const0_rtx)
18463 return copy_to_mode_reg (mode, const0_rtx);
18464 if (CONST_INT_P (val))
18466 HOST_WIDE_INT v = INTVAL (val) & 255;
18470 if (mode == DImode)
18471 v |= (v << 16) << 16;
18472 return copy_to_mode_reg (mode, gen_int_mode (v, mode));
18475 if (valmode == VOIDmode)
18477 if (valmode != QImode)
18478 val = gen_lowpart (QImode, val);
18479 if (mode == QImode)
18481 if (!TARGET_PARTIAL_REG_STALL)
18483 if (ix86_cost->mult_init[mode == DImode ? 3 : 2]
18484 + ix86_cost->mult_bit * (mode == DImode ? 8 : 4)
18485 <= (ix86_cost->shift_const + ix86_cost->add) * nops
18486 + (COSTS_N_INSNS (TARGET_PARTIAL_REG_STALL == 0)))
18488 rtx reg = convert_modes (mode, QImode, val, true);
18489 tmp = promote_duplicated_reg (mode, const1_rtx);
18490 return expand_simple_binop (mode, MULT, reg, tmp, NULL, 1,
18495 rtx reg = convert_modes (mode, QImode, val, true);
18497 if (!TARGET_PARTIAL_REG_STALL)
18498 if (mode == SImode)
18499 emit_insn (gen_movsi_insv_1 (reg, reg));
18501 emit_insn (gen_movdi_insv_1_rex64 (reg, reg));
18504 tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (8),
18505 NULL, 1, OPTAB_DIRECT);
18507 expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
18509 tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (16),
18510 NULL, 1, OPTAB_DIRECT);
18511 reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
18512 if (mode == SImode)
18514 tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (32),
18515 NULL, 1, OPTAB_DIRECT);
18516 reg = expand_simple_binop (mode, IOR, reg, tmp, reg, 1, OPTAB_DIRECT);
18521 /* Duplicate value VAL using promote_duplicated_reg into maximal size that will
18522 be needed by main loop copying SIZE_NEEDED chunks and prologue getting
18523 alignment from ALIGN to DESIRED_ALIGN. */
18525 promote_duplicated_reg_to_size (rtx val, int size_needed, int desired_align, int align)
18530 && (size_needed > 4 || (desired_align > align && desired_align > 4)))
18531 promoted_val = promote_duplicated_reg (DImode, val);
18532 else if (size_needed > 2 || (desired_align > align && desired_align > 2))
18533 promoted_val = promote_duplicated_reg (SImode, val);
18534 else if (size_needed > 1 || (desired_align > align && desired_align > 1))
18535 promoted_val = promote_duplicated_reg (HImode, val);
18537 promoted_val = val;
18539 return promoted_val;
18542 /* Expand string clear operation (bzero). Use i386 string operations when
18543 profitable. See expand_movmem comment for explanation of individual
18544 steps performed. */
18546 ix86_expand_setmem (rtx dst, rtx count_exp, rtx val_exp, rtx align_exp,
18547 rtx expected_align_exp, rtx expected_size_exp)
18552 rtx jump_around_label = NULL;
18553 HOST_WIDE_INT align = 1;
18554 unsigned HOST_WIDE_INT count = 0;
18555 HOST_WIDE_INT expected_size = -1;
18556 int size_needed = 0, epilogue_size_needed;
18557 int desired_align = 0, align_bytes = 0;
18558 enum stringop_alg alg;
18559 rtx promoted_val = NULL;
18560 bool force_loopy_epilogue = false;
18562 bool need_zero_guard = false;
18564 if (CONST_INT_P (align_exp))
18565 align = INTVAL (align_exp);
18566 /* i386 can do misaligned access on reasonably increased cost. */
18567 if (CONST_INT_P (expected_align_exp)
18568 && INTVAL (expected_align_exp) > align)
18569 align = INTVAL (expected_align_exp);
18570 if (CONST_INT_P (count_exp))
18571 count = expected_size = INTVAL (count_exp);
18572 if (CONST_INT_P (expected_size_exp) && count == 0)
18573 expected_size = INTVAL (expected_size_exp);
18575 /* Make sure we don't need to care about overflow later on. */
18576 if (count > ((unsigned HOST_WIDE_INT) 1 << 30))
18579 /* Step 0: Decide on preferred algorithm, desired alignment and
18580 size of chunks to be copied by main loop. */
18582 alg = decide_alg (count, expected_size, true, &dynamic_check);
18583 desired_align = decide_alignment (align, alg, expected_size);
18585 if (!TARGET_ALIGN_STRINGOPS)
18586 align = desired_align;
18588 if (alg == libcall)
18590 gcc_assert (alg != no_stringop);
18592 count_exp = copy_to_mode_reg (counter_mode (count_exp), count_exp);
18593 destreg = copy_to_mode_reg (Pmode, XEXP (dst, 0));
18598 gcc_unreachable ();
18600 need_zero_guard = true;
18601 size_needed = GET_MODE_SIZE (Pmode);
18603 case unrolled_loop:
18604 need_zero_guard = true;
18605 size_needed = GET_MODE_SIZE (Pmode) * 4;
18607 case rep_prefix_8_byte:
18610 case rep_prefix_4_byte:
18613 case rep_prefix_1_byte:
18617 need_zero_guard = true;
18621 epilogue_size_needed = size_needed;
18623 /* Step 1: Prologue guard. */
18625 /* Alignment code needs count to be in register. */
18626 if (CONST_INT_P (count_exp) && desired_align > align)
18628 if (INTVAL (count_exp) > desired_align
18629 && INTVAL (count_exp) > size_needed)
18632 = get_mem_align_offset (dst, desired_align * BITS_PER_UNIT);
18633 if (align_bytes <= 0)
18636 align_bytes = desired_align - align_bytes;
18638 if (align_bytes == 0)
18640 enum machine_mode mode = SImode;
18641 if (TARGET_64BIT && (count & ~0xffffffff))
18643 count_exp = force_reg (mode, count_exp);
18646 /* Do the cheap promotion to allow better CSE across the
18647 main loop and epilogue (ie one load of the big constant in the
18648 front of all code. */
18649 if (CONST_INT_P (val_exp))
18650 promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
18651 desired_align, align);
18652 /* Ensure that alignment prologue won't copy past end of block. */
18653 if (size_needed > 1 || (desired_align > 1 && desired_align > align))
18655 epilogue_size_needed = MAX (size_needed - 1, desired_align - align);
18656 /* Epilogue always copies COUNT_EXP & (EPILOGUE_SIZE_NEEDED - 1) bytes.
18657 Make sure it is power of 2. */
18658 epilogue_size_needed = smallest_pow2_greater_than (epilogue_size_needed);
18660 /* To improve performance of small blocks, we jump around the VAL
18661 promoting mode. This mean that if the promoted VAL is not constant,
18662 we might not use it in the epilogue and have to use byte
18664 if (epilogue_size_needed > 2 && !promoted_val)
18665 force_loopy_epilogue = true;
18668 if (count < (unsigned HOST_WIDE_INT)epilogue_size_needed)
18670 /* If main algorithm works on QImode, no epilogue is needed.
18671 For small sizes just don't align anything. */
18672 if (size_needed == 1)
18673 desired_align = align;
18680 label = gen_label_rtx ();
18681 emit_cmp_and_jump_insns (count_exp,
18682 GEN_INT (epilogue_size_needed),
18683 LTU, 0, counter_mode (count_exp), 1, label);
18684 if (expected_size == -1 || expected_size <= epilogue_size_needed)
18685 predict_jump (REG_BR_PROB_BASE * 60 / 100);
18687 predict_jump (REG_BR_PROB_BASE * 20 / 100);
18690 if (dynamic_check != -1)
18692 rtx hot_label = gen_label_rtx ();
18693 jump_around_label = gen_label_rtx ();
18694 emit_cmp_and_jump_insns (count_exp, GEN_INT (dynamic_check - 1),
18695 LEU, 0, counter_mode (count_exp), 1, hot_label);
18696 predict_jump (REG_BR_PROB_BASE * 90 / 100);
18697 set_storage_via_libcall (dst, count_exp, val_exp, false);
18698 emit_jump (jump_around_label);
18699 emit_label (hot_label);
18702 /* Step 2: Alignment prologue. */
18704 /* Do the expensive promotion once we branched off the small blocks. */
18706 promoted_val = promote_duplicated_reg_to_size (val_exp, size_needed,
18707 desired_align, align);
18708 gcc_assert (desired_align >= 1 && align >= 1);
18710 if (desired_align > align)
18712 if (align_bytes == 0)
18714 /* Except for the first move in epilogue, we no longer know
18715 constant offset in aliasing info. It don't seems to worth
18716 the pain to maintain it for the first move, so throw away
18718 dst = change_address (dst, BLKmode, destreg);
18719 expand_setmem_prologue (dst, destreg, promoted_val, count_exp, align,
18724 /* If we know how many bytes need to be stored before dst is
18725 sufficiently aligned, maintain aliasing info accurately. */
18726 dst = expand_constant_setmem_prologue (dst, destreg, promoted_val,
18727 desired_align, align_bytes);
18728 count_exp = plus_constant (count_exp, -align_bytes);
18729 count -= align_bytes;
18731 if (need_zero_guard
18732 && (count < (unsigned HOST_WIDE_INT) size_needed
18733 || (align_bytes == 0
18734 && count < ((unsigned HOST_WIDE_INT) size_needed
18735 + desired_align - align))))
18737 /* It is possible that we copied enough so the main loop will not
18739 gcc_assert (size_needed > 1);
18740 if (label == NULL_RTX)
18741 label = gen_label_rtx ();
18742 emit_cmp_and_jump_insns (count_exp,
18743 GEN_INT (size_needed),
18744 LTU, 0, counter_mode (count_exp), 1, label);
18745 if (expected_size == -1
18746 || expected_size < (desired_align - align) / 2 + size_needed)
18747 predict_jump (REG_BR_PROB_BASE * 20 / 100);
18749 predict_jump (REG_BR_PROB_BASE * 60 / 100);
18752 if (label && size_needed == 1)
18754 emit_label (label);
18755 LABEL_NUSES (label) = 1;
18757 promoted_val = val_exp;
18758 epilogue_size_needed = 1;
18760 else if (label == NULL_RTX)
18761 epilogue_size_needed = size_needed;
18763 /* Step 3: Main loop. */
18769 gcc_unreachable ();
18771 expand_set_or_movmem_via_loop (dst, NULL, destreg, NULL, promoted_val,
18772 count_exp, QImode, 1, expected_size);
18775 expand_set_or_movmem_via_loop (dst, NULL, destreg, NULL, promoted_val,
18776 count_exp, Pmode, 1, expected_size);
18778 case unrolled_loop:
18779 expand_set_or_movmem_via_loop (dst, NULL, destreg, NULL, promoted_val,
18780 count_exp, Pmode, 4, expected_size);
18782 case rep_prefix_8_byte:
18783 expand_setmem_via_rep_stos (dst, destreg, promoted_val, count_exp,
18786 case rep_prefix_4_byte:
18787 expand_setmem_via_rep_stos (dst, destreg, promoted_val, count_exp,
18790 case rep_prefix_1_byte:
18791 expand_setmem_via_rep_stos (dst, destreg, promoted_val, count_exp,
18795 /* Adjust properly the offset of src and dest memory for aliasing. */
18796 if (CONST_INT_P (count_exp))
18797 dst = adjust_automodify_address_nv (dst, BLKmode, destreg,
18798 (count / size_needed) * size_needed);
18800 dst = change_address (dst, BLKmode, destreg);
18802 /* Step 4: Epilogue to copy the remaining bytes. */
18806 /* When the main loop is done, COUNT_EXP might hold original count,
18807 while we want to copy only COUNT_EXP & SIZE_NEEDED bytes.
18808 Epilogue code will actually copy COUNT_EXP & EPILOGUE_SIZE_NEEDED
18809 bytes. Compensate if needed. */
18811 if (size_needed < epilogue_size_needed)
18814 expand_simple_binop (counter_mode (count_exp), AND, count_exp,
18815 GEN_INT (size_needed - 1), count_exp, 1,
18817 if (tmp != count_exp)
18818 emit_move_insn (count_exp, tmp);
18820 emit_label (label);
18821 LABEL_NUSES (label) = 1;
18824 if (count_exp != const0_rtx && epilogue_size_needed > 1)
18826 if (force_loopy_epilogue)
18827 expand_setmem_epilogue_via_loop (dst, destreg, val_exp, count_exp,
18828 epilogue_size_needed);
18830 expand_setmem_epilogue (dst, destreg, promoted_val, count_exp,
18831 epilogue_size_needed);
18833 if (jump_around_label)
18834 emit_label (jump_around_label);
18838 /* Expand the appropriate insns for doing strlen if not just doing
18841 out = result, initialized with the start address
18842 align_rtx = alignment of the address.
18843 scratch = scratch register, initialized with the startaddress when
18844 not aligned, otherwise undefined
18846 This is just the body. It needs the initializations mentioned above and
18847 some address computing at the end. These things are done in i386.md. */
18850 ix86_expand_strlensi_unroll_1 (rtx out, rtx src, rtx align_rtx)
18854 rtx align_2_label = NULL_RTX;
18855 rtx align_3_label = NULL_RTX;
18856 rtx align_4_label = gen_label_rtx ();
18857 rtx end_0_label = gen_label_rtx ();
18859 rtx tmpreg = gen_reg_rtx (SImode);
18860 rtx scratch = gen_reg_rtx (SImode);
18864 if (CONST_INT_P (align_rtx))
18865 align = INTVAL (align_rtx);
18867 /* Loop to check 1..3 bytes for null to get an aligned pointer. */
18869 /* Is there a known alignment and is it less than 4? */
18872 rtx scratch1 = gen_reg_rtx (Pmode);
18873 emit_move_insn (scratch1, out);
18874 /* Is there a known alignment and is it not 2? */
18877 align_3_label = gen_label_rtx (); /* Label when aligned to 3-byte */
18878 align_2_label = gen_label_rtx (); /* Label when aligned to 2-byte */
18880 /* Leave just the 3 lower bits. */
18881 align_rtx = expand_binop (Pmode, and_optab, scratch1, GEN_INT (3),
18882 NULL_RTX, 0, OPTAB_WIDEN);
18884 emit_cmp_and_jump_insns (align_rtx, const0_rtx, EQ, NULL,
18885 Pmode, 1, align_4_label);
18886 emit_cmp_and_jump_insns (align_rtx, const2_rtx, EQ, NULL,
18887 Pmode, 1, align_2_label);
18888 emit_cmp_and_jump_insns (align_rtx, const2_rtx, GTU, NULL,
18889 Pmode, 1, align_3_label);
18893 /* Since the alignment is 2, we have to check 2 or 0 bytes;
18894 check if is aligned to 4 - byte. */
18896 align_rtx = expand_binop (Pmode, and_optab, scratch1, const2_rtx,
18897 NULL_RTX, 0, OPTAB_WIDEN);
18899 emit_cmp_and_jump_insns (align_rtx, const0_rtx, EQ, NULL,
18900 Pmode, 1, align_4_label);
18903 mem = change_address (src, QImode, out);
18905 /* Now compare the bytes. */
18907 /* Compare the first n unaligned byte on a byte per byte basis. */
18908 emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL,
18909 QImode, 1, end_0_label);
18911 /* Increment the address. */
18912 emit_insn ((*ix86_gen_add3) (out, out, const1_rtx));
18914 /* Not needed with an alignment of 2 */
18917 emit_label (align_2_label);
18919 emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1,
18922 emit_insn ((*ix86_gen_add3) (out, out, const1_rtx));
18924 emit_label (align_3_label);
18927 emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1,
18930 emit_insn ((*ix86_gen_add3) (out, out, const1_rtx));
18933 /* Generate loop to check 4 bytes at a time. It is not a good idea to
18934 align this loop. It gives only huge programs, but does not help to
18936 emit_label (align_4_label);
18938 mem = change_address (src, SImode, out);
18939 emit_move_insn (scratch, mem);
18940 emit_insn ((*ix86_gen_add3) (out, out, GEN_INT (4)));
18942 /* This formula yields a nonzero result iff one of the bytes is zero.
18943 This saves three branches inside loop and many cycles. */
18945 emit_insn (gen_addsi3 (tmpreg, scratch, GEN_INT (-0x01010101)));
18946 emit_insn (gen_one_cmplsi2 (scratch, scratch));
18947 emit_insn (gen_andsi3 (tmpreg, tmpreg, scratch));
18948 emit_insn (gen_andsi3 (tmpreg, tmpreg,
18949 gen_int_mode (0x80808080, SImode)));
18950 emit_cmp_and_jump_insns (tmpreg, const0_rtx, EQ, 0, SImode, 1,
18955 rtx reg = gen_reg_rtx (SImode);
18956 rtx reg2 = gen_reg_rtx (Pmode);
18957 emit_move_insn (reg, tmpreg);
18958 emit_insn (gen_lshrsi3 (reg, reg, GEN_INT (16)));
18960 /* If zero is not in the first two bytes, move two bytes forward. */
18961 emit_insn (gen_testsi_ccno_1 (tmpreg, GEN_INT (0x8080)));
18962 tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
18963 tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
18964 emit_insn (gen_rtx_SET (VOIDmode, tmpreg,
18965 gen_rtx_IF_THEN_ELSE (SImode, tmp,
18968 /* Emit lea manually to avoid clobbering of flags. */
18969 emit_insn (gen_rtx_SET (SImode, reg2,
18970 gen_rtx_PLUS (Pmode, out, const2_rtx)));
18972 tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
18973 tmp = gen_rtx_EQ (VOIDmode, tmp, const0_rtx);
18974 emit_insn (gen_rtx_SET (VOIDmode, out,
18975 gen_rtx_IF_THEN_ELSE (Pmode, tmp,
18982 rtx end_2_label = gen_label_rtx ();
18983 /* Is zero in the first two bytes? */
18985 emit_insn (gen_testsi_ccno_1 (tmpreg, GEN_INT (0x8080)));
18986 tmp = gen_rtx_REG (CCNOmode, FLAGS_REG);
18987 tmp = gen_rtx_NE (VOIDmode, tmp, const0_rtx);
18988 tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
18989 gen_rtx_LABEL_REF (VOIDmode, end_2_label),
18991 tmp = emit_jump_insn (gen_rtx_SET (VOIDmode, pc_rtx, tmp));
18992 JUMP_LABEL (tmp) = end_2_label;
18994 /* Not in the first two. Move two bytes forward. */
18995 emit_insn (gen_lshrsi3 (tmpreg, tmpreg, GEN_INT (16)));
18996 emit_insn ((*ix86_gen_add3) (out, out, const2_rtx));
18998 emit_label (end_2_label);
19002 /* Avoid branch in fixing the byte. */
19003 tmpreg = gen_lowpart (QImode, tmpreg);
19004 emit_insn (gen_addqi3_cc (tmpreg, tmpreg, tmpreg));
19005 cmp = gen_rtx_LTU (Pmode, gen_rtx_REG (CCmode, FLAGS_REG), const0_rtx);
19006 emit_insn ((*ix86_gen_sub3_carry) (out, out, GEN_INT (3), cmp));
19008 emit_label (end_0_label);
19011 /* Expand strlen. */
19014 ix86_expand_strlen (rtx out, rtx src, rtx eoschar, rtx align)
19016 rtx addr, scratch1, scratch2, scratch3, scratch4;
19018 /* The generic case of strlen expander is long. Avoid it's
19019 expanding unless TARGET_INLINE_ALL_STRINGOPS. */
19021 if (TARGET_UNROLL_STRLEN && eoschar == const0_rtx && optimize > 1
19022 && !TARGET_INLINE_ALL_STRINGOPS
19023 && !optimize_insn_for_size_p ()
19024 && (!CONST_INT_P (align) || INTVAL (align) < 4))
19027 addr = force_reg (Pmode, XEXP (src, 0));
19028 scratch1 = gen_reg_rtx (Pmode);
19030 if (TARGET_UNROLL_STRLEN && eoschar == const0_rtx && optimize > 1
19031 && !optimize_insn_for_size_p ())
19033 /* Well it seems that some optimizer does not combine a call like
19034 foo(strlen(bar), strlen(bar));
19035 when the move and the subtraction is done here. It does calculate
19036 the length just once when these instructions are done inside of
19037 output_strlen_unroll(). But I think since &bar[strlen(bar)] is
19038 often used and I use one fewer register for the lifetime of
19039 output_strlen_unroll() this is better. */
19041 emit_move_insn (out, addr);
19043 ix86_expand_strlensi_unroll_1 (out, src, align);
19045 /* strlensi_unroll_1 returns the address of the zero at the end of
19046 the string, like memchr(), so compute the length by subtracting
19047 the start address. */
19048 emit_insn ((*ix86_gen_sub3) (out, out, addr));
19054 /* Can't use this if the user has appropriated eax, ecx, or edi. */
19055 if (fixed_regs[AX_REG] || fixed_regs[CX_REG] || fixed_regs[DI_REG])
19058 scratch2 = gen_reg_rtx (Pmode);
19059 scratch3 = gen_reg_rtx (Pmode);
19060 scratch4 = force_reg (Pmode, constm1_rtx);
19062 emit_move_insn (scratch3, addr);
19063 eoschar = force_reg (QImode, eoschar);
19065 src = replace_equiv_address_nv (src, scratch3);
19067 /* If .md starts supporting :P, this can be done in .md. */
19068 unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (4, src, eoschar, align,
19069 scratch4), UNSPEC_SCAS);
19070 emit_insn (gen_strlenqi_1 (scratch1, scratch3, unspec));
19071 emit_insn ((*ix86_gen_one_cmpl2) (scratch2, scratch1));
19072 emit_insn ((*ix86_gen_add3) (out, scratch2, constm1_rtx));
19077 /* For given symbol (function) construct code to compute address of it's PLT
19078 entry in large x86-64 PIC model. */
19080 construct_plt_address (rtx symbol)
19082 rtx tmp = gen_reg_rtx (Pmode);
19083 rtx unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, symbol), UNSPEC_PLTOFF);
19085 gcc_assert (GET_CODE (symbol) == SYMBOL_REF);
19086 gcc_assert (ix86_cmodel == CM_LARGE_PIC);
19088 emit_move_insn (tmp, gen_rtx_CONST (Pmode, unspec));
19089 emit_insn (gen_adddi3 (tmp, tmp, pic_offset_table_rtx));
19094 ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1,
19096 rtx pop, int sibcall)
19098 rtx use = NULL, call;
19100 if (pop == const0_rtx)
19102 gcc_assert (!TARGET_64BIT || !pop);
19104 if (TARGET_MACHO && !TARGET_64BIT)
19107 if (flag_pic && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF)
19108 fnaddr = machopic_indirect_call_target (fnaddr);
19113 /* Static functions and indirect calls don't need the pic register. */
19114 if (flag_pic && (!TARGET_64BIT || ix86_cmodel == CM_LARGE_PIC)
19115 && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF
19116 && ! SYMBOL_REF_LOCAL_P (XEXP (fnaddr, 0)))
19117 use_reg (&use, pic_offset_table_rtx);
19120 if (TARGET_64BIT && INTVAL (callarg2) >= 0)
19122 rtx al = gen_rtx_REG (QImode, AX_REG);
19123 emit_move_insn (al, callarg2);
19124 use_reg (&use, al);
19127 if (ix86_cmodel == CM_LARGE_PIC
19128 && GET_CODE (fnaddr) == MEM
19129 && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF
19130 && !local_symbolic_operand (XEXP (fnaddr, 0), VOIDmode))
19131 fnaddr = gen_rtx_MEM (QImode, construct_plt_address (XEXP (fnaddr, 0)));
19132 else if (! call_insn_operand (XEXP (fnaddr, 0), Pmode))
19134 fnaddr = copy_to_mode_reg (Pmode, XEXP (fnaddr, 0));
19135 fnaddr = gen_rtx_MEM (QImode, fnaddr);
19137 if (sibcall && TARGET_64BIT
19138 && !constant_call_address_operand (XEXP (fnaddr, 0), Pmode))
19141 addr = copy_to_mode_reg (Pmode, XEXP (fnaddr, 0));
19142 fnaddr = gen_rtx_REG (Pmode, R11_REG);
19143 emit_move_insn (fnaddr, addr);
19144 fnaddr = gen_rtx_MEM (QImode, fnaddr);
19147 call = gen_rtx_CALL (VOIDmode, fnaddr, callarg1);
19149 call = gen_rtx_SET (VOIDmode, retval, call);
19152 pop = gen_rtx_PLUS (Pmode, stack_pointer_rtx, pop);
19153 pop = gen_rtx_SET (VOIDmode, stack_pointer_rtx, pop);
19154 call = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, call, pop));
19157 && ix86_cfun_abi () == MS_ABI
19158 && (!callarg2 || INTVAL (callarg2) != -2))
19160 /* We need to represent that SI and DI registers are clobbered
19162 static int clobbered_registers[] = {
19163 XMM6_REG, XMM7_REG, XMM8_REG,
19164 XMM9_REG, XMM10_REG, XMM11_REG,
19165 XMM12_REG, XMM13_REG, XMM14_REG,
19166 XMM15_REG, SI_REG, DI_REG
19169 rtx vec[ARRAY_SIZE (clobbered_registers) + 2];
19170 rtx unspec = gen_rtx_UNSPEC (VOIDmode, gen_rtvec (1, const0_rtx),
19171 UNSPEC_MS_TO_SYSV_CALL);
19175 for (i = 0; i < ARRAY_SIZE (clobbered_registers); i++)
19176 vec[i + 2] = gen_rtx_CLOBBER (SSE_REGNO_P (clobbered_registers[i])
19179 (SSE_REGNO_P (clobbered_registers[i])
19181 clobbered_registers[i]));
19183 call = gen_rtx_PARALLEL (VOIDmode,
19184 gen_rtvec_v (ARRAY_SIZE (clobbered_registers)
19188 call = emit_call_insn (call);
19190 CALL_INSN_FUNCTION_USAGE (call) = use;
19194 /* Clear stack slot assignments remembered from previous functions.
19195 This is called from INIT_EXPANDERS once before RTL is emitted for each
19198 static struct machine_function *
19199 ix86_init_machine_status (void)
19201 struct machine_function *f;
19203 f = GGC_CNEW (struct machine_function);
19204 f->use_fast_prologue_epilogue_nregs = -1;
19205 f->tls_descriptor_call_expanded_p = 0;
19206 f->call_abi = ix86_abi;
19211 /* Return a MEM corresponding to a stack slot with mode MODE.
19212 Allocate a new slot if necessary.
19214 The RTL for a function can have several slots available: N is
19215 which slot to use. */
19218 assign_386_stack_local (enum machine_mode mode, enum ix86_stack_slot n)
19220 struct stack_local_entry *s;
19222 gcc_assert (n < MAX_386_STACK_LOCALS);
19224 /* Virtual slot is valid only before vregs are instantiated. */
19225 gcc_assert ((n == SLOT_VIRTUAL) == !virtuals_instantiated);
19227 for (s = ix86_stack_locals; s; s = s->next)
19228 if (s->mode == mode && s->n == n)
19229 return copy_rtx (s->rtl);
19231 s = (struct stack_local_entry *)
19232 ggc_alloc (sizeof (struct stack_local_entry));
19235 s->rtl = assign_stack_local (mode, GET_MODE_SIZE (mode), 0);
19237 s->next = ix86_stack_locals;
19238 ix86_stack_locals = s;
19242 /* Construct the SYMBOL_REF for the tls_get_addr function. */
19244 static GTY(()) rtx ix86_tls_symbol;
19246 ix86_tls_get_addr (void)
19249 if (!ix86_tls_symbol)
19251 ix86_tls_symbol = gen_rtx_SYMBOL_REF (Pmode,
19252 (TARGET_ANY_GNU_TLS
19254 ? "___tls_get_addr"
19255 : "__tls_get_addr");
19258 return ix86_tls_symbol;
19261 /* Construct the SYMBOL_REF for the _TLS_MODULE_BASE_ symbol. */
19263 static GTY(()) rtx ix86_tls_module_base_symbol;
19265 ix86_tls_module_base (void)
19268 if (!ix86_tls_module_base_symbol)
19270 ix86_tls_module_base_symbol = gen_rtx_SYMBOL_REF (Pmode,
19271 "_TLS_MODULE_BASE_");
19272 SYMBOL_REF_FLAGS (ix86_tls_module_base_symbol)
19273 |= TLS_MODEL_GLOBAL_DYNAMIC << SYMBOL_FLAG_TLS_SHIFT;
19276 return ix86_tls_module_base_symbol;
19279 /* Calculate the length of the memory address in the instruction
19280 encoding. Does not include the one-byte modrm, opcode, or prefix. */
19283 memory_address_length (rtx addr)
19285 struct ix86_address parts;
19286 rtx base, index, disp;
19290 if (GET_CODE (addr) == PRE_DEC
19291 || GET_CODE (addr) == POST_INC
19292 || GET_CODE (addr) == PRE_MODIFY
19293 || GET_CODE (addr) == POST_MODIFY)
19296 ok = ix86_decompose_address (addr, &parts);
19299 if (parts.base && GET_CODE (parts.base) == SUBREG)
19300 parts.base = SUBREG_REG (parts.base);
19301 if (parts.index && GET_CODE (parts.index) == SUBREG)
19302 parts.index = SUBREG_REG (parts.index);
19305 index = parts.index;
19310 - esp as the base always wants an index,
19311 - ebp as the base always wants a displacement,
19312 - r12 as the base always wants an index,
19313 - r13 as the base always wants a displacement. */
19315 /* Register Indirect. */
19316 if (base && !index && !disp)
19318 /* esp (for its index) and ebp (for its displacement) need
19319 the two-byte modrm form. Similarly for r12 and r13 in 64-bit
19322 && (addr == arg_pointer_rtx
19323 || addr == frame_pointer_rtx
19324 || REGNO (addr) == SP_REG
19325 || REGNO (addr) == BP_REG
19326 || REGNO (addr) == R12_REG
19327 || REGNO (addr) == R13_REG))
19331 /* Direct Addressing. */
19332 else if (disp && !base && !index)
19337 /* Find the length of the displacement constant. */
19340 if (base && satisfies_constraint_K (disp))
19345 /* ebp always wants a displacement. Similarly r13. */
19346 else if (REG_P (base)
19347 && (REGNO (base) == BP_REG || REGNO (base) == R13_REG))
19350 /* An index requires the two-byte modrm form.... */
19352 /* ...like esp (or r12), which always wants an index. */
19353 || base == arg_pointer_rtx
19354 || base == frame_pointer_rtx
19356 && (REGNO (base) == SP_REG || REGNO (base) == R12_REG)))
19363 /* Compute default value for "length_immediate" attribute. When SHORTFORM
19364 is set, expect that insn have 8bit immediate alternative. */
19366 ix86_attr_length_immediate_default (rtx insn, int shortform)
19370 extract_insn_cached (insn);
19371 for (i = recog_data.n_operands - 1; i >= 0; --i)
19372 if (CONSTANT_P (recog_data.operand[i]))
19375 if (shortform && satisfies_constraint_K (recog_data.operand[i]))
19379 switch (get_attr_mode (insn))
19390 /* Immediates for DImode instructions are encoded as 32bit sign extended values. */
19395 fatal_insn ("unknown insn mode", insn);
19401 /* Compute default value for "length_address" attribute. */
19403 ix86_attr_length_address_default (rtx insn)
19407 if (get_attr_type (insn) == TYPE_LEA)
19409 rtx set = PATTERN (insn), addr;
19411 if (GET_CODE (set) == PARALLEL)
19412 set = XVECEXP (set, 0, 0);
19414 gcc_assert (GET_CODE (set) == SET);
19416 addr = SET_SRC (set);
19417 if (TARGET_64BIT && get_attr_mode (insn) == MODE_SI)
19419 if (GET_CODE (addr) == ZERO_EXTEND)
19420 addr = XEXP (addr, 0);
19421 if (GET_CODE (addr) == SUBREG)
19422 addr = SUBREG_REG (addr);
19425 return memory_address_length (addr);
19428 extract_insn_cached (insn);
19429 for (i = recog_data.n_operands - 1; i >= 0; --i)
19430 if (MEM_P (recog_data.operand[i]))
19432 return memory_address_length (XEXP (recog_data.operand[i], 0));
19438 /* Compute default value for "length_vex" attribute. It includes
19439 2 or 3 byte VEX prefix and 1 opcode byte. */
19442 ix86_attr_length_vex_default (rtx insn, int has_0f_opcode,
19447 /* Only 0f opcode can use 2 byte VEX prefix and VEX W bit uses 3
19448 byte VEX prefix. */
19449 if (!has_0f_opcode || has_vex_w)
19452 /* We can always use 2 byte VEX prefix in 32bit. */
19456 extract_insn_cached (insn);
19458 for (i = recog_data.n_operands - 1; i >= 0; --i)
19459 if (REG_P (recog_data.operand[i]))
19461 /* REX.W bit uses 3 byte VEX prefix. */
19462 if (GET_MODE (recog_data.operand[i]) == DImode)
19467 /* REX.X or REX.B bits use 3 byte VEX prefix. */
19468 if (MEM_P (recog_data.operand[i])
19469 && x86_extended_reg_mentioned_p (recog_data.operand[i]))
19476 /* Return the maximum number of instructions a cpu can issue. */
19479 ix86_issue_rate (void)
19483 case PROCESSOR_PENTIUM:
19484 case PROCESSOR_ATOM:
19488 case PROCESSOR_PENTIUMPRO:
19489 case PROCESSOR_PENTIUM4:
19490 case PROCESSOR_ATHLON:
19492 case PROCESSOR_AMDFAM10:
19493 case PROCESSOR_NOCONA:
19494 case PROCESSOR_GENERIC32:
19495 case PROCESSOR_GENERIC64:
19498 case PROCESSOR_CORE2:
19506 /* A subroutine of ix86_adjust_cost -- return true iff INSN reads flags set
19507 by DEP_INSN and nothing set by DEP_INSN. */
19510 ix86_flags_dependent (rtx insn, rtx dep_insn, enum attr_type insn_type)
19514 /* Simplify the test for uninteresting insns. */
19515 if (insn_type != TYPE_SETCC
19516 && insn_type != TYPE_ICMOV
19517 && insn_type != TYPE_FCMOV
19518 && insn_type != TYPE_IBR)
19521 if ((set = single_set (dep_insn)) != 0)
19523 set = SET_DEST (set);
19526 else if (GET_CODE (PATTERN (dep_insn)) == PARALLEL
19527 && XVECLEN (PATTERN (dep_insn), 0) == 2
19528 && GET_CODE (XVECEXP (PATTERN (dep_insn), 0, 0)) == SET
19529 && GET_CODE (XVECEXP (PATTERN (dep_insn), 0, 1)) == SET)
19531 set = SET_DEST (XVECEXP (PATTERN (dep_insn), 0, 0));
19532 set2 = SET_DEST (XVECEXP (PATTERN (dep_insn), 0, 0));
19537 if (!REG_P (set) || REGNO (set) != FLAGS_REG)
19540 /* This test is true if the dependent insn reads the flags but
19541 not any other potentially set register. */
19542 if (!reg_overlap_mentioned_p (set, PATTERN (insn)))
19545 if (set2 && reg_overlap_mentioned_p (set2, PATTERN (insn)))
19551 /* Return true iff USE_INSN has a memory address with operands set by
19555 ix86_agi_dependent (rtx set_insn, rtx use_insn)
19558 extract_insn_cached (use_insn);
19559 for (i = recog_data.n_operands - 1; i >= 0; --i)
19560 if (MEM_P (recog_data.operand[i]))
19562 rtx addr = XEXP (recog_data.operand[i], 0);
19563 return modified_in_p (addr, set_insn) != 0;
19569 ix86_adjust_cost (rtx insn, rtx link, rtx dep_insn, int cost)
19571 enum attr_type insn_type, dep_insn_type;
19572 enum attr_memory memory;
19574 int dep_insn_code_number;
19576 /* Anti and output dependencies have zero cost on all CPUs. */
19577 if (REG_NOTE_KIND (link) != 0)
19580 dep_insn_code_number = recog_memoized (dep_insn);
19582 /* If we can't recognize the insns, we can't really do anything. */
19583 if (dep_insn_code_number < 0 || recog_memoized (insn) < 0)
19586 insn_type = get_attr_type (insn);
19587 dep_insn_type = get_attr_type (dep_insn);
19591 case PROCESSOR_PENTIUM:
19592 /* Address Generation Interlock adds a cycle of latency. */
19593 if (insn_type == TYPE_LEA)
19595 rtx addr = PATTERN (insn);
19597 if (GET_CODE (addr) == PARALLEL)
19598 addr = XVECEXP (addr, 0, 0);
19600 gcc_assert (GET_CODE (addr) == SET);
19602 addr = SET_SRC (addr);
19603 if (modified_in_p (addr, dep_insn))
19606 else if (ix86_agi_dependent (dep_insn, insn))
19609 /* ??? Compares pair with jump/setcc. */
19610 if (ix86_flags_dependent (insn, dep_insn, insn_type))
19613 /* Floating point stores require value to be ready one cycle earlier. */
19614 if (insn_type == TYPE_FMOV
19615 && get_attr_memory (insn) == MEMORY_STORE
19616 && !ix86_agi_dependent (dep_insn, insn))
19620 case PROCESSOR_PENTIUMPRO:
19621 memory = get_attr_memory (insn);
19623 /* INT->FP conversion is expensive. */
19624 if (get_attr_fp_int_src (dep_insn))
19627 /* There is one cycle extra latency between an FP op and a store. */
19628 if (insn_type == TYPE_FMOV
19629 && (set = single_set (dep_insn)) != NULL_RTX
19630 && (set2 = single_set (insn)) != NULL_RTX
19631 && rtx_equal_p (SET_DEST (set), SET_SRC (set2))
19632 && MEM_P (SET_DEST (set2)))
19635 /* Show ability of reorder buffer to hide latency of load by executing
19636 in parallel with previous instruction in case
19637 previous instruction is not needed to compute the address. */
19638 if ((memory == MEMORY_LOAD || memory == MEMORY_BOTH)
19639 && !ix86_agi_dependent (dep_insn, insn))
19641 /* Claim moves to take one cycle, as core can issue one load
19642 at time and the next load can start cycle later. */
19643 if (dep_insn_type == TYPE_IMOV
19644 || dep_insn_type == TYPE_FMOV)
19652 memory = get_attr_memory (insn);
19654 /* The esp dependency is resolved before the instruction is really
19656 if ((insn_type == TYPE_PUSH || insn_type == TYPE_POP)
19657 && (dep_insn_type == TYPE_PUSH || dep_insn_type == TYPE_POP))
19660 /* INT->FP conversion is expensive. */
19661 if (get_attr_fp_int_src (dep_insn))
19664 /* Show ability of reorder buffer to hide latency of load by executing
19665 in parallel with previous instruction in case
19666 previous instruction is not needed to compute the address. */
19667 if ((memory == MEMORY_LOAD || memory == MEMORY_BOTH)
19668 && !ix86_agi_dependent (dep_insn, insn))
19670 /* Claim moves to take one cycle, as core can issue one load
19671 at time and the next load can start cycle later. */
19672 if (dep_insn_type == TYPE_IMOV
19673 || dep_insn_type == TYPE_FMOV)
19682 case PROCESSOR_ATHLON:
19684 case PROCESSOR_AMDFAM10:
19685 case PROCESSOR_ATOM:
19686 case PROCESSOR_GENERIC32:
19687 case PROCESSOR_GENERIC64:
19688 memory = get_attr_memory (insn);
19690 /* Show ability of reorder buffer to hide latency of load by executing
19691 in parallel with previous instruction in case
19692 previous instruction is not needed to compute the address. */
19693 if ((memory == MEMORY_LOAD || memory == MEMORY_BOTH)
19694 && !ix86_agi_dependent (dep_insn, insn))
19696 enum attr_unit unit = get_attr_unit (insn);
19699 /* Because of the difference between the length of integer and
19700 floating unit pipeline preparation stages, the memory operands
19701 for floating point are cheaper.
19703 ??? For Athlon it the difference is most probably 2. */
19704 if (unit == UNIT_INTEGER || unit == UNIT_UNKNOWN)
19707 loadcost = TARGET_ATHLON ? 2 : 0;
19709 if (cost >= loadcost)
19722 /* How many alternative schedules to try. This should be as wide as the
19723 scheduling freedom in the DFA, but no wider. Making this value too
19724 large results extra work for the scheduler. */
19727 ia32_multipass_dfa_lookahead (void)
19731 case PROCESSOR_PENTIUM:
19734 case PROCESSOR_PENTIUMPRO:
19744 /* Compute the alignment given to a constant that is being placed in memory.
19745 EXP is the constant and ALIGN is the alignment that the object would
19747 The value of this function is used instead of that alignment to align
19751 ix86_constant_alignment (tree exp, int align)
19753 if (TREE_CODE (exp) == REAL_CST || TREE_CODE (exp) == VECTOR_CST
19754 || TREE_CODE (exp) == INTEGER_CST)
19756 if (TYPE_MODE (TREE_TYPE (exp)) == DFmode && align < 64)
19758 else if (ALIGN_MODE_128 (TYPE_MODE (TREE_TYPE (exp))) && align < 128)
19761 else if (!optimize_size && TREE_CODE (exp) == STRING_CST
19762 && TREE_STRING_LENGTH (exp) >= 31 && align < BITS_PER_WORD)
19763 return BITS_PER_WORD;
19768 /* Compute the alignment for a static variable.
19769 TYPE is the data type, and ALIGN is the alignment that
19770 the object would ordinarily have. The value of this function is used
19771 instead of that alignment to align the object. */
19774 ix86_data_alignment (tree type, int align)
19776 int max_align = optimize_size ? BITS_PER_WORD : MIN (256, MAX_OFILE_ALIGNMENT);
19778 if (AGGREGATE_TYPE_P (type)
19779 && TYPE_SIZE (type)
19780 && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST
19781 && (TREE_INT_CST_LOW (TYPE_SIZE (type)) >= (unsigned) max_align
19782 || TREE_INT_CST_HIGH (TYPE_SIZE (type)))
19783 && align < max_align)
19786 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
19787 to 16byte boundary. */
19790 if (AGGREGATE_TYPE_P (type)
19791 && TYPE_SIZE (type)
19792 && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST
19793 && (TREE_INT_CST_LOW (TYPE_SIZE (type)) >= 128
19794 || TREE_INT_CST_HIGH (TYPE_SIZE (type))) && align < 128)
19798 if (TREE_CODE (type) == ARRAY_TYPE)
19800 if (TYPE_MODE (TREE_TYPE (type)) == DFmode && align < 64)
19802 if (ALIGN_MODE_128 (TYPE_MODE (TREE_TYPE (type))) && align < 128)
19805 else if (TREE_CODE (type) == COMPLEX_TYPE)
19808 if (TYPE_MODE (type) == DCmode && align < 64)
19810 if ((TYPE_MODE (type) == XCmode
19811 || TYPE_MODE (type) == TCmode) && align < 128)
19814 else if ((TREE_CODE (type) == RECORD_TYPE
19815 || TREE_CODE (type) == UNION_TYPE
19816 || TREE_CODE (type) == QUAL_UNION_TYPE)
19817 && TYPE_FIELDS (type))
19819 if (DECL_MODE (TYPE_FIELDS (type)) == DFmode && align < 64)
19821 if (ALIGN_MODE_128 (DECL_MODE (TYPE_FIELDS (type))) && align < 128)
19824 else if (TREE_CODE (type) == REAL_TYPE || TREE_CODE (type) == VECTOR_TYPE
19825 || TREE_CODE (type) == INTEGER_TYPE)
19827 if (TYPE_MODE (type) == DFmode && align < 64)
19829 if (ALIGN_MODE_128 (TYPE_MODE (type)) && align < 128)
19836 /* Compute the alignment for a local variable or a stack slot. EXP is
19837 the data type or decl itself, MODE is the widest mode available and
19838 ALIGN is the alignment that the object would ordinarily have. The
19839 value of this macro is used instead of that alignment to align the
19843 ix86_local_alignment (tree exp, enum machine_mode mode,
19844 unsigned int align)
19848 if (exp && DECL_P (exp))
19850 type = TREE_TYPE (exp);
19859 /* Don't do dynamic stack realignment for long long objects with
19860 -mpreferred-stack-boundary=2. */
19863 && ix86_preferred_stack_boundary < 64
19864 && (mode == DImode || (type && TYPE_MODE (type) == DImode))
19865 && (!type || !TYPE_USER_ALIGN (type))
19866 && (!decl || !DECL_USER_ALIGN (decl)))
19869 /* If TYPE is NULL, we are allocating a stack slot for caller-save
19870 register in MODE. We will return the largest alignment of XF
19874 if (mode == XFmode && align < GET_MODE_ALIGNMENT (DFmode))
19875 align = GET_MODE_ALIGNMENT (DFmode);
19879 /* x86-64 ABI requires arrays greater than 16 bytes to be aligned
19880 to 16byte boundary. */
19883 if (AGGREGATE_TYPE_P (type)
19884 && TYPE_SIZE (type)
19885 && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST
19886 && (TREE_INT_CST_LOW (TYPE_SIZE (type)) >= 16
19887 || TREE_INT_CST_HIGH (TYPE_SIZE (type))) && align < 128)
19890 if (TREE_CODE (type) == ARRAY_TYPE)
19892 if (TYPE_MODE (TREE_TYPE (type)) == DFmode && align < 64)
19894 if (ALIGN_MODE_128 (TYPE_MODE (TREE_TYPE (type))) && align < 128)
19897 else if (TREE_CODE (type) == COMPLEX_TYPE)
19899 if (TYPE_MODE (type) == DCmode && align < 64)
19901 if ((TYPE_MODE (type) == XCmode
19902 || TYPE_MODE (type) == TCmode) && align < 128)
19905 else if ((TREE_CODE (type) == RECORD_TYPE
19906 || TREE_CODE (type) == UNION_TYPE
19907 || TREE_CODE (type) == QUAL_UNION_TYPE)
19908 && TYPE_FIELDS (type))
19910 if (DECL_MODE (TYPE_FIELDS (type)) == DFmode && align < 64)
19912 if (ALIGN_MODE_128 (DECL_MODE (TYPE_FIELDS (type))) && align < 128)
19915 else if (TREE_CODE (type) == REAL_TYPE || TREE_CODE (type) == VECTOR_TYPE
19916 || TREE_CODE (type) == INTEGER_TYPE)
19919 if (TYPE_MODE (type) == DFmode && align < 64)
19921 if (ALIGN_MODE_128 (TYPE_MODE (type)) && align < 128)
19927 /* Emit RTL insns to initialize the variable parts of a trampoline.
19928 FNADDR is an RTX for the address of the function's pure code.
19929 CXT is an RTX for the static chain value for the function. */
19931 x86_initialize_trampoline (rtx tramp, rtx fnaddr, rtx cxt)
19935 /* Compute offset from the end of the jmp to the target function. */
19936 rtx disp = expand_binop (SImode, sub_optab, fnaddr,
19937 plus_constant (tramp, 10),
19938 NULL_RTX, 1, OPTAB_DIRECT);
19939 emit_move_insn (gen_rtx_MEM (QImode, tramp),
19940 gen_int_mode (0xb9, QImode));
19941 emit_move_insn (gen_rtx_MEM (SImode, plus_constant (tramp, 1)), cxt);
19942 emit_move_insn (gen_rtx_MEM (QImode, plus_constant (tramp, 5)),
19943 gen_int_mode (0xe9, QImode));
19944 emit_move_insn (gen_rtx_MEM (SImode, plus_constant (tramp, 6)), disp);
19949 /* Try to load address using shorter movl instead of movabs.
19950 We may want to support movq for kernel mode, but kernel does not use
19951 trampolines at the moment. */
19952 if (x86_64_zext_immediate_operand (fnaddr, VOIDmode))
19954 fnaddr = copy_to_mode_reg (DImode, fnaddr);
19955 emit_move_insn (gen_rtx_MEM (HImode, plus_constant (tramp, offset)),
19956 gen_int_mode (0xbb41, HImode));
19957 emit_move_insn (gen_rtx_MEM (SImode, plus_constant (tramp, offset + 2)),
19958 gen_lowpart (SImode, fnaddr));
19963 emit_move_insn (gen_rtx_MEM (HImode, plus_constant (tramp, offset)),
19964 gen_int_mode (0xbb49, HImode));
19965 emit_move_insn (gen_rtx_MEM (DImode, plus_constant (tramp, offset + 2)),
19969 /* Load static chain using movabs to r10. */
19970 emit_move_insn (gen_rtx_MEM (HImode, plus_constant (tramp, offset)),
19971 gen_int_mode (0xba49, HImode));
19972 emit_move_insn (gen_rtx_MEM (DImode, plus_constant (tramp, offset + 2)),
19975 /* Jump to the r11 */
19976 emit_move_insn (gen_rtx_MEM (HImode, plus_constant (tramp, offset)),
19977 gen_int_mode (0xff49, HImode));
19978 emit_move_insn (gen_rtx_MEM (QImode, plus_constant (tramp, offset+2)),
19979 gen_int_mode (0xe3, QImode));
19981 gcc_assert (offset <= TRAMPOLINE_SIZE);
19984 #ifdef ENABLE_EXECUTE_STACK
19985 emit_library_call (gen_rtx_SYMBOL_REF (Pmode, "__enable_execute_stack"),
19986 LCT_NORMAL, VOIDmode, 1, tramp, Pmode);
19990 /* Codes for all the SSE/MMX builtins. */
19993 IX86_BUILTIN_ADDPS,
19994 IX86_BUILTIN_ADDSS,
19995 IX86_BUILTIN_DIVPS,
19996 IX86_BUILTIN_DIVSS,
19997 IX86_BUILTIN_MULPS,
19998 IX86_BUILTIN_MULSS,
19999 IX86_BUILTIN_SUBPS,
20000 IX86_BUILTIN_SUBSS,
20002 IX86_BUILTIN_CMPEQPS,
20003 IX86_BUILTIN_CMPLTPS,
20004 IX86_BUILTIN_CMPLEPS,
20005 IX86_BUILTIN_CMPGTPS,
20006 IX86_BUILTIN_CMPGEPS,
20007 IX86_BUILTIN_CMPNEQPS,
20008 IX86_BUILTIN_CMPNLTPS,
20009 IX86_BUILTIN_CMPNLEPS,
20010 IX86_BUILTIN_CMPNGTPS,
20011 IX86_BUILTIN_CMPNGEPS,
20012 IX86_BUILTIN_CMPORDPS,
20013 IX86_BUILTIN_CMPUNORDPS,
20014 IX86_BUILTIN_CMPEQSS,
20015 IX86_BUILTIN_CMPLTSS,
20016 IX86_BUILTIN_CMPLESS,
20017 IX86_BUILTIN_CMPNEQSS,
20018 IX86_BUILTIN_CMPNLTSS,
20019 IX86_BUILTIN_CMPNLESS,
20020 IX86_BUILTIN_CMPNGTSS,
20021 IX86_BUILTIN_CMPNGESS,
20022 IX86_BUILTIN_CMPORDSS,
20023 IX86_BUILTIN_CMPUNORDSS,
20025 IX86_BUILTIN_COMIEQSS,
20026 IX86_BUILTIN_COMILTSS,
20027 IX86_BUILTIN_COMILESS,
20028 IX86_BUILTIN_COMIGTSS,
20029 IX86_BUILTIN_COMIGESS,
20030 IX86_BUILTIN_COMINEQSS,
20031 IX86_BUILTIN_UCOMIEQSS,
20032 IX86_BUILTIN_UCOMILTSS,
20033 IX86_BUILTIN_UCOMILESS,
20034 IX86_BUILTIN_UCOMIGTSS,
20035 IX86_BUILTIN_UCOMIGESS,
20036 IX86_BUILTIN_UCOMINEQSS,
20038 IX86_BUILTIN_CVTPI2PS,
20039 IX86_BUILTIN_CVTPS2PI,
20040 IX86_BUILTIN_CVTSI2SS,
20041 IX86_BUILTIN_CVTSI642SS,
20042 IX86_BUILTIN_CVTSS2SI,
20043 IX86_BUILTIN_CVTSS2SI64,
20044 IX86_BUILTIN_CVTTPS2PI,
20045 IX86_BUILTIN_CVTTSS2SI,
20046 IX86_BUILTIN_CVTTSS2SI64,
20048 IX86_BUILTIN_MAXPS,
20049 IX86_BUILTIN_MAXSS,
20050 IX86_BUILTIN_MINPS,
20051 IX86_BUILTIN_MINSS,
20053 IX86_BUILTIN_LOADUPS,
20054 IX86_BUILTIN_STOREUPS,
20055 IX86_BUILTIN_MOVSS,
20057 IX86_BUILTIN_MOVHLPS,
20058 IX86_BUILTIN_MOVLHPS,
20059 IX86_BUILTIN_LOADHPS,
20060 IX86_BUILTIN_LOADLPS,
20061 IX86_BUILTIN_STOREHPS,
20062 IX86_BUILTIN_STORELPS,
20064 IX86_BUILTIN_MASKMOVQ,
20065 IX86_BUILTIN_MOVMSKPS,
20066 IX86_BUILTIN_PMOVMSKB,
20068 IX86_BUILTIN_MOVNTPS,
20069 IX86_BUILTIN_MOVNTQ,
20071 IX86_BUILTIN_LOADDQU,
20072 IX86_BUILTIN_STOREDQU,
20074 IX86_BUILTIN_PACKSSWB,
20075 IX86_BUILTIN_PACKSSDW,
20076 IX86_BUILTIN_PACKUSWB,
20078 IX86_BUILTIN_PADDB,
20079 IX86_BUILTIN_PADDW,
20080 IX86_BUILTIN_PADDD,
20081 IX86_BUILTIN_PADDQ,
20082 IX86_BUILTIN_PADDSB,
20083 IX86_BUILTIN_PADDSW,
20084 IX86_BUILTIN_PADDUSB,
20085 IX86_BUILTIN_PADDUSW,
20086 IX86_BUILTIN_PSUBB,
20087 IX86_BUILTIN_PSUBW,
20088 IX86_BUILTIN_PSUBD,
20089 IX86_BUILTIN_PSUBQ,
20090 IX86_BUILTIN_PSUBSB,
20091 IX86_BUILTIN_PSUBSW,
20092 IX86_BUILTIN_PSUBUSB,
20093 IX86_BUILTIN_PSUBUSW,
20096 IX86_BUILTIN_PANDN,
20100 IX86_BUILTIN_PAVGB,
20101 IX86_BUILTIN_PAVGW,
20103 IX86_BUILTIN_PCMPEQB,
20104 IX86_BUILTIN_PCMPEQW,
20105 IX86_BUILTIN_PCMPEQD,
20106 IX86_BUILTIN_PCMPGTB,
20107 IX86_BUILTIN_PCMPGTW,
20108 IX86_BUILTIN_PCMPGTD,
20110 IX86_BUILTIN_PMADDWD,
20112 IX86_BUILTIN_PMAXSW,
20113 IX86_BUILTIN_PMAXUB,
20114 IX86_BUILTIN_PMINSW,
20115 IX86_BUILTIN_PMINUB,
20117 IX86_BUILTIN_PMULHUW,
20118 IX86_BUILTIN_PMULHW,
20119 IX86_BUILTIN_PMULLW,
20121 IX86_BUILTIN_PSADBW,
20122 IX86_BUILTIN_PSHUFW,
20124 IX86_BUILTIN_PSLLW,
20125 IX86_BUILTIN_PSLLD,
20126 IX86_BUILTIN_PSLLQ,
20127 IX86_BUILTIN_PSRAW,
20128 IX86_BUILTIN_PSRAD,
20129 IX86_BUILTIN_PSRLW,
20130 IX86_BUILTIN_PSRLD,
20131 IX86_BUILTIN_PSRLQ,
20132 IX86_BUILTIN_PSLLWI,
20133 IX86_BUILTIN_PSLLDI,
20134 IX86_BUILTIN_PSLLQI,
20135 IX86_BUILTIN_PSRAWI,
20136 IX86_BUILTIN_PSRADI,
20137 IX86_BUILTIN_PSRLWI,
20138 IX86_BUILTIN_PSRLDI,
20139 IX86_BUILTIN_PSRLQI,
20141 IX86_BUILTIN_PUNPCKHBW,
20142 IX86_BUILTIN_PUNPCKHWD,
20143 IX86_BUILTIN_PUNPCKHDQ,
20144 IX86_BUILTIN_PUNPCKLBW,
20145 IX86_BUILTIN_PUNPCKLWD,
20146 IX86_BUILTIN_PUNPCKLDQ,
20148 IX86_BUILTIN_SHUFPS,
20150 IX86_BUILTIN_RCPPS,
20151 IX86_BUILTIN_RCPSS,
20152 IX86_BUILTIN_RSQRTPS,
20153 IX86_BUILTIN_RSQRTPS_NR,
20154 IX86_BUILTIN_RSQRTSS,
20155 IX86_BUILTIN_RSQRTF,
20156 IX86_BUILTIN_SQRTPS,
20157 IX86_BUILTIN_SQRTPS_NR,
20158 IX86_BUILTIN_SQRTSS,
20160 IX86_BUILTIN_UNPCKHPS,
20161 IX86_BUILTIN_UNPCKLPS,
20163 IX86_BUILTIN_ANDPS,
20164 IX86_BUILTIN_ANDNPS,
20166 IX86_BUILTIN_XORPS,
20169 IX86_BUILTIN_LDMXCSR,
20170 IX86_BUILTIN_STMXCSR,
20171 IX86_BUILTIN_SFENCE,
20173 /* 3DNow! Original */
20174 IX86_BUILTIN_FEMMS,
20175 IX86_BUILTIN_PAVGUSB,
20176 IX86_BUILTIN_PF2ID,
20177 IX86_BUILTIN_PFACC,
20178 IX86_BUILTIN_PFADD,
20179 IX86_BUILTIN_PFCMPEQ,
20180 IX86_BUILTIN_PFCMPGE,
20181 IX86_BUILTIN_PFCMPGT,
20182 IX86_BUILTIN_PFMAX,
20183 IX86_BUILTIN_PFMIN,
20184 IX86_BUILTIN_PFMUL,
20185 IX86_BUILTIN_PFRCP,
20186 IX86_BUILTIN_PFRCPIT1,
20187 IX86_BUILTIN_PFRCPIT2,
20188 IX86_BUILTIN_PFRSQIT1,
20189 IX86_BUILTIN_PFRSQRT,
20190 IX86_BUILTIN_PFSUB,
20191 IX86_BUILTIN_PFSUBR,
20192 IX86_BUILTIN_PI2FD,
20193 IX86_BUILTIN_PMULHRW,
20195 /* 3DNow! Athlon Extensions */
20196 IX86_BUILTIN_PF2IW,
20197 IX86_BUILTIN_PFNACC,
20198 IX86_BUILTIN_PFPNACC,
20199 IX86_BUILTIN_PI2FW,
20200 IX86_BUILTIN_PSWAPDSI,
20201 IX86_BUILTIN_PSWAPDSF,
20204 IX86_BUILTIN_ADDPD,
20205 IX86_BUILTIN_ADDSD,
20206 IX86_BUILTIN_DIVPD,
20207 IX86_BUILTIN_DIVSD,
20208 IX86_BUILTIN_MULPD,
20209 IX86_BUILTIN_MULSD,
20210 IX86_BUILTIN_SUBPD,
20211 IX86_BUILTIN_SUBSD,
20213 IX86_BUILTIN_CMPEQPD,
20214 IX86_BUILTIN_CMPLTPD,
20215 IX86_BUILTIN_CMPLEPD,
20216 IX86_BUILTIN_CMPGTPD,
20217 IX86_BUILTIN_CMPGEPD,
20218 IX86_BUILTIN_CMPNEQPD,
20219 IX86_BUILTIN_CMPNLTPD,
20220 IX86_BUILTIN_CMPNLEPD,
20221 IX86_BUILTIN_CMPNGTPD,
20222 IX86_BUILTIN_CMPNGEPD,
20223 IX86_BUILTIN_CMPORDPD,
20224 IX86_BUILTIN_CMPUNORDPD,
20225 IX86_BUILTIN_CMPEQSD,
20226 IX86_BUILTIN_CMPLTSD,
20227 IX86_BUILTIN_CMPLESD,
20228 IX86_BUILTIN_CMPNEQSD,
20229 IX86_BUILTIN_CMPNLTSD,
20230 IX86_BUILTIN_CMPNLESD,
20231 IX86_BUILTIN_CMPORDSD,
20232 IX86_BUILTIN_CMPUNORDSD,
20234 IX86_BUILTIN_COMIEQSD,
20235 IX86_BUILTIN_COMILTSD,
20236 IX86_BUILTIN_COMILESD,
20237 IX86_BUILTIN_COMIGTSD,
20238 IX86_BUILTIN_COMIGESD,
20239 IX86_BUILTIN_COMINEQSD,
20240 IX86_BUILTIN_UCOMIEQSD,
20241 IX86_BUILTIN_UCOMILTSD,
20242 IX86_BUILTIN_UCOMILESD,
20243 IX86_BUILTIN_UCOMIGTSD,
20244 IX86_BUILTIN_UCOMIGESD,
20245 IX86_BUILTIN_UCOMINEQSD,
20247 IX86_BUILTIN_MAXPD,
20248 IX86_BUILTIN_MAXSD,
20249 IX86_BUILTIN_MINPD,
20250 IX86_BUILTIN_MINSD,
20252 IX86_BUILTIN_ANDPD,
20253 IX86_BUILTIN_ANDNPD,
20255 IX86_BUILTIN_XORPD,
20257 IX86_BUILTIN_SQRTPD,
20258 IX86_BUILTIN_SQRTSD,
20260 IX86_BUILTIN_UNPCKHPD,
20261 IX86_BUILTIN_UNPCKLPD,
20263 IX86_BUILTIN_SHUFPD,
20265 IX86_BUILTIN_LOADUPD,
20266 IX86_BUILTIN_STOREUPD,
20267 IX86_BUILTIN_MOVSD,
20269 IX86_BUILTIN_LOADHPD,
20270 IX86_BUILTIN_LOADLPD,
20272 IX86_BUILTIN_CVTDQ2PD,
20273 IX86_BUILTIN_CVTDQ2PS,
20275 IX86_BUILTIN_CVTPD2DQ,
20276 IX86_BUILTIN_CVTPD2PI,
20277 IX86_BUILTIN_CVTPD2PS,
20278 IX86_BUILTIN_CVTTPD2DQ,
20279 IX86_BUILTIN_CVTTPD2PI,
20281 IX86_BUILTIN_CVTPI2PD,
20282 IX86_BUILTIN_CVTSI2SD,
20283 IX86_BUILTIN_CVTSI642SD,
20285 IX86_BUILTIN_CVTSD2SI,
20286 IX86_BUILTIN_CVTSD2SI64,
20287 IX86_BUILTIN_CVTSD2SS,
20288 IX86_BUILTIN_CVTSS2SD,
20289 IX86_BUILTIN_CVTTSD2SI,
20290 IX86_BUILTIN_CVTTSD2SI64,
20292 IX86_BUILTIN_CVTPS2DQ,
20293 IX86_BUILTIN_CVTPS2PD,
20294 IX86_BUILTIN_CVTTPS2DQ,
20296 IX86_BUILTIN_MOVNTI,
20297 IX86_BUILTIN_MOVNTPD,
20298 IX86_BUILTIN_MOVNTDQ,
20300 IX86_BUILTIN_MOVQ128,
20303 IX86_BUILTIN_MASKMOVDQU,
20304 IX86_BUILTIN_MOVMSKPD,
20305 IX86_BUILTIN_PMOVMSKB128,
20307 IX86_BUILTIN_PACKSSWB128,
20308 IX86_BUILTIN_PACKSSDW128,
20309 IX86_BUILTIN_PACKUSWB128,
20311 IX86_BUILTIN_PADDB128,
20312 IX86_BUILTIN_PADDW128,
20313 IX86_BUILTIN_PADDD128,
20314 IX86_BUILTIN_PADDQ128,
20315 IX86_BUILTIN_PADDSB128,
20316 IX86_BUILTIN_PADDSW128,
20317 IX86_BUILTIN_PADDUSB128,
20318 IX86_BUILTIN_PADDUSW128,
20319 IX86_BUILTIN_PSUBB128,
20320 IX86_BUILTIN_PSUBW128,
20321 IX86_BUILTIN_PSUBD128,
20322 IX86_BUILTIN_PSUBQ128,
20323 IX86_BUILTIN_PSUBSB128,
20324 IX86_BUILTIN_PSUBSW128,
20325 IX86_BUILTIN_PSUBUSB128,
20326 IX86_BUILTIN_PSUBUSW128,
20328 IX86_BUILTIN_PAND128,
20329 IX86_BUILTIN_PANDN128,
20330 IX86_BUILTIN_POR128,
20331 IX86_BUILTIN_PXOR128,
20333 IX86_BUILTIN_PAVGB128,
20334 IX86_BUILTIN_PAVGW128,
20336 IX86_BUILTIN_PCMPEQB128,
20337 IX86_BUILTIN_PCMPEQW128,
20338 IX86_BUILTIN_PCMPEQD128,
20339 IX86_BUILTIN_PCMPGTB128,
20340 IX86_BUILTIN_PCMPGTW128,
20341 IX86_BUILTIN_PCMPGTD128,
20343 IX86_BUILTIN_PMADDWD128,
20345 IX86_BUILTIN_PMAXSW128,
20346 IX86_BUILTIN_PMAXUB128,
20347 IX86_BUILTIN_PMINSW128,
20348 IX86_BUILTIN_PMINUB128,
20350 IX86_BUILTIN_PMULUDQ,
20351 IX86_BUILTIN_PMULUDQ128,
20352 IX86_BUILTIN_PMULHUW128,
20353 IX86_BUILTIN_PMULHW128,
20354 IX86_BUILTIN_PMULLW128,
20356 IX86_BUILTIN_PSADBW128,
20357 IX86_BUILTIN_PSHUFHW,
20358 IX86_BUILTIN_PSHUFLW,
20359 IX86_BUILTIN_PSHUFD,
20361 IX86_BUILTIN_PSLLDQI128,
20362 IX86_BUILTIN_PSLLWI128,
20363 IX86_BUILTIN_PSLLDI128,
20364 IX86_BUILTIN_PSLLQI128,
20365 IX86_BUILTIN_PSRAWI128,
20366 IX86_BUILTIN_PSRADI128,
20367 IX86_BUILTIN_PSRLDQI128,
20368 IX86_BUILTIN_PSRLWI128,
20369 IX86_BUILTIN_PSRLDI128,
20370 IX86_BUILTIN_PSRLQI128,
20372 IX86_BUILTIN_PSLLDQ128,
20373 IX86_BUILTIN_PSLLW128,
20374 IX86_BUILTIN_PSLLD128,
20375 IX86_BUILTIN_PSLLQ128,
20376 IX86_BUILTIN_PSRAW128,
20377 IX86_BUILTIN_PSRAD128,
20378 IX86_BUILTIN_PSRLW128,
20379 IX86_BUILTIN_PSRLD128,
20380 IX86_BUILTIN_PSRLQ128,
20382 IX86_BUILTIN_PUNPCKHBW128,
20383 IX86_BUILTIN_PUNPCKHWD128,
20384 IX86_BUILTIN_PUNPCKHDQ128,
20385 IX86_BUILTIN_PUNPCKHQDQ128,
20386 IX86_BUILTIN_PUNPCKLBW128,
20387 IX86_BUILTIN_PUNPCKLWD128,
20388 IX86_BUILTIN_PUNPCKLDQ128,
20389 IX86_BUILTIN_PUNPCKLQDQ128,
20391 IX86_BUILTIN_CLFLUSH,
20392 IX86_BUILTIN_MFENCE,
20393 IX86_BUILTIN_LFENCE,
20396 IX86_BUILTIN_ADDSUBPS,
20397 IX86_BUILTIN_HADDPS,
20398 IX86_BUILTIN_HSUBPS,
20399 IX86_BUILTIN_MOVSHDUP,
20400 IX86_BUILTIN_MOVSLDUP,
20401 IX86_BUILTIN_ADDSUBPD,
20402 IX86_BUILTIN_HADDPD,
20403 IX86_BUILTIN_HSUBPD,
20404 IX86_BUILTIN_LDDQU,
20406 IX86_BUILTIN_MONITOR,
20407 IX86_BUILTIN_MWAIT,
20410 IX86_BUILTIN_PHADDW,
20411 IX86_BUILTIN_PHADDD,
20412 IX86_BUILTIN_PHADDSW,
20413 IX86_BUILTIN_PHSUBW,
20414 IX86_BUILTIN_PHSUBD,
20415 IX86_BUILTIN_PHSUBSW,
20416 IX86_BUILTIN_PMADDUBSW,
20417 IX86_BUILTIN_PMULHRSW,
20418 IX86_BUILTIN_PSHUFB,
20419 IX86_BUILTIN_PSIGNB,
20420 IX86_BUILTIN_PSIGNW,
20421 IX86_BUILTIN_PSIGND,
20422 IX86_BUILTIN_PALIGNR,
20423 IX86_BUILTIN_PABSB,
20424 IX86_BUILTIN_PABSW,
20425 IX86_BUILTIN_PABSD,
20427 IX86_BUILTIN_PHADDW128,
20428 IX86_BUILTIN_PHADDD128,
20429 IX86_BUILTIN_PHADDSW128,
20430 IX86_BUILTIN_PHSUBW128,
20431 IX86_BUILTIN_PHSUBD128,
20432 IX86_BUILTIN_PHSUBSW128,
20433 IX86_BUILTIN_PMADDUBSW128,
20434 IX86_BUILTIN_PMULHRSW128,
20435 IX86_BUILTIN_PSHUFB128,
20436 IX86_BUILTIN_PSIGNB128,
20437 IX86_BUILTIN_PSIGNW128,
20438 IX86_BUILTIN_PSIGND128,
20439 IX86_BUILTIN_PALIGNR128,
20440 IX86_BUILTIN_PABSB128,
20441 IX86_BUILTIN_PABSW128,
20442 IX86_BUILTIN_PABSD128,
20444 /* AMDFAM10 - SSE4A New Instructions. */
20445 IX86_BUILTIN_MOVNTSD,
20446 IX86_BUILTIN_MOVNTSS,
20447 IX86_BUILTIN_EXTRQI,
20448 IX86_BUILTIN_EXTRQ,
20449 IX86_BUILTIN_INSERTQI,
20450 IX86_BUILTIN_INSERTQ,
20453 IX86_BUILTIN_BLENDPD,
20454 IX86_BUILTIN_BLENDPS,
20455 IX86_BUILTIN_BLENDVPD,
20456 IX86_BUILTIN_BLENDVPS,
20457 IX86_BUILTIN_PBLENDVB128,
20458 IX86_BUILTIN_PBLENDW128,
20463 IX86_BUILTIN_INSERTPS128,
20465 IX86_BUILTIN_MOVNTDQA,
20466 IX86_BUILTIN_MPSADBW128,
20467 IX86_BUILTIN_PACKUSDW128,
20468 IX86_BUILTIN_PCMPEQQ,
20469 IX86_BUILTIN_PHMINPOSUW128,
20471 IX86_BUILTIN_PMAXSB128,
20472 IX86_BUILTIN_PMAXSD128,
20473 IX86_BUILTIN_PMAXUD128,
20474 IX86_BUILTIN_PMAXUW128,
20476 IX86_BUILTIN_PMINSB128,
20477 IX86_BUILTIN_PMINSD128,
20478 IX86_BUILTIN_PMINUD128,
20479 IX86_BUILTIN_PMINUW128,
20481 IX86_BUILTIN_PMOVSXBW128,
20482 IX86_BUILTIN_PMOVSXBD128,
20483 IX86_BUILTIN_PMOVSXBQ128,
20484 IX86_BUILTIN_PMOVSXWD128,
20485 IX86_BUILTIN_PMOVSXWQ128,
20486 IX86_BUILTIN_PMOVSXDQ128,
20488 IX86_BUILTIN_PMOVZXBW128,
20489 IX86_BUILTIN_PMOVZXBD128,
20490 IX86_BUILTIN_PMOVZXBQ128,
20491 IX86_BUILTIN_PMOVZXWD128,
20492 IX86_BUILTIN_PMOVZXWQ128,
20493 IX86_BUILTIN_PMOVZXDQ128,
20495 IX86_BUILTIN_PMULDQ128,
20496 IX86_BUILTIN_PMULLD128,
20498 IX86_BUILTIN_ROUNDPD,
20499 IX86_BUILTIN_ROUNDPS,
20500 IX86_BUILTIN_ROUNDSD,
20501 IX86_BUILTIN_ROUNDSS,
20503 IX86_BUILTIN_PTESTZ,
20504 IX86_BUILTIN_PTESTC,
20505 IX86_BUILTIN_PTESTNZC,
20507 IX86_BUILTIN_VEC_INIT_V2SI,
20508 IX86_BUILTIN_VEC_INIT_V4HI,
20509 IX86_BUILTIN_VEC_INIT_V8QI,
20510 IX86_BUILTIN_VEC_EXT_V2DF,
20511 IX86_BUILTIN_VEC_EXT_V2DI,
20512 IX86_BUILTIN_VEC_EXT_V4SF,
20513 IX86_BUILTIN_VEC_EXT_V4SI,
20514 IX86_BUILTIN_VEC_EXT_V8HI,
20515 IX86_BUILTIN_VEC_EXT_V2SI,
20516 IX86_BUILTIN_VEC_EXT_V4HI,
20517 IX86_BUILTIN_VEC_EXT_V16QI,
20518 IX86_BUILTIN_VEC_SET_V2DI,
20519 IX86_BUILTIN_VEC_SET_V4SF,
20520 IX86_BUILTIN_VEC_SET_V4SI,
20521 IX86_BUILTIN_VEC_SET_V8HI,
20522 IX86_BUILTIN_VEC_SET_V4HI,
20523 IX86_BUILTIN_VEC_SET_V16QI,
20525 IX86_BUILTIN_VEC_PACK_SFIX,
20528 IX86_BUILTIN_CRC32QI,
20529 IX86_BUILTIN_CRC32HI,
20530 IX86_BUILTIN_CRC32SI,
20531 IX86_BUILTIN_CRC32DI,
20533 IX86_BUILTIN_PCMPESTRI128,
20534 IX86_BUILTIN_PCMPESTRM128,
20535 IX86_BUILTIN_PCMPESTRA128,
20536 IX86_BUILTIN_PCMPESTRC128,
20537 IX86_BUILTIN_PCMPESTRO128,
20538 IX86_BUILTIN_PCMPESTRS128,
20539 IX86_BUILTIN_PCMPESTRZ128,
20540 IX86_BUILTIN_PCMPISTRI128,
20541 IX86_BUILTIN_PCMPISTRM128,
20542 IX86_BUILTIN_PCMPISTRA128,
20543 IX86_BUILTIN_PCMPISTRC128,
20544 IX86_BUILTIN_PCMPISTRO128,
20545 IX86_BUILTIN_PCMPISTRS128,
20546 IX86_BUILTIN_PCMPISTRZ128,
20548 IX86_BUILTIN_PCMPGTQ,
20550 /* AES instructions */
20551 IX86_BUILTIN_AESENC128,
20552 IX86_BUILTIN_AESENCLAST128,
20553 IX86_BUILTIN_AESDEC128,
20554 IX86_BUILTIN_AESDECLAST128,
20555 IX86_BUILTIN_AESIMC128,
20556 IX86_BUILTIN_AESKEYGENASSIST128,
20558 /* PCLMUL instruction */
20559 IX86_BUILTIN_PCLMULQDQ128,
20562 IX86_BUILTIN_ADDPD256,
20563 IX86_BUILTIN_ADDPS256,
20564 IX86_BUILTIN_ADDSUBPD256,
20565 IX86_BUILTIN_ADDSUBPS256,
20566 IX86_BUILTIN_ANDPD256,
20567 IX86_BUILTIN_ANDPS256,
20568 IX86_BUILTIN_ANDNPD256,
20569 IX86_BUILTIN_ANDNPS256,
20570 IX86_BUILTIN_BLENDPD256,
20571 IX86_BUILTIN_BLENDPS256,
20572 IX86_BUILTIN_BLENDVPD256,
20573 IX86_BUILTIN_BLENDVPS256,
20574 IX86_BUILTIN_DIVPD256,
20575 IX86_BUILTIN_DIVPS256,
20576 IX86_BUILTIN_DPPS256,
20577 IX86_BUILTIN_HADDPD256,
20578 IX86_BUILTIN_HADDPS256,
20579 IX86_BUILTIN_HSUBPD256,
20580 IX86_BUILTIN_HSUBPS256,
20581 IX86_BUILTIN_MAXPD256,
20582 IX86_BUILTIN_MAXPS256,
20583 IX86_BUILTIN_MINPD256,
20584 IX86_BUILTIN_MINPS256,
20585 IX86_BUILTIN_MULPD256,
20586 IX86_BUILTIN_MULPS256,
20587 IX86_BUILTIN_ORPD256,
20588 IX86_BUILTIN_ORPS256,
20589 IX86_BUILTIN_SHUFPD256,
20590 IX86_BUILTIN_SHUFPS256,
20591 IX86_BUILTIN_SUBPD256,
20592 IX86_BUILTIN_SUBPS256,
20593 IX86_BUILTIN_XORPD256,
20594 IX86_BUILTIN_XORPS256,
20595 IX86_BUILTIN_CMPSD,
20596 IX86_BUILTIN_CMPSS,
20597 IX86_BUILTIN_CMPPD,
20598 IX86_BUILTIN_CMPPS,
20599 IX86_BUILTIN_CMPPD256,
20600 IX86_BUILTIN_CMPPS256,
20601 IX86_BUILTIN_CVTDQ2PD256,
20602 IX86_BUILTIN_CVTDQ2PS256,
20603 IX86_BUILTIN_CVTPD2PS256,
20604 IX86_BUILTIN_CVTPS2DQ256,
20605 IX86_BUILTIN_CVTPS2PD256,
20606 IX86_BUILTIN_CVTTPD2DQ256,
20607 IX86_BUILTIN_CVTPD2DQ256,
20608 IX86_BUILTIN_CVTTPS2DQ256,
20609 IX86_BUILTIN_EXTRACTF128PD256,
20610 IX86_BUILTIN_EXTRACTF128PS256,
20611 IX86_BUILTIN_EXTRACTF128SI256,
20612 IX86_BUILTIN_VZEROALL,
20613 IX86_BUILTIN_VZEROUPPER,
20614 IX86_BUILTIN_VZEROUPPER_REX64,
20615 IX86_BUILTIN_VPERMILVARPD,
20616 IX86_BUILTIN_VPERMILVARPS,
20617 IX86_BUILTIN_VPERMILVARPD256,
20618 IX86_BUILTIN_VPERMILVARPS256,
20619 IX86_BUILTIN_VPERMILPD,
20620 IX86_BUILTIN_VPERMILPS,
20621 IX86_BUILTIN_VPERMILPD256,
20622 IX86_BUILTIN_VPERMILPS256,
20623 IX86_BUILTIN_VPERM2F128PD256,
20624 IX86_BUILTIN_VPERM2F128PS256,
20625 IX86_BUILTIN_VPERM2F128SI256,
20626 IX86_BUILTIN_VBROADCASTSS,
20627 IX86_BUILTIN_VBROADCASTSD256,
20628 IX86_BUILTIN_VBROADCASTSS256,
20629 IX86_BUILTIN_VBROADCASTPD256,
20630 IX86_BUILTIN_VBROADCASTPS256,
20631 IX86_BUILTIN_VINSERTF128PD256,
20632 IX86_BUILTIN_VINSERTF128PS256,
20633 IX86_BUILTIN_VINSERTF128SI256,
20634 IX86_BUILTIN_LOADUPD256,
20635 IX86_BUILTIN_LOADUPS256,
20636 IX86_BUILTIN_STOREUPD256,
20637 IX86_BUILTIN_STOREUPS256,
20638 IX86_BUILTIN_LDDQU256,
20639 IX86_BUILTIN_MOVNTDQ256,
20640 IX86_BUILTIN_MOVNTPD256,
20641 IX86_BUILTIN_MOVNTPS256,
20642 IX86_BUILTIN_LOADDQU256,
20643 IX86_BUILTIN_STOREDQU256,
20644 IX86_BUILTIN_MASKLOADPD,
20645 IX86_BUILTIN_MASKLOADPS,
20646 IX86_BUILTIN_MASKSTOREPD,
20647 IX86_BUILTIN_MASKSTOREPS,
20648 IX86_BUILTIN_MASKLOADPD256,
20649 IX86_BUILTIN_MASKLOADPS256,
20650 IX86_BUILTIN_MASKSTOREPD256,
20651 IX86_BUILTIN_MASKSTOREPS256,
20652 IX86_BUILTIN_MOVSHDUP256,
20653 IX86_BUILTIN_MOVSLDUP256,
20654 IX86_BUILTIN_MOVDDUP256,
20656 IX86_BUILTIN_SQRTPD256,
20657 IX86_BUILTIN_SQRTPS256,
20658 IX86_BUILTIN_SQRTPS_NR256,
20659 IX86_BUILTIN_RSQRTPS256,
20660 IX86_BUILTIN_RSQRTPS_NR256,
20662 IX86_BUILTIN_RCPPS256,
20664 IX86_BUILTIN_ROUNDPD256,
20665 IX86_BUILTIN_ROUNDPS256,
20667 IX86_BUILTIN_UNPCKHPD256,
20668 IX86_BUILTIN_UNPCKLPD256,
20669 IX86_BUILTIN_UNPCKHPS256,
20670 IX86_BUILTIN_UNPCKLPS256,
20672 IX86_BUILTIN_SI256_SI,
20673 IX86_BUILTIN_PS256_PS,
20674 IX86_BUILTIN_PD256_PD,
20675 IX86_BUILTIN_SI_SI256,
20676 IX86_BUILTIN_PS_PS256,
20677 IX86_BUILTIN_PD_PD256,
20679 IX86_BUILTIN_VTESTZPD,
20680 IX86_BUILTIN_VTESTCPD,
20681 IX86_BUILTIN_VTESTNZCPD,
20682 IX86_BUILTIN_VTESTZPS,
20683 IX86_BUILTIN_VTESTCPS,
20684 IX86_BUILTIN_VTESTNZCPS,
20685 IX86_BUILTIN_VTESTZPD256,
20686 IX86_BUILTIN_VTESTCPD256,
20687 IX86_BUILTIN_VTESTNZCPD256,
20688 IX86_BUILTIN_VTESTZPS256,
20689 IX86_BUILTIN_VTESTCPS256,
20690 IX86_BUILTIN_VTESTNZCPS256,
20691 IX86_BUILTIN_PTESTZ256,
20692 IX86_BUILTIN_PTESTC256,
20693 IX86_BUILTIN_PTESTNZC256,
20695 IX86_BUILTIN_MOVMSKPD256,
20696 IX86_BUILTIN_MOVMSKPS256,
20698 /* TFmode support builtins. */
20700 IX86_BUILTIN_HUGE_VALQ,
20701 IX86_BUILTIN_FABSQ,
20702 IX86_BUILTIN_COPYSIGNQ,
20704 /* SSE5 instructions */
20705 IX86_BUILTIN_FMADDSS,
20706 IX86_BUILTIN_FMADDSD,
20707 IX86_BUILTIN_FMADDPS,
20708 IX86_BUILTIN_FMADDPD,
20709 IX86_BUILTIN_FMSUBSS,
20710 IX86_BUILTIN_FMSUBSD,
20711 IX86_BUILTIN_FMSUBPS,
20712 IX86_BUILTIN_FMSUBPD,
20713 IX86_BUILTIN_FNMADDSS,
20714 IX86_BUILTIN_FNMADDSD,
20715 IX86_BUILTIN_FNMADDPS,
20716 IX86_BUILTIN_FNMADDPD,
20717 IX86_BUILTIN_FNMSUBSS,
20718 IX86_BUILTIN_FNMSUBSD,
20719 IX86_BUILTIN_FNMSUBPS,
20720 IX86_BUILTIN_FNMSUBPD,
20721 IX86_BUILTIN_PCMOV,
20722 IX86_BUILTIN_PCMOV_V2DI,
20723 IX86_BUILTIN_PCMOV_V4SI,
20724 IX86_BUILTIN_PCMOV_V8HI,
20725 IX86_BUILTIN_PCMOV_V16QI,
20726 IX86_BUILTIN_PCMOV_V4SF,
20727 IX86_BUILTIN_PCMOV_V2DF,
20728 IX86_BUILTIN_PPERM,
20729 IX86_BUILTIN_PERMPS,
20730 IX86_BUILTIN_PERMPD,
20731 IX86_BUILTIN_PMACSSWW,
20732 IX86_BUILTIN_PMACSWW,
20733 IX86_BUILTIN_PMACSSWD,
20734 IX86_BUILTIN_PMACSWD,
20735 IX86_BUILTIN_PMACSSDD,
20736 IX86_BUILTIN_PMACSDD,
20737 IX86_BUILTIN_PMACSSDQL,
20738 IX86_BUILTIN_PMACSSDQH,
20739 IX86_BUILTIN_PMACSDQL,
20740 IX86_BUILTIN_PMACSDQH,
20741 IX86_BUILTIN_PMADCSSWD,
20742 IX86_BUILTIN_PMADCSWD,
20743 IX86_BUILTIN_PHADDBW,
20744 IX86_BUILTIN_PHADDBD,
20745 IX86_BUILTIN_PHADDBQ,
20746 IX86_BUILTIN_PHADDWD,
20747 IX86_BUILTIN_PHADDWQ,
20748 IX86_BUILTIN_PHADDDQ,
20749 IX86_BUILTIN_PHADDUBW,
20750 IX86_BUILTIN_PHADDUBD,
20751 IX86_BUILTIN_PHADDUBQ,
20752 IX86_BUILTIN_PHADDUWD,
20753 IX86_BUILTIN_PHADDUWQ,
20754 IX86_BUILTIN_PHADDUDQ,
20755 IX86_BUILTIN_PHSUBBW,
20756 IX86_BUILTIN_PHSUBWD,
20757 IX86_BUILTIN_PHSUBDQ,
20758 IX86_BUILTIN_PROTB,
20759 IX86_BUILTIN_PROTW,
20760 IX86_BUILTIN_PROTD,
20761 IX86_BUILTIN_PROTQ,
20762 IX86_BUILTIN_PROTB_IMM,
20763 IX86_BUILTIN_PROTW_IMM,
20764 IX86_BUILTIN_PROTD_IMM,
20765 IX86_BUILTIN_PROTQ_IMM,
20766 IX86_BUILTIN_PSHLB,
20767 IX86_BUILTIN_PSHLW,
20768 IX86_BUILTIN_PSHLD,
20769 IX86_BUILTIN_PSHLQ,
20770 IX86_BUILTIN_PSHAB,
20771 IX86_BUILTIN_PSHAW,
20772 IX86_BUILTIN_PSHAD,
20773 IX86_BUILTIN_PSHAQ,
20774 IX86_BUILTIN_FRCZSS,
20775 IX86_BUILTIN_FRCZSD,
20776 IX86_BUILTIN_FRCZPS,
20777 IX86_BUILTIN_FRCZPD,
20778 IX86_BUILTIN_CVTPH2PS,
20779 IX86_BUILTIN_CVTPS2PH,
20781 IX86_BUILTIN_COMEQSS,
20782 IX86_BUILTIN_COMNESS,
20783 IX86_BUILTIN_COMLTSS,
20784 IX86_BUILTIN_COMLESS,
20785 IX86_BUILTIN_COMGTSS,
20786 IX86_BUILTIN_COMGESS,
20787 IX86_BUILTIN_COMUEQSS,
20788 IX86_BUILTIN_COMUNESS,
20789 IX86_BUILTIN_COMULTSS,
20790 IX86_BUILTIN_COMULESS,
20791 IX86_BUILTIN_COMUGTSS,
20792 IX86_BUILTIN_COMUGESS,
20793 IX86_BUILTIN_COMORDSS,
20794 IX86_BUILTIN_COMUNORDSS,
20795 IX86_BUILTIN_COMFALSESS,
20796 IX86_BUILTIN_COMTRUESS,
20798 IX86_BUILTIN_COMEQSD,
20799 IX86_BUILTIN_COMNESD,
20800 IX86_BUILTIN_COMLTSD,
20801 IX86_BUILTIN_COMLESD,
20802 IX86_BUILTIN_COMGTSD,
20803 IX86_BUILTIN_COMGESD,
20804 IX86_BUILTIN_COMUEQSD,
20805 IX86_BUILTIN_COMUNESD,
20806 IX86_BUILTIN_COMULTSD,
20807 IX86_BUILTIN_COMULESD,
20808 IX86_BUILTIN_COMUGTSD,
20809 IX86_BUILTIN_COMUGESD,
20810 IX86_BUILTIN_COMORDSD,
20811 IX86_BUILTIN_COMUNORDSD,
20812 IX86_BUILTIN_COMFALSESD,
20813 IX86_BUILTIN_COMTRUESD,
20815 IX86_BUILTIN_COMEQPS,
20816 IX86_BUILTIN_COMNEPS,
20817 IX86_BUILTIN_COMLTPS,
20818 IX86_BUILTIN_COMLEPS,
20819 IX86_BUILTIN_COMGTPS,
20820 IX86_BUILTIN_COMGEPS,
20821 IX86_BUILTIN_COMUEQPS,
20822 IX86_BUILTIN_COMUNEPS,
20823 IX86_BUILTIN_COMULTPS,
20824 IX86_BUILTIN_COMULEPS,
20825 IX86_BUILTIN_COMUGTPS,
20826 IX86_BUILTIN_COMUGEPS,
20827 IX86_BUILTIN_COMORDPS,
20828 IX86_BUILTIN_COMUNORDPS,
20829 IX86_BUILTIN_COMFALSEPS,
20830 IX86_BUILTIN_COMTRUEPS,
20832 IX86_BUILTIN_COMEQPD,
20833 IX86_BUILTIN_COMNEPD,
20834 IX86_BUILTIN_COMLTPD,
20835 IX86_BUILTIN_COMLEPD,
20836 IX86_BUILTIN_COMGTPD,
20837 IX86_BUILTIN_COMGEPD,
20838 IX86_BUILTIN_COMUEQPD,
20839 IX86_BUILTIN_COMUNEPD,
20840 IX86_BUILTIN_COMULTPD,
20841 IX86_BUILTIN_COMULEPD,
20842 IX86_BUILTIN_COMUGTPD,
20843 IX86_BUILTIN_COMUGEPD,
20844 IX86_BUILTIN_COMORDPD,
20845 IX86_BUILTIN_COMUNORDPD,
20846 IX86_BUILTIN_COMFALSEPD,
20847 IX86_BUILTIN_COMTRUEPD,
20849 IX86_BUILTIN_PCOMEQUB,
20850 IX86_BUILTIN_PCOMNEUB,
20851 IX86_BUILTIN_PCOMLTUB,
20852 IX86_BUILTIN_PCOMLEUB,
20853 IX86_BUILTIN_PCOMGTUB,
20854 IX86_BUILTIN_PCOMGEUB,
20855 IX86_BUILTIN_PCOMFALSEUB,
20856 IX86_BUILTIN_PCOMTRUEUB,
20857 IX86_BUILTIN_PCOMEQUW,
20858 IX86_BUILTIN_PCOMNEUW,
20859 IX86_BUILTIN_PCOMLTUW,
20860 IX86_BUILTIN_PCOMLEUW,
20861 IX86_BUILTIN_PCOMGTUW,
20862 IX86_BUILTIN_PCOMGEUW,
20863 IX86_BUILTIN_PCOMFALSEUW,
20864 IX86_BUILTIN_PCOMTRUEUW,
20865 IX86_BUILTIN_PCOMEQUD,
20866 IX86_BUILTIN_PCOMNEUD,
20867 IX86_BUILTIN_PCOMLTUD,
20868 IX86_BUILTIN_PCOMLEUD,
20869 IX86_BUILTIN_PCOMGTUD,
20870 IX86_BUILTIN_PCOMGEUD,
20871 IX86_BUILTIN_PCOMFALSEUD,
20872 IX86_BUILTIN_PCOMTRUEUD,
20873 IX86_BUILTIN_PCOMEQUQ,
20874 IX86_BUILTIN_PCOMNEUQ,
20875 IX86_BUILTIN_PCOMLTUQ,
20876 IX86_BUILTIN_PCOMLEUQ,
20877 IX86_BUILTIN_PCOMGTUQ,
20878 IX86_BUILTIN_PCOMGEUQ,
20879 IX86_BUILTIN_PCOMFALSEUQ,
20880 IX86_BUILTIN_PCOMTRUEUQ,
20882 IX86_BUILTIN_PCOMEQB,
20883 IX86_BUILTIN_PCOMNEB,
20884 IX86_BUILTIN_PCOMLTB,
20885 IX86_BUILTIN_PCOMLEB,
20886 IX86_BUILTIN_PCOMGTB,
20887 IX86_BUILTIN_PCOMGEB,
20888 IX86_BUILTIN_PCOMFALSEB,
20889 IX86_BUILTIN_PCOMTRUEB,
20890 IX86_BUILTIN_PCOMEQW,
20891 IX86_BUILTIN_PCOMNEW,
20892 IX86_BUILTIN_PCOMLTW,
20893 IX86_BUILTIN_PCOMLEW,
20894 IX86_BUILTIN_PCOMGTW,
20895 IX86_BUILTIN_PCOMGEW,
20896 IX86_BUILTIN_PCOMFALSEW,
20897 IX86_BUILTIN_PCOMTRUEW,
20898 IX86_BUILTIN_PCOMEQD,
20899 IX86_BUILTIN_PCOMNED,
20900 IX86_BUILTIN_PCOMLTD,
20901 IX86_BUILTIN_PCOMLED,
20902 IX86_BUILTIN_PCOMGTD,
20903 IX86_BUILTIN_PCOMGED,
20904 IX86_BUILTIN_PCOMFALSED,
20905 IX86_BUILTIN_PCOMTRUED,
20906 IX86_BUILTIN_PCOMEQQ,
20907 IX86_BUILTIN_PCOMNEQ,
20908 IX86_BUILTIN_PCOMLTQ,
20909 IX86_BUILTIN_PCOMLEQ,
20910 IX86_BUILTIN_PCOMGTQ,
20911 IX86_BUILTIN_PCOMGEQ,
20912 IX86_BUILTIN_PCOMFALSEQ,
20913 IX86_BUILTIN_PCOMTRUEQ,
20918 /* Table for the ix86 builtin decls. */
20919 static GTY(()) tree ix86_builtins[(int) IX86_BUILTIN_MAX];
20921 /* Table of all of the builtin functions that are possible with different ISA's
20922 but are waiting to be built until a function is declared to use that
20924 struct GTY(()) builtin_isa {
20925 tree type; /* builtin type to use in the declaration */
20926 const char *name; /* function name */
20927 int isa; /* isa_flags this builtin is defined for */
20928 bool const_p; /* true if the declaration is constant */
20931 static GTY(()) struct builtin_isa ix86_builtins_isa[(int) IX86_BUILTIN_MAX];
20934 /* Add an ix86 target builtin function with CODE, NAME and TYPE. Save the MASK
20935 * of which isa_flags to use in the ix86_builtins_isa array. Stores the
20936 * function decl in the ix86_builtins array. Returns the function decl or
20937 * NULL_TREE, if the builtin was not added.
20939 * If the front end has a special hook for builtin functions, delay adding
20940 * builtin functions that aren't in the current ISA until the ISA is changed
20941 * with function specific optimization. Doing so, can save about 300K for the
20942 * default compiler. When the builtin is expanded, check at that time whether
20945 * If the front end doesn't have a special hook, record all builtins, even if
20946 * it isn't an instruction set in the current ISA in case the user uses
20947 * function specific options for a different ISA, so that we don't get scope
20948 * errors if a builtin is added in the middle of a function scope. */
20951 def_builtin (int mask, const char *name, tree type, enum ix86_builtins code)
20953 tree decl = NULL_TREE;
20955 if (!(mask & OPTION_MASK_ISA_64BIT) || TARGET_64BIT)
20957 ix86_builtins_isa[(int) code].isa = mask;
20959 if ((mask & ix86_isa_flags) != 0
20960 || (lang_hooks.builtin_function
20961 == lang_hooks.builtin_function_ext_scope))
20964 decl = add_builtin_function (name, type, code, BUILT_IN_MD, NULL,
20966 ix86_builtins[(int) code] = decl;
20967 ix86_builtins_isa[(int) code].type = NULL_TREE;
20971 ix86_builtins[(int) code] = NULL_TREE;
20972 ix86_builtins_isa[(int) code].const_p = false;
20973 ix86_builtins_isa[(int) code].type = type;
20974 ix86_builtins_isa[(int) code].name = name;
20981 /* Like def_builtin, but also marks the function decl "const". */
20984 def_builtin_const (int mask, const char *name, tree type,
20985 enum ix86_builtins code)
20987 tree decl = def_builtin (mask, name, type, code);
20989 TREE_READONLY (decl) = 1;
20991 ix86_builtins_isa[(int) code].const_p = true;
20996 /* Add any new builtin functions for a given ISA that may not have been
20997 declared. This saves a bit of space compared to adding all of the
20998 declarations to the tree, even if we didn't use them. */
21001 ix86_add_new_builtins (int isa)
21006 for (i = 0; i < (int)IX86_BUILTIN_MAX; i++)
21008 if ((ix86_builtins_isa[i].isa & isa) != 0
21009 && ix86_builtins_isa[i].type != NULL_TREE)
21011 decl = add_builtin_function_ext_scope (ix86_builtins_isa[i].name,
21012 ix86_builtins_isa[i].type,
21013 i, BUILT_IN_MD, NULL,
21016 ix86_builtins[i] = decl;
21017 ix86_builtins_isa[i].type = NULL_TREE;
21018 if (ix86_builtins_isa[i].const_p)
21019 TREE_READONLY (decl) = 1;
21024 /* Bits for builtin_description.flag. */
21026 /* Set when we don't support the comparison natively, and should
21027 swap_comparison in order to support it. */
21028 #define BUILTIN_DESC_SWAP_OPERANDS 1
21030 struct builtin_description
21032 const unsigned int mask;
21033 const enum insn_code icode;
21034 const char *const name;
21035 const enum ix86_builtins code;
21036 const enum rtx_code comparison;
21040 static const struct builtin_description bdesc_comi[] =
21042 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comieq", IX86_BUILTIN_COMIEQSS, UNEQ, 0 },
21043 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comilt", IX86_BUILTIN_COMILTSS, UNLT, 0 },
21044 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comile", IX86_BUILTIN_COMILESS, UNLE, 0 },
21045 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comigt", IX86_BUILTIN_COMIGTSS, GT, 0 },
21046 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comige", IX86_BUILTIN_COMIGESS, GE, 0 },
21047 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_comi, "__builtin_ia32_comineq", IX86_BUILTIN_COMINEQSS, LTGT, 0 },
21048 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomieq", IX86_BUILTIN_UCOMIEQSS, UNEQ, 0 },
21049 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomilt", IX86_BUILTIN_UCOMILTSS, UNLT, 0 },
21050 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomile", IX86_BUILTIN_UCOMILESS, UNLE, 0 },
21051 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomigt", IX86_BUILTIN_UCOMIGTSS, GT, 0 },
21052 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomige", IX86_BUILTIN_UCOMIGESS, GE, 0 },
21053 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_ucomi, "__builtin_ia32_ucomineq", IX86_BUILTIN_UCOMINEQSS, LTGT, 0 },
21054 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdeq", IX86_BUILTIN_COMIEQSD, UNEQ, 0 },
21055 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdlt", IX86_BUILTIN_COMILTSD, UNLT, 0 },
21056 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdle", IX86_BUILTIN_COMILESD, UNLE, 0 },
21057 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdgt", IX86_BUILTIN_COMIGTSD, GT, 0 },
21058 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdge", IX86_BUILTIN_COMIGESD, GE, 0 },
21059 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_comi, "__builtin_ia32_comisdneq", IX86_BUILTIN_COMINEQSD, LTGT, 0 },
21060 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdeq", IX86_BUILTIN_UCOMIEQSD, UNEQ, 0 },
21061 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdlt", IX86_BUILTIN_UCOMILTSD, UNLT, 0 },
21062 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdle", IX86_BUILTIN_UCOMILESD, UNLE, 0 },
21063 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdgt", IX86_BUILTIN_UCOMIGTSD, GT, 0 },
21064 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdge", IX86_BUILTIN_UCOMIGESD, GE, 0 },
21065 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ucomi, "__builtin_ia32_ucomisdneq", IX86_BUILTIN_UCOMINEQSD, LTGT, 0 },
21068 static const struct builtin_description bdesc_pcmpestr[] =
21071 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestri128", IX86_BUILTIN_PCMPESTRI128, UNKNOWN, 0 },
21072 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrm128", IX86_BUILTIN_PCMPESTRM128, UNKNOWN, 0 },
21073 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestria128", IX86_BUILTIN_PCMPESTRA128, UNKNOWN, (int) CCAmode },
21074 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestric128", IX86_BUILTIN_PCMPESTRC128, UNKNOWN, (int) CCCmode },
21075 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestrio128", IX86_BUILTIN_PCMPESTRO128, UNKNOWN, (int) CCOmode },
21076 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestris128", IX86_BUILTIN_PCMPESTRS128, UNKNOWN, (int) CCSmode },
21077 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpestr, "__builtin_ia32_pcmpestriz128", IX86_BUILTIN_PCMPESTRZ128, UNKNOWN, (int) CCZmode },
21080 static const struct builtin_description bdesc_pcmpistr[] =
21083 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistri128", IX86_BUILTIN_PCMPISTRI128, UNKNOWN, 0 },
21084 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrm128", IX86_BUILTIN_PCMPISTRM128, UNKNOWN, 0 },
21085 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistria128", IX86_BUILTIN_PCMPISTRA128, UNKNOWN, (int) CCAmode },
21086 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistric128", IX86_BUILTIN_PCMPISTRC128, UNKNOWN, (int) CCCmode },
21087 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistrio128", IX86_BUILTIN_PCMPISTRO128, UNKNOWN, (int) CCOmode },
21088 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistris128", IX86_BUILTIN_PCMPISTRS128, UNKNOWN, (int) CCSmode },
21089 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_pcmpistr, "__builtin_ia32_pcmpistriz128", IX86_BUILTIN_PCMPISTRZ128, UNKNOWN, (int) CCZmode },
21092 /* Special builtin types */
21093 enum ix86_special_builtin_type
21095 SPECIAL_FTYPE_UNKNOWN,
21097 V32QI_FTYPE_PCCHAR,
21098 V16QI_FTYPE_PCCHAR,
21100 V8SF_FTYPE_PCFLOAT,
21102 V4DF_FTYPE_PCDOUBLE,
21103 V4SF_FTYPE_PCFLOAT,
21104 V2DF_FTYPE_PCDOUBLE,
21105 V8SF_FTYPE_PCV8SF_V8SF,
21106 V4DF_FTYPE_PCV4DF_V4DF,
21107 V4SF_FTYPE_V4SF_PCV2SF,
21108 V4SF_FTYPE_PCV4SF_V4SF,
21109 V2DF_FTYPE_V2DF_PCDOUBLE,
21110 V2DF_FTYPE_PCV2DF_V2DF,
21112 VOID_FTYPE_PV2SF_V4SF,
21113 VOID_FTYPE_PV4DI_V4DI,
21114 VOID_FTYPE_PV2DI_V2DI,
21115 VOID_FTYPE_PCHAR_V32QI,
21116 VOID_FTYPE_PCHAR_V16QI,
21117 VOID_FTYPE_PFLOAT_V8SF,
21118 VOID_FTYPE_PFLOAT_V4SF,
21119 VOID_FTYPE_PDOUBLE_V4DF,
21120 VOID_FTYPE_PDOUBLE_V2DF,
21122 VOID_FTYPE_PINT_INT,
21123 VOID_FTYPE_PV8SF_V8SF_V8SF,
21124 VOID_FTYPE_PV4DF_V4DF_V4DF,
21125 VOID_FTYPE_PV4SF_V4SF_V4SF,
21126 VOID_FTYPE_PV2DF_V2DF_V2DF
21129 /* Builtin types */
21130 enum ix86_builtin_type
21133 FLOAT128_FTYPE_FLOAT128,
21135 FLOAT128_FTYPE_FLOAT128_FLOAT128,
21136 INT_FTYPE_V8SF_V8SF_PTEST,
21137 INT_FTYPE_V4DI_V4DI_PTEST,
21138 INT_FTYPE_V4DF_V4DF_PTEST,
21139 INT_FTYPE_V4SF_V4SF_PTEST,
21140 INT_FTYPE_V2DI_V2DI_PTEST,
21141 INT_FTYPE_V2DF_V2DF_PTEST,
21173 V4SF_FTYPE_V4SF_VEC_MERGE,
21182 V2DF_FTYPE_V2DF_VEC_MERGE,
21193 V16QI_FTYPE_V16QI_V16QI,
21194 V16QI_FTYPE_V8HI_V8HI,
21195 V8QI_FTYPE_V8QI_V8QI,
21196 V8QI_FTYPE_V4HI_V4HI,
21197 V8HI_FTYPE_V8HI_V8HI,
21198 V8HI_FTYPE_V8HI_V8HI_COUNT,
21199 V8HI_FTYPE_V16QI_V16QI,
21200 V8HI_FTYPE_V4SI_V4SI,
21201 V8HI_FTYPE_V8HI_SI_COUNT,
21202 V8SF_FTYPE_V8SF_V8SF,
21203 V8SF_FTYPE_V8SF_V8SI,
21204 V4SI_FTYPE_V4SI_V4SI,
21205 V4SI_FTYPE_V4SI_V4SI_COUNT,
21206 V4SI_FTYPE_V8HI_V8HI,
21207 V4SI_FTYPE_V4SF_V4SF,
21208 V4SI_FTYPE_V2DF_V2DF,
21209 V4SI_FTYPE_V4SI_SI_COUNT,
21210 V4HI_FTYPE_V4HI_V4HI,
21211 V4HI_FTYPE_V4HI_V4HI_COUNT,
21212 V4HI_FTYPE_V8QI_V8QI,
21213 V4HI_FTYPE_V2SI_V2SI,
21214 V4HI_FTYPE_V4HI_SI_COUNT,
21215 V4DF_FTYPE_V4DF_V4DF,
21216 V4DF_FTYPE_V4DF_V4DI,
21217 V4SF_FTYPE_V4SF_V4SF,
21218 V4SF_FTYPE_V4SF_V4SF_SWAP,
21219 V4SF_FTYPE_V4SF_V4SI,
21220 V4SF_FTYPE_V4SF_V2SI,
21221 V4SF_FTYPE_V4SF_V2DF,
21222 V4SF_FTYPE_V4SF_DI,
21223 V4SF_FTYPE_V4SF_SI,
21224 V2DI_FTYPE_V2DI_V2DI,
21225 V2DI_FTYPE_V2DI_V2DI_COUNT,
21226 V2DI_FTYPE_V16QI_V16QI,
21227 V2DI_FTYPE_V4SI_V4SI,
21228 V2DI_FTYPE_V2DI_V16QI,
21229 V2DI_FTYPE_V2DF_V2DF,
21230 V2DI_FTYPE_V2DI_SI_COUNT,
21231 V2SI_FTYPE_V2SI_V2SI,
21232 V2SI_FTYPE_V2SI_V2SI_COUNT,
21233 V2SI_FTYPE_V4HI_V4HI,
21234 V2SI_FTYPE_V2SF_V2SF,
21235 V2SI_FTYPE_V2SI_SI_COUNT,
21236 V2DF_FTYPE_V2DF_V2DF,
21237 V2DF_FTYPE_V2DF_V2DF_SWAP,
21238 V2DF_FTYPE_V2DF_V4SF,
21239 V2DF_FTYPE_V2DF_V2DI,
21240 V2DF_FTYPE_V2DF_DI,
21241 V2DF_FTYPE_V2DF_SI,
21242 V2SF_FTYPE_V2SF_V2SF,
21243 V1DI_FTYPE_V1DI_V1DI,
21244 V1DI_FTYPE_V1DI_V1DI_COUNT,
21245 V1DI_FTYPE_V8QI_V8QI,
21246 V1DI_FTYPE_V2SI_V2SI,
21247 V1DI_FTYPE_V1DI_SI_COUNT,
21248 UINT64_FTYPE_UINT64_UINT64,
21249 UINT_FTYPE_UINT_UINT,
21250 UINT_FTYPE_UINT_USHORT,
21251 UINT_FTYPE_UINT_UCHAR,
21252 V8HI_FTYPE_V8HI_INT,
21253 V4SI_FTYPE_V4SI_INT,
21254 V4HI_FTYPE_V4HI_INT,
21255 V8SF_FTYPE_V8SF_INT,
21256 V4SI_FTYPE_V8SI_INT,
21257 V4SF_FTYPE_V8SF_INT,
21258 V2DF_FTYPE_V4DF_INT,
21259 V4DF_FTYPE_V4DF_INT,
21260 V4SF_FTYPE_V4SF_INT,
21261 V2DI_FTYPE_V2DI_INT,
21262 V2DI2TI_FTYPE_V2DI_INT,
21263 V2DF_FTYPE_V2DF_INT,
21264 V16QI_FTYPE_V16QI_V16QI_V16QI,
21265 V8SF_FTYPE_V8SF_V8SF_V8SF,
21266 V4DF_FTYPE_V4DF_V4DF_V4DF,
21267 V4SF_FTYPE_V4SF_V4SF_V4SF,
21268 V2DF_FTYPE_V2DF_V2DF_V2DF,
21269 V16QI_FTYPE_V16QI_V16QI_INT,
21270 V8SI_FTYPE_V8SI_V8SI_INT,
21271 V8SI_FTYPE_V8SI_V4SI_INT,
21272 V8HI_FTYPE_V8HI_V8HI_INT,
21273 V8SF_FTYPE_V8SF_V8SF_INT,
21274 V8SF_FTYPE_V8SF_V4SF_INT,
21275 V4SI_FTYPE_V4SI_V4SI_INT,
21276 V4DF_FTYPE_V4DF_V4DF_INT,
21277 V4DF_FTYPE_V4DF_V2DF_INT,
21278 V4SF_FTYPE_V4SF_V4SF_INT,
21279 V2DI_FTYPE_V2DI_V2DI_INT,
21280 V2DI2TI_FTYPE_V2DI_V2DI_INT,
21281 V1DI2DI_FTYPE_V1DI_V1DI_INT,
21282 V2DF_FTYPE_V2DF_V2DF_INT,
21283 V2DI_FTYPE_V2DI_UINT_UINT,
21284 V2DI_FTYPE_V2DI_V2DI_UINT_UINT
21287 /* Special builtins with variable number of arguments. */
21288 static const struct builtin_description bdesc_special_args[] =
21291 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_emms, "__builtin_ia32_emms", IX86_BUILTIN_EMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
21294 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_femms, "__builtin_ia32_femms", IX86_BUILTIN_FEMMS, UNKNOWN, (int) VOID_FTYPE_VOID },
21297 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movups, "__builtin_ia32_storeups", IX86_BUILTIN_STOREUPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
21298 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movntv4sf, "__builtin_ia32_movntps", IX86_BUILTIN_MOVNTPS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
21299 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movups, "__builtin_ia32_loadups", IX86_BUILTIN_LOADUPS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
21301 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadhps_exp, "__builtin_ia32_loadhps", IX86_BUILTIN_LOADHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
21302 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_loadlps_exp, "__builtin_ia32_loadlps", IX86_BUILTIN_LOADLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_PCV2SF },
21303 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storehps, "__builtin_ia32_storehps", IX86_BUILTIN_STOREHPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
21304 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_storelps, "__builtin_ia32_storelps", IX86_BUILTIN_STORELPS, UNKNOWN, (int) VOID_FTYPE_PV2SF_V4SF },
21306 /* SSE or 3DNow!A */
21307 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_sfence, "__builtin_ia32_sfence", IX86_BUILTIN_SFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
21308 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_sse_movntdi, "__builtin_ia32_movntq", IX86_BUILTIN_MOVNTQ, UNKNOWN, (int) VOID_FTYPE_PDI_DI },
21311 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lfence, "__builtin_ia32_lfence", IX86_BUILTIN_LFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
21312 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_mfence, 0, IX86_BUILTIN_MFENCE, UNKNOWN, (int) VOID_FTYPE_VOID },
21313 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movupd, "__builtin_ia32_storeupd", IX86_BUILTIN_STOREUPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
21314 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movdqu, "__builtin_ia32_storedqu", IX86_BUILTIN_STOREDQU, UNKNOWN, (int) VOID_FTYPE_PCHAR_V16QI },
21315 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2df, "__builtin_ia32_movntpd", IX86_BUILTIN_MOVNTPD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
21316 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntv2di, "__builtin_ia32_movntdq", IX86_BUILTIN_MOVNTDQ, UNKNOWN, (int) VOID_FTYPE_PV2DI_V2DI },
21317 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movntsi, "__builtin_ia32_movnti", IX86_BUILTIN_MOVNTI, UNKNOWN, (int) VOID_FTYPE_PINT_INT },
21318 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movupd, "__builtin_ia32_loadupd", IX86_BUILTIN_LOADUPD, UNKNOWN, (int) V2DF_FTYPE_PCDOUBLE },
21319 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movdqu, "__builtin_ia32_loaddqu", IX86_BUILTIN_LOADDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
21321 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadhpd_exp, "__builtin_ia32_loadhpd", IX86_BUILTIN_LOADHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
21322 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_loadlpd_exp, "__builtin_ia32_loadlpd", IX86_BUILTIN_LOADLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_PCDOUBLE },
21325 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_lddqu, "__builtin_ia32_lddqu", IX86_BUILTIN_LDDQU, UNKNOWN, (int) V16QI_FTYPE_PCCHAR },
21328 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_movntdqa, "__builtin_ia32_movntdqa", IX86_BUILTIN_MOVNTDQA, UNKNOWN, (int) V2DI_FTYPE_PV2DI },
21331 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv2df, "__builtin_ia32_movntsd", IX86_BUILTIN_MOVNTSD, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V2DF },
21332 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_vmmovntv4sf, "__builtin_ia32_movntss", IX86_BUILTIN_MOVNTSS, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V4SF },
21335 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroall, "__builtin_ia32_vzeroall", IX86_BUILTIN_VZEROALL, UNKNOWN, (int) VOID_FTYPE_VOID },
21336 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vzeroupper, 0, IX86_BUILTIN_VZEROUPPER, UNKNOWN, (int) VOID_FTYPE_VOID },
21337 { OPTION_MASK_ISA_AVX | OPTION_MASK_ISA_64BIT, CODE_FOR_avx_vzeroupper_rex64, 0, IX86_BUILTIN_VZEROUPPER_REX64, UNKNOWN, (int) VOID_FTYPE_VOID },
21339 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastss, "__builtin_ia32_vbroadcastss", IX86_BUILTIN_VBROADCASTSS, UNKNOWN, (int) V4SF_FTYPE_PCFLOAT },
21340 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastsd256, "__builtin_ia32_vbroadcastsd256", IX86_BUILTIN_VBROADCASTSD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
21341 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastss256, "__builtin_ia32_vbroadcastss256", IX86_BUILTIN_VBROADCASTSS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
21342 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_pd256, "__builtin_ia32_vbroadcastf128_pd256", IX86_BUILTIN_VBROADCASTPD256, UNKNOWN, (int) V4DF_FTYPE_PCV2DF },
21343 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vbroadcastf128_ps256, "__builtin_ia32_vbroadcastf128_ps256", IX86_BUILTIN_VBROADCASTPS256, UNKNOWN, (int) V8SF_FTYPE_PCV4SF },
21345 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movupd256, "__builtin_ia32_loadupd256", IX86_BUILTIN_LOADUPD256, UNKNOWN, (int) V4DF_FTYPE_PCDOUBLE },
21346 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movups256, "__builtin_ia32_loadups256", IX86_BUILTIN_LOADUPS256, UNKNOWN, (int) V8SF_FTYPE_PCFLOAT },
21347 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movupd256, "__builtin_ia32_storeupd256", IX86_BUILTIN_STOREUPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
21348 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movups256, "__builtin_ia32_storeups256", IX86_BUILTIN_STOREUPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
21349 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movdqu256, "__builtin_ia32_loaddqu256", IX86_BUILTIN_LOADDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
21350 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movdqu256, "__builtin_ia32_storedqu256", IX86_BUILTIN_STOREDQU256, UNKNOWN, (int) VOID_FTYPE_PCHAR_V32QI },
21351 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_lddqu256, "__builtin_ia32_lddqu256", IX86_BUILTIN_LDDQU256, UNKNOWN, (int) V32QI_FTYPE_PCCHAR },
21353 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4di, "__builtin_ia32_movntdq256", IX86_BUILTIN_MOVNTDQ256, UNKNOWN, (int) VOID_FTYPE_PV4DI_V4DI },
21354 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4df, "__builtin_ia32_movntpd256", IX86_BUILTIN_MOVNTPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF },
21355 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv8sf, "__builtin_ia32_movntps256", IX86_BUILTIN_MOVNTPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF },
21357 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd, "__builtin_ia32_maskloadpd", IX86_BUILTIN_MASKLOADPD, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DF },
21358 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps, "__builtin_ia32_maskloadps", IX86_BUILTIN_MASKLOADPS, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SF },
21359 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd256, "__builtin_ia32_maskloadpd256", IX86_BUILTIN_MASKLOADPD256, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DF },
21360 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps256, "__builtin_ia32_maskloadps256", IX86_BUILTIN_MASKLOADPS256, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SF },
21361 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd, "__builtin_ia32_maskstorepd", IX86_BUILTIN_MASKSTOREPD, UNKNOWN, (int) VOID_FTYPE_PV2DF_V2DF_V2DF },
21362 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps, "__builtin_ia32_maskstoreps", IX86_BUILTIN_MASKSTOREPS, UNKNOWN, (int) VOID_FTYPE_PV4SF_V4SF_V4SF },
21363 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd256, "__builtin_ia32_maskstorepd256", IX86_BUILTIN_MASKSTOREPD256, UNKNOWN, (int) VOID_FTYPE_PV4DF_V4DF_V4DF },
21364 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps256, "__builtin_ia32_maskstoreps256", IX86_BUILTIN_MASKSTOREPS256, UNKNOWN, (int) VOID_FTYPE_PV8SF_V8SF_V8SF },
21367 /* Builtins with variable number of arguments. */
21368 static const struct builtin_description bdesc_args[] =
21371 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv8qi3, "__builtin_ia32_paddb", IX86_BUILTIN_PADDB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
21372 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv4hi3, "__builtin_ia32_paddw", IX86_BUILTIN_PADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21373 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_addv2si3, "__builtin_ia32_paddd", IX86_BUILTIN_PADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
21374 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv8qi3, "__builtin_ia32_psubb", IX86_BUILTIN_PSUBB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
21375 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv4hi3, "__builtin_ia32_psubw", IX86_BUILTIN_PSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21376 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_subv2si3, "__builtin_ia32_psubd", IX86_BUILTIN_PSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
21378 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv8qi3, "__builtin_ia32_paddsb", IX86_BUILTIN_PADDSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
21379 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ssaddv4hi3, "__builtin_ia32_paddsw", IX86_BUILTIN_PADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21380 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv8qi3, "__builtin_ia32_psubsb", IX86_BUILTIN_PSUBSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
21381 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_sssubv4hi3, "__builtin_ia32_psubsw", IX86_BUILTIN_PSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21382 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv8qi3, "__builtin_ia32_paddusb", IX86_BUILTIN_PADDUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
21383 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_usaddv4hi3, "__builtin_ia32_paddusw", IX86_BUILTIN_PADDUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21384 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv8qi3, "__builtin_ia32_psubusb", IX86_BUILTIN_PSUBUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
21385 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ussubv4hi3, "__builtin_ia32_psubusw", IX86_BUILTIN_PSUBUSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21387 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_mulv4hi3, "__builtin_ia32_pmullw", IX86_BUILTIN_PMULLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21388 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_smulv4hi3_highpart, "__builtin_ia32_pmulhw", IX86_BUILTIN_PMULHW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21390 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andv2si3, "__builtin_ia32_pand", IX86_BUILTIN_PAND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
21391 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_andnotv2si3, "__builtin_ia32_pandn", IX86_BUILTIN_PANDN, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
21392 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_iorv2si3, "__builtin_ia32_por", IX86_BUILTIN_POR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
21393 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_xorv2si3, "__builtin_ia32_pxor", IX86_BUILTIN_PXOR, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
21395 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv8qi3, "__builtin_ia32_pcmpeqb", IX86_BUILTIN_PCMPEQB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
21396 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv4hi3, "__builtin_ia32_pcmpeqw", IX86_BUILTIN_PCMPEQW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21397 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_eqv2si3, "__builtin_ia32_pcmpeqd", IX86_BUILTIN_PCMPEQD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
21398 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv8qi3, "__builtin_ia32_pcmpgtb", IX86_BUILTIN_PCMPGTB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
21399 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv4hi3, "__builtin_ia32_pcmpgtw", IX86_BUILTIN_PCMPGTW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21400 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_gtv2si3, "__builtin_ia32_pcmpgtd", IX86_BUILTIN_PCMPGTD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
21402 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhbw, "__builtin_ia32_punpckhbw", IX86_BUILTIN_PUNPCKHBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
21403 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhwd, "__builtin_ia32_punpckhwd", IX86_BUILTIN_PUNPCKHWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21404 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckhdq, "__builtin_ia32_punpckhdq", IX86_BUILTIN_PUNPCKHDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
21405 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklbw, "__builtin_ia32_punpcklbw", IX86_BUILTIN_PUNPCKLBW, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
21406 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpcklwd, "__builtin_ia32_punpcklwd", IX86_BUILTIN_PUNPCKLWD, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI},
21407 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_punpckldq, "__builtin_ia32_punpckldq", IX86_BUILTIN_PUNPCKLDQ, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI},
21409 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packsswb, "__builtin_ia32_packsswb", IX86_BUILTIN_PACKSSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
21410 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packssdw, "__builtin_ia32_packssdw", IX86_BUILTIN_PACKSSDW, UNKNOWN, (int) V4HI_FTYPE_V2SI_V2SI },
21411 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_packuswb, "__builtin_ia32_packuswb", IX86_BUILTIN_PACKUSWB, UNKNOWN, (int) V8QI_FTYPE_V4HI_V4HI },
21413 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_pmaddwd, "__builtin_ia32_pmaddwd", IX86_BUILTIN_PMADDWD, UNKNOWN, (int) V2SI_FTYPE_V4HI_V4HI },
21415 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllwi", IX86_BUILTIN_PSLLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
21416 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslldi", IX86_BUILTIN_PSLLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
21417 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllqi", IX86_BUILTIN_PSLLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
21418 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv4hi3, "__builtin_ia32_psllw", IX86_BUILTIN_PSLLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
21419 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv2si3, "__builtin_ia32_pslld", IX86_BUILTIN_PSLLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
21420 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashlv1di3, "__builtin_ia32_psllq", IX86_BUILTIN_PSLLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
21422 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlwi", IX86_BUILTIN_PSRLWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
21423 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrldi", IX86_BUILTIN_PSRLDI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
21424 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlqi", IX86_BUILTIN_PSRLQI, UNKNOWN, (int) V1DI_FTYPE_V1DI_SI_COUNT },
21425 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv4hi3, "__builtin_ia32_psrlw", IX86_BUILTIN_PSRLW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
21426 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv2si3, "__builtin_ia32_psrld", IX86_BUILTIN_PSRLD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
21427 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_lshrv1di3, "__builtin_ia32_psrlq", IX86_BUILTIN_PSRLQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI_COUNT },
21429 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psrawi", IX86_BUILTIN_PSRAWI, UNKNOWN, (int) V4HI_FTYPE_V4HI_SI_COUNT },
21430 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psradi", IX86_BUILTIN_PSRADI, UNKNOWN, (int) V2SI_FTYPE_V2SI_SI_COUNT },
21431 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv4hi3, "__builtin_ia32_psraw", IX86_BUILTIN_PSRAW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI_COUNT },
21432 { OPTION_MASK_ISA_MMX, CODE_FOR_mmx_ashrv2si3, "__builtin_ia32_psrad", IX86_BUILTIN_PSRAD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI_COUNT },
21435 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pf2id, "__builtin_ia32_pf2id", IX86_BUILTIN_PF2ID, UNKNOWN, (int) V2SI_FTYPE_V2SF },
21436 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_floatv2si2, "__builtin_ia32_pi2fd", IX86_BUILTIN_PI2FD, UNKNOWN, (int) V2SF_FTYPE_V2SI },
21437 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpv2sf2, "__builtin_ia32_pfrcp", IX86_BUILTIN_PFRCP, UNKNOWN, (int) V2SF_FTYPE_V2SF },
21438 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqrtv2sf2, "__builtin_ia32_pfrsqrt", IX86_BUILTIN_PFRSQRT, UNKNOWN, (int) V2SF_FTYPE_V2SF },
21440 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgusb", IX86_BUILTIN_PAVGUSB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
21441 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_haddv2sf3, "__builtin_ia32_pfacc", IX86_BUILTIN_PFACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
21442 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_addv2sf3, "__builtin_ia32_pfadd", IX86_BUILTIN_PFADD, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
21443 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_eqv2sf3, "__builtin_ia32_pfcmpeq", IX86_BUILTIN_PFCMPEQ, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
21444 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gev2sf3, "__builtin_ia32_pfcmpge", IX86_BUILTIN_PFCMPGE, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
21445 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_gtv2sf3, "__builtin_ia32_pfcmpgt", IX86_BUILTIN_PFCMPGT, UNKNOWN, (int) V2SI_FTYPE_V2SF_V2SF },
21446 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_smaxv2sf3, "__builtin_ia32_pfmax", IX86_BUILTIN_PFMAX, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
21447 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_sminv2sf3, "__builtin_ia32_pfmin", IX86_BUILTIN_PFMIN, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
21448 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_mulv2sf3, "__builtin_ia32_pfmul", IX86_BUILTIN_PFMUL, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
21449 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit1v2sf3, "__builtin_ia32_pfrcpit1", IX86_BUILTIN_PFRCPIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
21450 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rcpit2v2sf3, "__builtin_ia32_pfrcpit2", IX86_BUILTIN_PFRCPIT2, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
21451 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_rsqit1v2sf3, "__builtin_ia32_pfrsqit1", IX86_BUILTIN_PFRSQIT1, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
21452 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subv2sf3, "__builtin_ia32_pfsub", IX86_BUILTIN_PFSUB, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
21453 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_subrv2sf3, "__builtin_ia32_pfsubr", IX86_BUILTIN_PFSUBR, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
21454 { OPTION_MASK_ISA_3DNOW, CODE_FOR_mmx_pmulhrwv4hi3, "__builtin_ia32_pmulhrw", IX86_BUILTIN_PMULHRW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21457 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pf2iw, "__builtin_ia32_pf2iw", IX86_BUILTIN_PF2IW, UNKNOWN, (int) V2SI_FTYPE_V2SF },
21458 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pi2fw, "__builtin_ia32_pi2fw", IX86_BUILTIN_PI2FW, UNKNOWN, (int) V2SF_FTYPE_V2SI },
21459 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2si2, "__builtin_ia32_pswapdsi", IX86_BUILTIN_PSWAPDSI, UNKNOWN, (int) V2SI_FTYPE_V2SI },
21460 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pswapdv2sf2, "__builtin_ia32_pswapdsf", IX86_BUILTIN_PSWAPDSF, UNKNOWN, (int) V2SF_FTYPE_V2SF },
21461 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_hsubv2sf3, "__builtin_ia32_pfnacc", IX86_BUILTIN_PFNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
21462 { OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_addsubv2sf3, "__builtin_ia32_pfpnacc", IX86_BUILTIN_PFPNACC, UNKNOWN, (int) V2SF_FTYPE_V2SF_V2SF },
21465 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movmskps, "__builtin_ia32_movmskps", IX86_BUILTIN_MOVMSKPS, UNKNOWN, (int) INT_FTYPE_V4SF },
21466 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_sqrtv4sf2, "__builtin_ia32_sqrtps", IX86_BUILTIN_SQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
21467 { OPTION_MASK_ISA_SSE, CODE_FOR_sqrtv4sf2, "__builtin_ia32_sqrtps_nr", IX86_BUILTIN_SQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
21468 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rsqrtv4sf2, "__builtin_ia32_rsqrtps", IX86_BUILTIN_RSQRTPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
21469 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtv4sf2, "__builtin_ia32_rsqrtps_nr", IX86_BUILTIN_RSQRTPS_NR, UNKNOWN, (int) V4SF_FTYPE_V4SF },
21470 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_rcpv4sf2, "__builtin_ia32_rcpps", IX86_BUILTIN_RCPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF },
21471 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtps2pi, "__builtin_ia32_cvtps2pi", IX86_BUILTIN_CVTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
21472 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtss2si, "__builtin_ia32_cvtss2si", IX86_BUILTIN_CVTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
21473 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtss2siq, "__builtin_ia32_cvtss2si64", IX86_BUILTIN_CVTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
21474 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttps2pi, "__builtin_ia32_cvttps2pi", IX86_BUILTIN_CVTTPS2PI, UNKNOWN, (int) V2SI_FTYPE_V4SF },
21475 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvttss2si, "__builtin_ia32_cvttss2si", IX86_BUILTIN_CVTTSS2SI, UNKNOWN, (int) INT_FTYPE_V4SF },
21476 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvttss2siq, "__builtin_ia32_cvttss2si64", IX86_BUILTIN_CVTTSS2SI64, UNKNOWN, (int) INT64_FTYPE_V4SF },
21478 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_shufps, "__builtin_ia32_shufps", IX86_BUILTIN_SHUFPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
21480 { OPTION_MASK_ISA_SSE, CODE_FOR_addv4sf3, "__builtin_ia32_addps", IX86_BUILTIN_ADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21481 { OPTION_MASK_ISA_SSE, CODE_FOR_subv4sf3, "__builtin_ia32_subps", IX86_BUILTIN_SUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21482 { OPTION_MASK_ISA_SSE, CODE_FOR_mulv4sf3, "__builtin_ia32_mulps", IX86_BUILTIN_MULPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21483 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_divv4sf3, "__builtin_ia32_divps", IX86_BUILTIN_DIVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21484 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmaddv4sf3, "__builtin_ia32_addss", IX86_BUILTIN_ADDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21485 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsubv4sf3, "__builtin_ia32_subss", IX86_BUILTIN_SUBSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21486 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmulv4sf3, "__builtin_ia32_mulss", IX86_BUILTIN_MULSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21487 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmdivv4sf3, "__builtin_ia32_divss", IX86_BUILTIN_DIVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21489 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpeqps", IX86_BUILTIN_CMPEQPS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
21490 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpltps", IX86_BUILTIN_CMPLTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
21491 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpleps", IX86_BUILTIN_CMPLEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
21492 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgtps", IX86_BUILTIN_CMPGTPS, LT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
21493 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpgeps", IX86_BUILTIN_CMPGEPS, LE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
21494 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpunordps", IX86_BUILTIN_CMPUNORDPS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
21495 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpneqps", IX86_BUILTIN_CMPNEQPS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
21496 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnltps", IX86_BUILTIN_CMPNLTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
21497 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpnleps", IX86_BUILTIN_CMPNLEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
21498 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngtps", IX86_BUILTIN_CMPNGTPS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
21499 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpngeps", IX86_BUILTIN_CMPNGEPS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP},
21500 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_maskcmpv4sf3, "__builtin_ia32_cmpordps", IX86_BUILTIN_CMPORDPS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
21501 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpeqss", IX86_BUILTIN_CMPEQSS, EQ, (int) V4SF_FTYPE_V4SF_V4SF },
21502 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpltss", IX86_BUILTIN_CMPLTSS, LT, (int) V4SF_FTYPE_V4SF_V4SF },
21503 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpless", IX86_BUILTIN_CMPLESS, LE, (int) V4SF_FTYPE_V4SF_V4SF },
21504 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpunordss", IX86_BUILTIN_CMPUNORDSS, UNORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
21505 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpneqss", IX86_BUILTIN_CMPNEQSS, NE, (int) V4SF_FTYPE_V4SF_V4SF },
21506 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnltss", IX86_BUILTIN_CMPNLTSS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF },
21507 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpnless", IX86_BUILTIN_CMPNLESS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF },
21508 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpngtss", IX86_BUILTIN_CMPNGTSS, UNGE, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
21509 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpngess", IX86_BUILTIN_CMPNGESS, UNGT, (int) V4SF_FTYPE_V4SF_V4SF_SWAP },
21510 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmmaskcmpv4sf3, "__builtin_ia32_cmpordss", IX86_BUILTIN_CMPORDSS, ORDERED, (int) V4SF_FTYPE_V4SF_V4SF },
21512 { OPTION_MASK_ISA_SSE, CODE_FOR_sminv4sf3, "__builtin_ia32_minps", IX86_BUILTIN_MINPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21513 { OPTION_MASK_ISA_SSE, CODE_FOR_smaxv4sf3, "__builtin_ia32_maxps", IX86_BUILTIN_MAXPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21514 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsminv4sf3, "__builtin_ia32_minss", IX86_BUILTIN_MINSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21515 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsmaxv4sf3, "__builtin_ia32_maxss", IX86_BUILTIN_MAXSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21517 { OPTION_MASK_ISA_SSE, CODE_FOR_andv4sf3, "__builtin_ia32_andps", IX86_BUILTIN_ANDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21518 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_andnotv4sf3, "__builtin_ia32_andnps", IX86_BUILTIN_ANDNPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21519 { OPTION_MASK_ISA_SSE, CODE_FOR_iorv4sf3, "__builtin_ia32_orps", IX86_BUILTIN_ORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21520 { OPTION_MASK_ISA_SSE, CODE_FOR_xorv4sf3, "__builtin_ia32_xorps", IX86_BUILTIN_XORPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21522 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movss, "__builtin_ia32_movss", IX86_BUILTIN_MOVSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21523 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movhlps_exp, "__builtin_ia32_movhlps", IX86_BUILTIN_MOVHLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21524 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_movlhps_exp, "__builtin_ia32_movlhps", IX86_BUILTIN_MOVLHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21525 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_unpckhps, "__builtin_ia32_unpckhps", IX86_BUILTIN_UNPCKHPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21526 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_unpcklps, "__builtin_ia32_unpcklps", IX86_BUILTIN_UNPCKLPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21528 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtpi2ps, "__builtin_ia32_cvtpi2ps", IX86_BUILTIN_CVTPI2PS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2SI },
21529 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_cvtsi2ss, "__builtin_ia32_cvtsi2ss", IX86_BUILTIN_CVTSI2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_SI },
21530 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_64BIT, CODE_FOR_sse_cvtsi2ssq, "__builtin_ia32_cvtsi642ss", IX86_BUILTIN_CVTSI642SS, UNKNOWN, V4SF_FTYPE_V4SF_DI },
21532 { OPTION_MASK_ISA_SSE, CODE_FOR_rsqrtsf2, "__builtin_ia32_rsqrtf", IX86_BUILTIN_RSQRTF, UNKNOWN, (int) FLOAT_FTYPE_FLOAT },
21534 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmsqrtv4sf2, "__builtin_ia32_sqrtss", IX86_BUILTIN_SQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
21535 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrsqrtv4sf2, "__builtin_ia32_rsqrtss", IX86_BUILTIN_RSQRTSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
21536 { OPTION_MASK_ISA_SSE, CODE_FOR_sse_vmrcpv4sf2, "__builtin_ia32_rcpss", IX86_BUILTIN_RCPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_VEC_MERGE },
21538 /* SSE MMX or 3Dnow!A */
21539 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv8qi3, "__builtin_ia32_pavgb", IX86_BUILTIN_PAVGB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
21540 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uavgv4hi3, "__builtin_ia32_pavgw", IX86_BUILTIN_PAVGW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21541 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umulv4hi3_highpart, "__builtin_ia32_pmulhuw", IX86_BUILTIN_PMULHUW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21543 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_umaxv8qi3, "__builtin_ia32_pmaxub", IX86_BUILTIN_PMAXUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
21544 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_smaxv4hi3, "__builtin_ia32_pmaxsw", IX86_BUILTIN_PMAXSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21545 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_uminv8qi3, "__builtin_ia32_pminub", IX86_BUILTIN_PMINUB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
21546 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_sminv4hi3, "__builtin_ia32_pminsw", IX86_BUILTIN_PMINSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21548 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_psadbw, "__builtin_ia32_psadbw", IX86_BUILTIN_PSADBW, UNKNOWN, (int) V1DI_FTYPE_V8QI_V8QI },
21549 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pmovmskb, "__builtin_ia32_pmovmskb", IX86_BUILTIN_PMOVMSKB, UNKNOWN, (int) INT_FTYPE_V8QI },
21551 { OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, CODE_FOR_mmx_pshufw, "__builtin_ia32_pshufw", IX86_BUILTIN_PSHUFW, UNKNOWN, (int) V4HI_FTYPE_V4HI_INT },
21554 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_shufpd, "__builtin_ia32_shufpd", IX86_BUILTIN_SHUFPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
21556 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movmskpd, "__builtin_ia32_movmskpd", IX86_BUILTIN_MOVMSKPD, UNKNOWN, (int) INT_FTYPE_V2DF },
21557 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmovmskb, "__builtin_ia32_pmovmskb128", IX86_BUILTIN_PMOVMSKB128, UNKNOWN, (int) INT_FTYPE_V16QI },
21558 { OPTION_MASK_ISA_SSE2, CODE_FOR_sqrtv2df2, "__builtin_ia32_sqrtpd", IX86_BUILTIN_SQRTPD, UNKNOWN, (int) V2DF_FTYPE_V2DF },
21559 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtdq2pd, "__builtin_ia32_cvtdq2pd", IX86_BUILTIN_CVTDQ2PD, UNKNOWN, (int) V2DF_FTYPE_V4SI },
21560 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtdq2ps, "__builtin_ia32_cvtdq2ps", IX86_BUILTIN_CVTDQ2PS, UNKNOWN, (int) V4SF_FTYPE_V4SI },
21562 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2dq, "__builtin_ia32_cvtpd2dq", IX86_BUILTIN_CVTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
21563 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2pi, "__builtin_ia32_cvtpd2pi", IX86_BUILTIN_CVTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
21564 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpd2ps, "__builtin_ia32_cvtpd2ps", IX86_BUILTIN_CVTPD2PS, UNKNOWN, (int) V4SF_FTYPE_V2DF },
21565 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2dq, "__builtin_ia32_cvttpd2dq", IX86_BUILTIN_CVTTPD2DQ, UNKNOWN, (int) V4SI_FTYPE_V2DF },
21566 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttpd2pi, "__builtin_ia32_cvttpd2pi", IX86_BUILTIN_CVTTPD2PI, UNKNOWN, (int) V2SI_FTYPE_V2DF },
21568 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtpi2pd, "__builtin_ia32_cvtpi2pd", IX86_BUILTIN_CVTPI2PD, UNKNOWN, (int) V2DF_FTYPE_V2SI },
21570 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2si, "__builtin_ia32_cvtsd2si", IX86_BUILTIN_CVTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
21571 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttsd2si, "__builtin_ia32_cvttsd2si", IX86_BUILTIN_CVTTSD2SI, UNKNOWN, (int) INT_FTYPE_V2DF },
21572 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsd2siq, "__builtin_ia32_cvtsd2si64", IX86_BUILTIN_CVTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
21573 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvttsd2siq, "__builtin_ia32_cvttsd2si64", IX86_BUILTIN_CVTTSD2SI64, UNKNOWN, (int) INT64_FTYPE_V2DF },
21575 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2dq, "__builtin_ia32_cvtps2dq", IX86_BUILTIN_CVTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
21576 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtps2pd, "__builtin_ia32_cvtps2pd", IX86_BUILTIN_CVTPS2PD, UNKNOWN, (int) V2DF_FTYPE_V4SF },
21577 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvttps2dq, "__builtin_ia32_cvttps2dq", IX86_BUILTIN_CVTTPS2DQ, UNKNOWN, (int) V4SI_FTYPE_V4SF },
21579 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2df3, "__builtin_ia32_addpd", IX86_BUILTIN_ADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21580 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2df3, "__builtin_ia32_subpd", IX86_BUILTIN_SUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21581 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv2df3, "__builtin_ia32_mulpd", IX86_BUILTIN_MULPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21582 { OPTION_MASK_ISA_SSE2, CODE_FOR_divv2df3, "__builtin_ia32_divpd", IX86_BUILTIN_DIVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21583 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmaddv2df3, "__builtin_ia32_addsd", IX86_BUILTIN_ADDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21584 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsubv2df3, "__builtin_ia32_subsd", IX86_BUILTIN_SUBSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21585 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmulv2df3, "__builtin_ia32_mulsd", IX86_BUILTIN_MULSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21586 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmdivv2df3, "__builtin_ia32_divsd", IX86_BUILTIN_DIVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21588 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpeqpd", IX86_BUILTIN_CMPEQPD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
21589 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpltpd", IX86_BUILTIN_CMPLTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
21590 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmplepd", IX86_BUILTIN_CMPLEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
21591 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgtpd", IX86_BUILTIN_CMPGTPD, LT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
21592 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpgepd", IX86_BUILTIN_CMPGEPD, LE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP},
21593 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpunordpd", IX86_BUILTIN_CMPUNORDPD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
21594 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpneqpd", IX86_BUILTIN_CMPNEQPD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
21595 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnltpd", IX86_BUILTIN_CMPNLTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
21596 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpnlepd", IX86_BUILTIN_CMPNLEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
21597 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngtpd", IX86_BUILTIN_CMPNGTPD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
21598 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpngepd", IX86_BUILTIN_CMPNGEPD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF_SWAP },
21599 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_maskcmpv2df3, "__builtin_ia32_cmpordpd", IX86_BUILTIN_CMPORDPD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
21600 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpeqsd", IX86_BUILTIN_CMPEQSD, EQ, (int) V2DF_FTYPE_V2DF_V2DF },
21601 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpltsd", IX86_BUILTIN_CMPLTSD, LT, (int) V2DF_FTYPE_V2DF_V2DF },
21602 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmplesd", IX86_BUILTIN_CMPLESD, LE, (int) V2DF_FTYPE_V2DF_V2DF },
21603 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpunordsd", IX86_BUILTIN_CMPUNORDSD, UNORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
21604 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpneqsd", IX86_BUILTIN_CMPNEQSD, NE, (int) V2DF_FTYPE_V2DF_V2DF },
21605 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnltsd", IX86_BUILTIN_CMPNLTSD, UNGE, (int) V2DF_FTYPE_V2DF_V2DF },
21606 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpnlesd", IX86_BUILTIN_CMPNLESD, UNGT, (int) V2DF_FTYPE_V2DF_V2DF },
21607 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmmaskcmpv2df3, "__builtin_ia32_cmpordsd", IX86_BUILTIN_CMPORDSD, ORDERED, (int) V2DF_FTYPE_V2DF_V2DF },
21609 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv2df3, "__builtin_ia32_minpd", IX86_BUILTIN_MINPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21610 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv2df3, "__builtin_ia32_maxpd", IX86_BUILTIN_MAXPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21611 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsminv2df3, "__builtin_ia32_minsd", IX86_BUILTIN_MINSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21612 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsmaxv2df3, "__builtin_ia32_maxsd", IX86_BUILTIN_MAXSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21614 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2df3, "__builtin_ia32_andpd", IX86_BUILTIN_ANDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21615 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2df3, "__builtin_ia32_andnpd", IX86_BUILTIN_ANDNPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21616 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2df3, "__builtin_ia32_orpd", IX86_BUILTIN_ORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21617 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2df3, "__builtin_ia32_xorpd", IX86_BUILTIN_XORPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21619 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_movsd, "__builtin_ia32_movsd", IX86_BUILTIN_MOVSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21620 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_unpckhpd_exp, "__builtin_ia32_unpckhpd", IX86_BUILTIN_UNPCKHPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21621 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_unpcklpd_exp, "__builtin_ia32_unpcklpd", IX86_BUILTIN_UNPCKLPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21623 { OPTION_MASK_ISA_SSE2, CODE_FOR_vec_pack_sfix_v2df, "__builtin_ia32_vec_pack_sfix", IX86_BUILTIN_VEC_PACK_SFIX, UNKNOWN, (int) V4SI_FTYPE_V2DF_V2DF },
21625 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv16qi3, "__builtin_ia32_paddb128", IX86_BUILTIN_PADDB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21626 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv8hi3, "__builtin_ia32_paddw128", IX86_BUILTIN_PADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21627 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv4si3, "__builtin_ia32_paddd128", IX86_BUILTIN_PADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21628 { OPTION_MASK_ISA_SSE2, CODE_FOR_addv2di3, "__builtin_ia32_paddq128", IX86_BUILTIN_PADDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21629 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv16qi3, "__builtin_ia32_psubb128", IX86_BUILTIN_PSUBB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21630 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv8hi3, "__builtin_ia32_psubw128", IX86_BUILTIN_PSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21631 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv4si3, "__builtin_ia32_psubd128", IX86_BUILTIN_PSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21632 { OPTION_MASK_ISA_SSE2, CODE_FOR_subv2di3, "__builtin_ia32_psubq128", IX86_BUILTIN_PSUBQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21634 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv16qi3, "__builtin_ia32_paddsb128", IX86_BUILTIN_PADDSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21635 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ssaddv8hi3, "__builtin_ia32_paddsw128", IX86_BUILTIN_PADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21636 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv16qi3, "__builtin_ia32_psubsb128", IX86_BUILTIN_PSUBSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21637 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_sssubv8hi3, "__builtin_ia32_psubsw128", IX86_BUILTIN_PSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21638 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv16qi3, "__builtin_ia32_paddusb128", IX86_BUILTIN_PADDUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21639 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_usaddv8hi3, "__builtin_ia32_paddusw128", IX86_BUILTIN_PADDUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21640 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv16qi3, "__builtin_ia32_psubusb128", IX86_BUILTIN_PSUBUSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21641 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ussubv8hi3, "__builtin_ia32_psubusw128", IX86_BUILTIN_PSUBUSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21643 { OPTION_MASK_ISA_SSE2, CODE_FOR_mulv8hi3, "__builtin_ia32_pmullw128", IX86_BUILTIN_PMULLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21644 { OPTION_MASK_ISA_SSE2, CODE_FOR_smulv8hi3_highpart, "__builtin_ia32_pmulhw128", IX86_BUILTIN_PMULHW128, UNKNOWN,(int) V8HI_FTYPE_V8HI_V8HI },
21646 { OPTION_MASK_ISA_SSE2, CODE_FOR_andv2di3, "__builtin_ia32_pand128", IX86_BUILTIN_PAND128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21647 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_andnotv2di3, "__builtin_ia32_pandn128", IX86_BUILTIN_PANDN128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21648 { OPTION_MASK_ISA_SSE2, CODE_FOR_iorv2di3, "__builtin_ia32_por128", IX86_BUILTIN_POR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21649 { OPTION_MASK_ISA_SSE2, CODE_FOR_xorv2di3, "__builtin_ia32_pxor128", IX86_BUILTIN_PXOR128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21651 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv16qi3, "__builtin_ia32_pavgb128", IX86_BUILTIN_PAVGB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21652 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_uavgv8hi3, "__builtin_ia32_pavgw128", IX86_BUILTIN_PAVGW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21654 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv16qi3, "__builtin_ia32_pcmpeqb128", IX86_BUILTIN_PCMPEQB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21655 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv8hi3, "__builtin_ia32_pcmpeqw128", IX86_BUILTIN_PCMPEQW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21656 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_eqv4si3, "__builtin_ia32_pcmpeqd128", IX86_BUILTIN_PCMPEQD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21657 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv16qi3, "__builtin_ia32_pcmpgtb128", IX86_BUILTIN_PCMPGTB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21658 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv8hi3, "__builtin_ia32_pcmpgtw128", IX86_BUILTIN_PCMPGTW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21659 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_gtv4si3, "__builtin_ia32_pcmpgtd128", IX86_BUILTIN_PCMPGTD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21661 { OPTION_MASK_ISA_SSE2, CODE_FOR_umaxv16qi3, "__builtin_ia32_pmaxub128", IX86_BUILTIN_PMAXUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21662 { OPTION_MASK_ISA_SSE2, CODE_FOR_smaxv8hi3, "__builtin_ia32_pmaxsw128", IX86_BUILTIN_PMAXSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21663 { OPTION_MASK_ISA_SSE2, CODE_FOR_uminv16qi3, "__builtin_ia32_pminub128", IX86_BUILTIN_PMINUB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21664 { OPTION_MASK_ISA_SSE2, CODE_FOR_sminv8hi3, "__builtin_ia32_pminsw128", IX86_BUILTIN_PMINSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21666 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_punpckhbw, "__builtin_ia32_punpckhbw128", IX86_BUILTIN_PUNPCKHBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21667 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_punpckhwd, "__builtin_ia32_punpckhwd128", IX86_BUILTIN_PUNPCKHWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21668 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_punpckhdq, "__builtin_ia32_punpckhdq128", IX86_BUILTIN_PUNPCKHDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21669 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_punpckhqdq, "__builtin_ia32_punpckhqdq128", IX86_BUILTIN_PUNPCKHQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21670 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_punpcklbw, "__builtin_ia32_punpcklbw128", IX86_BUILTIN_PUNPCKLBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21671 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_punpcklwd, "__builtin_ia32_punpcklwd128", IX86_BUILTIN_PUNPCKLWD128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21672 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_punpckldq, "__builtin_ia32_punpckldq128", IX86_BUILTIN_PUNPCKLDQ128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21673 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_punpcklqdq, "__builtin_ia32_punpcklqdq128", IX86_BUILTIN_PUNPCKLQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21675 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packsswb, "__builtin_ia32_packsswb128", IX86_BUILTIN_PACKSSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
21676 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packssdw, "__builtin_ia32_packssdw128", IX86_BUILTIN_PACKSSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
21677 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_packuswb, "__builtin_ia32_packuswb128", IX86_BUILTIN_PACKUSWB128, UNKNOWN, (int) V16QI_FTYPE_V8HI_V8HI },
21679 { OPTION_MASK_ISA_SSE2, CODE_FOR_umulv8hi3_highpart, "__builtin_ia32_pmulhuw128", IX86_BUILTIN_PMULHUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21680 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_psadbw, "__builtin_ia32_psadbw128", IX86_BUILTIN_PSADBW128, UNKNOWN, (int) V2DI_FTYPE_V16QI_V16QI },
21682 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_umulv1siv1di3, "__builtin_ia32_pmuludq", IX86_BUILTIN_PMULUDQ, UNKNOWN, (int) V1DI_FTYPE_V2SI_V2SI },
21683 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_umulv2siv2di3, "__builtin_ia32_pmuludq128", IX86_BUILTIN_PMULUDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
21685 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pmaddwd, "__builtin_ia32_pmaddwd128", IX86_BUILTIN_PMADDWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI_V8HI },
21687 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsi2sd, "__builtin_ia32_cvtsi2sd", IX86_BUILTIN_CVTSI2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_SI },
21688 { OPTION_MASK_ISA_SSE2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse2_cvtsi2sdq, "__builtin_ia32_cvtsi642sd", IX86_BUILTIN_CVTSI642SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_DI },
21689 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtsd2ss, "__builtin_ia32_cvtsd2ss", IX86_BUILTIN_CVTSD2SS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V2DF },
21690 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_cvtss2sd, "__builtin_ia32_cvtss2sd", IX86_BUILTIN_CVTSS2SD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V4SF },
21692 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_ashlti3, "__builtin_ia32_pslldqi128", IX86_BUILTIN_PSLLDQI128, UNKNOWN, (int) V2DI2TI_FTYPE_V2DI_INT },
21693 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllwi128", IX86_BUILTIN_PSLLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
21694 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslldi128", IX86_BUILTIN_PSLLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
21695 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllqi128", IX86_BUILTIN_PSLLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
21696 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv8hi3, "__builtin_ia32_psllw128", IX86_BUILTIN_PSLLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
21697 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv4si3, "__builtin_ia32_pslld128", IX86_BUILTIN_PSLLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
21698 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashlv2di3, "__builtin_ia32_psllq128", IX86_BUILTIN_PSLLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
21700 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_lshrti3, "__builtin_ia32_psrldqi128", IX86_BUILTIN_PSRLDQI128, UNKNOWN, (int) V2DI2TI_FTYPE_V2DI_INT },
21701 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlwi128", IX86_BUILTIN_PSRLWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
21702 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrldi128", IX86_BUILTIN_PSRLDI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
21703 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlqi128", IX86_BUILTIN_PSRLQI128, UNKNOWN, (int) V2DI_FTYPE_V2DI_SI_COUNT },
21704 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv8hi3, "__builtin_ia32_psrlw128", IX86_BUILTIN_PSRLW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
21705 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv4si3, "__builtin_ia32_psrld128", IX86_BUILTIN_PSRLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
21706 { OPTION_MASK_ISA_SSE2, CODE_FOR_lshrv2di3, "__builtin_ia32_psrlq128", IX86_BUILTIN_PSRLQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_COUNT },
21708 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psrawi128", IX86_BUILTIN_PSRAWI128, UNKNOWN, (int) V8HI_FTYPE_V8HI_SI_COUNT },
21709 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psradi128", IX86_BUILTIN_PSRADI128, UNKNOWN, (int) V4SI_FTYPE_V4SI_SI_COUNT },
21710 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv8hi3, "__builtin_ia32_psraw128", IX86_BUILTIN_PSRAW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_COUNT },
21711 { OPTION_MASK_ISA_SSE2, CODE_FOR_ashrv4si3, "__builtin_ia32_psrad128", IX86_BUILTIN_PSRAD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI_COUNT },
21713 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufd, "__builtin_ia32_pshufd", IX86_BUILTIN_PSHUFD, UNKNOWN, (int) V4SI_FTYPE_V4SI_INT },
21714 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshuflw, "__builtin_ia32_pshuflw", IX86_BUILTIN_PSHUFLW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
21715 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_pshufhw, "__builtin_ia32_pshufhw", IX86_BUILTIN_PSHUFHW, UNKNOWN, (int) V8HI_FTYPE_V8HI_INT },
21717 { OPTION_MASK_ISA_SSE2, CODE_FOR_sse2_vmsqrtv2df2, "__builtin_ia32_sqrtsd", IX86_BUILTIN_SQRTSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_VEC_MERGE },
21719 { OPTION_MASK_ISA_SSE2, CODE_FOR_abstf2, 0, IX86_BUILTIN_FABSQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128 },
21720 { OPTION_MASK_ISA_SSE2, CODE_FOR_copysigntf3, 0, IX86_BUILTIN_COPYSIGNQ, UNKNOWN, (int) FLOAT128_FTYPE_FLOAT128_FLOAT128 },
21722 { OPTION_MASK_ISA_SSE, CODE_FOR_sse2_movq128, "__builtin_ia32_movq128", IX86_BUILTIN_MOVQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
21725 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_addv1di3, "__builtin_ia32_paddq", IX86_BUILTIN_PADDQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
21726 { OPTION_MASK_ISA_SSE2, CODE_FOR_mmx_subv1di3, "__builtin_ia32_psubq", IX86_BUILTIN_PSUBQ, UNKNOWN, (int) V1DI_FTYPE_V1DI_V1DI },
21729 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movshdup, "__builtin_ia32_movshdup", IX86_BUILTIN_MOVSHDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF},
21730 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_movsldup, "__builtin_ia32_movsldup", IX86_BUILTIN_MOVSLDUP, UNKNOWN, (int) V4SF_FTYPE_V4SF },
21732 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv4sf3, "__builtin_ia32_addsubps", IX86_BUILTIN_ADDSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21733 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_addsubv2df3, "__builtin_ia32_addsubpd", IX86_BUILTIN_ADDSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21734 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv4sf3, "__builtin_ia32_haddps", IX86_BUILTIN_HADDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21735 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_haddv2df3, "__builtin_ia32_haddpd", IX86_BUILTIN_HADDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21736 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv4sf3, "__builtin_ia32_hsubps", IX86_BUILTIN_HSUBPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF },
21737 { OPTION_MASK_ISA_SSE3, CODE_FOR_sse3_hsubv2df3, "__builtin_ia32_hsubpd", IX86_BUILTIN_HSUBPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF },
21740 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv16qi2, "__builtin_ia32_pabsb128", IX86_BUILTIN_PABSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI },
21741 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8qi2, "__builtin_ia32_pabsb", IX86_BUILTIN_PABSB, UNKNOWN, (int) V8QI_FTYPE_V8QI },
21742 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv8hi2, "__builtin_ia32_pabsw128", IX86_BUILTIN_PABSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
21743 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4hi2, "__builtin_ia32_pabsw", IX86_BUILTIN_PABSW, UNKNOWN, (int) V4HI_FTYPE_V4HI },
21744 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv4si2, "__builtin_ia32_pabsd128", IX86_BUILTIN_PABSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI },
21745 { OPTION_MASK_ISA_SSSE3, CODE_FOR_absv2si2, "__builtin_ia32_pabsd", IX86_BUILTIN_PABSD, UNKNOWN, (int) V2SI_FTYPE_V2SI },
21747 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv8hi3, "__builtin_ia32_phaddw128", IX86_BUILTIN_PHADDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21748 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddwv4hi3, "__builtin_ia32_phaddw", IX86_BUILTIN_PHADDW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21749 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv4si3, "__builtin_ia32_phaddd128", IX86_BUILTIN_PHADDD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21750 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phadddv2si3, "__builtin_ia32_phaddd", IX86_BUILTIN_PHADDD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
21751 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv8hi3, "__builtin_ia32_phaddsw128", IX86_BUILTIN_PHADDSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21752 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phaddswv4hi3, "__builtin_ia32_phaddsw", IX86_BUILTIN_PHADDSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21753 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv8hi3, "__builtin_ia32_phsubw128", IX86_BUILTIN_PHSUBW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21754 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubwv4hi3, "__builtin_ia32_phsubw", IX86_BUILTIN_PHSUBW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21755 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv4si3, "__builtin_ia32_phsubd128", IX86_BUILTIN_PHSUBD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21756 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubdv2si3, "__builtin_ia32_phsubd", IX86_BUILTIN_PHSUBD, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
21757 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv8hi3, "__builtin_ia32_phsubsw128", IX86_BUILTIN_PHSUBSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21758 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_phsubswv4hi3, "__builtin_ia32_phsubsw", IX86_BUILTIN_PHSUBSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21759 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw128, "__builtin_ia32_pmaddubsw128", IX86_BUILTIN_PMADDUBSW128, UNKNOWN, (int) V8HI_FTYPE_V16QI_V16QI },
21760 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmaddubsw, "__builtin_ia32_pmaddubsw", IX86_BUILTIN_PMADDUBSW, UNKNOWN, (int) V4HI_FTYPE_V8QI_V8QI },
21761 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv8hi3, "__builtin_ia32_pmulhrsw128", IX86_BUILTIN_PMULHRSW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21762 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pmulhrswv4hi3, "__builtin_ia32_pmulhrsw", IX86_BUILTIN_PMULHRSW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21763 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv16qi3, "__builtin_ia32_pshufb128", IX86_BUILTIN_PSHUFB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21764 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_pshufbv8qi3, "__builtin_ia32_pshufb", IX86_BUILTIN_PSHUFB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
21765 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv16qi3, "__builtin_ia32_psignb128", IX86_BUILTIN_PSIGNB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21766 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8qi3, "__builtin_ia32_psignb", IX86_BUILTIN_PSIGNB, UNKNOWN, (int) V8QI_FTYPE_V8QI_V8QI },
21767 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv8hi3, "__builtin_ia32_psignw128", IX86_BUILTIN_PSIGNW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21768 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4hi3, "__builtin_ia32_psignw", IX86_BUILTIN_PSIGNW, UNKNOWN, (int) V4HI_FTYPE_V4HI_V4HI },
21769 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv4si3, "__builtin_ia32_psignd128", IX86_BUILTIN_PSIGND128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21770 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_psignv2si3, "__builtin_ia32_psignd", IX86_BUILTIN_PSIGND, UNKNOWN, (int) V2SI_FTYPE_V2SI_V2SI },
21773 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrti, "__builtin_ia32_palignr128", IX86_BUILTIN_PALIGNR128, UNKNOWN, (int) V2DI2TI_FTYPE_V2DI_V2DI_INT },
21774 { OPTION_MASK_ISA_SSSE3, CODE_FOR_ssse3_palignrdi, "__builtin_ia32_palignr", IX86_BUILTIN_PALIGNR, UNKNOWN, (int) V1DI2DI_FTYPE_V1DI_V1DI_INT },
21777 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendpd, "__builtin_ia32_blendpd", IX86_BUILTIN_BLENDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
21778 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendps, "__builtin_ia32_blendps", IX86_BUILTIN_BLENDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
21779 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvpd, "__builtin_ia32_blendvpd", IX86_BUILTIN_BLENDVPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_V2DF },
21780 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_blendvps, "__builtin_ia32_blendvps", IX86_BUILTIN_BLENDVPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_V4SF },
21781 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dppd, "__builtin_ia32_dppd", IX86_BUILTIN_DPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
21782 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_dpps, "__builtin_ia32_dpps", IX86_BUILTIN_DPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
21783 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_insertps, "__builtin_ia32_insertps128", IX86_BUILTIN_INSERTPS128, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
21784 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mpsadbw, "__builtin_ia32_mpsadbw128", IX86_BUILTIN_MPSADBW128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_INT },
21785 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendvb, "__builtin_ia32_pblendvb128", IX86_BUILTIN_PBLENDVB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI },
21786 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendw, "__builtin_ia32_pblendw128", IX86_BUILTIN_PBLENDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_INT },
21788 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_extendv8qiv8hi2, "__builtin_ia32_pmovsxbw128", IX86_BUILTIN_PMOVSXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
21789 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_extendv4qiv4si2, "__builtin_ia32_pmovsxbd128", IX86_BUILTIN_PMOVSXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
21790 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_extendv2qiv2di2, "__builtin_ia32_pmovsxbq128", IX86_BUILTIN_PMOVSXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
21791 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_extendv4hiv4si2, "__builtin_ia32_pmovsxwd128", IX86_BUILTIN_PMOVSXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
21792 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_extendv2hiv2di2, "__builtin_ia32_pmovsxwq128", IX86_BUILTIN_PMOVSXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
21793 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_extendv2siv2di2, "__builtin_ia32_pmovsxdq128", IX86_BUILTIN_PMOVSXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
21794 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv8qiv8hi2, "__builtin_ia32_pmovzxbw128", IX86_BUILTIN_PMOVZXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI },
21795 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4qiv4si2, "__builtin_ia32_pmovzxbd128", IX86_BUILTIN_PMOVZXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI },
21796 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2qiv2di2, "__builtin_ia32_pmovzxbq128", IX86_BUILTIN_PMOVZXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI },
21797 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4hiv4si2, "__builtin_ia32_pmovzxwd128", IX86_BUILTIN_PMOVZXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI },
21798 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2hiv2di2, "__builtin_ia32_pmovzxwq128", IX86_BUILTIN_PMOVZXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI },
21799 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2siv2di2, "__builtin_ia32_pmovzxdq128", IX86_BUILTIN_PMOVZXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI },
21800 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_phminposuw, "__builtin_ia32_phminposuw128", IX86_BUILTIN_PHMINPOSUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI },
21802 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_packusdw, "__builtin_ia32_packusdw128", IX86_BUILTIN_PACKUSDW128, UNKNOWN, (int) V8HI_FTYPE_V4SI_V4SI },
21803 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_eqv2di3, "__builtin_ia32_pcmpeqq", IX86_BUILTIN_PCMPEQQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21804 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv16qi3, "__builtin_ia32_pmaxsb128", IX86_BUILTIN_PMAXSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21805 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_smaxv4si3, "__builtin_ia32_pmaxsd128", IX86_BUILTIN_PMAXSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21806 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv4si3, "__builtin_ia32_pmaxud128", IX86_BUILTIN_PMAXUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21807 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_umaxv8hi3, "__builtin_ia32_pmaxuw128", IX86_BUILTIN_PMAXUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21808 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv16qi3, "__builtin_ia32_pminsb128", IX86_BUILTIN_PMINSB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI },
21809 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sminv4si3, "__builtin_ia32_pminsd128", IX86_BUILTIN_PMINSD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21810 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv4si3, "__builtin_ia32_pminud128", IX86_BUILTIN_PMINUD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21811 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_uminv8hi3, "__builtin_ia32_pminuw128", IX86_BUILTIN_PMINUW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI },
21812 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_mulv2siv2di3, "__builtin_ia32_pmuldq128", IX86_BUILTIN_PMULDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI_V4SI },
21813 { OPTION_MASK_ISA_SSE4_1, CODE_FOR_mulv4si3, "__builtin_ia32_pmulld128", IX86_BUILTIN_PMULLD128, UNKNOWN, (int) V4SI_FTYPE_V4SI_V4SI },
21815 /* SSE4.1 and SSE5 */
21816 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundpd, "__builtin_ia32_roundpd", IX86_BUILTIN_ROUNDPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
21817 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundps, "__builtin_ia32_roundps", IX86_BUILTIN_ROUNDPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
21818 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundsd, "__builtin_ia32_roundsd", IX86_BUILTIN_ROUNDSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
21819 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_roundss, "__builtin_ia32_roundss", IX86_BUILTIN_ROUNDSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
21821 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestz128", IX86_BUILTIN_PTESTZ, EQ, (int) INT_FTYPE_V2DI_V2DI_PTEST },
21822 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestc128", IX86_BUILTIN_PTESTC, LTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
21823 { OPTION_MASK_ISA_ROUND, CODE_FOR_sse4_1_ptest, "__builtin_ia32_ptestnzc128", IX86_BUILTIN_PTESTNZC, GTU, (int) INT_FTYPE_V2DI_V2DI_PTEST },
21826 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_gtv2di3, "__builtin_ia32_pcmpgtq", IX86_BUILTIN_PCMPGTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21827 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_crc32qi, "__builtin_ia32_crc32qi", IX86_BUILTIN_CRC32QI, UNKNOWN, (int) UINT_FTYPE_UINT_UCHAR },
21828 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_crc32hi, "__builtin_ia32_crc32hi", IX86_BUILTIN_CRC32HI, UNKNOWN, (int) UINT_FTYPE_UINT_USHORT },
21829 { OPTION_MASK_ISA_SSE4_2, CODE_FOR_sse4_2_crc32si, "__builtin_ia32_crc32si", IX86_BUILTIN_CRC32SI, UNKNOWN, (int) UINT_FTYPE_UINT_UINT },
21830 { OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_64BIT, CODE_FOR_sse4_2_crc32di, "__builtin_ia32_crc32di", IX86_BUILTIN_CRC32DI, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 },
21833 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrqi, "__builtin_ia32_extrqi", IX86_BUILTIN_EXTRQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_UINT_UINT },
21834 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_extrq, "__builtin_ia32_extrq", IX86_BUILTIN_EXTRQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V16QI },
21835 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertqi, "__builtin_ia32_insertqi", IX86_BUILTIN_INSERTQI, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_UINT_UINT },
21836 { OPTION_MASK_ISA_SSE4A, CODE_FOR_sse4a_insertq, "__builtin_ia32_insertq", IX86_BUILTIN_INSERTQ, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21839 { OPTION_MASK_ISA_SSE2, CODE_FOR_aeskeygenassist, 0, IX86_BUILTIN_AESKEYGENASSIST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_INT },
21840 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesimc, 0, IX86_BUILTIN_AESIMC128, UNKNOWN, (int) V2DI_FTYPE_V2DI },
21842 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenc, 0, IX86_BUILTIN_AESENC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21843 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesenclast, 0, IX86_BUILTIN_AESENCLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21844 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdec, 0, IX86_BUILTIN_AESDEC128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21845 { OPTION_MASK_ISA_SSE2, CODE_FOR_aesdeclast, 0, IX86_BUILTIN_AESDECLAST128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI },
21848 { OPTION_MASK_ISA_SSE2, CODE_FOR_pclmulqdq, 0, IX86_BUILTIN_PCLMULQDQ128, UNKNOWN, (int) V2DI_FTYPE_V2DI_V2DI_INT },
21851 { OPTION_MASK_ISA_AVX, CODE_FOR_addv4df3, "__builtin_ia32_addpd256", IX86_BUILTIN_ADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21852 { OPTION_MASK_ISA_AVX, CODE_FOR_addv8sf3, "__builtin_ia32_addps256", IX86_BUILTIN_ADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21853 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv4df3, "__builtin_ia32_addsubpd256", IX86_BUILTIN_ADDSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21854 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_addsubv8sf3, "__builtin_ia32_addsubps256", IX86_BUILTIN_ADDSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21855 { OPTION_MASK_ISA_AVX, CODE_FOR_andv4df3, "__builtin_ia32_andpd256", IX86_BUILTIN_ANDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21856 { OPTION_MASK_ISA_AVX, CODE_FOR_andv8sf3, "__builtin_ia32_andps256", IX86_BUILTIN_ANDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21857 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv4df3, "__builtin_ia32_andnpd256", IX86_BUILTIN_ANDNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21858 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_andnotv8sf3, "__builtin_ia32_andnps256", IX86_BUILTIN_ANDNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21859 { OPTION_MASK_ISA_AVX, CODE_FOR_divv4df3, "__builtin_ia32_divpd256", IX86_BUILTIN_DIVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21860 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_divv8sf3, "__builtin_ia32_divps256", IX86_BUILTIN_DIVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21861 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv4df3, "__builtin_ia32_haddpd256", IX86_BUILTIN_HADDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21862 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv8sf3, "__builtin_ia32_hsubps256", IX86_BUILTIN_HSUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21863 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_hsubv4df3, "__builtin_ia32_hsubpd256", IX86_BUILTIN_HSUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21864 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_haddv8sf3, "__builtin_ia32_haddps256", IX86_BUILTIN_HADDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21865 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv4df3, "__builtin_ia32_maxpd256", IX86_BUILTIN_MAXPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21866 { OPTION_MASK_ISA_AVX, CODE_FOR_smaxv8sf3, "__builtin_ia32_maxps256", IX86_BUILTIN_MAXPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21867 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv4df3, "__builtin_ia32_minpd256", IX86_BUILTIN_MINPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21868 { OPTION_MASK_ISA_AVX, CODE_FOR_sminv8sf3, "__builtin_ia32_minps256", IX86_BUILTIN_MINPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21869 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv4df3, "__builtin_ia32_mulpd256", IX86_BUILTIN_MULPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21870 { OPTION_MASK_ISA_AVX, CODE_FOR_mulv8sf3, "__builtin_ia32_mulps256", IX86_BUILTIN_MULPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21871 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv4df3, "__builtin_ia32_orpd256", IX86_BUILTIN_ORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21872 { OPTION_MASK_ISA_AVX, CODE_FOR_iorv8sf3, "__builtin_ia32_orps256", IX86_BUILTIN_ORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21873 { OPTION_MASK_ISA_AVX, CODE_FOR_subv4df3, "__builtin_ia32_subpd256", IX86_BUILTIN_SUBPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21874 { OPTION_MASK_ISA_AVX, CODE_FOR_subv8sf3, "__builtin_ia32_subps256", IX86_BUILTIN_SUBPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21875 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv4df3, "__builtin_ia32_xorpd256", IX86_BUILTIN_XORPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21876 { OPTION_MASK_ISA_AVX, CODE_FOR_xorv8sf3, "__builtin_ia32_xorps256", IX86_BUILTIN_XORPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21878 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv2df3, "__builtin_ia32_vpermilvarpd", IX86_BUILTIN_VPERMILVARPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DI },
21879 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4sf3, "__builtin_ia32_vpermilvarps", IX86_BUILTIN_VPERMILVARPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SI },
21880 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv4df3, "__builtin_ia32_vpermilvarpd256", IX86_BUILTIN_VPERMILVARPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DI },
21881 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilvarv8sf3, "__builtin_ia32_vpermilvarps256", IX86_BUILTIN_VPERMILVARPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SI },
21883 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendpd256, "__builtin_ia32_blendpd256", IX86_BUILTIN_BLENDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
21884 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendps256, "__builtin_ia32_blendps256", IX86_BUILTIN_BLENDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
21885 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvpd256, "__builtin_ia32_blendvpd256", IX86_BUILTIN_BLENDVPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_V4DF },
21886 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_blendvps256, "__builtin_ia32_blendvps256", IX86_BUILTIN_BLENDVPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_V8SF },
21887 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_dpps256, "__builtin_ia32_dpps256", IX86_BUILTIN_DPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
21888 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufpd256, "__builtin_ia32_shufpd256", IX86_BUILTIN_SHUFPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
21889 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_shufps256, "__builtin_ia32_shufps256", IX86_BUILTIN_SHUFPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
21890 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpsdv2df3, "__builtin_ia32_cmpsd", IX86_BUILTIN_CMPSD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
21891 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmpssv4sf3, "__builtin_ia32_cmpss", IX86_BUILTIN_CMPSS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
21892 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmppdv2df3, "__builtin_ia32_cmppd", IX86_BUILTIN_CMPPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_V2DF_INT },
21893 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmppsv4sf3, "__builtin_ia32_cmpps", IX86_BUILTIN_CMPPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_V4SF_INT },
21894 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmppdv4df3, "__builtin_ia32_cmppd256", IX86_BUILTIN_CMPPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
21895 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cmppsv8sf3, "__builtin_ia32_cmpps256", IX86_BUILTIN_CMPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
21896 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v4df, "__builtin_ia32_vextractf128_pd256", IX86_BUILTIN_EXTRACTF128PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF_INT },
21897 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8sf, "__builtin_ia32_vextractf128_ps256", IX86_BUILTIN_EXTRACTF128PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF_INT },
21898 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vextractf128v8si, "__builtin_ia32_vextractf128_si256", IX86_BUILTIN_EXTRACTF128SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI_INT },
21899 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtdq2pd256, "__builtin_ia32_cvtdq2pd256", IX86_BUILTIN_CVTDQ2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SI },
21900 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtdq2ps256, "__builtin_ia32_cvtdq2ps256", IX86_BUILTIN_CVTDQ2PS256, UNKNOWN, (int) V8SF_FTYPE_V8SI },
21901 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2ps256, "__builtin_ia32_cvtpd2ps256", IX86_BUILTIN_CVTPD2PS256, UNKNOWN, (int) V4SF_FTYPE_V4DF },
21902 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2dq256, "__builtin_ia32_cvtps2dq256", IX86_BUILTIN_CVTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
21903 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtps2pd256, "__builtin_ia32_cvtps2pd256", IX86_BUILTIN_CVTPS2PD256, UNKNOWN, (int) V4DF_FTYPE_V4SF },
21904 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvttpd2dq256, "__builtin_ia32_cvttpd2dq256", IX86_BUILTIN_CVTTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
21905 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvtpd2dq256, "__builtin_ia32_cvtpd2dq256", IX86_BUILTIN_CVTPD2DQ256, UNKNOWN, (int) V4SI_FTYPE_V4DF },
21906 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_cvttps2dq256, "__builtin_ia32_cvttps2dq256", IX86_BUILTIN_CVTTPS2DQ256, UNKNOWN, (int) V8SI_FTYPE_V8SF },
21907 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v4df3, "__builtin_ia32_vperm2f128_pd256", IX86_BUILTIN_VPERM2F128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF_INT },
21908 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8sf3, "__builtin_ia32_vperm2f128_ps256", IX86_BUILTIN_VPERM2F128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF_INT },
21909 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vperm2f128v8si3, "__builtin_ia32_vperm2f128_si256", IX86_BUILTIN_VPERM2F128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V8SI_INT },
21910 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv2df, "__builtin_ia32_vpermilpd", IX86_BUILTIN_VPERMILPD, UNKNOWN, (int) V2DF_FTYPE_V2DF_INT },
21911 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4sf, "__builtin_ia32_vpermilps", IX86_BUILTIN_VPERMILPS, UNKNOWN, (int) V4SF_FTYPE_V4SF_INT },
21912 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv4df, "__builtin_ia32_vpermilpd256", IX86_BUILTIN_VPERMILPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
21913 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vpermilv8sf, "__builtin_ia32_vpermilps256", IX86_BUILTIN_VPERMILPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
21914 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v4df, "__builtin_ia32_vinsertf128_pd256", IX86_BUILTIN_VINSERTF128PD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V2DF_INT },
21915 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8sf, "__builtin_ia32_vinsertf128_ps256", IX86_BUILTIN_VINSERTF128PS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V4SF_INT },
21916 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vinsertf128v8si, "__builtin_ia32_vinsertf128_si256", IX86_BUILTIN_VINSERTF128SI256, UNKNOWN, (int) V8SI_FTYPE_V8SI_V4SI_INT },
21918 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movshdup256, "__builtin_ia32_movshdup256", IX86_BUILTIN_MOVSHDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
21919 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movsldup256, "__builtin_ia32_movsldup256", IX86_BUILTIN_MOVSLDUP256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
21920 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movddup256, "__builtin_ia32_movddup256", IX86_BUILTIN_MOVDDUP256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
21922 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv4df2, "__builtin_ia32_sqrtpd256", IX86_BUILTIN_SQRTPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF },
21923 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_sqrtv8sf2, "__builtin_ia32_sqrtps256", IX86_BUILTIN_SQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
21924 { OPTION_MASK_ISA_AVX, CODE_FOR_sqrtv8sf2, "__builtin_ia32_sqrtps_nr256", IX86_BUILTIN_SQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
21925 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rsqrtv8sf2, "__builtin_ia32_rsqrtps256", IX86_BUILTIN_RSQRTPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
21926 { OPTION_MASK_ISA_AVX, CODE_FOR_rsqrtv8sf2, "__builtin_ia32_rsqrtps_nr256", IX86_BUILTIN_RSQRTPS_NR256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
21928 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_rcpv8sf2, "__builtin_ia32_rcpps256", IX86_BUILTIN_RCPPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF },
21930 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundpd256, "__builtin_ia32_roundpd256", IX86_BUILTIN_ROUNDPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_INT },
21931 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_roundps256, "__builtin_ia32_roundps256", IX86_BUILTIN_ROUNDPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_INT },
21933 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhpd256, "__builtin_ia32_unpckhpd256", IX86_BUILTIN_UNPCKHPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21934 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklpd256, "__builtin_ia32_unpcklpd256", IX86_BUILTIN_UNPCKLPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF },
21935 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpckhps256, "__builtin_ia32_unpckhps256", IX86_BUILTIN_UNPCKHPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21936 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_unpcklps256, "__builtin_ia32_unpcklps256", IX86_BUILTIN_UNPCKLPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF },
21938 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_si256_si, "__builtin_ia32_si256_si", IX86_BUILTIN_SI256_SI, UNKNOWN, (int) V8SI_FTYPE_V4SI },
21939 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ps256_ps, "__builtin_ia32_ps256_ps", IX86_BUILTIN_PS256_PS, UNKNOWN, (int) V8SF_FTYPE_V4SF },
21940 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_pd256_pd, "__builtin_ia32_pd256_pd", IX86_BUILTIN_PD256_PD, UNKNOWN, (int) V4DF_FTYPE_V2DF },
21941 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_si_si256, "__builtin_ia32_si_si256", IX86_BUILTIN_SI_SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI },
21942 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ps_ps256, "__builtin_ia32_ps_ps256", IX86_BUILTIN_PS_PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF },
21943 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_pd_pd256, "__builtin_ia32_pd_pd256", IX86_BUILTIN_PD_PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF },
21945 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestzpd", IX86_BUILTIN_VTESTZPD, EQ, (int) INT_FTYPE_V2DF_V2DF_PTEST },
21946 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestcpd", IX86_BUILTIN_VTESTCPD, LTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
21947 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestnzcpd", IX86_BUILTIN_VTESTNZCPD, GTU, (int) INT_FTYPE_V2DF_V2DF_PTEST },
21948 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestzps", IX86_BUILTIN_VTESTZPS, EQ, (int) INT_FTYPE_V4SF_V4SF_PTEST },
21949 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestcps", IX86_BUILTIN_VTESTCPS, LTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
21950 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps, "__builtin_ia32_vtestnzcps", IX86_BUILTIN_VTESTNZCPS, GTU, (int) INT_FTYPE_V4SF_V4SF_PTEST },
21951 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestzpd256", IX86_BUILTIN_VTESTZPD256, EQ, (int) INT_FTYPE_V4DF_V4DF_PTEST },
21952 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestcpd256", IX86_BUILTIN_VTESTCPD256, LTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
21953 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd256, "__builtin_ia32_vtestnzcpd256", IX86_BUILTIN_VTESTNZCPD256, GTU, (int) INT_FTYPE_V4DF_V4DF_PTEST },
21954 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestzps256", IX86_BUILTIN_VTESTZPS256, EQ, (int) INT_FTYPE_V8SF_V8SF_PTEST },
21955 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestcps256", IX86_BUILTIN_VTESTCPS256, LTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
21956 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestps256, "__builtin_ia32_vtestnzcps256", IX86_BUILTIN_VTESTNZCPS256, GTU, (int) INT_FTYPE_V8SF_V8SF_PTEST },
21957 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestz256", IX86_BUILTIN_PTESTZ256, EQ, (int) INT_FTYPE_V4DI_V4DI_PTEST },
21958 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestc256", IX86_BUILTIN_PTESTC256, LTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
21959 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ptest256, "__builtin_ia32_ptestnzc256", IX86_BUILTIN_PTESTNZC256, GTU, (int) INT_FTYPE_V4DI_V4DI_PTEST },
21961 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskpd256, "__builtin_ia32_movmskpd256", IX86_BUILTIN_MOVMSKPD256, UNKNOWN, (int) INT_FTYPE_V4DF },
21962 { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskps256, "__builtin_ia32_movmskps256", IX86_BUILTIN_MOVMSKPS256, UNKNOWN, (int) INT_FTYPE_V8SF },
21966 enum multi_arg_type {
21976 MULTI_ARG_3_PERMPS,
21977 MULTI_ARG_3_PERMPD,
21984 MULTI_ARG_2_DI_IMM,
21985 MULTI_ARG_2_SI_IMM,
21986 MULTI_ARG_2_HI_IMM,
21987 MULTI_ARG_2_QI_IMM,
21988 MULTI_ARG_2_SF_CMP,
21989 MULTI_ARG_2_DF_CMP,
21990 MULTI_ARG_2_DI_CMP,
21991 MULTI_ARG_2_SI_CMP,
21992 MULTI_ARG_2_HI_CMP,
21993 MULTI_ARG_2_QI_CMP,
22016 static const struct builtin_description bdesc_multi_arg[] =
22018 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_vmfmaddv4sf4, "__builtin_ia32_fmaddss", IX86_BUILTIN_FMADDSS, UNKNOWN, (int)MULTI_ARG_3_SF },
22019 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_vmfmaddv2df4, "__builtin_ia32_fmaddsd", IX86_BUILTIN_FMADDSD, UNKNOWN, (int)MULTI_ARG_3_DF },
22020 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_fmaddv4sf4, "__builtin_ia32_fmaddps", IX86_BUILTIN_FMADDPS, UNKNOWN, (int)MULTI_ARG_3_SF },
22021 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_fmaddv2df4, "__builtin_ia32_fmaddpd", IX86_BUILTIN_FMADDPD, UNKNOWN, (int)MULTI_ARG_3_DF },
22022 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_vmfmsubv4sf4, "__builtin_ia32_fmsubss", IX86_BUILTIN_FMSUBSS, UNKNOWN, (int)MULTI_ARG_3_SF },
22023 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_vmfmsubv2df4, "__builtin_ia32_fmsubsd", IX86_BUILTIN_FMSUBSD, UNKNOWN, (int)MULTI_ARG_3_DF },
22024 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_fmsubv4sf4, "__builtin_ia32_fmsubps", IX86_BUILTIN_FMSUBPS, UNKNOWN, (int)MULTI_ARG_3_SF },
22025 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_fmsubv2df4, "__builtin_ia32_fmsubpd", IX86_BUILTIN_FMSUBPD, UNKNOWN, (int)MULTI_ARG_3_DF },
22026 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_vmfnmaddv4sf4, "__builtin_ia32_fnmaddss", IX86_BUILTIN_FNMADDSS, UNKNOWN, (int)MULTI_ARG_3_SF },
22027 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_vmfnmaddv2df4, "__builtin_ia32_fnmaddsd", IX86_BUILTIN_FNMADDSD, UNKNOWN, (int)MULTI_ARG_3_DF },
22028 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_fnmaddv4sf4, "__builtin_ia32_fnmaddps", IX86_BUILTIN_FNMADDPS, UNKNOWN, (int)MULTI_ARG_3_SF },
22029 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_fnmaddv2df4, "__builtin_ia32_fnmaddpd", IX86_BUILTIN_FNMADDPD, UNKNOWN, (int)MULTI_ARG_3_DF },
22030 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_vmfnmsubv4sf4, "__builtin_ia32_fnmsubss", IX86_BUILTIN_FNMSUBSS, UNKNOWN, (int)MULTI_ARG_3_SF },
22031 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_vmfnmsubv2df4, "__builtin_ia32_fnmsubsd", IX86_BUILTIN_FNMSUBSD, UNKNOWN, (int)MULTI_ARG_3_DF },
22032 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_fnmsubv4sf4, "__builtin_ia32_fnmsubps", IX86_BUILTIN_FNMSUBPS, UNKNOWN, (int)MULTI_ARG_3_SF },
22033 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5i_fnmsubv2df4, "__builtin_ia32_fnmsubpd", IX86_BUILTIN_FNMSUBPD, UNKNOWN, (int)MULTI_ARG_3_DF },
22034 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcmov_v2di, "__builtin_ia32_pcmov", IX86_BUILTIN_PCMOV, UNKNOWN, (int)MULTI_ARG_3_DI },
22035 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcmov_v2di, "__builtin_ia32_pcmov_v2di", IX86_BUILTIN_PCMOV_V2DI, UNKNOWN, (int)MULTI_ARG_3_DI },
22036 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcmov_v4si, "__builtin_ia32_pcmov_v4si", IX86_BUILTIN_PCMOV_V4SI, UNKNOWN, (int)MULTI_ARG_3_SI },
22037 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcmov_v8hi, "__builtin_ia32_pcmov_v8hi", IX86_BUILTIN_PCMOV_V8HI, UNKNOWN, (int)MULTI_ARG_3_HI },
22038 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcmov_v16qi, "__builtin_ia32_pcmov_v16qi",IX86_BUILTIN_PCMOV_V16QI,UNKNOWN, (int)MULTI_ARG_3_QI },
22039 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcmov_v2df, "__builtin_ia32_pcmov_v2df", IX86_BUILTIN_PCMOV_V2DF, UNKNOWN, (int)MULTI_ARG_3_DF },
22040 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcmov_v4sf, "__builtin_ia32_pcmov_v4sf", IX86_BUILTIN_PCMOV_V4SF, UNKNOWN, (int)MULTI_ARG_3_SF },
22041 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pperm, "__builtin_ia32_pperm", IX86_BUILTIN_PPERM, UNKNOWN, (int)MULTI_ARG_3_QI },
22042 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_permv4sf, "__builtin_ia32_permps", IX86_BUILTIN_PERMPS, UNKNOWN, (int)MULTI_ARG_3_PERMPS },
22043 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_permv2df, "__builtin_ia32_permpd", IX86_BUILTIN_PERMPD, UNKNOWN, (int)MULTI_ARG_3_PERMPD },
22044 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacssww, "__builtin_ia32_pmacssww", IX86_BUILTIN_PMACSSWW, UNKNOWN, (int)MULTI_ARG_3_HI },
22045 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacsww, "__builtin_ia32_pmacsww", IX86_BUILTIN_PMACSWW, UNKNOWN, (int)MULTI_ARG_3_HI },
22046 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacsswd, "__builtin_ia32_pmacsswd", IX86_BUILTIN_PMACSSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
22047 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacswd, "__builtin_ia32_pmacswd", IX86_BUILTIN_PMACSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
22048 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacssdd, "__builtin_ia32_pmacssdd", IX86_BUILTIN_PMACSSDD, UNKNOWN, (int)MULTI_ARG_3_SI },
22049 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacsdd, "__builtin_ia32_pmacsdd", IX86_BUILTIN_PMACSDD, UNKNOWN, (int)MULTI_ARG_3_SI },
22050 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacssdql, "__builtin_ia32_pmacssdql", IX86_BUILTIN_PMACSSDQL, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
22051 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacssdqh, "__builtin_ia32_pmacssdqh", IX86_BUILTIN_PMACSSDQH, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
22052 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacsdql, "__builtin_ia32_pmacsdql", IX86_BUILTIN_PMACSDQL, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
22053 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmacsdqh, "__builtin_ia32_pmacsdqh", IX86_BUILTIN_PMACSDQH, UNKNOWN, (int)MULTI_ARG_3_SI_DI },
22054 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmadcsswd, "__builtin_ia32_pmadcsswd", IX86_BUILTIN_PMADCSSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
22055 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pmadcswd, "__builtin_ia32_pmadcswd", IX86_BUILTIN_PMADCSWD, UNKNOWN, (int)MULTI_ARG_3_HI_SI },
22056 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vrotlv2di3, "__builtin_ia32_protq", IX86_BUILTIN_PROTQ, UNKNOWN, (int)MULTI_ARG_2_DI },
22057 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vrotlv4si3, "__builtin_ia32_protd", IX86_BUILTIN_PROTD, UNKNOWN, (int)MULTI_ARG_2_SI },
22058 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vrotlv8hi3, "__builtin_ia32_protw", IX86_BUILTIN_PROTW, UNKNOWN, (int)MULTI_ARG_2_HI },
22059 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vrotlv16qi3, "__builtin_ia32_protb", IX86_BUILTIN_PROTB, UNKNOWN, (int)MULTI_ARG_2_QI },
22060 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_rotlv2di3, "__builtin_ia32_protqi", IX86_BUILTIN_PROTQ_IMM, UNKNOWN, (int)MULTI_ARG_2_DI_IMM },
22061 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_rotlv4si3, "__builtin_ia32_protdi", IX86_BUILTIN_PROTD_IMM, UNKNOWN, (int)MULTI_ARG_2_SI_IMM },
22062 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_rotlv8hi3, "__builtin_ia32_protwi", IX86_BUILTIN_PROTW_IMM, UNKNOWN, (int)MULTI_ARG_2_HI_IMM },
22063 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_rotlv16qi3, "__builtin_ia32_protbi", IX86_BUILTIN_PROTB_IMM, UNKNOWN, (int)MULTI_ARG_2_QI_IMM },
22064 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_ashlv2di3, "__builtin_ia32_pshaq", IX86_BUILTIN_PSHAQ, UNKNOWN, (int)MULTI_ARG_2_DI },
22065 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_ashlv4si3, "__builtin_ia32_pshad", IX86_BUILTIN_PSHAD, UNKNOWN, (int)MULTI_ARG_2_SI },
22066 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_ashlv8hi3, "__builtin_ia32_pshaw", IX86_BUILTIN_PSHAW, UNKNOWN, (int)MULTI_ARG_2_HI },
22067 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_ashlv16qi3, "__builtin_ia32_pshab", IX86_BUILTIN_PSHAB, UNKNOWN, (int)MULTI_ARG_2_QI },
22068 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_lshlv2di3, "__builtin_ia32_pshlq", IX86_BUILTIN_PSHLQ, UNKNOWN, (int)MULTI_ARG_2_DI },
22069 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_lshlv4si3, "__builtin_ia32_pshld", IX86_BUILTIN_PSHLD, UNKNOWN, (int)MULTI_ARG_2_SI },
22070 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_lshlv8hi3, "__builtin_ia32_pshlw", IX86_BUILTIN_PSHLW, UNKNOWN, (int)MULTI_ARG_2_HI },
22071 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_lshlv16qi3, "__builtin_ia32_pshlb", IX86_BUILTIN_PSHLB, UNKNOWN, (int)MULTI_ARG_2_QI },
22072 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmfrczv4sf2, "__builtin_ia32_frczss", IX86_BUILTIN_FRCZSS, UNKNOWN, (int)MULTI_ARG_2_SF },
22073 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmfrczv2df2, "__builtin_ia32_frczsd", IX86_BUILTIN_FRCZSD, UNKNOWN, (int)MULTI_ARG_2_DF },
22074 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_frczv4sf2, "__builtin_ia32_frczps", IX86_BUILTIN_FRCZPS, UNKNOWN, (int)MULTI_ARG_1_SF },
22075 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_frczv2df2, "__builtin_ia32_frczpd", IX86_BUILTIN_FRCZPD, UNKNOWN, (int)MULTI_ARG_1_DF },
22076 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_cvtph2ps, "__builtin_ia32_cvtph2ps", IX86_BUILTIN_CVTPH2PS, UNKNOWN, (int)MULTI_ARG_1_PH2PS },
22077 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_cvtps2ph, "__builtin_ia32_cvtps2ph", IX86_BUILTIN_CVTPS2PH, UNKNOWN, (int)MULTI_ARG_1_PS2PH },
22078 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phaddbw, "__builtin_ia32_phaddbw", IX86_BUILTIN_PHADDBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
22079 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phaddbd, "__builtin_ia32_phaddbd", IX86_BUILTIN_PHADDBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
22080 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phaddbq, "__builtin_ia32_phaddbq", IX86_BUILTIN_PHADDBQ, UNKNOWN, (int)MULTI_ARG_1_QI_DI },
22081 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phaddwd, "__builtin_ia32_phaddwd", IX86_BUILTIN_PHADDWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
22082 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phaddwq, "__builtin_ia32_phaddwq", IX86_BUILTIN_PHADDWQ, UNKNOWN, (int)MULTI_ARG_1_HI_DI },
22083 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phadddq, "__builtin_ia32_phadddq", IX86_BUILTIN_PHADDDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
22084 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phaddubw, "__builtin_ia32_phaddubw", IX86_BUILTIN_PHADDUBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
22085 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phaddubd, "__builtin_ia32_phaddubd", IX86_BUILTIN_PHADDUBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI },
22086 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phaddubq, "__builtin_ia32_phaddubq", IX86_BUILTIN_PHADDUBQ, UNKNOWN, (int)MULTI_ARG_1_QI_DI },
22087 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phadduwd, "__builtin_ia32_phadduwd", IX86_BUILTIN_PHADDUWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
22088 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phadduwq, "__builtin_ia32_phadduwq", IX86_BUILTIN_PHADDUWQ, UNKNOWN, (int)MULTI_ARG_1_HI_DI },
22089 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phaddudq, "__builtin_ia32_phaddudq", IX86_BUILTIN_PHADDUDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
22090 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phsubbw, "__builtin_ia32_phsubbw", IX86_BUILTIN_PHSUBBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI },
22091 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phsubwd, "__builtin_ia32_phsubwd", IX86_BUILTIN_PHSUBWD, UNKNOWN, (int)MULTI_ARG_1_HI_SI },
22092 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_phsubdq, "__builtin_ia32_phsubdq", IX86_BUILTIN_PHSUBDQ, UNKNOWN, (int)MULTI_ARG_1_SI_DI },
22094 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comeqss", IX86_BUILTIN_COMEQSS, EQ, (int)MULTI_ARG_2_SF_CMP },
22095 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comness", IX86_BUILTIN_COMNESS, NE, (int)MULTI_ARG_2_SF_CMP },
22096 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comneqss", IX86_BUILTIN_COMNESS, NE, (int)MULTI_ARG_2_SF_CMP },
22097 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comltss", IX86_BUILTIN_COMLTSS, LT, (int)MULTI_ARG_2_SF_CMP },
22098 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comless", IX86_BUILTIN_COMLESS, LE, (int)MULTI_ARG_2_SF_CMP },
22099 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comgtss", IX86_BUILTIN_COMGTSS, GT, (int)MULTI_ARG_2_SF_CMP },
22100 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comgess", IX86_BUILTIN_COMGESS, GE, (int)MULTI_ARG_2_SF_CMP },
22101 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comueqss", IX86_BUILTIN_COMUEQSS, UNEQ, (int)MULTI_ARG_2_SF_CMP },
22102 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comuness", IX86_BUILTIN_COMUNESS, LTGT, (int)MULTI_ARG_2_SF_CMP },
22103 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comuneqss", IX86_BUILTIN_COMUNESS, LTGT, (int)MULTI_ARG_2_SF_CMP },
22104 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comunltss", IX86_BUILTIN_COMULTSS, UNLT, (int)MULTI_ARG_2_SF_CMP },
22105 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comunless", IX86_BUILTIN_COMULESS, UNLE, (int)MULTI_ARG_2_SF_CMP },
22106 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comungtss", IX86_BUILTIN_COMUGTSS, UNGT, (int)MULTI_ARG_2_SF_CMP },
22107 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comungess", IX86_BUILTIN_COMUGESS, UNGE, (int)MULTI_ARG_2_SF_CMP },
22108 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comordss", IX86_BUILTIN_COMORDSS, ORDERED, (int)MULTI_ARG_2_SF_CMP },
22109 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv4sf3, "__builtin_ia32_comunordss", IX86_BUILTIN_COMUNORDSS, UNORDERED, (int)MULTI_ARG_2_SF_CMP },
22111 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comeqsd", IX86_BUILTIN_COMEQSD, EQ, (int)MULTI_ARG_2_DF_CMP },
22112 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comnesd", IX86_BUILTIN_COMNESD, NE, (int)MULTI_ARG_2_DF_CMP },
22113 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comneqsd", IX86_BUILTIN_COMNESD, NE, (int)MULTI_ARG_2_DF_CMP },
22114 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comltsd", IX86_BUILTIN_COMLTSD, LT, (int)MULTI_ARG_2_DF_CMP },
22115 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comlesd", IX86_BUILTIN_COMLESD, LE, (int)MULTI_ARG_2_DF_CMP },
22116 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comgtsd", IX86_BUILTIN_COMGTSD, GT, (int)MULTI_ARG_2_DF_CMP },
22117 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comgesd", IX86_BUILTIN_COMGESD, GE, (int)MULTI_ARG_2_DF_CMP },
22118 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comueqsd", IX86_BUILTIN_COMUEQSD, UNEQ, (int)MULTI_ARG_2_DF_CMP },
22119 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comunesd", IX86_BUILTIN_COMUNESD, LTGT, (int)MULTI_ARG_2_DF_CMP },
22120 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comuneqsd", IX86_BUILTIN_COMUNESD, LTGT, (int)MULTI_ARG_2_DF_CMP },
22121 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comunltsd", IX86_BUILTIN_COMULTSD, UNLT, (int)MULTI_ARG_2_DF_CMP },
22122 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comunlesd", IX86_BUILTIN_COMULESD, UNLE, (int)MULTI_ARG_2_DF_CMP },
22123 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comungtsd", IX86_BUILTIN_COMUGTSD, UNGT, (int)MULTI_ARG_2_DF_CMP },
22124 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comungesd", IX86_BUILTIN_COMUGESD, UNGE, (int)MULTI_ARG_2_DF_CMP },
22125 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comordsd", IX86_BUILTIN_COMORDSD, ORDERED, (int)MULTI_ARG_2_DF_CMP },
22126 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_vmmaskcmpv2df3, "__builtin_ia32_comunordsd", IX86_BUILTIN_COMUNORDSD, UNORDERED, (int)MULTI_ARG_2_DF_CMP },
22128 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comeqps", IX86_BUILTIN_COMEQPS, EQ, (int)MULTI_ARG_2_SF_CMP },
22129 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comneps", IX86_BUILTIN_COMNEPS, NE, (int)MULTI_ARG_2_SF_CMP },
22130 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comneqps", IX86_BUILTIN_COMNEPS, NE, (int)MULTI_ARG_2_SF_CMP },
22131 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comltps", IX86_BUILTIN_COMLTPS, LT, (int)MULTI_ARG_2_SF_CMP },
22132 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comleps", IX86_BUILTIN_COMLEPS, LE, (int)MULTI_ARG_2_SF_CMP },
22133 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comgtps", IX86_BUILTIN_COMGTPS, GT, (int)MULTI_ARG_2_SF_CMP },
22134 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comgeps", IX86_BUILTIN_COMGEPS, GE, (int)MULTI_ARG_2_SF_CMP },
22135 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comueqps", IX86_BUILTIN_COMUEQPS, UNEQ, (int)MULTI_ARG_2_SF_CMP },
22136 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comuneps", IX86_BUILTIN_COMUNEPS, LTGT, (int)MULTI_ARG_2_SF_CMP },
22137 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comuneqps", IX86_BUILTIN_COMUNEPS, LTGT, (int)MULTI_ARG_2_SF_CMP },
22138 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comunltps", IX86_BUILTIN_COMULTPS, UNLT, (int)MULTI_ARG_2_SF_CMP },
22139 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comunleps", IX86_BUILTIN_COMULEPS, UNLE, (int)MULTI_ARG_2_SF_CMP },
22140 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comungtps", IX86_BUILTIN_COMUGTPS, UNGT, (int)MULTI_ARG_2_SF_CMP },
22141 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comungeps", IX86_BUILTIN_COMUGEPS, UNGE, (int)MULTI_ARG_2_SF_CMP },
22142 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comordps", IX86_BUILTIN_COMORDPS, ORDERED, (int)MULTI_ARG_2_SF_CMP },
22143 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4sf3, "__builtin_ia32_comunordps", IX86_BUILTIN_COMUNORDPS, UNORDERED, (int)MULTI_ARG_2_SF_CMP },
22145 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comeqpd", IX86_BUILTIN_COMEQPD, EQ, (int)MULTI_ARG_2_DF_CMP },
22146 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comnepd", IX86_BUILTIN_COMNEPD, NE, (int)MULTI_ARG_2_DF_CMP },
22147 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comneqpd", IX86_BUILTIN_COMNEPD, NE, (int)MULTI_ARG_2_DF_CMP },
22148 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comltpd", IX86_BUILTIN_COMLTPD, LT, (int)MULTI_ARG_2_DF_CMP },
22149 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comlepd", IX86_BUILTIN_COMLEPD, LE, (int)MULTI_ARG_2_DF_CMP },
22150 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comgtpd", IX86_BUILTIN_COMGTPD, GT, (int)MULTI_ARG_2_DF_CMP },
22151 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comgepd", IX86_BUILTIN_COMGEPD, GE, (int)MULTI_ARG_2_DF_CMP },
22152 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comueqpd", IX86_BUILTIN_COMUEQPD, UNEQ, (int)MULTI_ARG_2_DF_CMP },
22153 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comunepd", IX86_BUILTIN_COMUNEPD, LTGT, (int)MULTI_ARG_2_DF_CMP },
22154 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comuneqpd", IX86_BUILTIN_COMUNEPD, LTGT, (int)MULTI_ARG_2_DF_CMP },
22155 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comunltpd", IX86_BUILTIN_COMULTPD, UNLT, (int)MULTI_ARG_2_DF_CMP },
22156 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comunlepd", IX86_BUILTIN_COMULEPD, UNLE, (int)MULTI_ARG_2_DF_CMP },
22157 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comungtpd", IX86_BUILTIN_COMUGTPD, UNGT, (int)MULTI_ARG_2_DF_CMP },
22158 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comungepd", IX86_BUILTIN_COMUGEPD, UNGE, (int)MULTI_ARG_2_DF_CMP },
22159 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comordpd", IX86_BUILTIN_COMORDPD, ORDERED, (int)MULTI_ARG_2_DF_CMP },
22160 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2df3, "__builtin_ia32_comunordpd", IX86_BUILTIN_COMUNORDPD, UNORDERED, (int)MULTI_ARG_2_DF_CMP },
22162 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv16qi3, "__builtin_ia32_pcomeqb", IX86_BUILTIN_PCOMEQB, EQ, (int)MULTI_ARG_2_QI_CMP },
22163 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv16qi3, "__builtin_ia32_pcomneb", IX86_BUILTIN_PCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
22164 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv16qi3, "__builtin_ia32_pcomneqb", IX86_BUILTIN_PCOMNEB, NE, (int)MULTI_ARG_2_QI_CMP },
22165 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv16qi3, "__builtin_ia32_pcomltb", IX86_BUILTIN_PCOMLTB, LT, (int)MULTI_ARG_2_QI_CMP },
22166 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv16qi3, "__builtin_ia32_pcomleb", IX86_BUILTIN_PCOMLEB, LE, (int)MULTI_ARG_2_QI_CMP },
22167 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv16qi3, "__builtin_ia32_pcomgtb", IX86_BUILTIN_PCOMGTB, GT, (int)MULTI_ARG_2_QI_CMP },
22168 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv16qi3, "__builtin_ia32_pcomgeb", IX86_BUILTIN_PCOMGEB, GE, (int)MULTI_ARG_2_QI_CMP },
22170 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv8hi3, "__builtin_ia32_pcomeqw", IX86_BUILTIN_PCOMEQW, EQ, (int)MULTI_ARG_2_HI_CMP },
22171 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv8hi3, "__builtin_ia32_pcomnew", IX86_BUILTIN_PCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
22172 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv8hi3, "__builtin_ia32_pcomneqw", IX86_BUILTIN_PCOMNEW, NE, (int)MULTI_ARG_2_HI_CMP },
22173 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv8hi3, "__builtin_ia32_pcomltw", IX86_BUILTIN_PCOMLTW, LT, (int)MULTI_ARG_2_HI_CMP },
22174 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv8hi3, "__builtin_ia32_pcomlew", IX86_BUILTIN_PCOMLEW, LE, (int)MULTI_ARG_2_HI_CMP },
22175 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv8hi3, "__builtin_ia32_pcomgtw", IX86_BUILTIN_PCOMGTW, GT, (int)MULTI_ARG_2_HI_CMP },
22176 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv8hi3, "__builtin_ia32_pcomgew", IX86_BUILTIN_PCOMGEW, GE, (int)MULTI_ARG_2_HI_CMP },
22178 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4si3, "__builtin_ia32_pcomeqd", IX86_BUILTIN_PCOMEQD, EQ, (int)MULTI_ARG_2_SI_CMP },
22179 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4si3, "__builtin_ia32_pcomned", IX86_BUILTIN_PCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
22180 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4si3, "__builtin_ia32_pcomneqd", IX86_BUILTIN_PCOMNED, NE, (int)MULTI_ARG_2_SI_CMP },
22181 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4si3, "__builtin_ia32_pcomltd", IX86_BUILTIN_PCOMLTD, LT, (int)MULTI_ARG_2_SI_CMP },
22182 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4si3, "__builtin_ia32_pcomled", IX86_BUILTIN_PCOMLED, LE, (int)MULTI_ARG_2_SI_CMP },
22183 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4si3, "__builtin_ia32_pcomgtd", IX86_BUILTIN_PCOMGTD, GT, (int)MULTI_ARG_2_SI_CMP },
22184 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv4si3, "__builtin_ia32_pcomged", IX86_BUILTIN_PCOMGED, GE, (int)MULTI_ARG_2_SI_CMP },
22186 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2di3, "__builtin_ia32_pcomeqq", IX86_BUILTIN_PCOMEQQ, EQ, (int)MULTI_ARG_2_DI_CMP },
22187 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2di3, "__builtin_ia32_pcomneq", IX86_BUILTIN_PCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
22188 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2di3, "__builtin_ia32_pcomneqq", IX86_BUILTIN_PCOMNEQ, NE, (int)MULTI_ARG_2_DI_CMP },
22189 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2di3, "__builtin_ia32_pcomltq", IX86_BUILTIN_PCOMLTQ, LT, (int)MULTI_ARG_2_DI_CMP },
22190 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2di3, "__builtin_ia32_pcomleq", IX86_BUILTIN_PCOMLEQ, LE, (int)MULTI_ARG_2_DI_CMP },
22191 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2di3, "__builtin_ia32_pcomgtq", IX86_BUILTIN_PCOMGTQ, GT, (int)MULTI_ARG_2_DI_CMP },
22192 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmpv2di3, "__builtin_ia32_pcomgeq", IX86_BUILTIN_PCOMGEQ, GE, (int)MULTI_ARG_2_DI_CMP },
22194 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v16qi3,"__builtin_ia32_pcomequb", IX86_BUILTIN_PCOMEQUB, EQ, (int)MULTI_ARG_2_QI_CMP },
22195 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v16qi3,"__builtin_ia32_pcomneub", IX86_BUILTIN_PCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
22196 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v16qi3,"__builtin_ia32_pcomnequb", IX86_BUILTIN_PCOMNEUB, NE, (int)MULTI_ARG_2_QI_CMP },
22197 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv16qi3, "__builtin_ia32_pcomltub", IX86_BUILTIN_PCOMLTUB, LTU, (int)MULTI_ARG_2_QI_CMP },
22198 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv16qi3, "__builtin_ia32_pcomleub", IX86_BUILTIN_PCOMLEUB, LEU, (int)MULTI_ARG_2_QI_CMP },
22199 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv16qi3, "__builtin_ia32_pcomgtub", IX86_BUILTIN_PCOMGTUB, GTU, (int)MULTI_ARG_2_QI_CMP },
22200 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv16qi3, "__builtin_ia32_pcomgeub", IX86_BUILTIN_PCOMGEUB, GEU, (int)MULTI_ARG_2_QI_CMP },
22202 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v8hi3, "__builtin_ia32_pcomequw", IX86_BUILTIN_PCOMEQUW, EQ, (int)MULTI_ARG_2_HI_CMP },
22203 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v8hi3, "__builtin_ia32_pcomneuw", IX86_BUILTIN_PCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
22204 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v8hi3, "__builtin_ia32_pcomnequw", IX86_BUILTIN_PCOMNEUW, NE, (int)MULTI_ARG_2_HI_CMP },
22205 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv8hi3, "__builtin_ia32_pcomltuw", IX86_BUILTIN_PCOMLTUW, LTU, (int)MULTI_ARG_2_HI_CMP },
22206 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv8hi3, "__builtin_ia32_pcomleuw", IX86_BUILTIN_PCOMLEUW, LEU, (int)MULTI_ARG_2_HI_CMP },
22207 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv8hi3, "__builtin_ia32_pcomgtuw", IX86_BUILTIN_PCOMGTUW, GTU, (int)MULTI_ARG_2_HI_CMP },
22208 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv8hi3, "__builtin_ia32_pcomgeuw", IX86_BUILTIN_PCOMGEUW, GEU, (int)MULTI_ARG_2_HI_CMP },
22210 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v4si3, "__builtin_ia32_pcomequd", IX86_BUILTIN_PCOMEQUD, EQ, (int)MULTI_ARG_2_SI_CMP },
22211 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v4si3, "__builtin_ia32_pcomneud", IX86_BUILTIN_PCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
22212 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v4si3, "__builtin_ia32_pcomnequd", IX86_BUILTIN_PCOMNEUD, NE, (int)MULTI_ARG_2_SI_CMP },
22213 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv4si3, "__builtin_ia32_pcomltud", IX86_BUILTIN_PCOMLTUD, LTU, (int)MULTI_ARG_2_SI_CMP },
22214 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv4si3, "__builtin_ia32_pcomleud", IX86_BUILTIN_PCOMLEUD, LEU, (int)MULTI_ARG_2_SI_CMP },
22215 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv4si3, "__builtin_ia32_pcomgtud", IX86_BUILTIN_PCOMGTUD, GTU, (int)MULTI_ARG_2_SI_CMP },
22216 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv4si3, "__builtin_ia32_pcomgeud", IX86_BUILTIN_PCOMGEUD, GEU, (int)MULTI_ARG_2_SI_CMP },
22218 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v2di3, "__builtin_ia32_pcomequq", IX86_BUILTIN_PCOMEQUQ, EQ, (int)MULTI_ARG_2_DI_CMP },
22219 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v2di3, "__builtin_ia32_pcomneuq", IX86_BUILTIN_PCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
22220 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_uns2v2di3, "__builtin_ia32_pcomnequq", IX86_BUILTIN_PCOMNEUQ, NE, (int)MULTI_ARG_2_DI_CMP },
22221 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv2di3, "__builtin_ia32_pcomltuq", IX86_BUILTIN_PCOMLTUQ, LTU, (int)MULTI_ARG_2_DI_CMP },
22222 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv2di3, "__builtin_ia32_pcomleuq", IX86_BUILTIN_PCOMLEUQ, LEU, (int)MULTI_ARG_2_DI_CMP },
22223 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv2di3, "__builtin_ia32_pcomgtuq", IX86_BUILTIN_PCOMGTUQ, GTU, (int)MULTI_ARG_2_DI_CMP },
22224 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_maskcmp_unsv2di3, "__builtin_ia32_pcomgeuq", IX86_BUILTIN_PCOMGEUQ, GEU, (int)MULTI_ARG_2_DI_CMP },
22226 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_com_tfv4sf3, "__builtin_ia32_comfalsess", IX86_BUILTIN_COMFALSESS, (enum rtx_code) COM_FALSE_S, (int)MULTI_ARG_2_SF_TF },
22227 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_com_tfv4sf3, "__builtin_ia32_comtruess", IX86_BUILTIN_COMTRUESS, (enum rtx_code) COM_TRUE_S, (int)MULTI_ARG_2_SF_TF },
22228 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_com_tfv4sf3, "__builtin_ia32_comfalseps", IX86_BUILTIN_COMFALSEPS, (enum rtx_code) COM_FALSE_P, (int)MULTI_ARG_2_SF_TF },
22229 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_com_tfv4sf3, "__builtin_ia32_comtrueps", IX86_BUILTIN_COMTRUEPS, (enum rtx_code) COM_TRUE_P, (int)MULTI_ARG_2_SF_TF },
22230 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_com_tfv2df3, "__builtin_ia32_comfalsesd", IX86_BUILTIN_COMFALSESD, (enum rtx_code) COM_FALSE_S, (int)MULTI_ARG_2_DF_TF },
22231 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_com_tfv2df3, "__builtin_ia32_comtruesd", IX86_BUILTIN_COMTRUESD, (enum rtx_code) COM_TRUE_S, (int)MULTI_ARG_2_DF_TF },
22232 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_com_tfv2df3, "__builtin_ia32_comfalsepd", IX86_BUILTIN_COMFALSEPD, (enum rtx_code) COM_FALSE_P, (int)MULTI_ARG_2_DF_TF },
22233 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_com_tfv2df3, "__builtin_ia32_comtruepd", IX86_BUILTIN_COMTRUEPD, (enum rtx_code) COM_TRUE_P, (int)MULTI_ARG_2_DF_TF },
22235 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv16qi3, "__builtin_ia32_pcomfalseb", IX86_BUILTIN_PCOMFALSEB, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
22236 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv8hi3, "__builtin_ia32_pcomfalsew", IX86_BUILTIN_PCOMFALSEW, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
22237 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv4si3, "__builtin_ia32_pcomfalsed", IX86_BUILTIN_PCOMFALSED, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
22238 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv2di3, "__builtin_ia32_pcomfalseq", IX86_BUILTIN_PCOMFALSEQ, (enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
22239 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv16qi3, "__builtin_ia32_pcomfalseub",IX86_BUILTIN_PCOMFALSEUB,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_QI_TF },
22240 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv8hi3, "__builtin_ia32_pcomfalseuw",IX86_BUILTIN_PCOMFALSEUW,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_HI_TF },
22241 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv4si3, "__builtin_ia32_pcomfalseud",IX86_BUILTIN_PCOMFALSEUD,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_SI_TF },
22242 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv2di3, "__builtin_ia32_pcomfalseuq",IX86_BUILTIN_PCOMFALSEUQ,(enum rtx_code) PCOM_FALSE, (int)MULTI_ARG_2_DI_TF },
22244 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv16qi3, "__builtin_ia32_pcomtrueb", IX86_BUILTIN_PCOMTRUEB, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
22245 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv8hi3, "__builtin_ia32_pcomtruew", IX86_BUILTIN_PCOMTRUEW, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
22246 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv4si3, "__builtin_ia32_pcomtrued", IX86_BUILTIN_PCOMTRUED, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
22247 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv2di3, "__builtin_ia32_pcomtrueq", IX86_BUILTIN_PCOMTRUEQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
22248 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv16qi3, "__builtin_ia32_pcomtrueub", IX86_BUILTIN_PCOMTRUEUB, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_QI_TF },
22249 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv8hi3, "__builtin_ia32_pcomtrueuw", IX86_BUILTIN_PCOMTRUEUW, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_HI_TF },
22250 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv4si3, "__builtin_ia32_pcomtrueud", IX86_BUILTIN_PCOMTRUEUD, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF },
22251 { OPTION_MASK_ISA_SSE5, CODE_FOR_sse5_pcom_tfv2di3, "__builtin_ia32_pcomtrueuq", IX86_BUILTIN_PCOMTRUEUQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF },
22254 /* Set up all the MMX/SSE builtins, even builtins for instructions that are not
22255 in the current target ISA to allow the user to compile particular modules
22256 with different target specific options that differ from the command line
22259 ix86_init_mmx_sse_builtins (void)
22261 const struct builtin_description * d;
22264 tree V16QI_type_node = build_vector_type_for_mode (char_type_node, V16QImode);
22265 tree V2SI_type_node = build_vector_type_for_mode (intSI_type_node, V2SImode);
22266 tree V1DI_type_node
22267 = build_vector_type_for_mode (long_long_integer_type_node, V1DImode);
22268 tree V2SF_type_node = build_vector_type_for_mode (float_type_node, V2SFmode);
22269 tree V2DI_type_node
22270 = build_vector_type_for_mode (long_long_integer_type_node, V2DImode);
22271 tree V2DF_type_node = build_vector_type_for_mode (double_type_node, V2DFmode);
22272 tree V4SF_type_node = build_vector_type_for_mode (float_type_node, V4SFmode);
22273 tree V4SI_type_node = build_vector_type_for_mode (intSI_type_node, V4SImode);
22274 tree V4HI_type_node = build_vector_type_for_mode (intHI_type_node, V4HImode);
22275 tree V8QI_type_node = build_vector_type_for_mode (char_type_node, V8QImode);
22276 tree V8HI_type_node = build_vector_type_for_mode (intHI_type_node, V8HImode);
22278 tree pchar_type_node = build_pointer_type (char_type_node);
22279 tree pcchar_type_node
22280 = build_pointer_type (build_type_variant (char_type_node, 1, 0));
22281 tree pfloat_type_node = build_pointer_type (float_type_node);
22282 tree pcfloat_type_node
22283 = build_pointer_type (build_type_variant (float_type_node, 1, 0));
22284 tree pv2sf_type_node = build_pointer_type (V2SF_type_node);
22285 tree pcv2sf_type_node
22286 = build_pointer_type (build_type_variant (V2SF_type_node, 1, 0));
22287 tree pv2di_type_node = build_pointer_type (V2DI_type_node);
22288 tree pdi_type_node = build_pointer_type (long_long_unsigned_type_node);
22291 tree int_ftype_v4sf_v4sf
22292 = build_function_type_list (integer_type_node,
22293 V4SF_type_node, V4SF_type_node, NULL_TREE);
22294 tree v4si_ftype_v4sf_v4sf
22295 = build_function_type_list (V4SI_type_node,
22296 V4SF_type_node, V4SF_type_node, NULL_TREE);
22297 /* MMX/SSE/integer conversions. */
22298 tree int_ftype_v4sf
22299 = build_function_type_list (integer_type_node,
22300 V4SF_type_node, NULL_TREE);
22301 tree int64_ftype_v4sf
22302 = build_function_type_list (long_long_integer_type_node,
22303 V4SF_type_node, NULL_TREE);
22304 tree int_ftype_v8qi
22305 = build_function_type_list (integer_type_node, V8QI_type_node, NULL_TREE);
22306 tree v4sf_ftype_v4sf_int
22307 = build_function_type_list (V4SF_type_node,
22308 V4SF_type_node, integer_type_node, NULL_TREE);
22309 tree v4sf_ftype_v4sf_int64
22310 = build_function_type_list (V4SF_type_node,
22311 V4SF_type_node, long_long_integer_type_node,
22313 tree v4sf_ftype_v4sf_v2si
22314 = build_function_type_list (V4SF_type_node,
22315 V4SF_type_node, V2SI_type_node, NULL_TREE);
22317 /* Miscellaneous. */
22318 tree v8qi_ftype_v4hi_v4hi
22319 = build_function_type_list (V8QI_type_node,
22320 V4HI_type_node, V4HI_type_node, NULL_TREE);
22321 tree v4hi_ftype_v2si_v2si
22322 = build_function_type_list (V4HI_type_node,
22323 V2SI_type_node, V2SI_type_node, NULL_TREE);
22324 tree v4sf_ftype_v4sf_v4sf_int
22325 = build_function_type_list (V4SF_type_node,
22326 V4SF_type_node, V4SF_type_node,
22327 integer_type_node, NULL_TREE);
22328 tree v2si_ftype_v4hi_v4hi
22329 = build_function_type_list (V2SI_type_node,
22330 V4HI_type_node, V4HI_type_node, NULL_TREE);
22331 tree v4hi_ftype_v4hi_int
22332 = build_function_type_list (V4HI_type_node,
22333 V4HI_type_node, integer_type_node, NULL_TREE);
22334 tree v2si_ftype_v2si_int
22335 = build_function_type_list (V2SI_type_node,
22336 V2SI_type_node, integer_type_node, NULL_TREE);
22337 tree v1di_ftype_v1di_int
22338 = build_function_type_list (V1DI_type_node,
22339 V1DI_type_node, integer_type_node, NULL_TREE);
22341 tree void_ftype_void
22342 = build_function_type (void_type_node, void_list_node);
22343 tree void_ftype_unsigned
22344 = build_function_type_list (void_type_node, unsigned_type_node, NULL_TREE);
22345 tree void_ftype_unsigned_unsigned
22346 = build_function_type_list (void_type_node, unsigned_type_node,
22347 unsigned_type_node, NULL_TREE);
22348 tree void_ftype_pcvoid_unsigned_unsigned
22349 = build_function_type_list (void_type_node, const_ptr_type_node,
22350 unsigned_type_node, unsigned_type_node,
22352 tree unsigned_ftype_void
22353 = build_function_type (unsigned_type_node, void_list_node);
22354 tree v2si_ftype_v4sf
22355 = build_function_type_list (V2SI_type_node, V4SF_type_node, NULL_TREE);
22356 /* Loads/stores. */
22357 tree void_ftype_v8qi_v8qi_pchar
22358 = build_function_type_list (void_type_node,
22359 V8QI_type_node, V8QI_type_node,
22360 pchar_type_node, NULL_TREE);
22361 tree v4sf_ftype_pcfloat
22362 = build_function_type_list (V4SF_type_node, pcfloat_type_node, NULL_TREE);
22363 tree v4sf_ftype_v4sf_pcv2sf
22364 = build_function_type_list (V4SF_type_node,
22365 V4SF_type_node, pcv2sf_type_node, NULL_TREE);
22366 tree void_ftype_pv2sf_v4sf
22367 = build_function_type_list (void_type_node,
22368 pv2sf_type_node, V4SF_type_node, NULL_TREE);
22369 tree void_ftype_pfloat_v4sf
22370 = build_function_type_list (void_type_node,
22371 pfloat_type_node, V4SF_type_node, NULL_TREE);
22372 tree void_ftype_pdi_di
22373 = build_function_type_list (void_type_node,
22374 pdi_type_node, long_long_unsigned_type_node,
22376 tree void_ftype_pv2di_v2di
22377 = build_function_type_list (void_type_node,
22378 pv2di_type_node, V2DI_type_node, NULL_TREE);
22379 /* Normal vector unops. */
22380 tree v4sf_ftype_v4sf
22381 = build_function_type_list (V4SF_type_node, V4SF_type_node, NULL_TREE);
22382 tree v16qi_ftype_v16qi
22383 = build_function_type_list (V16QI_type_node, V16QI_type_node, NULL_TREE);
22384 tree v8hi_ftype_v8hi
22385 = build_function_type_list (V8HI_type_node, V8HI_type_node, NULL_TREE);
22386 tree v4si_ftype_v4si
22387 = build_function_type_list (V4SI_type_node, V4SI_type_node, NULL_TREE);
22388 tree v8qi_ftype_v8qi
22389 = build_function_type_list (V8QI_type_node, V8QI_type_node, NULL_TREE);
22390 tree v4hi_ftype_v4hi
22391 = build_function_type_list (V4HI_type_node, V4HI_type_node, NULL_TREE);
22393 /* Normal vector binops. */
22394 tree v4sf_ftype_v4sf_v4sf
22395 = build_function_type_list (V4SF_type_node,
22396 V4SF_type_node, V4SF_type_node, NULL_TREE);
22397 tree v8qi_ftype_v8qi_v8qi
22398 = build_function_type_list (V8QI_type_node,
22399 V8QI_type_node, V8QI_type_node, NULL_TREE);
22400 tree v4hi_ftype_v4hi_v4hi
22401 = build_function_type_list (V4HI_type_node,
22402 V4HI_type_node, V4HI_type_node, NULL_TREE);
22403 tree v2si_ftype_v2si_v2si
22404 = build_function_type_list (V2SI_type_node,
22405 V2SI_type_node, V2SI_type_node, NULL_TREE);
22406 tree v1di_ftype_v1di_v1di
22407 = build_function_type_list (V1DI_type_node,
22408 V1DI_type_node, V1DI_type_node, NULL_TREE);
22409 tree v1di_ftype_v1di_v1di_int
22410 = build_function_type_list (V1DI_type_node,
22411 V1DI_type_node, V1DI_type_node,
22412 integer_type_node, NULL_TREE);
22413 tree v2si_ftype_v2sf
22414 = build_function_type_list (V2SI_type_node, V2SF_type_node, NULL_TREE);
22415 tree v2sf_ftype_v2si
22416 = build_function_type_list (V2SF_type_node, V2SI_type_node, NULL_TREE);
22417 tree v2si_ftype_v2si
22418 = build_function_type_list (V2SI_type_node, V2SI_type_node, NULL_TREE);
22419 tree v2sf_ftype_v2sf
22420 = build_function_type_list (V2SF_type_node, V2SF_type_node, NULL_TREE);
22421 tree v2sf_ftype_v2sf_v2sf
22422 = build_function_type_list (V2SF_type_node,
22423 V2SF_type_node, V2SF_type_node, NULL_TREE);
22424 tree v2si_ftype_v2sf_v2sf
22425 = build_function_type_list (V2SI_type_node,
22426 V2SF_type_node, V2SF_type_node, NULL_TREE);
22427 tree pint_type_node = build_pointer_type (integer_type_node);
22428 tree pdouble_type_node = build_pointer_type (double_type_node);
22429 tree pcdouble_type_node = build_pointer_type (
22430 build_type_variant (double_type_node, 1, 0));
22431 tree int_ftype_v2df_v2df
22432 = build_function_type_list (integer_type_node,
22433 V2DF_type_node, V2DF_type_node, NULL_TREE);
22435 tree void_ftype_pcvoid
22436 = build_function_type_list (void_type_node, const_ptr_type_node, NULL_TREE);
22437 tree v4sf_ftype_v4si
22438 = build_function_type_list (V4SF_type_node, V4SI_type_node, NULL_TREE);
22439 tree v4si_ftype_v4sf
22440 = build_function_type_list (V4SI_type_node, V4SF_type_node, NULL_TREE);
22441 tree v2df_ftype_v4si
22442 = build_function_type_list (V2DF_type_node, V4SI_type_node, NULL_TREE);
22443 tree v4si_ftype_v2df
22444 = build_function_type_list (V4SI_type_node, V2DF_type_node, NULL_TREE);
22445 tree v4si_ftype_v2df_v2df
22446 = build_function_type_list (V4SI_type_node,
22447 V2DF_type_node, V2DF_type_node, NULL_TREE);
22448 tree v2si_ftype_v2df
22449 = build_function_type_list (V2SI_type_node, V2DF_type_node, NULL_TREE);
22450 tree v4sf_ftype_v2df
22451 = build_function_type_list (V4SF_type_node, V2DF_type_node, NULL_TREE);
22452 tree v2df_ftype_v2si
22453 = build_function_type_list (V2DF_type_node, V2SI_type_node, NULL_TREE);
22454 tree v2df_ftype_v4sf
22455 = build_function_type_list (V2DF_type_node, V4SF_type_node, NULL_TREE);
22456 tree int_ftype_v2df
22457 = build_function_type_list (integer_type_node, V2DF_type_node, NULL_TREE);
22458 tree int64_ftype_v2df
22459 = build_function_type_list (long_long_integer_type_node,
22460 V2DF_type_node, NULL_TREE);
22461 tree v2df_ftype_v2df_int
22462 = build_function_type_list (V2DF_type_node,
22463 V2DF_type_node, integer_type_node, NULL_TREE);
22464 tree v2df_ftype_v2df_int64
22465 = build_function_type_list (V2DF_type_node,
22466 V2DF_type_node, long_long_integer_type_node,
22468 tree v4sf_ftype_v4sf_v2df
22469 = build_function_type_list (V4SF_type_node,
22470 V4SF_type_node, V2DF_type_node, NULL_TREE);
22471 tree v2df_ftype_v2df_v4sf
22472 = build_function_type_list (V2DF_type_node,
22473 V2DF_type_node, V4SF_type_node, NULL_TREE);
22474 tree v2df_ftype_v2df_v2df_int
22475 = build_function_type_list (V2DF_type_node,
22476 V2DF_type_node, V2DF_type_node,
22479 tree v2df_ftype_v2df_pcdouble
22480 = build_function_type_list (V2DF_type_node,
22481 V2DF_type_node, pcdouble_type_node, NULL_TREE);
22482 tree void_ftype_pdouble_v2df
22483 = build_function_type_list (void_type_node,
22484 pdouble_type_node, V2DF_type_node, NULL_TREE);
22485 tree void_ftype_pint_int
22486 = build_function_type_list (void_type_node,
22487 pint_type_node, integer_type_node, NULL_TREE);
22488 tree void_ftype_v16qi_v16qi_pchar
22489 = build_function_type_list (void_type_node,
22490 V16QI_type_node, V16QI_type_node,
22491 pchar_type_node, NULL_TREE);
22492 tree v2df_ftype_pcdouble
22493 = build_function_type_list (V2DF_type_node, pcdouble_type_node, NULL_TREE);
22494 tree v2df_ftype_v2df_v2df
22495 = build_function_type_list (V2DF_type_node,
22496 V2DF_type_node, V2DF_type_node, NULL_TREE);
22497 tree v16qi_ftype_v16qi_v16qi
22498 = build_function_type_list (V16QI_type_node,
22499 V16QI_type_node, V16QI_type_node, NULL_TREE);
22500 tree v8hi_ftype_v8hi_v8hi
22501 = build_function_type_list (V8HI_type_node,
22502 V8HI_type_node, V8HI_type_node, NULL_TREE);
22503 tree v4si_ftype_v4si_v4si
22504 = build_function_type_list (V4SI_type_node,
22505 V4SI_type_node, V4SI_type_node, NULL_TREE);
22506 tree v2di_ftype_v2di_v2di
22507 = build_function_type_list (V2DI_type_node,
22508 V2DI_type_node, V2DI_type_node, NULL_TREE);
22509 tree v2di_ftype_v2df_v2df
22510 = build_function_type_list (V2DI_type_node,
22511 V2DF_type_node, V2DF_type_node, NULL_TREE);
22512 tree v2df_ftype_v2df
22513 = build_function_type_list (V2DF_type_node, V2DF_type_node, NULL_TREE);
22514 tree v2di_ftype_v2di_int
22515 = build_function_type_list (V2DI_type_node,
22516 V2DI_type_node, integer_type_node, NULL_TREE);
22517 tree v2di_ftype_v2di_v2di_int
22518 = build_function_type_list (V2DI_type_node, V2DI_type_node,
22519 V2DI_type_node, integer_type_node, NULL_TREE);
22520 tree v4si_ftype_v4si_int
22521 = build_function_type_list (V4SI_type_node,
22522 V4SI_type_node, integer_type_node, NULL_TREE);
22523 tree v8hi_ftype_v8hi_int
22524 = build_function_type_list (V8HI_type_node,
22525 V8HI_type_node, integer_type_node, NULL_TREE);
22526 tree v4si_ftype_v8hi_v8hi
22527 = build_function_type_list (V4SI_type_node,
22528 V8HI_type_node, V8HI_type_node, NULL_TREE);
22529 tree v1di_ftype_v8qi_v8qi
22530 = build_function_type_list (V1DI_type_node,
22531 V8QI_type_node, V8QI_type_node, NULL_TREE);
22532 tree v1di_ftype_v2si_v2si
22533 = build_function_type_list (V1DI_type_node,
22534 V2SI_type_node, V2SI_type_node, NULL_TREE);
22535 tree v2di_ftype_v16qi_v16qi
22536 = build_function_type_list (V2DI_type_node,
22537 V16QI_type_node, V16QI_type_node, NULL_TREE);
22538 tree v2di_ftype_v4si_v4si
22539 = build_function_type_list (V2DI_type_node,
22540 V4SI_type_node, V4SI_type_node, NULL_TREE);
22541 tree int_ftype_v16qi
22542 = build_function_type_list (integer_type_node, V16QI_type_node, NULL_TREE);
22543 tree v16qi_ftype_pcchar
22544 = build_function_type_list (V16QI_type_node, pcchar_type_node, NULL_TREE);
22545 tree void_ftype_pchar_v16qi
22546 = build_function_type_list (void_type_node,
22547 pchar_type_node, V16QI_type_node, NULL_TREE);
22549 tree v2di_ftype_v2di_unsigned_unsigned
22550 = build_function_type_list (V2DI_type_node, V2DI_type_node,
22551 unsigned_type_node, unsigned_type_node,
22553 tree v2di_ftype_v2di_v2di_unsigned_unsigned
22554 = build_function_type_list (V2DI_type_node, V2DI_type_node, V2DI_type_node,
22555 unsigned_type_node, unsigned_type_node,
22557 tree v2di_ftype_v2di_v16qi
22558 = build_function_type_list (V2DI_type_node, V2DI_type_node, V16QI_type_node,
22560 tree v2df_ftype_v2df_v2df_v2df
22561 = build_function_type_list (V2DF_type_node,
22562 V2DF_type_node, V2DF_type_node,
22563 V2DF_type_node, NULL_TREE);
22564 tree v4sf_ftype_v4sf_v4sf_v4sf
22565 = build_function_type_list (V4SF_type_node,
22566 V4SF_type_node, V4SF_type_node,
22567 V4SF_type_node, NULL_TREE);
22568 tree v8hi_ftype_v16qi
22569 = build_function_type_list (V8HI_type_node, V16QI_type_node,
22571 tree v4si_ftype_v16qi
22572 = build_function_type_list (V4SI_type_node, V16QI_type_node,
22574 tree v2di_ftype_v16qi
22575 = build_function_type_list (V2DI_type_node, V16QI_type_node,
22577 tree v4si_ftype_v8hi
22578 = build_function_type_list (V4SI_type_node, V8HI_type_node,
22580 tree v2di_ftype_v8hi
22581 = build_function_type_list (V2DI_type_node, V8HI_type_node,
22583 tree v2di_ftype_v4si
22584 = build_function_type_list (V2DI_type_node, V4SI_type_node,
22586 tree v2di_ftype_pv2di
22587 = build_function_type_list (V2DI_type_node, pv2di_type_node,
22589 tree v16qi_ftype_v16qi_v16qi_int
22590 = build_function_type_list (V16QI_type_node, V16QI_type_node,
22591 V16QI_type_node, integer_type_node,
22593 tree v16qi_ftype_v16qi_v16qi_v16qi
22594 = build_function_type_list (V16QI_type_node, V16QI_type_node,
22595 V16QI_type_node, V16QI_type_node,
22597 tree v8hi_ftype_v8hi_v8hi_int
22598 = build_function_type_list (V8HI_type_node, V8HI_type_node,
22599 V8HI_type_node, integer_type_node,
22601 tree v4si_ftype_v4si_v4si_int
22602 = build_function_type_list (V4SI_type_node, V4SI_type_node,
22603 V4SI_type_node, integer_type_node,
22605 tree int_ftype_v2di_v2di
22606 = build_function_type_list (integer_type_node,
22607 V2DI_type_node, V2DI_type_node,
22609 tree int_ftype_v16qi_int_v16qi_int_int
22610 = build_function_type_list (integer_type_node,
22617 tree v16qi_ftype_v16qi_int_v16qi_int_int
22618 = build_function_type_list (V16QI_type_node,
22625 tree int_ftype_v16qi_v16qi_int
22626 = build_function_type_list (integer_type_node,
22632 /* SSE5 instructions */
22633 tree v2di_ftype_v2di_v2di_v2di
22634 = build_function_type_list (V2DI_type_node,
22640 tree v4si_ftype_v4si_v4si_v4si
22641 = build_function_type_list (V4SI_type_node,
22647 tree v4si_ftype_v4si_v4si_v2di
22648 = build_function_type_list (V4SI_type_node,
22654 tree v8hi_ftype_v8hi_v8hi_v8hi
22655 = build_function_type_list (V8HI_type_node,
22661 tree v8hi_ftype_v8hi_v8hi_v4si
22662 = build_function_type_list (V8HI_type_node,
22668 tree v2df_ftype_v2df_v2df_v16qi
22669 = build_function_type_list (V2DF_type_node,
22675 tree v4sf_ftype_v4sf_v4sf_v16qi
22676 = build_function_type_list (V4SF_type_node,
22682 tree v2di_ftype_v2di_si
22683 = build_function_type_list (V2DI_type_node,
22688 tree v4si_ftype_v4si_si
22689 = build_function_type_list (V4SI_type_node,
22694 tree v8hi_ftype_v8hi_si
22695 = build_function_type_list (V8HI_type_node,
22700 tree v16qi_ftype_v16qi_si
22701 = build_function_type_list (V16QI_type_node,
22705 tree v4sf_ftype_v4hi
22706 = build_function_type_list (V4SF_type_node,
22710 tree v4hi_ftype_v4sf
22711 = build_function_type_list (V4HI_type_node,
22715 tree v2di_ftype_v2di
22716 = build_function_type_list (V2DI_type_node, V2DI_type_node, NULL_TREE);
22718 tree v16qi_ftype_v8hi_v8hi
22719 = build_function_type_list (V16QI_type_node,
22720 V8HI_type_node, V8HI_type_node,
22722 tree v8hi_ftype_v4si_v4si
22723 = build_function_type_list (V8HI_type_node,
22724 V4SI_type_node, V4SI_type_node,
22726 tree v8hi_ftype_v16qi_v16qi
22727 = build_function_type_list (V8HI_type_node,
22728 V16QI_type_node, V16QI_type_node,
22730 tree v4hi_ftype_v8qi_v8qi
22731 = build_function_type_list (V4HI_type_node,
22732 V8QI_type_node, V8QI_type_node,
22734 tree unsigned_ftype_unsigned_uchar
22735 = build_function_type_list (unsigned_type_node,
22736 unsigned_type_node,
22737 unsigned_char_type_node,
22739 tree unsigned_ftype_unsigned_ushort
22740 = build_function_type_list (unsigned_type_node,
22741 unsigned_type_node,
22742 short_unsigned_type_node,
22744 tree unsigned_ftype_unsigned_unsigned
22745 = build_function_type_list (unsigned_type_node,
22746 unsigned_type_node,
22747 unsigned_type_node,
22749 tree uint64_ftype_uint64_uint64
22750 = build_function_type_list (long_long_unsigned_type_node,
22751 long_long_unsigned_type_node,
22752 long_long_unsigned_type_node,
22754 tree float_ftype_float
22755 = build_function_type_list (float_type_node,
22760 tree V32QI_type_node = build_vector_type_for_mode (char_type_node,
22762 tree V8SI_type_node = build_vector_type_for_mode (intSI_type_node,
22764 tree V8SF_type_node = build_vector_type_for_mode (float_type_node,
22766 tree V4DI_type_node = build_vector_type_for_mode (long_long_integer_type_node,
22768 tree V4DF_type_node = build_vector_type_for_mode (double_type_node,
22770 tree v8sf_ftype_v8sf
22771 = build_function_type_list (V8SF_type_node,
22774 tree v8si_ftype_v8sf
22775 = build_function_type_list (V8SI_type_node,
22778 tree v8sf_ftype_v8si
22779 = build_function_type_list (V8SF_type_node,
22782 tree v4si_ftype_v4df
22783 = build_function_type_list (V4SI_type_node,
22786 tree v4df_ftype_v4df
22787 = build_function_type_list (V4DF_type_node,
22790 tree v4df_ftype_v4si
22791 = build_function_type_list (V4DF_type_node,
22794 tree v4df_ftype_v4sf
22795 = build_function_type_list (V4DF_type_node,
22798 tree v4sf_ftype_v4df
22799 = build_function_type_list (V4SF_type_node,
22802 tree v8sf_ftype_v8sf_v8sf
22803 = build_function_type_list (V8SF_type_node,
22804 V8SF_type_node, V8SF_type_node,
22806 tree v4df_ftype_v4df_v4df
22807 = build_function_type_list (V4DF_type_node,
22808 V4DF_type_node, V4DF_type_node,
22810 tree v8sf_ftype_v8sf_int
22811 = build_function_type_list (V8SF_type_node,
22812 V8SF_type_node, integer_type_node,
22814 tree v4si_ftype_v8si_int
22815 = build_function_type_list (V4SI_type_node,
22816 V8SI_type_node, integer_type_node,
22818 tree v4df_ftype_v4df_int
22819 = build_function_type_list (V4DF_type_node,
22820 V4DF_type_node, integer_type_node,
22822 tree v4sf_ftype_v8sf_int
22823 = build_function_type_list (V4SF_type_node,
22824 V8SF_type_node, integer_type_node,
22826 tree v2df_ftype_v4df_int
22827 = build_function_type_list (V2DF_type_node,
22828 V4DF_type_node, integer_type_node,
22830 tree v8sf_ftype_v8sf_v8sf_int
22831 = build_function_type_list (V8SF_type_node,
22832 V8SF_type_node, V8SF_type_node,
22835 tree v8sf_ftype_v8sf_v8sf_v8sf
22836 = build_function_type_list (V8SF_type_node,
22837 V8SF_type_node, V8SF_type_node,
22840 tree v4df_ftype_v4df_v4df_v4df
22841 = build_function_type_list (V4DF_type_node,
22842 V4DF_type_node, V4DF_type_node,
22845 tree v8si_ftype_v8si_v8si_int
22846 = build_function_type_list (V8SI_type_node,
22847 V8SI_type_node, V8SI_type_node,
22850 tree v4df_ftype_v4df_v4df_int
22851 = build_function_type_list (V4DF_type_node,
22852 V4DF_type_node, V4DF_type_node,
22855 tree v8sf_ftype_pcfloat
22856 = build_function_type_list (V8SF_type_node,
22859 tree v4df_ftype_pcdouble
22860 = build_function_type_list (V4DF_type_node,
22861 pcdouble_type_node,
22863 tree pcv4sf_type_node
22864 = build_pointer_type (build_type_variant (V4SF_type_node, 1, 0));
22865 tree pcv2df_type_node
22866 = build_pointer_type (build_type_variant (V2DF_type_node, 1, 0));
22867 tree v8sf_ftype_pcv4sf
22868 = build_function_type_list (V8SF_type_node,
22871 tree v4df_ftype_pcv2df
22872 = build_function_type_list (V4DF_type_node,
22875 tree v32qi_ftype_pcchar
22876 = build_function_type_list (V32QI_type_node,
22879 tree void_ftype_pchar_v32qi
22880 = build_function_type_list (void_type_node,
22881 pchar_type_node, V32QI_type_node,
22883 tree v8si_ftype_v8si_v4si_int
22884 = build_function_type_list (V8SI_type_node,
22885 V8SI_type_node, V4SI_type_node,
22888 tree pv4di_type_node = build_pointer_type (V4DI_type_node);
22889 tree void_ftype_pv4di_v4di
22890 = build_function_type_list (void_type_node,
22891 pv4di_type_node, V4DI_type_node,
22893 tree v8sf_ftype_v8sf_v4sf_int
22894 = build_function_type_list (V8SF_type_node,
22895 V8SF_type_node, V4SF_type_node,
22898 tree v4df_ftype_v4df_v2df_int
22899 = build_function_type_list (V4DF_type_node,
22900 V4DF_type_node, V2DF_type_node,
22903 tree void_ftype_pfloat_v8sf
22904 = build_function_type_list (void_type_node,
22905 pfloat_type_node, V8SF_type_node,
22907 tree void_ftype_pdouble_v4df
22908 = build_function_type_list (void_type_node,
22909 pdouble_type_node, V4DF_type_node,
22911 tree pv8sf_type_node = build_pointer_type (V8SF_type_node);
22912 tree pv4sf_type_node = build_pointer_type (V4SF_type_node);
22913 tree pv4df_type_node = build_pointer_type (V4DF_type_node);
22914 tree pv2df_type_node = build_pointer_type (V2DF_type_node);
22915 tree pcv8sf_type_node
22916 = build_pointer_type (build_type_variant (V8SF_type_node, 1, 0));
22917 tree pcv4df_type_node
22918 = build_pointer_type (build_type_variant (V4DF_type_node, 1, 0));
22919 tree v8sf_ftype_pcv8sf_v8sf
22920 = build_function_type_list (V8SF_type_node,
22921 pcv8sf_type_node, V8SF_type_node,
22923 tree v4df_ftype_pcv4df_v4df
22924 = build_function_type_list (V4DF_type_node,
22925 pcv4df_type_node, V4DF_type_node,
22927 tree v4sf_ftype_pcv4sf_v4sf
22928 = build_function_type_list (V4SF_type_node,
22929 pcv4sf_type_node, V4SF_type_node,
22931 tree v2df_ftype_pcv2df_v2df
22932 = build_function_type_list (V2DF_type_node,
22933 pcv2df_type_node, V2DF_type_node,
22935 tree void_ftype_pv8sf_v8sf_v8sf
22936 = build_function_type_list (void_type_node,
22937 pv8sf_type_node, V8SF_type_node,
22940 tree void_ftype_pv4df_v4df_v4df
22941 = build_function_type_list (void_type_node,
22942 pv4df_type_node, V4DF_type_node,
22945 tree void_ftype_pv4sf_v4sf_v4sf
22946 = build_function_type_list (void_type_node,
22947 pv4sf_type_node, V4SF_type_node,
22950 tree void_ftype_pv2df_v2df_v2df
22951 = build_function_type_list (void_type_node,
22952 pv2df_type_node, V2DF_type_node,
22955 tree v4df_ftype_v2df
22956 = build_function_type_list (V4DF_type_node,
22959 tree v8sf_ftype_v4sf
22960 = build_function_type_list (V8SF_type_node,
22963 tree v8si_ftype_v4si
22964 = build_function_type_list (V8SI_type_node,
22967 tree v2df_ftype_v4df
22968 = build_function_type_list (V2DF_type_node,
22971 tree v4sf_ftype_v8sf
22972 = build_function_type_list (V4SF_type_node,
22975 tree v4si_ftype_v8si
22976 = build_function_type_list (V4SI_type_node,
22979 tree int_ftype_v4df
22980 = build_function_type_list (integer_type_node,
22983 tree int_ftype_v8sf
22984 = build_function_type_list (integer_type_node,
22987 tree int_ftype_v8sf_v8sf
22988 = build_function_type_list (integer_type_node,
22989 V8SF_type_node, V8SF_type_node,
22991 tree int_ftype_v4di_v4di
22992 = build_function_type_list (integer_type_node,
22993 V4DI_type_node, V4DI_type_node,
22995 tree int_ftype_v4df_v4df
22996 = build_function_type_list (integer_type_node,
22997 V4DF_type_node, V4DF_type_node,
22999 tree v8sf_ftype_v8sf_v8si
23000 = build_function_type_list (V8SF_type_node,
23001 V8SF_type_node, V8SI_type_node,
23003 tree v4df_ftype_v4df_v4di
23004 = build_function_type_list (V4DF_type_node,
23005 V4DF_type_node, V4DI_type_node,
23007 tree v4sf_ftype_v4sf_v4si
23008 = build_function_type_list (V4SF_type_node,
23009 V4SF_type_node, V4SI_type_node, NULL_TREE);
23010 tree v2df_ftype_v2df_v2di
23011 = build_function_type_list (V2DF_type_node,
23012 V2DF_type_node, V2DI_type_node, NULL_TREE);
23016 /* Add all special builtins with variable number of operands. */
23017 for (i = 0, d = bdesc_special_args;
23018 i < ARRAY_SIZE (bdesc_special_args);
23026 switch ((enum ix86_special_builtin_type) d->flag)
23028 case VOID_FTYPE_VOID:
23029 type = void_ftype_void;
23031 case V32QI_FTYPE_PCCHAR:
23032 type = v32qi_ftype_pcchar;
23034 case V16QI_FTYPE_PCCHAR:
23035 type = v16qi_ftype_pcchar;
23037 case V8SF_FTYPE_PCV4SF:
23038 type = v8sf_ftype_pcv4sf;
23040 case V8SF_FTYPE_PCFLOAT:
23041 type = v8sf_ftype_pcfloat;
23043 case V4DF_FTYPE_PCV2DF:
23044 type = v4df_ftype_pcv2df;
23046 case V4DF_FTYPE_PCDOUBLE:
23047 type = v4df_ftype_pcdouble;
23049 case V4SF_FTYPE_PCFLOAT:
23050 type = v4sf_ftype_pcfloat;
23052 case V2DI_FTYPE_PV2DI:
23053 type = v2di_ftype_pv2di;
23055 case V2DF_FTYPE_PCDOUBLE:
23056 type = v2df_ftype_pcdouble;
23058 case V8SF_FTYPE_PCV8SF_V8SF:
23059 type = v8sf_ftype_pcv8sf_v8sf;
23061 case V4DF_FTYPE_PCV4DF_V4DF:
23062 type = v4df_ftype_pcv4df_v4df;
23064 case V4SF_FTYPE_V4SF_PCV2SF:
23065 type = v4sf_ftype_v4sf_pcv2sf;
23067 case V4SF_FTYPE_PCV4SF_V4SF:
23068 type = v4sf_ftype_pcv4sf_v4sf;
23070 case V2DF_FTYPE_V2DF_PCDOUBLE:
23071 type = v2df_ftype_v2df_pcdouble;
23073 case V2DF_FTYPE_PCV2DF_V2DF:
23074 type = v2df_ftype_pcv2df_v2df;
23076 case VOID_FTYPE_PV2SF_V4SF:
23077 type = void_ftype_pv2sf_v4sf;
23079 case VOID_FTYPE_PV4DI_V4DI:
23080 type = void_ftype_pv4di_v4di;
23082 case VOID_FTYPE_PV2DI_V2DI:
23083 type = void_ftype_pv2di_v2di;
23085 case VOID_FTYPE_PCHAR_V32QI:
23086 type = void_ftype_pchar_v32qi;
23088 case VOID_FTYPE_PCHAR_V16QI:
23089 type = void_ftype_pchar_v16qi;
23091 case VOID_FTYPE_PFLOAT_V8SF:
23092 type = void_ftype_pfloat_v8sf;
23094 case VOID_FTYPE_PFLOAT_V4SF:
23095 type = void_ftype_pfloat_v4sf;
23097 case VOID_FTYPE_PDOUBLE_V4DF:
23098 type = void_ftype_pdouble_v4df;
23100 case VOID_FTYPE_PDOUBLE_V2DF:
23101 type = void_ftype_pdouble_v2df;
23103 case VOID_FTYPE_PDI_DI:
23104 type = void_ftype_pdi_di;
23106 case VOID_FTYPE_PINT_INT:
23107 type = void_ftype_pint_int;
23109 case VOID_FTYPE_PV8SF_V8SF_V8SF:
23110 type = void_ftype_pv8sf_v8sf_v8sf;
23112 case VOID_FTYPE_PV4DF_V4DF_V4DF:
23113 type = void_ftype_pv4df_v4df_v4df;
23115 case VOID_FTYPE_PV4SF_V4SF_V4SF:
23116 type = void_ftype_pv4sf_v4sf_v4sf;
23118 case VOID_FTYPE_PV2DF_V2DF_V2DF:
23119 type = void_ftype_pv2df_v2df_v2df;
23122 gcc_unreachable ();
23125 def_builtin (d->mask, d->name, type, d->code);
23128 /* Add all builtins with variable number of operands. */
23129 for (i = 0, d = bdesc_args;
23130 i < ARRAY_SIZE (bdesc_args);
23138 switch ((enum ix86_builtin_type) d->flag)
23140 case FLOAT_FTYPE_FLOAT:
23141 type = float_ftype_float;
23143 case INT_FTYPE_V8SF_V8SF_PTEST:
23144 type = int_ftype_v8sf_v8sf;
23146 case INT_FTYPE_V4DI_V4DI_PTEST:
23147 type = int_ftype_v4di_v4di;
23149 case INT_FTYPE_V4DF_V4DF_PTEST:
23150 type = int_ftype_v4df_v4df;
23152 case INT_FTYPE_V4SF_V4SF_PTEST:
23153 type = int_ftype_v4sf_v4sf;
23155 case INT_FTYPE_V2DI_V2DI_PTEST:
23156 type = int_ftype_v2di_v2di;
23158 case INT_FTYPE_V2DF_V2DF_PTEST:
23159 type = int_ftype_v2df_v2df;
23161 case INT64_FTYPE_V4SF:
23162 type = int64_ftype_v4sf;
23164 case INT64_FTYPE_V2DF:
23165 type = int64_ftype_v2df;
23167 case INT_FTYPE_V16QI:
23168 type = int_ftype_v16qi;
23170 case INT_FTYPE_V8QI:
23171 type = int_ftype_v8qi;
23173 case INT_FTYPE_V8SF:
23174 type = int_ftype_v8sf;
23176 case INT_FTYPE_V4DF:
23177 type = int_ftype_v4df;
23179 case INT_FTYPE_V4SF:
23180 type = int_ftype_v4sf;
23182 case INT_FTYPE_V2DF:
23183 type = int_ftype_v2df;
23185 case V16QI_FTYPE_V16QI:
23186 type = v16qi_ftype_v16qi;
23188 case V8SI_FTYPE_V8SF:
23189 type = v8si_ftype_v8sf;
23191 case V8SI_FTYPE_V4SI:
23192 type = v8si_ftype_v4si;
23194 case V8HI_FTYPE_V8HI:
23195 type = v8hi_ftype_v8hi;
23197 case V8HI_FTYPE_V16QI:
23198 type = v8hi_ftype_v16qi;
23200 case V8QI_FTYPE_V8QI:
23201 type = v8qi_ftype_v8qi;
23203 case V8SF_FTYPE_V8SF:
23204 type = v8sf_ftype_v8sf;
23206 case V8SF_FTYPE_V8SI:
23207 type = v8sf_ftype_v8si;
23209 case V8SF_FTYPE_V4SF:
23210 type = v8sf_ftype_v4sf;
23212 case V4SI_FTYPE_V4DF:
23213 type = v4si_ftype_v4df;
23215 case V4SI_FTYPE_V4SI:
23216 type = v4si_ftype_v4si;
23218 case V4SI_FTYPE_V16QI:
23219 type = v4si_ftype_v16qi;
23221 case V4SI_FTYPE_V8SI:
23222 type = v4si_ftype_v8si;
23224 case V4SI_FTYPE_V8HI:
23225 type = v4si_ftype_v8hi;
23227 case V4SI_FTYPE_V4SF:
23228 type = v4si_ftype_v4sf;
23230 case V4SI_FTYPE_V2DF:
23231 type = v4si_ftype_v2df;
23233 case V4HI_FTYPE_V4HI:
23234 type = v4hi_ftype_v4hi;
23236 case V4DF_FTYPE_V4DF:
23237 type = v4df_ftype_v4df;
23239 case V4DF_FTYPE_V4SI:
23240 type = v4df_ftype_v4si;
23242 case V4DF_FTYPE_V4SF:
23243 type = v4df_ftype_v4sf;
23245 case V4DF_FTYPE_V2DF:
23246 type = v4df_ftype_v2df;
23248 case V4SF_FTYPE_V4SF:
23249 case V4SF_FTYPE_V4SF_VEC_MERGE:
23250 type = v4sf_ftype_v4sf;
23252 case V4SF_FTYPE_V8SF:
23253 type = v4sf_ftype_v8sf;
23255 case V4SF_FTYPE_V4SI:
23256 type = v4sf_ftype_v4si;
23258 case V4SF_FTYPE_V4DF:
23259 type = v4sf_ftype_v4df;
23261 case V4SF_FTYPE_V2DF:
23262 type = v4sf_ftype_v2df;
23264 case V2DI_FTYPE_V2DI:
23265 type = v2di_ftype_v2di;
23267 case V2DI_FTYPE_V16QI:
23268 type = v2di_ftype_v16qi;
23270 case V2DI_FTYPE_V8HI:
23271 type = v2di_ftype_v8hi;
23273 case V2DI_FTYPE_V4SI:
23274 type = v2di_ftype_v4si;
23276 case V2SI_FTYPE_V2SI:
23277 type = v2si_ftype_v2si;
23279 case V2SI_FTYPE_V4SF:
23280 type = v2si_ftype_v4sf;
23282 case V2SI_FTYPE_V2DF:
23283 type = v2si_ftype_v2df;
23285 case V2SI_FTYPE_V2SF:
23286 type = v2si_ftype_v2sf;
23288 case V2DF_FTYPE_V4DF:
23289 type = v2df_ftype_v4df;
23291 case V2DF_FTYPE_V4SF:
23292 type = v2df_ftype_v4sf;
23294 case V2DF_FTYPE_V2DF:
23295 case V2DF_FTYPE_V2DF_VEC_MERGE:
23296 type = v2df_ftype_v2df;
23298 case V2DF_FTYPE_V2SI:
23299 type = v2df_ftype_v2si;
23301 case V2DF_FTYPE_V4SI:
23302 type = v2df_ftype_v4si;
23304 case V2SF_FTYPE_V2SF:
23305 type = v2sf_ftype_v2sf;
23307 case V2SF_FTYPE_V2SI:
23308 type = v2sf_ftype_v2si;
23310 case V16QI_FTYPE_V16QI_V16QI:
23311 type = v16qi_ftype_v16qi_v16qi;
23313 case V16QI_FTYPE_V8HI_V8HI:
23314 type = v16qi_ftype_v8hi_v8hi;
23316 case V8QI_FTYPE_V8QI_V8QI:
23317 type = v8qi_ftype_v8qi_v8qi;
23319 case V8QI_FTYPE_V4HI_V4HI:
23320 type = v8qi_ftype_v4hi_v4hi;
23322 case V8HI_FTYPE_V8HI_V8HI:
23323 case V8HI_FTYPE_V8HI_V8HI_COUNT:
23324 type = v8hi_ftype_v8hi_v8hi;
23326 case V8HI_FTYPE_V16QI_V16QI:
23327 type = v8hi_ftype_v16qi_v16qi;
23329 case V8HI_FTYPE_V4SI_V4SI:
23330 type = v8hi_ftype_v4si_v4si;
23332 case V8HI_FTYPE_V8HI_SI_COUNT:
23333 type = v8hi_ftype_v8hi_int;
23335 case V8SF_FTYPE_V8SF_V8SF:
23336 type = v8sf_ftype_v8sf_v8sf;
23338 case V8SF_FTYPE_V8SF_V8SI:
23339 type = v8sf_ftype_v8sf_v8si;
23341 case V4SI_FTYPE_V4SI_V4SI:
23342 case V4SI_FTYPE_V4SI_V4SI_COUNT:
23343 type = v4si_ftype_v4si_v4si;
23345 case V4SI_FTYPE_V8HI_V8HI:
23346 type = v4si_ftype_v8hi_v8hi;
23348 case V4SI_FTYPE_V4SF_V4SF:
23349 type = v4si_ftype_v4sf_v4sf;
23351 case V4SI_FTYPE_V2DF_V2DF:
23352 type = v4si_ftype_v2df_v2df;
23354 case V4SI_FTYPE_V4SI_SI_COUNT:
23355 type = v4si_ftype_v4si_int;
23357 case V4HI_FTYPE_V4HI_V4HI:
23358 case V4HI_FTYPE_V4HI_V4HI_COUNT:
23359 type = v4hi_ftype_v4hi_v4hi;
23361 case V4HI_FTYPE_V8QI_V8QI:
23362 type = v4hi_ftype_v8qi_v8qi;
23364 case V4HI_FTYPE_V2SI_V2SI:
23365 type = v4hi_ftype_v2si_v2si;
23367 case V4HI_FTYPE_V4HI_SI_COUNT:
23368 type = v4hi_ftype_v4hi_int;
23370 case V4DF_FTYPE_V4DF_V4DF:
23371 type = v4df_ftype_v4df_v4df;
23373 case V4DF_FTYPE_V4DF_V4DI:
23374 type = v4df_ftype_v4df_v4di;
23376 case V4SF_FTYPE_V4SF_V4SF:
23377 case V4SF_FTYPE_V4SF_V4SF_SWAP:
23378 type = v4sf_ftype_v4sf_v4sf;
23380 case V4SF_FTYPE_V4SF_V4SI:
23381 type = v4sf_ftype_v4sf_v4si;
23383 case V4SF_FTYPE_V4SF_V2SI:
23384 type = v4sf_ftype_v4sf_v2si;
23386 case V4SF_FTYPE_V4SF_V2DF:
23387 type = v4sf_ftype_v4sf_v2df;
23389 case V4SF_FTYPE_V4SF_DI:
23390 type = v4sf_ftype_v4sf_int64;
23392 case V4SF_FTYPE_V4SF_SI:
23393 type = v4sf_ftype_v4sf_int;
23395 case V2DI_FTYPE_V2DI_V2DI:
23396 case V2DI_FTYPE_V2DI_V2DI_COUNT:
23397 type = v2di_ftype_v2di_v2di;
23399 case V2DI_FTYPE_V16QI_V16QI:
23400 type = v2di_ftype_v16qi_v16qi;
23402 case V2DI_FTYPE_V4SI_V4SI:
23403 type = v2di_ftype_v4si_v4si;
23405 case V2DI_FTYPE_V2DI_V16QI:
23406 type = v2di_ftype_v2di_v16qi;
23408 case V2DI_FTYPE_V2DF_V2DF:
23409 type = v2di_ftype_v2df_v2df;
23411 case V2DI_FTYPE_V2DI_SI_COUNT:
23412 type = v2di_ftype_v2di_int;
23414 case V2SI_FTYPE_V2SI_V2SI:
23415 case V2SI_FTYPE_V2SI_V2SI_COUNT:
23416 type = v2si_ftype_v2si_v2si;
23418 case V2SI_FTYPE_V4HI_V4HI:
23419 type = v2si_ftype_v4hi_v4hi;
23421 case V2SI_FTYPE_V2SF_V2SF:
23422 type = v2si_ftype_v2sf_v2sf;
23424 case V2SI_FTYPE_V2SI_SI_COUNT:
23425 type = v2si_ftype_v2si_int;
23427 case V2DF_FTYPE_V2DF_V2DF:
23428 case V2DF_FTYPE_V2DF_V2DF_SWAP:
23429 type = v2df_ftype_v2df_v2df;
23431 case V2DF_FTYPE_V2DF_V4SF:
23432 type = v2df_ftype_v2df_v4sf;
23434 case V2DF_FTYPE_V2DF_V2DI:
23435 type = v2df_ftype_v2df_v2di;
23437 case V2DF_FTYPE_V2DF_DI:
23438 type = v2df_ftype_v2df_int64;
23440 case V2DF_FTYPE_V2DF_SI:
23441 type = v2df_ftype_v2df_int;
23443 case V2SF_FTYPE_V2SF_V2SF:
23444 type = v2sf_ftype_v2sf_v2sf;
23446 case V1DI_FTYPE_V1DI_V1DI:
23447 case V1DI_FTYPE_V1DI_V1DI_COUNT:
23448 type = v1di_ftype_v1di_v1di;
23450 case V1DI_FTYPE_V8QI_V8QI:
23451 type = v1di_ftype_v8qi_v8qi;
23453 case V1DI_FTYPE_V2SI_V2SI:
23454 type = v1di_ftype_v2si_v2si;
23456 case V1DI_FTYPE_V1DI_SI_COUNT:
23457 type = v1di_ftype_v1di_int;
23459 case UINT64_FTYPE_UINT64_UINT64:
23460 type = uint64_ftype_uint64_uint64;
23462 case UINT_FTYPE_UINT_UINT:
23463 type = unsigned_ftype_unsigned_unsigned;
23465 case UINT_FTYPE_UINT_USHORT:
23466 type = unsigned_ftype_unsigned_ushort;
23468 case UINT_FTYPE_UINT_UCHAR:
23469 type = unsigned_ftype_unsigned_uchar;
23471 case V8HI_FTYPE_V8HI_INT:
23472 type = v8hi_ftype_v8hi_int;
23474 case V8SF_FTYPE_V8SF_INT:
23475 type = v8sf_ftype_v8sf_int;
23477 case V4SI_FTYPE_V4SI_INT:
23478 type = v4si_ftype_v4si_int;
23480 case V4SI_FTYPE_V8SI_INT:
23481 type = v4si_ftype_v8si_int;
23483 case V4HI_FTYPE_V4HI_INT:
23484 type = v4hi_ftype_v4hi_int;
23486 case V4DF_FTYPE_V4DF_INT:
23487 type = v4df_ftype_v4df_int;
23489 case V4SF_FTYPE_V4SF_INT:
23490 type = v4sf_ftype_v4sf_int;
23492 case V4SF_FTYPE_V8SF_INT:
23493 type = v4sf_ftype_v8sf_int;
23495 case V2DI_FTYPE_V2DI_INT:
23496 case V2DI2TI_FTYPE_V2DI_INT:
23497 type = v2di_ftype_v2di_int;
23499 case V2DF_FTYPE_V2DF_INT:
23500 type = v2df_ftype_v2df_int;
23502 case V2DF_FTYPE_V4DF_INT:
23503 type = v2df_ftype_v4df_int;
23505 case V16QI_FTYPE_V16QI_V16QI_V16QI:
23506 type = v16qi_ftype_v16qi_v16qi_v16qi;
23508 case V8SF_FTYPE_V8SF_V8SF_V8SF:
23509 type = v8sf_ftype_v8sf_v8sf_v8sf;
23511 case V4DF_FTYPE_V4DF_V4DF_V4DF:
23512 type = v4df_ftype_v4df_v4df_v4df;
23514 case V4SF_FTYPE_V4SF_V4SF_V4SF:
23515 type = v4sf_ftype_v4sf_v4sf_v4sf;
23517 case V2DF_FTYPE_V2DF_V2DF_V2DF:
23518 type = v2df_ftype_v2df_v2df_v2df;
23520 case V16QI_FTYPE_V16QI_V16QI_INT:
23521 type = v16qi_ftype_v16qi_v16qi_int;
23523 case V8SI_FTYPE_V8SI_V8SI_INT:
23524 type = v8si_ftype_v8si_v8si_int;
23526 case V8SI_FTYPE_V8SI_V4SI_INT:
23527 type = v8si_ftype_v8si_v4si_int;
23529 case V8HI_FTYPE_V8HI_V8HI_INT:
23530 type = v8hi_ftype_v8hi_v8hi_int;
23532 case V8SF_FTYPE_V8SF_V8SF_INT:
23533 type = v8sf_ftype_v8sf_v8sf_int;
23535 case V8SF_FTYPE_V8SF_V4SF_INT:
23536 type = v8sf_ftype_v8sf_v4sf_int;
23538 case V4SI_FTYPE_V4SI_V4SI_INT:
23539 type = v4si_ftype_v4si_v4si_int;
23541 case V4DF_FTYPE_V4DF_V4DF_INT:
23542 type = v4df_ftype_v4df_v4df_int;
23544 case V4DF_FTYPE_V4DF_V2DF_INT:
23545 type = v4df_ftype_v4df_v2df_int;
23547 case V4SF_FTYPE_V4SF_V4SF_INT:
23548 type = v4sf_ftype_v4sf_v4sf_int;
23550 case V2DI_FTYPE_V2DI_V2DI_INT:
23551 case V2DI2TI_FTYPE_V2DI_V2DI_INT:
23552 type = v2di_ftype_v2di_v2di_int;
23554 case V2DF_FTYPE_V2DF_V2DF_INT:
23555 type = v2df_ftype_v2df_v2df_int;
23557 case V2DI_FTYPE_V2DI_UINT_UINT:
23558 type = v2di_ftype_v2di_unsigned_unsigned;
23560 case V2DI_FTYPE_V2DI_V2DI_UINT_UINT:
23561 type = v2di_ftype_v2di_v2di_unsigned_unsigned;
23563 case V1DI2DI_FTYPE_V1DI_V1DI_INT:
23564 type = v1di_ftype_v1di_v1di_int;
23567 gcc_unreachable ();
23570 def_builtin_const (d->mask, d->name, type, d->code);
23573 /* pcmpestr[im] insns. */
23574 for (i = 0, d = bdesc_pcmpestr;
23575 i < ARRAY_SIZE (bdesc_pcmpestr);
23578 if (d->code == IX86_BUILTIN_PCMPESTRM128)
23579 ftype = v16qi_ftype_v16qi_int_v16qi_int_int;
23581 ftype = int_ftype_v16qi_int_v16qi_int_int;
23582 def_builtin_const (d->mask, d->name, ftype, d->code);
23585 /* pcmpistr[im] insns. */
23586 for (i = 0, d = bdesc_pcmpistr;
23587 i < ARRAY_SIZE (bdesc_pcmpistr);
23590 if (d->code == IX86_BUILTIN_PCMPISTRM128)
23591 ftype = v16qi_ftype_v16qi_v16qi_int;
23593 ftype = int_ftype_v16qi_v16qi_int;
23594 def_builtin_const (d->mask, d->name, ftype, d->code);
23597 /* comi/ucomi insns. */
23598 for (i = 0, d = bdesc_comi; i < ARRAY_SIZE (bdesc_comi); i++, d++)
23599 if (d->mask == OPTION_MASK_ISA_SSE2)
23600 def_builtin_const (d->mask, d->name, int_ftype_v2df_v2df, d->code);
23602 def_builtin_const (d->mask, d->name, int_ftype_v4sf_v4sf, d->code);
23605 def_builtin (OPTION_MASK_ISA_SSE, "__builtin_ia32_ldmxcsr", void_ftype_unsigned, IX86_BUILTIN_LDMXCSR);
23606 def_builtin (OPTION_MASK_ISA_SSE, "__builtin_ia32_stmxcsr", unsigned_ftype_void, IX86_BUILTIN_STMXCSR);
23608 /* SSE or 3DNow!A */
23609 def_builtin (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, "__builtin_ia32_maskmovq", void_ftype_v8qi_v8qi_pchar, IX86_BUILTIN_MASKMOVQ);
23612 def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_maskmovdqu", void_ftype_v16qi_v16qi_pchar, IX86_BUILTIN_MASKMOVDQU);
23614 def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_clflush", void_ftype_pcvoid, IX86_BUILTIN_CLFLUSH);
23615 x86_mfence = def_builtin (OPTION_MASK_ISA_SSE2, "__builtin_ia32_mfence", void_ftype_void, IX86_BUILTIN_MFENCE);
23618 def_builtin (OPTION_MASK_ISA_SSE3, "__builtin_ia32_monitor", void_ftype_pcvoid_unsigned_unsigned, IX86_BUILTIN_MONITOR);
23619 def_builtin (OPTION_MASK_ISA_SSE3, "__builtin_ia32_mwait", void_ftype_unsigned_unsigned, IX86_BUILTIN_MWAIT);
23622 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesenc128", v2di_ftype_v2di_v2di, IX86_BUILTIN_AESENC128);
23623 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesenclast128", v2di_ftype_v2di_v2di, IX86_BUILTIN_AESENCLAST128);
23624 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesdec128", v2di_ftype_v2di_v2di, IX86_BUILTIN_AESDEC128);
23625 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesdeclast128", v2di_ftype_v2di_v2di, IX86_BUILTIN_AESDECLAST128);
23626 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesimc128", v2di_ftype_v2di, IX86_BUILTIN_AESIMC128);
23627 def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aeskeygenassist128", v2di_ftype_v2di_int, IX86_BUILTIN_AESKEYGENASSIST128);
23630 def_builtin_const (OPTION_MASK_ISA_PCLMUL, "__builtin_ia32_pclmulqdq128", v2di_ftype_v2di_v2di_int, IX86_BUILTIN_PCLMULQDQ128);
23633 def_builtin (OPTION_MASK_ISA_AVX, "__builtin_ia32_vzeroupper", void_ftype_void,
23634 TARGET_64BIT ? IX86_BUILTIN_VZEROUPPER_REX64 : IX86_BUILTIN_VZEROUPPER);
23636 /* Access to the vec_init patterns. */
23637 ftype = build_function_type_list (V2SI_type_node, integer_type_node,
23638 integer_type_node, NULL_TREE);
23639 def_builtin_const (OPTION_MASK_ISA_MMX, "__builtin_ia32_vec_init_v2si", ftype, IX86_BUILTIN_VEC_INIT_V2SI);
23641 ftype = build_function_type_list (V4HI_type_node, short_integer_type_node,
23642 short_integer_type_node,
23643 short_integer_type_node,
23644 short_integer_type_node, NULL_TREE);
23645 def_builtin_const (OPTION_MASK_ISA_MMX, "__builtin_ia32_vec_init_v4hi", ftype, IX86_BUILTIN_VEC_INIT_V4HI);
23647 ftype = build_function_type_list (V8QI_type_node, char_type_node,
23648 char_type_node, char_type_node,
23649 char_type_node, char_type_node,
23650 char_type_node, char_type_node,
23651 char_type_node, NULL_TREE);
23652 def_builtin_const (OPTION_MASK_ISA_MMX, "__builtin_ia32_vec_init_v8qi", ftype, IX86_BUILTIN_VEC_INIT_V8QI);
23654 /* Access to the vec_extract patterns. */
23655 ftype = build_function_type_list (double_type_node, V2DF_type_node,
23656 integer_type_node, NULL_TREE);
23657 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v2df", ftype, IX86_BUILTIN_VEC_EXT_V2DF);
23659 ftype = build_function_type_list (long_long_integer_type_node,
23660 V2DI_type_node, integer_type_node,
23662 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v2di", ftype, IX86_BUILTIN_VEC_EXT_V2DI);
23664 ftype = build_function_type_list (float_type_node, V4SF_type_node,
23665 integer_type_node, NULL_TREE);
23666 def_builtin_const (OPTION_MASK_ISA_SSE, "__builtin_ia32_vec_ext_v4sf", ftype, IX86_BUILTIN_VEC_EXT_V4SF);
23668 ftype = build_function_type_list (intSI_type_node, V4SI_type_node,
23669 integer_type_node, NULL_TREE);
23670 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v4si", ftype, IX86_BUILTIN_VEC_EXT_V4SI);
23672 ftype = build_function_type_list (intHI_type_node, V8HI_type_node,
23673 integer_type_node, NULL_TREE);
23674 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v8hi", ftype, IX86_BUILTIN_VEC_EXT_V8HI);
23676 ftype = build_function_type_list (intHI_type_node, V4HI_type_node,
23677 integer_type_node, NULL_TREE);
23678 def_builtin_const (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, "__builtin_ia32_vec_ext_v4hi", ftype, IX86_BUILTIN_VEC_EXT_V4HI);
23680 ftype = build_function_type_list (intSI_type_node, V2SI_type_node,
23681 integer_type_node, NULL_TREE);
23682 def_builtin_const (OPTION_MASK_ISA_MMX, "__builtin_ia32_vec_ext_v2si", ftype, IX86_BUILTIN_VEC_EXT_V2SI);
23684 ftype = build_function_type_list (intQI_type_node, V16QI_type_node,
23685 integer_type_node, NULL_TREE);
23686 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_ext_v16qi", ftype, IX86_BUILTIN_VEC_EXT_V16QI);
23688 /* Access to the vec_set patterns. */
23689 ftype = build_function_type_list (V2DI_type_node, V2DI_type_node,
23691 integer_type_node, NULL_TREE);
23692 def_builtin_const (OPTION_MASK_ISA_SSE4_1 | OPTION_MASK_ISA_64BIT, "__builtin_ia32_vec_set_v2di", ftype, IX86_BUILTIN_VEC_SET_V2DI);
23694 ftype = build_function_type_list (V4SF_type_node, V4SF_type_node,
23696 integer_type_node, NULL_TREE);
23697 def_builtin_const (OPTION_MASK_ISA_SSE4_1, "__builtin_ia32_vec_set_v4sf", ftype, IX86_BUILTIN_VEC_SET_V4SF);
23699 ftype = build_function_type_list (V4SI_type_node, V4SI_type_node,
23701 integer_type_node, NULL_TREE);
23702 def_builtin_const (OPTION_MASK_ISA_SSE4_1, "__builtin_ia32_vec_set_v4si", ftype, IX86_BUILTIN_VEC_SET_V4SI);
23704 ftype = build_function_type_list (V8HI_type_node, V8HI_type_node,
23706 integer_type_node, NULL_TREE);
23707 def_builtin_const (OPTION_MASK_ISA_SSE2, "__builtin_ia32_vec_set_v8hi", ftype, IX86_BUILTIN_VEC_SET_V8HI);
23709 ftype = build_function_type_list (V4HI_type_node, V4HI_type_node,
23711 integer_type_node, NULL_TREE);
23712 def_builtin_const (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A, "__builtin_ia32_vec_set_v4hi", ftype, IX86_BUILTIN_VEC_SET_V4HI);
23714 ftype = build_function_type_list (V16QI_type_node, V16QI_type_node,
23716 integer_type_node, NULL_TREE);
23717 def_builtin_const (OPTION_MASK_ISA_SSE4_1, "__builtin_ia32_vec_set_v16qi", ftype, IX86_BUILTIN_VEC_SET_V16QI);
23719 /* Add SSE5 multi-arg argument instructions */
23720 for (i = 0, d = bdesc_multi_arg; i < ARRAY_SIZE (bdesc_multi_arg); i++, d++)
23722 tree mtype = NULL_TREE;
23727 switch ((enum multi_arg_type)d->flag)
23729 case MULTI_ARG_3_SF: mtype = v4sf_ftype_v4sf_v4sf_v4sf; break;
23730 case MULTI_ARG_3_DF: mtype = v2df_ftype_v2df_v2df_v2df; break;
23731 case MULTI_ARG_3_DI: mtype = v2di_ftype_v2di_v2di_v2di; break;
23732 case MULTI_ARG_3_SI: mtype = v4si_ftype_v4si_v4si_v4si; break;
23733 case MULTI_ARG_3_SI_DI: mtype = v4si_ftype_v4si_v4si_v2di; break;
23734 case MULTI_ARG_3_HI: mtype = v8hi_ftype_v8hi_v8hi_v8hi; break;
23735 case MULTI_ARG_3_HI_SI: mtype = v8hi_ftype_v8hi_v8hi_v4si; break;
23736 case MULTI_ARG_3_QI: mtype = v16qi_ftype_v16qi_v16qi_v16qi; break;
23737 case MULTI_ARG_3_PERMPS: mtype = v4sf_ftype_v4sf_v4sf_v16qi; break;
23738 case MULTI_ARG_3_PERMPD: mtype = v2df_ftype_v2df_v2df_v16qi; break;
23739 case MULTI_ARG_2_SF: mtype = v4sf_ftype_v4sf_v4sf; break;
23740 case MULTI_ARG_2_DF: mtype = v2df_ftype_v2df_v2df; break;
23741 case MULTI_ARG_2_DI: mtype = v2di_ftype_v2di_v2di; break;
23742 case MULTI_ARG_2_SI: mtype = v4si_ftype_v4si_v4si; break;
23743 case MULTI_ARG_2_HI: mtype = v8hi_ftype_v8hi_v8hi; break;
23744 case MULTI_ARG_2_QI: mtype = v16qi_ftype_v16qi_v16qi; break;
23745 case MULTI_ARG_2_DI_IMM: mtype = v2di_ftype_v2di_si; break;
23746 case MULTI_ARG_2_SI_IMM: mtype = v4si_ftype_v4si_si; break;
23747 case MULTI_ARG_2_HI_IMM: mtype = v8hi_ftype_v8hi_si; break;
23748 case MULTI_ARG_2_QI_IMM: mtype = v16qi_ftype_v16qi_si; break;
23749 case MULTI_ARG_2_SF_CMP: mtype = v4sf_ftype_v4sf_v4sf; break;
23750 case MULTI_ARG_2_DF_CMP: mtype = v2df_ftype_v2df_v2df; break;
23751 case MULTI_ARG_2_DI_CMP: mtype = v2di_ftype_v2di_v2di; break;
23752 case MULTI_ARG_2_SI_CMP: mtype = v4si_ftype_v4si_v4si; break;
23753 case MULTI_ARG_2_HI_CMP: mtype = v8hi_ftype_v8hi_v8hi; break;
23754 case MULTI_ARG_2_QI_CMP: mtype = v16qi_ftype_v16qi_v16qi; break;
23755 case MULTI_ARG_2_SF_TF: mtype = v4sf_ftype_v4sf_v4sf; break;
23756 case MULTI_ARG_2_DF_TF: mtype = v2df_ftype_v2df_v2df; break;
23757 case MULTI_ARG_2_DI_TF: mtype = v2di_ftype_v2di_v2di; break;
23758 case MULTI_ARG_2_SI_TF: mtype = v4si_ftype_v4si_v4si; break;
23759 case MULTI_ARG_2_HI_TF: mtype = v8hi_ftype_v8hi_v8hi; break;
23760 case MULTI_ARG_2_QI_TF: mtype = v16qi_ftype_v16qi_v16qi; break;
23761 case MULTI_ARG_1_SF: mtype = v4sf_ftype_v4sf; break;
23762 case MULTI_ARG_1_DF: mtype = v2df_ftype_v2df; break;
23763 case MULTI_ARG_1_DI: mtype = v2di_ftype_v2di; break;
23764 case MULTI_ARG_1_SI: mtype = v4si_ftype_v4si; break;
23765 case MULTI_ARG_1_HI: mtype = v8hi_ftype_v8hi; break;
23766 case MULTI_ARG_1_QI: mtype = v16qi_ftype_v16qi; break;
23767 case MULTI_ARG_1_SI_DI: mtype = v2di_ftype_v4si; break;
23768 case MULTI_ARG_1_HI_DI: mtype = v2di_ftype_v8hi; break;
23769 case MULTI_ARG_1_HI_SI: mtype = v4si_ftype_v8hi; break;
23770 case MULTI_ARG_1_QI_DI: mtype = v2di_ftype_v16qi; break;
23771 case MULTI_ARG_1_QI_SI: mtype = v4si_ftype_v16qi; break;
23772 case MULTI_ARG_1_QI_HI: mtype = v8hi_ftype_v16qi; break;
23773 case MULTI_ARG_1_PH2PS: mtype = v4sf_ftype_v4hi; break;
23774 case MULTI_ARG_1_PS2PH: mtype = v4hi_ftype_v4sf; break;
23775 case MULTI_ARG_UNKNOWN:
23777 gcc_unreachable ();
23781 def_builtin_const (d->mask, d->name, mtype, d->code);
23785 /* Internal method for ix86_init_builtins. */
23788 ix86_init_builtins_va_builtins_abi (void)
23790 tree ms_va_ref, sysv_va_ref;
23791 tree fnvoid_va_end_ms, fnvoid_va_end_sysv;
23792 tree fnvoid_va_start_ms, fnvoid_va_start_sysv;
23793 tree fnvoid_va_copy_ms, fnvoid_va_copy_sysv;
23794 tree fnattr_ms = NULL_TREE, fnattr_sysv = NULL_TREE;
23798 fnattr_ms = build_tree_list (get_identifier ("ms_abi"), NULL_TREE);
23799 fnattr_sysv = build_tree_list (get_identifier ("sysv_abi"), NULL_TREE);
23800 ms_va_ref = build_reference_type (ms_va_list_type_node);
23802 build_pointer_type (TREE_TYPE (sysv_va_list_type_node));
23805 build_function_type_list (void_type_node, ms_va_ref, NULL_TREE);
23806 fnvoid_va_start_ms =
23807 build_varargs_function_type_list (void_type_node, ms_va_ref, NULL_TREE);
23808 fnvoid_va_end_sysv =
23809 build_function_type_list (void_type_node, sysv_va_ref, NULL_TREE);
23810 fnvoid_va_start_sysv =
23811 build_varargs_function_type_list (void_type_node, sysv_va_ref,
23813 fnvoid_va_copy_ms =
23814 build_function_type_list (void_type_node, ms_va_ref, ms_va_list_type_node,
23816 fnvoid_va_copy_sysv =
23817 build_function_type_list (void_type_node, sysv_va_ref,
23818 sysv_va_ref, NULL_TREE);
23820 add_builtin_function ("__builtin_ms_va_start", fnvoid_va_start_ms,
23821 BUILT_IN_VA_START, BUILT_IN_NORMAL, NULL, fnattr_ms);
23822 add_builtin_function ("__builtin_ms_va_end", fnvoid_va_end_ms,
23823 BUILT_IN_VA_END, BUILT_IN_NORMAL, NULL, fnattr_ms);
23824 add_builtin_function ("__builtin_ms_va_copy", fnvoid_va_copy_ms,
23825 BUILT_IN_VA_COPY, BUILT_IN_NORMAL, NULL, fnattr_ms);
23826 add_builtin_function ("__builtin_sysv_va_start", fnvoid_va_start_sysv,
23827 BUILT_IN_VA_START, BUILT_IN_NORMAL, NULL, fnattr_sysv);
23828 add_builtin_function ("__builtin_sysv_va_end", fnvoid_va_end_sysv,
23829 BUILT_IN_VA_END, BUILT_IN_NORMAL, NULL, fnattr_sysv);
23830 add_builtin_function ("__builtin_sysv_va_copy", fnvoid_va_copy_sysv,
23831 BUILT_IN_VA_COPY, BUILT_IN_NORMAL, NULL, fnattr_sysv);
23835 ix86_init_builtins (void)
23837 tree float128_type_node = make_node (REAL_TYPE);
23840 /* The __float80 type. */
23841 if (TYPE_MODE (long_double_type_node) == XFmode)
23842 (*lang_hooks.types.register_builtin_type) (long_double_type_node,
23846 /* The __float80 type. */
23847 tree float80_type_node = make_node (REAL_TYPE);
23849 TYPE_PRECISION (float80_type_node) = 80;
23850 layout_type (float80_type_node);
23851 (*lang_hooks.types.register_builtin_type) (float80_type_node,
23855 /* The __float128 type. */
23856 TYPE_PRECISION (float128_type_node) = 128;
23857 layout_type (float128_type_node);
23858 (*lang_hooks.types.register_builtin_type) (float128_type_node,
23861 /* TFmode support builtins. */
23862 ftype = build_function_type (float128_type_node, void_list_node);
23863 decl = add_builtin_function ("__builtin_infq", ftype,
23864 IX86_BUILTIN_INFQ, BUILT_IN_MD,
23866 ix86_builtins[(int) IX86_BUILTIN_INFQ] = decl;
23868 decl = add_builtin_function ("__builtin_huge_valq", ftype,
23869 IX86_BUILTIN_HUGE_VALQ, BUILT_IN_MD,
23871 ix86_builtins[(int) IX86_BUILTIN_HUGE_VALQ] = decl;
23873 /* We will expand them to normal call if SSE2 isn't available since
23874 they are used by libgcc. */
23875 ftype = build_function_type_list (float128_type_node,
23876 float128_type_node,
23878 decl = add_builtin_function ("__builtin_fabsq", ftype,
23879 IX86_BUILTIN_FABSQ, BUILT_IN_MD,
23880 "__fabstf2", NULL_TREE);
23881 ix86_builtins[(int) IX86_BUILTIN_FABSQ] = decl;
23882 TREE_READONLY (decl) = 1;
23884 ftype = build_function_type_list (float128_type_node,
23885 float128_type_node,
23886 float128_type_node,
23888 decl = add_builtin_function ("__builtin_copysignq", ftype,
23889 IX86_BUILTIN_COPYSIGNQ, BUILT_IN_MD,
23890 "__copysigntf3", NULL_TREE);
23891 ix86_builtins[(int) IX86_BUILTIN_COPYSIGNQ] = decl;
23892 TREE_READONLY (decl) = 1;
23894 ix86_init_mmx_sse_builtins ();
23896 ix86_init_builtins_va_builtins_abi ();
23899 /* Errors in the source file can cause expand_expr to return const0_rtx
23900 where we expect a vector. To avoid crashing, use one of the vector
23901 clear instructions. */
23903 safe_vector_operand (rtx x, enum machine_mode mode)
23905 if (x == const0_rtx)
23906 x = CONST0_RTX (mode);
23910 /* Subroutine of ix86_expand_builtin to take care of binop insns. */
23913 ix86_expand_binop_builtin (enum insn_code icode, tree exp, rtx target)
23916 tree arg0 = CALL_EXPR_ARG (exp, 0);
23917 tree arg1 = CALL_EXPR_ARG (exp, 1);
23918 rtx op0 = expand_normal (arg0);
23919 rtx op1 = expand_normal (arg1);
23920 enum machine_mode tmode = insn_data[icode].operand[0].mode;
23921 enum machine_mode mode0 = insn_data[icode].operand[1].mode;
23922 enum machine_mode mode1 = insn_data[icode].operand[2].mode;
23924 if (VECTOR_MODE_P (mode0))
23925 op0 = safe_vector_operand (op0, mode0);
23926 if (VECTOR_MODE_P (mode1))
23927 op1 = safe_vector_operand (op1, mode1);
23929 if (optimize || !target
23930 || GET_MODE (target) != tmode
23931 || ! (*insn_data[icode].operand[0].predicate) (target, tmode))
23932 target = gen_reg_rtx (tmode);
23934 if (GET_MODE (op1) == SImode && mode1 == TImode)
23936 rtx x = gen_reg_rtx (V4SImode);
23937 emit_insn (gen_sse2_loadd (x, op1));
23938 op1 = gen_lowpart (TImode, x);
23941 if (!(*insn_data[icode].operand[1].predicate) (op0, mode0))
23942 op0 = copy_to_mode_reg (mode0, op0);
23943 if (!(*insn_data[icode].operand[2].predicate) (op1, mode1))
23944 op1 = copy_to_mode_reg (mode1, op1);
23946 pat = GEN_FCN (icode) (target, op0, op1);
23955 /* Subroutine of ix86_expand_builtin to take care of 2-4 argument insns. */
23958 ix86_expand_multi_arg_builtin (enum insn_code icode, tree exp, rtx target,
23959 enum multi_arg_type m_type,
23960 enum rtx_code sub_code)
23965 bool comparison_p = false;
23967 bool last_arg_constant = false;
23968 int num_memory = 0;
23971 enum machine_mode mode;
23974 enum machine_mode tmode = insn_data[icode].operand[0].mode;
23978 case MULTI_ARG_3_SF:
23979 case MULTI_ARG_3_DF:
23980 case MULTI_ARG_3_DI:
23981 case MULTI_ARG_3_SI:
23982 case MULTI_ARG_3_SI_DI:
23983 case MULTI_ARG_3_HI:
23984 case MULTI_ARG_3_HI_SI:
23985 case MULTI_ARG_3_QI:
23986 case MULTI_ARG_3_PERMPS:
23987 case MULTI_ARG_3_PERMPD:
23991 case MULTI_ARG_2_SF:
23992 case MULTI_ARG_2_DF:
23993 case MULTI_ARG_2_DI:
23994 case MULTI_ARG_2_SI:
23995 case MULTI_ARG_2_HI:
23996 case MULTI_ARG_2_QI:
24000 case MULTI_ARG_2_DI_IMM:
24001 case MULTI_ARG_2_SI_IMM:
24002 case MULTI_ARG_2_HI_IMM:
24003 case MULTI_ARG_2_QI_IMM:
24005 last_arg_constant = true;
24008 case MULTI_ARG_1_SF:
24009 case MULTI_ARG_1_DF:
24010 case MULTI_ARG_1_DI:
24011 case MULTI_ARG_1_SI:
24012 case MULTI_ARG_1_HI:
24013 case MULTI_ARG_1_QI:
24014 case MULTI_ARG_1_SI_DI:
24015 case MULTI_ARG_1_HI_DI:
24016 case MULTI_ARG_1_HI_SI:
24017 case MULTI_ARG_1_QI_DI:
24018 case MULTI_ARG_1_QI_SI:
24019 case MULTI_ARG_1_QI_HI:
24020 case MULTI_ARG_1_PH2PS:
24021 case MULTI_ARG_1_PS2PH:
24025 case MULTI_ARG_2_SF_CMP:
24026 case MULTI_ARG_2_DF_CMP:
24027 case MULTI_ARG_2_DI_CMP:
24028 case MULTI_ARG_2_SI_CMP:
24029 case MULTI_ARG_2_HI_CMP:
24030 case MULTI_ARG_2_QI_CMP:
24032 comparison_p = true;
24035 case MULTI_ARG_2_SF_TF:
24036 case MULTI_ARG_2_DF_TF:
24037 case MULTI_ARG_2_DI_TF:
24038 case MULTI_ARG_2_SI_TF:
24039 case MULTI_ARG_2_HI_TF:
24040 case MULTI_ARG_2_QI_TF:
24045 case MULTI_ARG_UNKNOWN:
24047 gcc_unreachable ();
24050 if (optimize || !target
24051 || GET_MODE (target) != tmode
24052 || ! (*insn_data[icode].operand[0].predicate) (target, tmode))
24053 target = gen_reg_rtx (tmode);
24055 gcc_assert (nargs <= 4);
24057 for (i = 0; i < nargs; i++)
24059 tree arg = CALL_EXPR_ARG (exp, i);
24060 rtx op = expand_normal (arg);
24061 int adjust = (comparison_p) ? 1 : 0;
24062 enum machine_mode mode = insn_data[icode].operand[i+adjust+1].mode;
24064 if (last_arg_constant && i == nargs-1)
24066 if (GET_CODE (op) != CONST_INT)
24068 error ("last argument must be an immediate");
24069 return gen_reg_rtx (tmode);
24074 if (VECTOR_MODE_P (mode))
24075 op = safe_vector_operand (op, mode);
24077 /* If we aren't optimizing, only allow one memory operand to be
24079 if (memory_operand (op, mode))
24082 gcc_assert (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode);
24085 || ! (*insn_data[icode].operand[i+adjust+1].predicate) (op, mode)
24087 op = force_reg (mode, op);
24091 args[i].mode = mode;
24097 pat = GEN_FCN (icode) (target, args[0].op);
24102 pat = GEN_FCN (icode) (target, args[0].op, args[1].op,
24103 GEN_INT ((int)sub_code));
24104 else if (! comparison_p)
24105 pat = GEN_FCN (icode) (target, args[0].op, args[1].op);
24108 rtx cmp_op = gen_rtx_fmt_ee (sub_code, GET_MODE (target),
24112 pat = GEN_FCN (icode) (target, cmp_op, args[0].op, args[1].op);
24117 pat = GEN_FCN (icode) (target, args[0].op, args[1].op, args[2].op);
24121 gcc_unreachable ();
24131 /* Subroutine of ix86_expand_args_builtin to take care of scalar unop
24132 insns with vec_merge. */
24135 ix86_expand_unop_vec_merge_builtin (enum insn_code icode, tree exp,
24139 tree arg0 = CALL_EXPR_ARG (exp, 0);
24140 rtx op1, op0 = expand_normal (arg0);
24141 enum machine_mode tmode = insn_data[icode].operand[0].mode;
24142 enum machine_mode mode0 = insn_data[icode].operand[1].mode;
24144 if (optimize || !target
24145 || GET_MODE (target) != tmode
24146 || ! (*insn_data[icode].operand[0].predicate) (target, tmode))
24147 target = gen_reg_rtx (tmode);
24149 if (VECTOR_MODE_P (mode0))
24150 op0 = safe_vector_operand (op0, mode0);
24152 if ((optimize && !register_operand (op0, mode0))
24153 || ! (*insn_data[icode].operand[1].predicate) (op0, mode0))
24154 op0 = copy_to_mode_reg (mode0, op0);
24157 if (! (*insn_data[icode].operand[2].predicate) (op1, mode0))
24158 op1 = copy_to_mode_reg (mode0, op1);
24160 pat = GEN_FCN (icode) (target, op0, op1);
24167 /* Subroutine of ix86_expand_builtin to take care of comparison insns. */
24170 ix86_expand_sse_compare (const struct builtin_description *d,
24171 tree exp, rtx target, bool swap)
24174 tree arg0 = CALL_EXPR_ARG (exp, 0);
24175 tree arg1 = CALL_EXPR_ARG (exp, 1);
24176 rtx op0 = expand_normal (arg0);
24177 rtx op1 = expand_normal (arg1);
24179 enum machine_mode tmode = insn_data[d->icode].operand[0].mode;
24180 enum machine_mode mode0 = insn_data[d->icode].operand[1].mode;
24181 enum machine_mode mode1 = insn_data[d->icode].operand[2].mode;
24182 enum rtx_code comparison = d->comparison;
24184 if (VECTOR_MODE_P (mode0))
24185 op0 = safe_vector_operand (op0, mode0);
24186 if (VECTOR_MODE_P (mode1))
24187 op1 = safe_vector_operand (op1, mode1);
24189 /* Swap operands if we have a comparison that isn't available in
24193 rtx tmp = gen_reg_rtx (mode1);
24194 emit_move_insn (tmp, op1);
24199 if (optimize || !target
24200 || GET_MODE (target) != tmode
24201 || ! (*insn_data[d->icode].operand[0].predicate) (target, tmode))
24202 target = gen_reg_rtx (tmode);
24204 if ((optimize && !register_operand (op0, mode0))
24205 || ! (*insn_data[d->icode].operand[1].predicate) (op0, mode0))
24206 op0 = copy_to_mode_reg (mode0, op0);
24207 if ((optimize && !register_operand (op1, mode1))
24208 || ! (*insn_data[d->icode].operand[2].predicate) (op1, mode1))
24209 op1 = copy_to_mode_reg (mode1, op1);
24211 op2 = gen_rtx_fmt_ee (comparison, mode0, op0, op1);
24212 pat = GEN_FCN (d->icode) (target, op0, op1, op2);
24219 /* Subroutine of ix86_expand_builtin to take care of comi insns. */
24222 ix86_expand_sse_comi (const struct builtin_description *d, tree exp,
24226 tree arg0 = CALL_EXPR_ARG (exp, 0);
24227 tree arg1 = CALL_EXPR_ARG (exp, 1);
24228 rtx op0 = expand_normal (arg0);
24229 rtx op1 = expand_normal (arg1);
24230 enum machine_mode mode0 = insn_data[d->icode].operand[0].mode;
24231 enum machine_mode mode1 = insn_data[d->icode].operand[1].mode;
24232 enum rtx_code comparison = d->comparison;
24234 if (VECTOR_MODE_P (mode0))
24235 op0 = safe_vector_operand (op0, mode0);
24236 if (VECTOR_MODE_P (mode1))
24237 op1 = safe_vector_operand (op1, mode1);
24239 /* Swap operands if we have a comparison that isn't available in
24241 if (d->flag & BUILTIN_DESC_SWAP_OPERANDS)
24248 target = gen_reg_rtx (SImode);
24249 emit_move_insn (target, const0_rtx);
24250 target = gen_rtx_SUBREG (QImode, target, 0);
24252 if ((optimize && !register_operand (op0, mode0))
24253 || !(*insn_data[d->icode].operand[0].predicate) (op0, mode0))
24254 op0 = copy_to_mode_reg (mode0, op0);
24255 if ((optimize && !register_operand (op1, mode1))
24256 || !(*insn_data[d->icode].operand[1].predicate) (op1, mode1))
24257 op1 = copy_to_mode_reg (mode1, op1);
24259 pat = GEN_FCN (d->icode) (op0, op1);
24263 emit_insn (gen_rtx_SET (VOIDmode,
24264 gen_rtx_STRICT_LOW_PART (VOIDmode, target),
24265 gen_rtx_fmt_ee (comparison, QImode,
24269 return SUBREG_REG (target);
24272 /* Subroutine of ix86_expand_builtin to take care of ptest insns. */
24275 ix86_expand_sse_ptest (const struct builtin_description *d, tree exp,
24279 tree arg0 = CALL_EXPR_ARG (exp, 0);
24280 tree arg1 = CALL_EXPR_ARG (exp, 1);
24281 rtx op0 = expand_normal (arg0);
24282 rtx op1 = expand_normal (arg1);
24283 enum machine_mode mode0 = insn_data[d->icode].operand[0].mode;
24284 enum machine_mode mode1 = insn_data[d->icode].operand[1].mode;
24285 enum rtx_code comparison = d->comparison;
24287 if (VECTOR_MODE_P (mode0))
24288 op0 = safe_vector_operand (op0, mode0);
24289 if (VECTOR_MODE_P (mode1))
24290 op1 = safe_vector_operand (op1, mode1);
24292 target = gen_reg_rtx (SImode);
24293 emit_move_insn (target, const0_rtx);
24294 target = gen_rtx_SUBREG (QImode, target, 0);
24296 if ((optimize && !register_operand (op0, mode0))
24297 || !(*insn_data[d->icode].operand[0].predicate) (op0, mode0))
24298 op0 = copy_to_mode_reg (mode0, op0);
24299 if ((optimize && !register_operand (op1, mode1))
24300 || !(*insn_data[d->icode].operand[1].predicate) (op1, mode1))
24301 op1 = copy_to_mode_reg (mode1, op1);
24303 pat = GEN_FCN (d->icode) (op0, op1);
24307 emit_insn (gen_rtx_SET (VOIDmode,
24308 gen_rtx_STRICT_LOW_PART (VOIDmode, target),
24309 gen_rtx_fmt_ee (comparison, QImode,
24313 return SUBREG_REG (target);
24316 /* Subroutine of ix86_expand_builtin to take care of pcmpestr[im] insns. */
24319 ix86_expand_sse_pcmpestr (const struct builtin_description *d,
24320 tree exp, rtx target)
24323 tree arg0 = CALL_EXPR_ARG (exp, 0);
24324 tree arg1 = CALL_EXPR_ARG (exp, 1);
24325 tree arg2 = CALL_EXPR_ARG (exp, 2);
24326 tree arg3 = CALL_EXPR_ARG (exp, 3);
24327 tree arg4 = CALL_EXPR_ARG (exp, 4);
24328 rtx scratch0, scratch1;
24329 rtx op0 = expand_normal (arg0);
24330 rtx op1 = expand_normal (arg1);
24331 rtx op2 = expand_normal (arg2);
24332 rtx op3 = expand_normal (arg3);
24333 rtx op4 = expand_normal (arg4);
24334 enum machine_mode tmode0, tmode1, modev2, modei3, modev4, modei5, modeimm;
24336 tmode0 = insn_data[d->icode].operand[0].mode;
24337 tmode1 = insn_data[d->icode].operand[1].mode;
24338 modev2 = insn_data[d->icode].operand[2].mode;
24339 modei3 = insn_data[d->icode].operand[3].mode;
24340 modev4 = insn_data[d->icode].operand[4].mode;
24341 modei5 = insn_data[d->icode].operand[5].mode;
24342 modeimm = insn_data[d->icode].operand[6].mode;
24344 if (VECTOR_MODE_P (modev2))
24345 op0 = safe_vector_operand (op0, modev2);
24346 if (VECTOR_MODE_P (modev4))
24347 op2 = safe_vector_operand (op2, modev4);
24349 if (! (*insn_data[d->icode].operand[2].predicate) (op0, modev2))
24350 op0 = copy_to_mode_reg (modev2, op0);
24351 if (! (*insn_data[d->icode].operand[3].predicate) (op1, modei3))
24352 op1 = copy_to_mode_reg (modei3, op1);
24353 if ((optimize && !register_operand (op2, modev4))
24354 || !(*insn_data[d->icode].operand[4].predicate) (op2, modev4))
24355 op2 = copy_to_mode_reg (modev4, op2);
24356 if (! (*insn_data[d->icode].operand[5].predicate) (op3, modei5))
24357 op3 = copy_to_mode_reg (modei5, op3);
24359 if (! (*insn_data[d->icode].operand[6].predicate) (op4, modeimm))
24361 error ("the fifth argument must be a 8-bit immediate");
24365 if (d->code == IX86_BUILTIN_PCMPESTRI128)
24367 if (optimize || !target
24368 || GET_MODE (target) != tmode0
24369 || ! (*insn_data[d->icode].operand[0].predicate) (target, tmode0))
24370 target = gen_reg_rtx (tmode0);
24372 scratch1 = gen_reg_rtx (tmode1);
24374 pat = GEN_FCN (d->icode) (target, scratch1, op0, op1, op2, op3, op4);
24376 else if (d->code == IX86_BUILTIN_PCMPESTRM128)
24378 if (optimize || !target
24379 || GET_MODE (target) != tmode1
24380 || ! (*insn_data[d->icode].operand[1].predicate) (target, tmode1))
24381 target = gen_reg_rtx (tmode1);
24383 scratch0 = gen_reg_rtx (tmode0);
24385 pat = GEN_FCN (d->icode) (scratch0, target, op0, op1, op2, op3, op4);
24389 gcc_assert (d->flag);
24391 scratch0 = gen_reg_rtx (tmode0);
24392 scratch1 = gen_reg_rtx (tmode1);
24394 pat = GEN_FCN (d->icode) (scratch0, scratch1, op0, op1, op2, op3, op4);
24404 target = gen_reg_rtx (SImode);
24405 emit_move_insn (target, const0_rtx);
24406 target = gen_rtx_SUBREG (QImode, target, 0);
24409 (gen_rtx_SET (VOIDmode, gen_rtx_STRICT_LOW_PART (VOIDmode, target),
24410 gen_rtx_fmt_ee (EQ, QImode,
24411 gen_rtx_REG ((enum machine_mode) d->flag,
24414 return SUBREG_REG (target);
24421 /* Subroutine of ix86_expand_builtin to take care of pcmpistr[im] insns. */
24424 ix86_expand_sse_pcmpistr (const struct builtin_description *d,
24425 tree exp, rtx target)
24428 tree arg0 = CALL_EXPR_ARG (exp, 0);
24429 tree arg1 = CALL_EXPR_ARG (exp, 1);
24430 tree arg2 = CALL_EXPR_ARG (exp, 2);
24431 rtx scratch0, scratch1;
24432 rtx op0 = expand_normal (arg0);
24433 rtx op1 = expand_normal (arg1);
24434 rtx op2 = expand_normal (arg2);
24435 enum machine_mode tmode0, tmode1, modev2, modev3, modeimm;
24437 tmode0 = insn_data[d->icode].operand[0].mode;
24438 tmode1 = insn_data[d->icode].operand[1].mode;
24439 modev2 = insn_data[d->icode].operand[2].mode;
24440 modev3 = insn_data[d->icode].operand[3].mode;
24441 modeimm = insn_data[d->icode].operand[4].mode;
24443 if (VECTOR_MODE_P (modev2))
24444 op0 = safe_vector_operand (op0, modev2);
24445 if (VECTOR_MODE_P (modev3))
24446 op1 = safe_vector_operand (op1, modev3);
24448 if (! (*insn_data[d->icode].operand[2].predicate) (op0, modev2))
24449 op0 = copy_to_mode_reg (modev2, op0);
24450 if ((optimize && !register_operand (op1, modev3))
24451 || !(*insn_data[d->icode].operand[3].predicate) (op1, modev3))
24452 op1 = copy_to_mode_reg (modev3, op1);
24454 if (! (*insn_data[d->icode].operand[4].predicate) (op2, modeimm))
24456 error ("the third argument must be a 8-bit immediate");
24460 if (d->code == IX86_BUILTIN_PCMPISTRI128)
24462 if (optimize || !target
24463 || GET_MODE (target) != tmode0
24464 || ! (*insn_data[d->icode].operand[0].predicate) (target, tmode0))
24465 target = gen_reg_rtx (tmode0);
24467 scratch1 = gen_reg_rtx (tmode1);
24469 pat = GEN_FCN (d->icode) (target, scratch1, op0, op1, op2);
24471 else if (d->code == IX86_BUILTIN_PCMPISTRM128)
24473 if (optimize || !target
24474 || GET_MODE (target) != tmode1
24475 || ! (*insn_data[d->icode].operand[1].predicate) (target, tmode1))
24476 target = gen_reg_rtx (tmode1);
24478 scratch0 = gen_reg_rtx (tmode0);
24480 pat = GEN_FCN (d->icode) (scratch0, target, op0, op1, op2);
24484 gcc_assert (d->flag);
24486 scratch0 = gen_reg_rtx (tmode0);
24487 scratch1 = gen_reg_rtx (tmode1);
24489 pat = GEN_FCN (d->icode) (scratch0, scratch1, op0, op1, op2);
24499 target = gen_reg_rtx (SImode);
24500 emit_move_insn (target, const0_rtx);
24501 target = gen_rtx_SUBREG (QImode, target, 0);
24504 (gen_rtx_SET (VOIDmode, gen_rtx_STRICT_LOW_PART (VOIDmode, target),
24505 gen_rtx_fmt_ee (EQ, QImode,
24506 gen_rtx_REG ((enum machine_mode) d->flag,
24509 return SUBREG_REG (target);
24515 /* Subroutine of ix86_expand_builtin to take care of insns with
24516 variable number of operands. */
24519 ix86_expand_args_builtin (const struct builtin_description *d,
24520 tree exp, rtx target)
24522 rtx pat, real_target;
24523 unsigned int i, nargs;
24524 unsigned int nargs_constant = 0;
24525 int num_memory = 0;
24529 enum machine_mode mode;
24531 bool last_arg_count = false;
24532 enum insn_code icode = d->icode;
24533 const struct insn_data *insn_p = &insn_data[icode];
24534 enum machine_mode tmode = insn_p->operand[0].mode;
24535 enum machine_mode rmode = VOIDmode;
24537 enum rtx_code comparison = d->comparison;
24539 switch ((enum ix86_builtin_type) d->flag)
24541 case INT_FTYPE_V8SF_V8SF_PTEST:
24542 case INT_FTYPE_V4DI_V4DI_PTEST:
24543 case INT_FTYPE_V4DF_V4DF_PTEST:
24544 case INT_FTYPE_V4SF_V4SF_PTEST:
24545 case INT_FTYPE_V2DI_V2DI_PTEST:
24546 case INT_FTYPE_V2DF_V2DF_PTEST:
24547 return ix86_expand_sse_ptest (d, exp, target);
24548 case FLOAT128_FTYPE_FLOAT128:
24549 case FLOAT_FTYPE_FLOAT:
24550 case INT64_FTYPE_V4SF:
24551 case INT64_FTYPE_V2DF:
24552 case INT_FTYPE_V16QI:
24553 case INT_FTYPE_V8QI:
24554 case INT_FTYPE_V8SF:
24555 case INT_FTYPE_V4DF:
24556 case INT_FTYPE_V4SF:
24557 case INT_FTYPE_V2DF:
24558 case V16QI_FTYPE_V16QI:
24559 case V8SI_FTYPE_V8SF:
24560 case V8SI_FTYPE_V4SI:
24561 case V8HI_FTYPE_V8HI:
24562 case V8HI_FTYPE_V16QI:
24563 case V8QI_FTYPE_V8QI:
24564 case V8SF_FTYPE_V8SF:
24565 case V8SF_FTYPE_V8SI:
24566 case V8SF_FTYPE_V4SF:
24567 case V4SI_FTYPE_V4SI:
24568 case V4SI_FTYPE_V16QI:
24569 case V4SI_FTYPE_V4SF:
24570 case V4SI_FTYPE_V8SI:
24571 case V4SI_FTYPE_V8HI:
24572 case V4SI_FTYPE_V4DF:
24573 case V4SI_FTYPE_V2DF:
24574 case V4HI_FTYPE_V4HI:
24575 case V4DF_FTYPE_V4DF:
24576 case V4DF_FTYPE_V4SI:
24577 case V4DF_FTYPE_V4SF:
24578 case V4DF_FTYPE_V2DF:
24579 case V4SF_FTYPE_V4SF:
24580 case V4SF_FTYPE_V4SI:
24581 case V4SF_FTYPE_V8SF:
24582 case V4SF_FTYPE_V4DF:
24583 case V4SF_FTYPE_V2DF:
24584 case V2DI_FTYPE_V2DI:
24585 case V2DI_FTYPE_V16QI:
24586 case V2DI_FTYPE_V8HI:
24587 case V2DI_FTYPE_V4SI:
24588 case V2DF_FTYPE_V2DF:
24589 case V2DF_FTYPE_V4SI:
24590 case V2DF_FTYPE_V4DF:
24591 case V2DF_FTYPE_V4SF:
24592 case V2DF_FTYPE_V2SI:
24593 case V2SI_FTYPE_V2SI:
24594 case V2SI_FTYPE_V4SF:
24595 case V2SI_FTYPE_V2SF:
24596 case V2SI_FTYPE_V2DF:
24597 case V2SF_FTYPE_V2SF:
24598 case V2SF_FTYPE_V2SI:
24601 case V4SF_FTYPE_V4SF_VEC_MERGE:
24602 case V2DF_FTYPE_V2DF_VEC_MERGE:
24603 return ix86_expand_unop_vec_merge_builtin (icode, exp, target);
24604 case FLOAT128_FTYPE_FLOAT128_FLOAT128:
24605 case V16QI_FTYPE_V16QI_V16QI:
24606 case V16QI_FTYPE_V8HI_V8HI:
24607 case V8QI_FTYPE_V8QI_V8QI:
24608 case V8QI_FTYPE_V4HI_V4HI:
24609 case V8HI_FTYPE_V8HI_V8HI:
24610 case V8HI_FTYPE_V16QI_V16QI:
24611 case V8HI_FTYPE_V4SI_V4SI:
24612 case V8SF_FTYPE_V8SF_V8SF:
24613 case V8SF_FTYPE_V8SF_V8SI:
24614 case V4SI_FTYPE_V4SI_V4SI:
24615 case V4SI_FTYPE_V8HI_V8HI:
24616 case V4SI_FTYPE_V4SF_V4SF:
24617 case V4SI_FTYPE_V2DF_V2DF:
24618 case V4HI_FTYPE_V4HI_V4HI:
24619 case V4HI_FTYPE_V8QI_V8QI:
24620 case V4HI_FTYPE_V2SI_V2SI:
24621 case V4DF_FTYPE_V4DF_V4DF:
24622 case V4DF_FTYPE_V4DF_V4DI:
24623 case V4SF_FTYPE_V4SF_V4SF:
24624 case V4SF_FTYPE_V4SF_V4SI:
24625 case V4SF_FTYPE_V4SF_V2SI:
24626 case V4SF_FTYPE_V4SF_V2DF:
24627 case V4SF_FTYPE_V4SF_DI:
24628 case V4SF_FTYPE_V4SF_SI:
24629 case V2DI_FTYPE_V2DI_V2DI:
24630 case V2DI_FTYPE_V16QI_V16QI:
24631 case V2DI_FTYPE_V4SI_V4SI:
24632 case V2DI_FTYPE_V2DI_V16QI:
24633 case V2DI_FTYPE_V2DF_V2DF:
24634 case V2SI_FTYPE_V2SI_V2SI:
24635 case V2SI_FTYPE_V4HI_V4HI:
24636 case V2SI_FTYPE_V2SF_V2SF:
24637 case V2DF_FTYPE_V2DF_V2DF:
24638 case V2DF_FTYPE_V2DF_V4SF:
24639 case V2DF_FTYPE_V2DF_V2DI:
24640 case V2DF_FTYPE_V2DF_DI:
24641 case V2DF_FTYPE_V2DF_SI:
24642 case V2SF_FTYPE_V2SF_V2SF:
24643 case V1DI_FTYPE_V1DI_V1DI:
24644 case V1DI_FTYPE_V8QI_V8QI:
24645 case V1DI_FTYPE_V2SI_V2SI:
24646 if (comparison == UNKNOWN)
24647 return ix86_expand_binop_builtin (icode, exp, target);
24650 case V4SF_FTYPE_V4SF_V4SF_SWAP:
24651 case V2DF_FTYPE_V2DF_V2DF_SWAP:
24652 gcc_assert (comparison != UNKNOWN);
24656 case V8HI_FTYPE_V8HI_V8HI_COUNT:
24657 case V8HI_FTYPE_V8HI_SI_COUNT:
24658 case V4SI_FTYPE_V4SI_V4SI_COUNT:
24659 case V4SI_FTYPE_V4SI_SI_COUNT:
24660 case V4HI_FTYPE_V4HI_V4HI_COUNT:
24661 case V4HI_FTYPE_V4HI_SI_COUNT:
24662 case V2DI_FTYPE_V2DI_V2DI_COUNT:
24663 case V2DI_FTYPE_V2DI_SI_COUNT:
24664 case V2SI_FTYPE_V2SI_V2SI_COUNT:
24665 case V2SI_FTYPE_V2SI_SI_COUNT:
24666 case V1DI_FTYPE_V1DI_V1DI_COUNT:
24667 case V1DI_FTYPE_V1DI_SI_COUNT:
24669 last_arg_count = true;
24671 case UINT64_FTYPE_UINT64_UINT64:
24672 case UINT_FTYPE_UINT_UINT:
24673 case UINT_FTYPE_UINT_USHORT:
24674 case UINT_FTYPE_UINT_UCHAR:
24677 case V2DI2TI_FTYPE_V2DI_INT:
24680 nargs_constant = 1;
24682 case V8HI_FTYPE_V8HI_INT:
24683 case V8SF_FTYPE_V8SF_INT:
24684 case V4SI_FTYPE_V4SI_INT:
24685 case V4SI_FTYPE_V8SI_INT:
24686 case V4HI_FTYPE_V4HI_INT:
24687 case V4DF_FTYPE_V4DF_INT:
24688 case V4SF_FTYPE_V4SF_INT:
24689 case V4SF_FTYPE_V8SF_INT:
24690 case V2DI_FTYPE_V2DI_INT:
24691 case V2DF_FTYPE_V2DF_INT:
24692 case V2DF_FTYPE_V4DF_INT:
24694 nargs_constant = 1;
24696 case V16QI_FTYPE_V16QI_V16QI_V16QI:
24697 case V8SF_FTYPE_V8SF_V8SF_V8SF:
24698 case V4DF_FTYPE_V4DF_V4DF_V4DF:
24699 case V4SF_FTYPE_V4SF_V4SF_V4SF:
24700 case V2DF_FTYPE_V2DF_V2DF_V2DF:
24703 case V16QI_FTYPE_V16QI_V16QI_INT:
24704 case V8HI_FTYPE_V8HI_V8HI_INT:
24705 case V8SI_FTYPE_V8SI_V8SI_INT:
24706 case V8SI_FTYPE_V8SI_V4SI_INT:
24707 case V8SF_FTYPE_V8SF_V8SF_INT:
24708 case V8SF_FTYPE_V8SF_V4SF_INT:
24709 case V4SI_FTYPE_V4SI_V4SI_INT:
24710 case V4DF_FTYPE_V4DF_V4DF_INT:
24711 case V4DF_FTYPE_V4DF_V2DF_INT:
24712 case V4SF_FTYPE_V4SF_V4SF_INT:
24713 case V2DI_FTYPE_V2DI_V2DI_INT:
24714 case V2DF_FTYPE_V2DF_V2DF_INT:
24716 nargs_constant = 1;
24718 case V2DI2TI_FTYPE_V2DI_V2DI_INT:
24721 nargs_constant = 1;
24723 case V1DI2DI_FTYPE_V1DI_V1DI_INT:
24726 nargs_constant = 1;
24728 case V2DI_FTYPE_V2DI_UINT_UINT:
24730 nargs_constant = 2;
24732 case V2DI_FTYPE_V2DI_V2DI_UINT_UINT:
24734 nargs_constant = 2;
24737 gcc_unreachable ();
24740 gcc_assert (nargs <= ARRAY_SIZE (args));
24742 if (comparison != UNKNOWN)
24744 gcc_assert (nargs == 2);
24745 return ix86_expand_sse_compare (d, exp, target, swap);
24748 if (rmode == VOIDmode || rmode == tmode)
24752 || GET_MODE (target) != tmode
24753 || ! (*insn_p->operand[0].predicate) (target, tmode))
24754 target = gen_reg_rtx (tmode);
24755 real_target = target;
24759 target = gen_reg_rtx (rmode);
24760 real_target = simplify_gen_subreg (tmode, target, rmode, 0);
24763 for (i = 0; i < nargs; i++)
24765 tree arg = CALL_EXPR_ARG (exp, i);
24766 rtx op = expand_normal (arg);
24767 enum machine_mode mode = insn_p->operand[i + 1].mode;
24768 bool match = (*insn_p->operand[i + 1].predicate) (op, mode);
24770 if (last_arg_count && (i + 1) == nargs)
24772 /* SIMD shift insns take either an 8-bit immediate or
24773 register as count. But builtin functions take int as
24774 count. If count doesn't match, we put it in register. */
24777 op = simplify_gen_subreg (SImode, op, GET_MODE (op), 0);
24778 if (!(*insn_p->operand[i + 1].predicate) (op, mode))
24779 op = copy_to_reg (op);
24782 else if ((nargs - i) <= nargs_constant)
24787 case CODE_FOR_sse4_1_roundpd:
24788 case CODE_FOR_sse4_1_roundps:
24789 case CODE_FOR_sse4_1_roundsd:
24790 case CODE_FOR_sse4_1_roundss:
24791 case CODE_FOR_sse4_1_blendps:
24792 case CODE_FOR_avx_blendpd256:
24793 case CODE_FOR_avx_vpermilv4df:
24794 case CODE_FOR_avx_roundpd256:
24795 case CODE_FOR_avx_roundps256:
24796 error ("the last argument must be a 4-bit immediate");
24799 case CODE_FOR_sse4_1_blendpd:
24800 case CODE_FOR_avx_vpermilv2df:
24801 error ("the last argument must be a 2-bit immediate");
24804 case CODE_FOR_avx_vextractf128v4df:
24805 case CODE_FOR_avx_vextractf128v8sf:
24806 case CODE_FOR_avx_vextractf128v8si:
24807 case CODE_FOR_avx_vinsertf128v4df:
24808 case CODE_FOR_avx_vinsertf128v8sf:
24809 case CODE_FOR_avx_vinsertf128v8si:
24810 error ("the last argument must be a 1-bit immediate");
24813 case CODE_FOR_avx_cmpsdv2df3:
24814 case CODE_FOR_avx_cmpssv4sf3:
24815 case CODE_FOR_avx_cmppdv2df3:
24816 case CODE_FOR_avx_cmppsv4sf3:
24817 case CODE_FOR_avx_cmppdv4df3:
24818 case CODE_FOR_avx_cmppsv8sf3:
24819 error ("the last argument must be a 5-bit immediate");
24823 switch (nargs_constant)
24826 if ((nargs - i) == nargs_constant)
24828 error ("the next to last argument must be an 8-bit immediate");
24832 error ("the last argument must be an 8-bit immediate");
24835 gcc_unreachable ();
24842 if (VECTOR_MODE_P (mode))
24843 op = safe_vector_operand (op, mode);
24845 /* If we aren't optimizing, only allow one memory operand to
24847 if (memory_operand (op, mode))
24850 if (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode)
24852 if (optimize || !match || num_memory > 1)
24853 op = copy_to_mode_reg (mode, op);
24857 op = copy_to_reg (op);
24858 op = simplify_gen_subreg (mode, op, GET_MODE (op), 0);
24863 args[i].mode = mode;
24869 pat = GEN_FCN (icode) (real_target, args[0].op);
24872 pat = GEN_FCN (icode) (real_target, args[0].op, args[1].op);
24875 pat = GEN_FCN (icode) (real_target, args[0].op, args[1].op,
24879 pat = GEN_FCN (icode) (real_target, args[0].op, args[1].op,
24880 args[2].op, args[3].op);
24883 gcc_unreachable ();
24893 /* Subroutine of ix86_expand_builtin to take care of special insns
24894 with variable number of operands. */
24897 ix86_expand_special_args_builtin (const struct builtin_description *d,
24898 tree exp, rtx target)
24902 unsigned int i, nargs, arg_adjust, memory;
24906 enum machine_mode mode;
24908 enum insn_code icode = d->icode;
24909 bool last_arg_constant = false;
24910 const struct insn_data *insn_p = &insn_data[icode];
24911 enum machine_mode tmode = insn_p->operand[0].mode;
24912 enum { load, store } klass;
24914 switch ((enum ix86_special_builtin_type) d->flag)
24916 case VOID_FTYPE_VOID:
24917 emit_insn (GEN_FCN (icode) (target));
24919 case V2DI_FTYPE_PV2DI:
24920 case V32QI_FTYPE_PCCHAR:
24921 case V16QI_FTYPE_PCCHAR:
24922 case V8SF_FTYPE_PCV4SF:
24923 case V8SF_FTYPE_PCFLOAT:
24924 case V4SF_FTYPE_PCFLOAT:
24925 case V4DF_FTYPE_PCV2DF:
24926 case V4DF_FTYPE_PCDOUBLE:
24927 case V2DF_FTYPE_PCDOUBLE:
24932 case VOID_FTYPE_PV2SF_V4SF:
24933 case VOID_FTYPE_PV4DI_V4DI:
24934 case VOID_FTYPE_PV2DI_V2DI:
24935 case VOID_FTYPE_PCHAR_V32QI:
24936 case VOID_FTYPE_PCHAR_V16QI:
24937 case VOID_FTYPE_PFLOAT_V8SF:
24938 case VOID_FTYPE_PFLOAT_V4SF:
24939 case VOID_FTYPE_PDOUBLE_V4DF:
24940 case VOID_FTYPE_PDOUBLE_V2DF:
24941 case VOID_FTYPE_PDI_DI:
24942 case VOID_FTYPE_PINT_INT:
24945 /* Reserve memory operand for target. */
24946 memory = ARRAY_SIZE (args);
24948 case V4SF_FTYPE_V4SF_PCV2SF:
24949 case V2DF_FTYPE_V2DF_PCDOUBLE:
24954 case V8SF_FTYPE_PCV8SF_V8SF:
24955 case V4DF_FTYPE_PCV4DF_V4DF:
24956 case V4SF_FTYPE_PCV4SF_V4SF:
24957 case V2DF_FTYPE_PCV2DF_V2DF:
24962 case VOID_FTYPE_PV8SF_V8SF_V8SF:
24963 case VOID_FTYPE_PV4DF_V4DF_V4DF:
24964 case VOID_FTYPE_PV4SF_V4SF_V4SF:
24965 case VOID_FTYPE_PV2DF_V2DF_V2DF:
24968 /* Reserve memory operand for target. */
24969 memory = ARRAY_SIZE (args);
24972 gcc_unreachable ();
24975 gcc_assert (nargs <= ARRAY_SIZE (args));
24977 if (klass == store)
24979 arg = CALL_EXPR_ARG (exp, 0);
24980 op = expand_normal (arg);
24981 gcc_assert (target == 0);
24982 target = gen_rtx_MEM (tmode, copy_to_mode_reg (Pmode, op));
24990 || GET_MODE (target) != tmode
24991 || ! (*insn_p->operand[0].predicate) (target, tmode))
24992 target = gen_reg_rtx (tmode);
24995 for (i = 0; i < nargs; i++)
24997 enum machine_mode mode = insn_p->operand[i + 1].mode;
25000 arg = CALL_EXPR_ARG (exp, i + arg_adjust);
25001 op = expand_normal (arg);
25002 match = (*insn_p->operand[i + 1].predicate) (op, mode);
25004 if (last_arg_constant && (i + 1) == nargs)
25010 error ("the last argument must be an 8-bit immediate");
25018 /* This must be the memory operand. */
25019 op = gen_rtx_MEM (mode, copy_to_mode_reg (Pmode, op));
25020 gcc_assert (GET_MODE (op) == mode
25021 || GET_MODE (op) == VOIDmode);
25025 /* This must be register. */
25026 if (VECTOR_MODE_P (mode))
25027 op = safe_vector_operand (op, mode);
25029 gcc_assert (GET_MODE (op) == mode
25030 || GET_MODE (op) == VOIDmode);
25031 op = copy_to_mode_reg (mode, op);
25036 args[i].mode = mode;
25042 pat = GEN_FCN (icode) (target, args[0].op);
25045 pat = GEN_FCN (icode) (target, args[0].op, args[1].op);
25048 gcc_unreachable ();
25054 return klass == store ? 0 : target;
25057 /* Return the integer constant in ARG. Constrain it to be in the range
25058 of the subparts of VEC_TYPE; issue an error if not. */
25061 get_element_number (tree vec_type, tree arg)
25063 unsigned HOST_WIDE_INT elt, max = TYPE_VECTOR_SUBPARTS (vec_type) - 1;
25065 if (!host_integerp (arg, 1)
25066 || (elt = tree_low_cst (arg, 1), elt > max))
25068 error ("selector must be an integer constant in the range 0..%wi", max);
25075 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
25076 ix86_expand_vector_init. We DO have language-level syntax for this, in
25077 the form of (type){ init-list }. Except that since we can't place emms
25078 instructions from inside the compiler, we can't allow the use of MMX
25079 registers unless the user explicitly asks for it. So we do *not* define
25080 vec_set/vec_extract/vec_init patterns for MMX modes in mmx.md. Instead
25081 we have builtins invoked by mmintrin.h that gives us license to emit
25082 these sorts of instructions. */
25085 ix86_expand_vec_init_builtin (tree type, tree exp, rtx target)
25087 enum machine_mode tmode = TYPE_MODE (type);
25088 enum machine_mode inner_mode = GET_MODE_INNER (tmode);
25089 int i, n_elt = GET_MODE_NUNITS (tmode);
25090 rtvec v = rtvec_alloc (n_elt);
25092 gcc_assert (VECTOR_MODE_P (tmode));
25093 gcc_assert (call_expr_nargs (exp) == n_elt);
25095 for (i = 0; i < n_elt; ++i)
25097 rtx x = expand_normal (CALL_EXPR_ARG (exp, i));
25098 RTVEC_ELT (v, i) = gen_lowpart (inner_mode, x);
25101 if (!target || !register_operand (target, tmode))
25102 target = gen_reg_rtx (tmode);
25104 ix86_expand_vector_init (true, target, gen_rtx_PARALLEL (tmode, v));
25108 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
25109 ix86_expand_vector_extract. They would be redundant (for non-MMX) if we
25110 had a language-level syntax for referencing vector elements. */
25113 ix86_expand_vec_ext_builtin (tree exp, rtx target)
25115 enum machine_mode tmode, mode0;
25120 arg0 = CALL_EXPR_ARG (exp, 0);
25121 arg1 = CALL_EXPR_ARG (exp, 1);
25123 op0 = expand_normal (arg0);
25124 elt = get_element_number (TREE_TYPE (arg0), arg1);
25126 tmode = TYPE_MODE (TREE_TYPE (TREE_TYPE (arg0)));
25127 mode0 = TYPE_MODE (TREE_TYPE (arg0));
25128 gcc_assert (VECTOR_MODE_P (mode0));
25130 op0 = force_reg (mode0, op0);
25132 if (optimize || !target || !register_operand (target, tmode))
25133 target = gen_reg_rtx (tmode);
25135 ix86_expand_vector_extract (true, target, op0, elt);
25140 /* A subroutine of ix86_expand_builtin. These builtins are a wrapper around
25141 ix86_expand_vector_set. They would be redundant (for non-MMX) if we had
25142 a language-level syntax for referencing vector elements. */
25145 ix86_expand_vec_set_builtin (tree exp)
25147 enum machine_mode tmode, mode1;
25148 tree arg0, arg1, arg2;
25150 rtx op0, op1, target;
25152 arg0 = CALL_EXPR_ARG (exp, 0);
25153 arg1 = CALL_EXPR_ARG (exp, 1);
25154 arg2 = CALL_EXPR_ARG (exp, 2);
25156 tmode = TYPE_MODE (TREE_TYPE (arg0));
25157 mode1 = TYPE_MODE (TREE_TYPE (TREE_TYPE (arg0)));
25158 gcc_assert (VECTOR_MODE_P (tmode));
25160 op0 = expand_expr (arg0, NULL_RTX, tmode, EXPAND_NORMAL);
25161 op1 = expand_expr (arg1, NULL_RTX, mode1, EXPAND_NORMAL);
25162 elt = get_element_number (TREE_TYPE (arg0), arg2);
25164 if (GET_MODE (op1) != mode1 && GET_MODE (op1) != VOIDmode)
25165 op1 = convert_modes (mode1, GET_MODE (op1), op1, true);
25167 op0 = force_reg (tmode, op0);
25168 op1 = force_reg (mode1, op1);
25170 /* OP0 is the source of these builtin functions and shouldn't be
25171 modified. Create a copy, use it and return it as target. */
25172 target = gen_reg_rtx (tmode);
25173 emit_move_insn (target, op0);
25174 ix86_expand_vector_set (true, target, op1, elt);
25179 /* Expand an expression EXP that calls a built-in function,
25180 with result going to TARGET if that's convenient
25181 (and in mode MODE if that's convenient).
25182 SUBTARGET may be used as the target for computing one of EXP's operands.
25183 IGNORE is nonzero if the value is to be ignored. */
25186 ix86_expand_builtin (tree exp, rtx target, rtx subtarget ATTRIBUTE_UNUSED,
25187 enum machine_mode mode ATTRIBUTE_UNUSED,
25188 int ignore ATTRIBUTE_UNUSED)
25190 const struct builtin_description *d;
25192 enum insn_code icode;
25193 tree fndecl = TREE_OPERAND (CALL_EXPR_FN (exp), 0);
25194 tree arg0, arg1, arg2;
25195 rtx op0, op1, op2, pat;
25196 enum machine_mode mode0, mode1, mode2;
25197 unsigned int fcode = DECL_FUNCTION_CODE (fndecl);
25199 /* Determine whether the builtin function is available under the current ISA.
25200 Originally the builtin was not created if it wasn't applicable to the
25201 current ISA based on the command line switches. With function specific
25202 options, we need to check in the context of the function making the call
25203 whether it is supported. */
25204 if (ix86_builtins_isa[fcode].isa
25205 && !(ix86_builtins_isa[fcode].isa & ix86_isa_flags))
25207 char *opts = ix86_target_string (ix86_builtins_isa[fcode].isa, 0, NULL,
25208 NULL, NULL, false);
25211 error ("%qE needs unknown isa option", fndecl);
25214 gcc_assert (opts != NULL);
25215 error ("%qE needs isa option %s", fndecl, opts);
25223 case IX86_BUILTIN_MASKMOVQ:
25224 case IX86_BUILTIN_MASKMOVDQU:
25225 icode = (fcode == IX86_BUILTIN_MASKMOVQ
25226 ? CODE_FOR_mmx_maskmovq
25227 : CODE_FOR_sse2_maskmovdqu);
25228 /* Note the arg order is different from the operand order. */
25229 arg1 = CALL_EXPR_ARG (exp, 0);
25230 arg2 = CALL_EXPR_ARG (exp, 1);
25231 arg0 = CALL_EXPR_ARG (exp, 2);
25232 op0 = expand_normal (arg0);
25233 op1 = expand_normal (arg1);
25234 op2 = expand_normal (arg2);
25235 mode0 = insn_data[icode].operand[0].mode;
25236 mode1 = insn_data[icode].operand[1].mode;
25237 mode2 = insn_data[icode].operand[2].mode;
25239 op0 = force_reg (Pmode, op0);
25240 op0 = gen_rtx_MEM (mode1, op0);
25242 if (! (*insn_data[icode].operand[0].predicate) (op0, mode0))
25243 op0 = copy_to_mode_reg (mode0, op0);
25244 if (! (*insn_data[icode].operand[1].predicate) (op1, mode1))
25245 op1 = copy_to_mode_reg (mode1, op1);
25246 if (! (*insn_data[icode].operand[2].predicate) (op2, mode2))
25247 op2 = copy_to_mode_reg (mode2, op2);
25248 pat = GEN_FCN (icode) (op0, op1, op2);
25254 case IX86_BUILTIN_LDMXCSR:
25255 op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
25256 target = assign_386_stack_local (SImode, SLOT_VIRTUAL);
25257 emit_move_insn (target, op0);
25258 emit_insn (gen_sse_ldmxcsr (target));
25261 case IX86_BUILTIN_STMXCSR:
25262 target = assign_386_stack_local (SImode, SLOT_VIRTUAL);
25263 emit_insn (gen_sse_stmxcsr (target));
25264 return copy_to_mode_reg (SImode, target);
25266 case IX86_BUILTIN_CLFLUSH:
25267 arg0 = CALL_EXPR_ARG (exp, 0);
25268 op0 = expand_normal (arg0);
25269 icode = CODE_FOR_sse2_clflush;
25270 if (! (*insn_data[icode].operand[0].predicate) (op0, Pmode))
25271 op0 = copy_to_mode_reg (Pmode, op0);
25273 emit_insn (gen_sse2_clflush (op0));
25276 case IX86_BUILTIN_MONITOR:
25277 arg0 = CALL_EXPR_ARG (exp, 0);
25278 arg1 = CALL_EXPR_ARG (exp, 1);
25279 arg2 = CALL_EXPR_ARG (exp, 2);
25280 op0 = expand_normal (arg0);
25281 op1 = expand_normal (arg1);
25282 op2 = expand_normal (arg2);
25284 op0 = copy_to_mode_reg (Pmode, op0);
25286 op1 = copy_to_mode_reg (SImode, op1);
25288 op2 = copy_to_mode_reg (SImode, op2);
25289 emit_insn ((*ix86_gen_monitor) (op0, op1, op2));
25292 case IX86_BUILTIN_MWAIT:
25293 arg0 = CALL_EXPR_ARG (exp, 0);
25294 arg1 = CALL_EXPR_ARG (exp, 1);
25295 op0 = expand_normal (arg0);
25296 op1 = expand_normal (arg1);
25298 op0 = copy_to_mode_reg (SImode, op0);
25300 op1 = copy_to_mode_reg (SImode, op1);
25301 emit_insn (gen_sse3_mwait (op0, op1));
25304 case IX86_BUILTIN_VEC_INIT_V2SI:
25305 case IX86_BUILTIN_VEC_INIT_V4HI:
25306 case IX86_BUILTIN_VEC_INIT_V8QI:
25307 return ix86_expand_vec_init_builtin (TREE_TYPE (exp), exp, target);
25309 case IX86_BUILTIN_VEC_EXT_V2DF:
25310 case IX86_BUILTIN_VEC_EXT_V2DI:
25311 case IX86_BUILTIN_VEC_EXT_V4SF:
25312 case IX86_BUILTIN_VEC_EXT_V4SI:
25313 case IX86_BUILTIN_VEC_EXT_V8HI:
25314 case IX86_BUILTIN_VEC_EXT_V2SI:
25315 case IX86_BUILTIN_VEC_EXT_V4HI:
25316 case IX86_BUILTIN_VEC_EXT_V16QI:
25317 return ix86_expand_vec_ext_builtin (exp, target);
25319 case IX86_BUILTIN_VEC_SET_V2DI:
25320 case IX86_BUILTIN_VEC_SET_V4SF:
25321 case IX86_BUILTIN_VEC_SET_V4SI:
25322 case IX86_BUILTIN_VEC_SET_V8HI:
25323 case IX86_BUILTIN_VEC_SET_V4HI:
25324 case IX86_BUILTIN_VEC_SET_V16QI:
25325 return ix86_expand_vec_set_builtin (exp);
25327 case IX86_BUILTIN_INFQ:
25328 case IX86_BUILTIN_HUGE_VALQ:
25330 REAL_VALUE_TYPE inf;
25334 tmp = CONST_DOUBLE_FROM_REAL_VALUE (inf, mode);
25336 tmp = validize_mem (force_const_mem (mode, tmp));
25339 target = gen_reg_rtx (mode);
25341 emit_move_insn (target, tmp);
25349 for (i = 0, d = bdesc_special_args;
25350 i < ARRAY_SIZE (bdesc_special_args);
25352 if (d->code == fcode)
25353 return ix86_expand_special_args_builtin (d, exp, target);
25355 for (i = 0, d = bdesc_args;
25356 i < ARRAY_SIZE (bdesc_args);
25358 if (d->code == fcode)
25361 case IX86_BUILTIN_FABSQ:
25362 case IX86_BUILTIN_COPYSIGNQ:
25364 /* Emit a normal call if SSE2 isn't available. */
25365 return expand_call (exp, target, ignore);
25367 return ix86_expand_args_builtin (d, exp, target);
25370 for (i = 0, d = bdesc_comi; i < ARRAY_SIZE (bdesc_comi); i++, d++)
25371 if (d->code == fcode)
25372 return ix86_expand_sse_comi (d, exp, target);
25374 for (i = 0, d = bdesc_pcmpestr;
25375 i < ARRAY_SIZE (bdesc_pcmpestr);
25377 if (d->code == fcode)
25378 return ix86_expand_sse_pcmpestr (d, exp, target);
25380 for (i = 0, d = bdesc_pcmpistr;
25381 i < ARRAY_SIZE (bdesc_pcmpistr);
25383 if (d->code == fcode)
25384 return ix86_expand_sse_pcmpistr (d, exp, target);
25386 for (i = 0, d = bdesc_multi_arg; i < ARRAY_SIZE (bdesc_multi_arg); i++, d++)
25387 if (d->code == fcode)
25388 return ix86_expand_multi_arg_builtin (d->icode, exp, target,
25389 (enum multi_arg_type)d->flag,
25392 gcc_unreachable ();
25395 /* Returns a function decl for a vectorized version of the builtin function
25396 with builtin function code FN and the result vector type TYPE, or NULL_TREE
25397 if it is not available. */
25400 ix86_builtin_vectorized_function (unsigned int fn, tree type_out,
25403 enum machine_mode in_mode, out_mode;
25406 if (TREE_CODE (type_out) != VECTOR_TYPE
25407 || TREE_CODE (type_in) != VECTOR_TYPE)
25410 out_mode = TYPE_MODE (TREE_TYPE (type_out));
25411 out_n = TYPE_VECTOR_SUBPARTS (type_out);
25412 in_mode = TYPE_MODE (TREE_TYPE (type_in));
25413 in_n = TYPE_VECTOR_SUBPARTS (type_in);
25417 case BUILT_IN_SQRT:
25418 if (out_mode == DFmode && out_n == 2
25419 && in_mode == DFmode && in_n == 2)
25420 return ix86_builtins[IX86_BUILTIN_SQRTPD];
25423 case BUILT_IN_SQRTF:
25424 if (out_mode == SFmode && out_n == 4
25425 && in_mode == SFmode && in_n == 4)
25426 return ix86_builtins[IX86_BUILTIN_SQRTPS_NR];
25429 case BUILT_IN_LRINT:
25430 if (out_mode == SImode && out_n == 4
25431 && in_mode == DFmode && in_n == 2)
25432 return ix86_builtins[IX86_BUILTIN_VEC_PACK_SFIX];
25435 case BUILT_IN_LRINTF:
25436 if (out_mode == SImode && out_n == 4
25437 && in_mode == SFmode && in_n == 4)
25438 return ix86_builtins[IX86_BUILTIN_CVTPS2DQ];
25445 /* Dispatch to a handler for a vectorization library. */
25446 if (ix86_veclib_handler)
25447 return (*ix86_veclib_handler) ((enum built_in_function) fn, type_out,
25453 /* Handler for an SVML-style interface to
25454 a library with vectorized intrinsics. */
25457 ix86_veclibabi_svml (enum built_in_function fn, tree type_out, tree type_in)
25460 tree fntype, new_fndecl, args;
25463 enum machine_mode el_mode, in_mode;
25466 /* The SVML is suitable for unsafe math only. */
25467 if (!flag_unsafe_math_optimizations)
25470 el_mode = TYPE_MODE (TREE_TYPE (type_out));
25471 n = TYPE_VECTOR_SUBPARTS (type_out);
25472 in_mode = TYPE_MODE (TREE_TYPE (type_in));
25473 in_n = TYPE_VECTOR_SUBPARTS (type_in);
25474 if (el_mode != in_mode
25482 case BUILT_IN_LOG10:
25484 case BUILT_IN_TANH:
25486 case BUILT_IN_ATAN:
25487 case BUILT_IN_ATAN2:
25488 case BUILT_IN_ATANH:
25489 case BUILT_IN_CBRT:
25490 case BUILT_IN_SINH:
25492 case BUILT_IN_ASINH:
25493 case BUILT_IN_ASIN:
25494 case BUILT_IN_COSH:
25496 case BUILT_IN_ACOSH:
25497 case BUILT_IN_ACOS:
25498 if (el_mode != DFmode || n != 2)
25502 case BUILT_IN_EXPF:
25503 case BUILT_IN_LOGF:
25504 case BUILT_IN_LOG10F:
25505 case BUILT_IN_POWF:
25506 case BUILT_IN_TANHF:
25507 case BUILT_IN_TANF:
25508 case BUILT_IN_ATANF:
25509 case BUILT_IN_ATAN2F:
25510 case BUILT_IN_ATANHF:
25511 case BUILT_IN_CBRTF:
25512 case BUILT_IN_SINHF:
25513 case BUILT_IN_SINF:
25514 case BUILT_IN_ASINHF:
25515 case BUILT_IN_ASINF:
25516 case BUILT_IN_COSHF:
25517 case BUILT_IN_COSF:
25518 case BUILT_IN_ACOSHF:
25519 case BUILT_IN_ACOSF:
25520 if (el_mode != SFmode || n != 4)
25528 bname = IDENTIFIER_POINTER (DECL_NAME (implicit_built_in_decls[fn]));
25530 if (fn == BUILT_IN_LOGF)
25531 strcpy (name, "vmlsLn4");
25532 else if (fn == BUILT_IN_LOG)
25533 strcpy (name, "vmldLn2");
25536 sprintf (name, "vmls%s", bname+10);
25537 name[strlen (name)-1] = '4';
25540 sprintf (name, "vmld%s2", bname+10);
25542 /* Convert to uppercase. */
25546 for (args = DECL_ARGUMENTS (implicit_built_in_decls[fn]); args;
25547 args = TREE_CHAIN (args))
25551 fntype = build_function_type_list (type_out, type_in, NULL);
25553 fntype = build_function_type_list (type_out, type_in, type_in, NULL);
25555 /* Build a function declaration for the vectorized function. */
25556 new_fndecl = build_decl (FUNCTION_DECL, get_identifier (name), fntype);
25557 TREE_PUBLIC (new_fndecl) = 1;
25558 DECL_EXTERNAL (new_fndecl) = 1;
25559 DECL_IS_NOVOPS (new_fndecl) = 1;
25560 TREE_READONLY (new_fndecl) = 1;
25565 /* Handler for an ACML-style interface to
25566 a library with vectorized intrinsics. */
25569 ix86_veclibabi_acml (enum built_in_function fn, tree type_out, tree type_in)
25571 char name[20] = "__vr.._";
25572 tree fntype, new_fndecl, args;
25575 enum machine_mode el_mode, in_mode;
25578 /* The ACML is 64bits only and suitable for unsafe math only as
25579 it does not correctly support parts of IEEE with the required
25580 precision such as denormals. */
25582 || !flag_unsafe_math_optimizations)
25585 el_mode = TYPE_MODE (TREE_TYPE (type_out));
25586 n = TYPE_VECTOR_SUBPARTS (type_out);
25587 in_mode = TYPE_MODE (TREE_TYPE (type_in));
25588 in_n = TYPE_VECTOR_SUBPARTS (type_in);
25589 if (el_mode != in_mode
25599 case BUILT_IN_LOG2:
25600 case BUILT_IN_LOG10:
25603 if (el_mode != DFmode
25608 case BUILT_IN_SINF:
25609 case BUILT_IN_COSF:
25610 case BUILT_IN_EXPF:
25611 case BUILT_IN_POWF:
25612 case BUILT_IN_LOGF:
25613 case BUILT_IN_LOG2F:
25614 case BUILT_IN_LOG10F:
25617 if (el_mode != SFmode
25626 bname = IDENTIFIER_POINTER (DECL_NAME (implicit_built_in_decls[fn]));
25627 sprintf (name + 7, "%s", bname+10);
25630 for (args = DECL_ARGUMENTS (implicit_built_in_decls[fn]); args;
25631 args = TREE_CHAIN (args))
25635 fntype = build_function_type_list (type_out, type_in, NULL);
25637 fntype = build_function_type_list (type_out, type_in, type_in, NULL);
25639 /* Build a function declaration for the vectorized function. */
25640 new_fndecl = build_decl (FUNCTION_DECL, get_identifier (name), fntype);
25641 TREE_PUBLIC (new_fndecl) = 1;
25642 DECL_EXTERNAL (new_fndecl) = 1;
25643 DECL_IS_NOVOPS (new_fndecl) = 1;
25644 TREE_READONLY (new_fndecl) = 1;
25650 /* Returns a decl of a function that implements conversion of an integer vector
25651 into a floating-point vector, or vice-versa. TYPE is the type of the integer
25652 side of the conversion.
25653 Return NULL_TREE if it is not available. */
25656 ix86_vectorize_builtin_conversion (unsigned int code, tree type)
25658 if (TREE_CODE (type) != VECTOR_TYPE
25659 /* There are only conversions from/to signed integers. */
25660 || TYPE_UNSIGNED (TREE_TYPE (type)))
25666 switch (TYPE_MODE (type))
25669 return ix86_builtins[IX86_BUILTIN_CVTDQ2PS];
25674 case FIX_TRUNC_EXPR:
25675 switch (TYPE_MODE (type))
25678 return ix86_builtins[IX86_BUILTIN_CVTTPS2DQ];
25688 /* Returns a code for a target-specific builtin that implements
25689 reciprocal of the function, or NULL_TREE if not available. */
25692 ix86_builtin_reciprocal (unsigned int fn, bool md_fn,
25693 bool sqrt ATTRIBUTE_UNUSED)
25695 if (! (TARGET_SSE_MATH && TARGET_RECIP && !optimize_insn_for_size_p ()
25696 && flag_finite_math_only && !flag_trapping_math
25697 && flag_unsafe_math_optimizations))
25701 /* Machine dependent builtins. */
25704 /* Vectorized version of sqrt to rsqrt conversion. */
25705 case IX86_BUILTIN_SQRTPS_NR:
25706 return ix86_builtins[IX86_BUILTIN_RSQRTPS_NR];
25712 /* Normal builtins. */
25715 /* Sqrt to rsqrt conversion. */
25716 case BUILT_IN_SQRTF:
25717 return ix86_builtins[IX86_BUILTIN_RSQRTF];
25724 /* Store OPERAND to the memory after reload is completed. This means
25725 that we can't easily use assign_stack_local. */
25727 ix86_force_to_memory (enum machine_mode mode, rtx operand)
25731 gcc_assert (reload_completed);
25732 if (!TARGET_64BIT_MS_ABI && TARGET_RED_ZONE)
25734 result = gen_rtx_MEM (mode,
25735 gen_rtx_PLUS (Pmode,
25737 GEN_INT (-RED_ZONE_SIZE)));
25738 emit_move_insn (result, operand);
25740 else if ((TARGET_64BIT_MS_ABI || !TARGET_RED_ZONE) && TARGET_64BIT)
25746 operand = gen_lowpart (DImode, operand);
25750 gen_rtx_SET (VOIDmode,
25751 gen_rtx_MEM (DImode,
25752 gen_rtx_PRE_DEC (DImode,
25753 stack_pointer_rtx)),
25757 gcc_unreachable ();
25759 result = gen_rtx_MEM (mode, stack_pointer_rtx);
25768 split_di (&operand, 1, operands, operands + 1);
25770 gen_rtx_SET (VOIDmode,
25771 gen_rtx_MEM (SImode,
25772 gen_rtx_PRE_DEC (Pmode,
25773 stack_pointer_rtx)),
25776 gen_rtx_SET (VOIDmode,
25777 gen_rtx_MEM (SImode,
25778 gen_rtx_PRE_DEC (Pmode,
25779 stack_pointer_rtx)),
25784 /* Store HImodes as SImodes. */
25785 operand = gen_lowpart (SImode, operand);
25789 gen_rtx_SET (VOIDmode,
25790 gen_rtx_MEM (GET_MODE (operand),
25791 gen_rtx_PRE_DEC (SImode,
25792 stack_pointer_rtx)),
25796 gcc_unreachable ();
25798 result = gen_rtx_MEM (mode, stack_pointer_rtx);
25803 /* Free operand from the memory. */
25805 ix86_free_from_memory (enum machine_mode mode)
25807 if (!TARGET_RED_ZONE || TARGET_64BIT_MS_ABI)
25811 if (mode == DImode || TARGET_64BIT)
25815 /* Use LEA to deallocate stack space. In peephole2 it will be converted
25816 to pop or add instruction if registers are available. */
25817 emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx,
25818 gen_rtx_PLUS (Pmode, stack_pointer_rtx,
25823 /* Put float CONST_DOUBLE in the constant pool instead of fp regs.
25824 QImode must go into class Q_REGS.
25825 Narrow ALL_REGS to GENERAL_REGS. This supports allowing movsf and
25826 movdf to do mem-to-mem moves through integer regs. */
25828 ix86_preferred_reload_class (rtx x, enum reg_class regclass)
25830 enum machine_mode mode = GET_MODE (x);
25832 /* We're only allowed to return a subclass of CLASS. Many of the
25833 following checks fail for NO_REGS, so eliminate that early. */
25834 if (regclass == NO_REGS)
25837 /* All classes can load zeros. */
25838 if (x == CONST0_RTX (mode))
25841 /* Force constants into memory if we are loading a (nonzero) constant into
25842 an MMX or SSE register. This is because there are no MMX/SSE instructions
25843 to load from a constant. */
25845 && (MAYBE_MMX_CLASS_P (regclass) || MAYBE_SSE_CLASS_P (regclass)))
25848 /* Prefer SSE regs only, if we can use them for math. */
25849 if (TARGET_SSE_MATH && !TARGET_MIX_SSE_I387 && SSE_FLOAT_MODE_P (mode))
25850 return SSE_CLASS_P (regclass) ? regclass : NO_REGS;
25852 /* Floating-point constants need more complex checks. */
25853 if (GET_CODE (x) == CONST_DOUBLE && GET_MODE (x) != VOIDmode)
25855 /* General regs can load everything. */
25856 if (reg_class_subset_p (regclass, GENERAL_REGS))
25859 /* Floats can load 0 and 1 plus some others. Note that we eliminated
25860 zero above. We only want to wind up preferring 80387 registers if
25861 we plan on doing computation with them. */
25863 && standard_80387_constant_p (x))
25865 /* Limit class to non-sse. */
25866 if (regclass == FLOAT_SSE_REGS)
25868 if (regclass == FP_TOP_SSE_REGS)
25870 if (regclass == FP_SECOND_SSE_REGS)
25871 return FP_SECOND_REG;
25872 if (regclass == FLOAT_INT_REGS || regclass == FLOAT_REGS)
25879 /* Generally when we see PLUS here, it's the function invariant
25880 (plus soft-fp const_int). Which can only be computed into general
25882 if (GET_CODE (x) == PLUS)
25883 return reg_class_subset_p (regclass, GENERAL_REGS) ? regclass : NO_REGS;
25885 /* QImode constants are easy to load, but non-constant QImode data
25886 must go into Q_REGS. */
25887 if (GET_MODE (x) == QImode && !CONSTANT_P (x))
25889 if (reg_class_subset_p (regclass, Q_REGS))
25891 if (reg_class_subset_p (Q_REGS, regclass))
25899 /* Discourage putting floating-point values in SSE registers unless
25900 SSE math is being used, and likewise for the 387 registers. */
25902 ix86_preferred_output_reload_class (rtx x, enum reg_class regclass)
25904 enum machine_mode mode = GET_MODE (x);
25906 /* Restrict the output reload class to the register bank that we are doing
25907 math on. If we would like not to return a subset of CLASS, reject this
25908 alternative: if reload cannot do this, it will still use its choice. */
25909 mode = GET_MODE (x);
25910 if (TARGET_SSE_MATH && SSE_FLOAT_MODE_P (mode))
25911 return MAYBE_SSE_CLASS_P (regclass) ? SSE_REGS : NO_REGS;
25913 if (X87_FLOAT_MODE_P (mode))
25915 if (regclass == FP_TOP_SSE_REGS)
25917 else if (regclass == FP_SECOND_SSE_REGS)
25918 return FP_SECOND_REG;
25920 return FLOAT_CLASS_P (regclass) ? regclass : NO_REGS;
25926 static enum reg_class
25927 ix86_secondary_reload (bool in_p, rtx x, enum reg_class rclass,
25928 enum machine_mode mode,
25929 secondary_reload_info *sri ATTRIBUTE_UNUSED)
25931 /* QImode spills from non-QI registers require
25932 intermediate register on 32bit targets. */
25933 if (!in_p && mode == QImode && !TARGET_64BIT
25934 && (rclass == GENERAL_REGS
25935 || rclass == LEGACY_REGS
25936 || rclass == INDEX_REGS))
25945 if (regno >= FIRST_PSEUDO_REGISTER || GET_CODE (x) == SUBREG)
25946 regno = true_regnum (x);
25948 /* Return Q_REGS if the operand is in memory. */
25956 /* If we are copying between general and FP registers, we need a memory
25957 location. The same is true for SSE and MMX registers.
25959 To optimize register_move_cost performance, allow inline variant.
25961 The macro can't work reliably when one of the CLASSES is class containing
25962 registers from multiple units (SSE, MMX, integer). We avoid this by never
25963 combining those units in single alternative in the machine description.
25964 Ensure that this constraint holds to avoid unexpected surprises.
25966 When STRICT is false, we are being called from REGISTER_MOVE_COST, so do not
25967 enforce these sanity checks. */
25970 inline_secondary_memory_needed (enum reg_class class1, enum reg_class class2,
25971 enum machine_mode mode, int strict)
25973 if (MAYBE_FLOAT_CLASS_P (class1) != FLOAT_CLASS_P (class1)
25974 || MAYBE_FLOAT_CLASS_P (class2) != FLOAT_CLASS_P (class2)
25975 || MAYBE_SSE_CLASS_P (class1) != SSE_CLASS_P (class1)
25976 || MAYBE_SSE_CLASS_P (class2) != SSE_CLASS_P (class2)
25977 || MAYBE_MMX_CLASS_P (class1) != MMX_CLASS_P (class1)
25978 || MAYBE_MMX_CLASS_P (class2) != MMX_CLASS_P (class2))
25980 gcc_assert (!strict);
25984 if (FLOAT_CLASS_P (class1) != FLOAT_CLASS_P (class2))
25987 /* ??? This is a lie. We do have moves between mmx/general, and for
25988 mmx/sse2. But by saying we need secondary memory we discourage the
25989 register allocator from using the mmx registers unless needed. */
25990 if (MMX_CLASS_P (class1) != MMX_CLASS_P (class2))
25993 if (SSE_CLASS_P (class1) != SSE_CLASS_P (class2))
25995 /* SSE1 doesn't have any direct moves from other classes. */
25999 /* If the target says that inter-unit moves are more expensive
26000 than moving through memory, then don't generate them. */
26001 if (!TARGET_INTER_UNIT_MOVES)
26004 /* Between SSE and general, we have moves no larger than word size. */
26005 if (GET_MODE_SIZE (mode) > UNITS_PER_WORD)
26013 ix86_secondary_memory_needed (enum reg_class class1, enum reg_class class2,
26014 enum machine_mode mode, int strict)
26016 return inline_secondary_memory_needed (class1, class2, mode, strict);
26019 /* Return true if the registers in CLASS cannot represent the change from
26020 modes FROM to TO. */
26023 ix86_cannot_change_mode_class (enum machine_mode from, enum machine_mode to,
26024 enum reg_class regclass)
26029 /* x87 registers can't do subreg at all, as all values are reformatted
26030 to extended precision. */
26031 if (MAYBE_FLOAT_CLASS_P (regclass))
26034 if (MAYBE_SSE_CLASS_P (regclass) || MAYBE_MMX_CLASS_P (regclass))
26036 /* Vector registers do not support QI or HImode loads. If we don't
26037 disallow a change to these modes, reload will assume it's ok to
26038 drop the subreg from (subreg:SI (reg:HI 100) 0). This affects
26039 the vec_dupv4hi pattern. */
26040 if (GET_MODE_SIZE (from) < 4)
26043 /* Vector registers do not support subreg with nonzero offsets, which
26044 are otherwise valid for integer registers. Since we can't see
26045 whether we have a nonzero offset from here, prohibit all
26046 nonparadoxical subregs changing size. */
26047 if (GET_MODE_SIZE (to) < GET_MODE_SIZE (from))
26054 /* Return the cost of moving data of mode M between a
26055 register and memory. A value of 2 is the default; this cost is
26056 relative to those in `REGISTER_MOVE_COST'.
26058 This function is used extensively by register_move_cost that is used to
26059 build tables at startup. Make it inline in this case.
26060 When IN is 2, return maximum of in and out move cost.
26062 If moving between registers and memory is more expensive than
26063 between two registers, you should define this macro to express the
26066 Model also increased moving costs of QImode registers in non
26070 inline_memory_move_cost (enum machine_mode mode, enum reg_class regclass,
26074 if (FLOAT_CLASS_P (regclass))
26092 return MAX (ix86_cost->fp_load [index], ix86_cost->fp_store [index]);
26093 return in ? ix86_cost->fp_load [index] : ix86_cost->fp_store [index];
26095 if (SSE_CLASS_P (regclass))
26098 switch (GET_MODE_SIZE (mode))
26113 return MAX (ix86_cost->sse_load [index], ix86_cost->sse_store [index]);
26114 return in ? ix86_cost->sse_load [index] : ix86_cost->sse_store [index];
26116 if (MMX_CLASS_P (regclass))
26119 switch (GET_MODE_SIZE (mode))
26131 return MAX (ix86_cost->mmx_load [index], ix86_cost->mmx_store [index]);
26132 return in ? ix86_cost->mmx_load [index] : ix86_cost->mmx_store [index];
26134 switch (GET_MODE_SIZE (mode))
26137 if (Q_CLASS_P (regclass) || TARGET_64BIT)
26140 return ix86_cost->int_store[0];
26141 if (TARGET_PARTIAL_REG_DEPENDENCY
26142 && optimize_function_for_speed_p (cfun))
26143 cost = ix86_cost->movzbl_load;
26145 cost = ix86_cost->int_load[0];
26147 return MAX (cost, ix86_cost->int_store[0]);
26153 return MAX (ix86_cost->movzbl_load, ix86_cost->int_store[0] + 4);
26155 return ix86_cost->movzbl_load;
26157 return ix86_cost->int_store[0] + 4;
26162 return MAX (ix86_cost->int_load[1], ix86_cost->int_store[1]);
26163 return in ? ix86_cost->int_load[1] : ix86_cost->int_store[1];
26165 /* Compute number of 32bit moves needed. TFmode is moved as XFmode. */
26166 if (mode == TFmode)
26169 cost = MAX (ix86_cost->int_load[2] , ix86_cost->int_store[2]);
26171 cost = ix86_cost->int_load[2];
26173 cost = ix86_cost->int_store[2];
26174 return (cost * (((int) GET_MODE_SIZE (mode)
26175 + UNITS_PER_WORD - 1) / UNITS_PER_WORD));
26180 ix86_memory_move_cost (enum machine_mode mode, enum reg_class regclass, int in)
26182 return inline_memory_move_cost (mode, regclass, in);
26186 /* Return the cost of moving data from a register in class CLASS1 to
26187 one in class CLASS2.
26189 It is not required that the cost always equal 2 when FROM is the same as TO;
26190 on some machines it is expensive to move between registers if they are not
26191 general registers. */
26194 ix86_register_move_cost (enum machine_mode mode, enum reg_class class1,
26195 enum reg_class class2)
26197 /* In case we require secondary memory, compute cost of the store followed
26198 by load. In order to avoid bad register allocation choices, we need
26199 for this to be *at least* as high as the symmetric MEMORY_MOVE_COST. */
26201 if (inline_secondary_memory_needed (class1, class2, mode, 0))
26205 cost += inline_memory_move_cost (mode, class1, 2);
26206 cost += inline_memory_move_cost (mode, class2, 2);
26208 /* In case of copying from general_purpose_register we may emit multiple
26209 stores followed by single load causing memory size mismatch stall.
26210 Count this as arbitrarily high cost of 20. */
26211 if (CLASS_MAX_NREGS (class1, mode) > CLASS_MAX_NREGS (class2, mode))
26214 /* In the case of FP/MMX moves, the registers actually overlap, and we
26215 have to switch modes in order to treat them differently. */
26216 if ((MMX_CLASS_P (class1) && MAYBE_FLOAT_CLASS_P (class2))
26217 || (MMX_CLASS_P (class2) && MAYBE_FLOAT_CLASS_P (class1)))
26223 /* Moves between SSE/MMX and integer unit are expensive. */
26224 if (MMX_CLASS_P (class1) != MMX_CLASS_P (class2)
26225 || SSE_CLASS_P (class1) != SSE_CLASS_P (class2))
26227 /* ??? By keeping returned value relatively high, we limit the number
26228 of moves between integer and MMX/SSE registers for all targets.
26229 Additionally, high value prevents problem with x86_modes_tieable_p(),
26230 where integer modes in MMX/SSE registers are not tieable
26231 because of missing QImode and HImode moves to, from or between
26232 MMX/SSE registers. */
26233 return MAX (8, ix86_cost->mmxsse_to_integer);
26235 if (MAYBE_FLOAT_CLASS_P (class1))
26236 return ix86_cost->fp_move;
26237 if (MAYBE_SSE_CLASS_P (class1))
26238 return ix86_cost->sse_move;
26239 if (MAYBE_MMX_CLASS_P (class1))
26240 return ix86_cost->mmx_move;
26244 /* Return 1 if hard register REGNO can hold a value of machine-mode MODE. */
26247 ix86_hard_regno_mode_ok (int regno, enum machine_mode mode)
26249 /* Flags and only flags can only hold CCmode values. */
26250 if (CC_REGNO_P (regno))
26251 return GET_MODE_CLASS (mode) == MODE_CC;
26252 if (GET_MODE_CLASS (mode) == MODE_CC
26253 || GET_MODE_CLASS (mode) == MODE_RANDOM
26254 || GET_MODE_CLASS (mode) == MODE_PARTIAL_INT)
26256 if (FP_REGNO_P (regno))
26257 return VALID_FP_MODE_P (mode);
26258 if (SSE_REGNO_P (regno))
26260 /* We implement the move patterns for all vector modes into and
26261 out of SSE registers, even when no operation instructions
26262 are available. OImode move is available only when AVX is
26264 return ((TARGET_AVX && mode == OImode)
26265 || VALID_AVX256_REG_MODE (mode)
26266 || VALID_SSE_REG_MODE (mode)
26267 || VALID_SSE2_REG_MODE (mode)
26268 || VALID_MMX_REG_MODE (mode)
26269 || VALID_MMX_REG_MODE_3DNOW (mode));
26271 if (MMX_REGNO_P (regno))
26273 /* We implement the move patterns for 3DNOW modes even in MMX mode,
26274 so if the register is available at all, then we can move data of
26275 the given mode into or out of it. */
26276 return (VALID_MMX_REG_MODE (mode)
26277 || VALID_MMX_REG_MODE_3DNOW (mode));
26280 if (mode == QImode)
26282 /* Take care for QImode values - they can be in non-QI regs,
26283 but then they do cause partial register stalls. */
26284 if (regno <= BX_REG || TARGET_64BIT)
26286 if (!TARGET_PARTIAL_REG_STALL)
26288 return reload_in_progress || reload_completed;
26290 /* We handle both integer and floats in the general purpose registers. */
26291 else if (VALID_INT_MODE_P (mode))
26293 else if (VALID_FP_MODE_P (mode))
26295 else if (VALID_DFP_MODE_P (mode))
26297 /* Lots of MMX code casts 8 byte vector modes to DImode. If we then go
26298 on to use that value in smaller contexts, this can easily force a
26299 pseudo to be allocated to GENERAL_REGS. Since this is no worse than
26300 supporting DImode, allow it. */
26301 else if (VALID_MMX_REG_MODE_3DNOW (mode) || VALID_MMX_REG_MODE (mode))
26307 /* A subroutine of ix86_modes_tieable_p. Return true if MODE is a
26308 tieable integer mode. */
26311 ix86_tieable_integer_mode_p (enum machine_mode mode)
26320 return TARGET_64BIT || !TARGET_PARTIAL_REG_STALL;
26323 return TARGET_64BIT;
26330 /* Return true if MODE1 is accessible in a register that can hold MODE2
26331 without copying. That is, all register classes that can hold MODE2
26332 can also hold MODE1. */
26335 ix86_modes_tieable_p (enum machine_mode mode1, enum machine_mode mode2)
26337 if (mode1 == mode2)
26340 if (ix86_tieable_integer_mode_p (mode1)
26341 && ix86_tieable_integer_mode_p (mode2))
26344 /* MODE2 being XFmode implies fp stack or general regs, which means we
26345 can tie any smaller floating point modes to it. Note that we do not
26346 tie this with TFmode. */
26347 if (mode2 == XFmode)
26348 return mode1 == SFmode || mode1 == DFmode;
26350 /* MODE2 being DFmode implies fp stack, general or sse regs, which means
26351 that we can tie it with SFmode. */
26352 if (mode2 == DFmode)
26353 return mode1 == SFmode;
26355 /* If MODE2 is only appropriate for an SSE register, then tie with
26356 any other mode acceptable to SSE registers. */
26357 if (GET_MODE_SIZE (mode2) == 16
26358 && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode2))
26359 return (GET_MODE_SIZE (mode1) == 16
26360 && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode1));
26362 /* If MODE2 is appropriate for an MMX register, then tie
26363 with any other mode acceptable to MMX registers. */
26364 if (GET_MODE_SIZE (mode2) == 8
26365 && ix86_hard_regno_mode_ok (FIRST_MMX_REG, mode2))
26366 return (GET_MODE_SIZE (mode1) == 8
26367 && ix86_hard_regno_mode_ok (FIRST_MMX_REG, mode1));
26372 /* Compute a (partial) cost for rtx X. Return true if the complete
26373 cost has been computed, and false if subexpressions should be
26374 scanned. In either case, *TOTAL contains the cost result. */
26377 ix86_rtx_costs (rtx x, int code, int outer_code_i, int *total, bool speed)
26379 enum rtx_code outer_code = (enum rtx_code) outer_code_i;
26380 enum machine_mode mode = GET_MODE (x);
26381 const struct processor_costs *cost = speed ? ix86_cost : &ix86_size_cost;
26389 if (TARGET_64BIT && !x86_64_immediate_operand (x, VOIDmode))
26391 else if (TARGET_64BIT && !x86_64_zext_immediate_operand (x, VOIDmode))
26393 else if (flag_pic && SYMBOLIC_CONST (x)
26395 || (!GET_CODE (x) != LABEL_REF
26396 && (GET_CODE (x) != SYMBOL_REF
26397 || !SYMBOL_REF_LOCAL_P (x)))))
26404 if (mode == VOIDmode)
26407 switch (standard_80387_constant_p (x))
26412 default: /* Other constants */
26417 /* Start with (MEM (SYMBOL_REF)), since that's where
26418 it'll probably end up. Add a penalty for size. */
26419 *total = (COSTS_N_INSNS (1)
26420 + (flag_pic != 0 && !TARGET_64BIT)
26421 + (mode == SFmode ? 0 : mode == DFmode ? 1 : 2));
26427 /* The zero extensions is often completely free on x86_64, so make
26428 it as cheap as possible. */
26429 if (TARGET_64BIT && mode == DImode
26430 && GET_MODE (XEXP (x, 0)) == SImode)
26432 else if (TARGET_ZERO_EXTEND_WITH_AND)
26433 *total = cost->add;
26435 *total = cost->movzx;
26439 *total = cost->movsx;
26443 if (CONST_INT_P (XEXP (x, 1))
26444 && (GET_MODE (XEXP (x, 0)) != DImode || TARGET_64BIT))
26446 HOST_WIDE_INT value = INTVAL (XEXP (x, 1));
26449 *total = cost->add;
26452 if ((value == 2 || value == 3)
26453 && cost->lea <= cost->shift_const)
26455 *total = cost->lea;
26465 if (!TARGET_64BIT && GET_MODE (XEXP (x, 0)) == DImode)
26467 if (CONST_INT_P (XEXP (x, 1)))
26469 if (INTVAL (XEXP (x, 1)) > 32)
26470 *total = cost->shift_const + COSTS_N_INSNS (2);
26472 *total = cost->shift_const * 2;
26476 if (GET_CODE (XEXP (x, 1)) == AND)
26477 *total = cost->shift_var * 2;
26479 *total = cost->shift_var * 6 + COSTS_N_INSNS (2);
26484 if (CONST_INT_P (XEXP (x, 1)))
26485 *total = cost->shift_const;
26487 *total = cost->shift_var;
26492 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
26494 /* ??? SSE scalar cost should be used here. */
26495 *total = cost->fmul;
26498 else if (X87_FLOAT_MODE_P (mode))
26500 *total = cost->fmul;
26503 else if (FLOAT_MODE_P (mode))
26505 /* ??? SSE vector cost should be used here. */
26506 *total = cost->fmul;
26511 rtx op0 = XEXP (x, 0);
26512 rtx op1 = XEXP (x, 1);
26514 if (CONST_INT_P (XEXP (x, 1)))
26516 unsigned HOST_WIDE_INT value = INTVAL (XEXP (x, 1));
26517 for (nbits = 0; value != 0; value &= value - 1)
26521 /* This is arbitrary. */
26524 /* Compute costs correctly for widening multiplication. */
26525 if ((GET_CODE (op0) == SIGN_EXTEND || GET_CODE (op0) == ZERO_EXTEND)
26526 && GET_MODE_SIZE (GET_MODE (XEXP (op0, 0))) * 2
26527 == GET_MODE_SIZE (mode))
26529 int is_mulwiden = 0;
26530 enum machine_mode inner_mode = GET_MODE (op0);
26532 if (GET_CODE (op0) == GET_CODE (op1))
26533 is_mulwiden = 1, op1 = XEXP (op1, 0);
26534 else if (CONST_INT_P (op1))
26536 if (GET_CODE (op0) == SIGN_EXTEND)
26537 is_mulwiden = trunc_int_for_mode (INTVAL (op1), inner_mode)
26540 is_mulwiden = !(INTVAL (op1) & ~GET_MODE_MASK (inner_mode));
26544 op0 = XEXP (op0, 0), mode = GET_MODE (op0);
26547 *total = (cost->mult_init[MODE_INDEX (mode)]
26548 + nbits * cost->mult_bit
26549 + rtx_cost (op0, outer_code, speed) + rtx_cost (op1, outer_code, speed));
26558 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
26559 /* ??? SSE cost should be used here. */
26560 *total = cost->fdiv;
26561 else if (X87_FLOAT_MODE_P (mode))
26562 *total = cost->fdiv;
26563 else if (FLOAT_MODE_P (mode))
26564 /* ??? SSE vector cost should be used here. */
26565 *total = cost->fdiv;
26567 *total = cost->divide[MODE_INDEX (mode)];
26571 if (GET_MODE_CLASS (mode) == MODE_INT
26572 && GET_MODE_BITSIZE (mode) <= GET_MODE_BITSIZE (Pmode))
26574 if (GET_CODE (XEXP (x, 0)) == PLUS
26575 && GET_CODE (XEXP (XEXP (x, 0), 0)) == MULT
26576 && CONST_INT_P (XEXP (XEXP (XEXP (x, 0), 0), 1))
26577 && CONSTANT_P (XEXP (x, 1)))
26579 HOST_WIDE_INT val = INTVAL (XEXP (XEXP (XEXP (x, 0), 0), 1));
26580 if (val == 2 || val == 4 || val == 8)
26582 *total = cost->lea;
26583 *total += rtx_cost (XEXP (XEXP (x, 0), 1), outer_code, speed);
26584 *total += rtx_cost (XEXP (XEXP (XEXP (x, 0), 0), 0),
26585 outer_code, speed);
26586 *total += rtx_cost (XEXP (x, 1), outer_code, speed);
26590 else if (GET_CODE (XEXP (x, 0)) == MULT
26591 && CONST_INT_P (XEXP (XEXP (x, 0), 1)))
26593 HOST_WIDE_INT val = INTVAL (XEXP (XEXP (x, 0), 1));
26594 if (val == 2 || val == 4 || val == 8)
26596 *total = cost->lea;
26597 *total += rtx_cost (XEXP (XEXP (x, 0), 0), outer_code, speed);
26598 *total += rtx_cost (XEXP (x, 1), outer_code, speed);
26602 else if (GET_CODE (XEXP (x, 0)) == PLUS)
26604 *total = cost->lea;
26605 *total += rtx_cost (XEXP (XEXP (x, 0), 0), outer_code, speed);
26606 *total += rtx_cost (XEXP (XEXP (x, 0), 1), outer_code, speed);
26607 *total += rtx_cost (XEXP (x, 1), outer_code, speed);
26614 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
26616 /* ??? SSE cost should be used here. */
26617 *total = cost->fadd;
26620 else if (X87_FLOAT_MODE_P (mode))
26622 *total = cost->fadd;
26625 else if (FLOAT_MODE_P (mode))
26627 /* ??? SSE vector cost should be used here. */
26628 *total = cost->fadd;
26636 if (!TARGET_64BIT && mode == DImode)
26638 *total = (cost->add * 2
26639 + (rtx_cost (XEXP (x, 0), outer_code, speed)
26640 << (GET_MODE (XEXP (x, 0)) != DImode))
26641 + (rtx_cost (XEXP (x, 1), outer_code, speed)
26642 << (GET_MODE (XEXP (x, 1)) != DImode)));
26648 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
26650 /* ??? SSE cost should be used here. */
26651 *total = cost->fchs;
26654 else if (X87_FLOAT_MODE_P (mode))
26656 *total = cost->fchs;
26659 else if (FLOAT_MODE_P (mode))
26661 /* ??? SSE vector cost should be used here. */
26662 *total = cost->fchs;
26668 if (!TARGET_64BIT && mode == DImode)
26669 *total = cost->add * 2;
26671 *total = cost->add;
26675 if (GET_CODE (XEXP (x, 0)) == ZERO_EXTRACT
26676 && XEXP (XEXP (x, 0), 1) == const1_rtx
26677 && CONST_INT_P (XEXP (XEXP (x, 0), 2))
26678 && XEXP (x, 1) == const0_rtx)
26680 /* This kind of construct is implemented using test[bwl].
26681 Treat it as if we had an AND. */
26682 *total = (cost->add
26683 + rtx_cost (XEXP (XEXP (x, 0), 0), outer_code, speed)
26684 + rtx_cost (const1_rtx, outer_code, speed));
26690 if (!(SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH))
26695 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
26696 /* ??? SSE cost should be used here. */
26697 *total = cost->fabs;
26698 else if (X87_FLOAT_MODE_P (mode))
26699 *total = cost->fabs;
26700 else if (FLOAT_MODE_P (mode))
26701 /* ??? SSE vector cost should be used here. */
26702 *total = cost->fabs;
26706 if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)
26707 /* ??? SSE cost should be used here. */
26708 *total = cost->fsqrt;
26709 else if (X87_FLOAT_MODE_P (mode))
26710 *total = cost->fsqrt;
26711 else if (FLOAT_MODE_P (mode))
26712 /* ??? SSE vector cost should be used here. */
26713 *total = cost->fsqrt;
26717 if (XINT (x, 1) == UNSPEC_TP)
26728 static int current_machopic_label_num;
26730 /* Given a symbol name and its associated stub, write out the
26731 definition of the stub. */
26734 machopic_output_stub (FILE *file, const char *symb, const char *stub)
26736 unsigned int length;
26737 char *binder_name, *symbol_name, lazy_ptr_name[32];
26738 int label = ++current_machopic_label_num;
26740 /* For 64-bit we shouldn't get here. */
26741 gcc_assert (!TARGET_64BIT);
26743 /* Lose our funky encoding stuff so it doesn't contaminate the stub. */
26744 symb = (*targetm.strip_name_encoding) (symb);
26746 length = strlen (stub);
26747 binder_name = XALLOCAVEC (char, length + 32);
26748 GEN_BINDER_NAME_FOR_STUB (binder_name, stub, length);
26750 length = strlen (symb);
26751 symbol_name = XALLOCAVEC (char, length + 32);
26752 GEN_SYMBOL_NAME_FOR_SYMBOL (symbol_name, symb, length);
26754 sprintf (lazy_ptr_name, "L%d$lz", label);
26757 switch_to_section (darwin_sections[machopic_picsymbol_stub_section]);
26759 switch_to_section (darwin_sections[machopic_symbol_stub_section]);
26761 fprintf (file, "%s:\n", stub);
26762 fprintf (file, "\t.indirect_symbol %s\n", symbol_name);
26766 fprintf (file, "\tcall\tLPC$%d\nLPC$%d:\tpopl\t%%eax\n", label, label);
26767 fprintf (file, "\tmovl\t%s-LPC$%d(%%eax),%%edx\n", lazy_ptr_name, label);
26768 fprintf (file, "\tjmp\t*%%edx\n");
26771 fprintf (file, "\tjmp\t*%s\n", lazy_ptr_name);
26773 fprintf (file, "%s:\n", binder_name);
26777 fprintf (file, "\tlea\t%s-LPC$%d(%%eax),%%eax\n", lazy_ptr_name, label);
26778 fprintf (file, "\tpushl\t%%eax\n");
26781 fprintf (file, "\tpushl\t$%s\n", lazy_ptr_name);
26783 fprintf (file, "\tjmp\tdyld_stub_binding_helper\n");
26785 switch_to_section (darwin_sections[machopic_lazy_symbol_ptr_section]);
26786 fprintf (file, "%s:\n", lazy_ptr_name);
26787 fprintf (file, "\t.indirect_symbol %s\n", symbol_name);
26788 fprintf (file, "\t.long %s\n", binder_name);
26792 darwin_x86_file_end (void)
26794 darwin_file_end ();
26797 #endif /* TARGET_MACHO */
26799 /* Order the registers for register allocator. */
26802 x86_order_regs_for_local_alloc (void)
26807 /* First allocate the local general purpose registers. */
26808 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
26809 if (GENERAL_REGNO_P (i) && call_used_regs[i])
26810 reg_alloc_order [pos++] = i;
26812 /* Global general purpose registers. */
26813 for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
26814 if (GENERAL_REGNO_P (i) && !call_used_regs[i])
26815 reg_alloc_order [pos++] = i;
26817 /* x87 registers come first in case we are doing FP math
26819 if (!TARGET_SSE_MATH)
26820 for (i = FIRST_STACK_REG; i <= LAST_STACK_REG; i++)
26821 reg_alloc_order [pos++] = i;
26823 /* SSE registers. */
26824 for (i = FIRST_SSE_REG; i <= LAST_SSE_REG; i++)
26825 reg_alloc_order [pos++] = i;
26826 for (i = FIRST_REX_SSE_REG; i <= LAST_REX_SSE_REG; i++)
26827 reg_alloc_order [pos++] = i;
26829 /* x87 registers. */
26830 if (TARGET_SSE_MATH)
26831 for (i = FIRST_STACK_REG; i <= LAST_STACK_REG; i++)
26832 reg_alloc_order [pos++] = i;
26834 for (i = FIRST_MMX_REG; i <= LAST_MMX_REG; i++)
26835 reg_alloc_order [pos++] = i;
26837 /* Initialize the rest of array as we do not allocate some registers
26839 while (pos < FIRST_PSEUDO_REGISTER)
26840 reg_alloc_order [pos++] = 0;
26843 /* Handle a "ms_abi" or "sysv" attribute; arguments as in
26844 struct attribute_spec.handler. */
26846 ix86_handle_abi_attribute (tree *node, tree name,
26847 tree args ATTRIBUTE_UNUSED,
26848 int flags ATTRIBUTE_UNUSED, bool *no_add_attrs)
26850 if (TREE_CODE (*node) != FUNCTION_TYPE
26851 && TREE_CODE (*node) != METHOD_TYPE
26852 && TREE_CODE (*node) != FIELD_DECL
26853 && TREE_CODE (*node) != TYPE_DECL)
26855 warning (OPT_Wattributes, "%qE attribute only applies to functions",
26857 *no_add_attrs = true;
26862 warning (OPT_Wattributes, "%qE attribute only available for 64-bit",
26864 *no_add_attrs = true;
26868 /* Can combine regparm with all attributes but fastcall. */
26869 if (is_attribute_p ("ms_abi", name))
26871 if (lookup_attribute ("sysv_abi", TYPE_ATTRIBUTES (*node)))
26873 error ("ms_abi and sysv_abi attributes are not compatible");
26878 else if (is_attribute_p ("sysv_abi", name))
26880 if (lookup_attribute ("ms_abi", TYPE_ATTRIBUTES (*node)))
26882 error ("ms_abi and sysv_abi attributes are not compatible");
26891 /* Handle a "ms_struct" or "gcc_struct" attribute; arguments as in
26892 struct attribute_spec.handler. */
26894 ix86_handle_struct_attribute (tree *node, tree name,
26895 tree args ATTRIBUTE_UNUSED,
26896 int flags ATTRIBUTE_UNUSED, bool *no_add_attrs)
26899 if (DECL_P (*node))
26901 if (TREE_CODE (*node) == TYPE_DECL)
26902 type = &TREE_TYPE (*node);
26907 if (!(type && (TREE_CODE (*type) == RECORD_TYPE
26908 || TREE_CODE (*type) == UNION_TYPE)))
26910 warning (OPT_Wattributes, "%qE attribute ignored",
26912 *no_add_attrs = true;
26915 else if ((is_attribute_p ("ms_struct", name)
26916 && lookup_attribute ("gcc_struct", TYPE_ATTRIBUTES (*type)))
26917 || ((is_attribute_p ("gcc_struct", name)
26918 && lookup_attribute ("ms_struct", TYPE_ATTRIBUTES (*type)))))
26920 warning (OPT_Wattributes, "%qE incompatible attribute ignored",
26922 *no_add_attrs = true;
26929 ix86_ms_bitfield_layout_p (const_tree record_type)
26931 return (TARGET_MS_BITFIELD_LAYOUT &&
26932 !lookup_attribute ("gcc_struct", TYPE_ATTRIBUTES (record_type)))
26933 || lookup_attribute ("ms_struct", TYPE_ATTRIBUTES (record_type));
26936 /* Returns an expression indicating where the this parameter is
26937 located on entry to the FUNCTION. */
26940 x86_this_parameter (tree function)
26942 tree type = TREE_TYPE (function);
26943 bool aggr = aggregate_value_p (TREE_TYPE (type), type) != 0;
26948 const int *parm_regs;
26950 if (ix86_function_type_abi (type) == MS_ABI)
26951 parm_regs = x86_64_ms_abi_int_parameter_registers;
26953 parm_regs = x86_64_int_parameter_registers;
26954 return gen_rtx_REG (DImode, parm_regs[aggr]);
26957 nregs = ix86_function_regparm (type, function);
26959 if (nregs > 0 && !stdarg_p (type))
26963 if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (type)))
26964 regno = aggr ? DX_REG : CX_REG;
26972 return gen_rtx_MEM (SImode,
26973 plus_constant (stack_pointer_rtx, 4));
26976 return gen_rtx_REG (SImode, regno);
26979 return gen_rtx_MEM (SImode, plus_constant (stack_pointer_rtx, aggr ? 8 : 4));
26982 /* Determine whether x86_output_mi_thunk can succeed. */
26985 x86_can_output_mi_thunk (const_tree thunk ATTRIBUTE_UNUSED,
26986 HOST_WIDE_INT delta ATTRIBUTE_UNUSED,
26987 HOST_WIDE_INT vcall_offset, const_tree function)
26989 /* 64-bit can handle anything. */
26993 /* For 32-bit, everything's fine if we have one free register. */
26994 if (ix86_function_regparm (TREE_TYPE (function), function) < 3)
26997 /* Need a free register for vcall_offset. */
27001 /* Need a free register for GOT references. */
27002 if (flag_pic && !(*targetm.binds_local_p) (function))
27005 /* Otherwise ok. */
27009 /* Output the assembler code for a thunk function. THUNK_DECL is the
27010 declaration for the thunk function itself, FUNCTION is the decl for
27011 the target function. DELTA is an immediate constant offset to be
27012 added to THIS. If VCALL_OFFSET is nonzero, the word at
27013 *(*this + vcall_offset) should be added to THIS. */
27016 x86_output_mi_thunk (FILE *file ATTRIBUTE_UNUSED,
27017 tree thunk ATTRIBUTE_UNUSED, HOST_WIDE_INT delta,
27018 HOST_WIDE_INT vcall_offset, tree function)
27021 rtx this_param = x86_this_parameter (function);
27024 /* If VCALL_OFFSET, we'll need THIS in a register. Might as well
27025 pull it in now and let DELTA benefit. */
27026 if (REG_P (this_param))
27027 this_reg = this_param;
27028 else if (vcall_offset)
27030 /* Put the this parameter into %eax. */
27031 xops[0] = this_param;
27032 xops[1] = this_reg = gen_rtx_REG (Pmode, AX_REG);
27033 output_asm_insn ("mov%z1\t{%0, %1|%1, %0}", xops);
27036 this_reg = NULL_RTX;
27038 /* Adjust the this parameter by a fixed constant. */
27041 xops[0] = GEN_INT (delta);
27042 xops[1] = this_reg ? this_reg : this_param;
27045 if (!x86_64_general_operand (xops[0], DImode))
27047 tmp = gen_rtx_REG (DImode, R10_REG);
27049 output_asm_insn ("mov{q}\t{%1, %0|%0, %1}", xops);
27051 xops[1] = this_param;
27053 output_asm_insn ("add{q}\t{%0, %1|%1, %0}", xops);
27056 output_asm_insn ("add{l}\t{%0, %1|%1, %0}", xops);
27059 /* Adjust the this parameter by a value stored in the vtable. */
27063 tmp = gen_rtx_REG (DImode, R10_REG);
27066 int tmp_regno = CX_REG;
27067 if (lookup_attribute ("fastcall",
27068 TYPE_ATTRIBUTES (TREE_TYPE (function))))
27069 tmp_regno = AX_REG;
27070 tmp = gen_rtx_REG (SImode, tmp_regno);
27073 xops[0] = gen_rtx_MEM (Pmode, this_reg);
27075 output_asm_insn ("mov%z1\t{%0, %1|%1, %0}", xops);
27077 /* Adjust the this parameter. */
27078 xops[0] = gen_rtx_MEM (Pmode, plus_constant (tmp, vcall_offset));
27079 if (TARGET_64BIT && !memory_operand (xops[0], Pmode))
27081 rtx tmp2 = gen_rtx_REG (DImode, R11_REG);
27082 xops[0] = GEN_INT (vcall_offset);
27084 output_asm_insn ("mov{q}\t{%0, %1|%1, %0}", xops);
27085 xops[0] = gen_rtx_MEM (Pmode, gen_rtx_PLUS (Pmode, tmp, tmp2));
27087 xops[1] = this_reg;
27088 output_asm_insn ("add%z1\t{%0, %1|%1, %0}", xops);
27091 /* If necessary, drop THIS back to its stack slot. */
27092 if (this_reg && this_reg != this_param)
27094 xops[0] = this_reg;
27095 xops[1] = this_param;
27096 output_asm_insn ("mov%z1\t{%0, %1|%1, %0}", xops);
27099 xops[0] = XEXP (DECL_RTL (function), 0);
27102 if (!flag_pic || (*targetm.binds_local_p) (function))
27103 output_asm_insn ("jmp\t%P0", xops);
27104 /* All thunks should be in the same object as their target,
27105 and thus binds_local_p should be true. */
27106 else if (TARGET_64BIT && cfun->machine->call_abi == MS_ABI)
27107 gcc_unreachable ();
27110 tmp = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, xops[0]), UNSPEC_GOTPCREL);
27111 tmp = gen_rtx_CONST (Pmode, tmp);
27112 tmp = gen_rtx_MEM (QImode, tmp);
27114 output_asm_insn ("jmp\t%A0", xops);
27119 if (!flag_pic || (*targetm.binds_local_p) (function))
27120 output_asm_insn ("jmp\t%P0", xops);
27125 rtx sym_ref = XEXP (DECL_RTL (function), 0);
27126 tmp = (gen_rtx_SYMBOL_REF
27128 machopic_indirection_name (sym_ref, /*stub_p=*/true)));
27129 tmp = gen_rtx_MEM (QImode, tmp);
27131 output_asm_insn ("jmp\t%0", xops);
27134 #endif /* TARGET_MACHO */
27136 tmp = gen_rtx_REG (SImode, CX_REG);
27137 output_set_got (tmp, NULL_RTX);
27140 output_asm_insn ("mov{l}\t{%0@GOT(%1), %1|%1, %0@GOT[%1]}", xops);
27141 output_asm_insn ("jmp\t{*}%1", xops);
27147 x86_file_start (void)
27149 default_file_start ();
27151 darwin_file_start ();
27153 if (X86_FILE_START_VERSION_DIRECTIVE)
27154 fputs ("\t.version\t\"01.01\"\n", asm_out_file);
27155 if (X86_FILE_START_FLTUSED)
27156 fputs ("\t.global\t__fltused\n", asm_out_file);
27157 if (ix86_asm_dialect == ASM_INTEL)
27158 fputs ("\t.intel_syntax noprefix\n", asm_out_file);
27162 x86_field_alignment (tree field, int computed)
27164 enum machine_mode mode;
27165 tree type = TREE_TYPE (field);
27167 if (TARGET_64BIT || TARGET_ALIGN_DOUBLE)
27169 mode = TYPE_MODE (strip_array_types (type));
27170 if (mode == DFmode || mode == DCmode
27171 || GET_MODE_CLASS (mode) == MODE_INT
27172 || GET_MODE_CLASS (mode) == MODE_COMPLEX_INT)
27173 return MIN (32, computed);
27177 /* Output assembler code to FILE to increment profiler label # LABELNO
27178 for profiling a function entry. */
27180 x86_function_profiler (FILE *file, int labelno ATTRIBUTE_UNUSED)
27184 #ifndef NO_PROFILE_COUNTERS
27185 fprintf (file, "\tleaq\t%sP%d@(%%rip),%%r11\n", LPREFIX, labelno);
27188 if (DEFAULT_ABI == SYSV_ABI && flag_pic)
27189 fprintf (file, "\tcall\t*%s@GOTPCREL(%%rip)\n", MCOUNT_NAME);
27191 fprintf (file, "\tcall\t%s\n", MCOUNT_NAME);
27195 #ifndef NO_PROFILE_COUNTERS
27196 fprintf (file, "\tleal\t%sP%d@GOTOFF(%%ebx),%%%s\n",
27197 LPREFIX, labelno, PROFILE_COUNT_REGISTER);
27199 fprintf (file, "\tcall\t*%s@GOT(%%ebx)\n", MCOUNT_NAME);
27203 #ifndef NO_PROFILE_COUNTERS
27204 fprintf (file, "\tmovl\t$%sP%d,%%%s\n", LPREFIX, labelno,
27205 PROFILE_COUNT_REGISTER);
27207 fprintf (file, "\tcall\t%s\n", MCOUNT_NAME);
27211 #ifdef ASM_OUTPUT_MAX_SKIP_PAD
27212 /* We don't have exact information about the insn sizes, but we may assume
27213 quite safely that we are informed about all 1 byte insns and memory
27214 address sizes. This is enough to eliminate unnecessary padding in
27218 min_insn_size (rtx insn)
27222 if (!INSN_P (insn) || !active_insn_p (insn))
27225 /* Discard alignments we've emit and jump instructions. */
27226 if (GET_CODE (PATTERN (insn)) == UNSPEC_VOLATILE
27227 && XINT (PATTERN (insn), 1) == UNSPECV_ALIGN)
27230 && (GET_CODE (PATTERN (insn)) == ADDR_VEC
27231 || GET_CODE (PATTERN (insn)) == ADDR_DIFF_VEC))
27234 /* Important case - calls are always 5 bytes.
27235 It is common to have many calls in the row. */
27237 && symbolic_reference_mentioned_p (PATTERN (insn))
27238 && !SIBLING_CALL_P (insn))
27240 if (get_attr_length (insn) <= 1)
27243 /* For normal instructions we may rely on the sizes of addresses
27244 and the presence of symbol to require 4 bytes of encoding.
27245 This is not the case for jumps where references are PC relative. */
27246 if (!JUMP_P (insn))
27248 l = get_attr_length_address (insn);
27249 if (l < 4 && symbolic_reference_mentioned_p (PATTERN (insn)))
27258 /* AMD K8 core mispredicts jumps when there are more than 3 jumps in 16 byte
27262 ix86_avoid_jump_mispredicts (void)
27264 rtx insn, start = get_insns ();
27265 int nbytes = 0, njumps = 0;
27268 /* Look for all minimal intervals of instructions containing 4 jumps.
27269 The intervals are bounded by START and INSN. NBYTES is the total
27270 size of instructions in the interval including INSN and not including
27271 START. When the NBYTES is smaller than 16 bytes, it is possible
27272 that the end of START and INSN ends up in the same 16byte page.
27274 The smallest offset in the page INSN can start is the case where START
27275 ends on the offset 0. Offset of INSN is then NBYTES - sizeof (INSN).
27276 We add p2align to 16byte window with maxskip 15 - NBYTES + sizeof (INSN).
27278 for (insn = start; insn; insn = NEXT_INSN (insn))
27282 if (GET_CODE (insn) == CODE_LABEL)
27284 int align = label_to_alignment (insn);
27285 int max_skip = label_to_max_skip (insn);
27289 /* If align > 3, only up to 16 - max_skip - 1 bytes can be
27290 already in the current 16 byte page, because otherwise
27291 ASM_OUTPUT_MAX_SKIP_ALIGN could skip max_skip or fewer
27292 bytes to reach 16 byte boundary. */
27294 || (align <= 3 && max_skip != (1 << align) - 1))
27297 fprintf (dump_file, "Label %i with max_skip %i\n",
27298 INSN_UID (insn), max_skip);
27301 while (nbytes + max_skip >= 16)
27303 start = NEXT_INSN (start);
27304 if ((JUMP_P (start)
27305 && GET_CODE (PATTERN (start)) != ADDR_VEC
27306 && GET_CODE (PATTERN (start)) != ADDR_DIFF_VEC)
27308 njumps--, isjump = 1;
27311 nbytes -= min_insn_size (start);
27317 min_size = min_insn_size (insn);
27318 nbytes += min_size;
27320 fprintf (dump_file, "Insn %i estimated to %i bytes\n",
27321 INSN_UID (insn), min_size);
27323 && GET_CODE (PATTERN (insn)) != ADDR_VEC
27324 && GET_CODE (PATTERN (insn)) != ADDR_DIFF_VEC)
27332 start = NEXT_INSN (start);
27333 if ((JUMP_P (start)
27334 && GET_CODE (PATTERN (start)) != ADDR_VEC
27335 && GET_CODE (PATTERN (start)) != ADDR_DIFF_VEC)
27337 njumps--, isjump = 1;
27340 nbytes -= min_insn_size (start);
27342 gcc_assert (njumps >= 0);
27344 fprintf (dump_file, "Interval %i to %i has %i bytes\n",
27345 INSN_UID (start), INSN_UID (insn), nbytes);
27347 if (njumps == 3 && isjump && nbytes < 16)
27349 int padsize = 15 - nbytes + min_insn_size (insn);
27352 fprintf (dump_file, "Padding insn %i by %i bytes!\n",
27353 INSN_UID (insn), padsize);
27354 emit_insn_before (gen_pad (GEN_INT (padsize)), insn);
27360 /* AMD Athlon works faster
27361 when RET is not destination of conditional jump or directly preceded
27362 by other jump instruction. We avoid the penalty by inserting NOP just
27363 before the RET instructions in such cases. */
27365 ix86_pad_returns (void)
27370 FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR->preds)
27372 basic_block bb = e->src;
27373 rtx ret = BB_END (bb);
27375 bool replace = false;
27377 if (!JUMP_P (ret) || GET_CODE (PATTERN (ret)) != RETURN
27378 || optimize_bb_for_size_p (bb))
27380 for (prev = PREV_INSN (ret); prev; prev = PREV_INSN (prev))
27381 if (active_insn_p (prev) || LABEL_P (prev))
27383 if (prev && LABEL_P (prev))
27388 FOR_EACH_EDGE (e, ei, bb->preds)
27389 if (EDGE_FREQUENCY (e) && e->src->index >= 0
27390 && !(e->flags & EDGE_FALLTHRU))
27395 prev = prev_active_insn (ret);
27397 && ((JUMP_P (prev) && any_condjump_p (prev))
27400 /* Empty functions get branch mispredict even when the jump destination
27401 is not visible to us. */
27402 if (!prev && cfun->function_frequency > FUNCTION_FREQUENCY_UNLIKELY_EXECUTED)
27407 emit_insn_before (gen_return_internal_long (), ret);
27413 /* Implement machine specific optimizations. We implement padding of returns
27414 for K8 CPUs and pass to avoid 4 jumps in the single 16 byte window. */
27418 if (optimize && optimize_function_for_speed_p (cfun))
27420 if (TARGET_PAD_RETURNS)
27421 ix86_pad_returns ();
27422 #ifdef ASM_OUTPUT_MAX_SKIP_PAD
27423 if (TARGET_FOUR_JUMP_LIMIT)
27424 ix86_avoid_jump_mispredicts ();
27429 /* Return nonzero when QImode register that must be represented via REX prefix
27432 x86_extended_QIreg_mentioned_p (rtx insn)
27435 extract_insn_cached (insn);
27436 for (i = 0; i < recog_data.n_operands; i++)
27437 if (REG_P (recog_data.operand[i])
27438 && REGNO (recog_data.operand[i]) > BX_REG)
27443 /* Return nonzero when P points to register encoded via REX prefix.
27444 Called via for_each_rtx. */
27446 extended_reg_mentioned_1 (rtx *p, void *data ATTRIBUTE_UNUSED)
27448 unsigned int regno;
27451 regno = REGNO (*p);
27452 return REX_INT_REGNO_P (regno) || REX_SSE_REGNO_P (regno);
27455 /* Return true when INSN mentions register that must be encoded using REX
27458 x86_extended_reg_mentioned_p (rtx insn)
27460 return for_each_rtx (INSN_P (insn) ? &PATTERN (insn) : &insn,
27461 extended_reg_mentioned_1, NULL);
27464 /* Generate an unsigned DImode/SImode to FP conversion. This is the same code
27465 optabs would emit if we didn't have TFmode patterns. */
27468 x86_emit_floatuns (rtx operands[2])
27470 rtx neglab, donelab, i0, i1, f0, in, out;
27471 enum machine_mode mode, inmode;
27473 inmode = GET_MODE (operands[1]);
27474 gcc_assert (inmode == SImode || inmode == DImode);
27477 in = force_reg (inmode, operands[1]);
27478 mode = GET_MODE (out);
27479 neglab = gen_label_rtx ();
27480 donelab = gen_label_rtx ();
27481 f0 = gen_reg_rtx (mode);
27483 emit_cmp_and_jump_insns (in, const0_rtx, LT, const0_rtx, inmode, 0, neglab);
27485 expand_float (out, in, 0);
27487 emit_jump_insn (gen_jump (donelab));
27490 emit_label (neglab);
27492 i0 = expand_simple_binop (inmode, LSHIFTRT, in, const1_rtx, NULL,
27494 i1 = expand_simple_binop (inmode, AND, in, const1_rtx, NULL,
27496 i0 = expand_simple_binop (inmode, IOR, i0, i1, i0, 1, OPTAB_DIRECT);
27498 expand_float (f0, i0, 0);
27500 emit_insn (gen_rtx_SET (VOIDmode, out, gen_rtx_PLUS (mode, f0, f0)));
27502 emit_label (donelab);
27505 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
27506 with all elements equal to VAR. Return true if successful. */
27509 ix86_expand_vector_init_duplicate (bool mmx_ok, enum machine_mode mode,
27510 rtx target, rtx val)
27512 enum machine_mode hmode, smode, wsmode, wvmode;
27527 val = force_reg (GET_MODE_INNER (mode), val);
27528 x = gen_rtx_VEC_DUPLICATE (mode, val);
27529 emit_insn (gen_rtx_SET (VOIDmode, target, x));
27535 if (TARGET_SSE || TARGET_3DNOW_A)
27537 val = gen_lowpart (SImode, val);
27538 x = gen_rtx_TRUNCATE (HImode, val);
27539 x = gen_rtx_VEC_DUPLICATE (mode, x);
27540 emit_insn (gen_rtx_SET (VOIDmode, target, x));
27562 /* Extend HImode to SImode using a paradoxical SUBREG. */
27563 tmp1 = gen_reg_rtx (SImode);
27564 emit_move_insn (tmp1, gen_lowpart (SImode, val));
27565 /* Insert the SImode value as low element of V4SImode vector. */
27566 tmp2 = gen_reg_rtx (V4SImode);
27567 tmp1 = gen_rtx_VEC_MERGE (V4SImode,
27568 gen_rtx_VEC_DUPLICATE (V4SImode, tmp1),
27569 CONST0_RTX (V4SImode),
27571 emit_insn (gen_rtx_SET (VOIDmode, tmp2, tmp1));
27572 /* Cast the V4SImode vector back to a V8HImode vector. */
27573 tmp1 = gen_reg_rtx (V8HImode);
27574 emit_move_insn (tmp1, gen_lowpart (V8HImode, tmp2));
27575 /* Duplicate the low short through the whole low SImode word. */
27576 emit_insn (gen_sse2_punpcklwd (tmp1, tmp1, tmp1));
27577 /* Cast the V8HImode vector back to a V4SImode vector. */
27578 tmp2 = gen_reg_rtx (V4SImode);
27579 emit_move_insn (tmp2, gen_lowpart (V4SImode, tmp1));
27580 /* Replicate the low element of the V4SImode vector. */
27581 emit_insn (gen_sse2_pshufd (tmp2, tmp2, const0_rtx));
27582 /* Cast the V2SImode back to V8HImode, and store in target. */
27583 emit_move_insn (target, gen_lowpart (V8HImode, tmp2));
27594 /* Extend QImode to SImode using a paradoxical SUBREG. */
27595 tmp1 = gen_reg_rtx (SImode);
27596 emit_move_insn (tmp1, gen_lowpart (SImode, val));
27597 /* Insert the SImode value as low element of V4SImode vector. */
27598 tmp2 = gen_reg_rtx (V4SImode);
27599 tmp1 = gen_rtx_VEC_MERGE (V4SImode,
27600 gen_rtx_VEC_DUPLICATE (V4SImode, tmp1),
27601 CONST0_RTX (V4SImode),
27603 emit_insn (gen_rtx_SET (VOIDmode, tmp2, tmp1));
27604 /* Cast the V4SImode vector back to a V16QImode vector. */
27605 tmp1 = gen_reg_rtx (V16QImode);
27606 emit_move_insn (tmp1, gen_lowpart (V16QImode, tmp2));
27607 /* Duplicate the low byte through the whole low SImode word. */
27608 emit_insn (gen_sse2_punpcklbw (tmp1, tmp1, tmp1));
27609 emit_insn (gen_sse2_punpcklbw (tmp1, tmp1, tmp1));
27610 /* Cast the V16QImode vector back to a V4SImode vector. */
27611 tmp2 = gen_reg_rtx (V4SImode);
27612 emit_move_insn (tmp2, gen_lowpart (V4SImode, tmp1));
27613 /* Replicate the low element of the V4SImode vector. */
27614 emit_insn (gen_sse2_pshufd (tmp2, tmp2, const0_rtx));
27615 /* Cast the V2SImode back to V16QImode, and store in target. */
27616 emit_move_insn (target, gen_lowpart (V16QImode, tmp2));
27624 /* Replicate the value once into the next wider mode and recurse. */
27625 val = convert_modes (wsmode, smode, val, true);
27626 x = expand_simple_binop (wsmode, ASHIFT, val,
27627 GEN_INT (GET_MODE_BITSIZE (smode)),
27628 NULL_RTX, 1, OPTAB_LIB_WIDEN);
27629 val = expand_simple_binop (wsmode, IOR, val, x, x, 1, OPTAB_LIB_WIDEN);
27631 x = gen_reg_rtx (wvmode);
27632 if (!ix86_expand_vector_init_duplicate (mmx_ok, wvmode, x, val))
27633 gcc_unreachable ();
27634 emit_move_insn (target, gen_lowpart (mode, x));
27657 rtx tmp = gen_reg_rtx (hmode);
27658 ix86_expand_vector_init_duplicate (mmx_ok, hmode, tmp, val);
27659 emit_insn (gen_rtx_SET (VOIDmode, target,
27660 gen_rtx_VEC_CONCAT (mode, tmp, tmp)));
27669 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
27670 whose ONE_VAR element is VAR, and other elements are zero. Return true
27674 ix86_expand_vector_init_one_nonzero (bool mmx_ok, enum machine_mode mode,
27675 rtx target, rtx var, int one_var)
27677 enum machine_mode vsimode;
27680 bool use_vector_set = false;
27685 /* For SSE4.1, we normally use vector set. But if the second
27686 element is zero and inter-unit moves are OK, we use movq
27688 use_vector_set = (TARGET_64BIT
27690 && !(TARGET_INTER_UNIT_MOVES
27696 use_vector_set = TARGET_SSE4_1;
27699 use_vector_set = TARGET_SSE2;
27702 use_vector_set = TARGET_SSE || TARGET_3DNOW_A;
27709 use_vector_set = TARGET_AVX;
27712 /* Use ix86_expand_vector_set in 64bit mode only. */
27713 use_vector_set = TARGET_AVX && TARGET_64BIT;
27719 if (use_vector_set)
27721 emit_insn (gen_rtx_SET (VOIDmode, target, CONST0_RTX (mode)));
27722 var = force_reg (GET_MODE_INNER (mode), var);
27723 ix86_expand_vector_set (mmx_ok, target, var, one_var);
27739 var = force_reg (GET_MODE_INNER (mode), var);
27740 x = gen_rtx_VEC_CONCAT (mode, var, CONST0_RTX (GET_MODE_INNER (mode)));
27741 emit_insn (gen_rtx_SET (VOIDmode, target, x));
27746 if (!REG_P (target) || REGNO (target) < FIRST_PSEUDO_REGISTER)
27747 new_target = gen_reg_rtx (mode);
27749 new_target = target;
27750 var = force_reg (GET_MODE_INNER (mode), var);
27751 x = gen_rtx_VEC_DUPLICATE (mode, var);
27752 x = gen_rtx_VEC_MERGE (mode, x, CONST0_RTX (mode), const1_rtx);
27753 emit_insn (gen_rtx_SET (VOIDmode, new_target, x));
27756 /* We need to shuffle the value to the correct position, so
27757 create a new pseudo to store the intermediate result. */
27759 /* With SSE2, we can use the integer shuffle insns. */
27760 if (mode != V4SFmode && TARGET_SSE2)
27762 emit_insn (gen_sse2_pshufd_1 (new_target, new_target,
27764 GEN_INT (one_var == 1 ? 0 : 1),
27765 GEN_INT (one_var == 2 ? 0 : 1),
27766 GEN_INT (one_var == 3 ? 0 : 1)));
27767 if (target != new_target)
27768 emit_move_insn (target, new_target);
27772 /* Otherwise convert the intermediate result to V4SFmode and
27773 use the SSE1 shuffle instructions. */
27774 if (mode != V4SFmode)
27776 tmp = gen_reg_rtx (V4SFmode);
27777 emit_move_insn (tmp, gen_lowpart (V4SFmode, new_target));
27782 emit_insn (gen_sse_shufps_v4sf (tmp, tmp, tmp,
27784 GEN_INT (one_var == 1 ? 0 : 1),
27785 GEN_INT (one_var == 2 ? 0+4 : 1+4),
27786 GEN_INT (one_var == 3 ? 0+4 : 1+4)));
27788 if (mode != V4SFmode)
27789 emit_move_insn (target, gen_lowpart (V4SImode, tmp));
27790 else if (tmp != target)
27791 emit_move_insn (target, tmp);
27793 else if (target != new_target)
27794 emit_move_insn (target, new_target);
27799 vsimode = V4SImode;
27805 vsimode = V2SImode;
27811 /* Zero extend the variable element to SImode and recurse. */
27812 var = convert_modes (SImode, GET_MODE_INNER (mode), var, true);
27814 x = gen_reg_rtx (vsimode);
27815 if (!ix86_expand_vector_init_one_nonzero (mmx_ok, vsimode, x,
27817 gcc_unreachable ();
27819 emit_move_insn (target, gen_lowpart (mode, x));
27827 /* A subroutine of ix86_expand_vector_init. Store into TARGET a vector
27828 consisting of the values in VALS. It is known that all elements
27829 except ONE_VAR are constants. Return true if successful. */
27832 ix86_expand_vector_init_one_var (bool mmx_ok, enum machine_mode mode,
27833 rtx target, rtx vals, int one_var)
27835 rtx var = XVECEXP (vals, 0, one_var);
27836 enum machine_mode wmode;
27839 const_vec = copy_rtx (vals);
27840 XVECEXP (const_vec, 0, one_var) = CONST0_RTX (GET_MODE_INNER (mode));
27841 const_vec = gen_rtx_CONST_VECTOR (mode, XVEC (const_vec, 0));
27849 /* For the two element vectors, it's just as easy to use
27850 the general case. */
27854 /* Use ix86_expand_vector_set in 64bit mode only. */
27877 /* There's no way to set one QImode entry easily. Combine
27878 the variable value with its adjacent constant value, and
27879 promote to an HImode set. */
27880 x = XVECEXP (vals, 0, one_var ^ 1);
27883 var = convert_modes (HImode, QImode, var, true);
27884 var = expand_simple_binop (HImode, ASHIFT, var, GEN_INT (8),
27885 NULL_RTX, 1, OPTAB_LIB_WIDEN);
27886 x = GEN_INT (INTVAL (x) & 0xff);
27890 var = convert_modes (HImode, QImode, var, true);
27891 x = gen_int_mode (INTVAL (x) << 8, HImode);
27893 if (x != const0_rtx)
27894 var = expand_simple_binop (HImode, IOR, var, x, var,
27895 1, OPTAB_LIB_WIDEN);
27897 x = gen_reg_rtx (wmode);
27898 emit_move_insn (x, gen_lowpart (wmode, const_vec));
27899 ix86_expand_vector_set (mmx_ok, x, var, one_var >> 1);
27901 emit_move_insn (target, gen_lowpart (mode, x));
27908 emit_move_insn (target, const_vec);
27909 ix86_expand_vector_set (mmx_ok, target, var, one_var);
27913 /* A subroutine of ix86_expand_vector_init_general. Use vector
27914 concatenate to handle the most general case: all values variable,
27915 and none identical. */
27918 ix86_expand_vector_init_concat (enum machine_mode mode,
27919 rtx target, rtx *ops, int n)
27921 enum machine_mode cmode, hmode = VOIDmode;
27922 rtx first[8], second[4];
27962 gcc_unreachable ();
27965 if (!register_operand (ops[1], cmode))
27966 ops[1] = force_reg (cmode, ops[1]);
27967 if (!register_operand (ops[0], cmode))
27968 ops[0] = force_reg (cmode, ops[0]);
27969 emit_insn (gen_rtx_SET (VOIDmode, target,
27970 gen_rtx_VEC_CONCAT (mode, ops[0],
27990 gcc_unreachable ();
28006 gcc_unreachable ();
28011 /* FIXME: We process inputs backward to help RA. PR 36222. */
28014 for (; i > 0; i -= 2, j--)
28016 first[j] = gen_reg_rtx (cmode);
28017 v = gen_rtvec (2, ops[i - 1], ops[i]);
28018 ix86_expand_vector_init (false, first[j],
28019 gen_rtx_PARALLEL (cmode, v));
28025 gcc_assert (hmode != VOIDmode);
28026 for (i = j = 0; i < n; i += 2, j++)
28028 second[j] = gen_reg_rtx (hmode);
28029 ix86_expand_vector_init_concat (hmode, second [j],
28033 ix86_expand_vector_init_concat (mode, target, second, n);
28036 ix86_expand_vector_init_concat (mode, target, first, n);
28040 gcc_unreachable ();
28044 /* A subroutine of ix86_expand_vector_init_general. Use vector
28045 interleave to handle the most general case: all values variable,
28046 and none identical. */
28049 ix86_expand_vector_init_interleave (enum machine_mode mode,
28050 rtx target, rtx *ops, int n)
28052 enum machine_mode first_imode, second_imode, third_imode, inner_mode;
28055 rtx (*gen_load_even) (rtx, rtx, rtx);
28056 rtx (*gen_interleave_first_low) (rtx, rtx, rtx);
28057 rtx (*gen_interleave_second_low) (rtx, rtx, rtx);
28062 gen_load_even = gen_vec_setv8hi;
28063 gen_interleave_first_low = gen_vec_interleave_lowv4si;
28064 gen_interleave_second_low = gen_vec_interleave_lowv2di;
28065 inner_mode = HImode;
28066 first_imode = V4SImode;
28067 second_imode = V2DImode;
28068 third_imode = VOIDmode;
28071 gen_load_even = gen_vec_setv16qi;
28072 gen_interleave_first_low = gen_vec_interleave_lowv8hi;
28073 gen_interleave_second_low = gen_vec_interleave_lowv4si;
28074 inner_mode = QImode;
28075 first_imode = V8HImode;
28076 second_imode = V4SImode;
28077 third_imode = V2DImode;
28080 gcc_unreachable ();
28083 for (i = 0; i < n; i++)
28085 /* Extend the odd elment to SImode using a paradoxical SUBREG. */
28086 op0 = gen_reg_rtx (SImode);
28087 emit_move_insn (op0, gen_lowpart (SImode, ops [i + i]));
28089 /* Insert the SImode value as low element of V4SImode vector. */
28090 op1 = gen_reg_rtx (V4SImode);
28091 op0 = gen_rtx_VEC_MERGE (V4SImode,
28092 gen_rtx_VEC_DUPLICATE (V4SImode,
28094 CONST0_RTX (V4SImode),
28096 emit_insn (gen_rtx_SET (VOIDmode, op1, op0));
28098 /* Cast the V4SImode vector back to a vector in orignal mode. */
28099 op0 = gen_reg_rtx (mode);
28100 emit_move_insn (op0, gen_lowpart (mode, op1));
28102 /* Load even elements into the second positon. */
28103 emit_insn ((*gen_load_even) (op0,
28104 force_reg (inner_mode,
28108 /* Cast vector to FIRST_IMODE vector. */
28109 ops[i] = gen_reg_rtx (first_imode);
28110 emit_move_insn (ops[i], gen_lowpart (first_imode, op0));
28113 /* Interleave low FIRST_IMODE vectors. */
28114 for (i = j = 0; i < n; i += 2, j++)
28116 op0 = gen_reg_rtx (first_imode);
28117 emit_insn ((*gen_interleave_first_low) (op0, ops[i], ops[i + 1]));
28119 /* Cast FIRST_IMODE vector to SECOND_IMODE vector. */
28120 ops[j] = gen_reg_rtx (second_imode);
28121 emit_move_insn (ops[j], gen_lowpart (second_imode, op0));
28124 /* Interleave low SECOND_IMODE vectors. */
28125 switch (second_imode)
28128 for (i = j = 0; i < n / 2; i += 2, j++)
28130 op0 = gen_reg_rtx (second_imode);
28131 emit_insn ((*gen_interleave_second_low) (op0, ops[i],
28134 /* Cast the SECOND_IMODE vector to the THIRD_IMODE
28136 ops[j] = gen_reg_rtx (third_imode);
28137 emit_move_insn (ops[j], gen_lowpart (third_imode, op0));
28139 second_imode = V2DImode;
28140 gen_interleave_second_low = gen_vec_interleave_lowv2di;
28144 op0 = gen_reg_rtx (second_imode);
28145 emit_insn ((*gen_interleave_second_low) (op0, ops[0],
28148 /* Cast the SECOND_IMODE vector back to a vector on original
28150 emit_insn (gen_rtx_SET (VOIDmode, target,
28151 gen_lowpart (mode, op0)));
28155 gcc_unreachable ();
28159 /* A subroutine of ix86_expand_vector_init. Handle the most general case:
28160 all values variable, and none identical. */
28163 ix86_expand_vector_init_general (bool mmx_ok, enum machine_mode mode,
28164 rtx target, rtx vals)
28166 rtx ops[32], op0, op1;
28167 enum machine_mode half_mode = VOIDmode;
28174 if (!mmx_ok && !TARGET_SSE)
28186 n = GET_MODE_NUNITS (mode);
28187 for (i = 0; i < n; i++)
28188 ops[i] = XVECEXP (vals, 0, i);
28189 ix86_expand_vector_init_concat (mode, target, ops, n);
28193 half_mode = V16QImode;
28197 half_mode = V8HImode;
28201 n = GET_MODE_NUNITS (mode);
28202 for (i = 0; i < n; i++)
28203 ops[i] = XVECEXP (vals, 0, i);
28204 op0 = gen_reg_rtx (half_mode);
28205 op1 = gen_reg_rtx (half_mode);
28206 ix86_expand_vector_init_interleave (half_mode, op0, ops,
28208 ix86_expand_vector_init_interleave (half_mode, op1,
28209 &ops [n >> 1], n >> 2);
28210 emit_insn (gen_rtx_SET (VOIDmode, target,
28211 gen_rtx_VEC_CONCAT (mode, op0, op1)));
28215 if (!TARGET_SSE4_1)
28223 /* Don't use ix86_expand_vector_init_interleave if we can't
28224 move from GPR to SSE register directly. */
28225 if (!TARGET_INTER_UNIT_MOVES)
28228 n = GET_MODE_NUNITS (mode);
28229 for (i = 0; i < n; i++)
28230 ops[i] = XVECEXP (vals, 0, i);
28231 ix86_expand_vector_init_interleave (mode, target, ops, n >> 1);
28239 gcc_unreachable ();
28243 int i, j, n_elts, n_words, n_elt_per_word;
28244 enum machine_mode inner_mode;
28245 rtx words[4], shift;
28247 inner_mode = GET_MODE_INNER (mode);
28248 n_elts = GET_MODE_NUNITS (mode);
28249 n_words = GET_MODE_SIZE (mode) / UNITS_PER_WORD;
28250 n_elt_per_word = n_elts / n_words;
28251 shift = GEN_INT (GET_MODE_BITSIZE (inner_mode));
28253 for (i = 0; i < n_words; ++i)
28255 rtx word = NULL_RTX;
28257 for (j = 0; j < n_elt_per_word; ++j)
28259 rtx elt = XVECEXP (vals, 0, (i+1)*n_elt_per_word - j - 1);
28260 elt = convert_modes (word_mode, inner_mode, elt, true);
28266 word = expand_simple_binop (word_mode, ASHIFT, word, shift,
28267 word, 1, OPTAB_LIB_WIDEN);
28268 word = expand_simple_binop (word_mode, IOR, word, elt,
28269 word, 1, OPTAB_LIB_WIDEN);
28277 emit_move_insn (target, gen_lowpart (mode, words[0]));
28278 else if (n_words == 2)
28280 rtx tmp = gen_reg_rtx (mode);
28281 emit_clobber (tmp);
28282 emit_move_insn (gen_lowpart (word_mode, tmp), words[0]);
28283 emit_move_insn (gen_highpart (word_mode, tmp), words[1]);
28284 emit_move_insn (target, tmp);
28286 else if (n_words == 4)
28288 rtx tmp = gen_reg_rtx (V4SImode);
28289 gcc_assert (word_mode == SImode);
28290 vals = gen_rtx_PARALLEL (V4SImode, gen_rtvec_v (4, words));
28291 ix86_expand_vector_init_general (false, V4SImode, tmp, vals);
28292 emit_move_insn (target, gen_lowpart (mode, tmp));
28295 gcc_unreachable ();
28299 /* Initialize vector TARGET via VALS. Suppress the use of MMX
28300 instructions unless MMX_OK is true. */
28303 ix86_expand_vector_init (bool mmx_ok, rtx target, rtx vals)
28305 enum machine_mode mode = GET_MODE (target);
28306 enum machine_mode inner_mode = GET_MODE_INNER (mode);
28307 int n_elts = GET_MODE_NUNITS (mode);
28308 int n_var = 0, one_var = -1;
28309 bool all_same = true, all_const_zero = true;
28313 for (i = 0; i < n_elts; ++i)
28315 x = XVECEXP (vals, 0, i);
28316 if (!(CONST_INT_P (x)
28317 || GET_CODE (x) == CONST_DOUBLE
28318 || GET_CODE (x) == CONST_FIXED))
28319 n_var++, one_var = i;
28320 else if (x != CONST0_RTX (inner_mode))
28321 all_const_zero = false;
28322 if (i > 0 && !rtx_equal_p (x, XVECEXP (vals, 0, 0)))
28326 /* Constants are best loaded from the constant pool. */
28329 emit_move_insn (target, gen_rtx_CONST_VECTOR (mode, XVEC (vals, 0)));
28333 /* If all values are identical, broadcast the value. */
28335 && ix86_expand_vector_init_duplicate (mmx_ok, mode, target,
28336 XVECEXP (vals, 0, 0)))
28339 /* Values where only one field is non-constant are best loaded from
28340 the pool and overwritten via move later. */
28344 && ix86_expand_vector_init_one_nonzero (mmx_ok, mode, target,
28345 XVECEXP (vals, 0, one_var),
28349 if (ix86_expand_vector_init_one_var (mmx_ok, mode, target, vals, one_var))
28353 ix86_expand_vector_init_general (mmx_ok, mode, target, vals);
28357 ix86_expand_vector_set (bool mmx_ok, rtx target, rtx val, int elt)
28359 enum machine_mode mode = GET_MODE (target);
28360 enum machine_mode inner_mode = GET_MODE_INNER (mode);
28361 enum machine_mode half_mode;
28362 bool use_vec_merge = false;
28364 static rtx (*gen_extract[6][2]) (rtx, rtx)
28366 { gen_vec_extract_lo_v32qi, gen_vec_extract_hi_v32qi },
28367 { gen_vec_extract_lo_v16hi, gen_vec_extract_hi_v16hi },
28368 { gen_vec_extract_lo_v8si, gen_vec_extract_hi_v8si },
28369 { gen_vec_extract_lo_v4di, gen_vec_extract_hi_v4di },
28370 { gen_vec_extract_lo_v8sf, gen_vec_extract_hi_v8sf },
28371 { gen_vec_extract_lo_v4df, gen_vec_extract_hi_v4df }
28373 static rtx (*gen_insert[6][2]) (rtx, rtx, rtx)
28375 { gen_vec_set_lo_v32qi, gen_vec_set_hi_v32qi },
28376 { gen_vec_set_lo_v16hi, gen_vec_set_hi_v16hi },
28377 { gen_vec_set_lo_v8si, gen_vec_set_hi_v8si },
28378 { gen_vec_set_lo_v4di, gen_vec_set_hi_v4di },
28379 { gen_vec_set_lo_v8sf, gen_vec_set_hi_v8sf },
28380 { gen_vec_set_lo_v4df, gen_vec_set_hi_v4df }
28390 tmp = gen_reg_rtx (GET_MODE_INNER (mode));
28391 ix86_expand_vector_extract (true, tmp, target, 1 - elt);
28393 tmp = gen_rtx_VEC_CONCAT (mode, tmp, val);
28395 tmp = gen_rtx_VEC_CONCAT (mode, val, tmp);
28396 emit_insn (gen_rtx_SET (VOIDmode, target, tmp));
28402 use_vec_merge = TARGET_SSE4_1;
28410 /* For the two element vectors, we implement a VEC_CONCAT with
28411 the extraction of the other element. */
28413 tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (1 - elt)));
28414 tmp = gen_rtx_VEC_SELECT (inner_mode, target, tmp);
28417 op0 = val, op1 = tmp;
28419 op0 = tmp, op1 = val;
28421 tmp = gen_rtx_VEC_CONCAT (mode, op0, op1);
28422 emit_insn (gen_rtx_SET (VOIDmode, target, tmp));
28427 use_vec_merge = TARGET_SSE4_1;
28434 use_vec_merge = true;
28438 /* tmp = target = A B C D */
28439 tmp = copy_to_reg (target);
28440 /* target = A A B B */
28441 emit_insn (gen_sse_unpcklps (target, target, target));
28442 /* target = X A B B */
28443 ix86_expand_vector_set (false, target, val, 0);
28444 /* target = A X C D */
28445 emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
28446 GEN_INT (1), GEN_INT (0),
28447 GEN_INT (2+4), GEN_INT (3+4)));
28451 /* tmp = target = A B C D */
28452 tmp = copy_to_reg (target);
28453 /* tmp = X B C D */
28454 ix86_expand_vector_set (false, tmp, val, 0);
28455 /* target = A B X D */
28456 emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
28457 GEN_INT (0), GEN_INT (1),
28458 GEN_INT (0+4), GEN_INT (3+4)));
28462 /* tmp = target = A B C D */
28463 tmp = copy_to_reg (target);
28464 /* tmp = X B C D */
28465 ix86_expand_vector_set (false, tmp, val, 0);
28466 /* target = A B X D */
28467 emit_insn (gen_sse_shufps_v4sf (target, target, tmp,
28468 GEN_INT (0), GEN_INT (1),
28469 GEN_INT (2+4), GEN_INT (0+4)));
28473 gcc_unreachable ();
28478 use_vec_merge = TARGET_SSE4_1;
28482 /* Element 0 handled by vec_merge below. */
28485 use_vec_merge = true;
28491 /* With SSE2, use integer shuffles to swap element 0 and ELT,
28492 store into element 0, then shuffle them back. */
28496 order[0] = GEN_INT (elt);
28497 order[1] = const1_rtx;
28498 order[2] = const2_rtx;
28499 order[3] = GEN_INT (3);
28500 order[elt] = const0_rtx;
28502 emit_insn (gen_sse2_pshufd_1 (target, target, order[0],
28503 order[1], order[2], order[3]));
28505 ix86_expand_vector_set (false, target, val, 0);
28507 emit_insn (gen_sse2_pshufd_1 (target, target, order[0],
28508 order[1], order[2], order[3]));
28512 /* For SSE1, we have to reuse the V4SF code. */
28513 ix86_expand_vector_set (false, gen_lowpart (V4SFmode, target),
28514 gen_lowpart (SFmode, val), elt);
28519 use_vec_merge = TARGET_SSE2;
28522 use_vec_merge = mmx_ok && (TARGET_SSE || TARGET_3DNOW_A);
28526 use_vec_merge = TARGET_SSE4_1;
28533 half_mode = V16QImode;
28539 half_mode = V8HImode;
28545 half_mode = V4SImode;
28551 half_mode = V2DImode;
28557 half_mode = V4SFmode;
28563 half_mode = V2DFmode;
28569 /* Compute offset. */
28573 gcc_assert (i <= 1);
28575 /* Extract the half. */
28576 tmp = gen_reg_rtx (half_mode);
28577 emit_insn ((*gen_extract[j][i]) (tmp, target));
28579 /* Put val in tmp at elt. */
28580 ix86_expand_vector_set (false, tmp, val, elt);
28583 emit_insn ((*gen_insert[j][i]) (target, target, tmp));
28592 tmp = gen_rtx_VEC_DUPLICATE (mode, val);
28593 tmp = gen_rtx_VEC_MERGE (mode, tmp, target, GEN_INT (1 << elt));
28594 emit_insn (gen_rtx_SET (VOIDmode, target, tmp));
28598 rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode), false);
28600 emit_move_insn (mem, target);
28602 tmp = adjust_address (mem, inner_mode, elt*GET_MODE_SIZE (inner_mode));
28603 emit_move_insn (tmp, val);
28605 emit_move_insn (target, mem);
28610 ix86_expand_vector_extract (bool mmx_ok, rtx target, rtx vec, int elt)
28612 enum machine_mode mode = GET_MODE (vec);
28613 enum machine_mode inner_mode = GET_MODE_INNER (mode);
28614 bool use_vec_extr = false;
28627 use_vec_extr = true;
28631 use_vec_extr = TARGET_SSE4_1;
28643 tmp = gen_reg_rtx (mode);
28644 emit_insn (gen_sse_shufps_v4sf (tmp, vec, vec,
28645 GEN_INT (elt), GEN_INT (elt),
28646 GEN_INT (elt+4), GEN_INT (elt+4)));
28650 tmp = gen_reg_rtx (mode);
28651 emit_insn (gen_sse_unpckhps (tmp, vec, vec));
28655 gcc_unreachable ();
28658 use_vec_extr = true;
28663 use_vec_extr = TARGET_SSE4_1;
28677 tmp = gen_reg_rtx (mode);
28678 emit_insn (gen_sse2_pshufd_1 (tmp, vec,
28679 GEN_INT (elt), GEN_INT (elt),
28680 GEN_INT (elt), GEN_INT (elt)));
28684 tmp = gen_reg_rtx (mode);
28685 emit_insn (gen_sse2_punpckhdq (tmp, vec, vec));
28689 gcc_unreachable ();
28692 use_vec_extr = true;
28697 /* For SSE1, we have to reuse the V4SF code. */
28698 ix86_expand_vector_extract (false, gen_lowpart (SFmode, target),
28699 gen_lowpart (V4SFmode, vec), elt);
28705 use_vec_extr = TARGET_SSE2;
28708 use_vec_extr = mmx_ok && (TARGET_SSE || TARGET_3DNOW_A);
28712 use_vec_extr = TARGET_SSE4_1;
28716 /* ??? Could extract the appropriate HImode element and shift. */
28723 tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, GEN_INT (elt)));
28724 tmp = gen_rtx_VEC_SELECT (inner_mode, vec, tmp);
28726 /* Let the rtl optimizers know about the zero extension performed. */
28727 if (inner_mode == QImode || inner_mode == HImode)
28729 tmp = gen_rtx_ZERO_EXTEND (SImode, tmp);
28730 target = gen_lowpart (SImode, target);
28733 emit_insn (gen_rtx_SET (VOIDmode, target, tmp));
28737 rtx mem = assign_stack_temp (mode, GET_MODE_SIZE (mode), false);
28739 emit_move_insn (mem, vec);
28741 tmp = adjust_address (mem, inner_mode, elt*GET_MODE_SIZE (inner_mode));
28742 emit_move_insn (target, tmp);
28746 /* Expand a vector reduction on V4SFmode for SSE1. FN is the binary
28747 pattern to reduce; DEST is the destination; IN is the input vector. */
28750 ix86_expand_reduc_v4sf (rtx (*fn) (rtx, rtx, rtx), rtx dest, rtx in)
28752 rtx tmp1, tmp2, tmp3;
28754 tmp1 = gen_reg_rtx (V4SFmode);
28755 tmp2 = gen_reg_rtx (V4SFmode);
28756 tmp3 = gen_reg_rtx (V4SFmode);
28758 emit_insn (gen_sse_movhlps (tmp1, in, in));
28759 emit_insn (fn (tmp2, tmp1, in));
28761 emit_insn (gen_sse_shufps_v4sf (tmp3, tmp2, tmp2,
28762 GEN_INT (1), GEN_INT (1),
28763 GEN_INT (1+4), GEN_INT (1+4)));
28764 emit_insn (fn (dest, tmp2, tmp3));
28767 /* Target hook for scalar_mode_supported_p. */
28769 ix86_scalar_mode_supported_p (enum machine_mode mode)
28771 if (DECIMAL_FLOAT_MODE_P (mode))
28773 else if (mode == TFmode)
28776 return default_scalar_mode_supported_p (mode);
28779 /* Implements target hook vector_mode_supported_p. */
28781 ix86_vector_mode_supported_p (enum machine_mode mode)
28783 if (TARGET_SSE && VALID_SSE_REG_MODE (mode))
28785 if (TARGET_SSE2 && VALID_SSE2_REG_MODE (mode))
28787 if (TARGET_AVX && VALID_AVX256_REG_MODE (mode))
28789 if (TARGET_MMX && VALID_MMX_REG_MODE (mode))
28791 if (TARGET_3DNOW && VALID_MMX_REG_MODE_3DNOW (mode))
28796 /* Target hook for c_mode_for_suffix. */
28797 static enum machine_mode
28798 ix86_c_mode_for_suffix (char suffix)
28808 /* Worker function for TARGET_MD_ASM_CLOBBERS.
28810 We do this in the new i386 backend to maintain source compatibility
28811 with the old cc0-based compiler. */
28814 ix86_md_asm_clobbers (tree outputs ATTRIBUTE_UNUSED,
28815 tree inputs ATTRIBUTE_UNUSED,
28818 clobbers = tree_cons (NULL_TREE, build_string (5, "flags"),
28820 clobbers = tree_cons (NULL_TREE, build_string (4, "fpsr"),
28825 /* Implements target vector targetm.asm.encode_section_info. This
28826 is not used by netware. */
28828 static void ATTRIBUTE_UNUSED
28829 ix86_encode_section_info (tree decl, rtx rtl, int first)
28831 default_encode_section_info (decl, rtl, first);
28833 if (TREE_CODE (decl) == VAR_DECL
28834 && (TREE_STATIC (decl) || DECL_EXTERNAL (decl))
28835 && ix86_in_large_data_p (decl))
28836 SYMBOL_REF_FLAGS (XEXP (rtl, 0)) |= SYMBOL_FLAG_FAR_ADDR;
28839 /* Worker function for REVERSE_CONDITION. */
28842 ix86_reverse_condition (enum rtx_code code, enum machine_mode mode)
28844 return (mode != CCFPmode && mode != CCFPUmode
28845 ? reverse_condition (code)
28846 : reverse_condition_maybe_unordered (code));
28849 /* Output code to perform an x87 FP register move, from OPERANDS[1]
28853 output_387_reg_move (rtx insn, rtx *operands)
28855 if (REG_P (operands[0]))
28857 if (REG_P (operands[1])
28858 && find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
28860 if (REGNO (operands[0]) == FIRST_STACK_REG)
28861 return output_387_ffreep (operands, 0);
28862 return "fstp\t%y0";
28864 if (STACK_TOP_P (operands[0]))
28865 return "fld%Z1\t%y1";
28868 else if (MEM_P (operands[0]))
28870 gcc_assert (REG_P (operands[1]));
28871 if (find_regno_note (insn, REG_DEAD, REGNO (operands[1])))
28872 return "fstp%Z0\t%y0";
28875 /* There is no non-popping store to memory for XFmode.
28876 So if we need one, follow the store with a load. */
28877 if (GET_MODE (operands[0]) == XFmode)
28878 return "fstp%Z0\t%y0\n\tfld%Z0\t%y0";
28880 return "fst%Z0\t%y0";
28887 /* Output code to perform a conditional jump to LABEL, if C2 flag in
28888 FP status register is set. */
28891 ix86_emit_fp_unordered_jump (rtx label)
28893 rtx reg = gen_reg_rtx (HImode);
28896 emit_insn (gen_x86_fnstsw_1 (reg));
28898 if (TARGET_SAHF && (TARGET_USE_SAHF || optimize_insn_for_size_p ()))
28900 emit_insn (gen_x86_sahf_1 (reg));
28902 temp = gen_rtx_REG (CCmode, FLAGS_REG);
28903 temp = gen_rtx_UNORDERED (VOIDmode, temp, const0_rtx);
28907 emit_insn (gen_testqi_ext_ccno_0 (reg, GEN_INT (0x04)));
28909 temp = gen_rtx_REG (CCNOmode, FLAGS_REG);
28910 temp = gen_rtx_NE (VOIDmode, temp, const0_rtx);
28913 temp = gen_rtx_IF_THEN_ELSE (VOIDmode, temp,
28914 gen_rtx_LABEL_REF (VOIDmode, label),
28916 temp = gen_rtx_SET (VOIDmode, pc_rtx, temp);
28918 emit_jump_insn (temp);
28919 predict_jump (REG_BR_PROB_BASE * 10 / 100);
28922 /* Output code to perform a log1p XFmode calculation. */
28924 void ix86_emit_i387_log1p (rtx op0, rtx op1)
28926 rtx label1 = gen_label_rtx ();
28927 rtx label2 = gen_label_rtx ();
28929 rtx tmp = gen_reg_rtx (XFmode);
28930 rtx tmp2 = gen_reg_rtx (XFmode);
28933 emit_insn (gen_absxf2 (tmp, op1));
28934 test = gen_rtx_GE (VOIDmode, tmp,
28935 CONST_DOUBLE_FROM_REAL_VALUE (
28936 REAL_VALUE_ATOF ("0.29289321881345247561810596348408353", XFmode),
28938 emit_jump_insn (gen_cbranchxf4 (test, XEXP (test, 0), XEXP (test, 1), label1));
28940 emit_move_insn (tmp2, standard_80387_constant_rtx (4)); /* fldln2 */
28941 emit_insn (gen_fyl2xp1xf3_i387 (op0, op1, tmp2));
28942 emit_jump (label2);
28944 emit_label (label1);
28945 emit_move_insn (tmp, CONST1_RTX (XFmode));
28946 emit_insn (gen_addxf3 (tmp, op1, tmp));
28947 emit_move_insn (tmp2, standard_80387_constant_rtx (4)); /* fldln2 */
28948 emit_insn (gen_fyl2xxf3_i387 (op0, tmp, tmp2));
28950 emit_label (label2);
28953 /* Output code to perform a Newton-Rhapson approximation of a single precision
28954 floating point divide [http://en.wikipedia.org/wiki/N-th_root_algorithm]. */
28956 void ix86_emit_swdivsf (rtx res, rtx a, rtx b, enum machine_mode mode)
28958 rtx x0, x1, e0, e1, two;
28960 x0 = gen_reg_rtx (mode);
28961 e0 = gen_reg_rtx (mode);
28962 e1 = gen_reg_rtx (mode);
28963 x1 = gen_reg_rtx (mode);
28965 two = CONST_DOUBLE_FROM_REAL_VALUE (dconst2, SFmode);
28967 if (VECTOR_MODE_P (mode))
28968 two = ix86_build_const_vector (SFmode, true, two);
28970 two = force_reg (mode, two);
28972 /* a / b = a * rcp(b) * (2.0 - b * rcp(b)) */
28974 /* x0 = rcp(b) estimate */
28975 emit_insn (gen_rtx_SET (VOIDmode, x0,
28976 gen_rtx_UNSPEC (mode, gen_rtvec (1, b),
28979 emit_insn (gen_rtx_SET (VOIDmode, e0,
28980 gen_rtx_MULT (mode, x0, b)));
28982 emit_insn (gen_rtx_SET (VOIDmode, e1,
28983 gen_rtx_MINUS (mode, two, e0)));
28985 emit_insn (gen_rtx_SET (VOIDmode, x1,
28986 gen_rtx_MULT (mode, x0, e1)));
28988 emit_insn (gen_rtx_SET (VOIDmode, res,
28989 gen_rtx_MULT (mode, a, x1)));
28992 /* Output code to perform a Newton-Rhapson approximation of a
28993 single precision floating point [reciprocal] square root. */
28995 void ix86_emit_swsqrtsf (rtx res, rtx a, enum machine_mode mode,
28998 rtx x0, e0, e1, e2, e3, mthree, mhalf;
29001 x0 = gen_reg_rtx (mode);
29002 e0 = gen_reg_rtx (mode);
29003 e1 = gen_reg_rtx (mode);
29004 e2 = gen_reg_rtx (mode);
29005 e3 = gen_reg_rtx (mode);
29007 real_from_integer (&r, VOIDmode, -3, -1, 0);
29008 mthree = CONST_DOUBLE_FROM_REAL_VALUE (r, SFmode);
29010 real_arithmetic (&r, NEGATE_EXPR, &dconsthalf, NULL);
29011 mhalf = CONST_DOUBLE_FROM_REAL_VALUE (r, SFmode);
29013 if (VECTOR_MODE_P (mode))
29015 mthree = ix86_build_const_vector (SFmode, true, mthree);
29016 mhalf = ix86_build_const_vector (SFmode, true, mhalf);
29019 /* sqrt(a) = -0.5 * a * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0)
29020 rsqrt(a) = -0.5 * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0) */
29022 /* x0 = rsqrt(a) estimate */
29023 emit_insn (gen_rtx_SET (VOIDmode, x0,
29024 gen_rtx_UNSPEC (mode, gen_rtvec (1, a),
29027 /* If (a == 0.0) Filter out infinity to prevent NaN for sqrt(0.0). */
29032 zero = gen_reg_rtx (mode);
29033 mask = gen_reg_rtx (mode);
29035 zero = force_reg (mode, CONST0_RTX(mode));
29036 emit_insn (gen_rtx_SET (VOIDmode, mask,
29037 gen_rtx_NE (mode, zero, a)));
29039 emit_insn (gen_rtx_SET (VOIDmode, x0,
29040 gen_rtx_AND (mode, x0, mask)));
29044 emit_insn (gen_rtx_SET (VOIDmode, e0,
29045 gen_rtx_MULT (mode, x0, a)));
29047 emit_insn (gen_rtx_SET (VOIDmode, e1,
29048 gen_rtx_MULT (mode, e0, x0)));
29051 mthree = force_reg (mode, mthree);
29052 emit_insn (gen_rtx_SET (VOIDmode, e2,
29053 gen_rtx_PLUS (mode, e1, mthree)));
29055 mhalf = force_reg (mode, mhalf);
29057 /* e3 = -.5 * x0 */
29058 emit_insn (gen_rtx_SET (VOIDmode, e3,
29059 gen_rtx_MULT (mode, x0, mhalf)));
29061 /* e3 = -.5 * e0 */
29062 emit_insn (gen_rtx_SET (VOIDmode, e3,
29063 gen_rtx_MULT (mode, e0, mhalf)));
29064 /* ret = e2 * e3 */
29065 emit_insn (gen_rtx_SET (VOIDmode, res,
29066 gen_rtx_MULT (mode, e2, e3)));
29069 /* Solaris implementation of TARGET_ASM_NAMED_SECTION. */
29071 static void ATTRIBUTE_UNUSED
29072 i386_solaris_elf_named_section (const char *name, unsigned int flags,
29075 /* With Binutils 2.15, the "@unwind" marker must be specified on
29076 every occurrence of the ".eh_frame" section, not just the first
29079 && strcmp (name, ".eh_frame") == 0)
29081 fprintf (asm_out_file, "\t.section\t%s,\"%s\",@unwind\n", name,
29082 flags & SECTION_WRITE ? "aw" : "a");
29085 default_elf_asm_named_section (name, flags, decl);
29088 /* Return the mangling of TYPE if it is an extended fundamental type. */
29090 static const char *
29091 ix86_mangle_type (const_tree type)
29093 type = TYPE_MAIN_VARIANT (type);
29095 if (TREE_CODE (type) != VOID_TYPE && TREE_CODE (type) != BOOLEAN_TYPE
29096 && TREE_CODE (type) != INTEGER_TYPE && TREE_CODE (type) != REAL_TYPE)
29099 switch (TYPE_MODE (type))
29102 /* __float128 is "g". */
29105 /* "long double" or __float80 is "e". */
29112 /* For 32-bit code we can save PIC register setup by using
29113 __stack_chk_fail_local hidden function instead of calling
29114 __stack_chk_fail directly. 64-bit code doesn't need to setup any PIC
29115 register, so it is better to call __stack_chk_fail directly. */
29118 ix86_stack_protect_fail (void)
29120 return TARGET_64BIT
29121 ? default_external_stack_protect_fail ()
29122 : default_hidden_stack_protect_fail ();
29125 /* Select a format to encode pointers in exception handling data. CODE
29126 is 0 for data, 1 for code labels, 2 for function pointers. GLOBAL is
29127 true if the symbol may be affected by dynamic relocations.
29129 ??? All x86 object file formats are capable of representing this.
29130 After all, the relocation needed is the same as for the call insn.
29131 Whether or not a particular assembler allows us to enter such, I
29132 guess we'll have to see. */
29134 asm_preferred_eh_data_format (int code, int global)
29138 int type = DW_EH_PE_sdata8;
29140 || ix86_cmodel == CM_SMALL_PIC
29141 || (ix86_cmodel == CM_MEDIUM_PIC && (global || code)))
29142 type = DW_EH_PE_sdata4;
29143 return (global ? DW_EH_PE_indirect : 0) | DW_EH_PE_pcrel | type;
29145 if (ix86_cmodel == CM_SMALL
29146 || (ix86_cmodel == CM_MEDIUM && code))
29147 return DW_EH_PE_udata4;
29148 return DW_EH_PE_absptr;
29151 /* Expand copysign from SIGN to the positive value ABS_VALUE
29152 storing in RESULT. If MASK is non-null, it shall be a mask to mask out
29155 ix86_sse_copysign_to_positive (rtx result, rtx abs_value, rtx sign, rtx mask)
29157 enum machine_mode mode = GET_MODE (sign);
29158 rtx sgn = gen_reg_rtx (mode);
29159 if (mask == NULL_RTX)
29161 mask = ix86_build_signbit_mask (mode, VECTOR_MODE_P (mode), false);
29162 if (!VECTOR_MODE_P (mode))
29164 /* We need to generate a scalar mode mask in this case. */
29165 rtx tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
29166 tmp = gen_rtx_VEC_SELECT (mode, mask, tmp);
29167 mask = gen_reg_rtx (mode);
29168 emit_insn (gen_rtx_SET (VOIDmode, mask, tmp));
29172 mask = gen_rtx_NOT (mode, mask);
29173 emit_insn (gen_rtx_SET (VOIDmode, sgn,
29174 gen_rtx_AND (mode, mask, sign)));
29175 emit_insn (gen_rtx_SET (VOIDmode, result,
29176 gen_rtx_IOR (mode, abs_value, sgn)));
29179 /* Expand fabs (OP0) and return a new rtx that holds the result. The
29180 mask for masking out the sign-bit is stored in *SMASK, if that is
29183 ix86_expand_sse_fabs (rtx op0, rtx *smask)
29185 enum machine_mode mode = GET_MODE (op0);
29188 xa = gen_reg_rtx (mode);
29189 mask = ix86_build_signbit_mask (mode, VECTOR_MODE_P (mode), true);
29190 if (!VECTOR_MODE_P (mode))
29192 /* We need to generate a scalar mode mask in this case. */
29193 rtx tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const0_rtx));
29194 tmp = gen_rtx_VEC_SELECT (mode, mask, tmp);
29195 mask = gen_reg_rtx (mode);
29196 emit_insn (gen_rtx_SET (VOIDmode, mask, tmp));
29198 emit_insn (gen_rtx_SET (VOIDmode, xa,
29199 gen_rtx_AND (mode, op0, mask)));
29207 /* Expands a comparison of OP0 with OP1 using comparison code CODE,
29208 swapping the operands if SWAP_OPERANDS is true. The expanded
29209 code is a forward jump to a newly created label in case the
29210 comparison is true. The generated label rtx is returned. */
29212 ix86_expand_sse_compare_and_jump (enum rtx_code code, rtx op0, rtx op1,
29213 bool swap_operands)
29224 label = gen_label_rtx ();
29225 tmp = gen_rtx_REG (CCFPUmode, FLAGS_REG);
29226 emit_insn (gen_rtx_SET (VOIDmode, tmp,
29227 gen_rtx_COMPARE (CCFPUmode, op0, op1)));
29228 tmp = gen_rtx_fmt_ee (code, VOIDmode, tmp, const0_rtx);
29229 tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp,
29230 gen_rtx_LABEL_REF (VOIDmode, label), pc_rtx);
29231 tmp = emit_jump_insn (gen_rtx_SET (VOIDmode, pc_rtx, tmp));
29232 JUMP_LABEL (tmp) = label;
29237 /* Expand a mask generating SSE comparison instruction comparing OP0 with OP1
29238 using comparison code CODE. Operands are swapped for the comparison if
29239 SWAP_OPERANDS is true. Returns a rtx for the generated mask. */
29241 ix86_expand_sse_compare_mask (enum rtx_code code, rtx op0, rtx op1,
29242 bool swap_operands)
29244 enum machine_mode mode = GET_MODE (op0);
29245 rtx mask = gen_reg_rtx (mode);
29254 if (mode == DFmode)
29255 emit_insn (gen_sse2_maskcmpdf3 (mask, op0, op1,
29256 gen_rtx_fmt_ee (code, mode, op0, op1)));
29258 emit_insn (gen_sse_maskcmpsf3 (mask, op0, op1,
29259 gen_rtx_fmt_ee (code, mode, op0, op1)));
29264 /* Generate and return a rtx of mode MODE for 2**n where n is the number
29265 of bits of the mantissa of MODE, which must be one of DFmode or SFmode. */
29267 ix86_gen_TWO52 (enum machine_mode mode)
29269 REAL_VALUE_TYPE TWO52r;
29272 real_ldexp (&TWO52r, &dconst1, mode == DFmode ? 52 : 23);
29273 TWO52 = const_double_from_real_value (TWO52r, mode);
29274 TWO52 = force_reg (mode, TWO52);
29279 /* Expand SSE sequence for computing lround from OP1 storing
29282 ix86_expand_lround (rtx op0, rtx op1)
29284 /* C code for the stuff we're doing below:
29285 tmp = op1 + copysign (nextafter (0.5, 0.0), op1)
29288 enum machine_mode mode = GET_MODE (op1);
29289 const struct real_format *fmt;
29290 REAL_VALUE_TYPE pred_half, half_minus_pred_half;
29293 /* load nextafter (0.5, 0.0) */
29294 fmt = REAL_MODE_FORMAT (mode);
29295 real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
29296 REAL_ARITHMETIC (pred_half, MINUS_EXPR, dconsthalf, half_minus_pred_half);
29298 /* adj = copysign (0.5, op1) */
29299 adj = force_reg (mode, const_double_from_real_value (pred_half, mode));
29300 ix86_sse_copysign_to_positive (adj, adj, force_reg (mode, op1), NULL_RTX);
29302 /* adj = op1 + adj */
29303 adj = expand_simple_binop (mode, PLUS, adj, op1, NULL_RTX, 0, OPTAB_DIRECT);
29305 /* op0 = (imode)adj */
29306 expand_fix (op0, adj, 0);
29309 /* Expand SSE2 sequence for computing lround from OPERAND1 storing
29312 ix86_expand_lfloorceil (rtx op0, rtx op1, bool do_floor)
29314 /* C code for the stuff we're doing below (for do_floor):
29316 xi -= (double)xi > op1 ? 1 : 0;
29319 enum machine_mode fmode = GET_MODE (op1);
29320 enum machine_mode imode = GET_MODE (op0);
29321 rtx ireg, freg, label, tmp;
29323 /* reg = (long)op1 */
29324 ireg = gen_reg_rtx (imode);
29325 expand_fix (ireg, op1, 0);
29327 /* freg = (double)reg */
29328 freg = gen_reg_rtx (fmode);
29329 expand_float (freg, ireg, 0);
29331 /* ireg = (freg > op1) ? ireg - 1 : ireg */
29332 label = ix86_expand_sse_compare_and_jump (UNLE,
29333 freg, op1, !do_floor);
29334 tmp = expand_simple_binop (imode, do_floor ? MINUS : PLUS,
29335 ireg, const1_rtx, NULL_RTX, 0, OPTAB_DIRECT);
29336 emit_move_insn (ireg, tmp);
29338 emit_label (label);
29339 LABEL_NUSES (label) = 1;
29341 emit_move_insn (op0, ireg);
29344 /* Expand rint (IEEE round to nearest) rounding OPERAND1 and storing the
29345 result in OPERAND0. */
29347 ix86_expand_rint (rtx operand0, rtx operand1)
29349 /* C code for the stuff we're doing below:
29350 xa = fabs (operand1);
29351 if (!isless (xa, 2**52))
29353 xa = xa + 2**52 - 2**52;
29354 return copysign (xa, operand1);
29356 enum machine_mode mode = GET_MODE (operand0);
29357 rtx res, xa, label, TWO52, mask;
29359 res = gen_reg_rtx (mode);
29360 emit_move_insn (res, operand1);
29362 /* xa = abs (operand1) */
29363 xa = ix86_expand_sse_fabs (res, &mask);
29365 /* if (!isless (xa, TWO52)) goto label; */
29366 TWO52 = ix86_gen_TWO52 (mode);
29367 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
29369 xa = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
29370 xa = expand_simple_binop (mode, MINUS, xa, TWO52, xa, 0, OPTAB_DIRECT);
29372 ix86_sse_copysign_to_positive (res, xa, res, mask);
29374 emit_label (label);
29375 LABEL_NUSES (label) = 1;
29377 emit_move_insn (operand0, res);
29380 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
29383 ix86_expand_floorceildf_32 (rtx operand0, rtx operand1, bool do_floor)
29385 /* C code for the stuff we expand below.
29386 double xa = fabs (x), x2;
29387 if (!isless (xa, TWO52))
29389 xa = xa + TWO52 - TWO52;
29390 x2 = copysign (xa, x);
29399 enum machine_mode mode = GET_MODE (operand0);
29400 rtx xa, TWO52, tmp, label, one, res, mask;
29402 TWO52 = ix86_gen_TWO52 (mode);
29404 /* Temporary for holding the result, initialized to the input
29405 operand to ease control flow. */
29406 res = gen_reg_rtx (mode);
29407 emit_move_insn (res, operand1);
29409 /* xa = abs (operand1) */
29410 xa = ix86_expand_sse_fabs (res, &mask);
29412 /* if (!isless (xa, TWO52)) goto label; */
29413 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
29415 /* xa = xa + TWO52 - TWO52; */
29416 xa = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
29417 xa = expand_simple_binop (mode, MINUS, xa, TWO52, xa, 0, OPTAB_DIRECT);
29419 /* xa = copysign (xa, operand1) */
29420 ix86_sse_copysign_to_positive (xa, xa, res, mask);
29422 /* generate 1.0 or -1.0 */
29423 one = force_reg (mode,
29424 const_double_from_real_value (do_floor
29425 ? dconst1 : dconstm1, mode));
29427 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
29428 tmp = ix86_expand_sse_compare_mask (UNGT, xa, res, !do_floor);
29429 emit_insn (gen_rtx_SET (VOIDmode, tmp,
29430 gen_rtx_AND (mode, one, tmp)));
29431 /* We always need to subtract here to preserve signed zero. */
29432 tmp = expand_simple_binop (mode, MINUS,
29433 xa, tmp, NULL_RTX, 0, OPTAB_DIRECT);
29434 emit_move_insn (res, tmp);
29436 emit_label (label);
29437 LABEL_NUSES (label) = 1;
29439 emit_move_insn (operand0, res);
29442 /* Expand SSE2 sequence for computing floor or ceil from OPERAND1 storing
29445 ix86_expand_floorceil (rtx operand0, rtx operand1, bool do_floor)
29447 /* C code for the stuff we expand below.
29448 double xa = fabs (x), x2;
29449 if (!isless (xa, TWO52))
29451 x2 = (double)(long)x;
29458 if (HONOR_SIGNED_ZEROS (mode))
29459 return copysign (x2, x);
29462 enum machine_mode mode = GET_MODE (operand0);
29463 rtx xa, xi, TWO52, tmp, label, one, res, mask;
29465 TWO52 = ix86_gen_TWO52 (mode);
29467 /* Temporary for holding the result, initialized to the input
29468 operand to ease control flow. */
29469 res = gen_reg_rtx (mode);
29470 emit_move_insn (res, operand1);
29472 /* xa = abs (operand1) */
29473 xa = ix86_expand_sse_fabs (res, &mask);
29475 /* if (!isless (xa, TWO52)) goto label; */
29476 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
29478 /* xa = (double)(long)x */
29479 xi = gen_reg_rtx (mode == DFmode ? DImode : SImode);
29480 expand_fix (xi, res, 0);
29481 expand_float (xa, xi, 0);
29484 one = force_reg (mode, const_double_from_real_value (dconst1, mode));
29486 /* Compensate: xa = xa - (xa > operand1 ? 1 : 0) */
29487 tmp = ix86_expand_sse_compare_mask (UNGT, xa, res, !do_floor);
29488 emit_insn (gen_rtx_SET (VOIDmode, tmp,
29489 gen_rtx_AND (mode, one, tmp)));
29490 tmp = expand_simple_binop (mode, do_floor ? MINUS : PLUS,
29491 xa, tmp, NULL_RTX, 0, OPTAB_DIRECT);
29492 emit_move_insn (res, tmp);
29494 if (HONOR_SIGNED_ZEROS (mode))
29495 ix86_sse_copysign_to_positive (res, res, force_reg (mode, operand1), mask);
29497 emit_label (label);
29498 LABEL_NUSES (label) = 1;
29500 emit_move_insn (operand0, res);
29503 /* Expand SSE sequence for computing round from OPERAND1 storing
29504 into OPERAND0. Sequence that works without relying on DImode truncation
29505 via cvttsd2siq that is only available on 64bit targets. */
29507 ix86_expand_rounddf_32 (rtx operand0, rtx operand1)
29509 /* C code for the stuff we expand below.
29510 double xa = fabs (x), xa2, x2;
29511 if (!isless (xa, TWO52))
29513 Using the absolute value and copying back sign makes
29514 -0.0 -> -0.0 correct.
29515 xa2 = xa + TWO52 - TWO52;
29520 else if (dxa > 0.5)
29522 x2 = copysign (xa2, x);
29525 enum machine_mode mode = GET_MODE (operand0);
29526 rtx xa, xa2, dxa, TWO52, tmp, label, half, mhalf, one, res, mask;
29528 TWO52 = ix86_gen_TWO52 (mode);
29530 /* Temporary for holding the result, initialized to the input
29531 operand to ease control flow. */
29532 res = gen_reg_rtx (mode);
29533 emit_move_insn (res, operand1);
29535 /* xa = abs (operand1) */
29536 xa = ix86_expand_sse_fabs (res, &mask);
29538 /* if (!isless (xa, TWO52)) goto label; */
29539 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
29541 /* xa2 = xa + TWO52 - TWO52; */
29542 xa2 = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
29543 xa2 = expand_simple_binop (mode, MINUS, xa2, TWO52, xa2, 0, OPTAB_DIRECT);
29545 /* dxa = xa2 - xa; */
29546 dxa = expand_simple_binop (mode, MINUS, xa2, xa, NULL_RTX, 0, OPTAB_DIRECT);
29548 /* generate 0.5, 1.0 and -0.5 */
29549 half = force_reg (mode, const_double_from_real_value (dconsthalf, mode));
29550 one = expand_simple_binop (mode, PLUS, half, half, NULL_RTX, 0, OPTAB_DIRECT);
29551 mhalf = expand_simple_binop (mode, MINUS, half, one, NULL_RTX,
29555 tmp = gen_reg_rtx (mode);
29556 /* xa2 = xa2 - (dxa > 0.5 ? 1 : 0) */
29557 tmp = ix86_expand_sse_compare_mask (UNGT, dxa, half, false);
29558 emit_insn (gen_rtx_SET (VOIDmode, tmp,
29559 gen_rtx_AND (mode, one, tmp)));
29560 xa2 = expand_simple_binop (mode, MINUS, xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
29561 /* xa2 = xa2 + (dxa <= -0.5 ? 1 : 0) */
29562 tmp = ix86_expand_sse_compare_mask (UNGE, mhalf, dxa, false);
29563 emit_insn (gen_rtx_SET (VOIDmode, tmp,
29564 gen_rtx_AND (mode, one, tmp)));
29565 xa2 = expand_simple_binop (mode, PLUS, xa2, tmp, NULL_RTX, 0, OPTAB_DIRECT);
29567 /* res = copysign (xa2, operand1) */
29568 ix86_sse_copysign_to_positive (res, xa2, force_reg (mode, operand1), mask);
29570 emit_label (label);
29571 LABEL_NUSES (label) = 1;
29573 emit_move_insn (operand0, res);
29576 /* Expand SSE sequence for computing trunc from OPERAND1 storing
29579 ix86_expand_trunc (rtx operand0, rtx operand1)
29581 /* C code for SSE variant we expand below.
29582 double xa = fabs (x), x2;
29583 if (!isless (xa, TWO52))
29585 x2 = (double)(long)x;
29586 if (HONOR_SIGNED_ZEROS (mode))
29587 return copysign (x2, x);
29590 enum machine_mode mode = GET_MODE (operand0);
29591 rtx xa, xi, TWO52, label, res, mask;
29593 TWO52 = ix86_gen_TWO52 (mode);
29595 /* Temporary for holding the result, initialized to the input
29596 operand to ease control flow. */
29597 res = gen_reg_rtx (mode);
29598 emit_move_insn (res, operand1);
29600 /* xa = abs (operand1) */
29601 xa = ix86_expand_sse_fabs (res, &mask);
29603 /* if (!isless (xa, TWO52)) goto label; */
29604 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
29606 /* x = (double)(long)x */
29607 xi = gen_reg_rtx (mode == DFmode ? DImode : SImode);
29608 expand_fix (xi, res, 0);
29609 expand_float (res, xi, 0);
29611 if (HONOR_SIGNED_ZEROS (mode))
29612 ix86_sse_copysign_to_positive (res, res, force_reg (mode, operand1), mask);
29614 emit_label (label);
29615 LABEL_NUSES (label) = 1;
29617 emit_move_insn (operand0, res);
29620 /* Expand SSE sequence for computing trunc from OPERAND1 storing
29623 ix86_expand_truncdf_32 (rtx operand0, rtx operand1)
29625 enum machine_mode mode = GET_MODE (operand0);
29626 rtx xa, mask, TWO52, label, one, res, smask, tmp;
29628 /* C code for SSE variant we expand below.
29629 double xa = fabs (x), x2;
29630 if (!isless (xa, TWO52))
29632 xa2 = xa + TWO52 - TWO52;
29636 x2 = copysign (xa2, x);
29640 TWO52 = ix86_gen_TWO52 (mode);
29642 /* Temporary for holding the result, initialized to the input
29643 operand to ease control flow. */
29644 res = gen_reg_rtx (mode);
29645 emit_move_insn (res, operand1);
29647 /* xa = abs (operand1) */
29648 xa = ix86_expand_sse_fabs (res, &smask);
29650 /* if (!isless (xa, TWO52)) goto label; */
29651 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
29653 /* res = xa + TWO52 - TWO52; */
29654 tmp = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT);
29655 tmp = expand_simple_binop (mode, MINUS, tmp, TWO52, tmp, 0, OPTAB_DIRECT);
29656 emit_move_insn (res, tmp);
29659 one = force_reg (mode, const_double_from_real_value (dconst1, mode));
29661 /* Compensate: res = xa2 - (res > xa ? 1 : 0) */
29662 mask = ix86_expand_sse_compare_mask (UNGT, res, xa, false);
29663 emit_insn (gen_rtx_SET (VOIDmode, mask,
29664 gen_rtx_AND (mode, mask, one)));
29665 tmp = expand_simple_binop (mode, MINUS,
29666 res, mask, NULL_RTX, 0, OPTAB_DIRECT);
29667 emit_move_insn (res, tmp);
29669 /* res = copysign (res, operand1) */
29670 ix86_sse_copysign_to_positive (res, res, force_reg (mode, operand1), smask);
29672 emit_label (label);
29673 LABEL_NUSES (label) = 1;
29675 emit_move_insn (operand0, res);
29678 /* Expand SSE sequence for computing round from OPERAND1 storing
29681 ix86_expand_round (rtx operand0, rtx operand1)
29683 /* C code for the stuff we're doing below:
29684 double xa = fabs (x);
29685 if (!isless (xa, TWO52))
29687 xa = (double)(long)(xa + nextafter (0.5, 0.0));
29688 return copysign (xa, x);
29690 enum machine_mode mode = GET_MODE (operand0);
29691 rtx res, TWO52, xa, label, xi, half, mask;
29692 const struct real_format *fmt;
29693 REAL_VALUE_TYPE pred_half, half_minus_pred_half;
29695 /* Temporary for holding the result, initialized to the input
29696 operand to ease control flow. */
29697 res = gen_reg_rtx (mode);
29698 emit_move_insn (res, operand1);
29700 TWO52 = ix86_gen_TWO52 (mode);
29701 xa = ix86_expand_sse_fabs (res, &mask);
29702 label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false);
29704 /* load nextafter (0.5, 0.0) */
29705 fmt = REAL_MODE_FORMAT (mode);
29706 real_2expN (&half_minus_pred_half, -(fmt->p) - 1, mode);
29707 REAL_ARITHMETIC (pred_half, MINUS_EXPR, dconsthalf, half_minus_pred_half);
29709 /* xa = xa + 0.5 */
29710 half = force_reg (mode, const_double_from_real_value (pred_half, mode));
29711 xa = expand_simple_binop (mode, PLUS, xa, half, NULL_RTX, 0, OPTAB_DIRECT);
29713 /* xa = (double)(int64_t)xa */
29714 xi = gen_reg_rtx (mode == DFmode ? DImode : SImode);
29715 expand_fix (xi, xa, 0);
29716 expand_float (xa, xi, 0);
29718 /* res = copysign (xa, operand1) */
29719 ix86_sse_copysign_to_positive (res, xa, force_reg (mode, operand1), mask);
29721 emit_label (label);
29722 LABEL_NUSES (label) = 1;
29724 emit_move_insn (operand0, res);
29728 /* Validate whether a SSE5 instruction is valid or not.
29729 OPERANDS is the array of operands.
29730 NUM is the number of operands.
29731 USES_OC0 is true if the instruction uses OC0 and provides 4 variants.
29732 NUM_MEMORY is the maximum number of memory operands to accept.
29733 when COMMUTATIVE is set, operand 1 and 2 can be swapped. */
29736 ix86_sse5_valid_op_p (rtx operands[], rtx insn ATTRIBUTE_UNUSED, int num,
29737 bool uses_oc0, int num_memory, bool commutative)
29743 /* Count the number of memory arguments */
29746 for (i = 0; i < num; i++)
29748 enum machine_mode mode = GET_MODE (operands[i]);
29749 if (register_operand (operands[i], mode))
29752 else if (memory_operand (operands[i], mode))
29754 mem_mask |= (1 << i);
29760 rtx pattern = PATTERN (insn);
29762 /* allow 0 for pcmov */
29763 if (GET_CODE (pattern) != SET
29764 || GET_CODE (SET_SRC (pattern)) != IF_THEN_ELSE
29766 || operands[i] != CONST0_RTX (mode))
29771 /* Special case pmacsdq{l,h} where we allow the 3rd argument to be
29772 a memory operation. */
29773 if (num_memory < 0)
29775 num_memory = -num_memory;
29776 if ((mem_mask & (1 << (num-1))) != 0)
29778 mem_mask &= ~(1 << (num-1));
29783 /* If there were no memory operations, allow the insn */
29787 /* Do not allow the destination register to be a memory operand. */
29788 else if (mem_mask & (1 << 0))
29791 /* If there are too many memory operations, disallow the instruction. While
29792 the hardware only allows 1 memory reference, before register allocation
29793 for some insns, we allow two memory operations sometimes in order to allow
29794 code like the following to be optimized:
29796 float fmadd (float *a, float *b, float *c) { return (*a * *b) + *c; }
29798 or similar cases that are vectorized into using the fmaddss
29800 else if (mem_count > num_memory)
29803 /* Don't allow more than one memory operation if not optimizing. */
29804 else if (mem_count > 1 && !optimize)
29807 else if (num == 4 && mem_count == 1)
29809 /* formats (destination is the first argument), example fmaddss:
29810 xmm1, xmm1, xmm2, xmm3/mem
29811 xmm1, xmm1, xmm2/mem, xmm3
29812 xmm1, xmm2, xmm3/mem, xmm1
29813 xmm1, xmm2/mem, xmm3, xmm1 */
29815 return ((mem_mask == (1 << 1))
29816 || (mem_mask == (1 << 2))
29817 || (mem_mask == (1 << 3)));
29819 /* format, example pmacsdd:
29820 xmm1, xmm2, xmm3/mem, xmm1 */
29822 return (mem_mask == (1 << 2) || mem_mask == (1 << 1));
29824 return (mem_mask == (1 << 2));
29827 else if (num == 4 && num_memory == 2)
29829 /* If there are two memory operations, we can load one of the memory ops
29830 into the destination register. This is for optimizing the
29831 multiply/add ops, which the combiner has optimized both the multiply
29832 and the add insns to have a memory operation. We have to be careful
29833 that the destination doesn't overlap with the inputs. */
29834 rtx op0 = operands[0];
29836 if (reg_mentioned_p (op0, operands[1])
29837 || reg_mentioned_p (op0, operands[2])
29838 || reg_mentioned_p (op0, operands[3]))
29841 /* formats (destination is the first argument), example fmaddss:
29842 xmm1, xmm1, xmm2, xmm3/mem
29843 xmm1, xmm1, xmm2/mem, xmm3
29844 xmm1, xmm2, xmm3/mem, xmm1
29845 xmm1, xmm2/mem, xmm3, xmm1
29847 For the oc0 case, we will load either operands[1] or operands[3] into
29848 operands[0], so any combination of 2 memory operands is ok. */
29852 /* format, example pmacsdd:
29853 xmm1, xmm2, xmm3/mem, xmm1
29855 For the integer multiply/add instructions be more restrictive and
29856 require operands[2] and operands[3] to be the memory operands. */
29858 return (mem_mask == ((1 << 1) | (1 << 3)) || ((1 << 2) | (1 << 3)));
29860 return (mem_mask == ((1 << 2) | (1 << 3)));
29863 else if (num == 3 && num_memory == 1)
29865 /* formats, example protb:
29866 xmm1, xmm2, xmm3/mem
29867 xmm1, xmm2/mem, xmm3 */
29869 return ((mem_mask == (1 << 1)) || (mem_mask == (1 << 2)));
29871 /* format, example comeq:
29872 xmm1, xmm2, xmm3/mem */
29874 return (mem_mask == (1 << 2));
29878 gcc_unreachable ();
29884 /* Fixup an SSE5 instruction that has 2 memory input references into a form the
29885 hardware will allow by using the destination register to load one of the
29886 memory operations. Presently this is used by the multiply/add routines to
29887 allow 2 memory references. */
29890 ix86_expand_sse5_multiple_memory (rtx operands[],
29892 enum machine_mode mode)
29894 rtx op0 = operands[0];
29896 || memory_operand (op0, mode)
29897 || reg_mentioned_p (op0, operands[1])
29898 || reg_mentioned_p (op0, operands[2])
29899 || reg_mentioned_p (op0, operands[3]))
29900 gcc_unreachable ();
29902 /* For 2 memory operands, pick either operands[1] or operands[3] to move into
29903 the destination register. */
29904 if (memory_operand (operands[1], mode))
29906 emit_move_insn (op0, operands[1]);
29909 else if (memory_operand (operands[3], mode))
29911 emit_move_insn (op0, operands[3]);
29915 gcc_unreachable ();
29921 /* Table of valid machine attributes. */
29922 static const struct attribute_spec ix86_attribute_table[] =
29924 /* { name, min_len, max_len, decl_req, type_req, fn_type_req, handler } */
29925 /* Stdcall attribute says callee is responsible for popping arguments
29926 if they are not variable. */
29927 { "stdcall", 0, 0, false, true, true, ix86_handle_cconv_attribute },
29928 /* Fastcall attribute says callee is responsible for popping arguments
29929 if they are not variable. */
29930 { "fastcall", 0, 0, false, true, true, ix86_handle_cconv_attribute },
29931 /* Cdecl attribute says the callee is a normal C declaration */
29932 { "cdecl", 0, 0, false, true, true, ix86_handle_cconv_attribute },
29933 /* Regparm attribute specifies how many integer arguments are to be
29934 passed in registers. */
29935 { "regparm", 1, 1, false, true, true, ix86_handle_cconv_attribute },
29936 /* Sseregparm attribute says we are using x86_64 calling conventions
29937 for FP arguments. */
29938 { "sseregparm", 0, 0, false, true, true, ix86_handle_cconv_attribute },
29939 /* force_align_arg_pointer says this function realigns the stack at entry. */
29940 { (const char *)&ix86_force_align_arg_pointer_string, 0, 0,
29941 false, true, true, ix86_handle_cconv_attribute },
29942 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
29943 { "dllimport", 0, 0, false, false, false, handle_dll_attribute },
29944 { "dllexport", 0, 0, false, false, false, handle_dll_attribute },
29945 { "shared", 0, 0, true, false, false, ix86_handle_shared_attribute },
29947 { "ms_struct", 0, 0, false, false, false, ix86_handle_struct_attribute },
29948 { "gcc_struct", 0, 0, false, false, false, ix86_handle_struct_attribute },
29949 #ifdef SUBTARGET_ATTRIBUTE_TABLE
29950 SUBTARGET_ATTRIBUTE_TABLE,
29952 /* ms_abi and sysv_abi calling convention function attributes. */
29953 { "ms_abi", 0, 0, false, true, true, ix86_handle_abi_attribute },
29954 { "sysv_abi", 0, 0, false, true, true, ix86_handle_abi_attribute },
29956 { NULL, 0, 0, false, false, false, NULL }
29959 /* Implement targetm.vectorize.builtin_vectorization_cost. */
29961 x86_builtin_vectorization_cost (bool runtime_test)
29963 /* If the branch of the runtime test is taken - i.e. - the vectorized
29964 version is skipped - this incurs a misprediction cost (because the
29965 vectorized version is expected to be the fall-through). So we subtract
29966 the latency of a mispredicted branch from the costs that are incured
29967 when the vectorized version is executed.
29969 TODO: The values in individual target tables have to be tuned or new
29970 fields may be needed. For eg. on K8, the default branch path is the
29971 not-taken path. If the taken path is predicted correctly, the minimum
29972 penalty of going down the taken-path is 1 cycle. If the taken-path is
29973 not predicted correctly, then the minimum penalty is 10 cycles. */
29977 return (-(ix86_cost->cond_taken_branch_cost));
29983 /* This function returns the calling abi specific va_list type node.
29984 It returns the FNDECL specific va_list type. */
29987 ix86_fn_abi_va_list (tree fndecl)
29990 return va_list_type_node;
29991 gcc_assert (fndecl != NULL_TREE);
29993 if (ix86_function_abi ((const_tree) fndecl) == MS_ABI)
29994 return ms_va_list_type_node;
29996 return sysv_va_list_type_node;
29999 /* Returns the canonical va_list type specified by TYPE. If there
30000 is no valid TYPE provided, it return NULL_TREE. */
30003 ix86_canonical_va_list_type (tree type)
30007 /* Resolve references and pointers to va_list type. */
30008 if (INDIRECT_REF_P (type))
30009 type = TREE_TYPE (type);
30010 else if (POINTER_TYPE_P (type) && POINTER_TYPE_P (TREE_TYPE(type)))
30011 type = TREE_TYPE (type);
30015 wtype = va_list_type_node;
30016 gcc_assert (wtype != NULL_TREE);
30018 if (TREE_CODE (wtype) == ARRAY_TYPE)
30020 /* If va_list is an array type, the argument may have decayed
30021 to a pointer type, e.g. by being passed to another function.
30022 In that case, unwrap both types so that we can compare the
30023 underlying records. */
30024 if (TREE_CODE (htype) == ARRAY_TYPE
30025 || POINTER_TYPE_P (htype))
30027 wtype = TREE_TYPE (wtype);
30028 htype = TREE_TYPE (htype);
30031 if (TYPE_MAIN_VARIANT (wtype) == TYPE_MAIN_VARIANT (htype))
30032 return va_list_type_node;
30033 wtype = sysv_va_list_type_node;
30034 gcc_assert (wtype != NULL_TREE);
30036 if (TREE_CODE (wtype) == ARRAY_TYPE)
30038 /* If va_list is an array type, the argument may have decayed
30039 to a pointer type, e.g. by being passed to another function.
30040 In that case, unwrap both types so that we can compare the
30041 underlying records. */
30042 if (TREE_CODE (htype) == ARRAY_TYPE
30043 || POINTER_TYPE_P (htype))
30045 wtype = TREE_TYPE (wtype);
30046 htype = TREE_TYPE (htype);
30049 if (TYPE_MAIN_VARIANT (wtype) == TYPE_MAIN_VARIANT (htype))
30050 return sysv_va_list_type_node;
30051 wtype = ms_va_list_type_node;
30052 gcc_assert (wtype != NULL_TREE);
30054 if (TREE_CODE (wtype) == ARRAY_TYPE)
30056 /* If va_list is an array type, the argument may have decayed
30057 to a pointer type, e.g. by being passed to another function.
30058 In that case, unwrap both types so that we can compare the
30059 underlying records. */
30060 if (TREE_CODE (htype) == ARRAY_TYPE
30061 || POINTER_TYPE_P (htype))
30063 wtype = TREE_TYPE (wtype);
30064 htype = TREE_TYPE (htype);
30067 if (TYPE_MAIN_VARIANT (wtype) == TYPE_MAIN_VARIANT (htype))
30068 return ms_va_list_type_node;
30071 return std_canonical_va_list_type (type);
30074 /* Iterate through the target-specific builtin types for va_list.
30075 IDX denotes the iterator, *PTREE is set to the result type of
30076 the va_list builtin, and *PNAME to its internal type.
30077 Returns zero if there is no element for this index, otherwise
30078 IDX should be increased upon the next call.
30079 Note, do not iterate a base builtin's name like __builtin_va_list.
30080 Used from c_common_nodes_and_builtins. */
30083 ix86_enum_va_list (int idx, const char **pname, tree *ptree)
30089 *ptree = ms_va_list_type_node;
30090 *pname = "__builtin_ms_va_list";
30093 *ptree = sysv_va_list_type_node;
30094 *pname = "__builtin_sysv_va_list";
30102 /* Initialize the GCC target structure. */
30103 #undef TARGET_RETURN_IN_MEMORY
30104 #define TARGET_RETURN_IN_MEMORY ix86_return_in_memory
30106 #undef TARGET_LEGITIMIZE_ADDRESS
30107 #define TARGET_LEGITIMIZE_ADDRESS ix86_legitimize_address
30109 #undef TARGET_ATTRIBUTE_TABLE
30110 #define TARGET_ATTRIBUTE_TABLE ix86_attribute_table
30111 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
30112 # undef TARGET_MERGE_DECL_ATTRIBUTES
30113 # define TARGET_MERGE_DECL_ATTRIBUTES merge_dllimport_decl_attributes
30116 #undef TARGET_COMP_TYPE_ATTRIBUTES
30117 #define TARGET_COMP_TYPE_ATTRIBUTES ix86_comp_type_attributes
30119 #undef TARGET_INIT_BUILTINS
30120 #define TARGET_INIT_BUILTINS ix86_init_builtins
30121 #undef TARGET_EXPAND_BUILTIN
30122 #define TARGET_EXPAND_BUILTIN ix86_expand_builtin
30124 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION
30125 #define TARGET_VECTORIZE_BUILTIN_VECTORIZED_FUNCTION \
30126 ix86_builtin_vectorized_function
30128 #undef TARGET_VECTORIZE_BUILTIN_CONVERSION
30129 #define TARGET_VECTORIZE_BUILTIN_CONVERSION ix86_vectorize_builtin_conversion
30131 #undef TARGET_BUILTIN_RECIPROCAL
30132 #define TARGET_BUILTIN_RECIPROCAL ix86_builtin_reciprocal
30134 #undef TARGET_ASM_FUNCTION_EPILOGUE
30135 #define TARGET_ASM_FUNCTION_EPILOGUE ix86_output_function_epilogue
30137 #undef TARGET_ENCODE_SECTION_INFO
30138 #ifndef SUBTARGET_ENCODE_SECTION_INFO
30139 #define TARGET_ENCODE_SECTION_INFO ix86_encode_section_info
30141 #define TARGET_ENCODE_SECTION_INFO SUBTARGET_ENCODE_SECTION_INFO
30144 #undef TARGET_ASM_OPEN_PAREN
30145 #define TARGET_ASM_OPEN_PAREN ""
30146 #undef TARGET_ASM_CLOSE_PAREN
30147 #define TARGET_ASM_CLOSE_PAREN ""
30149 #undef TARGET_ASM_ALIGNED_HI_OP
30150 #define TARGET_ASM_ALIGNED_HI_OP ASM_SHORT
30151 #undef TARGET_ASM_ALIGNED_SI_OP
30152 #define TARGET_ASM_ALIGNED_SI_OP ASM_LONG
30154 #undef TARGET_ASM_ALIGNED_DI_OP
30155 #define TARGET_ASM_ALIGNED_DI_OP ASM_QUAD
30158 #undef TARGET_ASM_UNALIGNED_HI_OP
30159 #define TARGET_ASM_UNALIGNED_HI_OP TARGET_ASM_ALIGNED_HI_OP
30160 #undef TARGET_ASM_UNALIGNED_SI_OP
30161 #define TARGET_ASM_UNALIGNED_SI_OP TARGET_ASM_ALIGNED_SI_OP
30162 #undef TARGET_ASM_UNALIGNED_DI_OP
30163 #define TARGET_ASM_UNALIGNED_DI_OP TARGET_ASM_ALIGNED_DI_OP
30165 #undef TARGET_SCHED_ADJUST_COST
30166 #define TARGET_SCHED_ADJUST_COST ix86_adjust_cost
30167 #undef TARGET_SCHED_ISSUE_RATE
30168 #define TARGET_SCHED_ISSUE_RATE ix86_issue_rate
30169 #undef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD
30170 #define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DFA_LOOKAHEAD \
30171 ia32_multipass_dfa_lookahead
30173 #undef TARGET_FUNCTION_OK_FOR_SIBCALL
30174 #define TARGET_FUNCTION_OK_FOR_SIBCALL ix86_function_ok_for_sibcall
30177 #undef TARGET_HAVE_TLS
30178 #define TARGET_HAVE_TLS true
30180 #undef TARGET_CANNOT_FORCE_CONST_MEM
30181 #define TARGET_CANNOT_FORCE_CONST_MEM ix86_cannot_force_const_mem
30182 #undef TARGET_USE_BLOCKS_FOR_CONSTANT_P
30183 #define TARGET_USE_BLOCKS_FOR_CONSTANT_P hook_bool_mode_const_rtx_true
30185 #undef TARGET_DELEGITIMIZE_ADDRESS
30186 #define TARGET_DELEGITIMIZE_ADDRESS ix86_delegitimize_address
30188 #undef TARGET_MS_BITFIELD_LAYOUT_P
30189 #define TARGET_MS_BITFIELD_LAYOUT_P ix86_ms_bitfield_layout_p
30192 #undef TARGET_BINDS_LOCAL_P
30193 #define TARGET_BINDS_LOCAL_P darwin_binds_local_p
30195 #if TARGET_DLLIMPORT_DECL_ATTRIBUTES
30196 #undef TARGET_BINDS_LOCAL_P
30197 #define TARGET_BINDS_LOCAL_P i386_pe_binds_local_p
30200 #undef TARGET_ASM_OUTPUT_MI_THUNK
30201 #define TARGET_ASM_OUTPUT_MI_THUNK x86_output_mi_thunk
30202 #undef TARGET_ASM_CAN_OUTPUT_MI_THUNK
30203 #define TARGET_ASM_CAN_OUTPUT_MI_THUNK x86_can_output_mi_thunk
30205 #undef TARGET_ASM_FILE_START
30206 #define TARGET_ASM_FILE_START x86_file_start
30208 #undef TARGET_DEFAULT_TARGET_FLAGS
30209 #define TARGET_DEFAULT_TARGET_FLAGS \
30211 | TARGET_SUBTARGET_DEFAULT \
30212 | TARGET_TLS_DIRECT_SEG_REFS_DEFAULT)
30214 #undef TARGET_HANDLE_OPTION
30215 #define TARGET_HANDLE_OPTION ix86_handle_option
30217 #undef TARGET_RTX_COSTS
30218 #define TARGET_RTX_COSTS ix86_rtx_costs
30219 #undef TARGET_ADDRESS_COST
30220 #define TARGET_ADDRESS_COST ix86_address_cost
30222 #undef TARGET_FIXED_CONDITION_CODE_REGS
30223 #define TARGET_FIXED_CONDITION_CODE_REGS ix86_fixed_condition_code_regs
30224 #undef TARGET_CC_MODES_COMPATIBLE
30225 #define TARGET_CC_MODES_COMPATIBLE ix86_cc_modes_compatible
30227 #undef TARGET_MACHINE_DEPENDENT_REORG
30228 #define TARGET_MACHINE_DEPENDENT_REORG ix86_reorg
30230 #undef TARGET_BUILTIN_SETJMP_FRAME_VALUE
30231 #define TARGET_BUILTIN_SETJMP_FRAME_VALUE ix86_builtin_setjmp_frame_value
30233 #undef TARGET_BUILD_BUILTIN_VA_LIST
30234 #define TARGET_BUILD_BUILTIN_VA_LIST ix86_build_builtin_va_list
30236 #undef TARGET_FN_ABI_VA_LIST
30237 #define TARGET_FN_ABI_VA_LIST ix86_fn_abi_va_list
30239 #undef TARGET_CANONICAL_VA_LIST_TYPE
30240 #define TARGET_CANONICAL_VA_LIST_TYPE ix86_canonical_va_list_type
30242 #undef TARGET_EXPAND_BUILTIN_VA_START
30243 #define TARGET_EXPAND_BUILTIN_VA_START ix86_va_start
30245 #undef TARGET_MD_ASM_CLOBBERS
30246 #define TARGET_MD_ASM_CLOBBERS ix86_md_asm_clobbers
30248 #undef TARGET_PROMOTE_PROTOTYPES
30249 #define TARGET_PROMOTE_PROTOTYPES hook_bool_const_tree_true
30250 #undef TARGET_STRUCT_VALUE_RTX
30251 #define TARGET_STRUCT_VALUE_RTX ix86_struct_value_rtx
30252 #undef TARGET_SETUP_INCOMING_VARARGS
30253 #define TARGET_SETUP_INCOMING_VARARGS ix86_setup_incoming_varargs
30254 #undef TARGET_MUST_PASS_IN_STACK
30255 #define TARGET_MUST_PASS_IN_STACK ix86_must_pass_in_stack
30256 #undef TARGET_PASS_BY_REFERENCE
30257 #define TARGET_PASS_BY_REFERENCE ix86_pass_by_reference
30258 #undef TARGET_INTERNAL_ARG_POINTER
30259 #define TARGET_INTERNAL_ARG_POINTER ix86_internal_arg_pointer
30260 #undef TARGET_UPDATE_STACK_BOUNDARY
30261 #define TARGET_UPDATE_STACK_BOUNDARY ix86_update_stack_boundary
30262 #undef TARGET_GET_DRAP_RTX
30263 #define TARGET_GET_DRAP_RTX ix86_get_drap_rtx
30264 #undef TARGET_DWARF_HANDLE_FRAME_UNSPEC
30265 #define TARGET_DWARF_HANDLE_FRAME_UNSPEC ix86_dwarf_handle_frame_unspec
30266 #undef TARGET_STRICT_ARGUMENT_NAMING
30267 #define TARGET_STRICT_ARGUMENT_NAMING hook_bool_CUMULATIVE_ARGS_true
30269 #undef TARGET_GIMPLIFY_VA_ARG_EXPR
30270 #define TARGET_GIMPLIFY_VA_ARG_EXPR ix86_gimplify_va_arg
30272 #undef TARGET_SCALAR_MODE_SUPPORTED_P
30273 #define TARGET_SCALAR_MODE_SUPPORTED_P ix86_scalar_mode_supported_p
30275 #undef TARGET_VECTOR_MODE_SUPPORTED_P
30276 #define TARGET_VECTOR_MODE_SUPPORTED_P ix86_vector_mode_supported_p
30278 #undef TARGET_C_MODE_FOR_SUFFIX
30279 #define TARGET_C_MODE_FOR_SUFFIX ix86_c_mode_for_suffix
30282 #undef TARGET_ASM_OUTPUT_DWARF_DTPREL
30283 #define TARGET_ASM_OUTPUT_DWARF_DTPREL i386_output_dwarf_dtprel
30286 #ifdef SUBTARGET_INSERT_ATTRIBUTES
30287 #undef TARGET_INSERT_ATTRIBUTES
30288 #define TARGET_INSERT_ATTRIBUTES SUBTARGET_INSERT_ATTRIBUTES
30291 #undef TARGET_MANGLE_TYPE
30292 #define TARGET_MANGLE_TYPE ix86_mangle_type
30294 #undef TARGET_STACK_PROTECT_FAIL
30295 #define TARGET_STACK_PROTECT_FAIL ix86_stack_protect_fail
30297 #undef TARGET_FUNCTION_VALUE
30298 #define TARGET_FUNCTION_VALUE ix86_function_value
30300 #undef TARGET_SECONDARY_RELOAD
30301 #define TARGET_SECONDARY_RELOAD ix86_secondary_reload
30303 #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST
30304 #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST x86_builtin_vectorization_cost
30306 #undef TARGET_SET_CURRENT_FUNCTION
30307 #define TARGET_SET_CURRENT_FUNCTION ix86_set_current_function
30309 #undef TARGET_OPTION_VALID_ATTRIBUTE_P
30310 #define TARGET_OPTION_VALID_ATTRIBUTE_P ix86_valid_target_attribute_p
30312 #undef TARGET_OPTION_SAVE
30313 #define TARGET_OPTION_SAVE ix86_function_specific_save
30315 #undef TARGET_OPTION_RESTORE
30316 #define TARGET_OPTION_RESTORE ix86_function_specific_restore
30318 #undef TARGET_OPTION_PRINT
30319 #define TARGET_OPTION_PRINT ix86_function_specific_print
30321 #undef TARGET_OPTION_CAN_INLINE_P
30322 #define TARGET_OPTION_CAN_INLINE_P ix86_can_inline_p
30324 #undef TARGET_EXPAND_TO_RTL_HOOK
30325 #define TARGET_EXPAND_TO_RTL_HOOK ix86_maybe_switch_abi
30327 #undef TARGET_LEGITIMATE_ADDRESS_P
30328 #define TARGET_LEGITIMATE_ADDRESS_P ix86_legitimate_address_p
30330 struct gcc_target targetm = TARGET_INITIALIZER;
30332 #include "gt-i386.h"
OSDN Git Service
