1 ;; ARM NEON coprocessor Machine Description
2 ;; Copyright (C) 2006, 2007, 2008, 2009, 2010, 2012
3 ;; Free Software Foundation, Inc.
4 ;; Written by CodeSourcery.
6 ;; This file is part of GCC.
8 ;; GCC is free software; you can redistribute it and/or modify it
9 ;; under the terms of the GNU General Public License as published by
10 ;; the Free Software Foundation; either version 3, or (at your option)
13 ;; GCC is distributed in the hope that it will be useful, but
14 ;; WITHOUT ANY WARRANTY; without even the implied warranty of
15 ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 ;; General Public License for more details.
18 ;; You should have received a copy of the GNU General Public License
19 ;; along with GCC; see the file COPYING3. If not see
20 ;; <http://www.gnu.org/licenses/>.
22 ;; Enumerators for unspecs.
23 (define_c_enum "unspec" [
25 UNSPEC_ASHIFT_UNSIGNED
146 UNSPEC_MISALIGNED_ACCESS
152 ;; Attribute used to permit string comparisons against <VQH_mnem> in
153 ;; neon_type attribute definitions.
154 (define_attr "vqh_mnem" "vadd,vmin,vmax" (const_string "vadd"))
156 (define_insn "*neon_mov<mode>"
157 [(set (match_operand:VD 0 "nonimmediate_operand"
158 "=w,Uv,w, w, ?r,?w,?r,?r, ?Us")
159 (match_operand:VD 1 "general_operand"
160 " w,w, Dn,Uvi, w, r, r, Usi,r"))]
162 && (register_operand (operands[0], <MODE>mode)
163 || register_operand (operands[1], <MODE>mode))"
165 if (which_alternative == 2)
168 static char templ[40];
170 is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
171 &operands[1], &width);
173 gcc_assert (is_valid != 0);
176 return "vmov.f32\t%P0, %1 @ <mode>";
178 sprintf (templ, "vmov.i%d\t%%P0, %%1 @ <mode>", width);
183 /* FIXME: If the memory layout is changed in big-endian mode, output_move_vfp
184 below must be changed to output_move_neon (which will use the
185 element/structure loads/stores), and the constraint changed to 'Um' instead
188 switch (which_alternative)
190 case 0: return "vmov\t%P0, %P1 @ <mode>";
191 case 1: case 3: return output_move_vfp (operands);
192 case 2: gcc_unreachable ();
193 case 4: return "vmov\t%Q0, %R0, %P1 @ <mode>";
194 case 5: return "vmov\t%P0, %Q1, %R1 @ <mode>";
195 default: return output_move_double (operands, true, NULL);
198 [(set_attr "neon_type" "neon_int_1,*,neon_vmov,*,neon_mrrc,neon_mcr_2_mcrr,*,*,*")
199 (set_attr "type" "*,f_stored,*,f_loadd,*,*,alu,load2,store2")
200 (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
201 (set_attr "length" "4,4,4,4,4,4,8,8,8")
202 (set_attr "pool_range" "*,*,*,1020,*,*,*,1020,*")
203 (set_attr "neg_pool_range" "*,*,*,1004,*,*,*,1004,*")])
205 (define_insn "*neon_mov<mode>"
206 [(set (match_operand:VQXMOV 0 "nonimmediate_operand"
207 "=w,Un,w, w, ?r,?w,?r,?r, ?Us")
208 (match_operand:VQXMOV 1 "general_operand"
209 " w,w, Dn,Uni, w, r, r, Usi, r"))]
211 && (register_operand (operands[0], <MODE>mode)
212 || register_operand (operands[1], <MODE>mode))"
214 if (which_alternative == 2)
217 static char templ[40];
219 is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode,
220 &operands[1], &width);
222 gcc_assert (is_valid != 0);
225 return "vmov.f32\t%q0, %1 @ <mode>";
227 sprintf (templ, "vmov.i%d\t%%q0, %%1 @ <mode>", width);
232 switch (which_alternative)
234 case 0: return "vmov\t%q0, %q1 @ <mode>";
235 case 1: case 3: return output_move_neon (operands);
236 case 2: gcc_unreachable ();
237 case 4: return "vmov\t%Q0, %R0, %e1 @ <mode>\;vmov\t%J0, %K0, %f1";
238 case 5: return "vmov\t%e0, %Q1, %R1 @ <mode>\;vmov\t%f0, %J1, %K1";
239 default: return output_move_quad (operands);
242 [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_vmov,neon_ldm_2,\
243 neon_mrrc,neon_mcr_2_mcrr,*,*,*")
244 (set_attr "type" "*,*,*,*,*,*,alu,load4,store4")
245 (set_attr "insn" "*,*,*,*,*,*,mov,*,*")
246 (set_attr "length" "4,8,4,8,8,8,16,8,16")
247 (set_attr "pool_range" "*,*,*,1020,*,*,*,1020,*")
248 (set_attr "neg_pool_range" "*,*,*,996,*,*,*,996,*")])
250 (define_expand "movti"
251 [(set (match_operand:TI 0 "nonimmediate_operand" "")
252 (match_operand:TI 1 "general_operand" ""))]
255 if (can_create_pseudo_p ())
257 if (GET_CODE (operands[0]) != REG)
258 operands[1] = force_reg (TImode, operands[1]);
262 (define_expand "mov<mode>"
263 [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "")
264 (match_operand:VSTRUCT 1 "general_operand" ""))]
267 if (can_create_pseudo_p ())
269 if (GET_CODE (operands[0]) != REG)
270 operands[1] = force_reg (<MODE>mode, operands[1]);
274 (define_insn "*neon_mov<mode>"
275 [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "=w,Ut,w")
276 (match_operand:VSTRUCT 1 "general_operand" " w,w, Ut"))]
278 && (register_operand (operands[0], <MODE>mode)
279 || register_operand (operands[1], <MODE>mode))"
281 switch (which_alternative)
284 case 1: case 2: return output_move_neon (operands);
285 default: gcc_unreachable ();
288 [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_ldm_2")
289 (set (attr "length") (symbol_ref "arm_attr_length_move_neon (insn)"))])
292 [(set (match_operand:EI 0 "s_register_operand" "")
293 (match_operand:EI 1 "s_register_operand" ""))]
294 "TARGET_NEON && reload_completed"
295 [(set (match_dup 0) (match_dup 1))
296 (set (match_dup 2) (match_dup 3))]
298 int rdest = REGNO (operands[0]);
299 int rsrc = REGNO (operands[1]);
302 dest[0] = gen_rtx_REG (TImode, rdest);
303 src[0] = gen_rtx_REG (TImode, rsrc);
304 dest[1] = gen_rtx_REG (DImode, rdest + 4);
305 src[1] = gen_rtx_REG (DImode, rsrc + 4);
307 neon_disambiguate_copy (operands, dest, src, 2);
311 [(set (match_operand:OI 0 "s_register_operand" "")
312 (match_operand:OI 1 "s_register_operand" ""))]
313 "TARGET_NEON && reload_completed"
314 [(set (match_dup 0) (match_dup 1))
315 (set (match_dup 2) (match_dup 3))]
317 int rdest = REGNO (operands[0]);
318 int rsrc = REGNO (operands[1]);
321 dest[0] = gen_rtx_REG (TImode, rdest);
322 src[0] = gen_rtx_REG (TImode, rsrc);
323 dest[1] = gen_rtx_REG (TImode, rdest + 4);
324 src[1] = gen_rtx_REG (TImode, rsrc + 4);
326 neon_disambiguate_copy (operands, dest, src, 2);
330 [(set (match_operand:CI 0 "s_register_operand" "")
331 (match_operand:CI 1 "s_register_operand" ""))]
332 "TARGET_NEON && reload_completed"
333 [(set (match_dup 0) (match_dup 1))
334 (set (match_dup 2) (match_dup 3))
335 (set (match_dup 4) (match_dup 5))]
337 int rdest = REGNO (operands[0]);
338 int rsrc = REGNO (operands[1]);
341 dest[0] = gen_rtx_REG (TImode, rdest);
342 src[0] = gen_rtx_REG (TImode, rsrc);
343 dest[1] = gen_rtx_REG (TImode, rdest + 4);
344 src[1] = gen_rtx_REG (TImode, rsrc + 4);
345 dest[2] = gen_rtx_REG (TImode, rdest + 8);
346 src[2] = gen_rtx_REG (TImode, rsrc + 8);
348 neon_disambiguate_copy (operands, dest, src, 3);
352 [(set (match_operand:XI 0 "s_register_operand" "")
353 (match_operand:XI 1 "s_register_operand" ""))]
354 "TARGET_NEON && reload_completed"
355 [(set (match_dup 0) (match_dup 1))
356 (set (match_dup 2) (match_dup 3))
357 (set (match_dup 4) (match_dup 5))
358 (set (match_dup 6) (match_dup 7))]
360 int rdest = REGNO (operands[0]);
361 int rsrc = REGNO (operands[1]);
364 dest[0] = gen_rtx_REG (TImode, rdest);
365 src[0] = gen_rtx_REG (TImode, rsrc);
366 dest[1] = gen_rtx_REG (TImode, rdest + 4);
367 src[1] = gen_rtx_REG (TImode, rsrc + 4);
368 dest[2] = gen_rtx_REG (TImode, rdest + 8);
369 src[2] = gen_rtx_REG (TImode, rsrc + 8);
370 dest[3] = gen_rtx_REG (TImode, rdest + 12);
371 src[3] = gen_rtx_REG (TImode, rsrc + 12);
373 neon_disambiguate_copy (operands, dest, src, 4);
376 (define_expand "movmisalign<mode>"
377 [(set (match_operand:VDQX 0 "neon_struct_or_register_operand")
378 (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_or_register_operand")]
379 UNSPEC_MISALIGNED_ACCESS))]
380 "TARGET_NEON && !BYTES_BIG_ENDIAN"
382 /* This pattern is not permitted to fail during expansion: if both arguments
383 are non-registers (e.g. memory := constant, which can be created by the
384 auto-vectorizer), force operand 1 into a register. */
385 if (!s_register_operand (operands[0], <MODE>mode)
386 && !s_register_operand (operands[1], <MODE>mode))
387 operands[1] = force_reg (<MODE>mode, operands[1]);
390 (define_insn "*movmisalign<mode>_neon_store"
391 [(set (match_operand:VDX 0 "neon_struct_operand" "=Um")
392 (unspec:VDX [(match_operand:VDX 1 "s_register_operand" " w")]
393 UNSPEC_MISALIGNED_ACCESS))]
394 "TARGET_NEON && !BYTES_BIG_ENDIAN"
395 "vst1.<V_sz_elem>\t{%P1}, %A0"
396 [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
398 (define_insn "*movmisalign<mode>_neon_load"
399 [(set (match_operand:VDX 0 "s_register_operand" "=w")
400 (unspec:VDX [(match_operand:VDX 1 "neon_struct_operand" " Um")]
401 UNSPEC_MISALIGNED_ACCESS))]
402 "TARGET_NEON && !BYTES_BIG_ENDIAN"
403 "vld1.<V_sz_elem>\t{%P0}, %A1"
404 [(set_attr "neon_type" "neon_vld1_1_2_regs")])
406 (define_insn "*movmisalign<mode>_neon_store"
407 [(set (match_operand:VQX 0 "neon_struct_operand" "=Um")
408 (unspec:VQX [(match_operand:VQX 1 "s_register_operand" " w")]
409 UNSPEC_MISALIGNED_ACCESS))]
410 "TARGET_NEON && !BYTES_BIG_ENDIAN"
411 "vst1.<V_sz_elem>\t{%q1}, %A0"
412 [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
414 (define_insn "*movmisalign<mode>_neon_load"
415 [(set (match_operand:VQX 0 "s_register_operand" "=w")
416 (unspec:VQX [(match_operand:VQX 1 "neon_struct_operand" " Um")]
417 UNSPEC_MISALIGNED_ACCESS))]
418 "TARGET_NEON && !BYTES_BIG_ENDIAN"
419 "vld1.<V_sz_elem>\t{%q0}, %A1"
420 [(set_attr "neon_type" "neon_vld1_1_2_regs")])
422 (define_insn "vec_set<mode>_internal"
423 [(set (match_operand:VD 0 "s_register_operand" "=w")
426 (match_operand:<V_elem> 1 "s_register_operand" "r"))
427 (match_operand:VD 3 "s_register_operand" "0")
428 (match_operand:SI 2 "immediate_operand" "i")))]
431 int elt = ffs ((int) INTVAL (operands[2])) - 1;
432 if (BYTES_BIG_ENDIAN)
433 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
434 operands[2] = GEN_INT (elt);
436 return "vmov%?.<V_sz_elem>\t%P0[%c2], %1";
438 [(set_attr "predicable" "yes")
439 (set_attr "neon_type" "neon_mcr")])
441 (define_insn "vec_set<mode>_internal"
442 [(set (match_operand:VQ 0 "s_register_operand" "=w")
445 (match_operand:<V_elem> 1 "s_register_operand" "r"))
446 (match_operand:VQ 3 "s_register_operand" "0")
447 (match_operand:SI 2 "immediate_operand" "i")))]
450 HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
451 int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
452 int elt = elem % half_elts;
453 int hi = (elem / half_elts) * 2;
454 int regno = REGNO (operands[0]);
456 if (BYTES_BIG_ENDIAN)
457 elt = half_elts - 1 - elt;
459 operands[0] = gen_rtx_REG (<V_HALF>mode, regno + hi);
460 operands[2] = GEN_INT (elt);
462 return "vmov%?.<V_sz_elem>\t%P0[%c2], %1";
464 [(set_attr "predicable" "yes")
465 (set_attr "neon_type" "neon_mcr")]
468 (define_insn "vec_setv2di_internal"
469 [(set (match_operand:V2DI 0 "s_register_operand" "=w")
472 (match_operand:DI 1 "s_register_operand" "r"))
473 (match_operand:V2DI 3 "s_register_operand" "0")
474 (match_operand:SI 2 "immediate_operand" "i")))]
477 HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1;
478 int regno = REGNO (operands[0]) + 2 * elem;
480 operands[0] = gen_rtx_REG (DImode, regno);
482 return "vmov%?\t%P0, %Q1, %R1";
484 [(set_attr "predicable" "yes")
485 (set_attr "neon_type" "neon_mcr_2_mcrr")]
488 (define_expand "vec_set<mode>"
489 [(match_operand:VDQ 0 "s_register_operand" "")
490 (match_operand:<V_elem> 1 "s_register_operand" "")
491 (match_operand:SI 2 "immediate_operand" "")]
494 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]);
495 emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
496 GEN_INT (elem), operands[0]));
500 (define_insn "vec_extract<mode>"
501 [(set (match_operand:<V_elem> 0 "s_register_operand" "=r")
503 (match_operand:VD 1 "s_register_operand" "w")
504 (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
507 if (BYTES_BIG_ENDIAN)
509 int elt = INTVAL (operands[2]);
510 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
511 operands[2] = GEN_INT (elt);
513 return "vmov%?.<V_uf_sclr>\t%0, %P1[%c2]";
515 [(set_attr "predicable" "yes")
516 (set_attr "neon_type" "neon_bp_simple")]
519 (define_insn "vec_extract<mode>"
520 [(set (match_operand:<V_elem> 0 "s_register_operand" "=r")
522 (match_operand:VQ 1 "s_register_operand" "w")
523 (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
526 int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2;
527 int elt = INTVAL (operands[2]) % half_elts;
528 int hi = (INTVAL (operands[2]) / half_elts) * 2;
529 int regno = REGNO (operands[1]);
531 if (BYTES_BIG_ENDIAN)
532 elt = half_elts - 1 - elt;
534 operands[1] = gen_rtx_REG (<V_HALF>mode, regno + hi);
535 operands[2] = GEN_INT (elt);
537 return "vmov%?.<V_uf_sclr>\t%0, %P1[%c2]";
539 [(set_attr "predicable" "yes")
540 (set_attr "neon_type" "neon_bp_simple")]
543 (define_insn "vec_extractv2di"
544 [(set (match_operand:DI 0 "s_register_operand" "=r")
546 (match_operand:V2DI 1 "s_register_operand" "w")
547 (parallel [(match_operand:SI 2 "immediate_operand" "i")])))]
550 int regno = REGNO (operands[1]) + 2 * INTVAL (operands[2]);
552 operands[1] = gen_rtx_REG (DImode, regno);
554 return "vmov%?\t%Q0, %R0, %P1 @ v2di";
556 [(set_attr "predicable" "yes")
557 (set_attr "neon_type" "neon_int_1")]
560 (define_expand "vec_init<mode>"
561 [(match_operand:VDQ 0 "s_register_operand" "")
562 (match_operand 1 "" "")]
565 neon_expand_vector_init (operands[0], operands[1]);
569 ;; Doubleword and quadword arithmetic.
571 ;; NOTE: some other instructions also support 64-bit integer
572 ;; element size, which we could potentially use for "long long" operations.
574 (define_insn "*add<mode>3_neon"
575 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
576 (plus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
577 (match_operand:VDQ 2 "s_register_operand" "w")))]
578 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
579 "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
580 [(set (attr "neon_type")
581 (if_then_else (match_test "<Is_float_mode>")
582 (if_then_else (match_test "<Is_d_reg>")
583 (const_string "neon_fp_vadd_ddd_vabs_dd")
584 (const_string "neon_fp_vadd_qqq_vabs_qq"))
585 (const_string "neon_int_1")))]
588 (define_insn "adddi3_neon"
589 [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w")
590 (plus:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,w")
591 (match_operand:DI 2 "s_register_operand" "w,r,0,w")))
592 (clobber (reg:CC CC_REGNUM))]
595 switch (which_alternative)
597 case 0: /* fall through */
598 case 3: return "vadd.i64\t%P0, %P1, %P2";
601 default: gcc_unreachable ();
604 [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1")
605 (set_attr "conds" "*,clob,clob,*")
606 (set_attr "length" "*,8,8,*")
607 (set_attr "arch" "nota8,*,*,onlya8")]
610 (define_insn "*sub<mode>3_neon"
611 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
612 (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
613 (match_operand:VDQ 2 "s_register_operand" "w")))]
614 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
615 "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
616 [(set (attr "neon_type")
617 (if_then_else (match_test "<Is_float_mode>")
618 (if_then_else (match_test "<Is_d_reg>")
619 (const_string "neon_fp_vadd_ddd_vabs_dd")
620 (const_string "neon_fp_vadd_qqq_vabs_qq"))
621 (const_string "neon_int_2")))]
624 (define_insn "subdi3_neon"
625 [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r,?w")
626 (minus:DI (match_operand:DI 1 "s_register_operand" "w,0,r,0,w")
627 (match_operand:DI 2 "s_register_operand" "w,r,0,0,w")))
628 (clobber (reg:CC CC_REGNUM))]
631 switch (which_alternative)
633 case 0: /* fall through */
634 case 4: return "vsub.i64\t%P0, %P1, %P2";
635 case 1: /* fall through */
636 case 2: /* fall through */
637 case 3: return "subs\\t%Q0, %Q1, %Q2\;sbc\\t%R0, %R1, %R2";
638 default: gcc_unreachable ();
641 [(set_attr "neon_type" "neon_int_2,*,*,*,neon_int_2")
642 (set_attr "conds" "*,clob,clob,clob,*")
643 (set_attr "length" "*,8,8,8,*")
644 (set_attr "arch" "nota8,*,*,*,onlya8")]
647 (define_insn "*mul<mode>3_neon"
648 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
649 (mult:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
650 (match_operand:VDQ 2 "s_register_operand" "w")))]
651 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
652 "vmul.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
653 [(set (attr "neon_type")
654 (if_then_else (match_test "<Is_float_mode>")
655 (if_then_else (match_test "<Is_d_reg>")
656 (const_string "neon_fp_vadd_ddd_vabs_dd")
657 (const_string "neon_fp_vadd_qqq_vabs_qq"))
658 (if_then_else (match_test "<Is_d_reg>")
660 (match_test "<Scalar_mul_8_16>")
661 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
662 (const_string "neon_mul_qqq_8_16_32_ddd_32"))
663 (if_then_else (match_test "<Scalar_mul_8_16>")
664 (const_string "neon_mul_qqq_8_16_32_ddd_32")
665 (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
668 (define_insn "mul<mode>3add<mode>_neon"
669 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
670 (plus:VDQ (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
671 (match_operand:VDQ 3 "s_register_operand" "w"))
672 (match_operand:VDQ 1 "s_register_operand" "0")))]
673 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
674 "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
675 [(set (attr "neon_type")
676 (if_then_else (match_test "<Is_float_mode>")
677 (if_then_else (match_test "<Is_d_reg>")
678 (const_string "neon_fp_vmla_ddd")
679 (const_string "neon_fp_vmla_qqq"))
680 (if_then_else (match_test "<Is_d_reg>")
682 (match_test "<Scalar_mul_8_16>")
683 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
684 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
685 (if_then_else (match_test "<Scalar_mul_8_16>")
686 (const_string "neon_mla_qqq_8_16")
687 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
690 (define_insn "mul<mode>3neg<mode>add<mode>_neon"
691 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
692 (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "0")
693 (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w")
694 (match_operand:VDQ 3 "s_register_operand" "w"))))]
695 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
696 "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
697 [(set (attr "neon_type")
698 (if_then_else (match_test "<Is_float_mode>")
699 (if_then_else (match_test "<Is_d_reg>")
700 (const_string "neon_fp_vmla_ddd")
701 (const_string "neon_fp_vmla_qqq"))
702 (if_then_else (match_test "<Is_d_reg>")
704 (match_test "<Scalar_mul_8_16>")
705 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
706 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
707 (if_then_else (match_test "<Scalar_mul_8_16>")
708 (const_string "neon_mla_qqq_8_16")
709 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
712 (define_insn "ior<mode>3"
713 [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
714 (ior:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
715 (match_operand:VDQ 2 "neon_logic_op2" "w,Dl")))]
718 switch (which_alternative)
720 case 0: return "vorr\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
721 case 1: return neon_output_logic_immediate ("vorr", &operands[2],
722 <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode));
723 default: gcc_unreachable ();
726 [(set_attr "neon_type" "neon_int_1")]
729 (define_insn "iordi3_neon"
730 [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r,?w,?w")
731 (ior:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r,w,0")
732 (match_operand:DI 2 "neon_logic_op2" "w,Dl,r,r,w,Dl")))]
735 switch (which_alternative)
737 case 0: /* fall through */
738 case 4: return "vorr\t%P0, %P1, %P2";
739 case 1: /* fall through */
740 case 5: return neon_output_logic_immediate ("vorr", &operands[2],
741 DImode, 0, VALID_NEON_QREG_MODE (DImode));
744 default: gcc_unreachable ();
747 [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*,neon_int_1,neon_int_1")
748 (set_attr "length" "*,*,8,8,*,*")
749 (set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")]
752 ;; The concrete forms of the Neon immediate-logic instructions are vbic and
753 ;; vorr. We support the pseudo-instruction vand instead, because that
754 ;; corresponds to the canonical form the middle-end expects to use for
755 ;; immediate bitwise-ANDs.
757 (define_insn "and<mode>3"
758 [(set (match_operand:VDQ 0 "s_register_operand" "=w,w")
759 (and:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0")
760 (match_operand:VDQ 2 "neon_inv_logic_op2" "w,DL")))]
763 switch (which_alternative)
765 case 0: return "vand\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
766 case 1: return neon_output_logic_immediate ("vand", &operands[2],
767 <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode));
768 default: gcc_unreachable ();
771 [(set_attr "neon_type" "neon_int_1")]
774 (define_insn "anddi3_neon"
775 [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r,?w,?w")
776 (and:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r,w,0")
777 (match_operand:DI 2 "neon_inv_logic_op2" "w,DL,r,r,w,DL")))]
780 switch (which_alternative)
782 case 0: /* fall through */
783 case 4: return "vand\t%P0, %P1, %P2";
784 case 1: /* fall through */
785 case 5: return neon_output_logic_immediate ("vand", &operands[2],
786 DImode, 1, VALID_NEON_QREG_MODE (DImode));
789 default: gcc_unreachable ();
792 [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*,neon_int_1,neon_int_1")
793 (set_attr "length" "*,*,8,8,*,*")
794 (set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")]
797 (define_insn "orn<mode>3_neon"
798 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
799 (ior:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
800 (match_operand:VDQ 1 "s_register_operand" "w")))]
802 "vorn\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
803 [(set_attr "neon_type" "neon_int_1")]
806 ;; TODO: investigate whether we should disable
807 ;; this and bicdi3_neon for the A8 in line with the other
809 (define_insn_and_split "orndi3_neon"
810 [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r")
811 (ior:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,0,0,r"))
812 (match_operand:DI 1 "s_register_operand" "w,r,r,0")))]
820 (TARGET_NEON && !(IS_VFP_REGNUM (REGNO (operands[0]))))"
821 [(set (match_dup 0) (ior:SI (not:SI (match_dup 2)) (match_dup 1)))
822 (set (match_dup 3) (ior:SI (not:SI (match_dup 4)) (match_dup 5)))]
827 operands[3] = gen_highpart (SImode, operands[0]);
828 operands[0] = gen_lowpart (SImode, operands[0]);
829 operands[4] = gen_highpart (SImode, operands[2]);
830 operands[2] = gen_lowpart (SImode, operands[2]);
831 operands[5] = gen_highpart (SImode, operands[1]);
832 operands[1] = gen_lowpart (SImode, operands[1]);
836 emit_insn (gen_one_cmpldi2 (operands[0], operands[2]));
837 emit_insn (gen_iordi3 (operands[0], operands[1], operands[0]));
841 [(set_attr "neon_type" "neon_int_1,*,*,*")
842 (set_attr "length" "*,16,8,8")
843 (set_attr "arch" "any,a,t2,t2")]
846 (define_insn "bic<mode>3_neon"
847 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
848 (and:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w"))
849 (match_operand:VDQ 1 "s_register_operand" "w")))]
851 "vbic\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
852 [(set_attr "neon_type" "neon_int_1")]
855 ;; Compare to *anddi_notdi_di.
856 (define_insn "bicdi3_neon"
857 [(set (match_operand:DI 0 "s_register_operand" "=w,?=&r,?&r")
858 (and:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,r,0"))
859 (match_operand:DI 1 "s_register_operand" "w,0,r")))]
865 [(set_attr "neon_type" "neon_int_1,*,*")
866 (set_attr "length" "*,8,8")]
869 (define_insn "xor<mode>3"
870 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
871 (xor:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
872 (match_operand:VDQ 2 "s_register_operand" "w")))]
874 "veor\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
875 [(set_attr "neon_type" "neon_int_1")]
878 (define_insn "xordi3_neon"
879 [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w")
880 (xor:DI (match_operand:DI 1 "s_register_operand" "%w,0,r,w")
881 (match_operand:DI 2 "s_register_operand" "w,r,r,w")))]
888 [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1")
889 (set_attr "length" "*,8,8,*")
890 (set_attr "arch" "nota8,*,*,onlya8")]
893 (define_insn "one_cmpl<mode>2"
894 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
895 (not:VDQ (match_operand:VDQ 1 "s_register_operand" "w")))]
897 "vmvn\t%<V_reg>0, %<V_reg>1"
898 [(set_attr "neon_type" "neon_int_1")]
901 (define_insn "abs<mode>2"
902 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
903 (abs:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
905 "vabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
906 [(set (attr "neon_type")
907 (if_then_else (match_test "<Is_float_mode>")
908 (if_then_else (match_test "<Is_d_reg>")
909 (const_string "neon_fp_vadd_ddd_vabs_dd")
910 (const_string "neon_fp_vadd_qqq_vabs_qq"))
911 (const_string "neon_int_3")))]
914 (define_insn "neg<mode>2"
915 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
916 (neg:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))]
918 "vneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
919 [(set (attr "neon_type")
920 (if_then_else (match_test "<Is_float_mode>")
921 (if_then_else (match_test "<Is_d_reg>")
922 (const_string "neon_fp_vadd_ddd_vabs_dd")
923 (const_string "neon_fp_vadd_qqq_vabs_qq"))
924 (const_string "neon_int_3")))]
927 (define_insn "*umin<mode>3_neon"
928 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
929 (umin:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
930 (match_operand:VDQIW 2 "s_register_operand" "w")))]
932 "vmin.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
933 [(set_attr "neon_type" "neon_int_5")]
936 (define_insn "*umax<mode>3_neon"
937 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
938 (umax:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
939 (match_operand:VDQIW 2 "s_register_operand" "w")))]
941 "vmax.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
942 [(set_attr "neon_type" "neon_int_5")]
945 (define_insn "*smin<mode>3_neon"
946 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
947 (smin:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
948 (match_operand:VDQW 2 "s_register_operand" "w")))]
950 "vmin.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
951 [(set (attr "neon_type")
952 (if_then_else (match_test "<Is_float_mode>")
953 (const_string "neon_fp_vadd_ddd_vabs_dd")
954 (const_string "neon_int_5")))]
957 (define_insn "*smax<mode>3_neon"
958 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
959 (smax:VDQW (match_operand:VDQW 1 "s_register_operand" "w")
960 (match_operand:VDQW 2 "s_register_operand" "w")))]
962 "vmax.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
963 [(set (attr "neon_type")
964 (if_then_else (match_test "<Is_float_mode>")
965 (const_string "neon_fp_vadd_ddd_vabs_dd")
966 (const_string "neon_int_5")))]
969 ; TODO: V2DI shifts are current disabled because there are bugs in the
970 ; generic vectorizer code. It ends up creating a V2DI constructor with
973 (define_insn "vashl<mode>3"
974 [(set (match_operand:VDQIW 0 "s_register_operand" "=w,w")
975 (ashift:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w,w")
976 (match_operand:VDQIW 2 "imm_lshift_or_reg_neon" "w,Dn")))]
979 switch (which_alternative)
981 case 0: return "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2";
982 case 1: return neon_output_shift_immediate ("vshl", 'i', &operands[2],
984 VALID_NEON_QREG_MODE (<MODE>mode),
986 default: gcc_unreachable ();
989 [(set (attr "neon_type")
990 (if_then_else (match_test "<Is_d_reg>")
991 (const_string "neon_vshl_ddd")
992 (const_string "neon_shift_3")))]
995 (define_insn "vashr<mode>3_imm"
996 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
997 (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
998 (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
1001 return neon_output_shift_immediate ("vshr", 's', &operands[2],
1002 <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
1005 [(set (attr "neon_type")
1006 (if_then_else (match_test "<Is_d_reg>")
1007 (const_string "neon_vshl_ddd")
1008 (const_string "neon_shift_3")))]
1011 (define_insn "vlshr<mode>3_imm"
1012 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1013 (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")
1014 (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))]
1017 return neon_output_shift_immediate ("vshr", 'u', &operands[2],
1018 <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode),
1021 [(set (attr "neon_type")
1022 (if_then_else (match_test "<Is_d_reg>")
1023 (const_string "neon_vshl_ddd")
1024 (const_string "neon_shift_3")))]
1027 ; Used for implementing logical shift-right, which is a left-shift by a negative
1028 ; amount, with signed operands. This is essentially the same as ashl<mode>3
1029 ; above, but using an unspec in case GCC tries anything tricky with negative
1032 (define_insn "ashl<mode>3_signed"
1033 [(set (match_operand:VDQI 0 "s_register_operand" "=w")
1034 (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
1035 (match_operand:VDQI 2 "s_register_operand" "w")]
1036 UNSPEC_ASHIFT_SIGNED))]
1038 "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1039 [(set (attr "neon_type")
1040 (if_then_else (match_test "<Is_d_reg>")
1041 (const_string "neon_vshl_ddd")
1042 (const_string "neon_shift_3")))]
1045 ; Used for implementing logical shift-right, which is a left-shift by a negative
1046 ; amount, with unsigned operands.
1048 (define_insn "ashl<mode>3_unsigned"
1049 [(set (match_operand:VDQI 0 "s_register_operand" "=w")
1050 (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w")
1051 (match_operand:VDQI 2 "s_register_operand" "w")]
1052 UNSPEC_ASHIFT_UNSIGNED))]
1054 "vshl.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1055 [(set (attr "neon_type")
1056 (if_then_else (match_test "<Is_d_reg>")
1057 (const_string "neon_vshl_ddd")
1058 (const_string "neon_shift_3")))]
1061 (define_expand "vashr<mode>3"
1062 [(set (match_operand:VDQIW 0 "s_register_operand" "")
1063 (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
1064 (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
1067 if (s_register_operand (operands[2], <MODE>mode))
1069 rtx neg = gen_reg_rtx (<MODE>mode);
1070 emit_insn (gen_neg<mode>2 (neg, operands[2]));
1071 emit_insn (gen_ashl<mode>3_signed (operands[0], operands[1], neg));
1074 emit_insn (gen_vashr<mode>3_imm (operands[0], operands[1], operands[2]));
1078 (define_expand "vlshr<mode>3"
1079 [(set (match_operand:VDQIW 0 "s_register_operand" "")
1080 (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "")
1081 (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))]
1084 if (s_register_operand (operands[2], <MODE>mode))
1086 rtx neg = gen_reg_rtx (<MODE>mode);
1087 emit_insn (gen_neg<mode>2 (neg, operands[2]));
1088 emit_insn (gen_ashl<mode>3_unsigned (operands[0], operands[1], neg));
1091 emit_insn (gen_vlshr<mode>3_imm (operands[0], operands[1], operands[2]));
1095 ;; Widening operations
1097 (define_insn "widen_ssum<mode>3"
1098 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1099 (plus:<V_widen> (sign_extend:<V_widen>
1100 (match_operand:VW 1 "s_register_operand" "%w"))
1101 (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1103 "vaddw.<V_s_elem>\t%q0, %q2, %P1"
1104 [(set_attr "neon_type" "neon_int_3")]
1107 (define_insn "widen_usum<mode>3"
1108 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1109 (plus:<V_widen> (zero_extend:<V_widen>
1110 (match_operand:VW 1 "s_register_operand" "%w"))
1111 (match_operand:<V_widen> 2 "s_register_operand" "w")))]
1113 "vaddw.<V_u_elem>\t%q0, %q2, %P1"
1114 [(set_attr "neon_type" "neon_int_3")]
1117 ;; VEXT can be used to synthesize coarse whole-vector shifts with 8-bit
1118 ;; shift-count granularity. That's good enough for the middle-end's current
1121 ;; Note that it's not safe to perform such an operation in big-endian mode,
1122 ;; due to element-ordering issues.
1124 (define_expand "vec_shr_<mode>"
1125 [(match_operand:VDQ 0 "s_register_operand" "")
1126 (match_operand:VDQ 1 "s_register_operand" "")
1127 (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1128 "TARGET_NEON && !BYTES_BIG_ENDIAN"
1131 HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1132 const int width = GET_MODE_BITSIZE (<MODE>mode);
1133 const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1134 rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1135 (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1137 if (num_bits == width)
1139 emit_move_insn (operands[0], operands[1]);
1143 zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1144 operands[0] = gen_lowpart (bvecmode, operands[0]);
1145 operands[1] = gen_lowpart (bvecmode, operands[1]);
1147 emit_insn (gen_ext (operands[0], operands[1], zero_reg,
1148 GEN_INT (num_bits / BITS_PER_UNIT)));
1152 (define_expand "vec_shl_<mode>"
1153 [(match_operand:VDQ 0 "s_register_operand" "")
1154 (match_operand:VDQ 1 "s_register_operand" "")
1155 (match_operand:SI 2 "const_multiple_of_8_operand" "")]
1156 "TARGET_NEON && !BYTES_BIG_ENDIAN"
1159 HOST_WIDE_INT num_bits = INTVAL (operands[2]);
1160 const int width = GET_MODE_BITSIZE (<MODE>mode);
1161 const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode;
1162 rtx (*gen_ext) (rtx, rtx, rtx, rtx) =
1163 (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi;
1167 emit_move_insn (operands[0], CONST0_RTX (<MODE>mode));
1171 num_bits = width - num_bits;
1173 zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode));
1174 operands[0] = gen_lowpart (bvecmode, operands[0]);
1175 operands[1] = gen_lowpart (bvecmode, operands[1]);
1177 emit_insn (gen_ext (operands[0], zero_reg, operands[1],
1178 GEN_INT (num_bits / BITS_PER_UNIT)));
1182 ;; Helpers for quad-word reduction operations
1184 ; Add (or smin, smax...) the low N/2 elements of the N-element vector
1185 ; operand[1] to the high N/2 elements of same. Put the result in operand[0], an
1186 ; N/2-element vector.
1188 (define_insn "quad_halves_<code>v4si"
1189 [(set (match_operand:V2SI 0 "s_register_operand" "=w")
1191 (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w")
1192 (parallel [(const_int 0) (const_int 1)]))
1193 (vec_select:V2SI (match_dup 1)
1194 (parallel [(const_int 2) (const_int 3)]))))]
1196 "<VQH_mnem>.<VQH_sign>32\t%P0, %e1, %f1"
1197 [(set_attr "vqh_mnem" "<VQH_mnem>")
1198 (set (attr "neon_type")
1199 (if_then_else (eq_attr "vqh_mnem" "vadd")
1200 (const_string "neon_int_1") (const_string "neon_int_5")))]
1203 (define_insn "quad_halves_<code>v4sf"
1204 [(set (match_operand:V2SF 0 "s_register_operand" "=w")
1206 (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w")
1207 (parallel [(const_int 0) (const_int 1)]))
1208 (vec_select:V2SF (match_dup 1)
1209 (parallel [(const_int 2) (const_int 3)]))))]
1210 "TARGET_NEON && flag_unsafe_math_optimizations"
1211 "<VQH_mnem>.f32\t%P0, %e1, %f1"
1212 [(set_attr "vqh_mnem" "<VQH_mnem>")
1213 (set (attr "neon_type")
1214 (if_then_else (eq_attr "vqh_mnem" "vadd")
1215 (const_string "neon_int_1") (const_string "neon_int_5")))]
1218 (define_insn "quad_halves_<code>v8hi"
1219 [(set (match_operand:V4HI 0 "s_register_operand" "+w")
1221 (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w")
1222 (parallel [(const_int 0) (const_int 1)
1223 (const_int 2) (const_int 3)]))
1224 (vec_select:V4HI (match_dup 1)
1225 (parallel [(const_int 4) (const_int 5)
1226 (const_int 6) (const_int 7)]))))]
1228 "<VQH_mnem>.<VQH_sign>16\t%P0, %e1, %f1"
1229 [(set_attr "vqh_mnem" "<VQH_mnem>")
1230 (set (attr "neon_type")
1231 (if_then_else (eq_attr "vqh_mnem" "vadd")
1232 (const_string "neon_int_1") (const_string "neon_int_5")))]
1235 (define_insn "quad_halves_<code>v16qi"
1236 [(set (match_operand:V8QI 0 "s_register_operand" "+w")
1238 (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w")
1239 (parallel [(const_int 0) (const_int 1)
1240 (const_int 2) (const_int 3)
1241 (const_int 4) (const_int 5)
1242 (const_int 6) (const_int 7)]))
1243 (vec_select:V8QI (match_dup 1)
1244 (parallel [(const_int 8) (const_int 9)
1245 (const_int 10) (const_int 11)
1246 (const_int 12) (const_int 13)
1247 (const_int 14) (const_int 15)]))))]
1249 "<VQH_mnem>.<VQH_sign>8\t%P0, %e1, %f1"
1250 [(set_attr "vqh_mnem" "<VQH_mnem>")
1251 (set (attr "neon_type")
1252 (if_then_else (eq_attr "vqh_mnem" "vadd")
1253 (const_string "neon_int_1") (const_string "neon_int_5")))]
1256 (define_expand "move_hi_quad_<mode>"
1257 [(match_operand:ANY128 0 "s_register_operand" "")
1258 (match_operand:<V_HALF> 1 "s_register_operand" "")]
1261 emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0], <MODE>mode,
1262 GET_MODE_SIZE (<V_HALF>mode)),
1267 (define_expand "move_lo_quad_<mode>"
1268 [(match_operand:ANY128 0 "s_register_operand" "")
1269 (match_operand:<V_HALF> 1 "s_register_operand" "")]
1272 emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0],
1278 ;; Reduction operations
1280 (define_expand "reduc_splus_<mode>"
1281 [(match_operand:VD 0 "s_register_operand" "")
1282 (match_operand:VD 1 "s_register_operand" "")]
1283 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1285 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1286 &gen_neon_vpadd_internal<mode>);
1290 (define_expand "reduc_splus_<mode>"
1291 [(match_operand:VQ 0 "s_register_operand" "")
1292 (match_operand:VQ 1 "s_register_operand" "")]
1293 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1294 && !BYTES_BIG_ENDIAN"
1296 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1297 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1299 emit_insn (gen_quad_halves_plus<mode> (step1, operands[1]));
1300 emit_insn (gen_reduc_splus_<V_half> (res_d, step1));
1301 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1306 (define_insn "reduc_splus_v2di"
1307 [(set (match_operand:V2DI 0 "s_register_operand" "=w")
1308 (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")]
1310 "TARGET_NEON && !BYTES_BIG_ENDIAN"
1311 "vadd.i64\t%e0, %e1, %f1"
1312 [(set_attr "neon_type" "neon_int_1")]
1315 ;; NEON does not distinguish between signed and unsigned addition except on
1316 ;; widening operations.
1317 (define_expand "reduc_uplus_<mode>"
1318 [(match_operand:VDQI 0 "s_register_operand" "")
1319 (match_operand:VDQI 1 "s_register_operand" "")]
1320 "TARGET_NEON && (<Is_d_reg> || !BYTES_BIG_ENDIAN)"
1322 emit_insn (gen_reduc_splus_<mode> (operands[0], operands[1]));
1326 (define_expand "reduc_smin_<mode>"
1327 [(match_operand:VD 0 "s_register_operand" "")
1328 (match_operand:VD 1 "s_register_operand" "")]
1329 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1331 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1332 &gen_neon_vpsmin<mode>);
1336 (define_expand "reduc_smin_<mode>"
1337 [(match_operand:VQ 0 "s_register_operand" "")
1338 (match_operand:VQ 1 "s_register_operand" "")]
1339 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1340 && !BYTES_BIG_ENDIAN"
1342 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1343 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1345 emit_insn (gen_quad_halves_smin<mode> (step1, operands[1]));
1346 emit_insn (gen_reduc_smin_<V_half> (res_d, step1));
1347 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1352 (define_expand "reduc_smax_<mode>"
1353 [(match_operand:VD 0 "s_register_operand" "")
1354 (match_operand:VD 1 "s_register_operand" "")]
1355 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1357 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1358 &gen_neon_vpsmax<mode>);
1362 (define_expand "reduc_smax_<mode>"
1363 [(match_operand:VQ 0 "s_register_operand" "")
1364 (match_operand:VQ 1 "s_register_operand" "")]
1365 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)
1366 && !BYTES_BIG_ENDIAN"
1368 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1369 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1371 emit_insn (gen_quad_halves_smax<mode> (step1, operands[1]));
1372 emit_insn (gen_reduc_smax_<V_half> (res_d, step1));
1373 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1378 (define_expand "reduc_umin_<mode>"
1379 [(match_operand:VDI 0 "s_register_operand" "")
1380 (match_operand:VDI 1 "s_register_operand" "")]
1383 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1384 &gen_neon_vpumin<mode>);
1388 (define_expand "reduc_umin_<mode>"
1389 [(match_operand:VQI 0 "s_register_operand" "")
1390 (match_operand:VQI 1 "s_register_operand" "")]
1391 "TARGET_NEON && !BYTES_BIG_ENDIAN"
1393 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1394 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1396 emit_insn (gen_quad_halves_umin<mode> (step1, operands[1]));
1397 emit_insn (gen_reduc_umin_<V_half> (res_d, step1));
1398 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1403 (define_expand "reduc_umax_<mode>"
1404 [(match_operand:VDI 0 "s_register_operand" "")
1405 (match_operand:VDI 1 "s_register_operand" "")]
1408 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode,
1409 &gen_neon_vpumax<mode>);
1413 (define_expand "reduc_umax_<mode>"
1414 [(match_operand:VQI 0 "s_register_operand" "")
1415 (match_operand:VQI 1 "s_register_operand" "")]
1416 "TARGET_NEON && !BYTES_BIG_ENDIAN"
1418 rtx step1 = gen_reg_rtx (<V_HALF>mode);
1419 rtx res_d = gen_reg_rtx (<V_HALF>mode);
1421 emit_insn (gen_quad_halves_umax<mode> (step1, operands[1]));
1422 emit_insn (gen_reduc_umax_<V_half> (res_d, step1));
1423 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d));
1428 (define_insn "neon_vpadd_internal<mode>"
1429 [(set (match_operand:VD 0 "s_register_operand" "=w")
1430 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1431 (match_operand:VD 2 "s_register_operand" "w")]
1434 "vpadd.<V_if_elem>\t%P0, %P1, %P2"
1435 ;; Assume this schedules like vadd.
1436 [(set (attr "neon_type")
1437 (if_then_else (match_test "<Is_float_mode>")
1438 (if_then_else (match_test "<Is_d_reg>")
1439 (const_string "neon_fp_vadd_ddd_vabs_dd")
1440 (const_string "neon_fp_vadd_qqq_vabs_qq"))
1441 (const_string "neon_int_1")))]
1444 (define_insn "neon_vpsmin<mode>"
1445 [(set (match_operand:VD 0 "s_register_operand" "=w")
1446 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1447 (match_operand:VD 2 "s_register_operand" "w")]
1450 "vpmin.<V_s_elem>\t%P0, %P1, %P2"
1451 ;; Assume this schedules like vmin.
1452 [(set (attr "neon_type")
1453 (if_then_else (match_test "<Is_float_mode>")
1454 (const_string "neon_fp_vadd_ddd_vabs_dd")
1455 (const_string "neon_int_5")))]
1458 (define_insn "neon_vpsmax<mode>"
1459 [(set (match_operand:VD 0 "s_register_operand" "=w")
1460 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
1461 (match_operand:VD 2 "s_register_operand" "w")]
1464 "vpmax.<V_s_elem>\t%P0, %P1, %P2"
1465 ;; Assume this schedules like vmax.
1466 [(set (attr "neon_type")
1467 (if_then_else (match_test "<Is_float_mode>")
1468 (const_string "neon_fp_vadd_ddd_vabs_dd")
1469 (const_string "neon_int_5")))]
1472 (define_insn "neon_vpumin<mode>"
1473 [(set (match_operand:VDI 0 "s_register_operand" "=w")
1474 (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1475 (match_operand:VDI 2 "s_register_operand" "w")]
1478 "vpmin.<V_u_elem>\t%P0, %P1, %P2"
1479 ;; Assume this schedules like umin.
1480 [(set_attr "neon_type" "neon_int_5")]
1483 (define_insn "neon_vpumax<mode>"
1484 [(set (match_operand:VDI 0 "s_register_operand" "=w")
1485 (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w")
1486 (match_operand:VDI 2 "s_register_operand" "w")]
1489 "vpmax.<V_u_elem>\t%P0, %P1, %P2"
1490 ;; Assume this schedules like umax.
1491 [(set_attr "neon_type" "neon_int_5")]
1494 ;; Saturating arithmetic
1496 ; NOTE: Neon supports many more saturating variants of instructions than the
1497 ; following, but these are all GCC currently understands.
1498 ; FIXME: Actually, GCC doesn't know how to create saturating add/sub by itself
1499 ; yet either, although these patterns may be used by intrinsics when they're
1502 (define_insn "*ss_add<mode>_neon"
1503 [(set (match_operand:VD 0 "s_register_operand" "=w")
1504 (ss_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1505 (match_operand:VD 2 "s_register_operand" "w")))]
1507 "vqadd.<V_s_elem>\t%P0, %P1, %P2"
1508 [(set_attr "neon_type" "neon_int_4")]
1511 (define_insn "*us_add<mode>_neon"
1512 [(set (match_operand:VD 0 "s_register_operand" "=w")
1513 (us_plus:VD (match_operand:VD 1 "s_register_operand" "w")
1514 (match_operand:VD 2 "s_register_operand" "w")))]
1516 "vqadd.<V_u_elem>\t%P0, %P1, %P2"
1517 [(set_attr "neon_type" "neon_int_4")]
1520 (define_insn "*ss_sub<mode>_neon"
1521 [(set (match_operand:VD 0 "s_register_operand" "=w")
1522 (ss_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1523 (match_operand:VD 2 "s_register_operand" "w")))]
1525 "vqsub.<V_s_elem>\t%P0, %P1, %P2"
1526 [(set_attr "neon_type" "neon_int_5")]
1529 (define_insn "*us_sub<mode>_neon"
1530 [(set (match_operand:VD 0 "s_register_operand" "=w")
1531 (us_minus:VD (match_operand:VD 1 "s_register_operand" "w")
1532 (match_operand:VD 2 "s_register_operand" "w")))]
1534 "vqsub.<V_u_elem>\t%P0, %P1, %P2"
1535 [(set_attr "neon_type" "neon_int_5")]
1538 ;; Conditional instructions. These are comparisons with conditional moves for
1539 ;; vectors. They perform the assignment:
1541 ;; Vop0 = (Vop4 <op3> Vop5) ? Vop1 : Vop2;
1543 ;; where op3 is <, <=, ==, !=, >= or >. Operations are performed
1546 (define_expand "vcond<mode><mode>"
1547 [(set (match_operand:VDQW 0 "s_register_operand" "")
1549 (match_operator 3 "arm_comparison_operator"
1550 [(match_operand:VDQW 4 "s_register_operand" "")
1551 (match_operand:VDQW 5 "nonmemory_operand" "")])
1552 (match_operand:VDQW 1 "s_register_operand" "")
1553 (match_operand:VDQW 2 "s_register_operand" "")))]
1554 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
1557 int inverse = 0, immediate_zero = 0;
1558 /* See the description of "magic" bits in the 'T' case of
1559 arm_print_operand. */
1560 HOST_WIDE_INT magic_word = (<MODE>mode == V2SFmode || <MODE>mode == V4SFmode)
1562 rtx magic_rtx = GEN_INT (magic_word);
1564 mask = gen_reg_rtx (<V_cmp_result>mode);
1566 if (operands[5] == CONST0_RTX (<MODE>mode))
1568 else if (!REG_P (operands[5]))
1569 operands[5] = force_reg (<MODE>mode, operands[5]);
1571 switch (GET_CODE (operands[3]))
1574 emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1579 emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1584 emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1590 emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1593 emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1599 emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1602 emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1607 emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1617 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1620 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1626 (define_expand "vcondu<mode><mode>"
1627 [(set (match_operand:VDQIW 0 "s_register_operand" "")
1629 (match_operator 3 "arm_comparison_operator"
1630 [(match_operand:VDQIW 4 "s_register_operand" "")
1631 (match_operand:VDQIW 5 "s_register_operand" "")])
1632 (match_operand:VDQIW 1 "s_register_operand" "")
1633 (match_operand:VDQIW 2 "s_register_operand" "")))]
1637 int inverse = 0, immediate_zero = 0;
1639 mask = gen_reg_rtx (<V_cmp_result>mode);
1641 if (operands[5] == CONST0_RTX (<MODE>mode))
1643 else if (!REG_P (operands[5]))
1644 operands[5] = force_reg (<MODE>mode, operands[5]);
1646 switch (GET_CODE (operands[3]))
1649 emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5],
1654 emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5],
1659 emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1665 emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5],
1668 emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4],
1674 emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5],
1677 emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4],
1682 emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5],
1692 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2],
1695 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1],
1701 ;; Patterns for builtins.
1703 ; good for plain vadd, vaddq.
1705 (define_expand "neon_vadd<mode>"
1706 [(match_operand:VDQX 0 "s_register_operand" "=w")
1707 (match_operand:VDQX 1 "s_register_operand" "w")
1708 (match_operand:VDQX 2 "s_register_operand" "w")
1709 (match_operand:SI 3 "immediate_operand" "i")]
1712 if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1713 emit_insn (gen_add<mode>3 (operands[0], operands[1], operands[2]));
1715 emit_insn (gen_neon_vadd<mode>_unspec (operands[0], operands[1],
1720 ; Note that NEON operations don't support the full IEEE 754 standard: in
1721 ; particular, denormal values are flushed to zero. This means that GCC cannot
1722 ; use those instructions for autovectorization, etc. unless
1723 ; -funsafe-math-optimizations is in effect (in which case flush-to-zero
1724 ; behaviour is permissible). Intrinsic operations (provided by the arm_neon.h
1725 ; header) must work in either case: if -funsafe-math-optimizations is given,
1726 ; intrinsics expand to "canonical" RTL where possible, otherwise intrinsics
1727 ; expand to unspecs (which may potentially limit the extent to which they might
1728 ; be optimized by generic code).
1730 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1732 (define_insn "neon_vadd<mode>_unspec"
1733 [(set (match_operand:VDQX 0 "s_register_operand" "=w")
1734 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
1735 (match_operand:VDQX 2 "s_register_operand" "w")]
1738 "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1739 [(set (attr "neon_type")
1740 (if_then_else (match_test "<Is_float_mode>")
1741 (if_then_else (match_test "<Is_d_reg>")
1742 (const_string "neon_fp_vadd_ddd_vabs_dd")
1743 (const_string "neon_fp_vadd_qqq_vabs_qq"))
1744 (const_string "neon_int_1")))]
1747 ; operand 3 represents in bits:
1748 ; bit 0: signed (vs unsigned).
1749 ; bit 1: rounding (vs none).
1751 (define_insn "neon_vaddl<mode>"
1752 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1753 (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
1754 (match_operand:VDI 2 "s_register_operand" "w")
1755 (match_operand:SI 3 "immediate_operand" "i")]
1758 "vaddl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
1759 [(set_attr "neon_type" "neon_int_3")]
1762 (define_insn "neon_vaddw<mode>"
1763 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1764 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
1765 (match_operand:VDI 2 "s_register_operand" "w")
1766 (match_operand:SI 3 "immediate_operand" "i")]
1769 "vaddw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
1770 [(set_attr "neon_type" "neon_int_2")]
1775 (define_insn "neon_vhadd<mode>"
1776 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
1777 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
1778 (match_operand:VDQIW 2 "s_register_operand" "w")
1779 (match_operand:SI 3 "immediate_operand" "i")]
1782 "v%O3hadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1783 [(set_attr "neon_type" "neon_int_4")]
1786 (define_insn "neon_vqadd<mode>"
1787 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
1788 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
1789 (match_operand:VDQIX 2 "s_register_operand" "w")
1790 (match_operand:SI 3 "immediate_operand" "i")]
1793 "vqadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1794 [(set_attr "neon_type" "neon_int_4")]
1797 (define_insn "neon_vaddhn<mode>"
1798 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
1799 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
1800 (match_operand:VN 2 "s_register_operand" "w")
1801 (match_operand:SI 3 "immediate_operand" "i")]
1804 "v%O3addhn.<V_if_elem>\t%P0, %q1, %q2"
1805 [(set_attr "neon_type" "neon_int_4")]
1808 ;; We cannot replace this unspec with mul<mode>3 because of the odd
1809 ;; polynomial multiplication case that can specified by operand 3.
1810 (define_insn "neon_vmul<mode>"
1811 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
1812 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
1813 (match_operand:VDQW 2 "s_register_operand" "w")
1814 (match_operand:SI 3 "immediate_operand" "i")]
1817 "vmul.%F3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1818 [(set (attr "neon_type")
1819 (if_then_else (match_test "<Is_float_mode>")
1820 (if_then_else (match_test "<Is_d_reg>")
1821 (const_string "neon_fp_vadd_ddd_vabs_dd")
1822 (const_string "neon_fp_vadd_qqq_vabs_qq"))
1823 (if_then_else (match_test "<Is_d_reg>")
1825 (match_test "<Scalar_mul_8_16>")
1826 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
1827 (const_string "neon_mul_qqq_8_16_32_ddd_32"))
1828 (if_then_else (match_test "<Scalar_mul_8_16>")
1829 (const_string "neon_mul_qqq_8_16_32_ddd_32")
1830 (const_string "neon_mul_qqq_8_16_32_ddd_32")))))]
1833 (define_expand "neon_vmla<mode>"
1834 [(match_operand:VDQW 0 "s_register_operand" "=w")
1835 (match_operand:VDQW 1 "s_register_operand" "0")
1836 (match_operand:VDQW 2 "s_register_operand" "w")
1837 (match_operand:VDQW 3 "s_register_operand" "w")
1838 (match_operand:SI 4 "immediate_operand" "i")]
1841 if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1842 emit_insn (gen_mul<mode>3add<mode>_neon (operands[0], operands[1],
1843 operands[2], operands[3]));
1845 emit_insn (gen_neon_vmla<mode>_unspec (operands[0], operands[1],
1846 operands[2], operands[3]));
1850 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1852 (define_insn "neon_vmla<mode>_unspec"
1853 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
1854 (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
1855 (match_operand:VDQ 2 "s_register_operand" "w")
1856 (match_operand:VDQ 3 "s_register_operand" "w")]
1859 "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
1860 [(set (attr "neon_type")
1861 (if_then_else (match_test "<Is_float_mode>")
1862 (if_then_else (match_test "<Is_d_reg>")
1863 (const_string "neon_fp_vmla_ddd")
1864 (const_string "neon_fp_vmla_qqq"))
1865 (if_then_else (match_test "<Is_d_reg>")
1867 (match_test "<Scalar_mul_8_16>")
1868 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1869 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
1870 (if_then_else (match_test "<Scalar_mul_8_16>")
1871 (const_string "neon_mla_qqq_8_16")
1872 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
1875 (define_insn "neon_vmlal<mode>"
1876 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1877 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1878 (match_operand:VW 2 "s_register_operand" "w")
1879 (match_operand:VW 3 "s_register_operand" "w")
1880 (match_operand:SI 4 "immediate_operand" "i")]
1883 "vmlal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
1884 [(set (attr "neon_type")
1885 (if_then_else (match_test "<Scalar_mul_8_16>")
1886 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1887 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1890 (define_expand "neon_vmls<mode>"
1891 [(match_operand:VDQW 0 "s_register_operand" "=w")
1892 (match_operand:VDQW 1 "s_register_operand" "0")
1893 (match_operand:VDQW 2 "s_register_operand" "w")
1894 (match_operand:VDQW 3 "s_register_operand" "w")
1895 (match_operand:SI 4 "immediate_operand" "i")]
1898 if (!<Is_float_mode> || flag_unsafe_math_optimizations)
1899 emit_insn (gen_mul<mode>3neg<mode>add<mode>_neon (operands[0],
1900 operands[1], operands[2], operands[3]));
1902 emit_insn (gen_neon_vmls<mode>_unspec (operands[0], operands[1],
1903 operands[2], operands[3]));
1907 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
1909 (define_insn "neon_vmls<mode>_unspec"
1910 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
1911 (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0")
1912 (match_operand:VDQ 2 "s_register_operand" "w")
1913 (match_operand:VDQ 3 "s_register_operand" "w")]
1916 "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
1917 [(set (attr "neon_type")
1918 (if_then_else (match_test "<Is_float_mode>")
1919 (if_then_else (match_test "<Is_d_reg>")
1920 (const_string "neon_fp_vmla_ddd")
1921 (const_string "neon_fp_vmla_qqq"))
1922 (if_then_else (match_test "<Is_d_reg>")
1924 (match_test "<Scalar_mul_8_16>")
1925 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1926 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))
1928 (match_test "<Scalar_mul_8_16>")
1929 (const_string "neon_mla_qqq_8_16")
1930 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))]
1933 (define_insn "neon_vmlsl<mode>"
1934 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1935 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1936 (match_operand:VW 2 "s_register_operand" "w")
1937 (match_operand:VW 3 "s_register_operand" "w")
1938 (match_operand:SI 4 "immediate_operand" "i")]
1941 "vmlsl.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
1942 [(set (attr "neon_type")
1943 (if_then_else (match_test "<Scalar_mul_8_16>")
1944 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1945 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1948 (define_insn "neon_vqdmulh<mode>"
1949 [(set (match_operand:VMDQI 0 "s_register_operand" "=w")
1950 (unspec:VMDQI [(match_operand:VMDQI 1 "s_register_operand" "w")
1951 (match_operand:VMDQI 2 "s_register_operand" "w")
1952 (match_operand:SI 3 "immediate_operand" "i")]
1955 "vq%O3dmulh.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
1956 [(set (attr "neon_type")
1957 (if_then_else (match_test "<Is_d_reg>")
1958 (if_then_else (match_test "<Scalar_mul_8_16>")
1959 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
1960 (const_string "neon_mul_qqq_8_16_32_ddd_32"))
1961 (if_then_else (match_test "<Scalar_mul_8_16>")
1962 (const_string "neon_mul_qqq_8_16_32_ddd_32")
1963 (const_string "neon_mul_qqq_8_16_32_ddd_32"))))]
1966 (define_insn "neon_vqdmlal<mode>"
1967 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1968 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1969 (match_operand:VMDI 2 "s_register_operand" "w")
1970 (match_operand:VMDI 3 "s_register_operand" "w")
1971 (match_operand:SI 4 "immediate_operand" "i")]
1974 "vqdmlal.<V_s_elem>\t%q0, %P2, %P3"
1975 [(set (attr "neon_type")
1976 (if_then_else (match_test "<Scalar_mul_8_16>")
1977 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1978 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1981 (define_insn "neon_vqdmlsl<mode>"
1982 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1983 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
1984 (match_operand:VMDI 2 "s_register_operand" "w")
1985 (match_operand:VMDI 3 "s_register_operand" "w")
1986 (match_operand:SI 4 "immediate_operand" "i")]
1989 "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3"
1990 [(set (attr "neon_type")
1991 (if_then_else (match_test "<Scalar_mul_8_16>")
1992 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long")
1993 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
1996 (define_insn "neon_vmull<mode>"
1997 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
1998 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
1999 (match_operand:VW 2 "s_register_operand" "w")
2000 (match_operand:SI 3 "immediate_operand" "i")]
2003 "vmull.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2004 [(set (attr "neon_type")
2005 (if_then_else (match_test "<Scalar_mul_8_16>")
2006 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2007 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
2010 (define_insn "neon_vqdmull<mode>"
2011 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2012 (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
2013 (match_operand:VMDI 2 "s_register_operand" "w")
2014 (match_operand:SI 3 "immediate_operand" "i")]
2017 "vqdmull.<V_s_elem>\t%q0, %P1, %P2"
2018 [(set (attr "neon_type")
2019 (if_then_else (match_test "<Scalar_mul_8_16>")
2020 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long")
2021 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
2024 (define_expand "neon_vsub<mode>"
2025 [(match_operand:VDQX 0 "s_register_operand" "=w")
2026 (match_operand:VDQX 1 "s_register_operand" "w")
2027 (match_operand:VDQX 2 "s_register_operand" "w")
2028 (match_operand:SI 3 "immediate_operand" "i")]
2031 if (!<Is_float_mode> || flag_unsafe_math_optimizations)
2032 emit_insn (gen_sub<mode>3 (operands[0], operands[1], operands[2]));
2034 emit_insn (gen_neon_vsub<mode>_unspec (operands[0], operands[1],
2039 ; Used for intrinsics when flag_unsafe_math_optimizations is false.
2041 (define_insn "neon_vsub<mode>_unspec"
2042 [(set (match_operand:VDQX 0 "s_register_operand" "=w")
2043 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
2044 (match_operand:VDQX 2 "s_register_operand" "w")]
2047 "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2048 [(set (attr "neon_type")
2049 (if_then_else (match_test "<Is_float_mode>")
2050 (if_then_else (match_test "<Is_d_reg>")
2051 (const_string "neon_fp_vadd_ddd_vabs_dd")
2052 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2053 (const_string "neon_int_2")))]
2056 (define_insn "neon_vsubl<mode>"
2057 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2058 (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w")
2059 (match_operand:VDI 2 "s_register_operand" "w")
2060 (match_operand:SI 3 "immediate_operand" "i")]
2063 "vsubl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2064 [(set_attr "neon_type" "neon_int_2")]
2067 (define_insn "neon_vsubw<mode>"
2068 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2069 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w")
2070 (match_operand:VDI 2 "s_register_operand" "w")
2071 (match_operand:SI 3 "immediate_operand" "i")]
2074 "vsubw.%T3%#<V_sz_elem>\t%q0, %q1, %P2"
2075 [(set_attr "neon_type" "neon_int_2")]
2078 (define_insn "neon_vqsub<mode>"
2079 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
2080 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
2081 (match_operand:VDQIX 2 "s_register_operand" "w")
2082 (match_operand:SI 3 "immediate_operand" "i")]
2085 "vqsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2086 [(set_attr "neon_type" "neon_int_5")]
2089 (define_insn "neon_vhsub<mode>"
2090 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2091 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2092 (match_operand:VDQIW 2 "s_register_operand" "w")
2093 (match_operand:SI 3 "immediate_operand" "i")]
2096 "vhsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2097 [(set_attr "neon_type" "neon_int_5")]
2100 (define_insn "neon_vsubhn<mode>"
2101 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
2102 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
2103 (match_operand:VN 2 "s_register_operand" "w")
2104 (match_operand:SI 3 "immediate_operand" "i")]
2107 "v%O3subhn.<V_if_elem>\t%P0, %q1, %q2"
2108 [(set_attr "neon_type" "neon_int_4")]
2111 (define_insn "neon_vceq<mode>"
2112 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2113 (unspec:<V_cmp_result>
2114 [(match_operand:VDQW 1 "s_register_operand" "w,w")
2115 (match_operand:VDQW 2 "nonmemory_operand" "w,Dz")
2116 (match_operand:SI 3 "immediate_operand" "i,i")]
2120 vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2121 vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, #0"
2122 [(set (attr "neon_type")
2123 (if_then_else (match_test "<Is_float_mode>")
2124 (if_then_else (match_test "<Is_d_reg>")
2125 (const_string "neon_fp_vadd_ddd_vabs_dd")
2126 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2127 (const_string "neon_int_5")))]
2130 (define_insn "neon_vcge<mode>"
2131 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2132 (unspec:<V_cmp_result>
2133 [(match_operand:VDQW 1 "s_register_operand" "w,w")
2134 (match_operand:VDQW 2 "nonmemory_operand" "w,Dz")
2135 (match_operand:SI 3 "immediate_operand" "i,i")]
2139 vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2140 vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2141 [(set (attr "neon_type")
2142 (if_then_else (match_test "<Is_float_mode>")
2143 (if_then_else (match_test "<Is_d_reg>")
2144 (const_string "neon_fp_vadd_ddd_vabs_dd")
2145 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2146 (const_string "neon_int_5")))]
2149 (define_insn "neon_vcgt<mode>"
2150 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w")
2151 (unspec:<V_cmp_result>
2152 [(match_operand:VDQW 1 "s_register_operand" "w,w")
2153 (match_operand:VDQW 2 "nonmemory_operand" "w,Dz")
2154 (match_operand:SI 3 "immediate_operand" "i,i")]
2158 vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2
2159 vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2160 [(set (attr "neon_type")
2161 (if_then_else (match_test "<Is_float_mode>")
2162 (if_then_else (match_test "<Is_d_reg>")
2163 (const_string "neon_fp_vadd_ddd_vabs_dd")
2164 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2165 (const_string "neon_int_5")))]
2168 ;; VCLE and VCLT only support comparisons with immediate zero (register
2169 ;; variants are VCGE and VCGT with operands reversed).
2171 (define_insn "neon_vcle<mode>"
2172 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2173 (unspec:<V_cmp_result>
2174 [(match_operand:VDQW 1 "s_register_operand" "w")
2175 (match_operand:VDQW 2 "nonmemory_operand" "Dz")
2176 (match_operand:SI 3 "immediate_operand" "i")]
2179 "vcle.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2180 [(set (attr "neon_type")
2181 (if_then_else (match_test "<Is_float_mode>")
2182 (if_then_else (match_test "<Is_d_reg>")
2183 (const_string "neon_fp_vadd_ddd_vabs_dd")
2184 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2185 (const_string "neon_int_5")))]
2188 (define_insn "neon_vclt<mode>"
2189 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2190 (unspec:<V_cmp_result>
2191 [(match_operand:VDQW 1 "s_register_operand" "w")
2192 (match_operand:VDQW 2 "nonmemory_operand" "Dz")
2193 (match_operand:SI 3 "immediate_operand" "i")]
2196 "vclt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0"
2197 [(set (attr "neon_type")
2198 (if_then_else (match_test "<Is_float_mode>")
2199 (if_then_else (match_test "<Is_d_reg>")
2200 (const_string "neon_fp_vadd_ddd_vabs_dd")
2201 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2202 (const_string "neon_int_5")))]
2205 (define_insn "neon_vcage<mode>"
2206 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2207 (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2208 (match_operand:VCVTF 2 "s_register_operand" "w")
2209 (match_operand:SI 3 "immediate_operand" "i")]
2212 "vacge.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2213 [(set (attr "neon_type")
2214 (if_then_else (match_test "<Is_d_reg>")
2215 (const_string "neon_fp_vadd_ddd_vabs_dd")
2216 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2219 (define_insn "neon_vcagt<mode>"
2220 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w")
2221 (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w")
2222 (match_operand:VCVTF 2 "s_register_operand" "w")
2223 (match_operand:SI 3 "immediate_operand" "i")]
2226 "vacgt.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2227 [(set (attr "neon_type")
2228 (if_then_else (match_test "<Is_d_reg>")
2229 (const_string "neon_fp_vadd_ddd_vabs_dd")
2230 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2233 (define_insn "neon_vtst<mode>"
2234 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2235 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2236 (match_operand:VDQIW 2 "s_register_operand" "w")
2237 (match_operand:SI 3 "immediate_operand" "i")]
2240 "vtst.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2241 [(set_attr "neon_type" "neon_int_4")]
2244 (define_insn "neon_vabd<mode>"
2245 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2246 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2247 (match_operand:VDQW 2 "s_register_operand" "w")
2248 (match_operand:SI 3 "immediate_operand" "i")]
2251 "vabd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2252 [(set (attr "neon_type")
2253 (if_then_else (match_test "<Is_float_mode>")
2254 (if_then_else (match_test "<Is_d_reg>")
2255 (const_string "neon_fp_vadd_ddd_vabs_dd")
2256 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2257 (const_string "neon_int_5")))]
2260 (define_insn "neon_vabdl<mode>"
2261 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2262 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
2263 (match_operand:VW 2 "s_register_operand" "w")
2264 (match_operand:SI 3 "immediate_operand" "i")]
2267 "vabdl.%T3%#<V_sz_elem>\t%q0, %P1, %P2"
2268 [(set_attr "neon_type" "neon_int_5")]
2271 (define_insn "neon_vaba<mode>"
2272 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2273 (plus:VDQIW (match_operand:VDQIW 1 "s_register_operand" "0")
2274 (unspec:VDQIW [(match_operand:VDQIW 2 "s_register_operand" "w")
2275 (match_operand:VDQIW 3 "s_register_operand" "w")
2276 (match_operand:SI 4 "immediate_operand" "i")]
2279 "vaba.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3"
2280 [(set (attr "neon_type")
2281 (if_then_else (match_test "<Is_d_reg>")
2282 (const_string "neon_vaba") (const_string "neon_vaba_qqq")))]
2285 (define_insn "neon_vabal<mode>"
2286 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
2287 (plus:<V_widen> (match_operand:<V_widen> 1 "s_register_operand" "0")
2288 (unspec:<V_widen> [(match_operand:VW 2 "s_register_operand" "w")
2289 (match_operand:VW 3 "s_register_operand" "w")
2290 (match_operand:SI 4 "immediate_operand" "i")]
2293 "vabal.%T4%#<V_sz_elem>\t%q0, %P2, %P3"
2294 [(set_attr "neon_type" "neon_vaba")]
2297 (define_insn "neon_vmax<mode>"
2298 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2299 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2300 (match_operand:VDQW 2 "s_register_operand" "w")
2301 (match_operand:SI 3 "immediate_operand" "i")]
2304 "vmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2305 [(set (attr "neon_type")
2306 (if_then_else (match_test "<Is_float_mode>")
2307 (if_then_else (match_test "<Is_d_reg>")
2308 (const_string "neon_fp_vadd_ddd_vabs_dd")
2309 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2310 (const_string "neon_int_5")))]
2313 (define_insn "neon_vmin<mode>"
2314 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2315 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w")
2316 (match_operand:VDQW 2 "s_register_operand" "w")
2317 (match_operand:SI 3 "immediate_operand" "i")]
2320 "vmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2321 [(set (attr "neon_type")
2322 (if_then_else (match_test "<Is_float_mode>")
2323 (if_then_else (match_test "<Is_d_reg>")
2324 (const_string "neon_fp_vadd_ddd_vabs_dd")
2325 (const_string "neon_fp_vadd_qqq_vabs_qq"))
2326 (const_string "neon_int_5")))]
2329 (define_expand "neon_vpadd<mode>"
2330 [(match_operand:VD 0 "s_register_operand" "=w")
2331 (match_operand:VD 1 "s_register_operand" "w")
2332 (match_operand:VD 2 "s_register_operand" "w")
2333 (match_operand:SI 3 "immediate_operand" "i")]
2336 emit_insn (gen_neon_vpadd_internal<mode> (operands[0], operands[1],
2341 (define_insn "neon_vpaddl<mode>"
2342 [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2343 (unspec:<V_double_width> [(match_operand:VDQIW 1 "s_register_operand" "w")
2344 (match_operand:SI 2 "immediate_operand" "i")]
2347 "vpaddl.%T2%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2348 ;; Assume this schedules like vaddl.
2349 [(set_attr "neon_type" "neon_int_3")]
2352 (define_insn "neon_vpadal<mode>"
2353 [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w")
2354 (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0")
2355 (match_operand:VDQIW 2 "s_register_operand" "w")
2356 (match_operand:SI 3 "immediate_operand" "i")]
2359 "vpadal.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2"
2360 ;; Assume this schedules like vpadd.
2361 [(set_attr "neon_type" "neon_int_1")]
2364 (define_insn "neon_vpmax<mode>"
2365 [(set (match_operand:VD 0 "s_register_operand" "=w")
2366 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2367 (match_operand:VD 2 "s_register_operand" "w")
2368 (match_operand:SI 3 "immediate_operand" "i")]
2371 "vpmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2372 ;; Assume this schedules like vmax.
2373 [(set (attr "neon_type")
2374 (if_then_else (match_test "<Is_float_mode>")
2375 (const_string "neon_fp_vadd_ddd_vabs_dd")
2376 (const_string "neon_int_5")))]
2379 (define_insn "neon_vpmin<mode>"
2380 [(set (match_operand:VD 0 "s_register_operand" "=w")
2381 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w")
2382 (match_operand:VD 2 "s_register_operand" "w")
2383 (match_operand:SI 3 "immediate_operand" "i")]
2386 "vpmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2387 ;; Assume this schedules like vmin.
2388 [(set (attr "neon_type")
2389 (if_then_else (match_test "<Is_float_mode>")
2390 (const_string "neon_fp_vadd_ddd_vabs_dd")
2391 (const_string "neon_int_5")))]
2394 (define_insn "neon_vrecps<mode>"
2395 [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2396 (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2397 (match_operand:VCVTF 2 "s_register_operand" "w")
2398 (match_operand:SI 3 "immediate_operand" "i")]
2401 "vrecps.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2402 [(set (attr "neon_type")
2403 (if_then_else (match_test "<Is_d_reg>")
2404 (const_string "neon_fp_vrecps_vrsqrts_ddd")
2405 (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2408 (define_insn "neon_vrsqrts<mode>"
2409 [(set (match_operand:VCVTF 0 "s_register_operand" "=w")
2410 (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w")
2411 (match_operand:VCVTF 2 "s_register_operand" "w")
2412 (match_operand:SI 3 "immediate_operand" "i")]
2415 "vrsqrts.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
2416 [(set (attr "neon_type")
2417 (if_then_else (match_test "<Is_d_reg>")
2418 (const_string "neon_fp_vrecps_vrsqrts_ddd")
2419 (const_string "neon_fp_vrecps_vrsqrts_qqq")))]
2422 (define_expand "neon_vabs<mode>"
2423 [(match_operand:VDQW 0 "s_register_operand" "")
2424 (match_operand:VDQW 1 "s_register_operand" "")
2425 (match_operand:SI 2 "immediate_operand" "")]
2428 emit_insn (gen_abs<mode>2 (operands[0], operands[1]));
2432 (define_insn "neon_vqabs<mode>"
2433 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2434 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2435 (match_operand:SI 2 "immediate_operand" "i")]
2438 "vqabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2439 [(set_attr "neon_type" "neon_vqneg_vqabs")]
2442 (define_expand "neon_vneg<mode>"
2443 [(match_operand:VDQW 0 "s_register_operand" "")
2444 (match_operand:VDQW 1 "s_register_operand" "")
2445 (match_operand:SI 2 "immediate_operand" "")]
2448 emit_insn (gen_neg<mode>2 (operands[0], operands[1]));
2452 (define_insn "neon_vqneg<mode>"
2453 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2454 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2455 (match_operand:SI 2 "immediate_operand" "i")]
2458 "vqneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2459 [(set_attr "neon_type" "neon_vqneg_vqabs")]
2462 (define_insn "neon_vcls<mode>"
2463 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2464 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w")
2465 (match_operand:SI 2 "immediate_operand" "i")]
2468 "vcls.<V_s_elem>\t%<V_reg>0, %<V_reg>1"
2469 [(set_attr "neon_type" "neon_int_1")]
2472 (define_insn "clz<mode>2"
2473 [(set (match_operand:VDQIW 0 "s_register_operand" "=w")
2474 (clz:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")))]
2476 "vclz.<V_if_elem>\t%<V_reg>0, %<V_reg>1"
2477 [(set_attr "neon_type" "neon_int_1")]
2480 (define_expand "neon_vclz<mode>"
2481 [(match_operand:VDQIW 0 "s_register_operand" "")
2482 (match_operand:VDQIW 1 "s_register_operand" "")
2483 (match_operand:SI 2 "immediate_operand" "")]
2486 emit_insn (gen_clz<mode>2 (operands[0], operands[1]));
2490 (define_insn "popcount<mode>2"
2491 [(set (match_operand:VE 0 "s_register_operand" "=w")
2492 (popcount:VE (match_operand:VE 1 "s_register_operand" "w")))]
2494 "vcnt.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
2495 [(set_attr "neon_type" "neon_int_1")]
2498 (define_expand "neon_vcnt<mode>"
2499 [(match_operand:VE 0 "s_register_operand" "=w")
2500 (match_operand:VE 1 "s_register_operand" "w")
2501 (match_operand:SI 2 "immediate_operand" "i")]
2504 emit_insn (gen_popcount<mode>2 (operands[0], operands[1]));
2508 (define_insn "neon_vrecpe<mode>"
2509 [(set (match_operand:V32 0 "s_register_operand" "=w")
2510 (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2511 (match_operand:SI 2 "immediate_operand" "i")]
2514 "vrecpe.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2515 [(set (attr "neon_type")
2516 (if_then_else (match_test "<Is_d_reg>")
2517 (const_string "neon_fp_vadd_ddd_vabs_dd")
2518 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2521 (define_insn "neon_vrsqrte<mode>"
2522 [(set (match_operand:V32 0 "s_register_operand" "=w")
2523 (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w")
2524 (match_operand:SI 2 "immediate_operand" "i")]
2527 "vrsqrte.<V_u_elem>\t%<V_reg>0, %<V_reg>1"
2528 [(set (attr "neon_type")
2529 (if_then_else (match_test "<Is_d_reg>")
2530 (const_string "neon_fp_vadd_ddd_vabs_dd")
2531 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2534 (define_expand "neon_vmvn<mode>"
2535 [(match_operand:VDQIW 0 "s_register_operand" "")
2536 (match_operand:VDQIW 1 "s_register_operand" "")
2537 (match_operand:SI 2 "immediate_operand" "")]
2540 emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[1]));
2544 (define_insn "neon_vget_lane<mode>_sext_internal"
2545 [(set (match_operand:SI 0 "s_register_operand" "=r")
2547 (vec_select:<V_elem>
2548 (match_operand:VD 1 "s_register_operand" "w")
2549 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2552 if (BYTES_BIG_ENDIAN)
2554 int elt = INTVAL (operands[2]);
2555 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2556 operands[2] = GEN_INT (elt);
2558 return "vmov%?.s<V_sz_elem>\t%0, %P1[%c2]";
2560 [(set_attr "predicable" "yes")
2561 (set_attr "neon_type" "neon_bp_simple")]
2564 (define_insn "neon_vget_lane<mode>_zext_internal"
2565 [(set (match_operand:SI 0 "s_register_operand" "=r")
2567 (vec_select:<V_elem>
2568 (match_operand:VD 1 "s_register_operand" "w")
2569 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2572 if (BYTES_BIG_ENDIAN)
2574 int elt = INTVAL (operands[2]);
2575 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt;
2576 operands[2] = GEN_INT (elt);
2578 return "vmov%?.u<V_sz_elem>\t%0, %P1[%c2]";
2580 [(set_attr "predicable" "yes")
2581 (set_attr "neon_type" "neon_bp_simple")]
2584 (define_insn "neon_vget_lane<mode>_sext_internal"
2585 [(set (match_operand:SI 0 "s_register_operand" "=r")
2587 (vec_select:<V_elem>
2588 (match_operand:VQ 1 "s_register_operand" "w")
2589 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2593 int regno = REGNO (operands[1]);
2594 unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2595 unsigned int elt = INTVAL (operands[2]);
2596 unsigned int elt_adj = elt % halfelts;
2598 if (BYTES_BIG_ENDIAN)
2599 elt_adj = halfelts - 1 - elt_adj;
2601 ops[0] = operands[0];
2602 ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2603 ops[2] = GEN_INT (elt_adj);
2604 output_asm_insn ("vmov%?.s<V_sz_elem>\t%0, %P1[%c2]", ops);
2608 [(set_attr "predicable" "yes")
2609 (set_attr "neon_type" "neon_bp_simple")]
2612 (define_insn "neon_vget_lane<mode>_zext_internal"
2613 [(set (match_operand:SI 0 "s_register_operand" "=r")
2615 (vec_select:<V_elem>
2616 (match_operand:VQ 1 "s_register_operand" "w")
2617 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2621 int regno = REGNO (operands[1]);
2622 unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2;
2623 unsigned int elt = INTVAL (operands[2]);
2624 unsigned int elt_adj = elt % halfelts;
2626 if (BYTES_BIG_ENDIAN)
2627 elt_adj = halfelts - 1 - elt_adj;
2629 ops[0] = operands[0];
2630 ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts));
2631 ops[2] = GEN_INT (elt_adj);
2632 output_asm_insn ("vmov%?.u<V_sz_elem>\t%0, %P1[%c2]", ops);
2636 [(set_attr "predicable" "yes")
2637 (set_attr "neon_type" "neon_bp_simple")]
2640 (define_expand "neon_vget_lane<mode>"
2641 [(match_operand:<V_ext> 0 "s_register_operand" "")
2642 (match_operand:VDQW 1 "s_register_operand" "")
2643 (match_operand:SI 2 "immediate_operand" "")
2644 (match_operand:SI 3 "immediate_operand" "")]
2647 HOST_WIDE_INT magic = INTVAL (operands[3]);
2650 neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<MODE>mode));
2652 if (BYTES_BIG_ENDIAN)
2654 /* The intrinsics are defined in terms of a model where the
2655 element ordering in memory is vldm order, whereas the generic
2656 RTL is defined in terms of a model where the element ordering
2657 in memory is array order. Convert the lane number to conform
2659 unsigned int elt = INTVAL (operands[2]);
2660 unsigned int reg_nelts
2661 = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2662 elt ^= reg_nelts - 1;
2663 operands[2] = GEN_INT (elt);
2666 if ((magic & 3) == 3 || GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode)) == 32)
2667 insn = gen_vec_extract<mode> (operands[0], operands[1], operands[2]);
2670 if ((magic & 1) != 0)
2671 insn = gen_neon_vget_lane<mode>_sext_internal (operands[0], operands[1],
2674 insn = gen_neon_vget_lane<mode>_zext_internal (operands[0], operands[1],
2681 ; Operand 3 (info word) is ignored because it does nothing useful with 64-bit
2684 (define_expand "neon_vget_lanedi"
2685 [(match_operand:DI 0 "s_register_operand" "=r")
2686 (match_operand:DI 1 "s_register_operand" "w")
2687 (match_operand:SI 2 "immediate_operand" "i")
2688 (match_operand:SI 3 "immediate_operand" "i")]
2691 neon_lane_bounds (operands[2], 0, 1);
2692 emit_move_insn (operands[0], operands[1]);
2696 (define_expand "neon_vget_lanev2di"
2697 [(match_operand:DI 0 "s_register_operand" "=r")
2698 (match_operand:V2DI 1 "s_register_operand" "w")
2699 (match_operand:SI 2 "immediate_operand" "i")
2700 (match_operand:SI 3 "immediate_operand" "i")]
2703 neon_lane_bounds (operands[2], 0, 2);
2704 emit_insn (gen_vec_extractv2di (operands[0], operands[1], operands[2]));
2708 (define_expand "neon_vset_lane<mode>"
2709 [(match_operand:VDQ 0 "s_register_operand" "=w")
2710 (match_operand:<V_elem> 1 "s_register_operand" "r")
2711 (match_operand:VDQ 2 "s_register_operand" "0")
2712 (match_operand:SI 3 "immediate_operand" "i")]
2715 unsigned int elt = INTVAL (operands[3]);
2716 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
2718 if (BYTES_BIG_ENDIAN)
2720 unsigned int reg_nelts
2721 = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode));
2722 elt ^= reg_nelts - 1;
2725 emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1],
2726 GEN_INT (1 << elt), operands[2]));
2730 ; See neon_vget_lanedi comment for reasons operands 2 & 3 are ignored.
2732 (define_expand "neon_vset_lanedi"
2733 [(match_operand:DI 0 "s_register_operand" "=w")
2734 (match_operand:DI 1 "s_register_operand" "r")
2735 (match_operand:DI 2 "s_register_operand" "0")
2736 (match_operand:SI 3 "immediate_operand" "i")]
2739 neon_lane_bounds (operands[3], 0, 1);
2740 emit_move_insn (operands[0], operands[1]);
2744 (define_expand "neon_vcreate<mode>"
2745 [(match_operand:VDX 0 "s_register_operand" "")
2746 (match_operand:DI 1 "general_operand" "")]
2749 rtx src = gen_lowpart (<MODE>mode, operands[1]);
2750 emit_move_insn (operands[0], src);
2754 (define_insn "neon_vdup_n<mode>"
2755 [(set (match_operand:VX 0 "s_register_operand" "=w")
2756 (vec_duplicate:VX (match_operand:<V_elem> 1 "s_register_operand" "r")))]
2758 "vdup%?.<V_sz_elem>\t%<V_reg>0, %1"
2759 ;; Assume this schedules like vmov.
2760 [(set_attr "predicable" "yes")
2761 (set_attr "neon_type" "neon_bp_simple")]
2764 (define_insn "neon_vdup_n<mode>"
2765 [(set (match_operand:V32 0 "s_register_operand" "=w,w")
2766 (vec_duplicate:V32 (match_operand:<V_elem> 1 "s_register_operand" "r,t")))]
2769 vdup%?.<V_sz_elem>\t%<V_reg>0, %1
2770 vdup%?.<V_sz_elem>\t%<V_reg>0, %y1"
2771 ;; Assume this schedules like vmov.
2772 [(set_attr "predicable" "yes")
2773 (set_attr "neon_type" "neon_bp_simple")]
2776 (define_expand "neon_vdup_ndi"
2777 [(match_operand:DI 0 "s_register_operand" "=w")
2778 (match_operand:DI 1 "s_register_operand" "r")]
2781 emit_move_insn (operands[0], operands[1]);
2786 (define_insn "neon_vdup_nv2di"
2787 [(set (match_operand:V2DI 0 "s_register_operand" "=w,w")
2788 (vec_duplicate:V2DI (match_operand:DI 1 "s_register_operand" "r,w")))]
2791 vmov%?\t%e0, %Q1, %R1\;vmov%?\t%f0, %Q1, %R1
2792 vmov%?\t%e0, %P1\;vmov%?\t%f0, %P1"
2793 [(set_attr "predicable" "yes")
2794 (set_attr "length" "8")
2795 (set_attr "neon_type" "neon_bp_simple")]
2798 (define_insn "neon_vdup_lane<mode>_internal"
2799 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
2801 (vec_select:<V_elem>
2802 (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
2803 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))]
2806 if (BYTES_BIG_ENDIAN)
2808 int elt = INTVAL (operands[2]);
2809 elt = GET_MODE_NUNITS (<V_double_vector_mode>mode) - 1 - elt;
2810 operands[2] = GEN_INT (elt);
2813 return "vdup.<V_sz_elem>\t%P0, %P1[%c2]";
2815 return "vdup.<V_sz_elem>\t%q0, %P1[%c2]";
2817 ;; Assume this schedules like vmov.
2818 [(set_attr "neon_type" "neon_bp_simple")]
2821 (define_expand "neon_vdup_lane<mode>"
2822 [(match_operand:VDQW 0 "s_register_operand" "=w")
2823 (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w")
2824 (match_operand:SI 2 "immediate_operand" "i")]
2827 neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<V_double_vector_mode>mode));
2828 if (BYTES_BIG_ENDIAN)
2830 unsigned int elt = INTVAL (operands[2]);
2831 unsigned int reg_nelts
2832 = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<V_double_vector_mode>mode));
2833 elt ^= reg_nelts - 1;
2834 operands[2] = GEN_INT (elt);
2836 emit_insn (gen_neon_vdup_lane<mode>_internal (operands[0], operands[1],
2841 ; Scalar index is ignored, since only zero is valid here.
2842 (define_expand "neon_vdup_lanedi"
2843 [(match_operand:DI 0 "s_register_operand" "=w")
2844 (match_operand:DI 1 "s_register_operand" "w")
2845 (match_operand:SI 2 "immediate_operand" "i")]
2848 neon_lane_bounds (operands[2], 0, 1);
2849 emit_move_insn (operands[0], operands[1]);
2853 ; Likewise for v2di, as the DImode second operand has only a single element.
2854 (define_expand "neon_vdup_lanev2di"
2855 [(match_operand:V2DI 0 "s_register_operand" "=w")
2856 (match_operand:DI 1 "s_register_operand" "w")
2857 (match_operand:SI 2 "immediate_operand" "i")]
2860 neon_lane_bounds (operands[2], 0, 1);
2861 emit_insn (gen_neon_vdup_nv2di (operands[0], operands[1]));
2865 ; Disabled before reload because we don't want combine doing something silly,
2866 ; but used by the post-reload expansion of neon_vcombine.
2867 (define_insn "*neon_vswp<mode>"
2868 [(set (match_operand:VDQX 0 "s_register_operand" "+w")
2869 (match_operand:VDQX 1 "s_register_operand" "+w"))
2870 (set (match_dup 1) (match_dup 0))]
2871 "TARGET_NEON && reload_completed"
2872 "vswp\t%<V_reg>0, %<V_reg>1"
2873 [(set (attr "neon_type")
2874 (if_then_else (match_test "<Is_d_reg>")
2875 (const_string "neon_bp_simple")
2876 (const_string "neon_bp_2cycle")))]
2879 ;; In this insn, operand 1 should be low, and operand 2 the high part of the
2881 ;; FIXME: A different implementation of this builtin could make it much
2882 ;; more likely that we wouldn't actually need to output anything (we could make
2883 ;; it so that the reg allocator puts things in the right places magically
2884 ;; instead). Lack of subregs for vectors makes that tricky though, I think.
2886 (define_insn_and_split "neon_vcombine<mode>"
2887 [(set (match_operand:<V_DOUBLE> 0 "s_register_operand" "=w")
2888 (vec_concat:<V_DOUBLE>
2889 (match_operand:VDX 1 "s_register_operand" "w")
2890 (match_operand:VDX 2 "s_register_operand" "w")))]
2893 "&& reload_completed"
2896 neon_split_vcombine (operands);
2900 (define_expand "neon_vget_high<mode>"
2901 [(match_operand:<V_HALF> 0 "s_register_operand")
2902 (match_operand:VQX 1 "s_register_operand")]
2905 emit_move_insn (operands[0],
2906 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
2907 GET_MODE_SIZE (<V_HALF>mode)));
2911 (define_expand "neon_vget_low<mode>"
2912 [(match_operand:<V_HALF> 0 "s_register_operand")
2913 (match_operand:VQX 1 "s_register_operand")]
2916 emit_move_insn (operands[0],
2917 simplify_gen_subreg (<V_HALF>mode, operands[1],
2922 (define_insn "float<mode><V_cvtto>2"
2923 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
2924 (float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
2925 "TARGET_NEON && !flag_rounding_math"
2926 "vcvt.f32.s32\t%<V_reg>0, %<V_reg>1"
2927 [(set (attr "neon_type")
2928 (if_then_else (match_test "<Is_d_reg>")
2929 (const_string "neon_fp_vadd_ddd_vabs_dd")
2930 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2933 (define_insn "floatuns<mode><V_cvtto>2"
2934 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
2935 (unsigned_float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))]
2936 "TARGET_NEON && !flag_rounding_math"
2937 "vcvt.f32.u32\t%<V_reg>0, %<V_reg>1"
2938 [(set (attr "neon_type")
2939 (if_then_else (match_test "<Is_d_reg>")
2940 (const_string "neon_fp_vadd_ddd_vabs_dd")
2941 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2944 (define_insn "fix_trunc<mode><V_cvtto>2"
2945 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
2946 (fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
2948 "vcvt.s32.f32\t%<V_reg>0, %<V_reg>1"
2949 [(set (attr "neon_type")
2950 (if_then_else (match_test "<Is_d_reg>")
2951 (const_string "neon_fp_vadd_ddd_vabs_dd")
2952 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2955 (define_insn "fixuns_trunc<mode><V_cvtto>2"
2956 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
2957 (unsigned_fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))]
2959 "vcvt.u32.f32\t%<V_reg>0, %<V_reg>1"
2960 [(set (attr "neon_type")
2961 (if_then_else (match_test "<Is_d_reg>")
2962 (const_string "neon_fp_vadd_ddd_vabs_dd")
2963 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2966 (define_insn "neon_vcvt<mode>"
2967 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
2968 (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
2969 (match_operand:SI 2 "immediate_operand" "i")]
2972 "vcvt.%T2%#32.f32\t%<V_reg>0, %<V_reg>1"
2973 [(set (attr "neon_type")
2974 (if_then_else (match_test "<Is_d_reg>")
2975 (const_string "neon_fp_vadd_ddd_vabs_dd")
2976 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2979 (define_insn "neon_vcvt<mode>"
2980 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
2981 (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
2982 (match_operand:SI 2 "immediate_operand" "i")]
2985 "vcvt.f32.%T2%#32\t%<V_reg>0, %<V_reg>1"
2986 [(set (attr "neon_type")
2987 (if_then_else (match_test "<Is_d_reg>")
2988 (const_string "neon_fp_vadd_ddd_vabs_dd")
2989 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
2992 (define_insn "neon_vcvt_n<mode>"
2993 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
2994 (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w")
2995 (match_operand:SI 2 "immediate_operand" "i")
2996 (match_operand:SI 3 "immediate_operand" "i")]
3000 neon_const_bounds (operands[2], 1, 33);
3001 return "vcvt.%T3%#32.f32\t%<V_reg>0, %<V_reg>1, %2";
3003 [(set (attr "neon_type")
3004 (if_then_else (match_test "<Is_d_reg>")
3005 (const_string "neon_fp_vadd_ddd_vabs_dd")
3006 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3009 (define_insn "neon_vcvt_n<mode>"
3010 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w")
3011 (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w")
3012 (match_operand:SI 2 "immediate_operand" "i")
3013 (match_operand:SI 3 "immediate_operand" "i")]
3017 neon_const_bounds (operands[2], 1, 33);
3018 return "vcvt.f32.%T3%#32\t%<V_reg>0, %<V_reg>1, %2";
3020 [(set (attr "neon_type")
3021 (if_then_else (match_test "<Is_d_reg>")
3022 (const_string "neon_fp_vadd_ddd_vabs_dd")
3023 (const_string "neon_fp_vadd_qqq_vabs_qq")))]
3026 (define_insn "neon_vmovn<mode>"
3027 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3028 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3029 (match_operand:SI 2 "immediate_operand" "i")]
3032 "vmovn.<V_if_elem>\t%P0, %q1"
3033 [(set_attr "neon_type" "neon_bp_simple")]
3036 (define_insn "neon_vqmovn<mode>"
3037 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3038 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3039 (match_operand:SI 2 "immediate_operand" "i")]
3042 "vqmovn.%T2%#<V_sz_elem>\t%P0, %q1"
3043 [(set_attr "neon_type" "neon_shift_2")]
3046 (define_insn "neon_vqmovun<mode>"
3047 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3048 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3049 (match_operand:SI 2 "immediate_operand" "i")]
3052 "vqmovun.<V_s_elem>\t%P0, %q1"
3053 [(set_attr "neon_type" "neon_shift_2")]
3056 (define_insn "neon_vmovl<mode>"
3057 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3058 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3059 (match_operand:SI 2 "immediate_operand" "i")]
3062 "vmovl.%T2%#<V_sz_elem>\t%q0, %P1"
3063 [(set_attr "neon_type" "neon_shift_1")]
3066 (define_insn "neon_vmul_lane<mode>"
3067 [(set (match_operand:VMD 0 "s_register_operand" "=w")
3068 (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "w")
3069 (match_operand:VMD 2 "s_register_operand"
3070 "<scalar_mul_constraint>")
3071 (match_operand:SI 3 "immediate_operand" "i")
3072 (match_operand:SI 4 "immediate_operand" "i")]
3076 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3077 return "vmul.<V_if_elem>\t%P0, %P1, %P2[%c3]";
3079 [(set (attr "neon_type")
3080 (if_then_else (match_test "<Is_float_mode>")
3081 (const_string "neon_fp_vmul_ddd")
3082 (if_then_else (match_test "<Scalar_mul_8_16>")
3083 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3084 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar"))))]
3087 (define_insn "neon_vmul_lane<mode>"
3088 [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3089 (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "w")
3090 (match_operand:<V_HALF> 2 "s_register_operand"
3091 "<scalar_mul_constraint>")
3092 (match_operand:SI 3 "immediate_operand" "i")
3093 (match_operand:SI 4 "immediate_operand" "i")]
3097 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<V_HALF>mode));
3098 return "vmul.<V_if_elem>\t%q0, %q1, %P2[%c3]";
3100 [(set (attr "neon_type")
3101 (if_then_else (match_test "<Is_float_mode>")
3102 (const_string "neon_fp_vmul_qqd")
3103 (if_then_else (match_test "<Scalar_mul_8_16>")
3104 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3105 (const_string "neon_mul_qqd_32_scalar"))))]
3108 (define_insn "neon_vmull_lane<mode>"
3109 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3110 (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3111 (match_operand:VMDI 2 "s_register_operand"
3112 "<scalar_mul_constraint>")
3113 (match_operand:SI 3 "immediate_operand" "i")
3114 (match_operand:SI 4 "immediate_operand" "i")]
3115 UNSPEC_VMULL_LANE))]
3118 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3119 return "vmull.%T4%#<V_sz_elem>\t%q0, %P1, %P2[%c3]";
3121 [(set (attr "neon_type")
3122 (if_then_else (match_test "<Scalar_mul_8_16>")
3123 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3124 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3127 (define_insn "neon_vqdmull_lane<mode>"
3128 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3129 (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w")
3130 (match_operand:VMDI 2 "s_register_operand"
3131 "<scalar_mul_constraint>")
3132 (match_operand:SI 3 "immediate_operand" "i")
3133 (match_operand:SI 4 "immediate_operand" "i")]
3134 UNSPEC_VQDMULL_LANE))]
3137 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3138 return "vqdmull.<V_s_elem>\t%q0, %P1, %P2[%c3]";
3140 [(set (attr "neon_type")
3141 (if_then_else (match_test "<Scalar_mul_8_16>")
3142 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3143 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3146 (define_insn "neon_vqdmulh_lane<mode>"
3147 [(set (match_operand:VMQI 0 "s_register_operand" "=w")
3148 (unspec:VMQI [(match_operand:VMQI 1 "s_register_operand" "w")
3149 (match_operand:<V_HALF> 2 "s_register_operand"
3150 "<scalar_mul_constraint>")
3151 (match_operand:SI 3 "immediate_operand" "i")
3152 (match_operand:SI 4 "immediate_operand" "i")]
3153 UNSPEC_VQDMULH_LANE))]
3156 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3157 return "vq%O4dmulh.%T4%#<V_sz_elem>\t%q0, %q1, %P2[%c3]";
3159 [(set (attr "neon_type")
3160 (if_then_else (match_test "<Scalar_mul_8_16>")
3161 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")
3162 (const_string "neon_mul_qqd_32_scalar")))]
3165 (define_insn "neon_vqdmulh_lane<mode>"
3166 [(set (match_operand:VMDI 0 "s_register_operand" "=w")
3167 (unspec:VMDI [(match_operand:VMDI 1 "s_register_operand" "w")
3168 (match_operand:VMDI 2 "s_register_operand"
3169 "<scalar_mul_constraint>")
3170 (match_operand:SI 3 "immediate_operand" "i")
3171 (match_operand:SI 4 "immediate_operand" "i")]
3172 UNSPEC_VQDMULH_LANE))]
3175 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3176 return "vq%O4dmulh.%T4%#<V_sz_elem>\t%P0, %P1, %P2[%c3]";
3178 [(set (attr "neon_type")
3179 (if_then_else (match_test "<Scalar_mul_8_16>")
3180 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar")
3181 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))]
3184 (define_insn "neon_vmla_lane<mode>"
3185 [(set (match_operand:VMD 0 "s_register_operand" "=w")
3186 (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3187 (match_operand:VMD 2 "s_register_operand" "w")
3188 (match_operand:VMD 3 "s_register_operand"
3189 "<scalar_mul_constraint>")
3190 (match_operand:SI 4 "immediate_operand" "i")
3191 (match_operand:SI 5 "immediate_operand" "i")]
3195 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3196 return "vmla.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3198 [(set (attr "neon_type")
3199 (if_then_else (match_test "<Is_float_mode>")
3200 (const_string "neon_fp_vmla_ddd_scalar")
3201 (if_then_else (match_test "<Scalar_mul_8_16>")
3202 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3203 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3206 (define_insn "neon_vmla_lane<mode>"
3207 [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3208 (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3209 (match_operand:VMQ 2 "s_register_operand" "w")
3210 (match_operand:<V_HALF> 3 "s_register_operand"
3211 "<scalar_mul_constraint>")
3212 (match_operand:SI 4 "immediate_operand" "i")
3213 (match_operand:SI 5 "immediate_operand" "i")]
3217 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3218 return "vmla.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3220 [(set (attr "neon_type")
3221 (if_then_else (match_test "<Is_float_mode>")
3222 (const_string "neon_fp_vmla_qqq_scalar")
3223 (if_then_else (match_test "<Scalar_mul_8_16>")
3224 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3225 (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3228 (define_insn "neon_vmlal_lane<mode>"
3229 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3230 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3231 (match_operand:VMDI 2 "s_register_operand" "w")
3232 (match_operand:VMDI 3 "s_register_operand"
3233 "<scalar_mul_constraint>")
3234 (match_operand:SI 4 "immediate_operand" "i")
3235 (match_operand:SI 5 "immediate_operand" "i")]
3236 UNSPEC_VMLAL_LANE))]
3239 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3240 return "vmlal.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3242 [(set (attr "neon_type")
3243 (if_then_else (match_test "<Scalar_mul_8_16>")
3244 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3245 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3248 (define_insn "neon_vqdmlal_lane<mode>"
3249 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3250 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3251 (match_operand:VMDI 2 "s_register_operand" "w")
3252 (match_operand:VMDI 3 "s_register_operand"
3253 "<scalar_mul_constraint>")
3254 (match_operand:SI 4 "immediate_operand" "i")
3255 (match_operand:SI 5 "immediate_operand" "i")]
3256 UNSPEC_VQDMLAL_LANE))]
3259 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3260 return "vqdmlal.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3262 [(set (attr "neon_type")
3263 (if_then_else (match_test "<Scalar_mul_8_16>")
3264 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3265 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3268 (define_insn "neon_vmls_lane<mode>"
3269 [(set (match_operand:VMD 0 "s_register_operand" "=w")
3270 (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0")
3271 (match_operand:VMD 2 "s_register_operand" "w")
3272 (match_operand:VMD 3 "s_register_operand"
3273 "<scalar_mul_constraint>")
3274 (match_operand:SI 4 "immediate_operand" "i")
3275 (match_operand:SI 5 "immediate_operand" "i")]
3279 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3280 return "vmls.<V_if_elem>\t%P0, %P2, %P3[%c4]";
3282 [(set (attr "neon_type")
3283 (if_then_else (match_test "<Is_float_mode>")
3284 (const_string "neon_fp_vmla_ddd_scalar")
3285 (if_then_else (match_test "<Scalar_mul_8_16>")
3286 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3287 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))]
3290 (define_insn "neon_vmls_lane<mode>"
3291 [(set (match_operand:VMQ 0 "s_register_operand" "=w")
3292 (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0")
3293 (match_operand:VMQ 2 "s_register_operand" "w")
3294 (match_operand:<V_HALF> 3 "s_register_operand"
3295 "<scalar_mul_constraint>")
3296 (match_operand:SI 4 "immediate_operand" "i")
3297 (match_operand:SI 5 "immediate_operand" "i")]
3301 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3302 return "vmls.<V_if_elem>\t%q0, %q2, %P3[%c4]";
3304 [(set (attr "neon_type")
3305 (if_then_else (match_test "<Is_float_mode>")
3306 (const_string "neon_fp_vmla_qqq_scalar")
3307 (if_then_else (match_test "<Scalar_mul_8_16>")
3308 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")
3309 (const_string "neon_mla_qqq_32_qqd_32_scalar"))))]
3312 (define_insn "neon_vmlsl_lane<mode>"
3313 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3314 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3315 (match_operand:VMDI 2 "s_register_operand" "w")
3316 (match_operand:VMDI 3 "s_register_operand"
3317 "<scalar_mul_constraint>")
3318 (match_operand:SI 4 "immediate_operand" "i")
3319 (match_operand:SI 5 "immediate_operand" "i")]
3320 UNSPEC_VMLSL_LANE))]
3323 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3324 return "vmlsl.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]";
3326 [(set (attr "neon_type")
3327 (if_then_else (match_test "<Scalar_mul_8_16>")
3328 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3329 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3332 (define_insn "neon_vqdmlsl_lane<mode>"
3333 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3334 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0")
3335 (match_operand:VMDI 2 "s_register_operand" "w")
3336 (match_operand:VMDI 3 "s_register_operand"
3337 "<scalar_mul_constraint>")
3338 (match_operand:SI 4 "immediate_operand" "i")
3339 (match_operand:SI 5 "immediate_operand" "i")]
3340 UNSPEC_VQDMLSL_LANE))]
3343 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode));
3344 return "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3[%c4]";
3346 [(set (attr "neon_type")
3347 (if_then_else (match_test "<Scalar_mul_8_16>")
3348 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar")
3349 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))]
3352 ; FIXME: For the "_n" multiply/multiply-accumulate insns, we copy a value in a
3353 ; core register into a temp register, then use a scalar taken from that. This
3354 ; isn't an optimal solution if e.g. the scalar has just been read from memory
3355 ; or extracted from another vector. The latter case it's currently better to
3356 ; use the "_lane" variant, and the former case can probably be implemented
3357 ; using vld1_lane, but that hasn't been done yet.
3359 (define_expand "neon_vmul_n<mode>"
3360 [(match_operand:VMD 0 "s_register_operand" "")
3361 (match_operand:VMD 1 "s_register_operand" "")
3362 (match_operand:<V_elem> 2 "s_register_operand" "")
3363 (match_operand:SI 3 "immediate_operand" "")]
3366 rtx tmp = gen_reg_rtx (<MODE>mode);
3367 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3368 emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3369 const0_rtx, const0_rtx));
3373 (define_expand "neon_vmul_n<mode>"
3374 [(match_operand:VMQ 0 "s_register_operand" "")
3375 (match_operand:VMQ 1 "s_register_operand" "")
3376 (match_operand:<V_elem> 2 "s_register_operand" "")
3377 (match_operand:SI 3 "immediate_operand" "")]
3380 rtx tmp = gen_reg_rtx (<V_HALF>mode);
3381 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3382 emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp,
3383 const0_rtx, const0_rtx));
3387 (define_expand "neon_vmull_n<mode>"
3388 [(match_operand:<V_widen> 0 "s_register_operand" "")
3389 (match_operand:VMDI 1 "s_register_operand" "")
3390 (match_operand:<V_elem> 2 "s_register_operand" "")
3391 (match_operand:SI 3 "immediate_operand" "")]
3394 rtx tmp = gen_reg_rtx (<MODE>mode);
3395 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3396 emit_insn (gen_neon_vmull_lane<mode> (operands[0], operands[1], tmp,
3397 const0_rtx, operands[3]));
3401 (define_expand "neon_vqdmull_n<mode>"
3402 [(match_operand:<V_widen> 0 "s_register_operand" "")
3403 (match_operand:VMDI 1 "s_register_operand" "")
3404 (match_operand:<V_elem> 2 "s_register_operand" "")
3405 (match_operand:SI 3 "immediate_operand" "")]
3408 rtx tmp = gen_reg_rtx (<MODE>mode);
3409 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3410 emit_insn (gen_neon_vqdmull_lane<mode> (operands[0], operands[1], tmp,
3411 const0_rtx, const0_rtx));
3415 (define_expand "neon_vqdmulh_n<mode>"
3416 [(match_operand:VMDI 0 "s_register_operand" "")
3417 (match_operand:VMDI 1 "s_register_operand" "")
3418 (match_operand:<V_elem> 2 "s_register_operand" "")
3419 (match_operand:SI 3 "immediate_operand" "")]
3422 rtx tmp = gen_reg_rtx (<MODE>mode);
3423 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx));
3424 emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3425 const0_rtx, operands[3]));
3429 (define_expand "neon_vqdmulh_n<mode>"
3430 [(match_operand:VMQI 0 "s_register_operand" "")
3431 (match_operand:VMQI 1 "s_register_operand" "")
3432 (match_operand:<V_elem> 2 "s_register_operand" "")
3433 (match_operand:SI 3 "immediate_operand" "")]
3436 rtx tmp = gen_reg_rtx (<V_HALF>mode);
3437 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx));
3438 emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp,
3439 const0_rtx, operands[3]));
3443 (define_expand "neon_vmla_n<mode>"
3444 [(match_operand:VMD 0 "s_register_operand" "")
3445 (match_operand:VMD 1 "s_register_operand" "")
3446 (match_operand:VMD 2 "s_register_operand" "")
3447 (match_operand:<V_elem> 3 "s_register_operand" "")
3448 (match_operand:SI 4 "immediate_operand" "")]
3451 rtx tmp = gen_reg_rtx (<MODE>mode);
3452 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3453 emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3454 tmp, const0_rtx, operands[4]));
3458 (define_expand "neon_vmla_n<mode>"
3459 [(match_operand:VMQ 0 "s_register_operand" "")
3460 (match_operand:VMQ 1 "s_register_operand" "")
3461 (match_operand:VMQ 2 "s_register_operand" "")
3462 (match_operand:<V_elem> 3 "s_register_operand" "")
3463 (match_operand:SI 4 "immediate_operand" "")]
3466 rtx tmp = gen_reg_rtx (<V_HALF>mode);
3467 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3468 emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2],
3469 tmp, const0_rtx, operands[4]));
3473 (define_expand "neon_vmlal_n<mode>"
3474 [(match_operand:<V_widen> 0 "s_register_operand" "")
3475 (match_operand:<V_widen> 1 "s_register_operand" "")
3476 (match_operand:VMDI 2 "s_register_operand" "")
3477 (match_operand:<V_elem> 3 "s_register_operand" "")
3478 (match_operand:SI 4 "immediate_operand" "")]
3481 rtx tmp = gen_reg_rtx (<MODE>mode);
3482 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3483 emit_insn (gen_neon_vmlal_lane<mode> (operands[0], operands[1], operands[2],
3484 tmp, const0_rtx, operands[4]));
3488 (define_expand "neon_vqdmlal_n<mode>"
3489 [(match_operand:<V_widen> 0 "s_register_operand" "")
3490 (match_operand:<V_widen> 1 "s_register_operand" "")
3491 (match_operand:VMDI 2 "s_register_operand" "")
3492 (match_operand:<V_elem> 3 "s_register_operand" "")
3493 (match_operand:SI 4 "immediate_operand" "")]
3496 rtx tmp = gen_reg_rtx (<MODE>mode);
3497 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3498 emit_insn (gen_neon_vqdmlal_lane<mode> (operands[0], operands[1], operands[2],
3499 tmp, const0_rtx, operands[4]));
3503 (define_expand "neon_vmls_n<mode>"
3504 [(match_operand:VMD 0 "s_register_operand" "")
3505 (match_operand:VMD 1 "s_register_operand" "")
3506 (match_operand:VMD 2 "s_register_operand" "")
3507 (match_operand:<V_elem> 3 "s_register_operand" "")
3508 (match_operand:SI 4 "immediate_operand" "")]
3511 rtx tmp = gen_reg_rtx (<MODE>mode);
3512 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3513 emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3514 tmp, const0_rtx, operands[4]));
3518 (define_expand "neon_vmls_n<mode>"
3519 [(match_operand:VMQ 0 "s_register_operand" "")
3520 (match_operand:VMQ 1 "s_register_operand" "")
3521 (match_operand:VMQ 2 "s_register_operand" "")
3522 (match_operand:<V_elem> 3 "s_register_operand" "")
3523 (match_operand:SI 4 "immediate_operand" "")]
3526 rtx tmp = gen_reg_rtx (<V_HALF>mode);
3527 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx));
3528 emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2],
3529 tmp, const0_rtx, operands[4]));
3533 (define_expand "neon_vmlsl_n<mode>"
3534 [(match_operand:<V_widen> 0 "s_register_operand" "")
3535 (match_operand:<V_widen> 1 "s_register_operand" "")
3536 (match_operand:VMDI 2 "s_register_operand" "")
3537 (match_operand:<V_elem> 3 "s_register_operand" "")
3538 (match_operand:SI 4 "immediate_operand" "")]
3541 rtx tmp = gen_reg_rtx (<MODE>mode);
3542 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3543 emit_insn (gen_neon_vmlsl_lane<mode> (operands[0], operands[1], operands[2],
3544 tmp, const0_rtx, operands[4]));
3548 (define_expand "neon_vqdmlsl_n<mode>"
3549 [(match_operand:<V_widen> 0 "s_register_operand" "")
3550 (match_operand:<V_widen> 1 "s_register_operand" "")
3551 (match_operand:VMDI 2 "s_register_operand" "")
3552 (match_operand:<V_elem> 3 "s_register_operand" "")
3553 (match_operand:SI 4 "immediate_operand" "")]
3556 rtx tmp = gen_reg_rtx (<MODE>mode);
3557 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx));
3558 emit_insn (gen_neon_vqdmlsl_lane<mode> (operands[0], operands[1], operands[2],
3559 tmp, const0_rtx, operands[4]));
3563 (define_insn "neon_vext<mode>"
3564 [(set (match_operand:VDQX 0 "s_register_operand" "=w")
3565 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")
3566 (match_operand:VDQX 2 "s_register_operand" "w")
3567 (match_operand:SI 3 "immediate_operand" "i")]
3571 neon_const_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode));
3572 return "vext.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2, %3";
3574 [(set (attr "neon_type")
3575 (if_then_else (match_test "<Is_d_reg>")
3576 (const_string "neon_bp_simple")
3577 (const_string "neon_bp_2cycle")))]
3580 (define_insn "neon_vrev64<mode>"
3581 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
3582 (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
3583 (match_operand:SI 2 "immediate_operand" "i")]
3586 "vrev64.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3587 [(set_attr "neon_type" "neon_bp_simple")]
3590 (define_insn "neon_vrev32<mode>"
3591 [(set (match_operand:VX 0 "s_register_operand" "=w")
3592 (unspec:VX [(match_operand:VX 1 "s_register_operand" "w")
3593 (match_operand:SI 2 "immediate_operand" "i")]
3596 "vrev32.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3597 [(set_attr "neon_type" "neon_bp_simple")]
3600 (define_insn "neon_vrev16<mode>"
3601 [(set (match_operand:VE 0 "s_register_operand" "=w")
3602 (unspec:VE [(match_operand:VE 1 "s_register_operand" "w")
3603 (match_operand:SI 2 "immediate_operand" "i")]
3606 "vrev16.<V_sz_elem>\t%<V_reg>0, %<V_reg>1"
3607 [(set_attr "neon_type" "neon_bp_simple")]
3610 ; vbsl_* intrinsics may compile to any of vbsl/vbif/vbit depending on register
3611 ; allocation. For an intrinsic of form:
3612 ; rD = vbsl_* (rS, rN, rM)
3613 ; We can use any of:
3614 ; vbsl rS, rN, rM (if D = S)
3615 ; vbit rD, rN, rS (if D = M, so 1-bits in rS choose bits from rN, else rM)
3616 ; vbif rD, rM, rS (if D = N, so 0-bits in rS choose bits from rM, else rN)
3618 (define_insn "neon_vbsl<mode>_internal"
3619 [(set (match_operand:VDQX 0 "s_register_operand" "=w,w,w")
3620 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" " 0,w,w")
3621 (match_operand:VDQX 2 "s_register_operand" " w,w,0")
3622 (match_operand:VDQX 3 "s_register_operand" " w,0,w")]
3626 vbsl\t%<V_reg>0, %<V_reg>2, %<V_reg>3
3627 vbit\t%<V_reg>0, %<V_reg>2, %<V_reg>1
3628 vbif\t%<V_reg>0, %<V_reg>3, %<V_reg>1"
3629 [(set_attr "neon_type" "neon_int_1")]
3632 (define_expand "neon_vbsl<mode>"
3633 [(set (match_operand:VDQX 0 "s_register_operand" "")
3634 (unspec:VDQX [(match_operand:<V_cmp_result> 1 "s_register_operand" "")
3635 (match_operand:VDQX 2 "s_register_operand" "")
3636 (match_operand:VDQX 3 "s_register_operand" "")]
3640 /* We can't alias operands together if they have different modes. */
3641 operands[1] = gen_lowpart (<MODE>mode, operands[1]);
3644 (define_insn "neon_vshl<mode>"
3645 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3646 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3647 (match_operand:VDQIX 2 "s_register_operand" "w")
3648 (match_operand:SI 3 "immediate_operand" "i")]
3651 "v%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3652 [(set (attr "neon_type")
3653 (if_then_else (match_test "<Is_d_reg>")
3654 (const_string "neon_vshl_ddd")
3655 (const_string "neon_shift_3")))]
3658 (define_insn "neon_vqshl<mode>"
3659 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3660 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3661 (match_operand:VDQIX 2 "s_register_operand" "w")
3662 (match_operand:SI 3 "immediate_operand" "i")]
3665 "vq%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"
3666 [(set (attr "neon_type")
3667 (if_then_else (match_test "<Is_d_reg>")
3668 (const_string "neon_shift_2")
3669 (const_string "neon_vqshl_vrshl_vqrshl_qqq")))]
3672 (define_insn "neon_vshr_n<mode>"
3673 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3674 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3675 (match_operand:SI 2 "immediate_operand" "i")
3676 (match_operand:SI 3 "immediate_operand" "i")]
3680 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) + 1);
3681 return "v%O3shr.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3683 [(set_attr "neon_type" "neon_shift_1")]
3686 (define_insn "neon_vshrn_n<mode>"
3687 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3688 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3689 (match_operand:SI 2 "immediate_operand" "i")
3690 (match_operand:SI 3 "immediate_operand" "i")]
3694 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3695 return "v%O3shrn.<V_if_elem>\t%P0, %q1, %2";
3697 [(set_attr "neon_type" "neon_shift_1")]
3700 (define_insn "neon_vqshrn_n<mode>"
3701 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3702 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3703 (match_operand:SI 2 "immediate_operand" "i")
3704 (match_operand:SI 3 "immediate_operand" "i")]
3708 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3709 return "vq%O3shrn.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3711 [(set_attr "neon_type" "neon_shift_2")]
3714 (define_insn "neon_vqshrun_n<mode>"
3715 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w")
3716 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w")
3717 (match_operand:SI 2 "immediate_operand" "i")
3718 (match_operand:SI 3 "immediate_operand" "i")]
3722 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1);
3723 return "vq%O3shrun.%T3%#<V_sz_elem>\t%P0, %q1, %2";
3725 [(set_attr "neon_type" "neon_shift_2")]
3728 (define_insn "neon_vshl_n<mode>"
3729 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3730 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3731 (match_operand:SI 2 "immediate_operand" "i")
3732 (match_operand:SI 3 "immediate_operand" "i")]
3736 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3737 return "vshl.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %2";
3739 [(set_attr "neon_type" "neon_shift_1")]
3742 (define_insn "neon_vqshl_n<mode>"
3743 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3744 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3745 (match_operand:SI 2 "immediate_operand" "i")
3746 (match_operand:SI 3 "immediate_operand" "i")]
3750 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3751 return "vqshl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3753 [(set_attr "neon_type" "neon_shift_2")]
3756 (define_insn "neon_vqshlu_n<mode>"
3757 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3758 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w")
3759 (match_operand:SI 2 "immediate_operand" "i")
3760 (match_operand:SI 3 "immediate_operand" "i")]
3764 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode));
3765 return "vqshlu.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2";
3767 [(set_attr "neon_type" "neon_shift_2")]
3770 (define_insn "neon_vshll_n<mode>"
3771 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w")
3772 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w")
3773 (match_operand:SI 2 "immediate_operand" "i")
3774 (match_operand:SI 3 "immediate_operand" "i")]
3778 /* The boundaries are: 0 < imm <= size. */
3779 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode) + 1);
3780 return "vshll.%T3%#<V_sz_elem>\t%q0, %P1, %2";
3782 [(set_attr "neon_type" "neon_shift_1")]
3785 (define_insn "neon_vsra_n<mode>"
3786 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3787 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3788 (match_operand:VDQIX 2 "s_register_operand" "w")
3789 (match_operand:SI 3 "immediate_operand" "i")
3790 (match_operand:SI 4 "immediate_operand" "i")]
3794 neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
3795 return "v%O4sra.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3797 [(set_attr "neon_type" "neon_vsra_vrsra")]
3800 (define_insn "neon_vsri_n<mode>"
3801 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3802 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3803 (match_operand:VDQIX 2 "s_register_operand" "w")
3804 (match_operand:SI 3 "immediate_operand" "i")]
3808 neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1);
3809 return "vsri.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3811 [(set (attr "neon_type")
3812 (if_then_else (match_test "<Is_d_reg>")
3813 (const_string "neon_shift_1")
3814 (const_string "neon_shift_3")))]
3817 (define_insn "neon_vsli_n<mode>"
3818 [(set (match_operand:VDQIX 0 "s_register_operand" "=w")
3819 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0")
3820 (match_operand:VDQIX 2 "s_register_operand" "w")
3821 (match_operand:SI 3 "immediate_operand" "i")]
3825 neon_const_bounds (operands[3], 0, neon_element_bits (<MODE>mode));
3826 return "vsli.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3";
3828 [(set (attr "neon_type")
3829 (if_then_else (match_test "<Is_d_reg>")
3830 (const_string "neon_shift_1")
3831 (const_string "neon_shift_3")))]
3834 (define_insn "neon_vtbl1v8qi"
3835 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3836 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "w")
3837 (match_operand:V8QI 2 "s_register_operand" "w")]
3840 "vtbl.8\t%P0, {%P1}, %P2"
3841 [(set_attr "neon_type" "neon_bp_2cycle")]
3844 (define_insn "neon_vtbl2v8qi"
3845 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3846 (unspec:V8QI [(match_operand:TI 1 "s_register_operand" "w")
3847 (match_operand:V8QI 2 "s_register_operand" "w")]
3852 int tabbase = REGNO (operands[1]);
3854 ops[0] = operands[0];
3855 ops[1] = gen_rtx_REG (V8QImode, tabbase);
3856 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3857 ops[3] = operands[2];
3858 output_asm_insn ("vtbl.8\t%P0, {%P1, %P2}, %P3", ops);
3862 [(set_attr "neon_type" "neon_bp_2cycle")]
3865 (define_insn "neon_vtbl3v8qi"
3866 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3867 (unspec:V8QI [(match_operand:EI 1 "s_register_operand" "w")
3868 (match_operand:V8QI 2 "s_register_operand" "w")]
3873 int tabbase = REGNO (operands[1]);
3875 ops[0] = operands[0];
3876 ops[1] = gen_rtx_REG (V8QImode, tabbase);
3877 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3878 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
3879 ops[4] = operands[2];
3880 output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
3884 [(set_attr "neon_type" "neon_bp_3cycle")]
3887 (define_insn "neon_vtbl4v8qi"
3888 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3889 (unspec:V8QI [(match_operand:OI 1 "s_register_operand" "w")
3890 (match_operand:V8QI 2 "s_register_operand" "w")]
3895 int tabbase = REGNO (operands[1]);
3897 ops[0] = operands[0];
3898 ops[1] = gen_rtx_REG (V8QImode, tabbase);
3899 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
3900 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
3901 ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
3902 ops[5] = operands[2];
3903 output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
3907 [(set_attr "neon_type" "neon_bp_3cycle")]
3910 ;; These three are used by the vec_perm infrastructure for V16QImode.
3911 (define_insn_and_split "neon_vtbl1v16qi"
3912 [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
3913 (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w")
3914 (match_operand:V16QI 2 "s_register_operand" "w")]
3918 "&& reload_completed"
3921 rtx op0, op1, op2, part0, part2;
3925 op1 = gen_lowpart (TImode, operands[1]);
3928 ofs = subreg_lowpart_offset (V8QImode, V16QImode);
3929 part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
3930 part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
3931 emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
3933 ofs = subreg_highpart_offset (V8QImode, V16QImode);
3934 part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
3935 part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
3936 emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
3940 (define_insn_and_split "neon_vtbl2v16qi"
3941 [(set (match_operand:V16QI 0 "s_register_operand" "=&w")
3942 (unspec:V16QI [(match_operand:OI 1 "s_register_operand" "w")
3943 (match_operand:V16QI 2 "s_register_operand" "w")]
3947 "&& reload_completed"
3950 rtx op0, op1, op2, part0, part2;
3957 ofs = subreg_lowpart_offset (V8QImode, V16QImode);
3958 part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
3959 part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
3960 emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
3962 ofs = subreg_highpart_offset (V8QImode, V16QImode);
3963 part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs);
3964 part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs);
3965 emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2));
3969 ;; ??? Logically we should extend the regular neon_vcombine pattern to
3970 ;; handle quad-word input modes, producing octa-word output modes. But
3971 ;; that requires us to add support for octa-word vector modes in moves.
3972 ;; That seems overkill for this one use in vec_perm.
3973 (define_insn_and_split "neon_vcombinev16qi"
3974 [(set (match_operand:OI 0 "s_register_operand" "=w")
3975 (unspec:OI [(match_operand:V16QI 1 "s_register_operand" "w")
3976 (match_operand:V16QI 2 "s_register_operand" "w")]
3980 "&& reload_completed"
3983 neon_split_vcombine (operands);
3987 (define_insn "neon_vtbx1v8qi"
3988 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
3989 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
3990 (match_operand:V8QI 2 "s_register_operand" "w")
3991 (match_operand:V8QI 3 "s_register_operand" "w")]
3994 "vtbx.8\t%P0, {%P2}, %P3"
3995 [(set_attr "neon_type" "neon_bp_2cycle")]
3998 (define_insn "neon_vtbx2v8qi"
3999 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4000 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4001 (match_operand:TI 2 "s_register_operand" "w")
4002 (match_operand:V8QI 3 "s_register_operand" "w")]
4007 int tabbase = REGNO (operands[2]);
4009 ops[0] = operands[0];
4010 ops[1] = gen_rtx_REG (V8QImode, tabbase);
4011 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4012 ops[3] = operands[3];
4013 output_asm_insn ("vtbx.8\t%P0, {%P1, %P2}, %P3", ops);
4017 [(set_attr "neon_type" "neon_bp_2cycle")]
4020 (define_insn "neon_vtbx3v8qi"
4021 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4022 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4023 (match_operand:EI 2 "s_register_operand" "w")
4024 (match_operand:V8QI 3 "s_register_operand" "w")]
4029 int tabbase = REGNO (operands[2]);
4031 ops[0] = operands[0];
4032 ops[1] = gen_rtx_REG (V8QImode, tabbase);
4033 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4034 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4035 ops[4] = operands[3];
4036 output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3}, %P4", ops);
4040 [(set_attr "neon_type" "neon_bp_3cycle")]
4043 (define_insn "neon_vtbx4v8qi"
4044 [(set (match_operand:V8QI 0 "s_register_operand" "=w")
4045 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0")
4046 (match_operand:OI 2 "s_register_operand" "w")
4047 (match_operand:V8QI 3 "s_register_operand" "w")]
4052 int tabbase = REGNO (operands[2]);
4054 ops[0] = operands[0];
4055 ops[1] = gen_rtx_REG (V8QImode, tabbase);
4056 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2);
4057 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4);
4058 ops[4] = gen_rtx_REG (V8QImode, tabbase + 6);
4059 ops[5] = operands[3];
4060 output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops);
4064 [(set_attr "neon_type" "neon_bp_3cycle")]
4067 (define_insn "neon_vtrn<mode>_internal"
4068 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4069 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4070 (match_operand:VDQW 2 "s_register_operand" "w")]
4072 (set (match_operand:VDQW 3 "s_register_operand" "=2")
4073 (unspec:VDQW [(match_dup 1) (match_dup 2)]
4076 "vtrn.<V_sz_elem>\t%<V_reg>0, %<V_reg>3"
4077 [(set (attr "neon_type")
4078 (if_then_else (match_test "<Is_d_reg>")
4079 (const_string "neon_bp_simple")
4080 (const_string "neon_bp_3cycle")))]
4083 (define_expand "neon_vtrn<mode>"
4084 [(match_operand:SI 0 "s_register_operand" "r")
4085 (match_operand:VDQW 1 "s_register_operand" "w")
4086 (match_operand:VDQW 2 "s_register_operand" "w")]
4089 neon_emit_pair_result_insn (<MODE>mode, gen_neon_vtrn<mode>_internal,
4090 operands[0], operands[1], operands[2]);
4094 (define_insn "neon_vzip<mode>_internal"
4095 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4096 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4097 (match_operand:VDQW 2 "s_register_operand" "w")]
4099 (set (match_operand:VDQW 3 "s_register_operand" "=2")
4100 (unspec:VDQW [(match_dup 1) (match_dup 2)]
4103 "vzip.<V_sz_elem>\t%<V_reg>0, %<V_reg>3"
4104 [(set (attr "neon_type")
4105 (if_then_else (match_test "<Is_d_reg>")
4106 (const_string "neon_bp_simple")
4107 (const_string "neon_bp_3cycle")))]
4110 (define_expand "neon_vzip<mode>"
4111 [(match_operand:SI 0 "s_register_operand" "r")
4112 (match_operand:VDQW 1 "s_register_operand" "w")
4113 (match_operand:VDQW 2 "s_register_operand" "w")]
4116 neon_emit_pair_result_insn (<MODE>mode, gen_neon_vzip<mode>_internal,
4117 operands[0], operands[1], operands[2]);
4121 (define_insn "neon_vuzp<mode>_internal"
4122 [(set (match_operand:VDQW 0 "s_register_operand" "=w")
4123 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0")
4124 (match_operand:VDQW 2 "s_register_operand" "w")]
4126 (set (match_operand:VDQW 3 "s_register_operand" "=2")
4127 (unspec:VDQW [(match_dup 1) (match_dup 2)]
4130 "vuzp.<V_sz_elem>\t%<V_reg>0, %<V_reg>3"
4131 [(set (attr "neon_type")
4132 (if_then_else (match_test "<Is_d_reg>")
4133 (const_string "neon_bp_simple")
4134 (const_string "neon_bp_3cycle")))]
4137 (define_expand "neon_vuzp<mode>"
4138 [(match_operand:SI 0 "s_register_operand" "r")
4139 (match_operand:VDQW 1 "s_register_operand" "w")
4140 (match_operand:VDQW 2 "s_register_operand" "w")]
4143 neon_emit_pair_result_insn (<MODE>mode, gen_neon_vuzp<mode>_internal,
4144 operands[0], operands[1], operands[2]);
4148 (define_expand "neon_vreinterpretv8qi<mode>"
4149 [(match_operand:V8QI 0 "s_register_operand" "")
4150 (match_operand:VDX 1 "s_register_operand" "")]
4153 neon_reinterpret (operands[0], operands[1]);
4157 (define_expand "neon_vreinterpretv4hi<mode>"
4158 [(match_operand:V4HI 0 "s_register_operand" "")
4159 (match_operand:VDX 1 "s_register_operand" "")]
4162 neon_reinterpret (operands[0], operands[1]);
4166 (define_expand "neon_vreinterpretv2si<mode>"
4167 [(match_operand:V2SI 0 "s_register_operand" "")
4168 (match_operand:VDX 1 "s_register_operand" "")]
4171 neon_reinterpret (operands[0], operands[1]);
4175 (define_expand "neon_vreinterpretv2sf<mode>"
4176 [(match_operand:V2SF 0 "s_register_operand" "")
4177 (match_operand:VDX 1 "s_register_operand" "")]
4180 neon_reinterpret (operands[0], operands[1]);
4184 (define_expand "neon_vreinterpretdi<mode>"
4185 [(match_operand:DI 0 "s_register_operand" "")
4186 (match_operand:VDX 1 "s_register_operand" "")]
4189 neon_reinterpret (operands[0], operands[1]);
4193 (define_expand "neon_vreinterpretv16qi<mode>"
4194 [(match_operand:V16QI 0 "s_register_operand" "")
4195 (match_operand:VQX 1 "s_register_operand" "")]
4198 neon_reinterpret (operands[0], operands[1]);
4202 (define_expand "neon_vreinterpretv8hi<mode>"
4203 [(match_operand:V8HI 0 "s_register_operand" "")
4204 (match_operand:VQX 1 "s_register_operand" "")]
4207 neon_reinterpret (operands[0], operands[1]);
4211 (define_expand "neon_vreinterpretv4si<mode>"
4212 [(match_operand:V4SI 0 "s_register_operand" "")
4213 (match_operand:VQX 1 "s_register_operand" "")]
4216 neon_reinterpret (operands[0], operands[1]);
4220 (define_expand "neon_vreinterpretv4sf<mode>"
4221 [(match_operand:V4SF 0 "s_register_operand" "")
4222 (match_operand:VQX 1 "s_register_operand" "")]
4225 neon_reinterpret (operands[0], operands[1]);
4229 (define_expand "neon_vreinterpretv2di<mode>"
4230 [(match_operand:V2DI 0 "s_register_operand" "")
4231 (match_operand:VQX 1 "s_register_operand" "")]
4234 neon_reinterpret (operands[0], operands[1]);
4238 (define_expand "vec_load_lanes<mode><mode>"
4239 [(set (match_operand:VDQX 0 "s_register_operand")
4240 (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand")]
4244 (define_insn "neon_vld1<mode>"
4245 [(set (match_operand:VDQX 0 "s_register_operand" "=w")
4246 (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand" "Um")]
4249 "vld1.<V_sz_elem>\t%h0, %A1"
4250 [(set_attr "neon_type" "neon_vld1_1_2_regs")]
4253 (define_insn "neon_vld1_lane<mode>"
4254 [(set (match_operand:VDX 0 "s_register_operand" "=w")
4255 (unspec:VDX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
4256 (match_operand:VDX 2 "s_register_operand" "0")
4257 (match_operand:SI 3 "immediate_operand" "i")]
4261 HOST_WIDE_INT lane = INTVAL (operands[3]);
4262 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4263 if (lane < 0 || lane >= max)
4264 error ("lane out of range");
4266 return "vld1.<V_sz_elem>\t%P0, %A1";
4268 return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
4270 [(set (attr "neon_type")
4271 (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4272 (const_string "neon_vld1_1_2_regs")
4273 (const_string "neon_vld1_vld2_lane")))]
4276 (define_insn "neon_vld1_lane<mode>"
4277 [(set (match_operand:VQX 0 "s_register_operand" "=w")
4278 (unspec:VQX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")
4279 (match_operand:VQX 2 "s_register_operand" "0")
4280 (match_operand:SI 3 "immediate_operand" "i")]
4284 HOST_WIDE_INT lane = INTVAL (operands[3]);
4285 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4286 int regno = REGNO (operands[0]);
4287 if (lane < 0 || lane >= max)
4288 error ("lane out of range");
4289 else if (lane >= max / 2)
4293 operands[3] = GEN_INT (lane);
4295 operands[0] = gen_rtx_REG (<V_HALF>mode, regno);
4297 return "vld1.<V_sz_elem>\t%P0, %A1";
4299 return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1";
4301 [(set (attr "neon_type")
4302 (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2))
4303 (const_string "neon_vld1_1_2_regs")
4304 (const_string "neon_vld1_vld2_lane")))]
4307 (define_insn "neon_vld1_dup<mode>"
4308 [(set (match_operand:VDX 0 "s_register_operand" "=w")
4309 (unspec:VDX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")]
4313 if (GET_MODE_NUNITS (<MODE>mode) > 1)
4314 return "vld1.<V_sz_elem>\t{%P0[]}, %A1";
4316 return "vld1.<V_sz_elem>\t%h0, %A1";
4318 [(set (attr "neon_type")
4319 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4320 (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4321 (const_string "neon_vld1_1_2_regs")))]
4324 (define_insn "neon_vld1_dup<mode>"
4325 [(set (match_operand:VQX 0 "s_register_operand" "=w")
4326 (unspec:VQX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um")]
4330 if (GET_MODE_NUNITS (<MODE>mode) > 2)
4331 return "vld1.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
4333 return "vld1.<V_sz_elem>\t%h0, %A1";
4335 [(set (attr "neon_type")
4336 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4337 (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4338 (const_string "neon_vld1_1_2_regs")))]
4341 (define_expand "vec_store_lanes<mode><mode>"
4342 [(set (match_operand:VDQX 0 "neon_struct_operand")
4343 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand")]
4347 (define_insn "neon_vst1<mode>"
4348 [(set (match_operand:VDQX 0 "neon_struct_operand" "=Um")
4349 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")]
4352 "vst1.<V_sz_elem>\t%h1, %A0"
4353 [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")])
4355 (define_insn "neon_vst1_lane<mode>"
4356 [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
4357 (vec_select:<V_elem>
4358 (match_operand:VDX 1 "s_register_operand" "w")
4359 (parallel [(match_operand:SI 2 "neon_lane_number" "i")])))]
4362 HOST_WIDE_INT lane = INTVAL (operands[2]);
4363 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4364 if (lane < 0 || lane >= max)
4365 error ("lane out of range");
4367 return "vst1.<V_sz_elem>\t{%P1}, %A0";
4369 return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
4371 [(set (attr "neon_type")
4372 (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 1))
4373 (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4374 (const_string "neon_vst1_vst2_lane")))])
4376 (define_insn "neon_vst1_lane<mode>"
4377 [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um")
4378 (vec_select:<V_elem>
4379 (match_operand:VQX 1 "s_register_operand" "w")
4380 (parallel [(match_operand:SI 2 "neon_lane_number" "i")])))]
4383 HOST_WIDE_INT lane = INTVAL (operands[2]);
4384 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4385 int regno = REGNO (operands[1]);
4386 if (lane < 0 || lane >= max)
4387 error ("lane out of range");
4388 else if (lane >= max / 2)
4392 operands[2] = GEN_INT (lane);
4394 operands[1] = gen_rtx_REG (<V_HALF>mode, regno);
4396 return "vst1.<V_sz_elem>\t{%P1}, %A0";
4398 return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0";
4400 [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4403 (define_expand "vec_load_lanesti<mode>"
4404 [(set (match_operand:TI 0 "s_register_operand")
4405 (unspec:TI [(match_operand:TI 1 "neon_struct_operand")
4406 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4410 (define_insn "neon_vld2<mode>"
4411 [(set (match_operand:TI 0 "s_register_operand" "=w")
4412 (unspec:TI [(match_operand:TI 1 "neon_struct_operand" "Um")
4413 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4417 if (<V_sz_elem> == 64)
4418 return "vld1.64\t%h0, %A1";
4420 return "vld2.<V_sz_elem>\t%h0, %A1";
4422 [(set (attr "neon_type")
4423 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4424 (const_string "neon_vld1_1_2_regs")
4425 (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")))]
4428 (define_expand "vec_load_lanesoi<mode>"
4429 [(set (match_operand:OI 0 "s_register_operand")
4430 (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
4431 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4435 (define_insn "neon_vld2<mode>"
4436 [(set (match_operand:OI 0 "s_register_operand" "=w")
4437 (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
4438 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4441 "vld2.<V_sz_elem>\t%h0, %A1"
4442 [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")])
4444 (define_insn "neon_vld2_lane<mode>"
4445 [(set (match_operand:TI 0 "s_register_operand" "=w")
4446 (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4447 (match_operand:TI 2 "s_register_operand" "0")
4448 (match_operand:SI 3 "immediate_operand" "i")
4449 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4453 HOST_WIDE_INT lane = INTVAL (operands[3]);
4454 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4455 int regno = REGNO (operands[0]);
4457 if (lane < 0 || lane >= max)
4458 error ("lane out of range");
4459 ops[0] = gen_rtx_REG (DImode, regno);
4460 ops[1] = gen_rtx_REG (DImode, regno + 2);
4461 ops[2] = operands[1];
4462 ops[3] = operands[3];
4463 output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
4466 [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4469 (define_insn "neon_vld2_lane<mode>"
4470 [(set (match_operand:OI 0 "s_register_operand" "=w")
4471 (unspec:OI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4472 (match_operand:OI 2 "s_register_operand" "0")
4473 (match_operand:SI 3 "immediate_operand" "i")
4474 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4478 HOST_WIDE_INT lane = INTVAL (operands[3]);
4479 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4480 int regno = REGNO (operands[0]);
4482 if (lane < 0 || lane >= max)
4483 error ("lane out of range");
4484 else if (lane >= max / 2)
4489 ops[0] = gen_rtx_REG (DImode, regno);
4490 ops[1] = gen_rtx_REG (DImode, regno + 4);
4491 ops[2] = operands[1];
4492 ops[3] = GEN_INT (lane);
4493 output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops);
4496 [(set_attr "neon_type" "neon_vld1_vld2_lane")]
4499 (define_insn "neon_vld2_dup<mode>"
4500 [(set (match_operand:TI 0 "s_register_operand" "=w")
4501 (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um")
4502 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4506 if (GET_MODE_NUNITS (<MODE>mode) > 1)
4507 return "vld2.<V_sz_elem>\t{%e0[], %f0[]}, %A1";
4509 return "vld1.<V_sz_elem>\t%h0, %A1";
4511 [(set (attr "neon_type")
4512 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4513 (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")
4514 (const_string "neon_vld1_1_2_regs")))]
4517 (define_expand "vec_store_lanesti<mode>"
4518 [(set (match_operand:TI 0 "neon_struct_operand")
4519 (unspec:TI [(match_operand:TI 1 "s_register_operand")
4520 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4524 (define_insn "neon_vst2<mode>"
4525 [(set (match_operand:TI 0 "neon_struct_operand" "=Um")
4526 (unspec:TI [(match_operand:TI 1 "s_register_operand" "w")
4527 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4531 if (<V_sz_elem> == 64)
4532 return "vst1.64\t%h1, %A0";
4534 return "vst2.<V_sz_elem>\t%h1, %A0";
4536 [(set (attr "neon_type")
4537 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4538 (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4539 (const_string "neon_vst1_1_2_regs_vst2_2_regs")))]
4542 (define_expand "vec_store_lanesoi<mode>"
4543 [(set (match_operand:OI 0 "neon_struct_operand")
4544 (unspec:OI [(match_operand:OI 1 "s_register_operand")
4545 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4549 (define_insn "neon_vst2<mode>"
4550 [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
4551 (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
4552 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4555 "vst2.<V_sz_elem>\t%h1, %A0"
4556 [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")]
4559 (define_insn "neon_vst2_lane<mode>"
4560 [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
4561 (unspec:<V_two_elem>
4562 [(match_operand:TI 1 "s_register_operand" "w")
4563 (match_operand:SI 2 "immediate_operand" "i")
4564 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4568 HOST_WIDE_INT lane = INTVAL (operands[2]);
4569 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4570 int regno = REGNO (operands[1]);
4572 if (lane < 0 || lane >= max)
4573 error ("lane out of range");
4574 ops[0] = operands[0];
4575 ops[1] = gen_rtx_REG (DImode, regno);
4576 ops[2] = gen_rtx_REG (DImode, regno + 2);
4577 ops[3] = operands[2];
4578 output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
4581 [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4584 (define_insn "neon_vst2_lane<mode>"
4585 [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um")
4586 (unspec:<V_two_elem>
4587 [(match_operand:OI 1 "s_register_operand" "w")
4588 (match_operand:SI 2 "immediate_operand" "i")
4589 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4593 HOST_WIDE_INT lane = INTVAL (operands[2]);
4594 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4595 int regno = REGNO (operands[1]);
4597 if (lane < 0 || lane >= max)
4598 error ("lane out of range");
4599 else if (lane >= max / 2)
4604 ops[0] = operands[0];
4605 ops[1] = gen_rtx_REG (DImode, regno);
4606 ops[2] = gen_rtx_REG (DImode, regno + 4);
4607 ops[3] = GEN_INT (lane);
4608 output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops);
4611 [(set_attr "neon_type" "neon_vst1_vst2_lane")]
4614 (define_expand "vec_load_lanesei<mode>"
4615 [(set (match_operand:EI 0 "s_register_operand")
4616 (unspec:EI [(match_operand:EI 1 "neon_struct_operand")
4617 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4621 (define_insn "neon_vld3<mode>"
4622 [(set (match_operand:EI 0 "s_register_operand" "=w")
4623 (unspec:EI [(match_operand:EI 1 "neon_struct_operand" "Um")
4624 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4628 if (<V_sz_elem> == 64)
4629 return "vld1.64\t%h0, %A1";
4631 return "vld3.<V_sz_elem>\t%h0, %A1";
4633 [(set (attr "neon_type")
4634 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4635 (const_string "neon_vld1_1_2_regs")
4636 (const_string "neon_vld3_vld4")))]
4639 (define_expand "vec_load_lanesci<mode>"
4640 [(match_operand:CI 0 "s_register_operand")
4641 (match_operand:CI 1 "neon_struct_operand")
4642 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4645 emit_insn (gen_neon_vld3<mode> (operands[0], operands[1]));
4649 (define_expand "neon_vld3<mode>"
4650 [(match_operand:CI 0 "s_register_operand")
4651 (match_operand:CI 1 "neon_struct_operand")
4652 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4657 mem = adjust_address (operands[1], EImode, 0);
4658 emit_insn (gen_neon_vld3qa<mode> (operands[0], mem));
4659 mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
4660 emit_insn (gen_neon_vld3qb<mode> (operands[0], mem, operands[0]));
4664 (define_insn "neon_vld3qa<mode>"
4665 [(set (match_operand:CI 0 "s_register_operand" "=w")
4666 (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
4667 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4671 int regno = REGNO (operands[0]);
4673 ops[0] = gen_rtx_REG (DImode, regno);
4674 ops[1] = gen_rtx_REG (DImode, regno + 4);
4675 ops[2] = gen_rtx_REG (DImode, regno + 8);
4676 ops[3] = operands[1];
4677 output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
4680 [(set_attr "neon_type" "neon_vld3_vld4")]
4683 (define_insn "neon_vld3qb<mode>"
4684 [(set (match_operand:CI 0 "s_register_operand" "=w")
4685 (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um")
4686 (match_operand:CI 2 "s_register_operand" "0")
4687 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4691 int regno = REGNO (operands[0]);
4693 ops[0] = gen_rtx_REG (DImode, regno + 2);
4694 ops[1] = gen_rtx_REG (DImode, regno + 6);
4695 ops[2] = gen_rtx_REG (DImode, regno + 10);
4696 ops[3] = operands[1];
4697 output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops);
4700 [(set_attr "neon_type" "neon_vld3_vld4")]
4703 (define_insn "neon_vld3_lane<mode>"
4704 [(set (match_operand:EI 0 "s_register_operand" "=w")
4705 (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
4706 (match_operand:EI 2 "s_register_operand" "0")
4707 (match_operand:SI 3 "immediate_operand" "i")
4708 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4712 HOST_WIDE_INT lane = INTVAL (operands[3]);
4713 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4714 int regno = REGNO (operands[0]);
4716 if (lane < 0 || lane >= max)
4717 error ("lane out of range");
4718 ops[0] = gen_rtx_REG (DImode, regno);
4719 ops[1] = gen_rtx_REG (DImode, regno + 2);
4720 ops[2] = gen_rtx_REG (DImode, regno + 4);
4721 ops[3] = operands[1];
4722 ops[4] = operands[3];
4723 output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %A3",
4727 [(set_attr "neon_type" "neon_vld3_vld4_lane")]
4730 (define_insn "neon_vld3_lane<mode>"
4731 [(set (match_operand:CI 0 "s_register_operand" "=w")
4732 (unspec:CI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
4733 (match_operand:CI 2 "s_register_operand" "0")
4734 (match_operand:SI 3 "immediate_operand" "i")
4735 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4739 HOST_WIDE_INT lane = INTVAL (operands[3]);
4740 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4741 int regno = REGNO (operands[0]);
4743 if (lane < 0 || lane >= max)
4744 error ("lane out of range");
4745 else if (lane >= max / 2)
4750 ops[0] = gen_rtx_REG (DImode, regno);
4751 ops[1] = gen_rtx_REG (DImode, regno + 4);
4752 ops[2] = gen_rtx_REG (DImode, regno + 8);
4753 ops[3] = operands[1];
4754 ops[4] = GEN_INT (lane);
4755 output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %A3",
4759 [(set_attr "neon_type" "neon_vld3_vld4_lane")]
4762 (define_insn "neon_vld3_dup<mode>"
4763 [(set (match_operand:EI 0 "s_register_operand" "=w")
4764 (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um")
4765 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4769 if (GET_MODE_NUNITS (<MODE>mode) > 1)
4771 int regno = REGNO (operands[0]);
4773 ops[0] = gen_rtx_REG (DImode, regno);
4774 ops[1] = gen_rtx_REG (DImode, regno + 2);
4775 ops[2] = gen_rtx_REG (DImode, regno + 4);
4776 ops[3] = operands[1];
4777 output_asm_insn ("vld3.<V_sz_elem>\t{%P0[], %P1[], %P2[]}, %A3", ops);
4781 return "vld1.<V_sz_elem>\t%h0, %A1";
4783 [(set (attr "neon_type")
4784 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
4785 (const_string "neon_vld3_vld4_all_lanes")
4786 (const_string "neon_vld1_1_2_regs")))])
4788 (define_expand "vec_store_lanesei<mode>"
4789 [(set (match_operand:EI 0 "neon_struct_operand")
4790 (unspec:EI [(match_operand:EI 1 "s_register_operand")
4791 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4795 (define_insn "neon_vst3<mode>"
4796 [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
4797 (unspec:EI [(match_operand:EI 1 "s_register_operand" "w")
4798 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4802 if (<V_sz_elem> == 64)
4803 return "vst1.64\t%h1, %A0";
4805 return "vst3.<V_sz_elem>\t%h1, %A0";
4807 [(set (attr "neon_type")
4808 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4809 (const_string "neon_vst1_1_2_regs_vst2_2_regs")
4810 (const_string "neon_vst2_4_regs_vst3_vst4")))])
4812 (define_expand "vec_store_lanesci<mode>"
4813 [(match_operand:CI 0 "neon_struct_operand")
4814 (match_operand:CI 1 "s_register_operand")
4815 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4818 emit_insn (gen_neon_vst3<mode> (operands[0], operands[1]));
4822 (define_expand "neon_vst3<mode>"
4823 [(match_operand:CI 0 "neon_struct_operand")
4824 (match_operand:CI 1 "s_register_operand")
4825 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4830 mem = adjust_address (operands[0], EImode, 0);
4831 emit_insn (gen_neon_vst3qa<mode> (mem, operands[1]));
4832 mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode));
4833 emit_insn (gen_neon_vst3qb<mode> (mem, operands[1]));
4837 (define_insn "neon_vst3qa<mode>"
4838 [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
4839 (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
4840 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4844 int regno = REGNO (operands[1]);
4846 ops[0] = operands[0];
4847 ops[1] = gen_rtx_REG (DImode, regno);
4848 ops[2] = gen_rtx_REG (DImode, regno + 4);
4849 ops[3] = gen_rtx_REG (DImode, regno + 8);
4850 output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
4853 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
4856 (define_insn "neon_vst3qb<mode>"
4857 [(set (match_operand:EI 0 "neon_struct_operand" "=Um")
4858 (unspec:EI [(match_operand:CI 1 "s_register_operand" "w")
4859 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4863 int regno = REGNO (operands[1]);
4865 ops[0] = operands[0];
4866 ops[1] = gen_rtx_REG (DImode, regno + 2);
4867 ops[2] = gen_rtx_REG (DImode, regno + 6);
4868 ops[3] = gen_rtx_REG (DImode, regno + 10);
4869 output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops);
4872 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
4875 (define_insn "neon_vst3_lane<mode>"
4876 [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
4877 (unspec:<V_three_elem>
4878 [(match_operand:EI 1 "s_register_operand" "w")
4879 (match_operand:SI 2 "immediate_operand" "i")
4880 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4884 HOST_WIDE_INT lane = INTVAL (operands[2]);
4885 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4886 int regno = REGNO (operands[1]);
4888 if (lane < 0 || lane >= max)
4889 error ("lane out of range");
4890 ops[0] = operands[0];
4891 ops[1] = gen_rtx_REG (DImode, regno);
4892 ops[2] = gen_rtx_REG (DImode, regno + 2);
4893 ops[3] = gen_rtx_REG (DImode, regno + 4);
4894 ops[4] = operands[2];
4895 output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %A0",
4899 [(set_attr "neon_type" "neon_vst3_vst4_lane")]
4902 (define_insn "neon_vst3_lane<mode>"
4903 [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um")
4904 (unspec:<V_three_elem>
4905 [(match_operand:CI 1 "s_register_operand" "w")
4906 (match_operand:SI 2 "immediate_operand" "i")
4907 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4911 HOST_WIDE_INT lane = INTVAL (operands[2]);
4912 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
4913 int regno = REGNO (operands[1]);
4915 if (lane < 0 || lane >= max)
4916 error ("lane out of range");
4917 else if (lane >= max / 2)
4922 ops[0] = operands[0];
4923 ops[1] = gen_rtx_REG (DImode, regno);
4924 ops[2] = gen_rtx_REG (DImode, regno + 4);
4925 ops[3] = gen_rtx_REG (DImode, regno + 8);
4926 ops[4] = GEN_INT (lane);
4927 output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %A0",
4931 [(set_attr "neon_type" "neon_vst3_vst4_lane")])
4933 (define_expand "vec_load_lanesoi<mode>"
4934 [(set (match_operand:OI 0 "s_register_operand")
4935 (unspec:OI [(match_operand:OI 1 "neon_struct_operand")
4936 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4940 (define_insn "neon_vld4<mode>"
4941 [(set (match_operand:OI 0 "s_register_operand" "=w")
4942 (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um")
4943 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4947 if (<V_sz_elem> == 64)
4948 return "vld1.64\t%h0, %A1";
4950 return "vld4.<V_sz_elem>\t%h0, %A1";
4952 [(set (attr "neon_type")
4953 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
4954 (const_string "neon_vld1_1_2_regs")
4955 (const_string "neon_vld3_vld4")))]
4958 (define_expand "vec_load_lanesxi<mode>"
4959 [(match_operand:XI 0 "s_register_operand")
4960 (match_operand:XI 1 "neon_struct_operand")
4961 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4964 emit_insn (gen_neon_vld4<mode> (operands[0], operands[1]));
4968 (define_expand "neon_vld4<mode>"
4969 [(match_operand:XI 0 "s_register_operand")
4970 (match_operand:XI 1 "neon_struct_operand")
4971 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4976 mem = adjust_address (operands[1], OImode, 0);
4977 emit_insn (gen_neon_vld4qa<mode> (operands[0], mem));
4978 mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
4979 emit_insn (gen_neon_vld4qb<mode> (operands[0], mem, operands[0]));
4983 (define_insn "neon_vld4qa<mode>"
4984 [(set (match_operand:XI 0 "s_register_operand" "=w")
4985 (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
4986 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
4990 int regno = REGNO (operands[0]);
4992 ops[0] = gen_rtx_REG (DImode, regno);
4993 ops[1] = gen_rtx_REG (DImode, regno + 4);
4994 ops[2] = gen_rtx_REG (DImode, regno + 8);
4995 ops[3] = gen_rtx_REG (DImode, regno + 12);
4996 ops[4] = operands[1];
4997 output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
5000 [(set_attr "neon_type" "neon_vld3_vld4")]
5003 (define_insn "neon_vld4qb<mode>"
5004 [(set (match_operand:XI 0 "s_register_operand" "=w")
5005 (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um")
5006 (match_operand:XI 2 "s_register_operand" "0")
5007 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5011 int regno = REGNO (operands[0]);
5013 ops[0] = gen_rtx_REG (DImode, regno + 2);
5014 ops[1] = gen_rtx_REG (DImode, regno + 6);
5015 ops[2] = gen_rtx_REG (DImode, regno + 10);
5016 ops[3] = gen_rtx_REG (DImode, regno + 14);
5017 ops[4] = operands[1];
5018 output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops);
5021 [(set_attr "neon_type" "neon_vld3_vld4")]
5024 (define_insn "neon_vld4_lane<mode>"
5025 [(set (match_operand:OI 0 "s_register_operand" "=w")
5026 (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5027 (match_operand:OI 2 "s_register_operand" "0")
5028 (match_operand:SI 3 "immediate_operand" "i")
5029 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5033 HOST_WIDE_INT lane = INTVAL (operands[3]);
5034 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5035 int regno = REGNO (operands[0]);
5037 if (lane < 0 || lane >= max)
5038 error ("lane out of range");
5039 ops[0] = gen_rtx_REG (DImode, regno);
5040 ops[1] = gen_rtx_REG (DImode, regno + 2);
5041 ops[2] = gen_rtx_REG (DImode, regno + 4);
5042 ops[3] = gen_rtx_REG (DImode, regno + 6);
5043 ops[4] = operands[1];
5044 ops[5] = operands[3];
5045 output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
5049 [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5052 (define_insn "neon_vld4_lane<mode>"
5053 [(set (match_operand:XI 0 "s_register_operand" "=w")
5054 (unspec:XI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5055 (match_operand:XI 2 "s_register_operand" "0")
5056 (match_operand:SI 3 "immediate_operand" "i")
5057 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5061 HOST_WIDE_INT lane = INTVAL (operands[3]);
5062 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5063 int regno = REGNO (operands[0]);
5065 if (lane < 0 || lane >= max)
5066 error ("lane out of range");
5067 else if (lane >= max / 2)
5072 ops[0] = gen_rtx_REG (DImode, regno);
5073 ops[1] = gen_rtx_REG (DImode, regno + 4);
5074 ops[2] = gen_rtx_REG (DImode, regno + 8);
5075 ops[3] = gen_rtx_REG (DImode, regno + 12);
5076 ops[4] = operands[1];
5077 ops[5] = GEN_INT (lane);
5078 output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4",
5082 [(set_attr "neon_type" "neon_vld3_vld4_lane")]
5085 (define_insn "neon_vld4_dup<mode>"
5086 [(set (match_operand:OI 0 "s_register_operand" "=w")
5087 (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um")
5088 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5092 if (GET_MODE_NUNITS (<MODE>mode) > 1)
5094 int regno = REGNO (operands[0]);
5096 ops[0] = gen_rtx_REG (DImode, regno);
5097 ops[1] = gen_rtx_REG (DImode, regno + 2);
5098 ops[2] = gen_rtx_REG (DImode, regno + 4);
5099 ops[3] = gen_rtx_REG (DImode, regno + 6);
5100 ops[4] = operands[1];
5101 output_asm_insn ("vld4.<V_sz_elem>\t{%P0[], %P1[], %P2[], %P3[]}, %A4",
5106 return "vld1.<V_sz_elem>\t%h0, %A1";
5108 [(set (attr "neon_type")
5109 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1"))
5110 (const_string "neon_vld3_vld4_all_lanes")
5111 (const_string "neon_vld1_1_2_regs")))]
5114 (define_expand "vec_store_lanesoi<mode>"
5115 [(set (match_operand:OI 0 "neon_struct_operand")
5116 (unspec:OI [(match_operand:OI 1 "s_register_operand")
5117 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5121 (define_insn "neon_vst4<mode>"
5122 [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5123 (unspec:OI [(match_operand:OI 1 "s_register_operand" "w")
5124 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5128 if (<V_sz_elem> == 64)
5129 return "vst1.64\t%h1, %A0";
5131 return "vst4.<V_sz_elem>\t%h1, %A0";
5133 [(set (attr "neon_type")
5134 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64"))
5135 (const_string "neon_vst1_1_2_regs_vst2_2_regs")
5136 (const_string "neon_vst2_4_regs_vst3_vst4")))]
5139 (define_expand "vec_store_lanesxi<mode>"
5140 [(match_operand:XI 0 "neon_struct_operand")
5141 (match_operand:XI 1 "s_register_operand")
5142 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5145 emit_insn (gen_neon_vst4<mode> (operands[0], operands[1]));
5149 (define_expand "neon_vst4<mode>"
5150 [(match_operand:XI 0 "neon_struct_operand")
5151 (match_operand:XI 1 "s_register_operand")
5152 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5157 mem = adjust_address (operands[0], OImode, 0);
5158 emit_insn (gen_neon_vst4qa<mode> (mem, operands[1]));
5159 mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode));
5160 emit_insn (gen_neon_vst4qb<mode> (mem, operands[1]));
5164 (define_insn "neon_vst4qa<mode>"
5165 [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5166 (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
5167 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5171 int regno = REGNO (operands[1]);
5173 ops[0] = operands[0];
5174 ops[1] = gen_rtx_REG (DImode, regno);
5175 ops[2] = gen_rtx_REG (DImode, regno + 4);
5176 ops[3] = gen_rtx_REG (DImode, regno + 8);
5177 ops[4] = gen_rtx_REG (DImode, regno + 12);
5178 output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
5181 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5184 (define_insn "neon_vst4qb<mode>"
5185 [(set (match_operand:OI 0 "neon_struct_operand" "=Um")
5186 (unspec:OI [(match_operand:XI 1 "s_register_operand" "w")
5187 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5191 int regno = REGNO (operands[1]);
5193 ops[0] = operands[0];
5194 ops[1] = gen_rtx_REG (DImode, regno + 2);
5195 ops[2] = gen_rtx_REG (DImode, regno + 6);
5196 ops[3] = gen_rtx_REG (DImode, regno + 10);
5197 ops[4] = gen_rtx_REG (DImode, regno + 14);
5198 output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops);
5201 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")]
5204 (define_insn "neon_vst4_lane<mode>"
5205 [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
5206 (unspec:<V_four_elem>
5207 [(match_operand:OI 1 "s_register_operand" "w")
5208 (match_operand:SI 2 "immediate_operand" "i")
5209 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5213 HOST_WIDE_INT lane = INTVAL (operands[2]);
5214 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5215 int regno = REGNO (operands[1]);
5217 if (lane < 0 || lane >= max)
5218 error ("lane out of range");
5219 ops[0] = operands[0];
5220 ops[1] = gen_rtx_REG (DImode, regno);
5221 ops[2] = gen_rtx_REG (DImode, regno + 2);
5222 ops[3] = gen_rtx_REG (DImode, regno + 4);
5223 ops[4] = gen_rtx_REG (DImode, regno + 6);
5224 ops[5] = operands[2];
5225 output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
5229 [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5232 (define_insn "neon_vst4_lane<mode>"
5233 [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um")
5234 (unspec:<V_four_elem>
5235 [(match_operand:XI 1 "s_register_operand" "w")
5236 (match_operand:SI 2 "immediate_operand" "i")
5237 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)]
5241 HOST_WIDE_INT lane = INTVAL (operands[2]);
5242 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode);
5243 int regno = REGNO (operands[1]);
5245 if (lane < 0 || lane >= max)
5246 error ("lane out of range");
5247 else if (lane >= max / 2)
5252 ops[0] = operands[0];
5253 ops[1] = gen_rtx_REG (DImode, regno);
5254 ops[2] = gen_rtx_REG (DImode, regno + 4);
5255 ops[3] = gen_rtx_REG (DImode, regno + 8);
5256 ops[4] = gen_rtx_REG (DImode, regno + 12);
5257 ops[5] = GEN_INT (lane);
5258 output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0",
5262 [(set_attr "neon_type" "neon_vst3_vst4_lane")]
5265 (define_expand "neon_vand<mode>"
5266 [(match_operand:VDQX 0 "s_register_operand" "")
5267 (match_operand:VDQX 1 "s_register_operand" "")
5268 (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5269 (match_operand:SI 3 "immediate_operand" "")]
5272 emit_insn (gen_and<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5276 (define_expand "neon_vorr<mode>"
5277 [(match_operand:VDQX 0 "s_register_operand" "")
5278 (match_operand:VDQX 1 "s_register_operand" "")
5279 (match_operand:VDQX 2 "neon_logic_op2" "")
5280 (match_operand:SI 3 "immediate_operand" "")]
5283 emit_insn (gen_ior<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5287 (define_expand "neon_veor<mode>"
5288 [(match_operand:VDQX 0 "s_register_operand" "")
5289 (match_operand:VDQX 1 "s_register_operand" "")
5290 (match_operand:VDQX 2 "s_register_operand" "")
5291 (match_operand:SI 3 "immediate_operand" "")]
5294 emit_insn (gen_xor<mode>3<V_suf64> (operands[0], operands[1], operands[2]));
5298 (define_expand "neon_vbic<mode>"
5299 [(match_operand:VDQX 0 "s_register_operand" "")
5300 (match_operand:VDQX 1 "s_register_operand" "")
5301 (match_operand:VDQX 2 "neon_logic_op2" "")
5302 (match_operand:SI 3 "immediate_operand" "")]
5305 emit_insn (gen_bic<mode>3_neon (operands[0], operands[1], operands[2]));
5309 (define_expand "neon_vorn<mode>"
5310 [(match_operand:VDQX 0 "s_register_operand" "")
5311 (match_operand:VDQX 1 "s_register_operand" "")
5312 (match_operand:VDQX 2 "neon_inv_logic_op2" "")
5313 (match_operand:SI 3 "immediate_operand" "")]
5316 emit_insn (gen_orn<mode>3_neon (operands[0], operands[1], operands[2]));
5320 (define_insn "neon_vec_unpack<US>_lo_<mode>"
5321 [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5322 (SE:<V_unpack> (vec_select:<V_HALF>
5323 (match_operand:VU 1 "register_operand" "w")
5324 (match_operand:VU 2 "vect_par_constant_low" ""))))]
5325 "TARGET_NEON && !BYTES_BIG_ENDIAN"
5326 "vmovl.<US><V_sz_elem> %q0, %e1"
5327 [(set_attr "neon_type" "neon_shift_1")]
5330 (define_insn "neon_vec_unpack<US>_hi_<mode>"
5331 [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5332 (SE:<V_unpack> (vec_select:<V_HALF>
5333 (match_operand:VU 1 "register_operand" "w")
5334 (match_operand:VU 2 "vect_par_constant_high" ""))))]
5335 "TARGET_NEON && !BYTES_BIG_ENDIAN"
5336 "vmovl.<US><V_sz_elem> %q0, %f1"
5337 [(set_attr "neon_type" "neon_shift_1")]
5340 (define_expand "vec_unpack<US>_hi_<mode>"
5341 [(match_operand:<V_unpack> 0 "register_operand" "")
5342 (SE:<V_unpack> (match_operand:VU 1 "register_operand"))]
5343 "TARGET_NEON && !BYTES_BIG_ENDIAN"
5345 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ;
5348 for (i = 0; i < (<V_mode_nunits>/2); i++)
5349 RTVEC_ELT (v, i) = GEN_INT ((<V_mode_nunits>/2) + i);
5351 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5352 emit_insn (gen_neon_vec_unpack<US>_hi_<mode> (operands[0],
5359 (define_expand "vec_unpack<US>_lo_<mode>"
5360 [(match_operand:<V_unpack> 0 "register_operand" "")
5361 (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))]
5362 "TARGET_NEON && !BYTES_BIG_ENDIAN"
5364 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ;
5367 for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5368 RTVEC_ELT (v, i) = GEN_INT (i);
5369 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5370 emit_insn (gen_neon_vec_unpack<US>_lo_<mode> (operands[0],
5377 (define_insn "neon_vec_<US>mult_lo_<mode>"
5378 [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5379 (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5380 (match_operand:VU 1 "register_operand" "w")
5381 (match_operand:VU 2 "vect_par_constant_low" "")))
5382 (SE:<V_unpack> (vec_select:<V_HALF>
5383 (match_operand:VU 3 "register_operand" "w")
5385 "TARGET_NEON && !BYTES_BIG_ENDIAN"
5386 "vmull.<US><V_sz_elem> %q0, %e1, %e3"
5387 [(set_attr "neon_type" "neon_shift_1")]
5390 (define_expand "vec_widen_<US>mult_lo_<mode>"
5391 [(match_operand:<V_unpack> 0 "register_operand" "")
5392 (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5393 (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5394 "TARGET_NEON && !BYTES_BIG_ENDIAN"
5396 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ;
5399 for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5400 RTVEC_ELT (v, i) = GEN_INT (i);
5401 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5403 emit_insn (gen_neon_vec_<US>mult_lo_<mode> (operands[0],
5411 (define_insn "neon_vec_<US>mult_hi_<mode>"
5412 [(set (match_operand:<V_unpack> 0 "register_operand" "=w")
5413 (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF>
5414 (match_operand:VU 1 "register_operand" "w")
5415 (match_operand:VU 2 "vect_par_constant_high" "")))
5416 (SE:<V_unpack> (vec_select:<V_HALF>
5417 (match_operand:VU 3 "register_operand" "w")
5419 "TARGET_NEON && !BYTES_BIG_ENDIAN"
5420 "vmull.<US><V_sz_elem> %q0, %f1, %f3"
5421 [(set_attr "neon_type" "neon_shift_1")]
5424 (define_expand "vec_widen_<US>mult_hi_<mode>"
5425 [(match_operand:<V_unpack> 0 "register_operand" "")
5426 (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5427 (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))]
5428 "TARGET_NEON && !BYTES_BIG_ENDIAN"
5430 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ;
5433 for (i = 0; i < (<V_mode_nunits>/2) ; i++)
5434 RTVEC_ELT (v, i) = GEN_INT (<V_mode_nunits>/2 + i);
5435 t1 = gen_rtx_PARALLEL (<MODE>mode, v);
5437 emit_insn (gen_neon_vec_<US>mult_hi_<mode> (operands[0],
5446 (define_insn "neon_vec_<US>shiftl_<mode>"
5447 [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5448 (SE:<V_widen> (ashift:VW (match_operand:VW 1 "register_operand" "w")
5449 (match_operand:<V_innermode> 2 "const_neon_scalar_shift_amount_operand" ""))))]
5452 return "vshll.<US><V_sz_elem> %q0, %P1, %2";
5454 [(set_attr "neon_type" "neon_shift_1")]
5457 (define_expand "vec_widen_<US>shiftl_lo_<mode>"
5458 [(match_operand:<V_unpack> 0 "register_operand" "")
5459 (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5460 (match_operand:SI 2 "immediate_operand" "i")]
5461 "TARGET_NEON && !BYTES_BIG_ENDIAN"
5463 emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5464 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode, 0),
5470 (define_expand "vec_widen_<US>shiftl_hi_<mode>"
5471 [(match_operand:<V_unpack> 0 "register_operand" "")
5472 (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))
5473 (match_operand:SI 2 "immediate_operand" "i")]
5474 "TARGET_NEON && !BYTES_BIG_ENDIAN"
5476 emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0],
5477 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode,
5478 GET_MODE_SIZE (<V_HALF>mode)),
5484 ;; Vectorize for non-neon-quad case
5485 (define_insn "neon_unpack<US>_<mode>"
5486 [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5487 (SE:<V_widen> (match_operand:VDI 1 "register_operand" "w")))]
5489 "vmovl.<US><V_sz_elem> %q0, %P1"
5490 [(set_attr "neon_type" "neon_shift_1")]
5493 (define_expand "vec_unpack<US>_lo_<mode>"
5494 [(match_operand:<V_double_width> 0 "register_operand" "")
5495 (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5498 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5499 emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5500 emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5506 (define_expand "vec_unpack<US>_hi_<mode>"
5507 [(match_operand:<V_double_width> 0 "register_operand" "")
5508 (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))]
5511 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5512 emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1]));
5513 emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5519 (define_insn "neon_vec_<US>mult_<mode>"
5520 [(set (match_operand:<V_widen> 0 "register_operand" "=w")
5521 (mult:<V_widen> (SE:<V_widen>
5522 (match_operand:VDI 1 "register_operand" "w"))
5524 (match_operand:VDI 2 "register_operand" "w"))))]
5526 "vmull.<US><V_sz_elem> %q0, %P1, %P2"
5527 [(set_attr "neon_type" "neon_shift_1")]
5530 (define_expand "vec_widen_<US>mult_hi_<mode>"
5531 [(match_operand:<V_double_width> 0 "register_operand" "")
5532 (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5533 (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5536 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5537 emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5538 emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5545 (define_expand "vec_widen_<US>mult_lo_<mode>"
5546 [(match_operand:<V_double_width> 0 "register_operand" "")
5547 (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5548 (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))]
5551 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5552 emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2]));
5553 emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5560 (define_expand "vec_widen_<US>shiftl_hi_<mode>"
5561 [(match_operand:<V_double_width> 0 "register_operand" "")
5562 (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5563 (match_operand:SI 2 "immediate_operand" "i")]
5566 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5567 emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5568 emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg));
5574 (define_expand "vec_widen_<US>shiftl_lo_<mode>"
5575 [(match_operand:<V_double_width> 0 "register_operand" "")
5576 (SE:<V_double_width> (match_operand:VDI 1 "register_operand" ""))
5577 (match_operand:SI 2 "immediate_operand" "i")]
5580 rtx tmpreg = gen_reg_rtx (<V_widen>mode);
5581 emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2]));
5582 emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg));
5588 ; FIXME: These instruction patterns can't be used safely in big-endian mode
5589 ; because the ordering of vector elements in Q registers is different from what
5590 ; the semantics of the instructions require.
5592 (define_insn "vec_pack_trunc_<mode>"
5593 [(set (match_operand:<V_narrow_pack> 0 "register_operand" "=&w")
5594 (vec_concat:<V_narrow_pack>
5595 (truncate:<V_narrow>
5596 (match_operand:VN 1 "register_operand" "w"))
5597 (truncate:<V_narrow>
5598 (match_operand:VN 2 "register_operand" "w"))))]
5599 "TARGET_NEON && !BYTES_BIG_ENDIAN"
5600 "vmovn.i<V_sz_elem>\t%e0, %q1\;vmovn.i<V_sz_elem>\t%f0, %q2"
5601 [(set_attr "neon_type" "neon_shift_1")
5602 (set_attr "length" "8")]
5605 ;; For the non-quad case.
5606 (define_insn "neon_vec_pack_trunc_<mode>"
5607 [(set (match_operand:<V_narrow> 0 "register_operand" "=w")
5608 (truncate:<V_narrow> (match_operand:VN 1 "register_operand" "w")))]
5609 "TARGET_NEON && !BYTES_BIG_ENDIAN"
5610 "vmovn.i<V_sz_elem>\t%P0, %q1"
5611 [(set_attr "neon_type" "neon_shift_1")]
5614 (define_expand "vec_pack_trunc_<mode>"
5615 [(match_operand:<V_narrow_pack> 0 "register_operand" "")
5616 (match_operand:VSHFT 1 "register_operand" "")
5617 (match_operand:VSHFT 2 "register_operand")]
5618 "TARGET_NEON && !BYTES_BIG_ENDIAN"
5620 rtx tempreg = gen_reg_rtx (<V_DOUBLE>mode);
5622 emit_insn (gen_move_lo_quad_<V_double> (tempreg, operands[1]));
5623 emit_insn (gen_move_hi_quad_<V_double> (tempreg, operands[2]));
5624 emit_insn (gen_neon_vec_pack_trunc_<V_double> (operands[0], tempreg));
5628 (define_insn "neon_vabd<mode>_2"
5629 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5630 (abs:VDQ (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w")
5631 (match_operand:VDQ 2 "s_register_operand" "w"))))]
5632 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5633 "vabd.<V_s_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5634 [(set (attr "neon_type")
5635 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5636 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
5637 (const_string "neon_fp_vadd_ddd_vabs_dd")
5638 (const_string "neon_fp_vadd_qqq_vabs_qq"))
5639 (const_string "neon_int_5")))]
5642 (define_insn "neon_vabd<mode>_3"
5643 [(set (match_operand:VDQ 0 "s_register_operand" "=w")
5644 (abs:VDQ (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w")
5645 (match_operand:VDQ 2 "s_register_operand" "w")]
5647 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)"
5648 "vabd.<V_if_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2"
5649 [(set (attr "neon_type")
5650 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0))
5651 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0))
5652 (const_string "neon_fp_vadd_ddd_vabs_dd")
5653 (const_string "neon_fp_vadd_qqq_vabs_qq"))
5654 (const_string "neon_int_5")))]