1
2
3
4
5 package arm64
6
7 import (
8 "math"
9
10 "cmd/compile/internal/base"
11 "cmd/compile/internal/ir"
12 "cmd/compile/internal/logopt"
13 "cmd/compile/internal/objw"
14 "cmd/compile/internal/ssa"
15 "cmd/compile/internal/ssa/block"
16 "cmd/compile/internal/ssa/ssaop"
17 "cmd/compile/internal/ssagen"
18 "cmd/compile/internal/types"
19 "cmd/internal/obj"
20 "cmd/internal/obj/arm64"
21 "internal/abi"
22 )
23
24
25 func loadByType(t *types.Type) obj.As {
26 if t.IsSIMD() {
27 switch t.Size() {
28 case 16:
29 return arm64.AFMOVQ
30 case 8:
31 return arm64.APLDR
32 case 32:
33 return arm64.AZLDR
34 }
35 } else if t.IsFloat() {
36 switch t.Size() {
37 case 4:
38 return arm64.AFMOVS
39 case 8:
40 return arm64.AFMOVD
41 }
42 } else {
43 switch t.Size() {
44 case 1:
45 if t.IsSigned() {
46 return arm64.AMOVB
47 } else {
48 return arm64.AMOVBU
49 }
50 case 2:
51 if t.IsSigned() {
52 return arm64.AMOVH
53 } else {
54 return arm64.AMOVHU
55 }
56 case 4:
57 if t.IsSigned() {
58 return arm64.AMOVW
59 } else {
60 return arm64.AMOVWU
61 }
62 case 8:
63 return arm64.AMOVD
64 }
65 }
66 panic("bad load type")
67 }
68
69
70 func storeByType(t *types.Type) obj.As {
71 if t.IsSIMD() {
72 switch t.Size() {
73 case 16:
74 return arm64.AFMOVQ
75 case 8:
76 return arm64.APSTR
77 case 32:
78 return arm64.AZSTR
79 }
80 } else if t.IsFloat() {
81 switch t.Size() {
82 case 4:
83 return arm64.AFMOVS
84 case 8:
85 return arm64.AFMOVD
86 }
87 } else {
88 switch t.Size() {
89 case 1:
90 return arm64.AMOVB
91 case 2:
92 return arm64.AMOVH
93 case 4:
94 return arm64.AMOVW
95 case 8:
96 return arm64.AMOVD
97 }
98 }
99 panic("bad store type")
100 }
101
102
103
104 func loadByType2(t *types.Type) obj.As {
105 if t.IsFloat() {
106 switch t.Size() {
107 case 4:
108 return arm64.AFLDPS
109 case 8:
110 return arm64.AFLDPD
111 }
112 } else {
113 switch t.Size() {
114 case 4:
115 return arm64.ALDPW
116 case 8:
117 return arm64.ALDP
118 }
119 }
120 return obj.AXXX
121 }
122
123
124
125 func storeByType2(t *types.Type) obj.As {
126 if t.IsFloat() {
127 switch t.Size() {
128 case 4:
129 return arm64.AFSTPS
130 case 8:
131 return arm64.AFSTPD
132 }
133 } else {
134 switch t.Size() {
135 case 4:
136 return arm64.ASTPW
137 case 8:
138 return arm64.ASTP
139 }
140 }
141 return obj.AXXX
142 }
143
144
145 func makeshift(v *ssa.Value, reg int16, typ int64, s int64) int64 {
146 if s < 0 || s >= 64 {
147 v.Fatalf("shift out of range: %d", s)
148 }
149 return int64(reg&31)<<16 | typ | (s&63)<<10
150 }
151
152
153 func genshift(s *ssagen.State, v *ssa.Value, as obj.As, r0, r1, r int16, typ int64, n int64) *obj.Prog {
154 p := s.Prog(as)
155 p.From.Type = obj.TYPE_SHIFT
156 p.From.Offset = makeshift(v, r1, typ, n)
157 p.Reg = r0
158 if r != 0 {
159 p.To.Type = obj.TYPE_REG
160 p.To.Reg = r
161 }
162 return p
163 }
164
165
166
167 func genIndexedOperand(op ssaop.Op, base, idx int16) obj.Addr {
168
169 mop := obj.Addr{Type: obj.TYPE_MEM, Reg: base}
170 switch op {
171 case ssaop.OpARM64MOVDloadidx8, ssaop.OpARM64MOVDstoreidx8,
172 ssaop.OpARM64FMOVDloadidx8, ssaop.OpARM64FMOVDstoreidx8:
173 mop.Index = arm64.REG_LSL | 3<<5 | idx&31
174 case ssaop.OpARM64MOVWloadidx4, ssaop.OpARM64MOVWUloadidx4, ssaop.OpARM64MOVWstoreidx4,
175 ssaop.OpARM64FMOVSloadidx4, ssaop.OpARM64FMOVSstoreidx4:
176 mop.Index = arm64.REG_LSL | 2<<5 | idx&31
177 case ssaop.OpARM64MOVHloadidx2, ssaop.OpARM64MOVHUloadidx2, ssaop.OpARM64MOVHstoreidx2:
178 mop.Index = arm64.REG_LSL | 1<<5 | idx&31
179 default:
180 mop.Index = idx
181 }
182 return mop
183 }
184
185
186 const simdSVEVectorLengthScaled = arm64.SIMDSVEVectorLengthScaled
187
188
189 func simdRegArng(reg int16, arng int16) int16 {
190 if reg < arm64.REG_F0 || arm64.REG_F31 < reg {
191 base.Fatalf("expected fp register: r%d", reg)
192 }
193 var err error
194 if reg, err = arm64.RegisterArrangement(reg, arng, false); err != nil {
195 base.Fatalf("bad simd register arrangement: %v", err)
196 }
197 return reg
198 }
199
200
201 func simdRegElem(reg int16, arng int16, idx int16) (res obj.Addr) {
202 if reg < arm64.REG_F0 || arm64.REG_F31 < reg {
203 base.Fatalf("expected fp register: r%d", reg)
204 }
205 elem, err := arm64.RegisterArrangement(reg, arng, true )
206 if err != nil {
207 base.Fatalf("bad simd register indexing arrangement: %v", err)
208 }
209 res.Type = obj.TYPE_REG
210 res.Class = arm64.C_ELEM
211 res.Index = idx
212 res.Reg = elem
213 return
214 }
215
216
217
218 func allLanes(arng int16) int16 {
219 switch arng {
220 case arm64.ARNG_B:
221 return arm64.ARNG_16B
222 case arm64.ARNG_H:
223 return arm64.ARNG_8H
224 case arm64.ARNG_S:
225 return arm64.ARNG_4S
226 case arm64.ARNG_D:
227 return arm64.ARNG_2D
228 default:
229 base.Fatalf("unsupported element arrangement: %d", arng)
230 return 0
231 }
232 }
233
234
235 func arngNarrow(arng int16) int16 {
236 switch arng {
237 case arm64.ARNG_8H:
238 return arm64.ARNG_8B
239 case arm64.ARNG_4S:
240 return arm64.ARNG_4H
241 case arm64.ARNG_2D:
242 return arm64.ARNG_2S
243 default:
244 base.Fatalf("unsupported narrow input arrangement: %d", arng)
245 return 0
246 }
247 }
248
249
250 func arngLong(arng int16) int16 {
251 switch arng {
252 case arm64.ARNG_8B:
253 return arm64.ARNG_8H
254 case arm64.ARNG_4H:
255 return arm64.ARNG_4S
256 case arm64.ARNG_2S:
257 return arm64.ARNG_2D
258 case arm64.ARNG_1D:
259 return arm64.ARNG_1Q
260 default:
261 base.Fatalf("unsupported long input arrangement: %d", arng)
262 return 0
263 }
264 }
265
266
267
268 func arngHalfLanes(arng int16) int16 {
269 switch arng {
270 case arm64.ARNG_16B:
271 return arm64.ARNG_8B
272 case arm64.ARNG_8H:
273 return arm64.ARNG_4H
274 case arm64.ARNG_4S:
275 return arm64.ARNG_2S
276 case arm64.ARNG_2D:
277 return arm64.ARNG_1D
278 default:
279 base.Fatalf("unsupported halfLanes input arrangement: %d", arng)
280 return 0
281 }
282 }
283
284
285
286 func arngTwiceLanes(arng int16) int16 {
287 switch arng {
288 case arm64.ARNG_8B:
289 return arm64.ARNG_16B
290 case arm64.ARNG_4H:
291 return arm64.ARNG_8H
292 case arm64.ARNG_2S:
293 return arm64.ARNG_4S
294 default:
295 base.Fatalf("unsupported twiceLanes input arrangement: %d", arng)
296 return 0
297 }
298 }
299
300
301 func simdV01Imm(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
302 p := s.Prog(v.Op.Asm())
303 p.From.Type = obj.TYPE_CONST
304 p.From.Offset = int64(v.AuxUInt8())
305 p.To.Type = obj.TYPE_REG
306 p.To.Reg = simdRegArng(v.Reg(), arrangement)
307 return p
308 }
309
310
311 func simdV11Asm(s *ssagen.State, asm obj.As, src, dst int16, arrangement int16) *obj.Prog {
312 p := s.Prog(asm)
313 p.From.Type = obj.TYPE_REG
314 p.From.Reg = simdRegArng(src, arrangement)
315 p.To.Type = obj.TYPE_REG
316 p.To.Reg = simdRegArng(dst, arrangement)
317 return p
318 }
319
320
321 func simdV11(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
322 return simdV11Asm(s, v.Op.Asm(), v.Args[0].Reg(), v.Reg(), arrangement)
323 }
324
325
326
327 func simdV11Imm(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
328 p := s.Prog(v.Op.Asm())
329 p.From.Type = obj.TYPE_CONST
330 p.From.Offset = int64(v.AuxUInt8())
331 p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
332 p.To.Type = obj.TYPE_REG
333 p.To.Reg = simdRegArng(v.Reg(), arrangement)
334 return p
335 }
336
337
338
339
340 func simdV11ImmIn1(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
341 p := s.Prog(v.Op.Asm())
342 p.From = simdRegElem(v.Args[0].Reg(), arrangement, int16(v.AuxUInt8()))
343 p.To.Type = obj.TYPE_REG
344 p.To.Reg = simdRegArng(v.Reg(), allLanes(arrangement))
345 return p
346 }
347
348
349 func simdV11Scalar(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
350 p := s.Prog(v.Op.Asm())
351 p.From.Type = obj.TYPE_REG
352 p.From.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
353 p.To.Type = obj.TYPE_REG
354 p.To.Reg = v.Reg() - arm64.REG_F0 + arm64.REG_V0
355 return p
356 }
357
358
359
360
361 func simdV11ScalarImmIn1(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
362 p := s.Prog(v.Op.Asm())
363 p.From = simdRegElem(v.Args[0].Reg(), arrangement, int16(v.AuxUInt8()))
364 p.To.Type = obj.TYPE_REG
365 p.To.Reg = v.Reg() - arm64.REG_F0 + arm64.REG_V0
366 p.To.Class = arm64.C_VREG
367 return p
368 }
369
370
371 func simdV21(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
372 p := s.Prog(v.Op.Asm())
373 p.From.Type = obj.TYPE_REG
374 p.From.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
375 p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
376 p.To.Type = obj.TYPE_REG
377 p.To.Reg = simdRegArng(v.Reg(), arrangement)
378 return p
379 }
380
381
382 func simdV21Imm(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
383 p := s.Prog(v.Op.Asm())
384 p.From.Type = obj.TYPE_CONST
385 p.From.Offset = int64(v.AuxUInt8())
386 p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
387 p.To.Type = obj.TYPE_REG
388 p.To.Reg = simdRegArng(v.Reg(), arrangement)
389 p.AddRestSource(obj.Addr{Type: obj.TYPE_REG, Reg: simdRegArng(v.Args[1].Reg(), arrangement)})
390 return p
391 }
392
393
394
395 func simdV31ResultInArg0(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
396 p := s.Prog(v.Op.Asm())
397 p.From.Type = obj.TYPE_REG
398 p.From.Reg = simdRegArng(v.Args[2].Reg(), arrangement)
399 p.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
400 p.To.Type = obj.TYPE_REG
401 p.To.Reg = simdRegArng(v.Reg(), arrangement)
402 return p
403 }
404
405
406 func simdV21List(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
407 if v.Op.Asm() != arm64.AVTBL {
408 panic("simdV21List: expected VTBL")
409 }
410 p := s.Prog(v.Op.Asm())
411 p.From.Type = obj.TYPE_REG
412 p.From.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
413
414
415 const listB16 = int64(1 << 30)
416 regList, _ := arm64.RegisterListOffset(int(v.Args[0].Reg()&31), 1, listB16, 0)
417 p.AddRestSource(obj.Addr{Type: obj.TYPE_REGLIST, Offset: regList})
418 p.To.Type = obj.TYPE_REG
419 p.To.Reg = simdRegArng(v.Reg(), arrangement)
420 return p
421 }
422
423
424
425 func simdV31ResultInArg0List(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
426 if v.Op.Asm() != arm64.AVTBX {
427 panic("simdV31ResultInArg0List: expected VTBX")
428 }
429 p := s.Prog(v.Op.Asm())
430 p.From.Type = obj.TYPE_REG
431 p.From.Reg = simdRegArng(v.Args[2].Reg(), arrangement)
432
433
434 const listB16 = int64(1 << 30)
435 regList, _ := arm64.RegisterListOffset(int(v.Args[1].Reg()&31), 1, listB16, 0)
436 p.AddRestSource(obj.Addr{Type: obj.TYPE_REGLIST, Offset: regList})
437 p.To.Type = obj.TYPE_REG
438 p.To.Reg = simdRegArng(v.Reg(), arrangement)
439 return p
440 }
441
442
443
444
445 func simdVfpvResultInArg0ImmOutIn1(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
446 p := s.Prog(v.Op.Asm())
447 p.To = simdRegElem(v.Reg(), arrangement, int16(v.AuxUInt8()))
448 p.From = simdRegElem(v.Args[1].Reg(), arrangement, 0)
449 return p
450 }
451
452
453
454 func simdVgpImmIn1(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
455 p := s.Prog(v.Op.Asm())
456 p.From = simdRegElem(v.Args[0].Reg(), arrangement, int16(v.AuxUInt8()))
457 p.To.Reg = v.Reg()
458 p.To.Type = obj.TYPE_REG
459 return p
460 }
461
462
463
464 func simdVgpvResultInArg0ImmOutIn0(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
465 p := s.Prog(v.Op.Asm())
466 p.To = simdRegElem(v.Reg(), arrangement, int16(v.AuxUInt8()))
467 p.From.Reg = v.Args[1].Reg()
468 p.From.Type = obj.TYPE_REG
469 return p
470 }
471
472
473
474
475 func simdV11Narrow(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
476 p := s.Prog(v.Op.Asm())
477 p.From.Type = obj.TYPE_REG
478 p.From.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
479 p.To.Type = obj.TYPE_REG
480 p.To.Reg = simdRegArng(v.Reg(), arngNarrow(arrangement))
481 return p
482 }
483
484
485
486 func simdV21Narrow2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
487 p := s.Prog(v.Op.Asm())
488 p.From.Type = obj.TYPE_REG
489 p.From.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
490 p.To.Type = obj.TYPE_REG
491 p.To.Reg = simdRegArng(v.Reg(), arngTwiceLanes(arngNarrow(arrangement)))
492 return p
493 }
494
495
496
497 func simdV11ImmNarrow(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
498 p := s.Prog(v.Op.Asm())
499 p.From.Type = obj.TYPE_CONST
500 p.From.Offset = int64(v.AuxUInt8())
501 p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
502 p.To.Type = obj.TYPE_REG
503 p.To.Reg = simdRegArng(v.Reg(), arngNarrow(arrangement))
504 return p
505 }
506
507
508
509 func simdV21ImmNarrow2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
510 p := s.Prog(v.Op.Asm())
511 p.From.Type = obj.TYPE_CONST
512 p.From.Offset = int64(v.AuxUInt8())
513 p.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
514 p.To.Type = obj.TYPE_REG
515 p.To.Reg = simdRegArng(v.Reg(), arngTwiceLanes(arngNarrow(arrangement)))
516 return p
517 }
518
519
520
521 func simdV11Long(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
522 src := arngHalfLanes(arrangement)
523 p := s.Prog(v.Op.Asm())
524 p.From.Type = obj.TYPE_REG
525 p.From.Reg = simdRegArng(v.Args[0].Reg(), src)
526 p.To.Type = obj.TYPE_REG
527 p.To.Reg = simdRegArng(v.Reg(), arngLong(src))
528 return p
529 }
530
531
532
533 func simdV11Long2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
534 p := s.Prog(v.Op.Asm())
535 p.From.Type = obj.TYPE_REG
536 p.From.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
537 p.To.Type = obj.TYPE_REG
538 p.To.Reg = simdRegArng(v.Reg(), arngLong(arngHalfLanes(arrangement)))
539 return p
540 }
541
542
543
544 func simdV11ImmLong(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
545 src := arngHalfLanes(arrangement)
546 p := s.Prog(v.Op.Asm())
547 p.From.Type = obj.TYPE_CONST
548 p.From.Offset = int64(v.AuxUInt8())
549 p.Reg = simdRegArng(v.Args[0].Reg(), src)
550 p.To.Type = obj.TYPE_REG
551 p.To.Reg = simdRegArng(v.Reg(), arngLong(src))
552 return p
553 }
554
555
556
557 func simdV11ImmLong2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
558 p := s.Prog(v.Op.Asm())
559 p.From.Type = obj.TYPE_CONST
560 p.From.Offset = int64(v.AuxUInt8())
561 p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
562 p.To.Type = obj.TYPE_REG
563 p.To.Reg = simdRegArng(v.Reg(), arngLong(arngHalfLanes(arrangement)))
564 return p
565 }
566
567
568
569 func simdV21Long(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
570 src := arngHalfLanes(arrangement)
571 p := s.Prog(v.Op.Asm())
572 p.From.Type = obj.TYPE_REG
573 p.From.Reg = simdRegArng(v.Args[1].Reg(), src)
574 p.Reg = simdRegArng(v.Args[0].Reg(), src)
575 p.To.Type = obj.TYPE_REG
576 p.To.Reg = simdRegArng(v.Reg(), arngLong(src))
577 return p
578 }
579
580
581
582 func simdV21Long2(s *ssagen.State, v *ssa.Value, arrangement int16) *obj.Prog {
583 p := s.Prog(v.Op.Asm())
584 p.From.Type = obj.TYPE_REG
585 p.From.Reg = simdRegArng(v.Args[1].Reg(), arrangement)
586 p.Reg = simdRegArng(v.Args[0].Reg(), arrangement)
587 p.To.Type = obj.TYPE_REG
588 p.To.Reg = simdRegArng(v.Reg(), arngLong(arngHalfLanes(arrangement)))
589 return p
590 }
591
592 func ssaGenValue(s *ssagen.State, v *ssa.Value) {
593 switch v.Op {
594 case ssaop.OpCopy, ssaop.OpARM64MOVDreg:
595 if v.Type.IsMemory() {
596 return
597 }
598 x := v.Args[0].Reg()
599 y := v.Reg()
600 if x == y {
601 return
602 }
603 as := arm64.AMOVD
604 if v.Type.IsFloat() {
605 switch v.Type.Size() {
606 case 4:
607 as = arm64.AFMOVS
608 case 8:
609 as = arm64.AFMOVD
610 default:
611 panic("bad float size")
612 }
613 } else if v.Type.IsSIMD() {
614 if v.Type.Size() == 16 {
615 simdV11Asm(s, arm64.AVMOV, x, y, arm64.ARNG_16B)
616 return
617 } else if v.Type.Size() == 32 {
618
619 p := s.Prog(arm64.AZORR)
620 p.From.Type = obj.TYPE_REG
621 p.From.Reg = zregArng(x, arm64.ARNG_D)
622 p.AddRestSourceReg(zregArng(x, arm64.ARNG_D))
623 p.To.Type = obj.TYPE_REG
624 p.To.Reg = zregArng(y, arm64.ARNG_D)
625 return
626 } else if v.Type.Size() == 8 {
627
628 p := s.Prog(arm64.APORR)
629 p.From.Type = obj.TYPE_REG
630 if x < arm64.REG_P0 || x > arm64.REG_P15 {
631 panic("bad P reg")
632 }
633 p.From.Reg = pregArng(x, arm64.ARNG_B)
634 p.AddRestSourceReg(pregArng(x, arm64.ARNG_B))
635 p.AddRestSourceReg(pregMask(x, arm64.PRED_Z))
636 p.To.Type = obj.TYPE_REG
637 p.To.Reg = pregArng(y, arm64.ARNG_B)
638 return
639 } else {
640 panic("bad simd size")
641 }
642 }
643 p := s.Prog(as)
644 p.From.Type = obj.TYPE_REG
645 p.From.Reg = x
646 p.To.Type = obj.TYPE_REG
647 p.To.Reg = y
648 case ssaop.OpARM64MOVDnop, ssaop.OpARM64ZERO:
649
650 case ssaop.OpARM64VMOVI16B:
651 simdV01Imm(s, v, arm64.ARNG_16B)
652 case ssaop.OpLoadReg:
653 if v.Type.IsFlags() {
654 v.Fatalf("load flags not implemented: %v", v.LongString())
655 return
656 }
657 p := s.Prog(loadByType(v.Type))
658 ssagen.AddrAuto(&p.From, v.Args[0])
659 p.To.Type = obj.TYPE_REG
660 if v.Type.IsSIMD() && (v.Type.Size() == 32 || v.Type.Size() == 8) {
661 p.To.Reg = pzreg(v.Reg())
662 } else {
663 p.To.Reg = v.Reg()
664 }
665 case ssaop.OpStoreReg:
666 if v.Type.IsFlags() {
667 v.Fatalf("store flags not implemented: %v", v.LongString())
668 return
669 }
670 p := s.Prog(storeByType(v.Type))
671 p.From.Type = obj.TYPE_REG
672 if v.Type.IsSIMD() && (v.Type.Size() == 32 || v.Type.Size() == 8) {
673 p.From.Reg = pzreg(v.Args[0].Reg())
674 } else {
675 p.From.Reg = v.Args[0].Reg()
676 }
677 ssagen.AddrAuto(&p.To, v)
678 case ssaop.OpArgIntReg, ssaop.OpArgFloatReg:
679 ssagen.CheckArgReg(v)
680
681
682 args := v.Block.Func.RegArgs
683 if len(args) == 0 {
684 break
685 }
686 v.Block.Func.RegArgs = nil
687
688 for i := 0; i < len(args); i++ {
689 a := args[i]
690
691 addr := ssagen.SpillSlotAddr(a, arm64.REGSP, base.Ctxt.Arch.FixedFrameSize)
692
693 if i < len(args)-1 {
694 b := args[i+1]
695 if a.Type.Size() == b.Type.Size() &&
696 a.Type.IsFloat() == b.Type.IsFloat() &&
697 b.Offset == a.Offset+a.Type.Size() {
698 ld := loadByType2(a.Type)
699 st := storeByType2(a.Type)
700 if ld != obj.AXXX && st != obj.AXXX {
701 s.FuncInfo().AddSpill(obj.RegSpill{Reg: a.Reg, Reg2: b.Reg, Addr: addr, Unspill: ld, Spill: st})
702 i++
703 continue
704 }
705 }
706 }
707 reg := a.Reg
708 if a.Type.IsSIMD() && a.Type.Size() == 32 {
709
710 reg = pzreg(reg)
711 }
712
713 s.FuncInfo().AddSpill(obj.RegSpill{Reg: reg, Addr: addr, Unspill: loadByType(a.Type), Spill: storeByType(a.Type)})
714 }
715
716 case ssaop.OpARM64ADD,
717 ssaop.OpARM64SUB,
718 ssaop.OpARM64AND,
719 ssaop.OpARM64OR,
720 ssaop.OpARM64XOR,
721 ssaop.OpARM64BIC,
722 ssaop.OpARM64EON,
723 ssaop.OpARM64ORN,
724 ssaop.OpARM64MUL,
725 ssaop.OpARM64MULW,
726 ssaop.OpARM64MNEG,
727 ssaop.OpARM64MNEGW,
728 ssaop.OpARM64MULH,
729 ssaop.OpARM64UMULH,
730 ssaop.OpARM64MULL,
731 ssaop.OpARM64UMULL,
732 ssaop.OpARM64DIV,
733 ssaop.OpARM64UDIV,
734 ssaop.OpARM64DIVW,
735 ssaop.OpARM64UDIVW,
736 ssaop.OpARM64MOD,
737 ssaop.OpARM64UMOD,
738 ssaop.OpARM64MODW,
739 ssaop.OpARM64UMODW,
740 ssaop.OpARM64SLL,
741 ssaop.OpARM64SRL,
742 ssaop.OpARM64SRA,
743 ssaop.OpARM64FADDS,
744 ssaop.OpARM64FADDD,
745 ssaop.OpARM64FSUBS,
746 ssaop.OpARM64FSUBD,
747 ssaop.OpARM64FMULS,
748 ssaop.OpARM64FMULD,
749 ssaop.OpARM64FNMULS,
750 ssaop.OpARM64FNMULD,
751 ssaop.OpARM64FDIVS,
752 ssaop.OpARM64FDIVD,
753 ssaop.OpARM64FMINS,
754 ssaop.OpARM64FMIND,
755 ssaop.OpARM64FMAXS,
756 ssaop.OpARM64FMAXD,
757 ssaop.OpARM64ROR,
758 ssaop.OpARM64RORW:
759 r := v.Reg()
760 r1 := v.Args[0].Reg()
761 r2 := v.Args[1].Reg()
762 p := s.Prog(v.Op.Asm())
763 p.From.Type = obj.TYPE_REG
764 p.From.Reg = r2
765 p.Reg = r1
766 p.To.Type = obj.TYPE_REG
767 p.To.Reg = r
768 case ssaop.OpARM64ZLDRload:
769
770 p := s.Prog(v.Op.Asm())
771 p.From.Type = obj.TYPE_MEM
772 p.From.Reg = v.Args[0].Reg()
773 p.From.Scale = simdSVEVectorLengthScaled
774 ssagen.AddAux(&p.From, v)
775 p.To.Type = obj.TYPE_REG
776 p.To.Reg = pzreg(v.Reg())
777 case ssaop.OpARM64ZSTRstore:
778
779 p := s.Prog(v.Op.Asm())
780 p.From.Type = obj.TYPE_REG
781 p.From.Reg = pzreg(v.Args[1].Reg())
782 p.To.Type = obj.TYPE_MEM
783 p.To.Reg = v.Args[0].Reg()
784 p.To.Scale = simdSVEVectorLengthScaled
785 ssagen.AddAux(&p.To, v)
786 case ssaop.OpARM64PLDRload:
787
788 p := s.Prog(v.Op.Asm())
789 p.From.Type = obj.TYPE_MEM
790 p.From.Reg = v.Args[0].Reg()
791 p.From.Scale = simdSVEVectorLengthScaled
792 ssagen.AddAux(&p.From, v)
793 p.To.Type = obj.TYPE_REG
794 p.To.Reg = v.Reg()
795 case ssaop.OpARM64PSTRstore:
796
797 p := s.Prog(v.Op.Asm())
798 p.From.Type = obj.TYPE_REG
799 p.From.Reg = v.Args[1].Reg()
800 p.To.Type = obj.TYPE_MEM
801 p.To.Reg = v.Args[0].Reg()
802 p.To.Scale = simdSVEVectorLengthScaled
803 ssagen.AddAux(&p.To, v)
804 case ssaop.OpARM64PPNEXTB:
805 simdPNext(s, v, arm64.ARNG_B)
806 case ssaop.OpARM64PPNEXTH:
807 simdPNext(s, v, arm64.ARNG_H)
808 case ssaop.OpARM64PPNEXTS:
809 simdPNext(s, v, arm64.ARNG_S)
810 case ssaop.OpARM64PPNEXTD:
811 simdPNext(s, v, arm64.ARNG_D)
812 case ssaop.OpARM64PBICSB:
813
814 p := s.Prog(v.Op.Asm())
815 p.From.Type = obj.TYPE_REG
816 p.From.Reg = pregArng(v.Args[2].Reg(), arm64.ARNG_B)
817 p.AddRestSourceReg(pregArng(v.Args[1].Reg(), arm64.ARNG_B))
818 p.AddRestSourceReg(pregMask(v.Args[0].Reg(), arm64.PRED_Z))
819 p.To.Type = obj.TYPE_REG
820 p.To.Reg = pregArng(v.Reg0(), arm64.ARNG_B)
821 case ssaop.OpARM64PPTEST:
822
823 p := s.Prog(v.Op.Asm())
824 p.From.Type = obj.TYPE_REG
825 p.From.Reg = pregArng(v.Args[1].Reg(), arm64.ARNG_B)
826 p.To.Type = obj.TYPE_REG
827 p.To.Reg = v.Args[0].Reg()
828 case ssaop.OpARM64PPFALSEB:
829
830 p := s.Prog(v.Op.Asm())
831 p.To.Type = obj.TYPE_REG
832 p.To.Reg = pregArng(v.Reg(), arm64.ARNG_B)
833 case ssaop.OpARM64ZDUPBconst:
834
835 simdZDupConst(s, v, arm64.ARNG_B)
836 case ssaop.OpARM64ZDUPHconst:
837 simdZDupConst(s, v, arm64.ARNG_H)
838 case ssaop.OpARM64ZDUPSconst:
839 simdZDupConst(s, v, arm64.ARNG_S)
840 case ssaop.OpARM64ZDUPDconst:
841 simdZDupConst(s, v, arm64.ARNG_D)
842 case ssaop.OpARM64RDVL:
843
844 p := s.Prog(v.Op.Asm())
845 p.From.Type = obj.TYPE_CONST
846 p.From.Offset = v.AuxInt
847 p.To.Type = obj.TYPE_REG
848 p.To.Reg = v.Reg()
849 case ssaop.OpARM64ZDUPB:
850 simdZDupGp(s, v, arm64.ARNG_B)
851 case ssaop.OpARM64ZDUPH:
852 simdZDupGp(s, v, arm64.ARNG_H)
853 case ssaop.OpARM64ZDUPS:
854 simdZDupGp(s, v, arm64.ARNG_S)
855 case ssaop.OpARM64ZDUPD:
856 simdZDupGp(s, v, arm64.ARNG_D)
857 case ssaop.OpARM64ZDUPIB:
858 simdZDupIndexed(s, v, arm64.ARNG_B)
859 case ssaop.OpARM64ZDUPIH:
860 simdZDupIndexed(s, v, arm64.ARNG_H)
861 case ssaop.OpARM64ZDUPIS:
862 simdZDupIndexed(s, v, arm64.ARNG_S)
863 case ssaop.OpARM64ZDUPID:
864 simdZDupIndexed(s, v, arm64.ARNG_D)
865 case ssaop.OpARM64ZCPYBconst:
866 simdZCpyConst(s, v, arm64.ARNG_B)
867 case ssaop.OpARM64ZCPYHconst:
868 simdZCpyConst(s, v, arm64.ARNG_H)
869 case ssaop.OpARM64ZCPYSconst:
870 simdZCpyConst(s, v, arm64.ARNG_S)
871 case ssaop.OpARM64ZCPYDconst:
872 simdZCpyConst(s, v, arm64.ARNG_D)
873 case ssaop.OpARM64ZCPYMergingBconst:
874 simdZCpyMergingConst(s, v, arm64.ARNG_B)
875 case ssaop.OpARM64ZCPYMergingHconst:
876 simdZCpyMergingConst(s, v, arm64.ARNG_H)
877 case ssaop.OpARM64ZCPYMergingSconst:
878 simdZCpyMergingConst(s, v, arm64.ARNG_S)
879 case ssaop.OpARM64ZCPYMergingDconst:
880 simdZCpyMergingConst(s, v, arm64.ARNG_D)
881 case ssaop.OpARM64ZCPYMergingB:
882 simdZCpyMergingGp(s, v, arm64.ARNG_B)
883 case ssaop.OpARM64ZCPYMergingH:
884 simdZCpyMergingGp(s, v, arm64.ARNG_H)
885 case ssaop.OpARM64ZCPYMergingS:
886 simdZCpyMergingGp(s, v, arm64.ARNG_S)
887 case ssaop.OpARM64ZCPYMergingD:
888 simdZCpyMergingGp(s, v, arm64.ARNG_D)
889 case ssaop.OpARM64ZCPYIMergingB:
890 simdZCpyMergingVec(s, v, arm64.ARNG_B)
891 case ssaop.OpARM64ZCPYIMergingH:
892 simdZCpyMergingVec(s, v, arm64.ARNG_H)
893 case ssaop.OpARM64ZCPYIMergingS:
894 simdZCpyMergingVec(s, v, arm64.ARNG_S)
895 case ssaop.OpARM64ZCPYIMergingD:
896 simdZCpyMergingVec(s, v, arm64.ARNG_D)
897 case ssaop.OpARM64ZSELB:
898 simdZ2kv(s, v, arm64.ARNG_B)
899 case ssaop.OpARM64ZSELH:
900 simdZ2kv(s, v, arm64.ARNG_H)
901 case ssaop.OpARM64ZSELS:
902 simdZ2kv(s, v, arm64.ARNG_S)
903 case ssaop.OpARM64ZSELD:
904 simdZ2kv(s, v, arm64.ARNG_D)
905 case ssaop.OpARM64PWHILELTB:
906 simdPWHILELT(s, v, arm64.ARNG_B)
907 case ssaop.OpARM64PWHILELTH:
908 simdPWHILELT(s, v, arm64.ARNG_H)
909 case ssaop.OpARM64PWHILELTS:
910 simdPWHILELT(s, v, arm64.ARNG_S)
911 case ssaop.OpARM64PWHILELTD:
912 simdPWHILELT(s, v, arm64.ARNG_D)
913 case ssaop.OpARM64ZLD1BPredload:
914
915
916 p := s.Prog(v.Op.Asm())
917 p.From.Type = obj.TYPE_MEM
918 p.From.Reg = v.Args[0].Reg()
919 p.From.Scale = simdSVEVectorLengthScaled
920 ssagen.AddAux(&p.From, v)
921 p.AddRestSourceReg(pregMask(v.Args[1].Reg(), arm64.PRED_Z))
922 p.To.Type = obj.TYPE_REGLIST
923 p.To.Offset, _ = arm64.RegisterListOffset(int(pzreg(v.Reg())), 1, regListArr("Z", "B"), 0)
924 case ssaop.OpARM64ZST1BPredstore:
925
926
927 p := s.Prog(v.Op.Asm())
928 p.From.Type = obj.TYPE_REGLIST
929 p.From.Offset, _ = arm64.RegisterListOffset(int(pzreg(v.Args[1].Reg())), 1, regListArr("Z", "B"), 0)
930 p.AddRestSourceReg(v.Args[2].Reg())
931 p.To.Type = obj.TYPE_MEM
932 p.To.Reg = v.Args[0].Reg()
933 p.To.Scale = simdSVEVectorLengthScaled
934 ssagen.AddAux(&p.To, v)
935 case ssaop.OpARM64FMADDS,
936 ssaop.OpARM64FMADDD,
937 ssaop.OpARM64FNMADDS,
938 ssaop.OpARM64FNMADDD,
939 ssaop.OpARM64FMSUBS,
940 ssaop.OpARM64FMSUBD,
941 ssaop.OpARM64FNMSUBS,
942 ssaop.OpARM64FNMSUBD,
943 ssaop.OpARM64MADD,
944 ssaop.OpARM64MADDW,
945 ssaop.OpARM64MSUB,
946 ssaop.OpARM64MSUBW:
947 rt := v.Reg()
948 ra := v.Args[0].Reg()
949 rm := v.Args[1].Reg()
950 rn := v.Args[2].Reg()
951 p := s.Prog(v.Op.Asm())
952 p.Reg = ra
953 p.From.Type = obj.TYPE_REG
954 p.From.Reg = rm
955 p.AddRestSourceReg(rn)
956 p.To.Type = obj.TYPE_REG
957 p.To.Reg = rt
958 case ssaop.OpARM64ADDconst,
959 ssaop.OpARM64SUBconst,
960 ssaop.OpARM64ANDconst,
961 ssaop.OpARM64ORconst,
962 ssaop.OpARM64XORconst,
963 ssaop.OpARM64SLLconst,
964 ssaop.OpARM64SRLconst,
965 ssaop.OpARM64SRAconst,
966 ssaop.OpARM64RORconst,
967 ssaop.OpARM64RORWconst:
968 p := s.Prog(v.Op.Asm())
969 p.From.Type = obj.TYPE_CONST
970 p.From.Offset = v.AuxInt
971 p.Reg = v.Args[0].Reg()
972 p.To.Type = obj.TYPE_REG
973 p.To.Reg = v.Reg()
974 case ssaop.OpARM64ADDSconstflags:
975 p := s.Prog(v.Op.Asm())
976 p.From.Type = obj.TYPE_CONST
977 p.From.Offset = v.AuxInt
978 p.Reg = v.Args[0].Reg()
979 p.To.Type = obj.TYPE_REG
980 p.To.Reg = v.Reg0()
981 case ssaop.OpARM64ADCzerocarry:
982 p := s.Prog(v.Op.Asm())
983 p.From.Type = obj.TYPE_REG
984 p.From.Reg = arm64.REGZERO
985 p.Reg = arm64.REGZERO
986 p.To.Type = obj.TYPE_REG
987 p.To.Reg = v.Reg()
988 case ssaop.OpARM64ADCSflags,
989 ssaop.OpARM64ADDSflags,
990 ssaop.OpARM64SBCSflags,
991 ssaop.OpARM64SUBSflags:
992 r := v.Reg0()
993 r1 := v.Args[0].Reg()
994 r2 := v.Args[1].Reg()
995 p := s.Prog(v.Op.Asm())
996 p.From.Type = obj.TYPE_REG
997 p.From.Reg = r2
998 p.Reg = r1
999 p.To.Type = obj.TYPE_REG
1000 p.To.Reg = r
1001 case ssaop.OpARM64NEGSflags:
1002 p := s.Prog(v.Op.Asm())
1003 p.From.Type = obj.TYPE_REG
1004 p.From.Reg = v.Args[0].Reg()
1005 p.To.Type = obj.TYPE_REG
1006 p.To.Reg = v.Reg0()
1007 case ssaop.OpARM64NGCzerocarry:
1008 p := s.Prog(v.Op.Asm())
1009 p.From.Type = obj.TYPE_REG
1010 p.From.Reg = arm64.REGZERO
1011 p.To.Type = obj.TYPE_REG
1012 p.To.Reg = v.Reg()
1013 case ssaop.OpARM64EXTRconst,
1014 ssaop.OpARM64EXTRWconst:
1015 p := s.Prog(v.Op.Asm())
1016 p.From.Type = obj.TYPE_CONST
1017 p.From.Offset = v.AuxInt
1018 p.AddRestSourceReg(v.Args[0].Reg())
1019 p.Reg = v.Args[1].Reg()
1020 p.To.Type = obj.TYPE_REG
1021 p.To.Reg = v.Reg()
1022 case ssaop.OpARM64MVNshiftLL, ssaop.OpARM64NEGshiftLL:
1023 genshift(s, v, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_LL, v.AuxInt)
1024 case ssaop.OpARM64MVNshiftRL, ssaop.OpARM64NEGshiftRL:
1025 genshift(s, v, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_LR, v.AuxInt)
1026 case ssaop.OpARM64MVNshiftRA, ssaop.OpARM64NEGshiftRA:
1027 genshift(s, v, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_AR, v.AuxInt)
1028 case ssaop.OpARM64MVNshiftRO:
1029 genshift(s, v, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_ROR, v.AuxInt)
1030 case ssaop.OpARM64ADDshiftLL,
1031 ssaop.OpARM64SUBshiftLL,
1032 ssaop.OpARM64ANDshiftLL,
1033 ssaop.OpARM64ORshiftLL,
1034 ssaop.OpARM64XORshiftLL,
1035 ssaop.OpARM64EONshiftLL,
1036 ssaop.OpARM64ORNshiftLL,
1037 ssaop.OpARM64BICshiftLL:
1038 genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_LL, v.AuxInt)
1039 case ssaop.OpARM64ADDshiftRL,
1040 ssaop.OpARM64SUBshiftRL,
1041 ssaop.OpARM64ANDshiftRL,
1042 ssaop.OpARM64ORshiftRL,
1043 ssaop.OpARM64XORshiftRL,
1044 ssaop.OpARM64EONshiftRL,
1045 ssaop.OpARM64ORNshiftRL,
1046 ssaop.OpARM64BICshiftRL:
1047 genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_LR, v.AuxInt)
1048 case ssaop.OpARM64ADDshiftRA,
1049 ssaop.OpARM64SUBshiftRA,
1050 ssaop.OpARM64ANDshiftRA,
1051 ssaop.OpARM64ORshiftRA,
1052 ssaop.OpARM64XORshiftRA,
1053 ssaop.OpARM64EONshiftRA,
1054 ssaop.OpARM64ORNshiftRA,
1055 ssaop.OpARM64BICshiftRA:
1056 genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_AR, v.AuxInt)
1057 case ssaop.OpARM64ANDshiftRO,
1058 ssaop.OpARM64ORshiftRO,
1059 ssaop.OpARM64XORshiftRO,
1060 ssaop.OpARM64EONshiftRO,
1061 ssaop.OpARM64ORNshiftRO,
1062 ssaop.OpARM64BICshiftRO:
1063 genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_ROR, v.AuxInt)
1064 case ssaop.OpARM64MOVDconst:
1065 p := s.Prog(v.Op.Asm())
1066 p.From.Type = obj.TYPE_CONST
1067 p.From.Offset = v.AuxInt
1068 p.To.Type = obj.TYPE_REG
1069 p.To.Reg = v.Reg()
1070 case ssaop.OpARM64FMOVSconst,
1071 ssaop.OpARM64FMOVDconst:
1072 p := s.Prog(v.Op.Asm())
1073 p.From.Type = obj.TYPE_FCONST
1074 p.From.Val = math.Float64frombits(uint64(v.AuxInt))
1075 p.To.Type = obj.TYPE_REG
1076 p.To.Reg = v.Reg()
1077 case ssaop.OpARM64FCMPS0,
1078 ssaop.OpARM64FCMPD0:
1079 p := s.Prog(v.Op.Asm())
1080 p.From.Type = obj.TYPE_FCONST
1081 p.From.Val = math.Float64frombits(0)
1082 p.Reg = v.Args[0].Reg()
1083 case ssaop.OpARM64CMP,
1084 ssaop.OpARM64CMPW,
1085 ssaop.OpARM64CMN,
1086 ssaop.OpARM64CMNW,
1087 ssaop.OpARM64TST,
1088 ssaop.OpARM64TSTW,
1089 ssaop.OpARM64FCMPS,
1090 ssaop.OpARM64FCMPD:
1091 p := s.Prog(v.Op.Asm())
1092 p.From.Type = obj.TYPE_REG
1093 p.From.Reg = v.Args[1].Reg()
1094 p.Reg = v.Args[0].Reg()
1095 case ssaop.OpARM64CMPconst,
1096 ssaop.OpARM64CMPWconst,
1097 ssaop.OpARM64CMNconst,
1098 ssaop.OpARM64CMNWconst,
1099 ssaop.OpARM64TSTconst,
1100 ssaop.OpARM64TSTWconst:
1101 p := s.Prog(v.Op.Asm())
1102 p.From.Type = obj.TYPE_CONST
1103 p.From.Offset = v.AuxInt
1104 p.Reg = v.Args[0].Reg()
1105 case ssaop.OpARM64CMPshiftLL, ssaop.OpARM64CMNshiftLL, ssaop.OpARM64TSTshiftLL:
1106 genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_LL, v.AuxInt)
1107 case ssaop.OpARM64CMPshiftRL, ssaop.OpARM64CMNshiftRL, ssaop.OpARM64TSTshiftRL:
1108 genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_LR, v.AuxInt)
1109 case ssaop.OpARM64CMPshiftRA, ssaop.OpARM64CMNshiftRA, ssaop.OpARM64TSTshiftRA:
1110 genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_AR, v.AuxInt)
1111 case ssaop.OpARM64TSTshiftRO:
1112 genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_ROR, v.AuxInt)
1113 case ssaop.OpARM64MOVDaddr:
1114 p := s.Prog(arm64.AMOVD)
1115 p.From.Type = obj.TYPE_ADDR
1116 p.From.Reg = v.Args[0].Reg()
1117 p.To.Type = obj.TYPE_REG
1118 p.To.Reg = v.Reg()
1119
1120 var wantreg string
1121
1122
1123
1124
1125
1126 switch v.Aux.(type) {
1127 default:
1128 v.Fatalf("aux is of unknown type %T", v.Aux)
1129 case *obj.LSym:
1130 wantreg = "SB"
1131 ssagen.AddAux(&p.From, v)
1132 case *ir.Name:
1133 wantreg = "SP"
1134 ssagen.AddAux(&p.From, v)
1135 case nil:
1136
1137 wantreg = "SP"
1138 p.From.Offset = v.AuxInt
1139 }
1140 if reg := v.Args[0].RegName(); reg != wantreg {
1141 v.Fatalf("bad reg %s for symbol type %T, want %s", reg, v.Aux, wantreg)
1142 }
1143 case ssaop.OpARM64MOVBload,
1144 ssaop.OpARM64MOVBUload,
1145 ssaop.OpARM64MOVHload,
1146 ssaop.OpARM64MOVHUload,
1147 ssaop.OpARM64MOVWload,
1148 ssaop.OpARM64MOVWUload,
1149 ssaop.OpARM64MOVDload,
1150 ssaop.OpARM64FMOVSload,
1151 ssaop.OpARM64FMOVDload,
1152 ssaop.OpARM64FMOVQload:
1153 p := s.Prog(v.Op.Asm())
1154 p.From.Type = obj.TYPE_MEM
1155 p.From.Reg = v.Args[0].Reg()
1156 ssagen.AddAux(&p.From, v)
1157 p.To.Type = obj.TYPE_REG
1158 p.To.Reg = v.Reg()
1159 case ssaop.OpARM64LDP, ssaop.OpARM64LDPW, ssaop.OpARM64LDPSW, ssaop.OpARM64FLDPD, ssaop.OpARM64FLDPS, ssaop.OpARM64FLDPQ:
1160 p := s.Prog(v.Op.Asm())
1161 p.From.Type = obj.TYPE_MEM
1162 p.From.Reg = v.Args[0].Reg()
1163 ssagen.AddAux(&p.From, v)
1164 p.To.Type = obj.TYPE_REGREG
1165 p.To.Reg = v.Reg0()
1166 p.To.Offset = int64(v.Reg1())
1167 case ssaop.OpARM64MOVBloadidx,
1168 ssaop.OpARM64MOVBUloadidx,
1169 ssaop.OpARM64MOVHloadidx,
1170 ssaop.OpARM64MOVHUloadidx,
1171 ssaop.OpARM64MOVWloadidx,
1172 ssaop.OpARM64MOVWUloadidx,
1173 ssaop.OpARM64MOVDloadidx,
1174 ssaop.OpARM64FMOVSloadidx,
1175 ssaop.OpARM64FMOVDloadidx,
1176 ssaop.OpARM64MOVHloadidx2,
1177 ssaop.OpARM64MOVHUloadidx2,
1178 ssaop.OpARM64MOVWloadidx4,
1179 ssaop.OpARM64MOVWUloadidx4,
1180 ssaop.OpARM64MOVDloadidx8,
1181 ssaop.OpARM64FMOVDloadidx8,
1182 ssaop.OpARM64FMOVSloadidx4:
1183 p := s.Prog(v.Op.Asm())
1184 p.From = genIndexedOperand(v.Op, v.Args[0].Reg(), v.Args[1].Reg())
1185 p.To.Type = obj.TYPE_REG
1186 p.To.Reg = v.Reg()
1187 case ssaop.OpARM64LDAR,
1188 ssaop.OpARM64LDARB,
1189 ssaop.OpARM64LDARW:
1190 p := s.Prog(v.Op.Asm())
1191 p.From.Type = obj.TYPE_MEM
1192 p.From.Reg = v.Args[0].Reg()
1193 ssagen.AddAux(&p.From, v)
1194 p.To.Type = obj.TYPE_REG
1195 p.To.Reg = v.Reg0()
1196 case ssaop.OpARM64MOVBstore,
1197 ssaop.OpARM64MOVHstore,
1198 ssaop.OpARM64MOVWstore,
1199 ssaop.OpARM64MOVDstore,
1200 ssaop.OpARM64FMOVSstore,
1201 ssaop.OpARM64FMOVDstore,
1202 ssaop.OpARM64FMOVQstore,
1203 ssaop.OpARM64STLRB,
1204 ssaop.OpARM64STLR,
1205 ssaop.OpARM64STLRW:
1206 p := s.Prog(v.Op.Asm())
1207 p.From.Type = obj.TYPE_REG
1208 p.From.Reg = v.Args[1].Reg()
1209 p.To.Type = obj.TYPE_MEM
1210 p.To.Reg = v.Args[0].Reg()
1211 ssagen.AddAux(&p.To, v)
1212 case ssaop.OpARM64MOVBstoreidx,
1213 ssaop.OpARM64MOVHstoreidx,
1214 ssaop.OpARM64MOVWstoreidx,
1215 ssaop.OpARM64MOVDstoreidx,
1216 ssaop.OpARM64FMOVSstoreidx,
1217 ssaop.OpARM64FMOVDstoreidx,
1218 ssaop.OpARM64MOVHstoreidx2,
1219 ssaop.OpARM64MOVWstoreidx4,
1220 ssaop.OpARM64FMOVSstoreidx4,
1221 ssaop.OpARM64MOVDstoreidx8,
1222 ssaop.OpARM64FMOVDstoreidx8:
1223 p := s.Prog(v.Op.Asm())
1224 p.To = genIndexedOperand(v.Op, v.Args[0].Reg(), v.Args[1].Reg())
1225 p.From.Type = obj.TYPE_REG
1226 p.From.Reg = v.Args[2].Reg()
1227 case ssaop.OpARM64STP, ssaop.OpARM64STPW, ssaop.OpARM64FSTPD, ssaop.OpARM64FSTPS, ssaop.OpARM64FSTPQ:
1228 p := s.Prog(v.Op.Asm())
1229 p.From.Type = obj.TYPE_REGREG
1230 p.From.Reg = v.Args[1].Reg()
1231 p.From.Offset = int64(v.Args[2].Reg())
1232 p.To.Type = obj.TYPE_MEM
1233 p.To.Reg = v.Args[0].Reg()
1234 ssagen.AddAux(&p.To, v)
1235 case ssaop.OpARM64BFI,
1236 ssaop.OpARM64BFXIL:
1237 p := s.Prog(v.Op.Asm())
1238 p.From.Type = obj.TYPE_CONST
1239 p.From.Offset = v.AuxInt >> 8
1240 p.AddRestSourceConst(v.AuxInt & 0xff)
1241 p.Reg = v.Args[1].Reg()
1242 p.To.Type = obj.TYPE_REG
1243 p.To.Reg = v.Reg()
1244 case ssaop.OpARM64SBFIZ,
1245 ssaop.OpARM64SBFX,
1246 ssaop.OpARM64UBFIZ,
1247 ssaop.OpARM64UBFX:
1248 p := s.Prog(v.Op.Asm())
1249 p.From.Type = obj.TYPE_CONST
1250 p.From.Offset = v.AuxInt >> 8
1251 p.AddRestSourceConst(v.AuxInt & 0xff)
1252 p.Reg = v.Args[0].Reg()
1253 p.To.Type = obj.TYPE_REG
1254 p.To.Reg = v.Reg()
1255 case ssaop.OpARM64LoweredAtomicExchange64,
1256 ssaop.OpARM64LoweredAtomicExchange32,
1257 ssaop.OpARM64LoweredAtomicExchange8:
1258
1259
1260
1261
1262
1263 var ld, st obj.As
1264 switch v.Op {
1265 case ssaop.OpARM64LoweredAtomicExchange8:
1266 ld = arm64.ALDAXRB
1267 st = arm64.ASTLXRB
1268 case ssaop.OpARM64LoweredAtomicExchange32:
1269 ld = arm64.ALDAXRW
1270 st = arm64.ASTLXRW
1271 case ssaop.OpARM64LoweredAtomicExchange64:
1272 ld = arm64.ALDAXR
1273 st = arm64.ASTLXR
1274 }
1275 r0 := v.Args[0].Reg()
1276 r1 := v.Args[1].Reg()
1277 out := v.Reg0()
1278 p := s.Prog(ld)
1279 p.From.Type = obj.TYPE_MEM
1280 p.From.Reg = r0
1281 p.To.Type = obj.TYPE_REG
1282 p.To.Reg = out
1283 p1 := s.Prog(st)
1284 p1.From.Type = obj.TYPE_REG
1285 p1.From.Reg = r1
1286 p1.To.Type = obj.TYPE_MEM
1287 p1.To.Reg = r0
1288 p1.RegTo2 = arm64.REGTMP
1289 p2 := s.Prog(arm64.ACBNZ)
1290 p2.From.Type = obj.TYPE_REG
1291 p2.From.Reg = arm64.REGTMP
1292 p2.To.Type = obj.TYPE_BRANCH
1293 p2.To.SetTarget(p)
1294 case ssaop.OpARM64LoweredAtomicExchange64Variant,
1295 ssaop.OpARM64LoweredAtomicExchange32Variant,
1296 ssaop.OpARM64LoweredAtomicExchange8Variant:
1297
1298 var swap obj.As
1299 switch v.Op {
1300 case ssaop.OpARM64LoweredAtomicExchange8Variant:
1301 swap = arm64.ASWPALB
1302 case ssaop.OpARM64LoweredAtomicExchange32Variant:
1303 swap = arm64.ASWPALW
1304 case ssaop.OpARM64LoweredAtomicExchange64Variant:
1305 swap = arm64.ASWPALD
1306 }
1307 r0 := v.Args[0].Reg()
1308 r1 := v.Args[1].Reg()
1309 out := v.Reg0()
1310
1311
1312 p := s.Prog(swap)
1313 p.From.Type = obj.TYPE_REG
1314 p.From.Reg = r1
1315 p.To.Type = obj.TYPE_MEM
1316 p.To.Reg = r0
1317 p.RegTo2 = out
1318
1319 case ssaop.OpARM64LoweredAtomicAdd64,
1320 ssaop.OpARM64LoweredAtomicAdd32:
1321
1322
1323
1324
1325 ld := arm64.ALDAXR
1326 st := arm64.ASTLXR
1327 if v.Op == ssaop.OpARM64LoweredAtomicAdd32 {
1328 ld = arm64.ALDAXRW
1329 st = arm64.ASTLXRW
1330 }
1331 r0 := v.Args[0].Reg()
1332 r1 := v.Args[1].Reg()
1333 out := v.Reg0()
1334 p := s.Prog(ld)
1335 p.From.Type = obj.TYPE_MEM
1336 p.From.Reg = r0
1337 p.To.Type = obj.TYPE_REG
1338 p.To.Reg = out
1339 p1 := s.Prog(arm64.AADD)
1340 p1.From.Type = obj.TYPE_REG
1341 p1.From.Reg = r1
1342 p1.To.Type = obj.TYPE_REG
1343 p1.To.Reg = out
1344 p2 := s.Prog(st)
1345 p2.From.Type = obj.TYPE_REG
1346 p2.From.Reg = out
1347 p2.To.Type = obj.TYPE_MEM
1348 p2.To.Reg = r0
1349 p2.RegTo2 = arm64.REGTMP
1350 p3 := s.Prog(arm64.ACBNZ)
1351 p3.From.Type = obj.TYPE_REG
1352 p3.From.Reg = arm64.REGTMP
1353 p3.To.Type = obj.TYPE_BRANCH
1354 p3.To.SetTarget(p)
1355 case ssaop.OpARM64LoweredAtomicAdd64Variant,
1356 ssaop.OpARM64LoweredAtomicAdd32Variant:
1357
1358
1359 op := arm64.ALDADDALD
1360 if v.Op == ssaop.OpARM64LoweredAtomicAdd32Variant {
1361 op = arm64.ALDADDALW
1362 }
1363 r0 := v.Args[0].Reg()
1364 r1 := v.Args[1].Reg()
1365 out := v.Reg0()
1366 p := s.Prog(op)
1367 p.From.Type = obj.TYPE_REG
1368 p.From.Reg = r1
1369 p.To.Type = obj.TYPE_MEM
1370 p.To.Reg = r0
1371 p.RegTo2 = out
1372 p1 := s.Prog(arm64.AADD)
1373 p1.From.Type = obj.TYPE_REG
1374 p1.From.Reg = r1
1375 p1.To.Type = obj.TYPE_REG
1376 p1.To.Reg = out
1377 case ssaop.OpARM64LoweredAtomicCas64,
1378 ssaop.OpARM64LoweredAtomicCas32:
1379
1380
1381
1382
1383
1384
1385
1386
1387 ld := arm64.ALDAXR
1388 st := arm64.ASTLXR
1389 cmp := arm64.ACMP
1390 if v.Op == ssaop.OpARM64LoweredAtomicCas32 {
1391 ld = arm64.ALDAXRW
1392 st = arm64.ASTLXRW
1393 cmp = arm64.ACMPW
1394 }
1395 r0 := v.Args[0].Reg()
1396 r1 := v.Args[1].Reg()
1397 r2 := v.Args[2].Reg()
1398 out := v.Reg0()
1399 p := s.Prog(ld)
1400 p.From.Type = obj.TYPE_MEM
1401 p.From.Reg = r0
1402 p.To.Type = obj.TYPE_REG
1403 p.To.Reg = arm64.REGTMP
1404 p1 := s.Prog(cmp)
1405 p1.From.Type = obj.TYPE_REG
1406 p1.From.Reg = r1
1407 p1.Reg = arm64.REGTMP
1408 p2 := s.Prog(arm64.ABNE)
1409 p2.To.Type = obj.TYPE_BRANCH
1410 p3 := s.Prog(st)
1411 p3.From.Type = obj.TYPE_REG
1412 p3.From.Reg = r2
1413 p3.To.Type = obj.TYPE_MEM
1414 p3.To.Reg = r0
1415 p3.RegTo2 = arm64.REGTMP
1416 p4 := s.Prog(arm64.ACBNZ)
1417 p4.From.Type = obj.TYPE_REG
1418 p4.From.Reg = arm64.REGTMP
1419 p4.To.Type = obj.TYPE_BRANCH
1420 p4.To.SetTarget(p)
1421 p5 := s.Prog(arm64.ACSET)
1422 p5.From.Type = obj.TYPE_SPECIAL
1423 p5.From.Offset = int64(arm64.SPOP_EQ)
1424 p5.To.Type = obj.TYPE_REG
1425 p5.To.Reg = out
1426 p2.To.SetTarget(p5)
1427 case ssaop.OpARM64LoweredAtomicCas64Variant,
1428 ssaop.OpARM64LoweredAtomicCas32Variant:
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438 cas := arm64.ACASALD
1439 cmp := arm64.ACMP
1440 mov := arm64.AMOVD
1441 if v.Op == ssaop.OpARM64LoweredAtomicCas32Variant {
1442 cas = arm64.ACASALW
1443 cmp = arm64.ACMPW
1444 mov = arm64.AMOVW
1445 }
1446 r0 := v.Args[0].Reg()
1447 r1 := v.Args[1].Reg()
1448 r2 := v.Args[2].Reg()
1449 out := v.Reg0()
1450
1451
1452 p := s.Prog(mov)
1453 p.From.Type = obj.TYPE_REG
1454 p.From.Reg = r1
1455 p.To.Type = obj.TYPE_REG
1456 p.To.Reg = arm64.REGTMP
1457
1458
1459 p1 := s.Prog(cas)
1460 p1.From.Type = obj.TYPE_REG
1461 p1.From.Reg = arm64.REGTMP
1462 p1.To.Type = obj.TYPE_MEM
1463 p1.To.Reg = r0
1464 p1.RegTo2 = r2
1465
1466
1467 p2 := s.Prog(cmp)
1468 p2.From.Type = obj.TYPE_REG
1469 p2.From.Reg = r1
1470 p2.Reg = arm64.REGTMP
1471
1472
1473 p3 := s.Prog(arm64.ACSET)
1474 p3.From.Type = obj.TYPE_SPECIAL
1475 p3.From.Offset = int64(arm64.SPOP_EQ)
1476 p3.To.Type = obj.TYPE_REG
1477 p3.To.Reg = out
1478
1479 case ssaop.OpARM64LoweredAtomicAnd64,
1480 ssaop.OpARM64LoweredAtomicOr64,
1481 ssaop.OpARM64LoweredAtomicAnd32,
1482 ssaop.OpARM64LoweredAtomicOr32,
1483 ssaop.OpARM64LoweredAtomicAnd8,
1484 ssaop.OpARM64LoweredAtomicOr8:
1485
1486
1487
1488
1489
1490
1491 ld := arm64.ALDAXR
1492 st := arm64.ASTLXR
1493 if v.Op == ssaop.OpARM64LoweredAtomicAnd32 || v.Op == ssaop.OpARM64LoweredAtomicOr32 {
1494 ld = arm64.ALDAXRW
1495 st = arm64.ASTLXRW
1496 }
1497 if v.Op == ssaop.OpARM64LoweredAtomicAnd8 || v.Op == ssaop.OpARM64LoweredAtomicOr8 {
1498 ld = arm64.ALDAXRB
1499 st = arm64.ASTLXRB
1500 }
1501 r0 := v.Args[0].Reg()
1502 r1 := v.Args[1].Reg()
1503 out := v.Reg0()
1504 tmp := v.RegTmp()
1505 p := s.Prog(ld)
1506 p.From.Type = obj.TYPE_MEM
1507 p.From.Reg = r0
1508 p.To.Type = obj.TYPE_REG
1509 p.To.Reg = out
1510 p1 := s.Prog(v.Op.Asm())
1511 p1.From.Type = obj.TYPE_REG
1512 p1.From.Reg = r1
1513 p1.Reg = out
1514 p1.To.Type = obj.TYPE_REG
1515 p1.To.Reg = tmp
1516 p2 := s.Prog(st)
1517 p2.From.Type = obj.TYPE_REG
1518 p2.From.Reg = tmp
1519 p2.To.Type = obj.TYPE_MEM
1520 p2.To.Reg = r0
1521 p2.RegTo2 = arm64.REGTMP
1522 p3 := s.Prog(arm64.ACBNZ)
1523 p3.From.Type = obj.TYPE_REG
1524 p3.From.Reg = arm64.REGTMP
1525 p3.To.Type = obj.TYPE_BRANCH
1526 p3.To.SetTarget(p)
1527
1528 case ssaop.OpARM64LoweredAtomicAnd8Variant,
1529 ssaop.OpARM64LoweredAtomicAnd32Variant,
1530 ssaop.OpARM64LoweredAtomicAnd64Variant:
1531
1532 atomic_clear := arm64.ALDCLRALD
1533 if v.Op == ssaop.OpARM64LoweredAtomicAnd32Variant {
1534 atomic_clear = arm64.ALDCLRALW
1535 }
1536 if v.Op == ssaop.OpARM64LoweredAtomicAnd8Variant {
1537 atomic_clear = arm64.ALDCLRALB
1538 }
1539 r0 := v.Args[0].Reg()
1540 r1 := v.Args[1].Reg()
1541 out := v.Reg0()
1542
1543
1544 p := s.Prog(arm64.AMVN)
1545 p.From.Type = obj.TYPE_REG
1546 p.From.Reg = r1
1547 p.To.Type = obj.TYPE_REG
1548 p.To.Reg = arm64.REGTMP
1549
1550
1551 p1 := s.Prog(atomic_clear)
1552 p1.From.Type = obj.TYPE_REG
1553 p1.From.Reg = arm64.REGTMP
1554 p1.To.Type = obj.TYPE_MEM
1555 p1.To.Reg = r0
1556 p1.RegTo2 = out
1557
1558 case ssaop.OpARM64LoweredAtomicOr8Variant,
1559 ssaop.OpARM64LoweredAtomicOr32Variant,
1560 ssaop.OpARM64LoweredAtomicOr64Variant:
1561
1562 atomic_or := arm64.ALDORALD
1563 if v.Op == ssaop.OpARM64LoweredAtomicOr32Variant {
1564 atomic_or = arm64.ALDORALW
1565 }
1566 if v.Op == ssaop.OpARM64LoweredAtomicOr8Variant {
1567 atomic_or = arm64.ALDORALB
1568 }
1569 r0 := v.Args[0].Reg()
1570 r1 := v.Args[1].Reg()
1571 out := v.Reg0()
1572
1573
1574 p := s.Prog(atomic_or)
1575 p.From.Type = obj.TYPE_REG
1576 p.From.Reg = r1
1577 p.To.Type = obj.TYPE_MEM
1578 p.To.Reg = r0
1579 p.RegTo2 = out
1580
1581 case ssaop.OpARM64MOVBreg,
1582 ssaop.OpARM64MOVBUreg,
1583 ssaop.OpARM64MOVHreg,
1584 ssaop.OpARM64MOVHUreg,
1585 ssaop.OpARM64MOVWreg,
1586 ssaop.OpARM64MOVWUreg:
1587 a := v.Args[0]
1588 for a.Op == ssaop.OpCopy || a.Op == ssaop.OpARM64MOVDreg {
1589 a = a.Args[0]
1590 }
1591 if a.Op == ssaop.OpLoadReg {
1592 t := a.Type
1593 switch {
1594 case v.Op == ssaop.OpARM64MOVBreg && t.Size() == 1 && t.IsSigned(),
1595 v.Op == ssaop.OpARM64MOVBUreg && t.Size() == 1 && !t.IsSigned(),
1596 v.Op == ssaop.OpARM64MOVHreg && t.Size() == 2 && t.IsSigned(),
1597 v.Op == ssaop.OpARM64MOVHUreg && t.Size() == 2 && !t.IsSigned(),
1598 v.Op == ssaop.OpARM64MOVWreg && t.Size() == 4 && t.IsSigned(),
1599 v.Op == ssaop.OpARM64MOVWUreg && t.Size() == 4 && !t.IsSigned():
1600
1601 if v.Reg() == v.Args[0].Reg() {
1602 return
1603 }
1604 p := s.Prog(arm64.AMOVD)
1605 p.From.Type = obj.TYPE_REG
1606 p.From.Reg = v.Args[0].Reg()
1607 p.To.Type = obj.TYPE_REG
1608 p.To.Reg = v.Reg()
1609 return
1610 default:
1611 }
1612 }
1613 fallthrough
1614 case ssaop.OpARM64MVN,
1615 ssaop.OpARM64NEG,
1616 ssaop.OpARM64FABSD,
1617 ssaop.OpARM64FABSS,
1618 ssaop.OpARM64FMOVDfpgp,
1619 ssaop.OpARM64FMOVDgpfp,
1620 ssaop.OpARM64FMOVSfpgp,
1621 ssaop.OpARM64FMOVSgpfp,
1622 ssaop.OpARM64FNEGS,
1623 ssaop.OpARM64FNEGD,
1624 ssaop.OpARM64FSQRTS,
1625 ssaop.OpARM64FSQRTD,
1626 ssaop.OpARM64FCVTZSSW,
1627 ssaop.OpARM64FCVTZSDW,
1628 ssaop.OpARM64FCVTZUSW,
1629 ssaop.OpARM64FCVTZUDW,
1630 ssaop.OpARM64FCVTZSS,
1631 ssaop.OpARM64FCVTZSD,
1632 ssaop.OpARM64FCVTZUS,
1633 ssaop.OpARM64FCVTZUD,
1634 ssaop.OpARM64SCVTFWS,
1635 ssaop.OpARM64SCVTFWD,
1636 ssaop.OpARM64SCVTFS,
1637 ssaop.OpARM64SCVTFD,
1638 ssaop.OpARM64UCVTFWS,
1639 ssaop.OpARM64UCVTFWD,
1640 ssaop.OpARM64UCVTFS,
1641 ssaop.OpARM64UCVTFD,
1642 ssaop.OpARM64FCVTSD,
1643 ssaop.OpARM64FCVTDS,
1644 ssaop.OpARM64REV,
1645 ssaop.OpARM64REVW,
1646 ssaop.OpARM64REV16,
1647 ssaop.OpARM64REV16W,
1648 ssaop.OpARM64RBIT,
1649 ssaop.OpARM64RBITW,
1650 ssaop.OpARM64CLZ,
1651 ssaop.OpARM64CLZW,
1652 ssaop.OpARM64FRINTAD,
1653 ssaop.OpARM64FRINTMD,
1654 ssaop.OpARM64FRINTND,
1655 ssaop.OpARM64FRINTPD,
1656 ssaop.OpARM64FRINTZD,
1657 ssaop.OpARM64FRINTAS,
1658 ssaop.OpARM64FRINTMS,
1659 ssaop.OpARM64FRINTNS,
1660 ssaop.OpARM64FRINTPS,
1661 ssaop.OpARM64FRINTZS:
1662 p := s.Prog(v.Op.Asm())
1663 p.From.Type = obj.TYPE_REG
1664 p.From.Reg = v.Args[0].Reg()
1665 p.To.Type = obj.TYPE_REG
1666 p.To.Reg = v.Reg()
1667 case ssaop.OpARM64LoweredRound32F, ssaop.OpARM64LoweredRound64F:
1668
1669 case ssaop.OpARM64VCNT:
1670 simdV11(s, v, arm64.ARNG_8B)
1671 case ssaop.OpARM64VUADDLV:
1672 simdV11Scalar(s, v, arm64.ARNG_8B)
1673 case ssaop.OpARM64CSEL, ssaop.OpARM64CSEL0, ssaop.OpARM64FCSELD, ssaop.OpARM64FCSELS:
1674 r1 := int16(arm64.REGZERO)
1675 if v.Op != ssaop.OpARM64CSEL0 {
1676 r1 = v.Args[1].Reg()
1677 }
1678 p := s.Prog(v.Op.Asm())
1679 p.From.Type = obj.TYPE_SPECIAL
1680 condCode := condBits[ssaop.Op(v.AuxInt)]
1681 p.From.Offset = int64(condCode)
1682 p.Reg = v.Args[0].Reg()
1683 p.AddRestSourceReg(r1)
1684 p.To.Type = obj.TYPE_REG
1685 p.To.Reg = v.Reg()
1686 case ssaop.OpARM64CSINC, ssaop.OpARM64CSINV, ssaop.OpARM64CSNEG:
1687 p := s.Prog(v.Op.Asm())
1688 p.From.Type = obj.TYPE_SPECIAL
1689 condCode := condBits[ssaop.Op(v.AuxInt)]
1690 p.From.Offset = int64(condCode)
1691 p.Reg = v.Args[0].Reg()
1692 p.AddRestSourceReg(v.Args[1].Reg())
1693 p.To.Type = obj.TYPE_REG
1694 p.To.Reg = v.Reg()
1695 case ssaop.OpARM64CSETM:
1696 p := s.Prog(arm64.ACSETM)
1697 p.From.Type = obj.TYPE_SPECIAL
1698 condCode := condBits[ssaop.Op(v.AuxInt)]
1699 p.From.Offset = int64(condCode)
1700 p.To.Type = obj.TYPE_REG
1701 p.To.Reg = v.Reg()
1702 case ssaop.OpARM64CCMP,
1703 ssaop.OpARM64CCMN,
1704 ssaop.OpARM64CCMPconst,
1705 ssaop.OpARM64CCMNconst,
1706 ssaop.OpARM64CCMPW,
1707 ssaop.OpARM64CCMNW,
1708 ssaop.OpARM64CCMPWconst,
1709 ssaop.OpARM64CCMNWconst:
1710 p := s.Prog(v.Op.Asm())
1711 p.Reg = v.Args[0].Reg()
1712 params := v.AuxArm64ConditionalParams()
1713 p.From.Type = obj.TYPE_SPECIAL
1714 p.From.Offset = int64(condBits[params.Cond])
1715 constValue, ok := params.ConstValue()
1716 if ok {
1717 p.AddRestSourceConst(constValue)
1718 } else {
1719 p.AddRestSourceReg(v.Args[1].Reg())
1720 }
1721 p.To.Type = obj.TYPE_CONST
1722 p.To.Offset = params.Nzcv()
1723 case ssaop.OpARM64LoweredZero:
1724 ptrReg := v.Args[0].Reg()
1725 n := v.AuxInt
1726 if n < 16 {
1727 v.Fatalf("Zero too small %d", n)
1728 }
1729
1730
1731 var off int64
1732 for n >= 16 {
1733
1734 zero16(s, ptrReg, off, false)
1735 off += 16
1736 n -= 16
1737 }
1738
1739
1740
1741 if n > 8 {
1742
1743 zero8(s, ptrReg, off)
1744 off += 8
1745 n -= 8
1746 }
1747 if n != 0 {
1748
1749
1750 zero8(s, ptrReg, off+n-8)
1751 }
1752 case ssaop.OpARM64LoweredZeroLoop:
1753 ptrReg := v.Args[0].Reg()
1754 countReg := v.RegTmp()
1755 n := v.AuxInt
1756 loopSize := int64(64)
1757 if n < 3*loopSize {
1758
1759
1760
1761
1762
1763
1764
1765
1766 v.Fatalf("ZeroLoop size too small %d", n)
1767 }
1768
1769
1770
1771 p := s.Prog(arm64.AMOVD)
1772 p.From.Type = obj.TYPE_CONST
1773 p.From.Offset = n / loopSize
1774 p.To.Type = obj.TYPE_REG
1775 p.To.Reg = countReg
1776 cntInit := p
1777
1778
1779
1780 for range loopSize / 16 {
1781
1782 zero16(s, ptrReg, 0, true)
1783
1784
1785
1786
1787 }
1788
1789
1790 p = s.Prog(arm64.ASUB)
1791 p.From.Type = obj.TYPE_CONST
1792 p.From.Offset = 1
1793 p.To.Type = obj.TYPE_REG
1794 p.To.Reg = countReg
1795
1796
1797 p = s.Prog(arm64.ACBNZ)
1798 p.From.Type = obj.TYPE_REG
1799 p.From.Reg = countReg
1800 p.To.Type = obj.TYPE_BRANCH
1801 p.To.SetTarget(cntInit.Link)
1802
1803
1804 n %= loopSize
1805
1806
1807 var off int64
1808 for n >= 16 {
1809
1810 zero16(s, ptrReg, off, false)
1811 off += 16
1812 n -= 16
1813 }
1814 if n > 8 {
1815
1816
1817
1818 zero8(s, ptrReg, off)
1819 off += 8
1820 n -= 8
1821 }
1822 if n != 0 {
1823
1824
1825 zero8(s, ptrReg, off+n-8)
1826 }
1827
1828
1829
1830
1831
1832
1833
1834
1835 case ssaop.OpARM64LoweredMove:
1836 dstReg := v.Args[0].Reg()
1837 srcReg := v.Args[1].Reg()
1838 if dstReg == srcReg {
1839 break
1840 }
1841 tmpReg1 := int16(arm64.REG_R25)
1842 tmpFReg1 := int16(arm64.REG_F16)
1843 tmpFReg2 := int16(arm64.REG_F17)
1844 n := v.AuxInt
1845 if n < 16 {
1846 v.Fatalf("Move too small %d", n)
1847 }
1848
1849
1850 var off int64
1851 for n >= 32 {
1852
1853
1854 move32(s, srcReg, dstReg, tmpFReg1, tmpFReg2, off, false)
1855 off += 32
1856 n -= 32
1857 }
1858 for n >= 16 {
1859
1860
1861 move16(s, srcReg, dstReg, tmpFReg1, off, false)
1862 off += 16
1863 n -= 16
1864 }
1865 if n > 8 {
1866
1867
1868 move8(s, srcReg, dstReg, tmpReg1, off)
1869 off += 8
1870 n -= 8
1871 }
1872 if n != 0 {
1873
1874
1875 move8(s, srcReg, dstReg, tmpReg1, off+n-8)
1876 }
1877 case ssaop.OpARM64LoweredMoveLoop:
1878 dstReg := v.Args[0].Reg()
1879 srcReg := v.Args[1].Reg()
1880 if dstReg == srcReg {
1881 break
1882 }
1883 countReg := int16(arm64.REG_R24)
1884 tmpReg1 := int16(arm64.REG_R25)
1885 tmpFReg1 := int16(arm64.REG_F16)
1886 tmpFReg2 := int16(arm64.REG_F17)
1887 n := v.AuxInt
1888 loopSize := int64(64)
1889 if n < 3*loopSize {
1890
1891
1892
1893
1894
1895
1896
1897
1898 v.Fatalf("ZeroLoop size too small %d", n)
1899 }
1900
1901
1902
1903 p := s.Prog(arm64.AMOVD)
1904 p.From.Type = obj.TYPE_CONST
1905 p.From.Offset = n / loopSize
1906 p.To.Type = obj.TYPE_REG
1907 p.To.Reg = countReg
1908 cntInit := p
1909
1910
1911
1912 for range loopSize / 32 {
1913
1914
1915 move32(s, srcReg, dstReg, tmpFReg1, tmpFReg2, 0, true)
1916 }
1917
1918
1919 p = s.Prog(arm64.ASUB)
1920 p.From.Type = obj.TYPE_CONST
1921 p.From.Offset = 1
1922 p.To.Type = obj.TYPE_REG
1923 p.To.Reg = countReg
1924
1925
1926 p = s.Prog(arm64.ACBNZ)
1927 p.From.Type = obj.TYPE_REG
1928 p.From.Reg = countReg
1929 p.To.Type = obj.TYPE_BRANCH
1930 p.To.SetTarget(cntInit.Link)
1931
1932
1933 n %= loopSize
1934
1935
1936 var off int64
1937 for n >= 32 {
1938
1939
1940 move32(s, srcReg, dstReg, tmpFReg1, tmpFReg2, off, false)
1941 off += 32
1942 n -= 32
1943 }
1944 for n >= 16 {
1945
1946
1947 move16(s, srcReg, dstReg, tmpFReg1, off, false)
1948 off += 16
1949 n -= 16
1950 }
1951 if n > 8 {
1952
1953
1954 move8(s, srcReg, dstReg, tmpReg1, off)
1955 off += 8
1956 n -= 8
1957 }
1958 if n != 0 {
1959
1960
1961 move8(s, srcReg, dstReg, tmpReg1, off+n-8)
1962 }
1963
1964 case ssaop.OpARM64CALLstatic, ssaop.OpARM64CALLclosure, ssaop.OpARM64CALLinter:
1965 s.Call(v)
1966 case ssaop.OpARM64CALLtail, ssaop.OpARM64CALLtailinter:
1967 s.TailCall(v)
1968 case ssaop.OpARM64LoweredWB:
1969 p := s.Prog(obj.ACALL)
1970 p.To.Type = obj.TYPE_MEM
1971 p.To.Name = obj.NAME_EXTERN
1972
1973 p.To.Sym = ir.Syms.GCWriteBarrier[v.AuxInt-1]
1974 case ssaop.OpARM64LoweredMemEq:
1975 p := s.Prog(obj.ACALL)
1976 p.To.Type = obj.TYPE_MEM
1977 p.To.Name = obj.NAME_EXTERN
1978 p.To.Sym = ir.Syms.Memequal
1979
1980 case ssaop.OpARM64LoweredPanicBoundsRR, ssaop.OpARM64LoweredPanicBoundsRC, ssaop.OpARM64LoweredPanicBoundsCR, ssaop.OpARM64LoweredPanicBoundsCC:
1981
1982 code, signed := ssa.BoundsKind(v.AuxInt).Code()
1983 xIsReg := false
1984 yIsReg := false
1985 xVal := 0
1986 yVal := 0
1987 switch v.Op {
1988 case ssaop.OpARM64LoweredPanicBoundsRR:
1989 xIsReg = true
1990 xVal = int(v.Args[0].Reg() - arm64.REG_R0)
1991 yIsReg = true
1992 yVal = int(v.Args[1].Reg() - arm64.REG_R0)
1993 case ssaop.OpARM64LoweredPanicBoundsRC:
1994 xIsReg = true
1995 xVal = int(v.Args[0].Reg() - arm64.REG_R0)
1996 c := v.Aux.(ssa.PanicBoundsC).C
1997 if c >= 0 && c <= abi.BoundsMaxConst {
1998 yVal = int(c)
1999 } else {
2000
2001 yIsReg = true
2002 if yVal == xVal {
2003 yVal = 1
2004 }
2005 p := s.Prog(arm64.AMOVD)
2006 p.From.Type = obj.TYPE_CONST
2007 p.From.Offset = c
2008 p.To.Type = obj.TYPE_REG
2009 p.To.Reg = arm64.REG_R0 + int16(yVal)
2010 }
2011 case ssaop.OpARM64LoweredPanicBoundsCR:
2012 yIsReg = true
2013 yVal = int(v.Args[0].Reg() - arm64.REG_R0)
2014 c := v.Aux.(ssa.PanicBoundsC).C
2015 if c >= 0 && c <= abi.BoundsMaxConst {
2016 xVal = int(c)
2017 } else {
2018
2019 xIsReg = true
2020 if xVal == yVal {
2021 xVal = 1
2022 }
2023 p := s.Prog(arm64.AMOVD)
2024 p.From.Type = obj.TYPE_CONST
2025 p.From.Offset = c
2026 p.To.Type = obj.TYPE_REG
2027 p.To.Reg = arm64.REG_R0 + int16(xVal)
2028 }
2029 case ssaop.OpARM64LoweredPanicBoundsCC:
2030 c := v.Aux.(ssa.PanicBoundsCC).Cx
2031 if c >= 0 && c <= abi.BoundsMaxConst {
2032 xVal = int(c)
2033 } else {
2034
2035 xIsReg = true
2036 p := s.Prog(arm64.AMOVD)
2037 p.From.Type = obj.TYPE_CONST
2038 p.From.Offset = c
2039 p.To.Type = obj.TYPE_REG
2040 p.To.Reg = arm64.REG_R0 + int16(xVal)
2041 }
2042 c = v.Aux.(ssa.PanicBoundsCC).Cy
2043 if c >= 0 && c <= abi.BoundsMaxConst {
2044 yVal = int(c)
2045 } else {
2046
2047 yIsReg = true
2048 yVal = 1
2049 p := s.Prog(arm64.AMOVD)
2050 p.From.Type = obj.TYPE_CONST
2051 p.From.Offset = c
2052 p.To.Type = obj.TYPE_REG
2053 p.To.Reg = arm64.REG_R0 + int16(yVal)
2054 }
2055 }
2056 c := abi.BoundsEncode(code, signed, xIsReg, yIsReg, xVal, yVal)
2057
2058 p := s.Prog(obj.APCDATA)
2059 p.From.SetConst(abi.PCDATA_PanicBounds)
2060 p.To.SetConst(int64(c))
2061 p = s.Prog(obj.ACALL)
2062 p.To.Type = obj.TYPE_MEM
2063 p.To.Name = obj.NAME_EXTERN
2064 p.To.Sym = ir.Syms.PanicBounds
2065
2066 case ssaop.OpARM64LoweredNilCheck:
2067
2068 p := s.Prog(arm64.AMOVB)
2069 p.From.Type = obj.TYPE_MEM
2070 p.From.Reg = v.Args[0].Reg()
2071 ssagen.AddAux(&p.From, v)
2072 p.To.Type = obj.TYPE_REG
2073 p.To.Reg = arm64.REGTMP
2074 if logopt.Enabled() {
2075 logopt.LogOpt(v.Pos, "nilcheck", "genssa", v.Block.Func.Name)
2076 }
2077 if base.Debug.Nil != 0 && v.Pos.Line() > 1 {
2078 base.WarnfAt(v.Pos, "generated nil check")
2079 }
2080 case ssaop.OpARM64Equal,
2081 ssaop.OpARM64NotEqual,
2082 ssaop.OpARM64LessThan,
2083 ssaop.OpARM64LessEqual,
2084 ssaop.OpARM64GreaterThan,
2085 ssaop.OpARM64GreaterEqual,
2086 ssaop.OpARM64LessThanU,
2087 ssaop.OpARM64LessEqualU,
2088 ssaop.OpARM64GreaterThanU,
2089 ssaop.OpARM64GreaterEqualU,
2090 ssaop.OpARM64LessThanF,
2091 ssaop.OpARM64LessEqualF,
2092 ssaop.OpARM64GreaterThanF,
2093 ssaop.OpARM64GreaterEqualF,
2094 ssaop.OpARM64NotLessThanF,
2095 ssaop.OpARM64NotLessEqualF,
2096 ssaop.OpARM64NotGreaterThanF,
2097 ssaop.OpARM64NotGreaterEqualF,
2098 ssaop.OpARM64LessThanNoov,
2099 ssaop.OpARM64GreaterEqualNoov:
2100
2101
2102
2103 p := s.Prog(arm64.ACSET)
2104 p.From.Type = obj.TYPE_SPECIAL
2105 condCode := condBits[v.Op]
2106 p.From.Offset = int64(condCode)
2107 p.To.Type = obj.TYPE_REG
2108 p.To.Reg = v.Reg()
2109 case ssaop.OpARM64PRFM:
2110 p := s.Prog(v.Op.Asm())
2111 p.From.Type = obj.TYPE_MEM
2112 p.From.Reg = v.Args[0].Reg()
2113 p.To.Type = obj.TYPE_CONST
2114 p.To.Offset = v.AuxInt
2115 case ssaop.OpARM64LoweredGetClosurePtr:
2116
2117 ssagen.CheckLoweredGetClosurePtr(v)
2118 case ssaop.OpARM64LoweredGetCallerSP:
2119
2120 p := s.Prog(arm64.AMOVD)
2121 p.From.Type = obj.TYPE_ADDR
2122 p.From.Offset = -base.Ctxt.Arch.FixedFrameSize
2123 p.From.Name = obj.NAME_PARAM
2124 p.To.Type = obj.TYPE_REG
2125 p.To.Reg = v.Reg()
2126 case ssaop.OpARM64LoweredGetCallerPC:
2127 p := s.Prog(obj.AGETCALLERPC)
2128 p.To.Type = obj.TYPE_REG
2129 p.To.Reg = v.Reg()
2130 case ssaop.OpARM64DMB:
2131 p := s.Prog(v.Op.Asm())
2132 p.From.Type = obj.TYPE_CONST
2133 p.From.Offset = v.AuxInt
2134 case ssaop.OpARM64FlagConstant:
2135 v.Fatalf("FlagConstant op should never make it to codegen %v", v.LongString())
2136 case ssaop.OpARM64InvertFlags:
2137 v.Fatalf("InvertFlags should never make it to codegen %v", v.LongString())
2138 case ssaop.OpClobber:
2139
2140
2141
2142 p := s.Prog(arm64.AMOVW)
2143 p.From.Type = obj.TYPE_CONST
2144 p.From.Offset = 0xdeaddead
2145 p.To.Type = obj.TYPE_REG
2146 p.To.Reg = arm64.REGTMP
2147 p = s.Prog(arm64.AMOVW)
2148 p.From.Type = obj.TYPE_REG
2149 p.From.Reg = arm64.REGTMP
2150 p.To.Type = obj.TYPE_MEM
2151 p.To.Reg = arm64.REGSP
2152 ssagen.AddAux(&p.To, v)
2153 p = s.Prog(arm64.AMOVW)
2154 p.From.Type = obj.TYPE_REG
2155 p.From.Reg = arm64.REGTMP
2156 p.To.Type = obj.TYPE_MEM
2157 p.To.Reg = arm64.REGSP
2158 ssagen.AddAux2(&p.To, v, v.AuxInt+4)
2159 case ssaop.OpClobberReg:
2160 x := uint64(0xdeaddeaddeaddead)
2161 p := s.Prog(arm64.AMOVD)
2162 p.From.Type = obj.TYPE_CONST
2163 p.From.Offset = int64(x)
2164 p.To.Type = obj.TYPE_REG
2165 p.To.Reg = v.Reg()
2166 default:
2167 if !ssaGenSIMDValue(s, v) && !ssaGenSIMDSVEValue(s, v) {
2168 v.Fatalf("genValue not implemented: %s", v.LongString())
2169 }
2170 }
2171 }
2172
2173
2174
2175 func simdZ21(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2176 p := s.Prog(v.Op.Asm())
2177 p.From.Type = obj.TYPE_REG
2178 p.From.Reg = zregArng(v.Args[1].Reg(), arng)
2179 p.AddRestSourceReg(zregArng(v.Args[0].Reg(), arng))
2180 p.To.Type = obj.TYPE_REG
2181 p.To.Reg = zregArng(v.Reg(), arng)
2182 return p
2183 }
2184
2185
2186
2187 func simdZ21List(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2188 if arng != arm64.ARNG_B {
2189 base.Fatalf("simdZ21List: unsupported arrangement %d", arng)
2190 }
2191 p := s.Prog(v.Op.Asm())
2192 p.From.Type = obj.TYPE_REG
2193 p.From.Reg = zregArng(v.Args[1].Reg(), arng)
2194
2195 regList, _ := arm64.RegisterListOffset(int(v.Args[0].Reg()&31), 1, regListArr("Z", "B"), 0)
2196 p.AddRestSource(obj.Addr{Type: obj.TYPE_REGLIST, Offset: regList})
2197 p.To.Type = obj.TYPE_REG
2198 p.To.Reg = zregArng(v.Reg(), arng)
2199 return p
2200 }
2201
2202
2203
2204 func simdZDupConst(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2205 p := s.Prog(v.Op.Asm())
2206 p.From.Type = obj.TYPE_CONST
2207 p.From.Offset = v.AuxInt
2208 p.To.Type = obj.TYPE_REG
2209 p.To.Reg = zregArng(v.Reg(), arng)
2210 return p
2211 }
2212
2213
2214
2215 func simdZDupGp(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2216 p := s.Prog(v.Op.Asm())
2217 p.From.Type = obj.TYPE_REG
2218 p.From.Reg = v.Args[0].Reg()
2219 p.To.Type = obj.TYPE_REG
2220 p.To.Reg = zregArng(v.Reg(), arng)
2221 return p
2222 }
2223
2224
2225
2226 func simdZDupIndexed(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2227 p := s.Prog(v.Op.Asm())
2228 p.From.Type = obj.TYPE_REG
2229 p.From.Reg = zregArngElem(v.Args[0].Reg(), arng)
2230 p.From.Index = int16(v.AuxInt)
2231 p.To.Type = obj.TYPE_REG
2232 p.To.Reg = zregArng(v.Reg(), arng)
2233 return p
2234 }
2235
2236
2237
2238 func simdZCpyConst(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2239 p := s.Prog(v.Op.Asm())
2240 p.From.Type = obj.TYPE_CONST
2241 p.From.Offset = v.AuxInt
2242 p.AddRestSourceReg(pregMask(v.Args[0].Reg(), arm64.PRED_Z))
2243 p.To.Type = obj.TYPE_REG
2244 p.To.Reg = zregArng(v.Reg(), arng)
2245 return p
2246 }
2247
2248
2249
2250
2251
2252 func simdZCpyMergingConst(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2253 p := s.Prog(v.Op.Asm())
2254 p.From.Type = obj.TYPE_CONST
2255 p.From.Offset = v.AuxInt
2256 p.AddRestSourceReg(pregMask(v.Args[1].Reg(), arm64.PRED_M))
2257 p.To.Type = obj.TYPE_REG
2258 p.To.Reg = zregArng(v.Reg(), arng)
2259 return p
2260 }
2261
2262
2263
2264
2265
2266 func simdZCpyMergingGp(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2267 p := s.Prog(v.Op.Asm())
2268 p.From.Type = obj.TYPE_REG
2269 p.From.Reg = v.Args[1].Reg()
2270 p.AddRestSourceReg(pregMask(v.Args[2].Reg(), arm64.PRED_M))
2271 p.To.Type = obj.TYPE_REG
2272 p.To.Reg = zregArng(v.Reg(), arng)
2273 return p
2274 }
2275
2276
2277
2278
2279
2280 func simdZCpyMergingVec(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2281 p := s.Prog(v.Op.Asm())
2282 p.From.Type = obj.TYPE_REG
2283 p.From.Reg = v.Args[1].Reg() - arm64.REG_F0 + arm64.REG_V0
2284 p.AddRestSourceReg(pregMask(v.Args[2].Reg(), arm64.PRED_M))
2285 p.To.Type = obj.TYPE_REG
2286 p.To.Reg = zregArng(v.Reg(), arng)
2287 return p
2288 }
2289
2290
2291
2292
2293
2294
2295 func simdZ2kv(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2296 p := s.Prog(v.Op.Asm())
2297 p.From.Type = obj.TYPE_REG
2298 p.From.Reg = zregArng(v.Args[1].Reg(), arng)
2299 p.AddRestSourceReg(zregArng(v.Args[0].Reg(), arng))
2300 p.AddRestSourceReg(v.Args[2].Reg())
2301 p.To.Type = obj.TYPE_REG
2302 p.To.Reg = zregArng(v.Reg(), arng)
2303 return p
2304 }
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317 func simdZ2kvPred(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2318 x, y := v.Args[0].Reg(), v.Args[1].Reg()
2319 d := v.Reg()
2320 switch d {
2321 case x:
2322 case y:
2323
2324
2325
2326 if v.Op.ResultInArg0() {
2327 v.Fatalf("destination is arg1 of resultInArg0 op %v", v.Op)
2328 }
2329 x, y = y, x
2330 default:
2331 mp := s.Prog(arm64.AZMOVPRFX)
2332 mp.From.Type = obj.TYPE_REG
2333 mp.From.Reg = pzreg(x)
2334 mp.To.Type = obj.TYPE_REG
2335 mp.To.Reg = pzreg(d)
2336 }
2337 p := s.Prog(v.Op.Asm())
2338 p.From.Type = obj.TYPE_REG
2339 p.From.Reg = zregArng(y, arng)
2340 p.AddRestSourceReg(zregArng(d, arng))
2341 p.AddRestSourceReg(pregMask(v.Args[2].Reg(), arm64.PRED_M))
2342 p.To.Type = obj.TYPE_REG
2343 p.To.Reg = zregArng(d, arng)
2344 return p
2345 }
2346
2347
2348
2349
2350
2351
2352 func simdZkv(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2353 return sveUnaryPred(s, v, arng, v.Args[0].Reg(), v.Args[1].Reg(), arm64.PRED_M)
2354 }
2355
2356
2357
2358
2359
2360 func simdZkfp(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2361 p := s.Prog(v.Op.Asm())
2362 p.From.Type = obj.TYPE_REG
2363 p.From.Reg = zregArng(v.Args[0].Reg(), arng)
2364 p.AddRestSourceReg(v.Args[1].Reg())
2365 p.To.Type = obj.TYPE_REG
2366 p.To.Reg = v.Reg() - arm64.REG_F0 + arm64.REG_V0
2367 return p
2368 }
2369
2370
2371
2372 func simdZ11Imm(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2373 p := s.Prog(v.Op.Asm())
2374 p.From.Type = obj.TYPE_CONST
2375 p.From.Offset = int64(v.AuxUInt8())
2376 p.AddRestSourceReg(zregArng(v.Args[0].Reg(), arng))
2377 p.To.Type = obj.TYPE_REG
2378 p.To.Reg = zregArng(v.Reg(), arng)
2379 return p
2380 }
2381
2382
2383
2384
2385 func simdZkvPredImm(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2386 d := v.Reg()
2387 if d != v.Args[0].Reg() {
2388 v.Fatalf("destination is not arg0 of destructive op %v", v.Op)
2389 }
2390 p := s.Prog(v.Op.Asm())
2391 p.From.Type = obj.TYPE_CONST
2392 p.From.Offset = int64(v.AuxUInt8())
2393 p.AddRestSourceReg(zregArng(d, arng))
2394 p.AddRestSourceReg(pregMask(v.Args[1].Reg(), arm64.PRED_M))
2395 p.To.Type = obj.TYPE_REG
2396 p.To.Reg = zregArng(d, arng)
2397 return p
2398 }
2399
2400
2401
2402
2403 func simdZ21Wide(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2404 p := s.Prog(v.Op.Asm())
2405 p.From.Type = obj.TYPE_REG
2406 p.From.Reg = zregArng(v.Args[1].Reg(), arm64.ARNG_D)
2407 p.AddRestSourceReg(zregArng(v.Args[0].Reg(), arng))
2408 p.To.Type = obj.TYPE_REG
2409 p.To.Reg = zregArng(v.Reg(), arng)
2410 return p
2411 }
2412
2413
2414
2415
2416 func simdZ2kvPredWide(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2417 d := v.Reg()
2418 if d != v.Args[0].Reg() {
2419 v.Fatalf("destination is not arg0 of destructive op %v", v.Op)
2420 }
2421 p := s.Prog(v.Op.Asm())
2422 p.From.Type = obj.TYPE_REG
2423 p.From.Reg = zregArng(v.Args[1].Reg(), arm64.ARNG_D)
2424 p.AddRestSourceReg(zregArng(d, arng))
2425 p.AddRestSourceReg(pregMask(v.Args[2].Reg(), arm64.PRED_M))
2426 p.To.Type = obj.TYPE_REG
2427 p.To.Reg = zregArng(d, arng)
2428 return p
2429 }
2430
2431
2432
2433
2434
2435
2436 func simdZ2kvPredResultInArg0(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2437 return sveUnaryPred(s, v, arng, v.Args[1].Reg(), v.Args[2].Reg(), arm64.PRED_M)
2438 }
2439
2440
2441 func sveUnaryPred(s *ssagen.State, v *ssa.Value, arng int16, zn, pg int16, qual int16) *obj.Prog {
2442 p := s.Prog(v.Op.Asm())
2443 p.From.Type = obj.TYPE_REG
2444 p.From.Reg = zregArng(zn, arng)
2445 p.AddRestSourceReg(pregMask(pg, qual))
2446 p.To.Type = obj.TYPE_REG
2447 p.To.Reg = zregArng(v.Reg(), arng)
2448 return p
2449 }
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459 func simdZ3kvPredAcc(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2460 p := s.Prog(v.Op.Asm())
2461 p.From.Type = obj.TYPE_REG
2462 p.From.Reg = zregArng(v.Args[2].Reg(), arng)
2463 p.AddRestSourceReg(zregArng(v.Args[1].Reg(), arng))
2464 p.AddRestSourceReg(pregMask(v.Args[3].Reg(), arm64.PRED_M))
2465 p.To.Type = obj.TYPE_REG
2466 p.To.Reg = zregArng(v.Reg(), arng)
2467 return p
2468 }
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487 func simdZ3kvPredResultInArg0(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2488 d := v.Reg()
2489 x, y := v.Args[1].Reg(), v.Args[2].Reg()
2490 pg := v.Args[3].Reg()
2491 zn, zm := x, y
2492 if d == zm {
2493 zn, zm = zm, zn
2494 }
2495 if d != zn {
2496 mp := s.Prog(arm64.AZMOVPRFX)
2497 mp.From.Type = obj.TYPE_REG
2498 mp.From.Reg = zregArng(zn, arng)
2499 mp.AddRestSourceReg(pregMask(pg, arm64.PRED_M))
2500 mp.To.Type = obj.TYPE_REG
2501 mp.To.Reg = zregArng(d, arng)
2502 }
2503 p := s.Prog(v.Op.Asm())
2504 p.From.Type = obj.TYPE_REG
2505 p.From.Reg = zregArng(zm, arng)
2506 p.AddRestSourceReg(zregArng(d, arng))
2507 p.AddRestSourceReg(pregMask(pg, arm64.PRED_M))
2508 p.To.Type = obj.TYPE_REG
2509 p.To.Reg = zregArng(d, arng)
2510 return p
2511 }
2512
2513
2514
2515
2516 func simdPNext(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2517 p := s.Prog(v.Op.Asm())
2518 p.From.Type = obj.TYPE_REG
2519 p.From.Reg = pregArng(v.Args[0].Reg(), arng)
2520 p.AddRestSourceReg(v.Args[1].Reg())
2521 p.To.Type = obj.TYPE_REG
2522 p.To.Reg = pregArng(v.Reg(), arng)
2523 return p
2524 }
2525
2526
2527
2528
2529 func simdPWHILELT(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2530 p := s.Prog(v.Op.Asm())
2531 p.From.Type = obj.TYPE_REG
2532 p.From.Reg = v.Args[1].Reg()
2533 p.AddRestSourceReg(v.Args[0].Reg())
2534 p.To.Type = obj.TYPE_REG
2535 p.To.Reg = pregArng(v.Reg0(), arng)
2536 return p
2537 }
2538
2539
2540
2541
2542 func simdZ2kk(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
2543 p := s.Prog(v.Op.Asm())
2544 p.From.Type = obj.TYPE_REG
2545 p.From.Reg = zregArng(v.Args[1].Reg(), arng)
2546 p.AddRestSourceReg(zregArng(v.Args[0].Reg(), arng))
2547 p.AddRestSourceReg(pregMask(v.Args[2].Reg(), arm64.PRED_Z))
2548 p.To.Type = obj.TYPE_REG
2549 p.To.Reg = pregArng(v.Reg(), arng)
2550 return p
2551 }
2552
2553 var condBits = map[ssaop.Op]arm64.SpecialOperand{
2554 ssaop.OpARM64Equal: arm64.SPOP_EQ,
2555 ssaop.OpARM64NotEqual: arm64.SPOP_NE,
2556 ssaop.OpARM64LessThan: arm64.SPOP_LT,
2557 ssaop.OpARM64LessThanU: arm64.SPOP_LO,
2558 ssaop.OpARM64LessEqual: arm64.SPOP_LE,
2559 ssaop.OpARM64LessEqualU: arm64.SPOP_LS,
2560 ssaop.OpARM64GreaterThan: arm64.SPOP_GT,
2561 ssaop.OpARM64GreaterThanU: arm64.SPOP_HI,
2562 ssaop.OpARM64GreaterEqual: arm64.SPOP_GE,
2563 ssaop.OpARM64GreaterEqualU: arm64.SPOP_HS,
2564 ssaop.OpARM64LessThanF: arm64.SPOP_MI,
2565 ssaop.OpARM64LessEqualF: arm64.SPOP_LS,
2566 ssaop.OpARM64GreaterThanF: arm64.SPOP_GT,
2567 ssaop.OpARM64GreaterEqualF: arm64.SPOP_GE,
2568
2569
2570 ssaop.OpARM64NotLessThanF: arm64.SPOP_PL,
2571 ssaop.OpARM64NotLessEqualF: arm64.SPOP_HI,
2572 ssaop.OpARM64NotGreaterThanF: arm64.SPOP_LE,
2573 ssaop.OpARM64NotGreaterEqualF: arm64.SPOP_LT,
2574
2575 ssaop.OpARM64LessThanNoov: arm64.SPOP_MI,
2576 ssaop.OpARM64GreaterEqualNoov: arm64.SPOP_PL,
2577 }
2578
2579 var blockJump = map[block.BlockKind]struct {
2580 asm, invasm obj.As
2581 }{
2582 block.BlockARM64EQ: {arm64.ABEQ, arm64.ABNE},
2583 block.BlockARM64NE: {arm64.ABNE, arm64.ABEQ},
2584 block.BlockARM64LT: {arm64.ABLT, arm64.ABGE},
2585 block.BlockARM64GE: {arm64.ABGE, arm64.ABLT},
2586 block.BlockARM64LE: {arm64.ABLE, arm64.ABGT},
2587 block.BlockARM64GT: {arm64.ABGT, arm64.ABLE},
2588 block.BlockARM64ULT: {arm64.ABLO, arm64.ABHS},
2589 block.BlockARM64UGE: {arm64.ABHS, arm64.ABLO},
2590 block.BlockARM64UGT: {arm64.ABHI, arm64.ABLS},
2591 block.BlockARM64ULE: {arm64.ABLS, arm64.ABHI},
2592 block.BlockARM64Z: {arm64.ACBZ, arm64.ACBNZ},
2593 block.BlockARM64NZ: {arm64.ACBNZ, arm64.ACBZ},
2594 block.BlockARM64ZW: {arm64.ACBZW, arm64.ACBNZW},
2595 block.BlockARM64NZW: {arm64.ACBNZW, arm64.ACBZW},
2596 block.BlockARM64TBZ: {arm64.ATBZ, arm64.ATBNZ},
2597 block.BlockARM64TBNZ: {arm64.ATBNZ, arm64.ATBZ},
2598 block.BlockARM64FLT: {arm64.ABMI, arm64.ABPL},
2599 block.BlockARM64FGE: {arm64.ABGE, arm64.ABLT},
2600 block.BlockARM64FLE: {arm64.ABLS, arm64.ABHI},
2601 block.BlockARM64FGT: {arm64.ABGT, arm64.ABLE},
2602 block.BlockARM64LTnoov: {arm64.ABMI, arm64.ABPL},
2603 block.BlockARM64GEnoov: {arm64.ABPL, arm64.ABMI},
2604 }
2605
2606
2607 var leJumps = [2][2]ssagen.IndexJump{
2608 {{Jump: arm64.ABEQ, Index: 0}, {Jump: arm64.ABPL, Index: 1}},
2609 {{Jump: arm64.ABMI, Index: 0}, {Jump: arm64.ABEQ, Index: 0}},
2610 }
2611
2612
2613 var gtJumps = [2][2]ssagen.IndexJump{
2614 {{Jump: arm64.ABMI, Index: 1}, {Jump: arm64.ABEQ, Index: 1}},
2615 {{Jump: arm64.ABEQ, Index: 1}, {Jump: arm64.ABPL, Index: 0}},
2616 }
2617
2618 func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
2619 switch b.Kind {
2620 case block.BlockPlain, block.BlockDefer:
2621 if b.Succs[0].Block() != next {
2622 p := s.Prog(obj.AJMP)
2623 p.To.Type = obj.TYPE_BRANCH
2624 s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
2625 }
2626
2627 case block.BlockExit, block.BlockRetJmp:
2628
2629 case block.BlockRet:
2630 s.Prog(obj.ARET)
2631
2632 case block.BlockARM64EQ, block.BlockARM64NE,
2633 block.BlockARM64LT, block.BlockARM64GE,
2634 block.BlockARM64LE, block.BlockARM64GT,
2635 block.BlockARM64ULT, block.BlockARM64UGT,
2636 block.BlockARM64ULE, block.BlockARM64UGE,
2637 block.BlockARM64Z, block.BlockARM64NZ,
2638 block.BlockARM64ZW, block.BlockARM64NZW,
2639 block.BlockARM64FLT, block.BlockARM64FGE,
2640 block.BlockARM64FLE, block.BlockARM64FGT,
2641 block.BlockARM64LTnoov, block.BlockARM64GEnoov:
2642 jmp := blockJump[b.Kind]
2643 var p *obj.Prog
2644 switch next {
2645 case b.Succs[0].Block():
2646 p = s.Br(jmp.invasm, b.Succs[1].Block())
2647 case b.Succs[1].Block():
2648 p = s.Br(jmp.asm, b.Succs[0].Block())
2649 default:
2650 if b.Likely != ssa.BranchUnlikely {
2651 p = s.Br(jmp.asm, b.Succs[0].Block())
2652 s.Br(obj.AJMP, b.Succs[1].Block())
2653 } else {
2654 p = s.Br(jmp.invasm, b.Succs[1].Block())
2655 s.Br(obj.AJMP, b.Succs[0].Block())
2656 }
2657 }
2658 if !b.Controls[0].Type.IsFlags() {
2659 p.From.Type = obj.TYPE_REG
2660 p.From.Reg = b.Controls[0].Reg()
2661 }
2662 case block.BlockARM64TBZ, block.BlockARM64TBNZ:
2663 jmp := blockJump[b.Kind]
2664 var p *obj.Prog
2665 switch next {
2666 case b.Succs[0].Block():
2667 p = s.Br(jmp.invasm, b.Succs[1].Block())
2668 case b.Succs[1].Block():
2669 p = s.Br(jmp.asm, b.Succs[0].Block())
2670 default:
2671 if b.Likely != ssa.BranchUnlikely {
2672 p = s.Br(jmp.asm, b.Succs[0].Block())
2673 s.Br(obj.AJMP, b.Succs[1].Block())
2674 } else {
2675 p = s.Br(jmp.invasm, b.Succs[1].Block())
2676 s.Br(obj.AJMP, b.Succs[0].Block())
2677 }
2678 }
2679 p.From.Offset = b.AuxInt
2680 p.From.Type = obj.TYPE_CONST
2681 p.Reg = b.Controls[0].Reg()
2682
2683 case block.BlockARM64LEnoov:
2684 s.CombJump(b, next, &leJumps)
2685 case block.BlockARM64GTnoov:
2686 s.CombJump(b, next, >Jumps)
2687
2688 case block.BlockARM64JUMPTABLE:
2689
2690
2691 p := s.Prog(arm64.AMOVD)
2692 p.From = genIndexedOperand(ssaop.OpARM64MOVDloadidx8, b.Controls[1].Reg(), b.Controls[0].Reg())
2693 p.To.Type = obj.TYPE_REG
2694 p.To.Reg = arm64.REGTMP
2695 p = s.Prog(obj.AJMP)
2696 p.To.Type = obj.TYPE_MEM
2697 p.To.Reg = arm64.REGTMP
2698
2699 s.JumpTables = append(s.JumpTables, b)
2700
2701 default:
2702 b.Fatalf("branch not implemented: %s", b.LongString())
2703 }
2704 }
2705
2706 func loadRegResult(s *ssagen.State, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
2707 p := s.Prog(loadByType(t))
2708 p.From.Type = obj.TYPE_MEM
2709 p.From.Name = obj.NAME_AUTO
2710 p.From.Sym = n.Linksym()
2711 p.From.Offset = n.FrameOffset() + off
2712 p.To.Type = obj.TYPE_REG
2713 p.To.Reg = reg
2714 return p
2715 }
2716
2717 func spillArgReg(pp *objw.Progs, p *obj.Prog, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog {
2718 p = pp.Append(p, storeByType(t), obj.TYPE_REG, reg, 0, obj.TYPE_MEM, 0, n.FrameOffset()+off)
2719 p.To.Name = obj.NAME_PARAM
2720 p.To.Sym = n.Linksym()
2721 p.Pos = p.Pos.WithNotStmt()
2722 return p
2723 }
2724
2725
2726
2727 func zero16(s *ssagen.State, reg int16, off int64, postInc bool) {
2728
2729 p := s.Prog(arm64.ASTP)
2730 p.From.Type = obj.TYPE_REGREG
2731 p.From.Reg = arm64.REGZERO
2732 p.From.Offset = int64(arm64.REGZERO)
2733 p.To.Type = obj.TYPE_MEM
2734 p.To.Reg = reg
2735 p.To.Offset = off
2736 if postInc {
2737 if off != 0 {
2738 panic("can't postinc with non-zero offset")
2739 }
2740
2741 p.Scond = arm64.C_XPOST
2742 p.To.Offset = 16
2743 }
2744 }
2745
2746
2747 func zero8(s *ssagen.State, reg int16, off int64) {
2748
2749 p := s.Prog(arm64.AMOVD)
2750 p.From.Type = obj.TYPE_REG
2751 p.From.Reg = arm64.REGZERO
2752 p.To.Type = obj.TYPE_MEM
2753 p.To.Reg = reg
2754 p.To.Offset = off
2755 }
2756
2757
2758
2759
2760 func move32(s *ssagen.State, src, dst, tmp1, tmp2 int16, off int64, postInc bool) {
2761
2762 ld := s.Prog(arm64.AFLDPQ)
2763 ld.From.Type = obj.TYPE_MEM
2764 ld.From.Reg = src
2765 ld.From.Offset = off
2766 ld.To.Type = obj.TYPE_REGREG
2767 ld.To.Reg = tmp1
2768 ld.To.Offset = int64(tmp2)
2769
2770 st := s.Prog(arm64.AFSTPQ)
2771 st.From.Type = obj.TYPE_REGREG
2772 st.From.Reg = tmp1
2773 st.From.Offset = int64(tmp2)
2774 st.To.Type = obj.TYPE_MEM
2775 st.To.Reg = dst
2776 st.To.Offset = off
2777 if postInc {
2778 if off != 0 {
2779 panic("can't postinc with non-zero offset")
2780 }
2781 ld.Scond = arm64.C_XPOST
2782 st.Scond = arm64.C_XPOST
2783 ld.From.Offset = 32
2784 st.To.Offset = 32
2785 }
2786 }
2787
2788
2789
2790
2791 func move16(s *ssagen.State, src, dst, tmp1 int16, off int64, postInc bool) {
2792
2793 ld := s.Prog(arm64.AFMOVQ)
2794 ld.From.Type = obj.TYPE_MEM
2795 ld.From.Reg = src
2796 ld.From.Offset = off
2797 ld.To.Type = obj.TYPE_REG
2798 ld.To.Reg = tmp1
2799
2800 st := s.Prog(arm64.AFMOVQ)
2801 st.From.Type = obj.TYPE_REG
2802 st.From.Reg = tmp1
2803 st.To.Type = obj.TYPE_MEM
2804 st.To.Reg = dst
2805 st.To.Offset = off
2806 if postInc {
2807 if off != 0 {
2808 panic("can't postinc with non-zero offset")
2809 }
2810 ld.Scond = arm64.C_XPOST
2811 st.Scond = arm64.C_XPOST
2812 ld.From.Offset = 16
2813 st.To.Offset = 16
2814 }
2815 }
2816
2817
2818
2819 func move8(s *ssagen.State, src, dst, tmp int16, off int64) {
2820
2821 ld := s.Prog(arm64.AMOVD)
2822 ld.From.Type = obj.TYPE_MEM
2823 ld.From.Reg = src
2824 ld.From.Offset = off
2825 ld.To.Type = obj.TYPE_REG
2826 ld.To.Reg = tmp
2827
2828 st := s.Prog(arm64.AMOVD)
2829 st.From.Type = obj.TYPE_REG
2830 st.From.Reg = tmp
2831 st.To.Type = obj.TYPE_MEM
2832 st.To.Reg = dst
2833 st.To.Offset = off
2834 }
2835
2836 func zregArng(r int16, arng int16) int16 {
2837 if r >= arm64.REG_F0 && r <= arm64.REG_F31 &&
2838 arng >= arm64.ARNG_B && arng <= arm64.ARNG_Q {
2839 return arm64.REG_ZARNG + (r - arm64.REG_F0) | (arng << 5)
2840 }
2841 panic("Bad Z reg with arrangement")
2842 }
2843
2844
2845
2846
2847 func zregArngElem(r int16, arng int16) int16 {
2848 if r >= arm64.REG_F0 && r <= arm64.REG_F31 &&
2849 arng >= arm64.ARNG_B && arng <= arm64.ARNG_Q {
2850 return arm64.REG_ZARNGELEM + (r - arm64.REG_F0) | (arng << 5)
2851 }
2852 panic("Bad Z reg with arrangement")
2853 }
2854
2855 func pregArng(r int16, mode int16) int16 {
2856 if r >= arm64.REG_P0 && r <= arm64.REG_P15 &&
2857 mode >= arm64.ARNG_B && mode <= arm64.ARNG_Q {
2858 return arm64.REG_PARNGZM + (r - arm64.REG_P0) | (mode << 5)
2859 }
2860 panic("Bad P reg with arrangement")
2861 }
2862
2863 func pregMask(r int16, mode int16) int16 {
2864 if r >= arm64.REG_P0 && r <= arm64.REG_P15 &&
2865 mode >= arm64.PRED_M && mode <= arm64.PRED_Z {
2866 return arm64.REG_PARNGZM + (r - arm64.REG_P0) | (mode << 5)
2867 }
2868 panic("Bad P reg with arrangement")
2869 }
2870
2871 func pzreg(r int16) int16 {
2872 if r >= arm64.REG_F0 && r <= arm64.REG_F31 {
2873 return r - arm64.REG_F0 + arm64.REG_Z0
2874 } else if r >= arm64.REG_P0 && r <= arm64.REG_P15 {
2875 return r
2876 }
2877 panic("Bad P or Z reg")
2878 }
2879
2880
2881 func regListArr(name, arng string) int64 {
2882 var curQ, curSize, prefix uint16
2883 if name[0] != 'V' && name[0] != 'Z' && name[0] != 'P' {
2884 panic("expect V0-V31, Z0-Z31, or P0-P15; found: " + name)
2885 }
2886 switch name[0] {
2887 case 'V':
2888 prefix = 0
2889 case 'Z':
2890 prefix = 1
2891 case 'P':
2892 prefix = 2
2893 }
2894 switch arng {
2895 case "B8":
2896 curSize = 0
2897 curQ = 0
2898 case "B16":
2899 curSize = 0
2900 curQ = 1
2901 case "H4":
2902 curSize = 1
2903 curQ = 0
2904 case "H8":
2905 curSize = 1
2906 curQ = 1
2907 case "S2":
2908 curSize = 2
2909 curQ = 0
2910 case "S4":
2911 curSize = 2
2912 curQ = 1
2913 case "D1":
2914 curSize = 3
2915 curQ = 0
2916 case "D2":
2917 curSize = 3
2918 curQ = 1
2919 case "B":
2920 curSize = 1
2921 curQ = 2
2922 case "H":
2923 curSize = 2
2924 curQ = 2
2925 case "S":
2926 curSize = 3
2927 curQ = 2
2928 case "D":
2929 curSize = 1
2930 curQ = 3
2931 case "Q":
2932 curSize = 2
2933 curQ = 3
2934 default:
2935 panic("invalid arrangement in ARM64 register list")
2936 }
2937 return (int64(prefix) << 32) | (int64(curQ) & 3 << 30) | (int64(curSize&3) << 10)
2938 }
2939
View as plain text