Text file
src/runtime/asm_ppc64x.s
1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 //go:build ppc64 || ppc64le
6
7 #include "go_asm.h"
8 #include "go_tls.h"
9 #include "funcdata.h"
10 #include "textflag.h"
11 #include "asm_ppc64x.h"
12
13 #ifdef GOOS_aix
14 #define cgoCalleeStackSize 48
15 #else
16 #define cgoCalleeStackSize 32
17 #endif
18
19 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
20 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer
21
22 // initialize essential registers
23 BL runtime·reginit(SB)
24
25 SUB $(FIXED_FRAME+16), R1
26 MOVD R2, 24(R1) // stash the TOC pointer away again now we've created a new frame
27 MOVW R3, FIXED_FRAME+0(R1) // argc
28 MOVD R4, FIXED_FRAME+8(R1) // argv
29
30 // create istack out of the given (operating system) stack.
31 // _cgo_init may update stackguard.
32 MOVD $runtime·g0(SB), g
33 BL runtime·save_g(SB)
34 MOVD $(-64*1024), R31
35 ADD R31, R1, R3
36 MOVD R3, g_stackguard0(g)
37 MOVD R3, g_stackguard1(g)
38 MOVD R3, (g_stack+stack_lo)(g)
39 MOVD R1, (g_stack+stack_hi)(g)
40
41 // If there is a _cgo_init, call it using the gcc ABI.
42 MOVD _cgo_init(SB), R12
43 CMP R12, $0
44 BEQ nocgo
45
46 #ifdef GO_PPC64X_HAS_FUNCDESC
47 // Load the real entry address from the first slot of the function descriptor.
48 MOVD 8(R12), R2
49 MOVD (R12), R12
50 #endif
51 MOVD R12, CTR // r12 = "global function entry point"
52 MOVD R13, R5 // arg 2: TLS base pointer
53 MOVD $setg_gcc<>(SB), R4 // arg 1: setg
54 MOVD g, R3 // arg 0: G
55 // C functions expect 32 (48 for AIX) bytes of space on caller
56 // stack frame and a 16-byte aligned R1
57 MOVD R1, R14 // save current stack
58 SUB $cgoCalleeStackSize, R1 // reserve the callee area
59 RLDCR $0, R1, $~15, R1 // 16-byte align
60 BL (CTR) // may clobber R0, R3-R12
61 MOVD R14, R1 // restore stack
62 #ifndef GOOS_aix
63 MOVD 24(R1), R2
64 #endif
65 XOR R0, R0 // fix R0
66
67 nocgo:
68 // update stackguard after _cgo_init
69 MOVD (g_stack+stack_lo)(g), R3
70 ADD $const_stackGuard, R3
71 MOVD R3, g_stackguard0(g)
72 MOVD R3, g_stackguard1(g)
73
74 // set the per-goroutine and per-mach "registers"
75 MOVD $runtime·m0(SB), R3
76
77 // save m->g0 = g0
78 MOVD g, m_g0(R3)
79 // save m0 to g0->m
80 MOVD R3, g_m(g)
81
82 BL runtime·check(SB)
83
84 // args are already prepared
85 BL runtime·args(SB)
86 BL runtime·osinit(SB)
87 BL runtime·schedinit(SB)
88
89 // create a new goroutine to start program
90 MOVD $runtime·mainPC(SB), R3 // entry
91 MOVDU R3, -8(R1)
92 MOVDU R0, -8(R1)
93 MOVDU R0, -8(R1)
94 MOVDU R0, -8(R1)
95 MOVDU R0, -8(R1)
96 BL runtime·newproc(SB)
97 ADD $(8+FIXED_FRAME), R1
98
99 // start this M
100 BL runtime·mstart(SB)
101 // Prevent dead-code elimination of debugCallV2 and debugPinnerV1, which are
102 // intended to be called by debuggers.
103 #ifdef GOARCH_ppc64le
104 MOVD $runtime·debugPinnerV1<ABIInternal>(SB), R31
105 MOVD $runtime·debugCallV2<ABIInternal>(SB), R31
106 #endif
107 MOVD R0, 0(R0)
108 RET
109
110 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
111 GLOBL runtime·mainPC(SB),RODATA,$8
112
113 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
114 TW $31, R0, R0
115 RET
116
117 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
118 RET
119
120 // Any changes must be reflected to runtime/cgo/gcc_aix_ppc64.S:.crosscall_ppc64
121 TEXT _cgo_reginit(SB),NOSPLIT|NOFRAME,$0-0
122 // crosscall_ppc64 and crosscall2 need to reginit, but can't
123 // get at the 'runtime.reginit' symbol.
124 BR runtime·reginit(SB)
125
126 TEXT runtime·reginit(SB),NOSPLIT|NOFRAME,$0-0
127 // set R0 to zero, it's expected by the toolchain
128 XOR R0, R0
129 RET
130
131 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
132 BL runtime·mstart0(SB)
133 RET // not reached
134
135 /*
136 * go-routine
137 */
138
139 // void gogo(Gobuf*)
140 // restore state from Gobuf; longjmp
141 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
142 MOVD buf+0(FP), R5
143 MOVD gobuf_g(R5), R6
144 MOVD 0(R6), R4 // make sure g != nil
145 BR gogo<>(SB)
146
147 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
148 MOVD R6, g
149 BL runtime·save_g(SB)
150
151 MOVD gobuf_sp(R5), R1
152 MOVD gobuf_lr(R5), R31
153 #ifndef GOOS_aix
154 MOVD 24(R1), R2 // restore R2
155 #endif
156 MOVD R31, LR
157 MOVD gobuf_ret(R5), R3
158 MOVD gobuf_ctxt(R5), R11
159 MOVD R0, gobuf_sp(R5)
160 MOVD R0, gobuf_ret(R5)
161 MOVD R0, gobuf_lr(R5)
162 MOVD R0, gobuf_ctxt(R5)
163 CMP R0, R0 // set condition codes for == test, needed by stack split
164 MOVD gobuf_pc(R5), R12
165 MOVD R12, CTR
166 BR (CTR)
167
168 // void mcall(fn func(*g))
169 // Switch to m->g0's stack, call fn(g).
170 // Fn must never return. It should gogo(&g->sched)
171 // to keep running g.
172 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
173 // Save caller state in g->sched
174 // R11 should be safe across save_g??
175 MOVD R3, R11
176 MOVD R1, (g_sched+gobuf_sp)(g)
177 MOVD LR, R31
178 MOVD R31, (g_sched+gobuf_pc)(g)
179 MOVD R0, (g_sched+gobuf_lr)(g)
180
181 // Switch to m->g0 & its stack, call fn.
182 MOVD g, R3
183 MOVD g_m(g), R8
184 MOVD m_g0(R8), g
185 BL runtime·save_g(SB)
186 CMP g, R3
187 BNE 2(PC)
188 BR runtime·badmcall(SB)
189 MOVD 0(R11), R12 // code pointer
190 MOVD R12, CTR
191 MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp
192 // Don't need to do anything special for regabiargs here
193 // R3 is g; stack is set anyway
194 MOVDU R3, -8(R1)
195 MOVDU R0, -8(R1)
196 MOVDU R0, -8(R1)
197 MOVDU R0, -8(R1)
198 MOVDU R0, -8(R1)
199 BL (CTR)
200 MOVD 24(R1), R2
201 BR runtime·badmcall2(SB)
202
203 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
204 // of the G stack. We need to distinguish the routine that
205 // lives at the bottom of the G stack from the one that lives
206 // at the top of the system stack because the one at the top of
207 // the system stack terminates the stack walk (see topofstack()).
208 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
209 // We have several undefs here so that 16 bytes past
210 // $runtime·systemstack_switch lies within them whether or not the
211 // instructions that derive r2 from r12 are there.
212 UNDEF
213 UNDEF
214 UNDEF
215 BL (LR) // make sure this function is not leaf
216 RET
217
218 // func systemstack(fn func())
219 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
220 MOVD fn+0(FP), R3 // R3 = fn
221 MOVD R3, R11 // context
222 MOVD g_m(g), R4 // R4 = m
223
224 MOVD m_gsignal(R4), R5 // R5 = gsignal
225 CMP g, R5
226 BEQ noswitch
227
228 MOVD m_g0(R4), R5 // R5 = g0
229 CMP g, R5
230 BEQ noswitch
231
232 MOVD m_curg(R4), R6
233 CMP g, R6
234 BEQ switch
235
236 // Bad: g is not gsignal, not g0, not curg. What is it?
237 // Hide call from linker nosplit analysis.
238 MOVD $runtime·badsystemstack(SB), R12
239 MOVD R12, CTR
240 BL (CTR)
241 BL runtime·abort(SB)
242
243 switch:
244 // save our state in g->sched. Pretend to
245 // be systemstack_switch if the G stack is scanned.
246 BL gosave_systemstack_switch<>(SB)
247
248 // switch to g0
249 MOVD R5, g
250 BL runtime·save_g(SB)
251 MOVD (g_sched+gobuf_sp)(g), R1
252
253 // call target function
254 MOVD 0(R11), R12 // code pointer
255 MOVD R12, CTR
256 BL (CTR)
257
258 // restore TOC pointer. It seems unlikely that we will use systemstack
259 // to call a function defined in another module, but the results of
260 // doing so would be so confusing that it's worth doing this.
261 MOVD g_m(g), R3
262 MOVD m_curg(R3), g
263 MOVD (g_sched+gobuf_sp)(g), R3
264 #ifndef GOOS_aix
265 MOVD 24(R3), R2
266 #endif
267 // switch back to g
268 MOVD g_m(g), R3
269 MOVD m_curg(R3), g
270 BL runtime·save_g(SB)
271 MOVD (g_sched+gobuf_sp)(g), R1
272 MOVD R0, (g_sched+gobuf_sp)(g)
273 RET
274
275 noswitch:
276 // already on m stack, just call directly
277 // On other arches we do a tail call here, but it appears to be
278 // impossible to tail call a function pointer in shared mode on
279 // ppc64 because the caller is responsible for restoring the TOC.
280 MOVD 0(R11), R12 // code pointer
281 MOVD R12, CTR
282 BL (CTR)
283 #ifndef GOOS_aix
284 MOVD 24(R1), R2
285 #endif
286 RET
287
288 // func switchToCrashStack0(fn func())
289 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
290 MOVD R3, R11 // context register
291 MOVD g_m(g), R3 // curm
292
293 // set g to gcrash
294 MOVD $runtime·gcrash(SB), g // g = &gcrash
295 CALL runtime·save_g(SB) // clobbers R31
296 MOVD R3, g_m(g) // g.m = curm
297 MOVD g, m_g0(R3) // curm.g0 = g
298
299 // switch to crashstack
300 MOVD (g_stack+stack_hi)(g), R3
301 SUB $(4*8), R3
302 MOVD R3, R1
303
304 // call target function
305 MOVD 0(R11), R12 // code pointer
306 MOVD R12, CTR
307 BL (CTR)
308
309 // should never return
310 CALL runtime·abort(SB)
311 UNDEF
312
313 /*
314 * support for morestack
315 */
316
317 // Called during function prolog when more stack is needed.
318 // Caller has already loaded:
319 // R3: framesize, R4: argsize, R5: LR
320 //
321 // The traceback routines see morestack on a g0 as being
322 // the top of a stack (for example, morestack calling newstack
323 // calling the scheduler calling newm calling gc), so we must
324 // record an argument size. For that purpose, it has no arguments.
325 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
326 // Called from f.
327 // Set g->sched to context in f.
328 MOVD R1, (g_sched+gobuf_sp)(g)
329 MOVD LR, R8
330 MOVD R8, (g_sched+gobuf_pc)(g)
331 MOVD R5, (g_sched+gobuf_lr)(g)
332 MOVD R11, (g_sched+gobuf_ctxt)(g)
333
334 // Cannot grow scheduler stack (m->g0).
335 MOVD g_m(g), R7
336 MOVD m_g0(R7), R8
337 CMP g, R8
338 BNE 3(PC)
339 BL runtime·badmorestackg0(SB)
340 BL runtime·abort(SB)
341
342 // Cannot grow signal stack (m->gsignal).
343 MOVD m_gsignal(R7), R8
344 CMP g, R8
345 BNE 3(PC)
346 BL runtime·badmorestackgsignal(SB)
347 BL runtime·abort(SB)
348
349 // Called from f.
350 // Set m->morebuf to f's caller.
351 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
352 MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
353 MOVD g, (m_morebuf+gobuf_g)(R7)
354
355 // Call newstack on m->g0's stack.
356 MOVD m_g0(R7), g
357 BL runtime·save_g(SB)
358 MOVD (g_sched+gobuf_sp)(g), R1
359 MOVDU R0, -(FIXED_FRAME+0)(R1) // create a call frame on g0
360 BL runtime·newstack(SB)
361
362 // Not reached, but make sure the return PC from the call to newstack
363 // is still in this function, and not the beginning of the next.
364 UNDEF
365
366 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
367 // Force SPWRITE. This function doesn't actually write SP,
368 // but it is called with a special calling convention where
369 // the caller doesn't save LR on stack but passes it as a
370 // register (R5), and the unwinder currently doesn't understand.
371 // Make it SPWRITE to stop unwinding. (See issue 54332)
372 // Use OR R0, R1 instead of MOVD R1, R1 as the MOVD instruction
373 // has a special affect on Power8,9,10 by lowering the thread
374 // priority and causing a slowdown in execution time
375
376 OR R0, R1
377 MOVD R0, R11
378 BR runtime·morestack(SB)
379
380 // reflectcall: call a function with the given argument list
381 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
382 // we don't have variable-sized frames, so we use a small number
383 // of constant-sized-frame functions to encode a few bits of size in the pc.
384 // Caution: ugly multiline assembly macros in your future!
385
386 #define DISPATCH(NAME,MAXSIZE) \
387 MOVD $MAXSIZE, R31; \
388 CMP R3, R31; \
389 BGT 4(PC); \
390 MOVD $NAME(SB), R12; \
391 MOVD R12, CTR; \
392 BR (CTR)
393 // Note: can't just "BR NAME(SB)" - bad inlining results.
394
395 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
396 MOVWZ frameSize+32(FP), R3
397 DISPATCH(runtime·call16, 16)
398 DISPATCH(runtime·call32, 32)
399 DISPATCH(runtime·call64, 64)
400 DISPATCH(runtime·call128, 128)
401 DISPATCH(runtime·call256, 256)
402 DISPATCH(runtime·call512, 512)
403 DISPATCH(runtime·call1024, 1024)
404 DISPATCH(runtime·call2048, 2048)
405 DISPATCH(runtime·call4096, 4096)
406 DISPATCH(runtime·call8192, 8192)
407 DISPATCH(runtime·call16384, 16384)
408 DISPATCH(runtime·call32768, 32768)
409 DISPATCH(runtime·call65536, 65536)
410 DISPATCH(runtime·call131072, 131072)
411 DISPATCH(runtime·call262144, 262144)
412 DISPATCH(runtime·call524288, 524288)
413 DISPATCH(runtime·call1048576, 1048576)
414 DISPATCH(runtime·call2097152, 2097152)
415 DISPATCH(runtime·call4194304, 4194304)
416 DISPATCH(runtime·call8388608, 8388608)
417 DISPATCH(runtime·call16777216, 16777216)
418 DISPATCH(runtime·call33554432, 33554432)
419 DISPATCH(runtime·call67108864, 67108864)
420 DISPATCH(runtime·call134217728, 134217728)
421 DISPATCH(runtime·call268435456, 268435456)
422 DISPATCH(runtime·call536870912, 536870912)
423 DISPATCH(runtime·call1073741824, 1073741824)
424 MOVD $runtime·badreflectcall(SB), R12
425 MOVD R12, CTR
426 BR (CTR)
427
428 #define CALLFN(NAME,MAXSIZE) \
429 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
430 NO_LOCAL_POINTERS; \
431 /* copy arguments to stack */ \
432 MOVD stackArgs+16(FP), R3; \
433 MOVWZ stackArgsSize+24(FP), R4; \
434 MOVD R1, R5; \
435 CMP R4, $8; \
436 BLT tailsetup; \
437 /* copy 8 at a time if possible */ \
438 ADD $(FIXED_FRAME-8), R5; \
439 SUB $8, R3; \
440 top: \
441 MOVDU 8(R3), R7; \
442 MOVDU R7, 8(R5); \
443 SUB $8, R4; \
444 CMP R4, $8; \
445 BGE top; \
446 /* handle remaining bytes */ \
447 CMP $0, R4; \
448 BEQ callfn; \
449 ADD $7, R3; \
450 ADD $7, R5; \
451 BR tail; \
452 tailsetup: \
453 CMP $0, R4; \
454 BEQ callfn; \
455 ADD $(FIXED_FRAME-1), R5; \
456 SUB $1, R3; \
457 tail: \
458 MOVBU 1(R3), R6; \
459 MOVBU R6, 1(R5); \
460 SUB $1, R4; \
461 CMP $0, R4; \
462 BGT tail; \
463 callfn: \
464 /* call function */ \
465 MOVD f+8(FP), R11; \
466 #ifdef GOOS_aix \
467 /* AIX won't trigger a SIGSEGV if R11 = nil */ \
468 /* So it manually triggers it */ \
469 CMP R11, $0 \
470 BNE 2(PC) \
471 MOVD R0, 0(R0) \
472 #endif \
473 MOVD regArgs+40(FP), R20; \
474 BL runtime·unspillArgs(SB); \
475 MOVD (R11), R12; \
476 MOVD R12, CTR; \
477 PCDATA $PCDATA_StackMapIndex, $0; \
478 BL (CTR); \
479 #ifndef GOOS_aix \
480 MOVD 24(R1), R2; \
481 #endif \
482 /* copy return values back */ \
483 MOVD regArgs+40(FP), R20; \
484 BL runtime·spillArgs(SB); \
485 MOVD stackArgsType+0(FP), R7; \
486 MOVD stackArgs+16(FP), R3; \
487 MOVWZ stackArgsSize+24(FP), R4; \
488 MOVWZ stackRetOffset+28(FP), R6; \
489 ADD $FIXED_FRAME, R1, R5; \
490 ADD R6, R5; \
491 ADD R6, R3; \
492 SUB R6, R4; \
493 BL callRet<>(SB); \
494 RET
495
496 // callRet copies return values back at the end of call*. This is a
497 // separate function so it can allocate stack space for the arguments
498 // to reflectcallmove. It does not follow the Go ABI; it expects its
499 // arguments in registers.
500 TEXT callRet<>(SB), NOSPLIT, $40-0
501 NO_LOCAL_POINTERS
502 MOVD R7, FIXED_FRAME+0(R1)
503 MOVD R3, FIXED_FRAME+8(R1)
504 MOVD R5, FIXED_FRAME+16(R1)
505 MOVD R4, FIXED_FRAME+24(R1)
506 MOVD R20, FIXED_FRAME+32(R1)
507 BL runtime·reflectcallmove(SB)
508 RET
509
510 CALLFN(·call16, 16)
511 CALLFN(·call32, 32)
512 CALLFN(·call64, 64)
513 CALLFN(·call128, 128)
514 CALLFN(·call256, 256)
515 CALLFN(·call512, 512)
516 CALLFN(·call1024, 1024)
517 CALLFN(·call2048, 2048)
518 CALLFN(·call4096, 4096)
519 CALLFN(·call8192, 8192)
520 CALLFN(·call16384, 16384)
521 CALLFN(·call32768, 32768)
522 CALLFN(·call65536, 65536)
523 CALLFN(·call131072, 131072)
524 CALLFN(·call262144, 262144)
525 CALLFN(·call524288, 524288)
526 CALLFN(·call1048576, 1048576)
527 CALLFN(·call2097152, 2097152)
528 CALLFN(·call4194304, 4194304)
529 CALLFN(·call8388608, 8388608)
530 CALLFN(·call16777216, 16777216)
531 CALLFN(·call33554432, 33554432)
532 CALLFN(·call67108864, 67108864)
533 CALLFN(·call134217728, 134217728)
534 CALLFN(·call268435456, 268435456)
535 CALLFN(·call536870912, 536870912)
536 CALLFN(·call1073741824, 1073741824)
537
538 TEXT runtime·procyield(SB),NOSPLIT|NOFRAME,$0-4
539 MOVW cycles+0(FP), R7
540 // POWER does not have a pause/yield instruction equivalent.
541 // Instead, we can lower the program priority by setting the
542 // Program Priority Register prior to the wait loop and set it
543 // back to default afterwards. On Linux, the default priority is
544 // medium-low. For details, see page 837 of the ISA 3.0.
545 OR R1, R1, R1 // Set PPR priority to low
546 again:
547 SUB $1, R7
548 CMP $0, R7
549 BNE again
550 OR R6, R6, R6 // Set PPR priority back to medium-low
551 RET
552
553 // Save state of caller into g->sched,
554 // but using fake PC from systemstack_switch.
555 // Must only be called from functions with no locals ($0)
556 // or else unwinding from systemstack_switch is incorrect.
557 // Smashes R31.
558 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
559 MOVD $runtime·systemstack_switch(SB), R31
560 ADD $16, R31 // get past prologue (including r2-setting instructions when they're there)
561 MOVD R31, (g_sched+gobuf_pc)(g)
562 MOVD R1, (g_sched+gobuf_sp)(g)
563 MOVD R0, (g_sched+gobuf_lr)(g)
564 MOVD R0, (g_sched+gobuf_ret)(g)
565 // Assert ctxt is zero. See func save.
566 MOVD (g_sched+gobuf_ctxt)(g), R31
567 CMP R31, $0
568 BEQ 2(PC)
569 BL runtime·abort(SB)
570 RET
571
572 #ifdef GOOS_aix
573 #define asmcgocallSaveOffset cgoCalleeStackSize + 8
574 #else
575 #define asmcgocallSaveOffset cgoCalleeStackSize
576 #endif
577
578 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
579 // Call fn(arg) aligned appropriately for the gcc ABI.
580 // Called on a system stack, and there may be no g yet (during needm).
581 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
582 MOVD fn+0(FP), R3
583 MOVD arg+8(FP), R4
584
585 MOVD R1, R15
586 SUB $(asmcgocallSaveOffset+8), R1
587 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI
588 MOVD R15, asmcgocallSaveOffset(R1)
589
590 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
591
592 // This is a "global call", so put the global entry point in r12
593 MOVD R3, R12
594
595 #ifdef GO_PPC64X_HAS_FUNCDESC
596 // Load the real entry address from the first slot of the function descriptor.
597 MOVD 8(R12), R2
598 MOVD (R12), R12
599 #endif
600 MOVD R12, CTR
601 MOVD R4, R3 // arg in r3
602 BL (CTR)
603
604 // C code can clobber R0, so set it back to 0. F27-F31 are
605 // callee save, so we don't need to recover those.
606 XOR R0, R0
607
608 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
609 #ifndef GOOS_aix
610 MOVD 24(R1), R2
611 #endif
612
613 RET
614
615 // func asmcgocall(fn, arg unsafe.Pointer) int32
616 // Call fn(arg) on the scheduler stack,
617 // aligned appropriately for the gcc ABI.
618 // See cgocall.go for more details.
619 TEXT ·asmcgocall<ABIInternal>(SB),NOSPLIT,$0-20
620 // R3 = fn
621 // R4 = arg
622
623 MOVD R1, R7 // save original stack pointer
624 CMP $0, g
625 BEQ nosave
626 MOVD g, R5
627
628 // Figure out if we need to switch to m->g0 stack.
629 // We get called to create new OS threads too, and those
630 // come in on the m->g0 stack already. Or we might already
631 // be on the m->gsignal stack.
632 MOVD g_m(g), R8
633 MOVD m_gsignal(R8), R6
634 CMP R6, g
635 BEQ nosave
636 MOVD m_g0(R8), R6
637 CMP R6, g
638 BEQ nosave
639
640 BL gosave_systemstack_switch<>(SB)
641 MOVD R6, g
642 BL runtime·save_g(SB)
643 MOVD (g_sched+gobuf_sp)(g), R1
644
645 // Now on a scheduling stack (a pthread-created stack).
646 #ifdef GOOS_aix
647 // Create a fake LR to improve backtrace.
648 MOVD $runtime·asmcgocall(SB), R6
649 MOVD R6, 16(R1)
650 // AIX also saves one argument on the stack.
651 SUB $8, R1
652 #endif
653 // Save room for two of our pointers, plus the callee
654 // save area that lives on the caller stack.
655 // Do arithmetics in R10 to hide from the assembler
656 // counting it as SP delta, which is irrelevant as we are
657 // on the system stack.
658 SUB $(asmcgocallSaveOffset+16), R1, R10
659 RLDCR $0, R10, $~15, R1 // 16-byte alignment for gcc ABI
660 MOVD R5, (asmcgocallSaveOffset+8)(R1) // save old g on stack
661 MOVD (g_stack+stack_hi)(R5), R5
662 SUB R7, R5
663 MOVD R5, asmcgocallSaveOffset(R1) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
664 #ifdef GOOS_aix
665 MOVD R7, 0(R1) // Save frame pointer to allow manual backtrace with gdb
666 #else
667 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
668 #endif
669 // This is a "global call", so put the global entry point in r12
670 MOVD R3, R12
671
672 #ifdef GO_PPC64X_HAS_FUNCDESC
673 // Load the real entry address from the first slot of the function descriptor.
674 MOVD 8(R12), R2
675 MOVD (R12), R12
676 #endif
677 MOVD R12, CTR
678 MOVD R4, R3 // arg in r3
679 BL (CTR)
680
681 // Reinitialise zero value register.
682 XOR R0, R0
683
684 // Restore g, stack pointer, toc pointer.
685 // R3 is errno, so don't touch it
686 MOVD (asmcgocallSaveOffset+8)(R1), g
687 MOVD (g_stack+stack_hi)(g), R5
688 MOVD asmcgocallSaveOffset(R1), R6
689 SUB R6, R5
690 #ifndef GOOS_aix
691 MOVD 24(R5), R2
692 #endif
693 MOVD R5, R1
694 BL runtime·save_g(SB)
695
696 // ret = R3
697 RET
698
699 nosave:
700 // Running on a system stack, perhaps even without a g.
701 // Having no g can happen during thread creation or thread teardown.
702 // This code is like the above sequence but without saving/restoring g
703 // and without worrying about the stack moving out from under us
704 // (because we're on a system stack, not a goroutine stack).
705 // The above code could be used directly if already on a system stack,
706 // but then the only path through this code would be a rare case.
707 // Using this code for all "already on system stack" calls exercises it more,
708 // which should help keep it correct.
709
710 SUB $(asmcgocallSaveOffset+8), R1, R10
711 RLDCR $0, R10, $~15, R1 // 16-byte alignment for gcc ABI
712 MOVD R7, asmcgocallSaveOffset(R1) // Save original stack pointer.
713
714 MOVD R3, R12 // fn
715 #ifdef GO_PPC64X_HAS_FUNCDESC
716 // Load the real entry address from the first slot of the function descriptor.
717 MOVD 8(R12), R2
718 MOVD (R12), R12
719 #endif
720 MOVD R12, CTR
721 MOVD R4, R3 // arg
722 BL (CTR)
723
724 // Reinitialise zero value register.
725 XOR R0, R0
726
727 MOVD asmcgocallSaveOffset(R1), R1 // Restore stack pointer.
728 #ifndef GOOS_aix
729 MOVD 24(R1), R2
730 #endif
731 // ret = R3
732 RET
733
734 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
735 // See cgocall.go for more details.
736 TEXT ·cgocallback(SB),NOSPLIT,$24-24
737 NO_LOCAL_POINTERS
738
739 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
740 // It is used to dropm while thread is exiting.
741 MOVD fn+0(FP), R5
742 CMP R5, $0
743 BNE loadg
744 // Restore the g from frame.
745 MOVD frame+8(FP), g
746 BR dropm
747
748 loadg:
749 // Load m and g from thread-local storage.
750 #ifndef GOOS_openbsd
751 MOVBZ runtime·iscgo(SB), R3
752 CMP R3, $0
753 BEQ nocgo
754 #endif
755 BL runtime·load_g(SB)
756 nocgo:
757
758 // If g is nil, Go did not create the current thread,
759 // or if this thread never called into Go on pthread platforms.
760 // Call needm to obtain one for temporary use.
761 // In this case, we're running on the thread stack, so there's
762 // lots of space, but the linker doesn't know. Hide the call from
763 // the linker analysis by using an indirect call.
764 CMP g, $0
765 BEQ needm
766
767 MOVD g_m(g), R8
768 MOVD R8, savedm-8(SP)
769 BR havem
770
771 needm:
772 MOVD g, savedm-8(SP) // g is zero, so is m.
773 MOVD $runtime·needAndBindM(SB), R12
774 MOVD R12, CTR
775 BL (CTR)
776
777 // Set m->sched.sp = SP, so that if a panic happens
778 // during the function we are about to execute, it will
779 // have a valid SP to run on the g0 stack.
780 // The next few lines (after the havem label)
781 // will save this SP onto the stack and then write
782 // the same SP back to m->sched.sp. That seems redundant,
783 // but if an unrecovered panic happens, unwindm will
784 // restore the g->sched.sp from the stack location
785 // and then systemstack will try to use it. If we don't set it here,
786 // that restored SP will be uninitialized (typically 0) and
787 // will not be usable.
788 MOVD g_m(g), R8
789 MOVD m_g0(R8), R3
790 MOVD R1, (g_sched+gobuf_sp)(R3)
791
792 havem:
793 // Now there's a valid m, and we're running on its m->g0.
794 // Save current m->g0->sched.sp on stack and then set it to SP.
795 // Save current sp in m->g0->sched.sp in preparation for
796 // switch back to m->curg stack.
797 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
798 MOVD m_g0(R8), R3
799 MOVD (g_sched+gobuf_sp)(R3), R4
800 MOVD R4, savedsp-24(SP) // must match frame size
801 MOVD R1, (g_sched+gobuf_sp)(R3)
802
803 // Switch to m->curg stack and call runtime.cgocallbackg.
804 // Because we are taking over the execution of m->curg
805 // but *not* resuming what had been running, we need to
806 // save that information (m->curg->sched) so we can restore it.
807 // We can restore m->curg->sched.sp easily, because calling
808 // runtime.cgocallbackg leaves SP unchanged upon return.
809 // To save m->curg->sched.pc, we push it onto the curg stack and
810 // open a frame the same size as cgocallback's g0 frame.
811 // Once we switch to the curg stack, the pushed PC will appear
812 // to be the return PC of cgocallback, so that the traceback
813 // will seamlessly trace back into the earlier calls.
814 MOVD m_curg(R8), g
815 BL runtime·save_g(SB)
816 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
817 MOVD (g_sched+gobuf_pc)(g), R5
818 MOVD R5, -(24+FIXED_FRAME)(R4) // "saved LR"; must match frame size
819 // Gather our arguments into registers.
820 MOVD fn+0(FP), R5
821 MOVD frame+8(FP), R6
822 MOVD ctxt+16(FP), R7
823 MOVD $-(24+FIXED_FRAME)(R4), R1 // switch stack; must match frame size
824 MOVD R5, FIXED_FRAME+0(R1)
825 MOVD R6, FIXED_FRAME+8(R1)
826 MOVD R7, FIXED_FRAME+16(R1)
827
828 MOVD $runtime·cgocallbackg(SB), R12
829 MOVD R12, CTR
830 CALL (CTR) // indirect call to bypass nosplit check. We're on a different stack now.
831
832 // Restore g->sched (== m->curg->sched) from saved values.
833 MOVD 0(R1), R5
834 MOVD R5, (g_sched+gobuf_pc)(g)
835 MOVD $(24+FIXED_FRAME)(R1), R4 // must match frame size
836 MOVD R4, (g_sched+gobuf_sp)(g)
837
838 // Switch back to m->g0's stack and restore m->g0->sched.sp.
839 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
840 // so we do not have to restore it.)
841 MOVD g_m(g), R8
842 MOVD m_g0(R8), g
843 BL runtime·save_g(SB)
844 MOVD (g_sched+gobuf_sp)(g), R1
845 MOVD savedsp-24(SP), R4 // must match frame size
846 MOVD R4, (g_sched+gobuf_sp)(g)
847
848 // If the m on entry was nil, we called needm above to borrow an m,
849 // 1. for the duration of the call on non-pthread platforms,
850 // 2. or the duration of the C thread alive on pthread platforms.
851 // If the m on entry wasn't nil,
852 // 1. the thread might be a Go thread,
853 // 2. or it wasn't the first call from a C thread on pthread platforms,
854 // since then we skip dropm to reuse the m in the first call.
855 MOVD savedm-8(SP), R6
856 CMP R6, $0
857 BNE droppedm
858
859 // Skip dropm to reuse it in the next call, when a pthread key has been created.
860 MOVD _cgo_pthread_key_created(SB), R6
861 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
862 CMP R6, $0
863 BEQ dropm
864 MOVD (R6), R6
865 CMP R6, $0
866 BNE droppedm
867
868 dropm:
869 MOVD $runtime·dropm(SB), R12
870 MOVD R12, CTR
871 BL (CTR)
872 droppedm:
873
874 // Done!
875 RET
876
877 // void setg(G*); set g. for use by needm.
878 TEXT runtime·setg(SB), NOSPLIT, $0-8
879 MOVD gg+0(FP), g
880 // This only happens if iscgo, so jump straight to save_g
881 BL runtime·save_g(SB)
882 RET
883
884 #ifdef GO_PPC64X_HAS_FUNCDESC
885 DEFINE_PPC64X_FUNCDESC(setg_gcc<>, _setg_gcc<>)
886 TEXT _setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
887 #else
888 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
889 #endif
890 // The standard prologue clobbers R31, which is callee-save in
891 // the C ABI, so we have to use $-8-0 and save LR ourselves.
892 MOVD LR, R4
893 // Also save g and R31, since they're callee-save in C ABI
894 MOVD R31, R5
895 MOVD g, R6
896
897 MOVD R3, g
898 BL runtime·save_g(SB)
899
900 MOVD R6, g
901 MOVD R5, R31
902 MOVD R4, LR
903 RET
904
905 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
906 MOVW (R0), R0
907 UNDEF
908
909 #define TBR 268
910
911 // int64 runtime·cputicks(void)
912 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
913 MOVD SPR(TBR), R3
914 MOVD R3, ret+0(FP)
915 RET
916
917 // spillArgs stores return values from registers to a *internal/abi.RegArgs in R20.
918 TEXT runtime·spillArgs(SB),NOSPLIT,$0-0
919 MOVD R3, 0(R20)
920 MOVD R4, 8(R20)
921 MOVD R5, 16(R20)
922 MOVD R6, 24(R20)
923 MOVD R7, 32(R20)
924 MOVD R8, 40(R20)
925 MOVD R9, 48(R20)
926 MOVD R10, 56(R20)
927 MOVD R14, 64(R20)
928 MOVD R15, 72(R20)
929 MOVD R16, 80(R20)
930 MOVD R17, 88(R20)
931 FMOVD F1, 96(R20)
932 FMOVD F2, 104(R20)
933 FMOVD F3, 112(R20)
934 FMOVD F4, 120(R20)
935 FMOVD F5, 128(R20)
936 FMOVD F6, 136(R20)
937 FMOVD F7, 144(R20)
938 FMOVD F8, 152(R20)
939 FMOVD F9, 160(R20)
940 FMOVD F10, 168(R20)
941 FMOVD F11, 176(R20)
942 FMOVD F12, 184(R20)
943 RET
944
945 // unspillArgs loads args into registers from a *internal/abi.RegArgs in R20.
946 TEXT runtime·unspillArgs(SB),NOSPLIT,$0-0
947 MOVD 0(R20), R3
948 MOVD 8(R20), R4
949 MOVD 16(R20), R5
950 MOVD 24(R20), R6
951 MOVD 32(R20), R7
952 MOVD 40(R20), R8
953 MOVD 48(R20), R9
954 MOVD 56(R20), R10
955 MOVD 64(R20), R14
956 MOVD 72(R20), R15
957 MOVD 80(R20), R16
958 MOVD 88(R20), R17
959 FMOVD 96(R20), F1
960 FMOVD 104(R20), F2
961 FMOVD 112(R20), F3
962 FMOVD 120(R20), F4
963 FMOVD 128(R20), F5
964 FMOVD 136(R20), F6
965 FMOVD 144(R20), F7
966 FMOVD 152(R20), F8
967 FMOVD 160(R20), F9
968 FMOVD 168(R20), F10
969 FMOVD 176(R20), F11
970 FMOVD 184(R20), F12
971 RET
972
973 // AES hashing not implemented for ppc64
974 TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
975 JMP runtime·memhashFallback<ABIInternal>(SB)
976 TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
977 JMP runtime·strhashFallback<ABIInternal>(SB)
978 TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
979 JMP runtime·memhash32Fallback<ABIInternal>(SB)
980 TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
981 JMP runtime·memhash64Fallback<ABIInternal>(SB)
982
983 TEXT runtime·return0(SB), NOSPLIT, $0
984 MOVW $0, R3
985 RET
986
987 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
988 // Must obey the gcc calling convention.
989 #ifdef GOOS_aix
990 // On AIX, _cgo_topofstack is defined in runtime/cgo, because it must
991 // be a longcall in order to prevent trampolines from ld.
992 TEXT __cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
993 #else
994 TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
995 #endif
996 // g (R30) and R31 are callee-save in the C ABI, so save them
997 MOVD g, R4
998 MOVD R31, R5
999 MOVD LR, R6
1000
1001 BL runtime·load_g(SB) // clobbers g (R30), R31
1002 MOVD g_m(g), R3
1003 MOVD m_curg(R3), R3
1004 MOVD (g_stack+stack_hi)(R3), R3
1005
1006 MOVD R4, g
1007 MOVD R5, R31
1008 MOVD R6, LR
1009 RET
1010
1011 // The top-most function running on a goroutine
1012 // returns to goexit+PCQuantum.
1013 //
1014 // When dynamically linking Go, it can be returned to from a function
1015 // implemented in a different module and so needs to reload the TOC pointer
1016 // from the stack (although this function declares that it does not set up x-a
1017 // frame, newproc1 does in fact allocate one for goexit and saves the TOC
1018 // pointer in the correct place).
1019 // goexit+_PCQuantum is halfway through the usual global entry point prologue
1020 // that derives r2 from r12 which is a bit silly, but not harmful.
1021 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
1022 MOVD 24(R1), R2
1023 BL runtime·goexit1(SB) // does not return
1024 // traceback from goexit1 must hit code range of goexit
1025 MOVD R0, R0 // NOP
1026
1027 // prepGoExitFrame saves the current TOC pointer (i.e. the TOC pointer for the
1028 // module containing runtime) to the frame that goexit will execute in when
1029 // the goroutine exits. It's implemented in assembly mainly because that's the
1030 // easiest way to get access to R2.
1031 TEXT runtime·prepGoExitFrame(SB),NOSPLIT,$0-8
1032 MOVD sp+0(FP), R3
1033 MOVD R2, 24(R3)
1034 RET
1035
1036 TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
1037 ADD $-8, R1
1038 MOVD R31, 0(R1)
1039 MOVD runtime·lastmoduledatap(SB), R4
1040 MOVD R3, moduledata_next(R4)
1041 MOVD R3, runtime·lastmoduledatap(SB)
1042 MOVD 0(R1), R31
1043 ADD $8, R1
1044 RET
1045
1046 TEXT ·checkASM(SB),NOSPLIT,$0-1
1047 MOVW $1, R3
1048 MOVB R3, ret+0(FP)
1049 RET
1050
1051 // gcWriteBarrier informs the GC about heap pointer writes.
1052 //
1053 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
1054 // number of bytes of buffer needed in R29, and returns a pointer
1055 // to the buffer space in R29.
1056 // It clobbers condition codes.
1057 // It does not clobber R0 through R17 (except special registers),
1058 // but may clobber any other register, *including* R31.
1059 TEXT gcWriteBarrier<>(SB),NOSPLIT,$120
1060 // The standard prologue clobbers R31.
1061 // We use R18, R19, and R31 as scratch registers.
1062 retry:
1063 MOVD g_m(g), R18
1064 MOVD m_p(R18), R18
1065 MOVD (p_wbBuf+wbBuf_next)(R18), R19
1066 MOVD (p_wbBuf+wbBuf_end)(R18), R31
1067 // Increment wbBuf.next position.
1068 ADD R29, R19
1069 // Is the buffer full?
1070 CMPU R31, R19
1071 BLT flush
1072 // Commit to the larger buffer.
1073 MOVD R19, (p_wbBuf+wbBuf_next)(R18)
1074 // Make return value (the original next position)
1075 SUB R29, R19, R29
1076 RET
1077
1078 flush:
1079 // Save registers R0 through R15 since these were not saved by the caller.
1080 // We don't save all registers on ppc64 because it takes too much space.
1081 MOVD R20, (FIXED_FRAME+0)(R1)
1082 MOVD R21, (FIXED_FRAME+8)(R1)
1083 // R0 is always 0, so no need to spill.
1084 // R1 is SP.
1085 // R2 is SB.
1086 MOVD R3, (FIXED_FRAME+16)(R1)
1087 MOVD R4, (FIXED_FRAME+24)(R1)
1088 MOVD R5, (FIXED_FRAME+32)(R1)
1089 MOVD R6, (FIXED_FRAME+40)(R1)
1090 MOVD R7, (FIXED_FRAME+48)(R1)
1091 MOVD R8, (FIXED_FRAME+56)(R1)
1092 MOVD R9, (FIXED_FRAME+64)(R1)
1093 MOVD R10, (FIXED_FRAME+72)(R1)
1094 // R11, R12 may be clobbered by external-linker-inserted trampoline
1095 // R13 is REGTLS
1096 MOVD R14, (FIXED_FRAME+80)(R1)
1097 MOVD R15, (FIXED_FRAME+88)(R1)
1098 MOVD R16, (FIXED_FRAME+96)(R1)
1099 MOVD R17, (FIXED_FRAME+104)(R1)
1100 MOVD R29, (FIXED_FRAME+112)(R1)
1101
1102 CALL runtime·wbBufFlush(SB)
1103
1104 MOVD (FIXED_FRAME+0)(R1), R20
1105 MOVD (FIXED_FRAME+8)(R1), R21
1106 MOVD (FIXED_FRAME+16)(R1), R3
1107 MOVD (FIXED_FRAME+24)(R1), R4
1108 MOVD (FIXED_FRAME+32)(R1), R5
1109 MOVD (FIXED_FRAME+40)(R1), R6
1110 MOVD (FIXED_FRAME+48)(R1), R7
1111 MOVD (FIXED_FRAME+56)(R1), R8
1112 MOVD (FIXED_FRAME+64)(R1), R9
1113 MOVD (FIXED_FRAME+72)(R1), R10
1114 MOVD (FIXED_FRAME+80)(R1), R14
1115 MOVD (FIXED_FRAME+88)(R1), R15
1116 MOVD (FIXED_FRAME+96)(R1), R16
1117 MOVD (FIXED_FRAME+104)(R1), R17
1118 MOVD (FIXED_FRAME+112)(R1), R29
1119 JMP retry
1120
1121 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
1122 MOVD $8, R29
1123 JMP gcWriteBarrier<>(SB)
1124 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
1125 MOVD $16, R29
1126 JMP gcWriteBarrier<>(SB)
1127 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
1128 MOVD $24, R29
1129 JMP gcWriteBarrier<>(SB)
1130 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
1131 MOVD $32, R29
1132 JMP gcWriteBarrier<>(SB)
1133 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
1134 MOVD $40, R29
1135 JMP gcWriteBarrier<>(SB)
1136 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
1137 MOVD $48, R29
1138 JMP gcWriteBarrier<>(SB)
1139 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
1140 MOVD $56, R29
1141 JMP gcWriteBarrier<>(SB)
1142 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
1143 MOVD $64, R29
1144 JMP gcWriteBarrier<>(SB)
1145
1146 DATA debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
1147 GLOBL debugCallFrameTooLarge<>(SB), RODATA, $20 // Size duplicated below
1148
1149 // debugCallV2 is the entry point for debugger-injected function
1150 // calls on running goroutines. It informs the runtime that a
1151 // debug call has been injected and creates a call frame for the
1152 // debugger to fill in.
1153 //
1154 // To inject a function call, a debugger should:
1155 // 1. Check that the goroutine is in state _Grunning and that
1156 // there are at least 320 bytes free on the stack.
1157 // 2. Set SP as SP-32.
1158 // 3. Store the current LR in (SP) (using the SP after step 2).
1159 // 4. Store the current PC in the LR register.
1160 // 5. Write the desired argument frame size at SP-32
1161 // 6. Save all machine registers (including flags and floating point registers)
1162 // so they can be restored later by the debugger.
1163 // 7. Set the PC to debugCallV2 and resume execution.
1164 //
1165 // If the goroutine is in state _Grunnable, then it's not generally
1166 // safe to inject a call because it may return out via other runtime
1167 // operations. Instead, the debugger should unwind the stack to find
1168 // the return to non-runtime code, add a temporary breakpoint there,
1169 // and inject the call once that breakpoint is hit.
1170 //
1171 // If the goroutine is in any other state, it's not safe to inject a call.
1172 //
1173 // This function communicates back to the debugger by setting R20 and
1174 // invoking TW to raise a breakpoint signal. Note that the signal PC of
1175 // the signal triggered by the TW instruction is the PC where the signal
1176 // is trapped, not the next PC, so to resume execution, the debugger needs
1177 // to set the signal PC to PC+4. See the comments in the implementation for
1178 // the protocol the debugger is expected to follow. InjectDebugCall in the
1179 // runtime tests demonstrates this protocol.
1180 // The debugger must ensure that any pointers passed to the function
1181 // obey escape analysis requirements. Specifically, it must not pass
1182 // a stack pointer to an escaping argument. debugCallV2 cannot check
1183 // this invariant.
1184 //
1185 // This is ABIInternal because Go code injects its PC directly into new
1186 // goroutine stacks.
1187 #ifdef GOARCH_ppc64le
1188 TEXT runtime·debugCallV2<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-0
1189 // save scratch register R31 first
1190 MOVD R31, -184(R1)
1191 MOVD 0(R1), R31
1192 // save caller LR
1193 MOVD R31, -304(R1)
1194 MOVD -32(R1), R31
1195 // save argument frame size
1196 MOVD R31, -192(R1)
1197 MOVD LR, R31
1198 MOVD R31, -320(R1)
1199 ADD $-320, R1
1200 // save all registers that can contain pointers
1201 // and the CR register
1202 MOVW CR, R31
1203 MOVD R31, 8(R1)
1204 MOVD R2, 24(R1)
1205 MOVD R3, 56(R1)
1206 MOVD R4, 64(R1)
1207 MOVD R5, 72(R1)
1208 MOVD R6, 80(R1)
1209 MOVD R7, 88(R1)
1210 MOVD R8, 96(R1)
1211 MOVD R9, 104(R1)
1212 MOVD R10, 112(R1)
1213 MOVD R11, 120(R1)
1214 MOVD R12, 144(R1)
1215 MOVD R13, 152(R1)
1216 MOVD R14, 160(R1)
1217 MOVD R15, 168(R1)
1218 MOVD R16, 176(R1)
1219 MOVD R17, 184(R1)
1220 MOVD R18, 192(R1)
1221 MOVD R19, 200(R1)
1222 MOVD R20, 208(R1)
1223 MOVD R21, 216(R1)
1224 MOVD R22, 224(R1)
1225 MOVD R23, 232(R1)
1226 MOVD R24, 240(R1)
1227 MOVD R25, 248(R1)
1228 MOVD R26, 256(R1)
1229 MOVD R27, 264(R1)
1230 MOVD R28, 272(R1)
1231 MOVD R29, 280(R1)
1232 MOVD g, 288(R1)
1233 MOVD LR, R31
1234 MOVD R31, 32(R1)
1235 CALL runtime·debugCallCheck(SB)
1236 MOVD 40(R1), R22
1237 XOR R0, R0
1238 CMP R22, $0
1239 BEQ good
1240 MOVD 48(R1), R22
1241 MOVD $8, R20
1242 TW $31, R0, R0
1243
1244 BR restore
1245
1246 good:
1247 #define DEBUG_CALL_DISPATCH(NAME,MAXSIZE) \
1248 MOVD $MAXSIZE, R23; \
1249 CMP R26, R23; \
1250 BGT 5(PC); \
1251 MOVD $NAME(SB), R26; \
1252 MOVD R26, 32(R1); \
1253 CALL runtime·debugCallWrap(SB); \
1254 BR restore
1255
1256 // the argument frame size
1257 MOVD 128(R1), R26
1258
1259 DEBUG_CALL_DISPATCH(debugCall32<>, 32)
1260 DEBUG_CALL_DISPATCH(debugCall64<>, 64)
1261 DEBUG_CALL_DISPATCH(debugCall128<>, 128)
1262 DEBUG_CALL_DISPATCH(debugCall256<>, 256)
1263 DEBUG_CALL_DISPATCH(debugCall512<>, 512)
1264 DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)
1265 DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)
1266 DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)
1267 DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)
1268 DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)
1269 DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)
1270 DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)
1271 // The frame size is too large. Report the error.
1272 MOVD $debugCallFrameTooLarge<>(SB), R22
1273 MOVD R22, 32(R1)
1274 MOVD $20, R22
1275 // length of debugCallFrameTooLarge string
1276 MOVD R22, 40(R1)
1277 MOVD $8, R20
1278 TW $31, R0, R0
1279 BR restore
1280 restore:
1281 MOVD $16, R20
1282 TW $31, R0, R0
1283 // restore all registers that can contain
1284 // pointers including CR
1285 MOVD 8(R1), R31
1286 MOVW R31, CR
1287 MOVD 24(R1), R2
1288 MOVD 56(R1), R3
1289 MOVD 64(R1), R4
1290 MOVD 72(R1), R5
1291 MOVD 80(R1), R6
1292 MOVD 88(R1), R7
1293 MOVD 96(R1), R8
1294 MOVD 104(R1), R9
1295 MOVD 112(R1), R10
1296 MOVD 120(R1), R11
1297 MOVD 144(R1), R12
1298 MOVD 152(R1), R13
1299 MOVD 160(R1), R14
1300 MOVD 168(R1), R15
1301 MOVD 176(R1), R16
1302 MOVD 184(R1), R17
1303 MOVD 192(R1), R18
1304 MOVD 200(R1), R19
1305 MOVD 208(R1), R20
1306 MOVD 216(R1), R21
1307 MOVD 224(R1), R22
1308 MOVD 232(R1), R23
1309 MOVD 240(R1), R24
1310 MOVD 248(R1), R25
1311 MOVD 256(R1), R26
1312 MOVD 264(R1), R27
1313 MOVD 272(R1), R28
1314 MOVD 280(R1), R29
1315 MOVD 288(R1), g
1316 MOVD 16(R1), R31
1317 // restore old LR
1318 MOVD R31, LR
1319 // restore caller PC
1320 MOVD 0(R1), CTR
1321 MOVD 136(R1), R31
1322 // Add 32 bytes more to compensate for SP change in saveSigContext
1323 ADD $352, R1
1324 JMP (CTR)
1325 #endif
1326 #define DEBUG_CALL_FN(NAME,MAXSIZE) \
1327 TEXT NAME(SB),WRAPPER,$MAXSIZE-0; \
1328 NO_LOCAL_POINTERS; \
1329 MOVD $0, R20; \
1330 TW $31, R0, R0 \
1331 MOVD $1, R20; \
1332 TW $31, R0, R0 \
1333 RET
1334 DEBUG_CALL_FN(debugCall32<>, 32)
1335 DEBUG_CALL_FN(debugCall64<>, 64)
1336 DEBUG_CALL_FN(debugCall128<>, 128)
1337 DEBUG_CALL_FN(debugCall256<>, 256)
1338 DEBUG_CALL_FN(debugCall512<>, 512)
1339 DEBUG_CALL_FN(debugCall1024<>, 1024)
1340 DEBUG_CALL_FN(debugCall2048<>, 2048)
1341 DEBUG_CALL_FN(debugCall4096<>, 4096)
1342 DEBUG_CALL_FN(debugCall8192<>, 8192)
1343 DEBUG_CALL_FN(debugCall16384<>, 16384)
1344 DEBUG_CALL_FN(debugCall32768<>, 32768)
1345 DEBUG_CALL_FN(debugCall65536<>, 65536)
1346
1347 #ifdef GOARCH_ppc64le
1348 // func debugCallPanicked(val interface{})
1349 TEXT runtime·debugCallPanicked(SB),NOSPLIT,$32-16
1350 // Copy the panic value to the top of stack at SP+32.
1351 MOVD val_type+0(FP), R31
1352 MOVD R31, 32(R1)
1353 MOVD val_data+8(FP), R31
1354 MOVD R31, 40(R1)
1355 MOVD $2, R20
1356 TW $31, R0, R0
1357 RET
1358 #endif
1359 // Note: these functions use a special calling convention to save generated code space.
1360 // Arguments are passed in registers, but the space for those arguments are allocated
1361 // in the caller's stack frame. These stubs write the args into that stack space and
1362 // then tail call to the corresponding runtime handler.
1363 // The tail call makes these stubs disappear in backtraces.
1364 TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
1365 JMP runtime·goPanicIndex<ABIInternal>(SB)
1366 TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
1367 JMP runtime·goPanicIndexU<ABIInternal>(SB)
1368 TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
1369 MOVD R4, R3
1370 MOVD R5, R4
1371 JMP runtime·goPanicSliceAlen<ABIInternal>(SB)
1372 TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
1373 MOVD R4, R3
1374 MOVD R5, R4
1375 JMP runtime·goPanicSliceAlenU<ABIInternal>(SB)
1376 TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
1377 MOVD R4, R3
1378 MOVD R5, R4
1379 JMP runtime·goPanicSliceAcap<ABIInternal>(SB)
1380 TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
1381 MOVD R4, R3
1382 MOVD R5, R4
1383 JMP runtime·goPanicSliceAcapU<ABIInternal>(SB)
1384 TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
1385 JMP runtime·goPanicSliceB<ABIInternal>(SB)
1386 TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
1387 JMP runtime·goPanicSliceBU<ABIInternal>(SB)
1388 TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
1389 MOVD R5, R3
1390 MOVD R6, R4
1391 JMP runtime·goPanicSlice3Alen<ABIInternal>(SB)
1392 TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
1393 MOVD R5, R3
1394 MOVD R6, R4
1395 JMP runtime·goPanicSlice3AlenU<ABIInternal>(SB)
1396 TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
1397 MOVD R5, R3
1398 MOVD R6, R4
1399 JMP runtime·goPanicSlice3Acap<ABIInternal>(SB)
1400 TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
1401 MOVD R5, R3
1402 MOVD R6, R4
1403 JMP runtime·goPanicSlice3AcapU<ABIInternal>(SB)
1404 TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
1405 MOVD R4, R3
1406 MOVD R5, R4
1407 JMP runtime·goPanicSlice3B<ABIInternal>(SB)
1408 TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
1409 MOVD R4, R3
1410 MOVD R5, R4
1411 JMP runtime·goPanicSlice3BU<ABIInternal>(SB)
1412 TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
1413 JMP runtime·goPanicSlice3C<ABIInternal>(SB)
1414 TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
1415 JMP runtime·goPanicSlice3CU<ABIInternal>(SB)
1416 TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16
1417 MOVD R5, R3
1418 MOVD R6, R4
1419 JMP runtime·goPanicSliceConvert<ABIInternal>(SB)
1420
1421 // These functions are used when internal linking cgo with external
1422 // objects compiled with the -Os on gcc. They reduce prologue/epilogue
1423 // size by deferring preservation of callee-save registers to a shared
1424 // function. These are defined in PPC64 ELFv2 2.3.3 (but also present
1425 // in ELFv1)
1426 //
1427 // These appear unused, but the linker will redirect calls to functions
1428 // like _savegpr0_14 or _restgpr1_14 to runtime.elf_savegpr0 or
1429 // runtime.elf_restgpr1 with an appropriate offset based on the number
1430 // register operations required when linking external objects which
1431 // make these calls. For GPR/FPR saves, the minimum register value is
1432 // 14, for VR it is 20.
1433 //
1434 // These are only used when linking such cgo code internally. Note, R12
1435 // and R0 may be used in different ways than regular ELF compliant
1436 // functions.
1437 TEXT runtime·elf_savegpr0(SB),NOSPLIT|NOFRAME,$0
1438 // R0 holds the LR of the caller's caller, R1 holds save location
1439 MOVD R14, -144(R1)
1440 MOVD R15, -136(R1)
1441 MOVD R16, -128(R1)
1442 MOVD R17, -120(R1)
1443 MOVD R18, -112(R1)
1444 MOVD R19, -104(R1)
1445 MOVD R20, -96(R1)
1446 MOVD R21, -88(R1)
1447 MOVD R22, -80(R1)
1448 MOVD R23, -72(R1)
1449 MOVD R24, -64(R1)
1450 MOVD R25, -56(R1)
1451 MOVD R26, -48(R1)
1452 MOVD R27, -40(R1)
1453 MOVD R28, -32(R1)
1454 MOVD R29, -24(R1)
1455 MOVD g, -16(R1)
1456 MOVD R31, -8(R1)
1457 MOVD R0, 16(R1)
1458 RET
1459 TEXT runtime·elf_restgpr0(SB),NOSPLIT|NOFRAME,$0
1460 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1461 MOVD -144(R1), R14
1462 MOVD -136(R1), R15
1463 MOVD -128(R1), R16
1464 MOVD -120(R1), R17
1465 MOVD -112(R1), R18
1466 MOVD -104(R1), R19
1467 MOVD -96(R1), R20
1468 MOVD -88(R1), R21
1469 MOVD -80(R1), R22
1470 MOVD -72(R1), R23
1471 MOVD -64(R1), R24
1472 MOVD -56(R1), R25
1473 MOVD -48(R1), R26
1474 MOVD -40(R1), R27
1475 MOVD -32(R1), R28
1476 MOVD -24(R1), R29
1477 MOVD -16(R1), g
1478 MOVD -8(R1), R31
1479 MOVD 16(R1), R0 // Load and return to saved LR
1480 MOVD R0, LR
1481 RET
1482 TEXT runtime·elf_savegpr1(SB),NOSPLIT|NOFRAME,$0
1483 // R12 holds the save location
1484 MOVD R14, -144(R12)
1485 MOVD R15, -136(R12)
1486 MOVD R16, -128(R12)
1487 MOVD R17, -120(R12)
1488 MOVD R18, -112(R12)
1489 MOVD R19, -104(R12)
1490 MOVD R20, -96(R12)
1491 MOVD R21, -88(R12)
1492 MOVD R22, -80(R12)
1493 MOVD R23, -72(R12)
1494 MOVD R24, -64(R12)
1495 MOVD R25, -56(R12)
1496 MOVD R26, -48(R12)
1497 MOVD R27, -40(R12)
1498 MOVD R28, -32(R12)
1499 MOVD R29, -24(R12)
1500 MOVD g, -16(R12)
1501 MOVD R31, -8(R12)
1502 RET
1503 TEXT runtime·elf_restgpr1(SB),NOSPLIT|NOFRAME,$0
1504 // R12 holds the save location
1505 MOVD -144(R12), R14
1506 MOVD -136(R12), R15
1507 MOVD -128(R12), R16
1508 MOVD -120(R12), R17
1509 MOVD -112(R12), R18
1510 MOVD -104(R12), R19
1511 MOVD -96(R12), R20
1512 MOVD -88(R12), R21
1513 MOVD -80(R12), R22
1514 MOVD -72(R12), R23
1515 MOVD -64(R12), R24
1516 MOVD -56(R12), R25
1517 MOVD -48(R12), R26
1518 MOVD -40(R12), R27
1519 MOVD -32(R12), R28
1520 MOVD -24(R12), R29
1521 MOVD -16(R12), g
1522 MOVD -8(R12), R31
1523 RET
1524 TEXT runtime·elf_savefpr(SB),NOSPLIT|NOFRAME,$0
1525 // R0 holds the LR of the caller's caller, R1 holds save location
1526 FMOVD F14, -144(R1)
1527 FMOVD F15, -136(R1)
1528 FMOVD F16, -128(R1)
1529 FMOVD F17, -120(R1)
1530 FMOVD F18, -112(R1)
1531 FMOVD F19, -104(R1)
1532 FMOVD F20, -96(R1)
1533 FMOVD F21, -88(R1)
1534 FMOVD F22, -80(R1)
1535 FMOVD F23, -72(R1)
1536 FMOVD F24, -64(R1)
1537 FMOVD F25, -56(R1)
1538 FMOVD F26, -48(R1)
1539 FMOVD F27, -40(R1)
1540 FMOVD F28, -32(R1)
1541 FMOVD F29, -24(R1)
1542 FMOVD F30, -16(R1)
1543 FMOVD F31, -8(R1)
1544 MOVD R0, 16(R1)
1545 RET
1546 TEXT runtime·elf_restfpr(SB),NOSPLIT|NOFRAME,$0
1547 // R1 holds save location. This returns to the LR saved on stack (bypassing the caller)
1548 FMOVD -144(R1), F14
1549 FMOVD -136(R1), F15
1550 FMOVD -128(R1), F16
1551 FMOVD -120(R1), F17
1552 FMOVD -112(R1), F18
1553 FMOVD -104(R1), F19
1554 FMOVD -96(R1), F20
1555 FMOVD -88(R1), F21
1556 FMOVD -80(R1), F22
1557 FMOVD -72(R1), F23
1558 FMOVD -64(R1), F24
1559 FMOVD -56(R1), F25
1560 FMOVD -48(R1), F26
1561 FMOVD -40(R1), F27
1562 FMOVD -32(R1), F28
1563 FMOVD -24(R1), F29
1564 FMOVD -16(R1), F30
1565 FMOVD -8(R1), F31
1566 MOVD 16(R1), R0 // Load and return to saved LR
1567 MOVD R0, LR
1568 RET
1569 TEXT runtime·elf_savevr(SB),NOSPLIT|NOFRAME,$0
1570 // R0 holds the save location, R12 is clobbered
1571 MOVD $-192, R12
1572 STVX V20, (R0+R12)
1573 MOVD $-176, R12
1574 STVX V21, (R0+R12)
1575 MOVD $-160, R12
1576 STVX V22, (R0+R12)
1577 MOVD $-144, R12
1578 STVX V23, (R0+R12)
1579 MOVD $-128, R12
1580 STVX V24, (R0+R12)
1581 MOVD $-112, R12
1582 STVX V25, (R0+R12)
1583 MOVD $-96, R12
1584 STVX V26, (R0+R12)
1585 MOVD $-80, R12
1586 STVX V27, (R0+R12)
1587 MOVD $-64, R12
1588 STVX V28, (R0+R12)
1589 MOVD $-48, R12
1590 STVX V29, (R0+R12)
1591 MOVD $-32, R12
1592 STVX V30, (R0+R12)
1593 MOVD $-16, R12
1594 STVX V31, (R0+R12)
1595 RET
1596 TEXT runtime·elf_restvr(SB),NOSPLIT|NOFRAME,$0
1597 // R0 holds the save location, R12 is clobbered
1598 MOVD $-192, R12
1599 LVX (R0+R12), V20
1600 MOVD $-176, R12
1601 LVX (R0+R12), V21
1602 MOVD $-160, R12
1603 LVX (R0+R12), V22
1604 MOVD $-144, R12
1605 LVX (R0+R12), V23
1606 MOVD $-128, R12
1607 LVX (R0+R12), V24
1608 MOVD $-112, R12
1609 LVX (R0+R12), V25
1610 MOVD $-96, R12
1611 LVX (R0+R12), V26
1612 MOVD $-80, R12
1613 LVX (R0+R12), V27
1614 MOVD $-64, R12
1615 LVX (R0+R12), V28
1616 MOVD $-48, R12
1617 LVX (R0+R12), V29
1618 MOVD $-32, R12
1619 LVX (R0+R12), V30
1620 MOVD $-16, R12
1621 LVX (R0+R12), V31
1622 RET
1623
View as plain text