Text file
src/runtime/asm_riscv64.s
1 // Copyright 2017 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "funcdata.h"
7 #include "textflag.h"
8 #include "cgo/abi_riscv64.h"
9
10
11 // When building with -buildmode=c-shared, this symbol is called when the shared
12 // library is loaded.
13 TEXT _rt0_riscv64_lib(SB),NOSPLIT,$224
14 // Preserve callee-save registers, along with X1 (LR).
15 MOV X1, (8*3)(X2)
16 SAVE_GPR((8*4))
17 SAVE_FPR((8*16))
18
19 // Initialize g as nil in case of using g later e.g. sigaction in cgo_sigaction.go
20 MOV X0, g
21
22 MOV A0, _rt0_riscv64_lib_argc<>(SB)
23 MOV A1, _rt0_riscv64_lib_argv<>(SB)
24
25 MOV $runtime·libInit(SB), T1
26 JALR RA, T1
27
28 // Restore callee-save registers, along with X1 (LR).
29 MOV (8*3)(X2), X1
30 RESTORE_GPR((8*4))
31 RESTORE_FPR((8*16))
32
33 RET
34
35 TEXT runtime·rt0_lib_go<ABIInternal>(SB),NOSPLIT,$0
36 MOV _rt0_riscv64_lib_argc<>(SB), A0
37 MOV _rt0_riscv64_lib_argv<>(SB), A1
38 MOV $runtime·rt0_go(SB), T0
39 JALR ZERO, T0
40
41 DATA _rt0_riscv64_lib_argc<>(SB)/8, $0
42 GLOBL _rt0_riscv64_lib_argc<>(SB),NOPTR, $8
43 DATA _rt0_riscv64_lib_argv<>(SB)/8, $0
44 GLOBL _rt0_riscv64_lib_argv<>(SB),NOPTR, $8
45
46 // func rt0_go()
47 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
48 // X2 = stack; A0 = argc; A1 = argv
49 SUB $24, X2
50 MOV A0, 8(X2) // argc
51 MOV A1, 16(X2) // argv
52
53 // create istack out of the given (operating system) stack.
54 // _cgo_init may update stackguard.
55 MOV $runtime·g0(SB), g
56 MOV $(-64*1024), T0
57 ADD T0, X2, T1
58 MOV T1, g_stackguard0(g)
59 MOV T1, g_stackguard1(g)
60 MOV T1, (g_stack+stack_lo)(g)
61 MOV X2, (g_stack+stack_hi)(g)
62
63 // if there is a _cgo_init, call it using the gcc ABI.
64 MOV _cgo_init(SB), T2
65 BEQ T2, ZERO, nocgo
66
67 MOV ZERO, A3 // arg 3: not used
68 MOV ZERO, A2 // arg 2: not used
69 MOV $setg_gcc<>(SB), A1 // arg 1: setg
70 MOV g, A0 // arg 0: G
71 JALR RA, T2
72
73 nocgo:
74 // update stackguard after _cgo_init
75 MOV (g_stack+stack_lo)(g), T0
76 ADD $const_stackGuard, T0
77 MOV T0, g_stackguard0(g)
78 MOV T0, g_stackguard1(g)
79
80 // set the per-goroutine and per-mach "registers"
81 MOV $runtime·m0(SB), T0
82
83 // save m->g0 = g0
84 MOV g, m_g0(T0)
85 // save m0 to g0->m
86 MOV T0, g_m(g)
87
88 CALL runtime·check(SB)
89
90 // args are already prepared
91 CALL runtime·args(SB)
92 CALL runtime·osinit(SB)
93 CALL runtime·schedinit(SB)
94
95 // create a new goroutine to start program
96 MOV $runtime·mainPC(SB), T0 // entry
97 SUB $16, X2
98 MOV T0, 8(X2)
99 MOV ZERO, 0(X2)
100 CALL runtime·newproc(SB)
101 ADD $16, X2
102
103 // start this M
104 CALL runtime·mstart(SB)
105
106 WORD $0 // crash if reached
107 RET
108
109 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
110 CALL runtime·mstart0(SB)
111 RET // not reached
112
113 // void setg_gcc(G*); set g called from gcc with g in A0
114 TEXT setg_gcc<>(SB),NOSPLIT,$0-0
115 MOV A0, g
116 CALL runtime·save_g(SB)
117 RET
118
119 // func cputicks() int64
120 TEXT runtime·cputicks<ABIInternal>(SB),NOSPLIT,$0-0
121 // RDTIME to emulate cpu ticks
122 // RDCYCLE reads counter that is per HART(core) based
123 // according to the riscv manual, see issue 46737
124 RDTIME X10
125 RET
126
127 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
128 // of the G stack. We need to distinguish the routine that
129 // lives at the bottom of the G stack from the one that lives
130 // at the top of the system stack because the one at the top of
131 // the system stack terminates the stack walk (see topofstack()).
132 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
133 UNDEF
134 JALR RA, ZERO // make sure this function is not leaf
135 RET
136
137 // func systemstack(fn func())
138 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
139 MOV fn+0(FP), CTXT // CTXT = fn
140 MOV g_m(g), T0 // T0 = m
141
142 MOV m_gsignal(T0), T1 // T1 = gsignal
143 BEQ g, T1, noswitch
144
145 MOV m_g0(T0), T1 // T1 = g0
146 BEQ g, T1, noswitch
147
148 MOV m_curg(T0), T2
149 BEQ g, T2, switch
150
151 // Bad: g is not gsignal, not g0, not curg. What is it?
152 // Hide call from linker nosplit analysis.
153 MOV $runtime·badsystemstack(SB), T1
154 JALR RA, T1
155
156 switch:
157 // save our state in g->sched. Pretend to
158 // be systemstack_switch if the G stack is scanned.
159 CALL gosave_systemstack_switch<>(SB)
160
161 // switch to g0
162 MOV T1, g
163 CALL runtime·save_g(SB)
164 MOV (g_sched+gobuf_sp)(g), T0
165 MOV T0, X2
166
167 // call target function
168 MOV 0(CTXT), T1 // code pointer
169 JALR RA, T1
170
171 // switch back to g
172 MOV g_m(g), T0
173 MOV m_curg(T0), g
174 CALL runtime·save_g(SB)
175 MOV (g_sched+gobuf_sp)(g), X2
176 MOV ZERO, (g_sched+gobuf_sp)(g)
177 RET
178
179 noswitch:
180 // already on m stack, just call directly
181 // Using a tail call here cleans up tracebacks since we won't stop
182 // at an intermediate systemstack.
183 MOV 0(CTXT), T1 // code pointer
184 ADD $8, X2
185 JMP (T1)
186
187 // func switchToCrashStack0(fn func())
188 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
189 MOV X10, CTXT // context register
190 MOV g_m(g), X11 // curm
191
192 // set g to gcrash
193 MOV $runtime·gcrash(SB), g // g = &gcrash
194 CALL runtime·save_g(SB) // clobbers X31
195 MOV X11, g_m(g) // g.m = curm
196 MOV g, m_g0(X11) // curm.g0 = g
197
198 // switch to crashstack
199 MOV (g_stack+stack_hi)(g), X11
200 SUB $(4*8), X11
201 MOV X11, X2
202
203 // call target function
204 MOV 0(CTXT), X10
205 JALR X1, X10
206
207 // should never return
208 CALL runtime·abort(SB)
209 UNDEF
210
211 /*
212 * support for morestack
213 */
214
215 // Called during function prolog when more stack is needed.
216 // Called with return address (i.e. caller's PC) in X5 (aka T0),
217 // and the LR register contains the caller's LR.
218 //
219 // The traceback routines see morestack on a g0 as being
220 // the top of a stack (for example, morestack calling newstack
221 // calling the scheduler calling newm calling gc), so we must
222 // record an argument size. For that purpose, it has no arguments.
223
224 // func morestack()
225 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
226 // Called from f.
227 // Set g->sched to context in f.
228 MOV X2, (g_sched+gobuf_sp)(g)
229 MOV T0, (g_sched+gobuf_pc)(g)
230 MOV RA, (g_sched+gobuf_lr)(g)
231 MOV CTXT, (g_sched+gobuf_ctxt)(g)
232
233 // Cannot grow scheduler stack (m->g0).
234 MOV g_m(g), A0
235 MOV m_g0(A0), A1
236 BNE g, A1, 3(PC)
237 CALL runtime·badmorestackg0(SB)
238 CALL runtime·abort(SB)
239
240 // Cannot grow signal stack (m->gsignal).
241 MOV m_gsignal(A0), A1
242 BNE g, A1, 3(PC)
243 CALL runtime·badmorestackgsignal(SB)
244 CALL runtime·abort(SB)
245
246 // Called from f.
247 // Set m->morebuf to f's caller.
248 MOV RA, (m_morebuf+gobuf_pc)(A0) // f's caller's PC
249 MOV X2, (m_morebuf+gobuf_sp)(A0) // f's caller's SP
250 MOV g, (m_morebuf+gobuf_g)(A0)
251
252 // Call newstack on m->g0's stack.
253 MOV m_g0(A0), g
254 CALL runtime·save_g(SB)
255 MOV (g_sched+gobuf_sp)(g), X2
256 // Create a stack frame on g0 to call newstack.
257 MOV ZERO, -8(X2) // Zero saved LR in frame
258 SUB $8, X2
259 CALL runtime·newstack(SB)
260
261 // Not reached, but make sure the return PC from the call to newstack
262 // is still in this function, and not the beginning of the next.
263 UNDEF
264
265 // func morestack_noctxt()
266 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
267 // Force SPWRITE. This function doesn't actually write SP,
268 // but it is called with a special calling convention where
269 // the caller doesn't save LR on stack but passes it as a
270 // register, and the unwinder currently doesn't understand.
271 // Make it SPWRITE to stop unwinding. (See issue 54332)
272 MOV X2, X2
273
274 MOV ZERO, CTXT
275 JMP runtime·morestack(SB)
276
277 // AES hashing not implemented for riscv64
278 TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
279 JMP runtime·memhashFallback<ABIInternal>(SB)
280 TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
281 JMP runtime·strhashFallback<ABIInternal>(SB)
282 TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
283 JMP runtime·memhash32Fallback<ABIInternal>(SB)
284 TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
285 JMP runtime·memhash64Fallback<ABIInternal>(SB)
286
287 // restore state from Gobuf; longjmp
288
289 // func gogo(buf *gobuf)
290 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
291 MOV buf+0(FP), T0
292 MOV gobuf_g(T0), T1
293 MOV 0(T1), ZERO // make sure g != nil
294 JMP gogo<>(SB)
295
296 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
297 MOV T1, g
298 CALL runtime·save_g(SB)
299
300 MOV gobuf_sp(T0), X2
301 MOV gobuf_lr(T0), RA
302 MOV gobuf_ctxt(T0), CTXT
303 MOV ZERO, gobuf_sp(T0)
304 MOV ZERO, gobuf_lr(T0)
305 MOV ZERO, gobuf_ctxt(T0)
306 MOV gobuf_pc(T0), T0
307 JALR ZERO, T0
308
309 // func procyieldAsm(cycles uint32)
310 TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0
311 RET
312
313 // Switch to m->g0's stack, call fn(g).
314 // Fn must never return. It should gogo(&g->sched)
315 // to keep running g.
316
317 // func mcall(fn func(*g))
318 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
319 MOV X10, CTXT
320
321 // Save caller state in g->sched
322 MOV X2, (g_sched+gobuf_sp)(g)
323 MOV RA, (g_sched+gobuf_pc)(g)
324 MOV ZERO, (g_sched+gobuf_lr)(g)
325
326 // Switch to m->g0 & its stack, call fn.
327 MOV g, X10
328 MOV g_m(g), T1
329 MOV m_g0(T1), g
330 CALL runtime·save_g(SB)
331 BNE g, X10, 2(PC)
332 JMP runtime·badmcall(SB)
333 MOV 0(CTXT), T1 // code pointer
334 MOV (g_sched+gobuf_sp)(g), X2 // sp = m->g0->sched.sp
335 // we don't need special macro for regabi since arg0(X10) = g
336 SUB $16, X2
337 MOV X10, 8(X2) // setup g
338 MOV ZERO, 0(X2) // clear return address
339 JALR RA, T1
340 JMP runtime·badmcall2(SB)
341
342 // Save state of caller into g->sched,
343 // but using fake PC from systemstack_switch.
344 // Must only be called from functions with no locals ($0)
345 // or else unwinding from systemstack_switch is incorrect.
346 // Smashes X31.
347 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
348 MOV $runtime·systemstack_switch(SB), X31
349 ADD $8, X31 // get past prologue
350 MOV X31, (g_sched+gobuf_pc)(g)
351 MOV X2, (g_sched+gobuf_sp)(g)
352 MOV ZERO, (g_sched+gobuf_lr)(g)
353 // Assert ctxt is zero. See func save.
354 MOV (g_sched+gobuf_ctxt)(g), X31
355 BEQ ZERO, X31, 2(PC)
356 CALL runtime·abort(SB)
357 RET
358
359 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
360 // Call fn(arg) aligned appropriately for the gcc ABI.
361 // Called on a system stack, and there may be no g yet (during needm).
362 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
363 MOV fn+0(FP), X11
364 MOV arg+8(FP), X10
365 JALR RA, (X11)
366 RET
367
368 // func asmcgocall(fn, arg unsafe.Pointer) int32
369 // Call fn(arg) on the scheduler stack,
370 // aligned appropriately for the gcc ABI.
371 // See cgocall.go for more details.
372 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
373 MOV fn+0(FP), X11
374 MOV arg+8(FP), X10
375
376 MOV X2, X8 // save original stack pointer
377 MOV g, X9
378
379 // Figure out if we need to switch to m->g0 stack.
380 // We get called to create new OS threads too, and those
381 // come in on the m->g0 stack already. Or we might already
382 // be on the m->gsignal stack.
383 BEQZ g, nosave
384 MOV g_m(g), X6
385 MOV m_gsignal(X6), X7
386 BEQ X7, g, g0
387 MOV m_g0(X6), X7
388 BEQ X7, g, g0
389
390 CALL gosave_systemstack_switch<>(SB)
391 MOV X7, g
392 CALL runtime·save_g(SB)
393 MOV (g_sched+gobuf_sp)(g), X2
394
395 // Now on a scheduling stack (a pthread-created stack).
396 g0:
397 // Save room for two of our pointers.
398 SUB $16, X2
399 MOV X9, 0(X2) // save old g on stack
400 MOV (g_stack+stack_hi)(X9), X9
401 SUB X8, X9, X8
402 MOV X8, 8(X2) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
403
404 JALR RA, (X11)
405
406 // Restore g, stack pointer. X10 is return value.
407 MOV 0(X2), g
408 CALL runtime·save_g(SB)
409 MOV (g_stack+stack_hi)(g), X5
410 MOV 8(X2), X6
411 SUB X6, X5, X6
412 MOV X6, X2
413
414 MOVW X10, ret+16(FP)
415 RET
416
417 nosave:
418 // Running on a system stack, perhaps even without a g.
419 // Having no g can happen during thread creation or thread teardown.
420 MOV fn+0(FP), X11
421 MOV arg+8(FP), X10
422 MOV X2, X8
423 SUB $16, X2
424 MOV ZERO, 0(X2) // Where above code stores g, in case someone looks during debugging.
425 MOV X8, 8(X2) // Save original stack pointer.
426 JALR RA, (X11)
427 MOV 8(X2), X2 // Restore stack pointer.
428 MOVW X10, ret+16(FP)
429 RET
430
431 // func asminit()
432 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
433 RET
434
435 // reflectcall: call a function with the given argument list
436 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
437 // we don't have variable-sized frames, so we use a small number
438 // of constant-sized-frame functions to encode a few bits of size in the pc.
439 // Caution: ugly multiline assembly macros in your future!
440
441 #define DISPATCH(NAME,MAXSIZE) \
442 MOV $MAXSIZE, T1 \
443 BLTU T1, T0, 3(PC) \
444 MOV $NAME(SB), T2; \
445 JALR ZERO, T2
446 // Note: can't just "BR NAME(SB)" - bad inlining results.
447
448 // func call(stackArgsType *rtype, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
449 TEXT reflect·call(SB), NOSPLIT, $0-0
450 JMP ·reflectcall(SB)
451
452 // func call(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
453 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
454 MOVWU frameSize+32(FP), T0
455 DISPATCH(runtime·call16, 16)
456 DISPATCH(runtime·call32, 32)
457 DISPATCH(runtime·call64, 64)
458 DISPATCH(runtime·call128, 128)
459 DISPATCH(runtime·call256, 256)
460 DISPATCH(runtime·call512, 512)
461 DISPATCH(runtime·call1024, 1024)
462 DISPATCH(runtime·call2048, 2048)
463 DISPATCH(runtime·call4096, 4096)
464 DISPATCH(runtime·call8192, 8192)
465 DISPATCH(runtime·call16384, 16384)
466 DISPATCH(runtime·call32768, 32768)
467 DISPATCH(runtime·call65536, 65536)
468 DISPATCH(runtime·call131072, 131072)
469 DISPATCH(runtime·call262144, 262144)
470 DISPATCH(runtime·call524288, 524288)
471 DISPATCH(runtime·call1048576, 1048576)
472 DISPATCH(runtime·call2097152, 2097152)
473 DISPATCH(runtime·call4194304, 4194304)
474 DISPATCH(runtime·call8388608, 8388608)
475 DISPATCH(runtime·call16777216, 16777216)
476 DISPATCH(runtime·call33554432, 33554432)
477 DISPATCH(runtime·call67108864, 67108864)
478 DISPATCH(runtime·call134217728, 134217728)
479 DISPATCH(runtime·call268435456, 268435456)
480 DISPATCH(runtime·call536870912, 536870912)
481 DISPATCH(runtime·call1073741824, 1073741824)
482 MOV $runtime·badreflectcall(SB), T2
483 JALR ZERO, T2
484
485 #define CALLFN(NAME,MAXSIZE) \
486 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
487 NO_LOCAL_POINTERS; \
488 /* copy arguments to stack */ \
489 MOV stackArgs+16(FP), A1; \
490 MOVWU stackArgsSize+24(FP), A2; \
491 MOV X2, A3; \
492 ADD $8, A3; \
493 ADD A3, A2; \
494 BEQ A3, A2, 6(PC); \
495 MOVBU (A1), A4; \
496 ADD $1, A1; \
497 MOVB A4, (A3); \
498 ADD $1, A3; \
499 JMP -5(PC); \
500 /* set up argument registers */ \
501 MOV regArgs+40(FP), X25; \
502 CALL ·unspillArgs(SB); \
503 /* call function */ \
504 MOV f+8(FP), CTXT; \
505 MOV (CTXT), X25; \
506 PCDATA $PCDATA_StackMapIndex, $0; \
507 JALR RA, X25; \
508 /* copy return values back */ \
509 MOV regArgs+40(FP), X25; \
510 CALL ·spillArgs(SB); \
511 MOV stackArgsType+0(FP), A5; \
512 MOV stackArgs+16(FP), A1; \
513 MOVWU stackArgsSize+24(FP), A2; \
514 MOVWU stackRetOffset+28(FP), A4; \
515 ADD $8, X2, A3; \
516 ADD A4, A3; \
517 ADD A4, A1; \
518 SUB A4, A2; \
519 CALL callRet<>(SB); \
520 RET
521
522 // callRet copies return values back at the end of call*. This is a
523 // separate function so it can allocate stack space for the arguments
524 // to reflectcallmove. It does not follow the Go ABI; it expects its
525 // arguments in registers.
526 TEXT callRet<>(SB), NOSPLIT, $40-0
527 NO_LOCAL_POINTERS
528 MOV A5, 8(X2)
529 MOV A1, 16(X2)
530 MOV A3, 24(X2)
531 MOV A2, 32(X2)
532 MOV X25, 40(X2)
533 CALL runtime·reflectcallmove(SB)
534 RET
535
536 CALLFN(·call16, 16)
537 CALLFN(·call32, 32)
538 CALLFN(·call64, 64)
539 CALLFN(·call128, 128)
540 CALLFN(·call256, 256)
541 CALLFN(·call512, 512)
542 CALLFN(·call1024, 1024)
543 CALLFN(·call2048, 2048)
544 CALLFN(·call4096, 4096)
545 CALLFN(·call8192, 8192)
546 CALLFN(·call16384, 16384)
547 CALLFN(·call32768, 32768)
548 CALLFN(·call65536, 65536)
549 CALLFN(·call131072, 131072)
550 CALLFN(·call262144, 262144)
551 CALLFN(·call524288, 524288)
552 CALLFN(·call1048576, 1048576)
553 CALLFN(·call2097152, 2097152)
554 CALLFN(·call4194304, 4194304)
555 CALLFN(·call8388608, 8388608)
556 CALLFN(·call16777216, 16777216)
557 CALLFN(·call33554432, 33554432)
558 CALLFN(·call67108864, 67108864)
559 CALLFN(·call134217728, 134217728)
560 CALLFN(·call268435456, 268435456)
561 CALLFN(·call536870912, 536870912)
562 CALLFN(·call1073741824, 1073741824)
563
564 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
565 // Must obey the gcc calling convention.
566 TEXT _cgo_topofstack(SB),NOSPLIT,$8
567 // g (X27) and REG_TMP (X31) might be clobbered by load_g.
568 // X27 is callee-save in the gcc calling convention, so save it.
569 MOV g, savedX27-8(SP)
570
571 CALL runtime·load_g(SB)
572 MOV g_m(g), X5
573 MOV m_curg(X5), X5
574 MOV (g_stack+stack_hi)(X5), X10 // return value in X10
575
576 MOV savedX27-8(SP), g
577 RET
578
579 // func goexit(neverCallThisFunction)
580 // The top-most function running on a goroutine, returns to goexit+PCQuantum*2.
581 // Note that the NOPs are written in a manner that will not be compressed,
582 // since the offset must be known by the runtime.
583 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
584 WORD $0x00000013 // NOP
585 JMP runtime·goexit1(SB) // does not return
586 // traceback from goexit1 must hit code range of goexit
587 WORD $0x00000013 // NOP
588
589 // This is called from .init_array and follows the platform, not the Go ABI.
590 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
591 // Use X31 as it is a scratch register in both the Go ABI and psABI.
592 MOV runtime·lastmoduledatap(SB), X31
593 MOV X10, moduledata_next(X31)
594 MOV X10, runtime·lastmoduledatap(SB)
595 RET
596
597 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
598 // See cgocall.go for more details.
599 TEXT ·cgocallback(SB),NOSPLIT,$24-24
600 NO_LOCAL_POINTERS
601
602 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
603 // It is used to dropm while thread is exiting.
604 MOV fn+0(FP), X7
605 BNE ZERO, X7, loadg
606 // Restore the g from frame.
607 MOV frame+8(FP), g
608 JMP dropm
609
610 loadg:
611 // Load m and g from thread-local storage.
612 MOVBU runtime·iscgo(SB), X5
613 BEQ ZERO, X5, nocgo
614 CALL runtime·load_g(SB)
615 nocgo:
616
617 // If g is nil, Go did not create the current thread,
618 // or if this thread never called into Go on pthread platforms.
619 // Call needm to obtain one for temporary use.
620 // In this case, we're running on the thread stack, so there's
621 // lots of space, but the linker doesn't know. Hide the call from
622 // the linker analysis by using an indirect call.
623 BEQ ZERO, g, needm
624
625 MOV g_m(g), X5
626 MOV X5, savedm-8(SP)
627 JMP havem
628
629 needm:
630 MOV g, savedm-8(SP) // g is zero, so is m.
631 MOV $runtime·needAndBindM(SB), X6
632 JALR RA, X6
633
634 // Set m->sched.sp = SP, so that if a panic happens
635 // during the function we are about to execute, it will
636 // have a valid SP to run on the g0 stack.
637 // The next few lines (after the havem label)
638 // will save this SP onto the stack and then write
639 // the same SP back to m->sched.sp. That seems redundant,
640 // but if an unrecovered panic happens, unwindm will
641 // restore the g->sched.sp from the stack location
642 // and then systemstack will try to use it. If we don't set it here,
643 // that restored SP will be uninitialized (typically 0) and
644 // will not be usable.
645 MOV g_m(g), X5
646 MOV m_g0(X5), X6
647 MOV X2, (g_sched+gobuf_sp)(X6)
648
649 havem:
650 // Now there's a valid m, and we're running on its m->g0.
651 // Save current m->g0->sched.sp on stack and then set it to SP.
652 // Save current sp in m->g0->sched.sp in preparation for
653 // switch back to m->curg stack.
654 // NOTE: unwindm knows that the saved g->sched.sp is at 8(X2) aka savedsp-24(SP).
655 MOV m_g0(X5), X6
656 MOV (g_sched+gobuf_sp)(X6), X7
657 MOV X7, savedsp-24(SP) // must match frame size
658 MOV X2, (g_sched+gobuf_sp)(X6)
659
660 // Switch to m->curg stack and call runtime.cgocallbackg.
661 // Because we are taking over the execution of m->curg
662 // but *not* resuming what had been running, we need to
663 // save that information (m->curg->sched) so we can restore it.
664 // We can restore m->curg->sched.sp easily, because calling
665 // runtime.cgocallbackg leaves SP unchanged upon return.
666 // To save m->curg->sched.pc, we push it onto the curg stack and
667 // open a frame the same size as cgocallback's g0 frame.
668 // Once we switch to the curg stack, the pushed PC will appear
669 // to be the return PC of cgocallback, so that the traceback
670 // will seamlessly trace back into the earlier calls.
671 MOV m_curg(X5), g
672 CALL runtime·save_g(SB)
673 MOV (g_sched+gobuf_sp)(g), X6 // prepare stack as X6
674 MOV (g_sched+gobuf_pc)(g), X7
675 MOV X7, -(24+8)(X6) // "saved LR"; must match frame size
676 // Gather our arguments into registers.
677 MOV fn+0(FP), X7
678 MOV frame+8(FP), X8
679 MOV ctxt+16(FP), X9
680 MOV $-(24+8)(X6), X2 // switch stack; must match frame size
681 MOV X7, 8(X2)
682 MOV X8, 16(X2)
683 MOV X9, 24(X2)
684 CALL runtime·cgocallbackg(SB)
685
686 // Restore g->sched (== m->curg->sched) from saved values.
687 MOV 0(X2), X7
688 MOV X7, (g_sched+gobuf_pc)(g)
689 MOV $(24+8)(X2), X6 // must match frame size
690 MOV X6, (g_sched+gobuf_sp)(g)
691
692 // Switch back to m->g0's stack and restore m->g0->sched.sp.
693 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
694 // so we do not have to restore it.)
695 MOV g_m(g), X5
696 MOV m_g0(X5), g
697 CALL runtime·save_g(SB)
698 MOV (g_sched+gobuf_sp)(g), X2
699 MOV savedsp-24(SP), X6 // must match frame size
700 MOV X6, (g_sched+gobuf_sp)(g)
701
702 // If the m on entry was nil, we called needm above to borrow an m,
703 // 1. for the duration of the call on non-pthread platforms,
704 // 2. or the duration of the C thread alive on pthread platforms.
705 // If the m on entry wasn't nil,
706 // 1. the thread might be a Go thread,
707 // 2. or it wasn't the first call from a C thread on pthread platforms,
708 // since then we skip dropm to reuse the m in the first call.
709 MOV savedm-8(SP), X5
710 BNE ZERO, X5, droppedm
711
712 // Skip dropm to reuse it in the next call, when a pthread key has been created.
713 MOV _cgo_pthread_key_created(SB), X5
714 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
715 BEQ ZERO, X5, dropm
716 MOV (X5), X5
717 BNE ZERO, X5, droppedm
718
719 dropm:
720 MOV $runtime·dropm(SB), X6
721 JALR RA, X6
722 droppedm:
723
724 // Done!
725 RET
726
727 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
728 EBREAK
729 RET
730
731 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
732 EBREAK
733 RET
734
735 // void setg(G*); set g. for use by needm.
736 TEXT runtime·setg(SB), NOSPLIT, $0-8
737 MOV gg+0(FP), g
738 // This only happens if iscgo, so jump straight to save_g
739 CALL runtime·save_g(SB)
740 RET
741
742 TEXT ·checkASM(SB),NOSPLIT,$0-1
743 MOV $1, T0
744 MOV T0, ret+0(FP)
745 RET
746
747 // spillArgs stores return values from registers to a *internal/abi.RegArgs in X25.
748 TEXT ·spillArgs(SB),NOSPLIT,$0-0
749 MOV X10, (0*8)(X25)
750 MOV X11, (1*8)(X25)
751 MOV X12, (2*8)(X25)
752 MOV X13, (3*8)(X25)
753 MOV X14, (4*8)(X25)
754 MOV X15, (5*8)(X25)
755 MOV X16, (6*8)(X25)
756 MOV X17, (7*8)(X25)
757 MOV X8, (8*8)(X25)
758 MOV X9, (9*8)(X25)
759 MOV X18, (10*8)(X25)
760 MOV X19, (11*8)(X25)
761 MOV X20, (12*8)(X25)
762 MOV X21, (13*8)(X25)
763 MOV X22, (14*8)(X25)
764 MOV X23, (15*8)(X25)
765 MOVD F10, (16*8)(X25)
766 MOVD F11, (17*8)(X25)
767 MOVD F12, (18*8)(X25)
768 MOVD F13, (19*8)(X25)
769 MOVD F14, (20*8)(X25)
770 MOVD F15, (21*8)(X25)
771 MOVD F16, (22*8)(X25)
772 MOVD F17, (23*8)(X25)
773 MOVD F8, (24*8)(X25)
774 MOVD F9, (25*8)(X25)
775 MOVD F18, (26*8)(X25)
776 MOVD F19, (27*8)(X25)
777 MOVD F20, (28*8)(X25)
778 MOVD F21, (29*8)(X25)
779 MOVD F22, (30*8)(X25)
780 MOVD F23, (31*8)(X25)
781 RET
782
783 // unspillArgs loads args into registers from a *internal/abi.RegArgs in X25.
784 TEXT ·unspillArgs(SB),NOSPLIT,$0-0
785 MOV (0*8)(X25), X10
786 MOV (1*8)(X25), X11
787 MOV (2*8)(X25), X12
788 MOV (3*8)(X25), X13
789 MOV (4*8)(X25), X14
790 MOV (5*8)(X25), X15
791 MOV (6*8)(X25), X16
792 MOV (7*8)(X25), X17
793 MOV (8*8)(X25), X8
794 MOV (9*8)(X25), X9
795 MOV (10*8)(X25), X18
796 MOV (11*8)(X25), X19
797 MOV (12*8)(X25), X20
798 MOV (13*8)(X25), X21
799 MOV (14*8)(X25), X22
800 MOV (15*8)(X25), X23
801 MOVD (16*8)(X25), F10
802 MOVD (17*8)(X25), F11
803 MOVD (18*8)(X25), F12
804 MOVD (19*8)(X25), F13
805 MOVD (20*8)(X25), F14
806 MOVD (21*8)(X25), F15
807 MOVD (22*8)(X25), F16
808 MOVD (23*8)(X25), F17
809 MOVD (24*8)(X25), F8
810 MOVD (25*8)(X25), F9
811 MOVD (26*8)(X25), F18
812 MOVD (27*8)(X25), F19
813 MOVD (28*8)(X25), F20
814 MOVD (29*8)(X25), F21
815 MOVD (30*8)(X25), F22
816 MOVD (31*8)(X25), F23
817 RET
818
819 // gcWriteBarrier informs the GC about heap pointer writes.
820 //
821 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
822 // number of bytes of buffer needed in X24, and returns a pointer
823 // to the buffer space in X24.
824 // It clobbers X31 aka T6 (the linker temp register - REG_TMP).
825 // The act of CALLing gcWriteBarrier will clobber RA (LR).
826 // It does not clobber any other general-purpose registers,
827 // but may clobber others (e.g., floating point registers).
828 TEXT gcWriteBarrier<>(SB),NOSPLIT,$208
829 // Save the registers clobbered by the fast path.
830 MOV A0, 24*8(X2)
831 MOV A1, 25*8(X2)
832 retry:
833 MOV g_m(g), A0
834 MOV m_p(A0), A0
835 MOV (p_wbBuf+wbBuf_next)(A0), A1
836 MOV (p_wbBuf+wbBuf_end)(A0), T6 // T6 is linker temp register (REG_TMP)
837 // Increment wbBuf.next position.
838 ADD X24, A1
839 // Is the buffer full?
840 BLTU T6, A1, flush
841 // Commit to the larger buffer.
842 MOV A1, (p_wbBuf+wbBuf_next)(A0)
843 // Make the return value (the original next position)
844 SUB X24, A1, X24
845 // Restore registers.
846 MOV 24*8(X2), A0
847 MOV 25*8(X2), A1
848 RET
849
850 flush:
851 // Save all general purpose registers since these could be
852 // clobbered by wbBufFlush and were not saved by the caller.
853 MOV T0, 1*8(X2)
854 MOV T1, 2*8(X2)
855 // X0 is zero register
856 // X1 is LR, saved by prologue
857 // X2 is SP
858 // X3 is GP
859 // X4 is TP
860 MOV X7, 3*8(X2)
861 MOV X8, 4*8(X2)
862 MOV X9, 5*8(X2)
863 // X10 already saved (A0)
864 // X11 already saved (A1)
865 MOV X12, 6*8(X2)
866 MOV X13, 7*8(X2)
867 MOV X14, 8*8(X2)
868 MOV X15, 9*8(X2)
869 MOV X16, 10*8(X2)
870 MOV X17, 11*8(X2)
871 MOV X18, 12*8(X2)
872 MOV X19, 13*8(X2)
873 MOV X20, 14*8(X2)
874 MOV X21, 15*8(X2)
875 MOV X22, 16*8(X2)
876 MOV X23, 17*8(X2)
877 MOV X24, 18*8(X2)
878 MOV X25, 19*8(X2)
879 MOV X26, 20*8(X2)
880 // X27 is g.
881 MOV X28, 21*8(X2)
882 MOV X29, 22*8(X2)
883 MOV X30, 23*8(X2)
884 // X31 is tmp register.
885
886 CALL runtime·wbBufFlush(SB)
887
888 MOV 1*8(X2), T0
889 MOV 2*8(X2), T1
890 MOV 3*8(X2), X7
891 MOV 4*8(X2), X8
892 MOV 5*8(X2), X9
893 MOV 6*8(X2), X12
894 MOV 7*8(X2), X13
895 MOV 8*8(X2), X14
896 MOV 9*8(X2), X15
897 MOV 10*8(X2), X16
898 MOV 11*8(X2), X17
899 MOV 12*8(X2), X18
900 MOV 13*8(X2), X19
901 MOV 14*8(X2), X20
902 MOV 15*8(X2), X21
903 MOV 16*8(X2), X22
904 MOV 17*8(X2), X23
905 MOV 18*8(X2), X24
906 MOV 19*8(X2), X25
907 MOV 20*8(X2), X26
908 MOV 21*8(X2), X28
909 MOV 22*8(X2), X29
910 MOV 23*8(X2), X30
911
912 JMP retry
913
914 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
915 MOV $8, X24
916 JMP gcWriteBarrier<>(SB)
917 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
918 MOV $16, X24
919 JMP gcWriteBarrier<>(SB)
920 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
921 MOV $24, X24
922 JMP gcWriteBarrier<>(SB)
923 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
924 MOV $32, X24
925 JMP gcWriteBarrier<>(SB)
926 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
927 MOV $40, X24
928 JMP gcWriteBarrier<>(SB)
929 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
930 MOV $48, X24
931 JMP gcWriteBarrier<>(SB)
932 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
933 MOV $56, X24
934 JMP gcWriteBarrier<>(SB)
935 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
936 MOV $64, X24
937 JMP gcWriteBarrier<>(SB)
938
939 TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$144-0
940 NO_LOCAL_POINTERS
941 // Save all 16 int registers that could have an index in them.
942 // They may be pointers, but if they are they are dead.
943 // Skip X0 aka ZERO, X1 aka LR, X2 aka SP, X3 aka GP, X4 aka TP.
944 MOV X5, 24(X2)
945 MOV X6, 32(X2)
946 MOV X7, 40(X2)
947 MOV X8, 48(X2)
948 MOV X9, 56(X2)
949 MOV X10, 64(X2)
950 MOV X11, 72(X2)
951 MOV X12, 80(X2)
952 MOV X13, 88(X2)
953 MOV X14, 96(X2)
954 MOV X15, 104(X2)
955 MOV X16, 112(X2)
956 MOV X17, 120(X2)
957 MOV X18, 128(X2)
958 MOV X19, 136(X2)
959 MOV X20, 144(X2)
960
961 MOV X1, X10 // PC immediately after call to panicBounds
962 ADD $24, X2, X11 // pointer to save area
963 CALL runtime·panicBounds64<ABIInternal>(SB)
964 RET
965
966 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
967 GLOBL runtime·mainPC(SB),RODATA,$8
968
View as plain text