Text file
src/runtime/asm_riscv64.s
1 // Copyright 2017 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "funcdata.h"
7 #include "textflag.h"
8 #include "cgo/abi_riscv64.h"
9
10
11 // When building with -buildmode=c-shared, this symbol is called when the shared
12 // library is loaded.
13 TEXT _rt0_riscv64_lib(SB),NOSPLIT,$224
14 // Preserve callee-save registers, along with X1 (LR).
15 MOV X1, (8*3)(X2)
16 SAVE_GPR((8*4))
17 SAVE_FPR((8*16))
18
19 // Initialize g as nil in case of using g later e.g. sigaction in cgo_sigaction.go
20 MOV X0, g
21
22 MOV A0, _rt0_riscv64_lib_argc<>(SB)
23 MOV A1, _rt0_riscv64_lib_argv<>(SB)
24
25 MOV $runtime·libInit(SB), T1
26 JALR RA, T1
27
28 // Restore callee-save registers, along with X1 (LR).
29 MOV (8*3)(X2), X1
30 RESTORE_GPR((8*4))
31 RESTORE_FPR((8*16))
32
33 RET
34
35 TEXT runtime·rt0_lib_go<ABIInternal>(SB),NOSPLIT,$0
36 MOV _rt0_riscv64_lib_argc<>(SB), A0
37 MOV _rt0_riscv64_lib_argv<>(SB), A1
38 MOV $runtime·rt0_go(SB), T0
39 JALR ZERO, T0
40
41 DATA _rt0_riscv64_lib_argc<>(SB)/8, $0
42 GLOBL _rt0_riscv64_lib_argc<>(SB),NOPTR, $8
43 DATA _rt0_riscv64_lib_argv<>(SB)/8, $0
44 GLOBL _rt0_riscv64_lib_argv<>(SB),NOPTR, $8
45
46 // func rt0_go()
47 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
48 // X2 = stack; A0 = argc; A1 = argv
49 SUB $24, X2
50 MOV A0, 8(X2) // argc
51 MOV A1, 16(X2) // argv
52
53 // create istack out of the given (operating system) stack.
54 // _cgo_init may update stackguard.
55 MOV $runtime·g0(SB), g
56 MOV $(-64*1024), T0
57 ADD T0, X2, T1
58 MOV T1, g_stackguard0(g)
59 MOV T1, g_stackguard1(g)
60 MOV T1, (g_stack+stack_lo)(g)
61 MOV X2, (g_stack+stack_hi)(g)
62
63 // if there is a _cgo_init, call it using the gcc ABI.
64 MOV _cgo_init(SB), T2
65 BEQ T2, ZERO, nocgo
66
67 MOV ZERO, A3 // arg 3: not used
68 MOV ZERO, A2 // arg 2: not used
69 MOV $setg_gcc<>(SB), A1 // arg 1: setg
70 MOV g, A0 // arg 0: G
71 JALR RA, T2
72
73 nocgo:
74 // update stackguard after _cgo_init
75 MOV (g_stack+stack_lo)(g), T0
76 ADD $const_stackGuard, T0
77 MOV T0, g_stackguard0(g)
78 MOV T0, g_stackguard1(g)
79
80 // set the per-goroutine and per-mach "registers"
81 MOV $runtime·m0(SB), T0
82
83 // save m->g0 = g0
84 MOV g, m_g0(T0)
85 // save m0 to g0->m
86 MOV T0, g_m(g)
87
88 CALL runtime·check(SB)
89
90 // args are already prepared
91 CALL runtime·args(SB)
92 CALL runtime·osinit(SB)
93 CALL runtime·schedinit(SB)
94
95 // create a new goroutine to start program
96 MOV $runtime·mainPC(SB), T0 // entry
97 SUB $16, X2
98 MOV T0, 8(X2)
99 MOV ZERO, 0(X2)
100 CALL runtime·newproc(SB)
101 ADD $16, X2
102
103 // start this M
104 CALL runtime·stackcheck(SB) // fault if stack check is wrong
105 CALL runtime·mstart(SB)
106
107 WORD $0 // crash if reached
108 RET
109
110 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
111 CALL runtime·mstart0(SB)
112 RET // not reached
113
114 // void setg_gcc(G*); set g called from gcc with g in A0
115 TEXT setg_gcc<>(SB),NOSPLIT,$0-0
116 MOV A0, g
117 CALL runtime·save_g(SB)
118 RET
119
120 // func cputicks() int64
121 TEXT runtime·cputicks<ABIInternal>(SB),NOSPLIT,$0-0
122 // RDTIME to emulate cpu ticks
123 // RDCYCLE reads counter that is per HART(core) based
124 // according to the riscv manual, see issue 46737
125 RDTIME X10
126 RET
127
128 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
129 // of the G stack. We need to distinguish the routine that
130 // lives at the bottom of the G stack from the one that lives
131 // at the top of the system stack because the one at the top of
132 // the system stack terminates the stack walk (see topofstack()).
133 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
134 UNDEF
135 JALR RA, ZERO // make sure this function is not leaf
136 RET
137
138 // func systemstack(fn func())
139 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
140 MOV fn+0(FP), CTXT // CTXT = fn
141 MOV g_m(g), T0 // T0 = m
142
143 MOV m_gsignal(T0), T1 // T1 = gsignal
144 BEQ g, T1, noswitch
145
146 MOV m_g0(T0), T1 // T1 = g0
147 BEQ g, T1, noswitch
148
149 MOV m_curg(T0), T2
150 BEQ g, T2, switch
151
152 // Bad: g is not gsignal, not g0, not curg. What is it?
153 // Hide call from linker nosplit analysis.
154 MOV $runtime·badsystemstack(SB), T1
155 JALR RA, T1
156
157 switch:
158 // save our state in g->sched. Pretend to
159 // be systemstack_switch if the G stack is scanned.
160 CALL gosave_systemstack_switch<>(SB)
161
162 // switch to g0
163 MOV T1, g
164 CALL runtime·save_g(SB)
165 MOV (g_sched+gobuf_sp)(g), T0
166 MOV T0, X2
167
168 // call target function
169 MOV 0(CTXT), T1 // code pointer
170 JALR RA, T1
171
172 // switch back to g
173 MOV g_m(g), T0
174 MOV m_curg(T0), g
175 CALL runtime·save_g(SB)
176 MOV (g_sched+gobuf_sp)(g), X2
177 MOV ZERO, (g_sched+gobuf_sp)(g)
178 RET
179
180 noswitch:
181 // already on m stack, just call directly
182 // Using a tail call here cleans up tracebacks since we won't stop
183 // at an intermediate systemstack.
184 MOV 0(CTXT), T1 // code pointer
185 ADD $8, X2
186 JMP (T1)
187
188 // func switchToCrashStack0(fn func())
189 TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8
190 MOV X10, CTXT // context register
191 MOV g_m(g), X11 // curm
192
193 // set g to gcrash
194 MOV $runtime·gcrash(SB), g // g = &gcrash
195 CALL runtime·save_g(SB) // clobbers X31
196 MOV X11, g_m(g) // g.m = curm
197 MOV g, m_g0(X11) // curm.g0 = g
198
199 // switch to crashstack
200 MOV (g_stack+stack_hi)(g), X11
201 SUB $(4*8), X11
202 MOV X11, X2
203
204 // call target function
205 MOV 0(CTXT), X10
206 JALR X1, X10
207
208 // should never return
209 CALL runtime·abort(SB)
210 UNDEF
211
212 /*
213 * support for morestack
214 */
215
216 // Called during function prolog when more stack is needed.
217 // Called with return address (i.e. caller's PC) in X5 (aka T0),
218 // and the LR register contains the caller's LR.
219 //
220 // The traceback routines see morestack on a g0 as being
221 // the top of a stack (for example, morestack calling newstack
222 // calling the scheduler calling newm calling gc), so we must
223 // record an argument size. For that purpose, it has no arguments.
224
225 // func morestack()
226 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
227 // Called from f.
228 // Set g->sched to context in f.
229 MOV X2, (g_sched+gobuf_sp)(g)
230 MOV T0, (g_sched+gobuf_pc)(g)
231 MOV RA, (g_sched+gobuf_lr)(g)
232 MOV CTXT, (g_sched+gobuf_ctxt)(g)
233
234 // Cannot grow scheduler stack (m->g0).
235 MOV g_m(g), A0
236 MOV m_g0(A0), A1
237 BNE g, A1, 3(PC)
238 CALL runtime·badmorestackg0(SB)
239 CALL runtime·abort(SB)
240
241 // Cannot grow signal stack (m->gsignal).
242 MOV m_gsignal(A0), A1
243 BNE g, A1, 3(PC)
244 CALL runtime·badmorestackgsignal(SB)
245 CALL runtime·abort(SB)
246
247 // Called from f.
248 // Set m->morebuf to f's caller.
249 MOV RA, (m_morebuf+gobuf_pc)(A0) // f's caller's PC
250 MOV X2, (m_morebuf+gobuf_sp)(A0) // f's caller's SP
251 MOV g, (m_morebuf+gobuf_g)(A0)
252
253 // Call newstack on m->g0's stack.
254 MOV m_g0(A0), g
255 CALL runtime·save_g(SB)
256 MOV (g_sched+gobuf_sp)(g), X2
257 // Create a stack frame on g0 to call newstack.
258 MOV ZERO, -8(X2) // Zero saved LR in frame
259 SUB $8, X2
260 CALL runtime·newstack(SB)
261
262 // Not reached, but make sure the return PC from the call to newstack
263 // is still in this function, and not the beginning of the next.
264 UNDEF
265
266 // func morestack_noctxt()
267 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
268 // Force SPWRITE. This function doesn't actually write SP,
269 // but it is called with a special calling convention where
270 // the caller doesn't save LR on stack but passes it as a
271 // register, and the unwinder currently doesn't understand.
272 // Make it SPWRITE to stop unwinding. (See issue 54332)
273 MOV X2, X2
274
275 MOV ZERO, CTXT
276 JMP runtime·morestack(SB)
277
278 // check that SP is in range [g->stack.lo, g->stack.hi]
279 TEXT runtime·stackcheck(SB), NOSPLIT|NOFRAME, $0-0
280 MOV (g_stack+stack_hi)(g), A0
281 BGEU A0, X2, 2(PC)
282 CALL runtime·abort(SB)
283
284 MOV (g_stack+stack_lo)(g), A0
285 BGTU X2, A0, 2(PC)
286 CALL runtime·abort(SB)
287 RET
288
289 // restore state from Gobuf; longjmp
290
291 // func gogo(buf *gobuf)
292 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
293 MOV buf+0(FP), T0
294 MOV gobuf_g(T0), T1
295 MOV 0(T1), ZERO // make sure g != nil
296 JMP gogo<>(SB)
297
298 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
299 MOV T1, g
300 CALL runtime·save_g(SB)
301
302 MOV gobuf_sp(T0), X2
303 MOV gobuf_lr(T0), RA
304 MOV gobuf_ctxt(T0), CTXT
305 MOV ZERO, gobuf_sp(T0)
306 MOV ZERO, gobuf_lr(T0)
307 MOV ZERO, gobuf_ctxt(T0)
308 MOV gobuf_pc(T0), T0
309 JALR ZERO, T0
310
311 // func procyieldAsm(cycles uint32)
312 TEXT runtime·procyieldAsm(SB),NOSPLIT,$0-0
313 RET
314
315 // Switch to m->g0's stack, call fn(g).
316 // Fn must never return. It should gogo(&g->sched)
317 // to keep running g.
318
319 // func mcall(fn func(*g))
320 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
321 MOV X10, CTXT
322
323 // Save caller state in g->sched
324 MOV X2, (g_sched+gobuf_sp)(g)
325 MOV RA, (g_sched+gobuf_pc)(g)
326 MOV ZERO, (g_sched+gobuf_lr)(g)
327
328 // Switch to m->g0 & its stack, call fn.
329 MOV g, X10
330 MOV g_m(g), T1
331 MOV m_g0(T1), g
332 CALL runtime·save_g(SB)
333 BNE g, X10, 2(PC)
334 JMP runtime·badmcall(SB)
335 MOV 0(CTXT), T1 // code pointer
336 MOV (g_sched+gobuf_sp)(g), X2 // sp = m->g0->sched.sp
337 // we don't need special macro for regabi since arg0(X10) = g
338 SUB $16, X2
339 MOV X10, 8(X2) // setup g
340 MOV ZERO, 0(X2) // clear return address
341 JALR RA, T1
342 JMP runtime·badmcall2(SB)
343
344 // Save state of caller into g->sched,
345 // but using fake PC from systemstack_switch.
346 // Must only be called from functions with no locals ($0)
347 // or else unwinding from systemstack_switch is incorrect.
348 // Smashes X31.
349 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
350 MOV $runtime·systemstack_switch(SB), X31
351 ADD $8, X31 // get past prologue
352 MOV X31, (g_sched+gobuf_pc)(g)
353 MOV X2, (g_sched+gobuf_sp)(g)
354 MOV ZERO, (g_sched+gobuf_lr)(g)
355 // Assert ctxt is zero. See func save.
356 MOV (g_sched+gobuf_ctxt)(g), X31
357 BEQ ZERO, X31, 2(PC)
358 CALL runtime·abort(SB)
359 RET
360
361 // func asmcgocall_no_g(fn, arg unsafe.Pointer)
362 // Call fn(arg) aligned appropriately for the gcc ABI.
363 // Called on a system stack, and there may be no g yet (during needm).
364 TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
365 MOV fn+0(FP), X11
366 MOV arg+8(FP), X10
367 MOV X2, X9 // save SP in X9 (callee-saved in C ABI)
368 ANDI $~15, X2 // align SP to 16 bytes per C ABI
369 MOV X0, X8 // clear frame pointer register (see asmcgocall)
370 JALR RA, (X11)
371 MOV X9, X2
372 RET
373
374 // func asmcgocall(fn, arg unsafe.Pointer) int32
375 // Call fn(arg) on the scheduler stack,
376 // aligned appropriately for the gcc ABI.
377 // See cgocall.go for more details.
378 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
379 MOV fn+0(FP), X11
380 MOV arg+8(FP), X10
381
382 MOV X2, X8 // save original stack pointer
383 MOV g, X9
384
385 // Figure out if we need to switch to m->g0 stack.
386 // We get called to create new OS threads too, and those
387 // come in on the m->g0 stack already. Or we might already
388 // be on the m->gsignal stack.
389 BEQZ g, nosave
390 MOV g_m(g), X6
391 MOV m_gsignal(X6), X7
392 BEQ X7, g, g0
393 MOV m_g0(X6), X7
394 BEQ X7, g, g0
395
396 CALL gosave_systemstack_switch<>(SB)
397 MOV X7, g
398 CALL runtime·save_g(SB)
399 MOV (g_sched+gobuf_sp)(g), X2
400
401 // Now on a scheduling stack (a pthread-created stack).
402 g0:
403 ANDI $~15, X2 // align SP to 16 bytes per C ABI
404 // Save room for two of our pointers.
405 SUB $16, X2
406 MOV X9, 0(X2) // save old g on stack
407 MOV (g_stack+stack_hi)(X9), X9
408 SUB X8, X9, X8
409 MOV X8, 8(X2) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
410
411 // Clear the frame pointer register before calling into C.
412 // At least some C unwinder does frame pointer unwinding.
413 // As Go currently doesn't use frame pointer on RISCV64,
414 // the C unwinder may see garbage value and may crash.
415 // Zero the frame pointer to tell the unwinder to stop
416 // (it is a stack switch anyway).
417 // If we enable frame pointers in Go, revisit this.
418 MOV X0, X8
419 JALR RA, (X11)
420
421 // Restore g, stack pointer. X10 is return value.
422 MOV 0(X2), g
423 CALL runtime·save_g(SB)
424 MOV (g_stack+stack_hi)(g), X5
425 MOV 8(X2), X6
426 SUB X6, X5, X6
427 MOV X6, X2
428
429 MOVW X10, ret+16(FP)
430 RET
431
432 nosave:
433 // Running on a system stack, perhaps even without a g.
434 // Having no g can happen during thread creation or thread teardown.
435 MOV fn+0(FP), X11
436 MOV arg+8(FP), X10
437 MOV X2, X8
438 ANDI $~15, X2 // align SP to 16 bytes per C ABI
439 SUB $16, X2
440 MOV ZERO, 0(X2) // Where above code stores g, in case someone looks during debugging.
441 MOV X8, 8(X2) // Save original stack pointer.
442 MOV X0, X8 // Clear frame pointer (see above)
443 JALR RA, (X11)
444 MOV 8(X2), X2 // Restore stack pointer.
445 MOVW X10, ret+16(FP)
446 RET
447
448 // func asminit()
449 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
450 RET
451
452 // reflectcall: call a function with the given argument list
453 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
454 // we don't have variable-sized frames, so we use a small number
455 // of constant-sized-frame functions to encode a few bits of size in the pc.
456 // Caution: ugly multiline assembly macros in your future!
457
458 #define DISPATCH(NAME,MAXSIZE) \
459 MOV $MAXSIZE, T1 \
460 BLTU T1, T0, 3(PC) \
461 MOV $NAME(SB), T2; \
462 JALR ZERO, T2
463 // Note: can't just "BR NAME(SB)" - bad inlining results.
464
465 // func call(stackArgsType *rtype, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
466 TEXT reflect·call(SB), NOSPLIT, $0-0
467 JMP ·reflectcall(SB)
468
469 // func call(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
470 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
471 MOVWU frameSize+32(FP), T0
472 DISPATCH(runtime·call16, 16)
473 DISPATCH(runtime·call32, 32)
474 DISPATCH(runtime·call64, 64)
475 DISPATCH(runtime·call128, 128)
476 DISPATCH(runtime·call256, 256)
477 DISPATCH(runtime·call512, 512)
478 DISPATCH(runtime·call1024, 1024)
479 DISPATCH(runtime·call2048, 2048)
480 DISPATCH(runtime·call4096, 4096)
481 DISPATCH(runtime·call8192, 8192)
482 DISPATCH(runtime·call16384, 16384)
483 DISPATCH(runtime·call32768, 32768)
484 DISPATCH(runtime·call65536, 65536)
485 DISPATCH(runtime·call131072, 131072)
486 DISPATCH(runtime·call262144, 262144)
487 DISPATCH(runtime·call524288, 524288)
488 DISPATCH(runtime·call1048576, 1048576)
489 DISPATCH(runtime·call2097152, 2097152)
490 DISPATCH(runtime·call4194304, 4194304)
491 DISPATCH(runtime·call8388608, 8388608)
492 DISPATCH(runtime·call16777216, 16777216)
493 DISPATCH(runtime·call33554432, 33554432)
494 DISPATCH(runtime·call67108864, 67108864)
495 DISPATCH(runtime·call134217728, 134217728)
496 DISPATCH(runtime·call268435456, 268435456)
497 DISPATCH(runtime·call536870912, 536870912)
498 DISPATCH(runtime·call1073741824, 1073741824)
499 MOV $runtime·badreflectcall(SB), T2
500 JALR ZERO, T2
501
502 #define CALLFN(NAME,MAXSIZE) \
503 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
504 NO_LOCAL_POINTERS; \
505 /* copy arguments to stack */ \
506 MOV stackArgs+16(FP), A1; \
507 MOVWU stackArgsSize+24(FP), A2; \
508 ADD $8, X2, A3; \
509 ADD A3, A2; \
510 /* dst is always 8 byte aligned, */ \
511 /* use aligned copies only if src is too */ \
512 AND $7, A1, A4; \
513 BNEZ A4, copy1; \
514 /* A6 = end - 16, loop while A3 <= A6 */ \
515 SUB $16, A2, A6; \
516 BLTU A6, A3, check8; \
517 copy16:; \
518 /* copy 16 bytes a time */ \
519 MOV (0*8)(A1), A4; \
520 MOV (1*8)(A1), A5; \
521 ADD $16, A1; \
522 MOV A4, (0*8)(A3); \
523 MOV A5, (1*8)(A3); \
524 ADD $16, A3; \
525 BGEU A6, A3, copy16; \
526 check8:; \
527 /* copy 8 more bytes if A3 <= end - 8 */ \
528 ADD $8, A6; \
529 BLTU A6, A3, copy1; \
530 MOV (A1), A4; \
531 ADD $8, A1; \
532 MOV A4, (A3); \
533 ADD $8, A3; \
534 copy1:; \
535 /* copy 1 byte a time for the rest */ \
536 BEQ A3, A2, callfn; \
537 copy1loop:; \
538 MOVBU (A1), A4; \
539 ADD $1, A1; \
540 MOVB A4, (A3); \
541 ADD $1, A3; \
542 BNE A3, A2, copy1loop; \
543 callfn:; \
544 /* set up argument registers */ \
545 MOV regArgs+40(FP), X25; \
546 CALL ·unspillArgs(SB); \
547 /* call function */ \
548 MOV f+8(FP), CTXT; \
549 MOV (CTXT), X25; \
550 PCDATA $PCDATA_StackMapIndex, $0; \
551 JALR RA, X25; \
552 /* copy return values back */ \
553 MOV regArgs+40(FP), X25; \
554 CALL ·spillArgs(SB); \
555 MOV stackArgsType+0(FP), A5; \
556 MOV stackArgs+16(FP), A1; \
557 MOVWU stackArgsSize+24(FP), A2; \
558 MOVWU stackRetOffset+28(FP), A4; \
559 ADD $8, X2, A3; \
560 ADD A4, A3; \
561 ADD A4, A1; \
562 SUB A4, A2; \
563 CALL callRet<>(SB); \
564 RET
565
566 // callRet copies return values back at the end of call*. This is a
567 // separate function so it can allocate stack space for the arguments
568 // to reflectcallmove. It does not follow the Go ABI; it expects its
569 // arguments in registers.
570 TEXT callRet<>(SB), NOSPLIT, $40-0
571 NO_LOCAL_POINTERS
572 MOV A5, 8(X2)
573 MOV A1, 16(X2)
574 MOV A3, 24(X2)
575 MOV A2, 32(X2)
576 MOV X25, 40(X2)
577 CALL runtime·reflectcallmove(SB)
578 RET
579
580 CALLFN(·call16, 16)
581 CALLFN(·call32, 32)
582 CALLFN(·call64, 64)
583 CALLFN(·call128, 128)
584 CALLFN(·call256, 256)
585 CALLFN(·call512, 512)
586 CALLFN(·call1024, 1024)
587 CALLFN(·call2048, 2048)
588 CALLFN(·call4096, 4096)
589 CALLFN(·call8192, 8192)
590 CALLFN(·call16384, 16384)
591 CALLFN(·call32768, 32768)
592 CALLFN(·call65536, 65536)
593 CALLFN(·call131072, 131072)
594 CALLFN(·call262144, 262144)
595 CALLFN(·call524288, 524288)
596 CALLFN(·call1048576, 1048576)
597 CALLFN(·call2097152, 2097152)
598 CALLFN(·call4194304, 4194304)
599 CALLFN(·call8388608, 8388608)
600 CALLFN(·call16777216, 16777216)
601 CALLFN(·call33554432, 33554432)
602 CALLFN(·call67108864, 67108864)
603 CALLFN(·call134217728, 134217728)
604 CALLFN(·call268435456, 268435456)
605 CALLFN(·call536870912, 536870912)
606 CALLFN(·call1073741824, 1073741824)
607
608 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
609 // Must obey the gcc calling convention.
610 TEXT _cgo_topofstack(SB),NOSPLIT,$8
611 // g (X27) and REG_TMP (X31) might be clobbered by load_g.
612 // X27 is callee-save in the gcc calling convention, so save it.
613 MOV g, savedX27-8(SP)
614
615 CALL runtime·load_g(SB)
616 MOV g_m(g), X5
617 MOV m_curg(X5), X5
618 MOV (g_stack+stack_hi)(X5), X10 // return value in X10
619
620 MOV savedX27-8(SP), g
621 RET
622
623 // func goexit(neverCallThisFunction)
624 // The top-most function running on a goroutine, returns to goexit+PCQuantum*2.
625 // Note that the NOPs are written in a manner that will not be compressed,
626 // since the offset must be known by the runtime.
627 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
628 WORD $0x00000013 // NOP
629 JMP runtime·goexit1(SB) // does not return
630 // traceback from goexit1 must hit code range of goexit
631 WORD $0x00000013 // NOP
632
633 // This is called from .init_array and follows the platform, not the Go ABI.
634 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
635 // Use X31 as it is a scratch register in both the Go ABI and psABI.
636 MOV runtime·lastmoduledatap(SB), X31
637 MOV X10, moduledata_next(X31)
638 MOV X10, runtime·lastmoduledatap(SB)
639 RET
640
641 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
642 // See cgocall.go for more details.
643 TEXT ·cgocallback(SB),NOSPLIT,$24-24
644 NO_LOCAL_POINTERS
645
646 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
647 // It is used to dropm while thread is exiting.
648 MOV fn+0(FP), X7
649 BNE ZERO, X7, loadg
650 // Restore the g from frame.
651 MOV frame+8(FP), g
652 JMP dropm
653
654 loadg:
655 // Load m and g from thread-local storage.
656 MOVBU runtime·iscgo(SB), X5
657 BEQ ZERO, X5, nocgo
658 CALL runtime·load_g(SB)
659 nocgo:
660
661 // If g is nil, Go did not create the current thread,
662 // or if this thread never called into Go on pthread platforms.
663 // Call needm to obtain one for temporary use.
664 // In this case, we're running on the thread stack, so there's
665 // lots of space, but the linker doesn't know. Hide the call from
666 // the linker analysis by using an indirect call.
667 BEQ ZERO, g, needm
668
669 MOV g_m(g), X5
670 MOV X5, savedm-8(SP)
671 JMP havem
672
673 needm:
674 MOV g, savedm-8(SP) // g is zero, so is m.
675 MOV $runtime·needAndBindM(SB), X6
676 JALR RA, X6
677
678 // Set m->sched.sp = SP, so that if a panic happens
679 // during the function we are about to execute, it will
680 // have a valid SP to run on the g0 stack.
681 // The next few lines (after the havem label)
682 // will save this SP onto the stack and then write
683 // the same SP back to m->sched.sp. That seems redundant,
684 // but if an unrecovered panic happens, unwindm will
685 // restore the g->sched.sp from the stack location
686 // and then systemstack will try to use it. If we don't set it here,
687 // that restored SP will be uninitialized (typically 0) and
688 // will not be usable.
689 MOV g_m(g), X5
690 MOV m_g0(X5), X6
691 MOV X2, (g_sched+gobuf_sp)(X6)
692
693 havem:
694 // Now there's a valid m, and we're running on its m->g0.
695 // Save current m->g0->sched.sp on stack and then set it to SP.
696 // Save current sp in m->g0->sched.sp in preparation for
697 // switch back to m->curg stack.
698 // NOTE: unwindm knows that the saved g->sched.sp is at 8(X2) aka savedsp-24(SP).
699 MOV m_g0(X5), X6
700 MOV (g_sched+gobuf_sp)(X6), X7
701 MOV X7, savedsp-24(SP) // must match frame size
702 MOV X2, (g_sched+gobuf_sp)(X6)
703
704 // Switch to m->curg stack and call runtime.cgocallbackg.
705 // Because we are taking over the execution of m->curg
706 // but *not* resuming what had been running, we need to
707 // save that information (m->curg->sched) so we can restore it.
708 // We can restore m->curg->sched.sp easily, because calling
709 // runtime.cgocallbackg leaves SP unchanged upon return.
710 // To save m->curg->sched.pc, we push it onto the curg stack and
711 // open a frame the same size as cgocallback's g0 frame.
712 // Once we switch to the curg stack, the pushed PC will appear
713 // to be the return PC of cgocallback, so that the traceback
714 // will seamlessly trace back into the earlier calls.
715 MOV m_curg(X5), g
716 CALL runtime·save_g(SB)
717 MOV (g_sched+gobuf_sp)(g), X6 // prepare stack as X6
718 MOV (g_sched+gobuf_pc)(g), X7
719 MOV X7, -(24+8)(X6) // "saved LR"; must match frame size
720 // Gather our arguments into registers.
721 MOV fn+0(FP), X7
722 MOV frame+8(FP), X8
723 MOV ctxt+16(FP), X9
724 MOV $-(24+8)(X6), X2 // switch stack; must match frame size
725 MOV X7, 8(X2)
726 MOV X8, 16(X2)
727 MOV X9, 24(X2)
728 CALL runtime·cgocallbackg(SB)
729
730 // Restore g->sched (== m->curg->sched) from saved values.
731 MOV 0(X2), X7
732 MOV X7, (g_sched+gobuf_pc)(g)
733 MOV $(24+8)(X2), X6 // must match frame size
734 MOV X6, (g_sched+gobuf_sp)(g)
735
736 // Switch back to m->g0's stack and restore m->g0->sched.sp.
737 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
738 // so we do not have to restore it.)
739 MOV g_m(g), X5
740 MOV m_g0(X5), g
741 CALL runtime·save_g(SB)
742 MOV (g_sched+gobuf_sp)(g), X2
743 MOV savedsp-24(SP), X6 // must match frame size
744 MOV X6, (g_sched+gobuf_sp)(g)
745
746 // If the m on entry was nil, we called needm above to borrow an m,
747 // 1. for the duration of the call on non-pthread platforms,
748 // 2. or the duration of the C thread alive on pthread platforms.
749 // If the m on entry wasn't nil,
750 // 1. the thread might be a Go thread,
751 // 2. or it wasn't the first call from a C thread on pthread platforms,
752 // since then we skip dropm to reuse the m in the first call.
753 MOV savedm-8(SP), X5
754 BNE ZERO, X5, droppedm
755
756 // Skip dropm to reuse it in the next call, when a pthread key has been created.
757 MOV _cgo_pthread_key_created(SB), X5
758 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
759 BEQ ZERO, X5, dropm
760 MOV (X5), X5
761 BNE ZERO, X5, droppedm
762
763 dropm:
764 MOV $runtime·dropm(SB), X6
765 JALR RA, X6
766 droppedm:
767
768 // Done!
769 RET
770
771 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
772 EBREAK
773 RET
774
775 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
776 EBREAK
777 RET
778
779 // void setg(G*); set g. for use by needm.
780 TEXT runtime·setg(SB), NOSPLIT, $0-8
781 MOV gg+0(FP), g
782 // This only happens if iscgo, so jump straight to save_g
783 CALL runtime·save_g(SB)
784 RET
785
786 // spillArgs stores return values from registers to a *internal/abi.RegArgs in X25.
787 TEXT ·spillArgs(SB),NOSPLIT,$0-0
788 MOV X10, (0*8)(X25)
789 MOV X11, (1*8)(X25)
790 MOV X12, (2*8)(X25)
791 MOV X13, (3*8)(X25)
792 MOV X14, (4*8)(X25)
793 MOV X15, (5*8)(X25)
794 MOV X16, (6*8)(X25)
795 MOV X17, (7*8)(X25)
796 MOV X8, (8*8)(X25)
797 MOV X9, (9*8)(X25)
798 MOV X18, (10*8)(X25)
799 MOV X19, (11*8)(X25)
800 MOV X20, (12*8)(X25)
801 MOV X21, (13*8)(X25)
802 MOV X22, (14*8)(X25)
803 MOV X23, (15*8)(X25)
804 MOVD F10, (16*8)(X25)
805 MOVD F11, (17*8)(X25)
806 MOVD F12, (18*8)(X25)
807 MOVD F13, (19*8)(X25)
808 MOVD F14, (20*8)(X25)
809 MOVD F15, (21*8)(X25)
810 MOVD F16, (22*8)(X25)
811 MOVD F17, (23*8)(X25)
812 MOVD F8, (24*8)(X25)
813 MOVD F9, (25*8)(X25)
814 MOVD F18, (26*8)(X25)
815 MOVD F19, (27*8)(X25)
816 MOVD F20, (28*8)(X25)
817 MOVD F21, (29*8)(X25)
818 MOVD F22, (30*8)(X25)
819 MOVD F23, (31*8)(X25)
820 RET
821
822 // unspillArgs loads args into registers from a *internal/abi.RegArgs in X25.
823 TEXT ·unspillArgs(SB),NOSPLIT,$0-0
824 MOV (0*8)(X25), X10
825 MOV (1*8)(X25), X11
826 MOV (2*8)(X25), X12
827 MOV (3*8)(X25), X13
828 MOV (4*8)(X25), X14
829 MOV (5*8)(X25), X15
830 MOV (6*8)(X25), X16
831 MOV (7*8)(X25), X17
832 MOV (8*8)(X25), X8
833 MOV (9*8)(X25), X9
834 MOV (10*8)(X25), X18
835 MOV (11*8)(X25), X19
836 MOV (12*8)(X25), X20
837 MOV (13*8)(X25), X21
838 MOV (14*8)(X25), X22
839 MOV (15*8)(X25), X23
840 MOVD (16*8)(X25), F10
841 MOVD (17*8)(X25), F11
842 MOVD (18*8)(X25), F12
843 MOVD (19*8)(X25), F13
844 MOVD (20*8)(X25), F14
845 MOVD (21*8)(X25), F15
846 MOVD (22*8)(X25), F16
847 MOVD (23*8)(X25), F17
848 MOVD (24*8)(X25), F8
849 MOVD (25*8)(X25), F9
850 MOVD (26*8)(X25), F18
851 MOVD (27*8)(X25), F19
852 MOVD (28*8)(X25), F20
853 MOVD (29*8)(X25), F21
854 MOVD (30*8)(X25), F22
855 MOVD (31*8)(X25), F23
856 RET
857
858 // gcWriteBarrier informs the GC about heap pointer writes.
859 //
860 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
861 // number of bytes of buffer needed in X24, and returns a pointer
862 // to the buffer space in X24.
863 // It clobbers X31 aka T6 (the linker temp register - REG_TMP).
864 // The act of CALLing gcWriteBarrier will clobber RA (LR).
865 // It does not clobber any other general-purpose registers,
866 // but may clobber others (e.g., floating point registers).
867 TEXT gcWriteBarrier<>(SB),NOSPLIT,$208
868 // Save the registers clobbered by the fast path.
869 MOV A0, 24*8(X2)
870 MOV A1, 25*8(X2)
871 retry:
872 MOV g_m(g), A0
873 MOV m_p(A0), A0
874 MOV (p_wbBuf+wbBuf_next)(A0), A1
875 MOV (p_wbBuf+wbBuf_end)(A0), T6 // T6 is linker temp register (REG_TMP)
876 // Increment wbBuf.next position.
877 ADD X24, A1
878 // Is the buffer full?
879 BLTU T6, A1, flush
880 // Commit to the larger buffer.
881 MOV A1, (p_wbBuf+wbBuf_next)(A0)
882 // Make the return value (the original next position)
883 SUB X24, A1, X24
884 // Restore registers.
885 MOV 24*8(X2), A0
886 MOV 25*8(X2), A1
887 RET
888
889 flush:
890 // Save all general purpose registers since these could be
891 // clobbered by wbBufFlush and were not saved by the caller.
892 MOV T0, 1*8(X2)
893 MOV T1, 2*8(X2)
894 // X0 is zero register
895 // X1 is LR, saved by prologue
896 // X2 is SP
897 // X3 is GP
898 // X4 is TP
899 MOV X7, 3*8(X2)
900 MOV X8, 4*8(X2)
901 MOV X9, 5*8(X2)
902 // X10 already saved (A0)
903 // X11 already saved (A1)
904 MOV X12, 6*8(X2)
905 MOV X13, 7*8(X2)
906 MOV X14, 8*8(X2)
907 MOV X15, 9*8(X2)
908 MOV X16, 10*8(X2)
909 MOV X17, 11*8(X2)
910 MOV X18, 12*8(X2)
911 MOV X19, 13*8(X2)
912 MOV X20, 14*8(X2)
913 MOV X21, 15*8(X2)
914 MOV X22, 16*8(X2)
915 MOV X23, 17*8(X2)
916 MOV X24, 18*8(X2)
917 MOV X25, 19*8(X2)
918 MOV X26, 20*8(X2)
919 // X27 is g.
920 MOV X28, 21*8(X2)
921 MOV X29, 22*8(X2)
922 MOV X30, 23*8(X2)
923 // X31 is tmp register.
924
925 CALL runtime·wbBufFlush(SB)
926
927 MOV 1*8(X2), T0
928 MOV 2*8(X2), T1
929 MOV 3*8(X2), X7
930 MOV 4*8(X2), X8
931 MOV 5*8(X2), X9
932 MOV 6*8(X2), X12
933 MOV 7*8(X2), X13
934 MOV 8*8(X2), X14
935 MOV 9*8(X2), X15
936 MOV 10*8(X2), X16
937 MOV 11*8(X2), X17
938 MOV 12*8(X2), X18
939 MOV 13*8(X2), X19
940 MOV 14*8(X2), X20
941 MOV 15*8(X2), X21
942 MOV 16*8(X2), X22
943 MOV 17*8(X2), X23
944 MOV 18*8(X2), X24
945 MOV 19*8(X2), X25
946 MOV 20*8(X2), X26
947 MOV 21*8(X2), X28
948 MOV 22*8(X2), X29
949 MOV 23*8(X2), X30
950
951 JMP retry
952
953 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
954 MOV $8, X24
955 JMP gcWriteBarrier<>(SB)
956 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
957 MOV $16, X24
958 JMP gcWriteBarrier<>(SB)
959 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
960 MOV $24, X24
961 JMP gcWriteBarrier<>(SB)
962 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
963 MOV $32, X24
964 JMP gcWriteBarrier<>(SB)
965 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
966 MOV $40, X24
967 JMP gcWriteBarrier<>(SB)
968 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
969 MOV $48, X24
970 JMP gcWriteBarrier<>(SB)
971 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
972 MOV $56, X24
973 JMP gcWriteBarrier<>(SB)
974 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
975 MOV $64, X24
976 JMP gcWriteBarrier<>(SB)
977
978 TEXT runtime·panicBounds<ABIInternal>(SB),NOSPLIT,$144-0
979 NO_LOCAL_POINTERS
980 // Save all 16 int registers that could have an index in them.
981 // They may be pointers, but if they are they are dead.
982 // Skip X0 aka ZERO, X1 aka LR, X2 aka SP, X3 aka GP, X4 aka TP.
983 MOV X5, 24(X2)
984 MOV X6, 32(X2)
985 MOV X7, 40(X2)
986 MOV X8, 48(X2)
987 MOV X9, 56(X2)
988 MOV X10, 64(X2)
989 MOV X11, 72(X2)
990 MOV X12, 80(X2)
991 MOV X13, 88(X2)
992 MOV X14, 96(X2)
993 MOV X15, 104(X2)
994 MOV X16, 112(X2)
995 MOV X17, 120(X2)
996 MOV X18, 128(X2)
997 MOV X19, 136(X2)
998 MOV X20, 144(X2)
999
1000 MOV X1, X10 // PC immediately after call to panicBounds
1001 ADD $24, X2, X11 // pointer to save area
1002 CALL runtime·panicBounds64<ABIInternal>(SB)
1003 RET
1004
1005 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
1006 GLOBL runtime·mainPC(SB),RODATA,$8
1007
View as plain text