1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
5 #include "zasm_GOOS_GOARCH.h"
9 // using frame size $-4 means do not save LR on stack.
10 TEXT runtime·rt0_go(SB),NOSPLIT,$-4
13 // copy arguments forward on an even stack
14 // use R13 instead of SP to avoid linker rewriting the offsets
15 MOVW 0(R13), R0 // argc
16 MOVW 4(R13), R1 // argv
17 SUB $64, R13 // plenty of scratch
19 MOVW R0, 60(R13) // save argc, argv away
24 MOVW $runtime·g0(SB), g
25 MOVW $runtime·m0(SB), R8
32 // create istack out of the OS stack
33 MOVW $(-8192+104)(R13), R0
34 MOVW R0, g_stackguard0(g)
35 MOVW R0, g_stackguard1(g)
36 MOVW R0, (g_stack+stack_lo)(g)
37 MOVW R13, (g_stack+stack_hi)(g)
39 BL runtime·emptyfunc(SB) // fault if stack check is wrong
42 // if there is an _cgo_init, call it.
43 MOVW _cgo_init(SB), R4
46 MRC 15, 0, R0, C13, C0, 3 // load TLS base pointer
47 MOVW R0, R3 // arg 3: TLS base pointer
48 MOVW $runtime·tlsg(SB), R2 // arg 2: tlsg
49 MOVW $setg_gcc<>(SB), R1 // arg 1: setg
50 MOVW g, R0 // arg 0: G
51 BL (R4) // will clobber R0-R3
55 // update stackguard after _cgo_init
56 MOVW (g_stack+stack_lo)(g), R0
57 ADD $const_StackGuard, R0
58 MOVW R0, g_stackguard0(g)
59 MOVW R0, g_stackguard1(g)
61 BL runtime·checkgoarm(SB)
71 BL runtime·schedinit(SB)
73 // create a new goroutine to start program
74 MOVW $runtime·main·f(SB), R0
79 MOVW.W R0, -4(R13) // push $0 as guard
80 BL runtime·newproc(SB)
81 MOVW $12(R13), R13 // pop args and LR
88 MOVW R0, (R1) // fail hard
90 DATA runtime·main·f+0(SB)/4,$runtime·main(SB)
91 GLOBL runtime·main·f(SB),RODATA,$4
93 TEXT runtime·breakpoint(SB),NOSPLIT,$0-0
94 // gdb won't skip this breakpoint instruction automatically,
95 // so you must manually "set $pc+=4" to skip it and continue.
97 WORD $0xe125be7f // BKPT 0x5bef, NACL_INSTR_ARM_BREAKPOINT
99 WORD $0xe7f001f0 // undefined instruction that gdb understands is a software breakpoint
103 TEXT runtime·asminit(SB),NOSPLIT,$0-0
104 // disable runfast (flush-to-zero) mode of vfp if runtime.goarm > 5
105 MOVB runtime·goarm(SB), R11
108 WORD $0xeef1ba10 // vmrs r11, fpscr
110 WORD $0xeee1ba10 // vmsr fpscr, r11
117 // void gosave(Gobuf*)
118 // save state in Gobuf; setjmp
119 TEXT runtime·gosave(SB),NOSPLIT,$-4-4
120 MOVW 0(FP), R0 // gobuf
121 MOVW SP, gobuf_sp(R0)
122 MOVW LR, gobuf_pc(R0)
125 MOVW R11, gobuf_lr(R0)
126 MOVW R11, gobuf_ret(R0)
127 MOVW R11, gobuf_ctxt(R0)
131 // restore state from Gobuf; longjmp
132 TEXT runtime·gogo(SB),NOSPLIT,$-4-4
133 MOVW 0(FP), R1 // gobuf
137 // NOTE: We updated g above, and we are about to update SP.
138 // Until LR and PC are also updated, the g/SP/LR/PC quadruple
139 // are out of sync and must not be used as the basis of a traceback.
140 // Sigprof skips the traceback when SP is not within g's bounds,
141 // and when the PC is inside this function, runtime.gogo.
142 // Since we are about to update SP, until we complete runtime.gogo
143 // we must not leave this function. In particular, no calls
144 // after this point: it must be straight-line code until the
145 // final B instruction.
146 // See large comment in sigprof for more details.
147 MOVW gobuf_sp(R1), SP // restore SP
148 MOVW gobuf_lr(R1), LR
149 MOVW gobuf_ret(R1), R0
150 MOVW gobuf_ctxt(R1), R7
152 MOVW R11, gobuf_sp(R1) // clear to help garbage collector
153 MOVW R11, gobuf_ret(R1)
154 MOVW R11, gobuf_lr(R1)
155 MOVW R11, gobuf_ctxt(R1)
156 MOVW gobuf_pc(R1), R11
157 CMP R11, R11 // set condition codes for == test, needed by stack split
160 // func mcall(fn func(*g))
161 // Switch to m->g0's stack, call fn(g).
162 // Fn must never return. It should gogo(&g->sched)
163 // to keep running g.
164 TEXT runtime·mcall(SB),NOSPLIT,$-4-4
165 // Save caller state in g->sched.
166 MOVW SP, (g_sched+gobuf_sp)(g)
167 MOVW LR, (g_sched+gobuf_pc)(g)
169 MOVW R11, (g_sched+gobuf_lr)(g)
170 MOVW g, (g_sched+gobuf_g)(g)
172 // Switch to m->g0 & its stack, call fn.
179 B runtime·badmcall(SB)
180 MOVB runtime·iscgo(SB), R11
182 BL.NE runtime·save_g(SB)
184 MOVW (g_sched+gobuf_sp)(g), SP
190 B runtime·badmcall2(SB)
193 // switchtoM is a dummy routine that onM leaves at the bottom
194 // of the G stack. We need to distinguish the routine that
195 // lives at the bottom of the G stack from the one that lives
196 // at the top of the M stack because the one at the top of
197 // the M stack terminates the stack walk (see topofstack()).
198 TEXT runtime·switchtoM(SB),NOSPLIT,$0-0
200 BL (R0) // clobber lr to ensure push {lr} is kept
203 // func onM_signalok(fn func())
204 TEXT runtime·onM_signalok(SB), NOSPLIT, $-4-4
206 MOVW m_gsignal(R1), R2
218 // func onM(fn func())
219 TEXT runtime·onM(SB),NOSPLIT,$0-4
220 MOVW fn+0(FP), R0 // R0 = fn
221 MOVW g_m(g), R1 // R1 = m
223 MOVW m_g0(R1), R2 // R2 = g0
231 // Not g0, not curg. Must be gsignal, but that's not allowed.
232 // Hide call from linker nosplit analysis.
233 MOVW $runtime·badonm(SB), R0
237 // save our state in g->sched. Pretend to
238 // be switchtoM if the G stack is scanned.
239 MOVW $runtime·switchtoM(SB), R3
240 ADD $4, R3, R3 // get past push {lr}
241 MOVW R3, (g_sched+gobuf_pc)(g)
242 MOVW SP, (g_sched+gobuf_sp)(g)
243 MOVW LR, (g_sched+gobuf_lr)(g)
244 MOVW g, (g_sched+gobuf_g)(g)
251 MOVW (g_sched+gobuf_sp)(R2), R3
252 // make it look like mstart called onM on g0, to stop traceback
254 MOVW $runtime·mstart(SB), R4
258 // call target function
267 MOVW (g_sched+gobuf_sp)(g), SP
269 MOVW R3, (g_sched+gobuf_sp)(g)
279 * support for morestack
282 // Called during function prolog when more stack is needed.
286 // NB. we do not save R0 because we've forced 5c to pass all arguments
288 // using frame size $-4 means do not save LR on stack.
290 // The traceback routines see morestack on a g0 as being
291 // the top of a stack (for example, morestack calling newstack
292 // calling the scheduler calling newm calling gc), so we must
293 // record an argument size. For that purpose, it has no arguments.
294 TEXT runtime·morestack(SB),NOSPLIT,$-4-0
295 // Cannot grow scheduler stack (m->g0).
299 BL.EQ runtime·abort(SB)
301 // Cannot grow signal stack (m->gsignal).
302 MOVW m_gsignal(R8), R4
304 BL.EQ runtime·abort(SB)
307 // Set g->sched to context in f.
308 MOVW R7, (g_sched+gobuf_ctxt)(g)
309 MOVW SP, (g_sched+gobuf_sp)(g)
310 MOVW LR, (g_sched+gobuf_pc)(g)
311 MOVW R3, (g_sched+gobuf_lr)(g)
314 // Set m->morebuf to f's caller.
315 MOVW R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC
316 MOVW SP, (m_morebuf+gobuf_sp)(R8) // f's caller's SP
317 MOVW $4(SP), R3 // f's argument pointer
318 MOVW g, (m_morebuf+gobuf_g)(R8)
320 // Call newstack on m->g0's stack.
323 MOVW (g_sched+gobuf_sp)(g), SP
324 BL runtime·newstack(SB)
326 // Not reached, but make sure the return PC from the call to newstack
327 // is still in this function, and not the beginning of the next.
330 TEXT runtime·morestack_noctxt(SB),NOSPLIT,$-4-0
332 B runtime·morestack(SB)
334 // reflectcall: call a function with the given argument list
335 // func call(f *FuncVal, arg *byte, argsize, retoffset uint32).
336 // we don't have variable-sized frames, so we use a small number
337 // of constant-sized-frame functions to encode a few bits of size in the pc.
338 // Caution: ugly multiline assembly macros in your future!
340 #define DISPATCH(NAME,MAXSIZE) \
343 MOVW $NAME(SB), R1; \
346 TEXT ·reflectcall(SB),NOSPLIT,$-4-16
347 MOVW argsize+8(FP), R0
348 DISPATCH(runtime·call16, 16)
349 DISPATCH(runtime·call32, 32)
350 DISPATCH(runtime·call64, 64)
351 DISPATCH(runtime·call128, 128)
352 DISPATCH(runtime·call256, 256)
353 DISPATCH(runtime·call512, 512)
354 DISPATCH(runtime·call1024, 1024)
355 DISPATCH(runtime·call2048, 2048)
356 DISPATCH(runtime·call4096, 4096)
357 DISPATCH(runtime·call8192, 8192)
358 DISPATCH(runtime·call16384, 16384)
359 DISPATCH(runtime·call32768, 32768)
360 DISPATCH(runtime·call65536, 65536)
361 DISPATCH(runtime·call131072, 131072)
362 DISPATCH(runtime·call262144, 262144)
363 DISPATCH(runtime·call524288, 524288)
364 DISPATCH(runtime·call1048576, 1048576)
365 DISPATCH(runtime·call2097152, 2097152)
366 DISPATCH(runtime·call4194304, 4194304)
367 DISPATCH(runtime·call8388608, 8388608)
368 DISPATCH(runtime·call16777216, 16777216)
369 DISPATCH(runtime·call33554432, 33554432)
370 DISPATCH(runtime·call67108864, 67108864)
371 DISPATCH(runtime·call134217728, 134217728)
372 DISPATCH(runtime·call268435456, 268435456)
373 DISPATCH(runtime·call536870912, 536870912)
374 DISPATCH(runtime·call1073741824, 1073741824)
375 MOVW $runtime·badreflectcall(SB), R1
378 #define CALLFN(NAME,MAXSIZE) \
379 TEXT NAME(SB), WRAPPER, $MAXSIZE-16; \
381 /* copy arguments to stack */ \
382 MOVW argptr+4(FP), R0; \
383 MOVW argsize+8(FP), R2; \
391 /* call function */ \
394 PCDATA $PCDATA_StackMapIndex, $0; \
396 /* copy return values back */ \
397 MOVW argptr+4(FP), R0; \
398 MOVW argsize+8(FP), R2; \
399 MOVW retoffset+12(FP), R3; \
414 CALLFN(·call128, 128)
415 CALLFN(·call256, 256)
416 CALLFN(·call512, 512)
417 CALLFN(·call1024, 1024)
418 CALLFN(·call2048, 2048)
419 CALLFN(·call4096, 4096)
420 CALLFN(·call8192, 8192)
421 CALLFN(·call16384, 16384)
422 CALLFN(·call32768, 32768)
423 CALLFN(·call65536, 65536)
424 CALLFN(·call131072, 131072)
425 CALLFN(·call262144, 262144)
426 CALLFN(·call524288, 524288)
427 CALLFN(·call1048576, 1048576)
428 CALLFN(·call2097152, 2097152)
429 CALLFN(·call4194304, 4194304)
430 CALLFN(·call8388608, 8388608)
431 CALLFN(·call16777216, 16777216)
432 CALLFN(·call33554432, 33554432)
433 CALLFN(·call67108864, 67108864)
434 CALLFN(·call134217728, 134217728)
435 CALLFN(·call268435456, 268435456)
436 CALLFN(·call536870912, 536870912)
437 CALLFN(·call1073741824, 1073741824)
439 // void jmpdefer(fn, sp);
440 // called from deferreturn.
441 // 1. grab stored LR for caller
442 // 2. sub 4 bytes to get back to BL deferreturn
444 // TODO(rsc): Push things on stack and then use pop
445 // to load all registers simultaneously, so that a profiling
446 // interrupt can never see mismatched SP/LR/PC.
447 // (And double-check that pop is atomic in that way.)
448 TEXT runtime·jmpdefer(SB),NOSPLIT,$0-8
450 MOVW $-4(LR), LR // BL deferreturn
453 MOVW $-4(SP), SP // SP is 4 below argp, due to saved LR
457 // Save state of caller into g->sched. Smashes R11.
458 TEXT gosave<>(SB),NOSPLIT,$0
459 MOVW LR, (g_sched+gobuf_pc)(g)
460 MOVW R13, (g_sched+gobuf_sp)(g)
462 MOVW R11, (g_sched+gobuf_lr)(g)
463 MOVW R11, (g_sched+gobuf_ret)(g)
464 MOVW R11, (g_sched+gobuf_ctxt)(g)
467 // asmcgocall(void(*fn)(void*), void *arg)
468 // Call fn(arg) on the scheduler stack,
469 // aligned appropriately for the gcc ABI.
470 // See cgocall.c for more details.
471 TEXT ·asmcgocall(SB),NOSPLIT,$0-8
477 TEXT ·asmcgocall_errno(SB),NOSPLIT,$0-12
484 TEXT asmcgocall<>(SB),NOSPLIT,$0-0
485 // fn in R1, arg in R0.
489 // Figure out if we need to switch to m->g0 stack.
490 // We get called to create new OS threads too, and those
491 // come in on the m->g0 stack already.
501 MOVW (g_sched+gobuf_sp)(g), R13
503 // Now on a scheduling stack (a pthread-created stack).
506 BIC $0x7, R13 // alignment for gcc ABI
507 MOVW R4, 20(R13) // save old g
508 MOVW (g_stack+stack_hi)(R4), R4
510 MOVW R4, 16(R13) // save depth in stack (can't just save SP, as stack might be copied during a callback)
513 // Restore registers, g, stack pointer.
517 MOVW (g_stack+stack_hi)(g), R1
524 // cgocallback(void (*fn)(void*), void *frame, uintptr framesize)
525 // Turn the fn into a Go func (by taking its address) and call
526 // cgocallback_gofunc.
527 TEXT runtime·cgocallback(SB),NOSPLIT,$12-12
532 MOVW framesize+8(FP), R0
534 MOVW $runtime·cgocallback_gofunc(SB), R0
538 // cgocallback_gofunc(void (*fn)(void*), void *frame, uintptr framesize)
539 // See cgocall.c for more details.
540 TEXT ·cgocallback_gofunc(SB),NOSPLIT,$8-12
543 // Load m and g from thread-local storage.
544 MOVB runtime·iscgo(SB), R0
546 BL.NE runtime·load_g(SB)
548 // If g is nil, Go did not create the current thread.
549 // Call needm to obtain one for temporary use.
550 // In this case, we're running on the thread stack, so there's
551 // lots of space, but the linker doesn't know. Hide the call from
552 // the linker analysis by using an indirect call.
555 MOVW g, savedm-4(SP) // g is zero, so is m.
556 MOVW $runtime·needm(SB), R0
559 // Set m->sched.sp = SP, so that if a panic happens
560 // during the function we are about to execute, it will
561 // have a valid SP to run on the g0 stack.
562 // The next few lines (after the havem label)
563 // will save this SP onto the stack and then write
564 // the same SP back to m->sched.sp. That seems redundant,
565 // but if an unrecovered panic happens, unwindm will
566 // restore the g->sched.sp from the stack location
567 // and then onM will try to use it. If we don't set it here,
568 // that restored SP will be uninitialized (typically 0) and
569 // will not be usable.
572 MOVW R13, (g_sched+gobuf_sp)(R3)
576 MOVW R8, savedm-4(SP)
577 // Now there's a valid m, and we're running on its m->g0.
578 // Save current m->g0->sched.sp on stack and then set it to SP.
579 // Save current sp in m->g0->sched.sp in preparation for
580 // switch back to m->curg stack.
581 // NOTE: unwindm knows that the saved g->sched.sp is at 4(R13) aka savedsp-8(SP).
583 MOVW (g_sched+gobuf_sp)(R3), R4
584 MOVW R4, savedsp-8(SP)
585 MOVW R13, (g_sched+gobuf_sp)(R3)
587 // Switch to m->curg stack and call runtime.cgocallbackg.
588 // Because we are taking over the execution of m->curg
589 // but *not* resuming what had been running, we need to
590 // save that information (m->curg->sched) so we can restore it.
591 // We can restore m->curg->sched.sp easily, because calling
592 // runtime.cgocallbackg leaves SP unchanged upon return.
593 // To save m->curg->sched.pc, we push it onto the stack.
594 // This has the added benefit that it looks to the traceback
595 // routine like cgocallbackg is going to return to that
596 // PC (because the frame we allocate below has the same
597 // size as cgocallback_gofunc's frame declared above)
598 // so that the traceback will seamlessly trace back into
599 // the earlier calls.
601 // In the new goroutine, -8(SP) and -4(SP) are unused.
604 MOVW (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
605 MOVW (g_sched+gobuf_pc)(g), R5
608 BL runtime·cgocallbackg(SB)
610 // Restore g->sched (== m->curg->sched) from saved values.
612 MOVW R5, (g_sched+gobuf_pc)(g)
614 MOVW R4, (g_sched+gobuf_sp)(g)
616 // Switch back to m->g0's stack and restore m->g0->sched.sp.
617 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
618 // so we do not have to restore it.)
622 MOVW (g_sched+gobuf_sp)(g), R13
623 MOVW savedsp-8(SP), R4
624 MOVW R4, (g_sched+gobuf_sp)(g)
626 // If the m on entry was nil, we called needm above to borrow an m
627 // for the duration of the call. Since the call is over, return it with dropm.
628 MOVW savedm-4(SP), R6
631 MOVW $runtime·dropm(SB), R0
637 // void setg(G*); set g. for use by needm.
638 TEXT runtime·setg(SB),NOSPLIT,$-4-4
642 TEXT setg<>(SB),NOSPLIT,$-4-0
645 // Save g to thread-local storage.
646 MOVB runtime·iscgo(SB), R0
654 TEXT runtime·getcallerpc(SB),NOSPLIT,$-4-4
659 TEXT runtime·gogetcallerpc(SB),NOSPLIT,$-4-8
663 TEXT runtime·setcallerpc(SB),NOSPLIT,$-4-8
668 TEXT runtime·getcallersp(SB),NOSPLIT,$-4-4
674 // func gogetcallersp(p unsafe.Pointer) uintptr
675 TEXT runtime·gogetcallersp(SB),NOSPLIT,$-4-8
681 TEXT runtime·emptyfunc(SB),0,$0-0
684 TEXT runtime·abort(SB),NOSPLIT,$-4-0
688 // bool armcas(int32 *val, int32 old, int32 new)
696 // To implement runtime·cas in sys_$GOOS_arm.s
697 // using the native instructions, use:
699 // TEXT runtime·cas(SB),NOSPLIT,$0
700 // B runtime·armcas(SB)
702 TEXT runtime·armcas(SB),NOSPLIT,$0-13
703 MOVW valptr+0(FP), R1
721 TEXT runtime·casuintptr(SB),NOSPLIT,$0-13
724 TEXT runtime·atomicloaduintptr(SB),NOSPLIT,$0-8
725 B runtime·atomicload(SB)
727 TEXT runtime·atomicloaduint(SB),NOSPLIT,$0-8
728 B runtime·atomicload(SB)
730 TEXT runtime·atomicstoreuintptr(SB),NOSPLIT,$0-8
731 B runtime·atomicstore(SB)
733 // AES hashing not implemented for ARM
734 TEXT runtime·aeshash(SB),NOSPLIT,$-4-0
737 TEXT runtime·aeshash32(SB),NOSPLIT,$-4-0
740 TEXT runtime·aeshash64(SB),NOSPLIT,$-4-0
743 TEXT runtime·aeshashstr(SB),NOSPLIT,$-4-0
747 TEXT runtime·memeq(SB),NOSPLIT,$-4-13
766 // eqstring tests whether two strings are equal.
767 // See runtime_test.go:eqstring_generic for
768 // equivalent Go code.
769 TEXT runtime·eqstring(SB),NOSPLIT,$-4-17
771 MOVW s2len+12(FP), R1
793 // void setg_gcc(G*); set g called from gcc.
794 TEXT setg_gcc<>(SB),NOSPLIT,$0
798 // TODO: share code with memeq?
799 TEXT bytes·Equal(SB),NOSPLIT,$0
801 MOVW b_len+16(FP), R3
803 CMP R1, R3 // unequal lengths are not equal
812 B.EQ equal // reached the end
828 TEXT bytes·IndexByte(SB),NOSPLIT,$0
831 MOVBU c+12(FP), R2 // byte to find
832 MOVW R0, R4 // store base for later
842 SUB $1, R0 // R0 will be one beyond the position we want
843 SUB R4, R0 // remove base
852 TEXT strings·IndexByte(SB),NOSPLIT,$0
855 MOVBU c+8(FP), R2 // byte to find
856 MOVW R0, R4 // store base for later
866 SUB $1, R0 // R0 will be one beyond the position we want
867 SUB R4, R0 // remove base
876 // A Duff's device for zeroing memory.
877 // The compiler jumps to computed addresses within
878 // this routine to zero chunks of memory. Do not
879 // change this code without also changing the code
880 // in ../../cmd/5g/ggen.c:clearfat.
882 // R1: ptr to memory to be zeroed
883 // R1 is updated as a side effect.
884 TEXT runtime·duffzero(SB),NOSPLIT,$0-0
1015 // A Duff's device for copying memory.
1016 // The compiler jumps to computed addresses within
1017 // this routine to copy chunks of memory. Source
1018 // and destination must not overlap. Do not
1019 // change this code without also changing the code
1020 // in ../../cmd/5g/cgen.c:sgen.
1021 // R0: scratch space
1022 // R1: ptr to source memory
1023 // R2: ptr to destination memory
1024 // R1 and R2 are updated as a side effect
1025 TEXT runtime·duffcopy(SB),NOSPLIT,$0-0
1284 TEXT runtime·fastrand1(SB),NOSPLIT,$-4-4
1286 MOVW m_fastrand(R1), R0
1288 EOR.MI $0x88888eef, R0
1289 MOVW R0, m_fastrand(R1)
1293 TEXT runtime·return0(SB),NOSPLIT,$0
1297 TEXT runtime·procyield(SB),NOSPLIT,$-4
1298 MOVW cycles+0(FP), R1
1307 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
1308 // Must obey the gcc calling convention.
1309 TEXT _cgo_topofstack(SB),NOSPLIT,$8
1310 // R11 and g register are clobbered by load_g. They are
1311 // callee-save in the gcc calling convention, so save them here.
1312 MOVW R11, saveR11-4(SP)
1315 BL runtime·load_g(SB)
1318 MOVW (g_stack+stack_hi)(R0), R0
1321 MOVW saveR11-4(SP), R11
1324 // The top-most function running on a goroutine
1325 // returns to goexit+PCQuantum.
1326 TEXT runtime·goexit(SB),NOSPLIT,$-4-0
1328 BL runtime·goexit1(SB) // does not return