1 // Copyright 2011 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
10 "runtime/internal/atomic"
14 //go:cgo_export_dynamic runtime.end _end
15 //go:cgo_export_dynamic runtime.etext _etext
16 //go:cgo_export_dynamic runtime.edata _edata
18 //go:cgo_import_dynamic libc____errno ___errno "libc.so"
19 //go:cgo_import_dynamic libc_clock_gettime clock_gettime "libc.so"
20 //go:cgo_import_dynamic libc_exit _exit "libc.so"
21 //go:cgo_import_dynamic libc_getcontext getcontext "libc.so"
22 //go:cgo_import_dynamic libc_kill kill "libc.so"
23 //go:cgo_import_dynamic libc_madvise madvise "libc.so"
24 //go:cgo_import_dynamic libc_malloc malloc "libc.so"
25 //go:cgo_import_dynamic libc_mmap mmap "libc.so"
26 //go:cgo_import_dynamic libc_munmap munmap "libc.so"
27 //go:cgo_import_dynamic libc_open open "libc.so"
28 //go:cgo_import_dynamic libc_pthread_attr_destroy pthread_attr_destroy "libc.so"
29 //go:cgo_import_dynamic libc_pthread_attr_getstack pthread_attr_getstack "libc.so"
30 //go:cgo_import_dynamic libc_pthread_attr_init pthread_attr_init "libc.so"
31 //go:cgo_import_dynamic libc_pthread_attr_setdetachstate pthread_attr_setdetachstate "libc.so"
32 //go:cgo_import_dynamic libc_pthread_attr_setstack pthread_attr_setstack "libc.so"
33 //go:cgo_import_dynamic libc_pthread_create pthread_create "libc.so"
34 //go:cgo_import_dynamic libc_pthread_self pthread_self "libc.so"
35 //go:cgo_import_dynamic libc_pthread_kill pthread_kill "libc.so"
36 //go:cgo_import_dynamic libc_raise raise "libc.so"
37 //go:cgo_import_dynamic libc_read read "libc.so"
38 //go:cgo_import_dynamic libc_select select "libc.so"
39 //go:cgo_import_dynamic libc_sched_yield sched_yield "libc.so"
40 //go:cgo_import_dynamic libc_sem_init sem_init "libc.so"
41 //go:cgo_import_dynamic libc_sem_post sem_post "libc.so"
42 //go:cgo_import_dynamic libc_sem_reltimedwait_np sem_reltimedwait_np "libc.so"
43 //go:cgo_import_dynamic libc_sem_wait sem_wait "libc.so"
44 //go:cgo_import_dynamic libc_setitimer setitimer "libc.so"
45 //go:cgo_import_dynamic libc_sigaction sigaction "libc.so"
46 //go:cgo_import_dynamic libc_sigaltstack sigaltstack "libc.so"
47 //go:cgo_import_dynamic libc_sigprocmask sigprocmask "libc.so"
48 //go:cgo_import_dynamic libc_sysconf sysconf "libc.so"
49 //go:cgo_import_dynamic libc_usleep usleep "libc.so"
50 //go:cgo_import_dynamic libc_write write "libc.so"
51 //go:cgo_import_dynamic libc_pipe2 pipe2 "libc.so"
53 //go:linkname libc____errno libc____errno
54 //go:linkname libc_clock_gettime libc_clock_gettime
55 //go:linkname libc_exit libc_exit
56 //go:linkname libc_getcontext libc_getcontext
57 //go:linkname libc_kill libc_kill
58 //go:linkname libc_madvise libc_madvise
59 //go:linkname libc_malloc libc_malloc
60 //go:linkname libc_mmap libc_mmap
61 //go:linkname libc_munmap libc_munmap
62 //go:linkname libc_open libc_open
63 //go:linkname libc_pthread_attr_destroy libc_pthread_attr_destroy
64 //go:linkname libc_pthread_attr_getstack libc_pthread_attr_getstack
65 //go:linkname libc_pthread_attr_init libc_pthread_attr_init
66 //go:linkname libc_pthread_attr_setdetachstate libc_pthread_attr_setdetachstate
67 //go:linkname libc_pthread_attr_setstack libc_pthread_attr_setstack
68 //go:linkname libc_pthread_create libc_pthread_create
69 //go:linkname libc_pthread_self libc_pthread_self
70 //go:linkname libc_pthread_kill libc_pthread_kill
71 //go:linkname libc_raise libc_raise
72 //go:linkname libc_read libc_read
73 //go:linkname libc_select libc_select
74 //go:linkname libc_sched_yield libc_sched_yield
75 //go:linkname libc_sem_init libc_sem_init
76 //go:linkname libc_sem_post libc_sem_post
77 //go:linkname libc_sem_reltimedwait_np libc_sem_reltimedwait_np
78 //go:linkname libc_sem_wait libc_sem_wait
79 //go:linkname libc_setitimer libc_setitimer
80 //go:linkname libc_sigaction libc_sigaction
81 //go:linkname libc_sigaltstack libc_sigaltstack
82 //go:linkname libc_sigprocmask libc_sigprocmask
83 //go:linkname libc_sysconf libc_sysconf
84 //go:linkname libc_usleep libc_usleep
85 //go:linkname libc_write libc_write
86 //go:linkname libc_pipe2 libc_pipe2
99 libc_pthread_attr_destroy,
100 libc_pthread_attr_getstack,
101 libc_pthread_attr_init,
102 libc_pthread_attr_setdetachstate,
103 libc_pthread_attr_setstack,
113 libc_sem_reltimedwait_np,
125 var sigset_all = sigset{[4]uint32{^uint32(0), ^uint32(0), ^uint32(0), ^uint32(0)}}
127 func getPageSize() uintptr {
128 n := int32(sysconf(__SC_PAGESIZE))
136 // Call miniterrno so that we can safely make system calls
137 // before calling minit on m0.
138 asmcgocall(unsafe.Pointer(abi.FuncPCABI0(miniterrno)), unsafe.Pointer(&libc____errno))
141 if physPageSize == 0 {
142 physPageSize = getPageSize()
146 func tstart_sysvicall(newm *m) uint32
148 // May run with m.p==nil, so write barriers are not allowed.
151 func newosproc(mp *m) {
160 if pthread_attr_init(&attr) != 0 {
161 throw("pthread_attr_init")
163 // Allocate a new 2MB stack.
164 if pthread_attr_setstack(&attr, 0, 0x200000) != 0 {
165 throw("pthread_attr_setstack")
167 // Read back the allocated stack.
168 if pthread_attr_getstack(&attr, unsafe.Pointer(&mp.g0.stack.hi), &size) != 0 {
169 throw("pthread_attr_getstack")
171 mp.g0.stack.lo = mp.g0.stack.hi - uintptr(size)
172 if pthread_attr_setdetachstate(&attr, _PTHREAD_CREATE_DETACHED) != 0 {
173 throw("pthread_attr_setdetachstate")
176 // Disable signals during create, so that the new thread starts
177 // with signals disabled. It will enable them in minit.
178 sigprocmask(_SIG_SETMASK, &sigset_all, &oset)
179 ret = retryOnEAGAIN(func() int32 {
180 return pthread_create(&tid, &attr, abi.FuncPCABI0(tstart_sysvicall), unsafe.Pointer(mp))
182 sigprocmask(_SIG_SETMASK, &oset, nil)
184 print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", ret, ")\n")
186 println("runtime: may need to increase max user processes (ulimit -u)")
192 func exitThread(wait *atomic.Uint32) {
193 // We should never reach exitThread on Solaris because we let
194 // libc clean up threads.
198 var urandom_dev = []byte("/dev/urandom\x00")
201 func getRandomData(r []byte) {
202 fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0)
203 n := read(fd, unsafe.Pointer(&r[0]), int32(len(r)))
205 extendRandom(r, int(n))
212 // Called to initialize a new m (including the bootstrap m).
213 // Called on the parent thread (main thread in case of bootstrap), can allocate memory.
214 func mpreinit(mp *m) {
215 mp.gsignal = malg(32 * 1024)
221 // Called to initialize a new m (including the bootstrap m).
222 // Called on the new thread, cannot allocate memory.
224 asmcgocall(unsafe.Pointer(abi.FuncPCABI0(miniterrno)), unsafe.Pointer(&libc____errno))
228 getg().m.procid = uint64(pthread_self())
231 // Called from dropm to undo the effect of an minit.
237 // Called from exitm, but not from drop, to undo the effect of thread-owned
238 // resources in minit, semacreate, or elsewhere. Do not take locks after calling this.
239 func mdestroy(mp *m) {
245 //go:nowritebarrierrec
246 func setsig(i uint32, fn uintptr) {
249 sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK | _SA_RESTART
250 sa.sa_mask = sigset_all
251 if fn == abi.FuncPCABIInternal(sighandler) { // abi.FuncPCABIInternal(sighandler) matches the callers in signal_unix.go
252 fn = abi.FuncPCABI0(sigtramp)
254 *((*uintptr)(unsafe.Pointer(&sa._funcptr))) = fn
255 sigaction(i, &sa, nil)
259 //go:nowritebarrierrec
260 func setsigstack(i uint32) {
262 sigaction(i, nil, &sa)
263 if sa.sa_flags&_SA_ONSTACK != 0 {
266 sa.sa_flags |= _SA_ONSTACK
267 sigaction(i, &sa, nil)
271 //go:nowritebarrierrec
272 func getsig(i uint32) uintptr {
274 sigaction(i, nil, &sa)
275 return *((*uintptr)(unsafe.Pointer(&sa._funcptr)))
278 // setSignalstackSP sets the ss_sp field of a stackt.
281 func setSignalstackSP(s *stackt, sp uintptr) {
282 *(*uintptr)(unsafe.Pointer(&s.ss_sp)) = sp
286 //go:nowritebarrierrec
287 func sigaddset(mask *sigset, i int) {
288 mask.__sigbits[(i-1)/32] |= 1 << ((uint32(i) - 1) & 31)
291 func sigdelset(mask *sigset, i int) {
292 mask.__sigbits[(i-1)/32] &^= 1 << ((uint32(i) - 1) & 31)
296 func (c *sigctxt) fixsigcode(sig uint32) {
299 func setProcessCPUProfiler(hz int32) {
300 setProcessCPUProfilerTimer(hz)
303 func setThreadCPUProfiler(hz int32) {
304 setThreadCPUProfilerHz(hz)
308 func validSIGPROF(mp *m, c *sigctxt) bool {
313 func semacreate(mp *m) {
314 if mp.waitsema != 0 {
320 // Call libc's malloc rather than malloc. This will
321 // allocate space on the C heap. We can't call malloc
322 // here because it could cause a deadlock.
323 mp.libcall.fn = uintptr(unsafe.Pointer(&libc_malloc))
325 mp.scratch = mscratch{}
326 mp.scratch.v[0] = unsafe.Sizeof(*sem)
327 mp.libcall.args = uintptr(unsafe.Pointer(&mp.scratch))
328 asmcgocall(unsafe.Pointer(&asmsysvicall6x), unsafe.Pointer(&mp.libcall))
329 sem = (*semt)(unsafe.Pointer(mp.libcall.r1))
330 if sem_init(sem, 0, 0) != 0 {
333 mp.waitsema = uintptr(unsafe.Pointer(sem))
337 func semasleep(ns int64) int32 {
340 mp.ts.tv_sec = ns / 1000000000
341 mp.ts.tv_nsec = ns % 1000000000
343 mp.libcall.fn = uintptr(unsafe.Pointer(&libc_sem_reltimedwait_np))
345 mp.scratch = mscratch{}
346 mp.scratch.v[0] = mp.waitsema
347 mp.scratch.v[1] = uintptr(unsafe.Pointer(&mp.ts))
348 mp.libcall.args = uintptr(unsafe.Pointer(&mp.scratch))
349 asmcgocall(unsafe.Pointer(&asmsysvicall6x), unsafe.Pointer(&mp.libcall))
351 if *mp.perrno == _ETIMEDOUT || *mp.perrno == _EAGAIN || *mp.perrno == _EINTR {
354 throw("sem_reltimedwait_np")
359 mp.libcall.fn = uintptr(unsafe.Pointer(&libc_sem_wait))
361 mp.scratch = mscratch{}
362 mp.scratch.v[0] = mp.waitsema
363 mp.libcall.args = uintptr(unsafe.Pointer(&mp.scratch))
364 asmcgocall(unsafe.Pointer(&asmsysvicall6x), unsafe.Pointer(&mp.libcall))
365 if mp.libcall.r1 == 0 {
368 if *mp.perrno == _EINTR {
377 func semawakeup(mp *m) {
378 if sem_post((*semt)(unsafe.Pointer(mp.waitsema))) != 0 {
384 func closefd(fd int32) int32 {
385 return int32(sysvicall1(&libc_close, uintptr(fd)))
390 sysvicall1(&libc_exit, uintptr(r))
394 func getcontext(context *ucontext) /* int32 */ {
395 sysvicall1(&libc_getcontext, uintptr(unsafe.Pointer(context)))
399 func madvise(addr unsafe.Pointer, n uintptr, flags int32) {
400 sysvicall3(&libc_madvise, uintptr(addr), uintptr(n), uintptr(flags))
404 func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) (unsafe.Pointer, int) {
405 p, err := doMmap(uintptr(addr), n, uintptr(prot), uintptr(flags), uintptr(fd), uintptr(off))
406 if p == ^uintptr(0) {
409 return unsafe.Pointer(p), 0
414 func doMmap(addr, n, prot, flags, fd, off uintptr) (uintptr, uintptr) {
416 libcall.fn = uintptr(unsafe.Pointer(&libc_mmap))
418 libcall.args = uintptr(noescape(unsafe.Pointer(&addr)))
419 asmcgocall(unsafe.Pointer(&asmsysvicall6x), unsafe.Pointer(&libcall))
420 return libcall.r1, libcall.err
424 func munmap(addr unsafe.Pointer, n uintptr) {
425 sysvicall2(&libc_munmap, uintptr(addr), uintptr(n))
434 func nanotime1() int64 {
436 sysvicall2(&libc_clock_gettime, _CLOCK_MONOTONIC, uintptr(unsafe.Pointer(&ts)))
437 return ts.tv_sec*1e9 + ts.tv_nsec
441 func open(path *byte, mode, perm int32) int32 {
442 return int32(sysvicall3(&libc_open, uintptr(unsafe.Pointer(path)), uintptr(mode), uintptr(perm)))
445 func pthread_attr_destroy(attr *pthreadattr) int32 {
446 return int32(sysvicall1(&libc_pthread_attr_destroy, uintptr(unsafe.Pointer(attr))))
449 func pthread_attr_getstack(attr *pthreadattr, addr unsafe.Pointer, size *uint64) int32 {
450 return int32(sysvicall3(&libc_pthread_attr_getstack, uintptr(unsafe.Pointer(attr)), uintptr(addr), uintptr(unsafe.Pointer(size))))
453 func pthread_attr_init(attr *pthreadattr) int32 {
454 return int32(sysvicall1(&libc_pthread_attr_init, uintptr(unsafe.Pointer(attr))))
457 func pthread_attr_setdetachstate(attr *pthreadattr, state int32) int32 {
458 return int32(sysvicall2(&libc_pthread_attr_setdetachstate, uintptr(unsafe.Pointer(attr)), uintptr(state)))
461 func pthread_attr_setstack(attr *pthreadattr, addr uintptr, size uint64) int32 {
462 return int32(sysvicall3(&libc_pthread_attr_setstack, uintptr(unsafe.Pointer(attr)), uintptr(addr), uintptr(size)))
465 func pthread_create(thread *pthread, attr *pthreadattr, fn uintptr, arg unsafe.Pointer) int32 {
466 return int32(sysvicall4(&libc_pthread_create, uintptr(unsafe.Pointer(thread)), uintptr(unsafe.Pointer(attr)), uintptr(fn), uintptr(arg)))
469 func pthread_self() pthread {
470 return pthread(sysvicall0(&libc_pthread_self))
473 func signalM(mp *m, sig int) {
474 sysvicall2(&libc_pthread_kill, uintptr(pthread(mp.procid)), uintptr(sig))
478 //go:nowritebarrierrec
479 func raise(sig uint32) /* int32 */ {
480 sysvicall1(&libc_raise, uintptr(sig))
483 func raiseproc(sig uint32) /* int32 */ {
484 pid := sysvicall0(&libc_getpid)
485 sysvicall2(&libc_kill, pid, uintptr(sig))
489 func read(fd int32, buf unsafe.Pointer, nbyte int32) int32 {
490 r1, err := sysvicall3Err(&libc_read, uintptr(fd), uintptr(buf), uintptr(nbyte))
491 if c := int32(r1); c >= 0 {
498 func sem_init(sem *semt, pshared int32, value uint32) int32 {
499 return int32(sysvicall3(&libc_sem_init, uintptr(unsafe.Pointer(sem)), uintptr(pshared), uintptr(value)))
503 func sem_post(sem *semt) int32 {
504 return int32(sysvicall1(&libc_sem_post, uintptr(unsafe.Pointer(sem))))
508 func sem_reltimedwait_np(sem *semt, timeout *timespec) int32 {
509 return int32(sysvicall2(&libc_sem_reltimedwait_np, uintptr(unsafe.Pointer(sem)), uintptr(unsafe.Pointer(timeout))))
513 func sem_wait(sem *semt) int32 {
514 return int32(sysvicall1(&libc_sem_wait, uintptr(unsafe.Pointer(sem))))
517 func setitimer(which int32, value *itimerval, ovalue *itimerval) /* int32 */ {
518 sysvicall3(&libc_setitimer, uintptr(which), uintptr(unsafe.Pointer(value)), uintptr(unsafe.Pointer(ovalue)))
522 //go:nowritebarrierrec
523 func sigaction(sig uint32, act *sigactiont, oact *sigactiont) /* int32 */ {
524 sysvicall3(&libc_sigaction, uintptr(sig), uintptr(unsafe.Pointer(act)), uintptr(unsafe.Pointer(oact)))
528 //go:nowritebarrierrec
529 func sigaltstack(ss *stackt, oss *stackt) /* int32 */ {
530 sysvicall2(&libc_sigaltstack, uintptr(unsafe.Pointer(ss)), uintptr(unsafe.Pointer(oss)))
534 //go:nowritebarrierrec
535 func sigprocmask(how int32, set *sigset, oset *sigset) /* int32 */ {
536 sysvicall3(&libc_sigprocmask, uintptr(how), uintptr(unsafe.Pointer(set)), uintptr(unsafe.Pointer(oset)))
539 func sysconf(name int32) int64 {
540 return int64(sysvicall1(&libc_sysconf, uintptr(name)))
543 func usleep1(usec uint32)
546 func usleep_no_g(µs uint32) {
551 func usleep(µs uint32) {
555 func walltime() (sec int64, nsec int32) {
557 sysvicall2(&libc_clock_gettime, _CLOCK_REALTIME, uintptr(unsafe.Pointer(&ts)))
558 return ts.tv_sec, int32(ts.tv_nsec)
562 func write1(fd uintptr, buf unsafe.Pointer, nbyte int32) int32 {
563 r1, err := sysvicall3Err(&libc_write, fd, uintptr(buf), uintptr(nbyte))
564 if c := int32(r1); c >= 0 {
571 func pipe2(flags int32) (r, w int32, errno int32) {
573 _, e := sysvicall2Err(&libc_pipe2, uintptr(noescape(unsafe.Pointer(&p))), uintptr(flags))
574 return p[0], p[1], int32(e)
578 func fcntl(fd, cmd, arg int32) (ret int32, errno int32) {
579 r1, err := sysvicall3Err(&libc_fcntl, uintptr(fd), uintptr(cmd), uintptr(arg))
580 return int32(r1), int32(err)
586 func osyield_no_g() {
592 sysvicall0(&libc_sched_yield)
595 //go:linkname executablePath os.executablePath
596 var executablePath string
598 func sysargs(argc int32, argv **byte) {
601 // skip over argv, envp to get to auxv
602 for argv_index(argv, n) != nil {
606 // skip NULL separator
609 // now argv+n is auxv
610 auxvp := (*[1 << 28]uintptr)(add(unsafe.Pointer(argv), uintptr(n)*goarch.PtrSize))
611 pairs := sysauxv(auxvp[:])
612 auxv = auxvp[: pairs*2 : pairs*2]
616 _AT_NULL = 0 // Terminates the vector
617 _AT_PAGESZ = 6 // Page size in bytes
618 _AT_SUN_EXECNAME = 2014 // exec() path name
621 func sysauxv(auxv []uintptr) (pairs int) {
623 for i = 0; auxv[i] != _AT_NULL; i += 2 {
624 tag, val := auxv[i], auxv[i+1]
628 case _AT_SUN_EXECNAME:
629 executablePath = gostringnocopy((*byte)(unsafe.Pointer(val)))
635 // sigPerThreadSyscall is only used on linux, so we assign a bogus signal
637 const sigPerThreadSyscall = 1 << 31
640 func runPerThreadSyscall() {
641 throw("runPerThreadSyscall only valid on linux")