// cgo writes a gcc-compiled function named GoF (not p.GoF, since gcc doesn't
// know about packages). The gcc-compiled C function f calls GoF.
//
-// GoF calls crosscall2(_cgoexp_GoF, frame, framesize). Crosscall2
-// (in cgo/gcc_$GOARCH.S, a gcc-compiled assembly file) is a two-argument
-// adapter from the gcc function call ABI to the 6c function call ABI.
-// It is called from gcc to call 6c functions. In this case it calls
-// _cgoexp_GoF(frame, framesize), still running on m->g0's stack
-// and outside the $GOMAXPROCS limit. Thus, this code cannot yet
-// call arbitrary Go code directly and must be careful not to allocate
-// memory or use up m->g0's stack.
+// GoF initializes "frame", a structure containing all of its
+// arguments and slots for p.GoF's results. It calls
+// crosscall2(_cgoexp_GoF, frame, framesize, ctxt) using the gcc ABI.
//
-// _cgoexp_GoF calls runtime.cgocallback(p.GoF, frame, framesize, ctxt).
-// (The reason for having _cgoexp_GoF instead of writing a crosscall3
-// to make this call directly is that _cgoexp_GoF, because it is compiled
-// with 6c instead of gcc, can refer to dotted names like
-// runtime.cgocallback and p.GoF.)
+// crosscall2 (in cgo/asm_$GOARCH.s) is a four-argument adapter from
+// the gcc function call ABI to the gc function call ABI. At this
+// point we're in the Go runtime, but we're still running on m.g0's
+// stack and outside the $GOMAXPROCS limit. crosscall2 calls
+// runtime.cgocallback(_cgoexp_GoF, frame, ctxt) using the gc ABI.
+// (crosscall2's framesize argument is no longer used, but there's one
+// case where SWIG calls crosscall2 directly and expects to pass this
+// argument. See _cgo_panic.)
//
-// runtime.cgocallback (in asm_$GOARCH.s) switches from m->g0's
-// stack to the original g (m->curg)'s stack, on which it calls
-// runtime.cgocallbackg(p.GoF, frame, framesize).
-// As part of the stack switch, runtime.cgocallback saves the current
-// SP as m->g0->sched.sp, so that any use of m->g0's stack during the
-// execution of the callback will be done below the existing stack frames.
-// Before overwriting m->g0->sched.sp, it pushes the old value on the
-// m->g0 stack, so that it can be restored later.
+// runtime.cgocallback (in asm_$GOARCH.s) switches from m.g0's stack
+// to the original g (m.curg)'s stack, on which it calls
+// runtime.cgocallbackg(_cgoexp_GoF, frame, ctxt). As part of the
+// stack switch, runtime.cgocallback saves the current SP as
+// m.g0.sched.sp, so that any use of m.g0's stack during the execution
+// of the callback will be done below the existing stack frames.
+// Before overwriting m.g0.sched.sp, it pushes the old value on the
+// m.g0 stack, so that it can be restored later.
//
// runtime.cgocallbackg (below) is now running on a real goroutine
-// stack (not an m->g0 stack). First it calls runtime.exitsyscall, which will
+// stack (not an m.g0 stack). First it calls runtime.exitsyscall, which will
// block until the $GOMAXPROCS limit allows running this goroutine.
// Once exitsyscall has returned, it is safe to do things like call the memory
-// allocator or invoke the Go callback function p.GoF. runtime.cgocallbackg
-// first defers a function to unwind m->g0.sched.sp, so that if p.GoF
-// panics, m->g0.sched.sp will be restored to its old value: the m->g0 stack
-// and the m->curg stack will be unwound in lock step.
-// Then it calls p.GoF. Finally it pops but does not execute the deferred
-// function, calls runtime.entersyscall, and returns to runtime.cgocallback.
+// allocator or invoke the Go callback function. runtime.cgocallbackg
+// first defers a function to unwind m.g0.sched.sp, so that if p.GoF
+// panics, m.g0.sched.sp will be restored to its old value: the m.g0 stack
+// and the m.curg stack will be unwound in lock step.
+// Then it calls _cgoexp_GoF(frame).
+//
+// _cgoexp_GoF, which was generated by cmd/cgo, unpacks the arguments
+// from frame, calls p.GoF, writes the results back to frame, and
+// returns. Now we start unwinding this whole process.
+//
+// runtime.cgocallbackg pops but does not execute the deferred
+// function to unwind m.g0.sched.sp, calls runtime.entersyscall, and
+// returns to runtime.cgocallback.
//
// After it regains control, runtime.cgocallback switches back to
-// m->g0's stack (the pointer is still in m->g0.sched.sp), restores the old
-// m->g0.sched.sp value from the stack, and returns to _cgoexp_GoF.
+// m.g0's stack (the pointer is still in m.g0.sched.sp), restores the old
+// m.g0.sched.sp value from the stack, and returns to crosscall2.
//
-// _cgoexp_GoF immediately returns to crosscall2, which restores the
-// callee-save registers for gcc and returns to GoF, which returns to f.
+// crosscall2 restores the callee-save registers for gcc and returns
+// to GoF, which unpacks any result values and returns to f.
package runtime
import (
- "runtime/internal/atomic"
+ "internal/goarch"
+ "internal/goexperiment"
"runtime/internal/sys"
"unsafe"
)
// Length must match arg.Max in x_cgo_callers in runtime/cgo/gcc_traceback.c.
type cgoCallers [32]uintptr
+// argset matches runtime/cgo/linux_syscall.c:argset_t
+type argset struct {
+ args unsafe.Pointer
+ retval uintptr
+}
+
+// wrapper for syscall package to call cgocall for libc (cgo) calls.
+//
+//go:linkname syscall_cgocaller syscall.cgocaller
+//go:nosplit
+//go:uintptrescapes
+func syscall_cgocaller(fn unsafe.Pointer, args ...uintptr) uintptr {
+ as := argset{args: unsafe.Pointer(&args[0])}
+ cgocall(fn, unsafe.Pointer(&as))
+ return as.retval
+}
+
+var ncgocall uint64 // number of cgo calls in total for dead m
+
// Call from Go to C.
+//
+// This must be nosplit because it's used for syscalls on some
+// platforms. Syscalls may have untyped arguments on the stack, so
+// it's not safe to grow or scan the stack.
+//
//go:nosplit
func cgocall(fn, arg unsafe.Pointer) int32 {
if !iscgo && GOOS != "solaris" && GOOS != "illumos" && GOOS != "windows" {
mp := getg().m
mp.ncgocall++
- mp.ncgo++
// Reset traceback.
mp.cgoCallers[0] = 0
// saved by entersyscall here.
entersyscall()
+ // Tell asynchronous preemption that we're entering external
+ // code. We do this after entersyscall because this may block
+ // and cause an async preemption to fail, but at this point a
+ // sync preemption will succeed (though this is not a matter
+ // of correctness).
+ osPreemptExtEnter(mp)
+
mp.incgo = true
+ // We use ncgo as a check during execution tracing for whether there is
+ // any C on the call stack, which there will be after this point. If
+ // there isn't, we can use frame pointer unwinding to collect call
+ // stacks efficiently. This will be the case for the first Go-to-C call
+ // on a stack, so it's preferable to update it here, after we emit a
+ // trace event in entersyscall above.
+ mp.ncgo++
+
errno := asmcgocall(fn, arg)
// Update accounting before exitsyscall because exitsyscall may
mp.incgo = false
mp.ncgo--
+ osPreemptExtExit(mp)
+
exitsyscall()
// Note that raceacquire must be called only after exitsyscall has
return errno
}
-// Call from C back to Go.
+// Set or reset the system stack bounds for a callback on sp.
+//
+// Must be nosplit because it is called by needm prior to fully initializing
+// the M.
+//
+//go:nosplit
+func callbackUpdateSystemStack(mp *m, sp uintptr, signal bool) {
+ g0 := mp.g0
+ if sp > g0.stack.lo && sp <= g0.stack.hi {
+ // Stack already in bounds, nothing to do.
+ return
+ }
+
+ if mp.ncgo > 0 {
+ // ncgo > 0 indicates that this M was in Go further up the stack
+ // (it called C and is now receiving a callback). It is not
+ // safe for the C call to change the stack out from under us.
+
+ // Note that this case isn't possible for signal == true, as
+ // that is always passing a new M from needm.
+
+ // Stack is bogus, but reset the bounds anyway so we can print.
+ hi := g0.stack.hi
+ lo := g0.stack.lo
+ g0.stack.hi = sp + 1024
+ g0.stack.lo = sp - 32*1024
+ g0.stackguard0 = g0.stack.lo + stackGuard
+ g0.stackguard1 = g0.stackguard0
+
+ print("M ", mp.id, " procid ", mp.procid, " runtime: cgocallback with sp=", hex(sp), " out of bounds [", hex(lo), ", ", hex(hi), "]")
+ print("\n")
+ exit(2)
+ }
+
+ // This M does not have Go further up the stack. However, it may have
+ // previously called into Go, initializing the stack bounds. Between
+ // that call returning and now the stack may have changed (perhaps the
+ // C thread is running a coroutine library). We need to update the
+ // stack bounds for this case.
+ //
+ // Set the stack bounds to match the current stack. If we don't
+ // actually know how big the stack is, like we don't know how big any
+ // scheduling stack is, but we assume there's at least 32 kB. If we
+ // can get a more accurate stack bound from pthread, use that, provided
+ // it actually contains SP..
+ g0.stack.hi = sp + 1024
+ g0.stack.lo = sp - 32*1024
+ if !signal && _cgo_getstackbound != nil {
+ // Don't adjust if called from the signal handler.
+ // We are on the signal stack, not the pthread stack.
+ // (We could get the stack bounds from sigaltstack, but
+ // we're getting out of the signal handler very soon
+ // anyway. Not worth it.)
+ var bounds [2]uintptr
+ asmcgocall(_cgo_getstackbound, unsafe.Pointer(&bounds))
+ // getstackbound is an unsupported no-op on Windows.
+ //
+ // Don't use these bounds if they don't contain SP. Perhaps we
+ // were called by something not using the standard thread
+ // stack.
+ if bounds[0] != 0 && sp > bounds[0] && sp <= bounds[1] {
+ g0.stack.lo = bounds[0]
+ g0.stack.hi = bounds[1]
+ }
+ }
+ g0.stackguard0 = g0.stack.lo + stackGuard
+ g0.stackguard1 = g0.stackguard0
+}
+
+// Call from C back to Go. fn must point to an ABIInternal Go entry-point.
+//
//go:nosplit
-func cgocallbackg(ctxt uintptr) {
+func cgocallbackg(fn, frame unsafe.Pointer, ctxt uintptr) {
gp := getg()
if gp != gp.m.curg {
println("runtime: bad g in cgocallback")
exit(2)
}
+ sp := gp.m.g0.sched.sp // system sp saved by cgocallback.
+ callbackUpdateSystemStack(gp.m, sp, false)
+
// The call from C is on gp.m's g0 stack, so we must ensure
// that we stay on that M. We have to do this before calling
// exitsyscall, since it would otherwise be free to move us to
- // a different M. The call to unlockOSThread is in unwindm.
+ // a different M. The call to unlockOSThread is in this function
+ // after cgocallbackg1, or in the case of panicking, in unwindm.
lockOSThread()
+ checkm := gp.m
+
// Save current syscall parameters, so m.syscall can be
// used again if callback decide to make syscall.
syscall := gp.m.syscall
savedpc := gp.syscallpc
exitsyscall() // coming out of cgo call
gp.m.incgo = false
+ if gp.m.isextra {
+ gp.m.isExtraInC = false
+ }
- cgocallbackg1(ctxt)
+ osPreemptExtExit(gp.m)
- // At this point unlockOSThread has been called.
+ if gp.nocgocallback {
+ panic("runtime: function marked with #cgo nocallback called back into Go")
+ }
+
+ cgocallbackg1(fn, frame, ctxt)
+
+ // At this point we're about to call unlockOSThread.
// The following code must not change to a different m.
// This is enforced by checking incgo in the schedule function.
-
gp.m.incgo = true
+ unlockOSThread()
+
+ if gp.m.isextra {
+ gp.m.isExtraInC = true
+ }
+
+ if gp.m != checkm {
+ throw("m changed unexpectedly in cgocallbackg")
+ }
+
+ osPreemptExtEnter(gp.m)
+
// going back to cgo call
reentersyscall(savedpc, uintptr(savedsp))
gp.m.syscall = syscall
}
-func cgocallbackg1(ctxt uintptr) {
+func cgocallbackg1(fn, frame unsafe.Pointer, ctxt uintptr) {
gp := getg()
- if gp.m.needextram || atomic.Load(&extraMWaiters) > 0 {
+
+ if gp.m.needextram || extraMWaiters.Load() > 0 {
gp.m.needextram = false
systemstack(newextram)
}
<-main_init_done
}
+ // Check whether the profiler needs to be turned on or off; this route to
+ // run Go code does not use runtime.execute, so bypasses the check there.
+ hz := sched.profilehz
+ if gp.m.profilehz != hz {
+ setThreadCPUProfiler(hz)
+ }
+
// Add entry to defer stack in case of panic.
restore := true
defer unwindm(&restore)
raceacquire(unsafe.Pointer(&racecgosync))
}
- type args struct {
- fn *funcval
- arg unsafe.Pointer
- argsize uintptr
- }
- var cb *args
-
- // Location of callback arguments depends on stack frame layout
- // and size of stack frame of cgocallback_gofunc.
- sp := gp.m.g0.sched.sp
- switch GOARCH {
- default:
- throw("cgocallbackg is unimplemented on arch")
- case "arm":
- // On arm, stack frame is two words and there's a saved LR between
- // SP and the stack frame and between the stack frame and the arguments.
- cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize))
- case "arm64":
- // On arm64, stack frame is four words and there's a saved LR between
- // SP and the stack frame and between the stack frame and the arguments.
- // Additional two words (16-byte alignment) are for saving FP.
- cb = (*args)(unsafe.Pointer(sp + 7*sys.PtrSize))
- case "amd64":
- // On amd64, stack frame is two words, plus caller PC.
- if framepointer_enabled {
- // In this case, there's also saved BP.
- cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize))
- break
- }
- cb = (*args)(unsafe.Pointer(sp + 3*sys.PtrSize))
- case "386":
- // On 386, stack frame is three words, plus caller PC.
- cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize))
- case "ppc64", "ppc64le", "s390x":
- // On ppc64 and s390x, the callback arguments are in the arguments area of
- // cgocallback's stack frame. The stack looks like this:
- // +--------------------+------------------------------+
- // | | ... |
- // | cgoexp_$fn +------------------------------+
- // | | fixed frame area |
- // +--------------------+------------------------------+
- // | | arguments area |
- // | cgocallback +------------------------------+ <- sp + 2*minFrameSize + 2*ptrSize
- // | | fixed frame area |
- // +--------------------+------------------------------+ <- sp + minFrameSize + 2*ptrSize
- // | | local variables (2 pointers) |
- // | cgocallback_gofunc +------------------------------+ <- sp + minFrameSize
- // | | fixed frame area |
- // +--------------------+------------------------------+ <- sp
- cb = (*args)(unsafe.Pointer(sp + 2*sys.MinFrameSize + 2*sys.PtrSize))
- case "mips64", "mips64le":
- // On mips64x, stack frame is two words and there's a saved LR between
- // SP and the stack frame and between the stack frame and the arguments.
- cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize))
- case "mips", "mipsle":
- // On mipsx, stack frame is two words and there's a saved LR between
- // SP and the stack frame and between the stack frame and the arguments.
- cb = (*args)(unsafe.Pointer(sp + 4*sys.PtrSize))
- }
-
- // Invoke callback.
- // NOTE(rsc): passing nil for argtype means that the copying of the
- // results back into cb.arg happens without any corresponding write barriers.
- // For cgo, cb.arg points into a C stack frame and therefore doesn't
- // hold any pointers that the GC can find anyway - the write barrier
- // would be a no-op.
- reflectcall(nil, unsafe.Pointer(cb.fn), cb.arg, uint32(cb.argsize), 0)
+ // Invoke callback. This function is generated by cmd/cgo and
+ // will unpack the argument frame and call the Go function.
+ var cb func(frame unsafe.Pointer)
+ cbFV := funcval{uintptr(fn)}
+ *(*unsafe.Pointer)(unsafe.Pointer(&cb)) = noescape(unsafe.Pointer(&cbFV))
+ cb(frame)
if raceenabled {
racereleasemerge(unsafe.Pointer(&racecgosync))
}
- if msanenabled {
- // Tell msan that we wrote to the entire argument block.
- // This tells msan that we set the results.
- // Since we have already called the function it doesn't
- // matter that we are writing to the non-result parameters.
- msanwrite(cb.arg, cb.argsize)
- }
// Do not unwind m->g0->sched.sp.
// Our caller, cgocallback, will do that.
// unwind of g's stack (see comment at top of file).
mp := acquirem()
sched := &mp.g0.sched
- switch GOARCH {
- default:
- throw("unwindm not implemented")
- case "386", "amd64", "arm", "ppc64", "ppc64le", "mips64", "mips64le", "s390x", "mips", "mipsle":
- sched.sp = *(*uintptr)(unsafe.Pointer(sched.sp + sys.MinFrameSize))
- case "arm64":
- sched.sp = *(*uintptr)(unsafe.Pointer(sched.sp + 16))
- }
+ sched.sp = *(*uintptr)(unsafe.Pointer(sched.sp + alignUp(sys.MinFrameSize, sys.StackAlign)))
// Do the accounting that cgocall will not have a chance to do
// during an unwind.
if mp.ncgo > 0 {
mp.incgo = false
mp.ncgo--
+ osPreemptExtExit(mp)
}
+ // Undo the call to lockOSThread in cgocallbackg, only on the
+ // panicking path. In normal return case cgocallbackg will call
+ // unlockOSThread, ensuring no preemption point after the unlock.
+ // Here we don't need to worry about preemption, because we're
+ // panicking out of the callback and unwinding the g0 stack,
+ // instead of reentering cgo (which requires the same thread).
+ unlockOSThread()
+
releasem(mp)
}
-
- // Undo the call to lockOSThread in cgocallbackg.
- // We must still stay on the same m.
- unlockOSThread()
}
-// called from assembly
+// called from assembly.
func badcgocallback() {
throw("misaligned stack in cgocallback")
}
-// called from (incomplete) assembly
+// called from (incomplete) assembly.
func cgounimpl() {
throw("cgo not implemented")
}
// We want to detect all cases where a program that does not use
// unsafe makes a cgo call passing a Go pointer to memory that
-// contains a Go pointer. Here a Go pointer is defined as a pointer
-// to memory allocated by the Go runtime. Programs that use unsafe
-// can evade this restriction easily, so we don't try to catch them.
-// The cgo program will rewrite all possibly bad pointer arguments to
-// call cgoCheckPointer, where we can catch cases of a Go pointer
-// pointing to a Go pointer.
+// contains an unpinned Go pointer. Here a Go pointer is defined as a
+// pointer to memory allocated by the Go runtime. Programs that use
+// unsafe can evade this restriction easily, so we don't try to catch
+// them. The cgo program will rewrite all possibly bad pointer
+// arguments to call cgoCheckPointer, where we can catch cases of a Go
+// pointer pointing to an unpinned Go pointer.
// Complicating matters, taking the address of a slice or array
// element permits the C program to access all elements of the slice
// pointers.)
// cgoCheckPointer checks if the argument contains a Go pointer that
-// points to a Go pointer, and panics if it does.
-func cgoCheckPointer(ptr interface{}, arg interface{}) {
- if debug.cgocheck == 0 {
+// points to an unpinned Go pointer, and panics if it does.
+func cgoCheckPointer(ptr any, arg any) {
+ if !goexperiment.CgoCheck2 && debug.cgocheck == 0 {
return
}
t := ep._type
top := true
- if arg != nil && (t.kind&kindMask == kindPtr || t.kind&kindMask == kindUnsafePointer) {
+ if arg != nil && (t.Kind_&kindMask == kindPtr || t.Kind_&kindMask == kindUnsafePointer) {
p := ep.data
- if t.kind&kindDirectIface == 0 {
+ if t.Kind_&kindDirectIface == 0 {
p = *(*unsafe.Pointer)(p)
}
if p == nil || !cgoIsGoPointer(p) {
return
}
aep := efaceOf(&arg)
- switch aep._type.kind & kindMask {
+ switch aep._type.Kind_ & kindMask {
case kindBool:
- if t.kind&kindMask == kindUnsafePointer {
+ if t.Kind_&kindMask == kindUnsafePointer {
// We don't know the type of the element.
break
}
pt := (*ptrtype)(unsafe.Pointer(t))
- cgoCheckArg(pt.elem, p, true, false, cgoCheckPointerFail)
+ cgoCheckArg(pt.Elem, p, true, false, cgoCheckPointerFail)
return
case kindSlice:
// Check the slice rather than the pointer.
}
}
- cgoCheckArg(t, ep.data, t.kind&kindDirectIface == 0, top, cgoCheckPointerFail)
+ cgoCheckArg(t, ep.data, t.Kind_&kindDirectIface == 0, top, cgoCheckPointerFail)
}
-const cgoCheckPointerFail = "cgo argument has Go pointer to Go pointer"
-const cgoResultFail = "cgo result has Go pointer"
+const cgoCheckPointerFail = "cgo argument has Go pointer to unpinned Go pointer"
+const cgoResultFail = "cgo result is unpinned Go pointer or points to unpinned Go pointer"
// cgoCheckArg is the real work of cgoCheckPointer. The argument p
// is either a pointer to the value (of type t), or the value itself,
// depending on indir. The top parameter is whether we are at the top
-// level, where Go pointers are allowed.
+// level, where Go pointers are allowed. Go pointers to pinned objects are
+// allowed as long as they don't reference other unpinned pointers.
func cgoCheckArg(t *_type, p unsafe.Pointer, indir, top bool, msg string) {
- if t.ptrdata == 0 || p == nil {
+ if t.PtrBytes == 0 || p == nil {
// If the type has no pointers there is nothing to do.
return
}
- switch t.kind & kindMask {
+ switch t.Kind_ & kindMask {
default:
throw("can't happen")
case kindArray:
at := (*arraytype)(unsafe.Pointer(t))
if !indir {
- if at.len != 1 {
+ if at.Len != 1 {
throw("can't happen")
}
- cgoCheckArg(at.elem, p, at.elem.kind&kindDirectIface == 0, top, msg)
+ cgoCheckArg(at.Elem, p, at.Elem.Kind_&kindDirectIface == 0, top, msg)
return
}
- for i := uintptr(0); i < at.len; i++ {
- cgoCheckArg(at.elem, p, true, top, msg)
- p = add(p, at.elem.size)
+ for i := uintptr(0); i < at.Len; i++ {
+ cgoCheckArg(at.Elem, p, true, top, msg)
+ p = add(p, at.Elem.Size_)
}
case kindChan, kindMap:
// These types contain internal pointers that will
if inheap(uintptr(unsafe.Pointer(it))) {
panic(errorString(msg))
}
- p = *(*unsafe.Pointer)(add(p, sys.PtrSize))
+ p = *(*unsafe.Pointer)(add(p, goarch.PtrSize))
if !cgoIsGoPointer(p) {
return
}
- if !top {
+ if !top && !isPinned(p) {
panic(errorString(msg))
}
- cgoCheckArg(it, p, it.kind&kindDirectIface == 0, false, msg)
+ cgoCheckArg(it, p, it.Kind_&kindDirectIface == 0, false, msg)
case kindSlice:
st := (*slicetype)(unsafe.Pointer(t))
s := (*slice)(p)
if p == nil || !cgoIsGoPointer(p) {
return
}
- if !top {
+ if !top && !isPinned(p) {
panic(errorString(msg))
}
- if st.elem.ptrdata == 0 {
+ if st.Elem.PtrBytes == 0 {
return
}
for i := 0; i < s.cap; i++ {
- cgoCheckArg(st.elem, p, true, false, msg)
- p = add(p, st.elem.size)
+ cgoCheckArg(st.Elem, p, true, false, msg)
+ p = add(p, st.Elem.Size_)
}
case kindString:
ss := (*stringStruct)(p)
if !cgoIsGoPointer(ss.str) {
return
}
- if !top {
+ if !top && !isPinned(ss.str) {
panic(errorString(msg))
}
case kindStruct:
st := (*structtype)(unsafe.Pointer(t))
if !indir {
- if len(st.fields) != 1 {
+ if len(st.Fields) != 1 {
throw("can't happen")
}
- cgoCheckArg(st.fields[0].typ, p, st.fields[0].typ.kind&kindDirectIface == 0, top, msg)
+ cgoCheckArg(st.Fields[0].Typ, p, st.Fields[0].Typ.Kind_&kindDirectIface == 0, top, msg)
return
}
- for _, f := range st.fields {
- if f.typ.ptrdata == 0 {
+ for _, f := range st.Fields {
+ if f.Typ.PtrBytes == 0 {
continue
}
- cgoCheckArg(f.typ, add(p, f.offset()), true, top, msg)
+ cgoCheckArg(f.Typ, add(p, f.Offset), true, top, msg)
}
case kindPtr, kindUnsafePointer:
if indir {
if !cgoIsGoPointer(p) {
return
}
- if !top {
+ if !top && !isPinned(p) {
panic(errorString(msg))
}
// cgoCheckUnknownPointer is called for an arbitrary pointer into Go
// memory. It checks whether that Go memory contains any other
-// pointer into Go memory. If it does, we panic.
+// pointer into unpinned Go memory. If it does, we panic.
// The return values are unused but useful to see in panic tracebacks.
func cgoCheckUnknownPointer(p unsafe.Pointer, msg string) (base, i uintptr) {
if inheap(uintptr(p)) {
if base == 0 {
return
}
- hbits := heapBitsForAddr(base)
- n := span.elemsize
- for i = uintptr(0); i < n; i += sys.PtrSize {
- if i != 1*sys.PtrSize && !hbits.morePointers() {
- // No more possible pointers.
- break
+ if goexperiment.AllocHeaders {
+ tp := span.typePointersOfUnchecked(base)
+ for {
+ var addr uintptr
+ if tp, addr = tp.next(base + span.elemsize); addr == 0 {
+ break
+ }
+ pp := *(*unsafe.Pointer)(unsafe.Pointer(addr))
+ if cgoIsGoPointer(pp) && !isPinned(pp) {
+ panic(errorString(msg))
+ }
}
- if hbits.isPointer() && cgoIsGoPointer(*(*unsafe.Pointer)(unsafe.Pointer(base + i))) {
- panic(errorString(msg))
+ } else {
+ n := span.elemsize
+ hbits := heapBitsForAddr(base, n)
+ for {
+ var addr uintptr
+ if hbits, addr = hbits.next(); addr == 0 {
+ break
+ }
+ pp := *(*unsafe.Pointer)(unsafe.Pointer(addr))
+ if cgoIsGoPointer(pp) && !isPinned(pp) {
+ panic(errorString(msg))
+ }
}
- hbits = hbits.next()
}
-
return
}
// cgoIsGoPointer reports whether the pointer is a Go pointer--a
// pointer to Go memory. We only care about Go memory that might
// contain pointers.
+//
//go:nosplit
//go:nowritebarrierrec
func cgoIsGoPointer(p unsafe.Pointer) bool {
}
// cgoInRange reports whether p is between start and end.
+//
//go:nosplit
//go:nowritebarrierrec
func cgoInRange(p unsafe.Pointer, start, end uintptr) bool {
}
// cgoCheckResult is called to check the result parameter of an
-// exported Go function. It panics if the result is or contains a Go
-// pointer.
-func cgoCheckResult(val interface{}) {
- if debug.cgocheck == 0 {
+// exported Go function. It panics if the result is or contains any
+// other pointer into unpinned Go memory.
+func cgoCheckResult(val any) {
+ if !goexperiment.CgoCheck2 && debug.cgocheck == 0 {
return
}
ep := efaceOf(&val)
t := ep._type
- cgoCheckArg(t, ep.data, t.kind&kindDirectIface == 0, false, cgoResultFail)
+ cgoCheckArg(t, ep.data, t.Kind_&kindDirectIface == 0, false, cgoResultFail)
}