#include <libc.h>
#include "go.h"
#include "../ld/textflag.h"
+#include "../../runtime/mgc0.h"
static Node* walkprint(Node*, NodeList**);
static Node* writebarrierfn(char*, Type*, Type*);
}
}
+void
+walkexprlistcheap(NodeList *l, NodeList **init)
+{
+ for(; l; l=l->next) {
+ l->n = cheapexpr(l->n, init);
+ walkexpr(&l->n, init);
+ }
+}
+
void
walkexpr(Node **np, NodeList **init)
{
int notfirst, et, op;
NodeList *calls;
- on = nil;
op = nn->op;
all = nn->list;
calls = nil;
notfirst = 0;
+ // Hoist all the argument evaluation up before the lock.
+ walkexprlistcheap(all, init);
+
+ calls = list(calls, mkcall("printlock", T, init));
+
for(l=all; l; l=l->next) {
if(notfirst) {
calls = list(calls, mkcall("printsp", T, init));
if(op == OPRINTN)
calls = list(calls, mkcall("printnl", T, nil));
+
+ calls = list(calls, mkcall("printunlock", T, init));
+
typechecklist(calls, Etop);
walkexprlist(calls, init);
{
Node *l, *r;
Type *t;
+ vlong x;
+ static Bvec *bv;
+ char name[32];
if(n->left && n->right && needwritebarrier(n->left, n->right)) {
t = n->left->type;
} else if(isinter(t)) {
n = mkcall1(writebarrierfn("writebarrieriface", t, n->right->type), T, init,
l, n->right);
- } else if(t->width == 2*widthptr) {
- n = mkcall1(writebarrierfn("writebarrierfat2", t, n->right->type), T, init,
- l, nodnil(), n->right);
- } else if(t->width == 3*widthptr) {
- n = mkcall1(writebarrierfn("writebarrierfat3", t, n->right->type), T, init,
- l, nodnil(), n->right);
- } else if(t->width == 4*widthptr) {
- n = mkcall1(writebarrierfn("writebarrierfat4", t, n->right->type), T, init,
+ } else if(t->width <= 4*widthptr) {
+ x = 0;
+ if(bv == nil)
+ bv = bvalloc(BitsPerPointer*4);
+ bvresetall(bv);
+ twobitwalktype1(t, &x, bv);
+ // The bvgets are looking for BitsPointer in successive slots.
+ enum {
+ PtrBit = 1,
+ };
+ if(BitsPointer != (1<<PtrBit))
+ fatal("wrong PtrBit");
+ switch(t->width/widthptr) {
+ default:
+ fatal("found writebarrierfat for %d-byte object of type %T", (int)t->width, t);
+ case 2:
+ snprint(name, sizeof name, "writebarrierfat%d%d",
+ bvget(bv, PtrBit), bvget(bv, BitsPerPointer+PtrBit));
+ break;
+ case 3:
+ snprint(name, sizeof name, "writebarrierfat%d%d%d",
+ bvget(bv, PtrBit), bvget(bv, BitsPerPointer+PtrBit), bvget(bv, 2*BitsPerPointer+PtrBit));
+ break;
+ case 4:
+ snprint(name, sizeof name, "writebarrierfat%d%d%d%d",
+ bvget(bv, PtrBit), bvget(bv, BitsPerPointer+PtrBit), bvget(bv, 2*BitsPerPointer+PtrBit), bvget(bv, 3*BitsPerPointer+PtrBit));
+ break;
+ }
+ n = mkcall1(writebarrierfn(name, t, n->right->type), T, init,
l, nodnil(), n->right);
} else {
r = n->right;
{
Node *nl, *nr, *nfrm, *nto, *nif, *nlen, *nwid, *fn;
NodeList *l;
+
+ if(haspointers(n->left->type->type)) {
+ fn = writebarrierfn("writebarriercopy", n->left->type, n->right->type);
+ return mkcall1(fn, n->type, init, typename(n->left->type->type), n->left, n->right);
+ }
if(runtimecall) {
if(n->right->type->etype == TSTRING)
echo '# sync -cpu=10'
go test sync -short -timeout=$(expr 120 \* $timeout_scale)s -cpu=10
- # Race detector only supported on Linux, FreeBSD and OS X,
- # and only on amd64, and only when cgo is enabled.
- # DISABLED until we get garbage collection working.
- case "$GOHOSTOS-$GOOS-$GOARCH-$CGO_ENABLED-XXX-DISABLED" in
- linux-linux-amd64-1 | freebsd-freebsd-amd64-1 | darwin-darwin-amd64-1)
- echo
- echo '# Testing race detector.'
- go test -race -i runtime/race flag
- go test -race -run=Output runtime/race
- go test -race -short flag
- esac
-
xcd() {
echo
echo '#' $1
[ "$CGO_ENABLED" != 1 ] ||
(xcd ../misc/cgo/test
# cgo tests inspect the traceback for runtime functions
+ extlink=0
export GOTRACEBACK=2
go test -ldflags '-linkmode=auto' || exit 1
# linkmode=internal fails on dragonfly since errno is a TLS relocation.
openbsd-386 | openbsd-amd64)
# test linkmode=external, but __thread not supported, so skip testtls.
go test -ldflags '-linkmode=external' || exit 1
+ extlink=1
;;
darwin-386 | darwin-amd64)
# linkmode=external fails on OS X 10.6 and earlier == Darwin
# 10.8 and earlier.
case $(uname -r) in
[0-9].* | 10.*) ;;
- *) go test -ldflags '-linkmode=external' || exit 1;;
+ *)
+ go test -ldflags '-linkmode=external' || exit 1
+ extlink=1
+ ;;
esac
;;
android-arm | dragonfly-386 | dragonfly-amd64 | freebsd-386 | freebsd-amd64 | freebsd-arm | linux-386 | linux-amd64 | linux-arm | netbsd-386 | netbsd-amd64)
go test -ldflags '-linkmode=external' || exit 1
go test -ldflags '-linkmode=auto' ../testtls || exit 1
go test -ldflags '-linkmode=external' ../testtls || exit 1
+ extlink=1
case "$GOHOSTOS-$GOARCH" in
netbsd-386 | netbsd-amd64) ;; # no static linking
esac
) || exit $?
-case "$GOHOSTOS-$GOOS-$GOARCH-$CGO_ENABLED" in
+ # Race detector only supported on Linux, FreeBSD and OS X,
+ # and only on amd64, and only when cgo is enabled.
+ # Delayed until here so we know whether to try external linking.
++# DISABLED until we get garbage collection working.
++case "$GOHOSTOS-$GOOS-$GOARCH-$CGO_ENABLED-XXX-DISABLED" in
+ linux-linux-amd64-1 | freebsd-freebsd-amd64-1 | darwin-darwin-amd64-1)
+ echo
+ echo '# Testing race detector.'
+ go test -race -i runtime/race flag os/exec
+ go test -race -run=Output runtime/race
+ go test -race -short flag os/exec
+
+ # Test with external linking; see issue 9133.
+ if [ "$extlink" = 1 ]; then
+ go test -race -short -ldflags=-linkmode=external flag os/exec
+ fi
+ esac
+
# This tests cgo -cdefs. That mode is not supported,
# so it's okay if it doesn't work on some systems.
# In particular, it works badly with clang on OS X.
// of the G stack. We need to distinguish the routine that
// lives at the bottom of the G stack from the one that lives
// at the top of the system stack because the one at the top of
- // the M stack terminates the stack walk (see topofstack()).
+ // the system stack terminates the stack walk (see topofstack()).
TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
RET
MOVL g(CX), AX
MOVL AX, ret+0(FP)
RET
+
+TEXT runtime·prefetcht0(SB),NOSPLIT,$0-4
+ MOVL addr+0(FP), AX
+ PREFETCHT0 (AX)
+ RET
+
+TEXT runtime·prefetcht1(SB),NOSPLIT,$0-4
+ MOVL addr+0(FP), AX
+ PREFETCHT1 (AX)
+ RET
+
+
+TEXT runtime·prefetcht2(SB),NOSPLIT,$0-4
+ MOVL addr+0(FP), AX
+ PREFETCHT2 (AX)
+ RET
+
+TEXT runtime·prefetchnta(SB),NOSPLIT,$0-4
+ MOVL addr+0(FP), AX
+ PREFETCHNTA (AX)
+ RET
// +build power64 power64le
- #include "zasm_GOOS_GOARCH.h"
+ #include "go_asm.h"
+ #include "go_tls.h"
#include "funcdata.h"
#include "textflag.h"
BL (CTR)
BR runtime·badmcall2(SB)
- // switchtoM is a dummy routine that onM leaves at the bottom
+ // systemstack_switch is a dummy routine that systemstack leaves at the bottom
// of the G stack. We need to distinguish the routine that
// lives at the bottom of the G stack from the one that lives
- // at the top of the M stack because the one at the top of
- // the M stack terminates the stack walk (see topofstack()).
- TEXT runtime·switchtoM(SB), NOSPLIT, $0-0
+ // at the top of the system stack because the one at the top of
+ // the system stack terminates the stack walk (see topofstack()).
+ TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
UNDEF
BL (LR) // make sure this function is not leaf
RETURN
- // func onM_signalok(fn func())
- TEXT runtime·onM_signalok(SB), NOSPLIT, $8-8
- MOVD g, R3 // R3 = g
- MOVD g_m(R3), R4 // R4 = g->m
- MOVD m_gsignal(R4), R4 // R4 = g->m->gsignal
- MOVD fn+0(FP), R11 // context for call below
- CMP R3, R4
- BEQ onsignal
- MOVD R11, 8(R1)
- BL runtime·onM(SB)
- RETURN
-
- onsignal:
- MOVD 0(R11), R3 // code pointer
- MOVD R3, CTR
- BL (CTR)
- RETURN
-
- // void onM(fn func())
- TEXT runtime·onM(SB), NOSPLIT, $0-8
+ // func systemstack(fn func())
+ TEXT runtime·systemstack(SB), NOSPLIT, $0-8
MOVD fn+0(FP), R3 // R3 = fn
MOVD R3, R11 // context
MOVD g_m(g), R4 // R4 = m
+ MOVD m_gsignal(R4), R5 // R5 = gsignal
+ CMP g, R5
+ BEQ noswitch
+
MOVD m_g0(R4), R5 // R5 = g0
CMP g, R5
- BEQ onm
+ BEQ noswitch
MOVD m_curg(R4), R6
CMP g, R6
- BEQ oncurg
+ BEQ switch
- // Not g0, not curg. Must be gsignal, but that's not allowed.
+ // Bad: g is not gsignal, not g0, not curg. What is it?
// Hide call from linker nosplit analysis.
- MOVD $runtime·badonm(SB), R3
+ MOVD $runtime·badsystemstack(SB), R3
MOVD R3, CTR
BL (CTR)
- oncurg:
+ switch:
// save our state in g->sched. Pretend to
- // be switchtoM if the G stack is scanned.
- MOVD $runtime·switchtoM(SB), R6
+ // be systemstack_switch if the G stack is scanned.
+ MOVD $runtime·systemstack_switch(SB), R6
ADD $8, R6 // get past prologue
MOVD R6, (g_sched+gobuf_pc)(g)
MOVD R1, (g_sched+gobuf_sp)(g)
// switch to g0
MOVD R5, g
MOVD (g_sched+gobuf_sp)(g), R3
- // make it look like mstart called onM on g0, to stop traceback
+ // make it look like mstart called systemstack on g0, to stop traceback
SUB $8, R3
MOVD $runtime·mstart(SB), R4
MOVD R4, 0(R3)
MOVD R0, (g_sched+gobuf_sp)(g)
RETURN
- onm:
+ noswitch:
// already on m stack, just call directly
MOVD 0(R11), R3 // code pointer
MOVD R3, CTR
MOVD R0, R0 // NOP
BL runtime·goexit1(SB) // does not return
+ TEXT runtime·getg(SB),NOSPLIT,$-8-8
+ MOVD g, ret+0(FP)
+ RETURN
++
+TEXT runtime·prefetcht0(SB),NOSPLIT,$0-8
+ RETURN
+
+TEXT runtime·prefetcht1(SB),NOSPLIT,$0-8
+ RETURN
+
+TEXT runtime·prefetcht2(SB),NOSPLIT,$0-8
+ RETURN
+
+TEXT runtime·prefetchnta(SB),NOSPLIT,$0-8
+ RETURN
// finalizers, etc.) to a file.
// The format of the dumped file is described at
- // http://code.google.com/p/go-wiki/wiki/heapdump14
+ // http://golang.org/s/go14heapdump.
package runtime
if n > uintptr(len(freemark)) {
gothrow("freemark array doesn't have enough entries")
}
- for l := s.freelist; l != nil; l = l.next {
- freemark[(uintptr(unsafe.Pointer(l))-p)/size] = true
+ for l := s.freelist; l.ptr() != nil; l = l.ptr().next {
+ freemark[(uintptr(l)-p)/size] = true
}
for j := uintptr(0); j < n; j, p = j+1, p+size {
if freemark[j] {
func lfstackpush(head *uint64, node *lfnode) {
node.pushcnt++
new := lfstackPack(node, node.pushcnt)
+ if node1, _ := lfstackUnpack(new); node1 != node {
+ println("runtime: lfstackpush invalid packing: node=", node, " cnt=", hex(node.pushcnt), " packed=", hex(new), " -> node=", node1, "\n")
+ gothrow("lfstackpush")
+ }
for {
old := atomicload64(head)
- node.next, _ = lfstackUnpack(old)
+ node.next = old
if cas64(head, old, new) {
break
}
return nil
}
node, _ := lfstackUnpack(old)
- node2 := (*lfnode)(atomicloadp(unsafe.Pointer(&node.next)))
- new := uint64(0)
- if node2 != nil {
- new = lfstackPack(node2, node2.pushcnt)
- }
- if cas64(head, old, new) {
+ next := atomicload64(&node.next)
+ if cas64(head, old, next) {
return unsafe.Pointer(node)
}
}
// Allocate a new maxTinySize block.
s = c.alloc[tinySizeClass]
v := s.freelist
- if v == nil {
+ if v.ptr() == nil {
systemstack(func() {
mCache_Refill(c, tinySizeClass)
})
s = c.alloc[tinySizeClass]
v = s.freelist
}
- s.freelist = v.next
+ s.freelist = v.ptr().next
s.ref++
//TODO: prefetch v.next
x = unsafe.Pointer(v)
size = uintptr(class_to_size[sizeclass])
s = c.alloc[sizeclass]
v := s.freelist
- if v == nil {
+ if v.ptr() == nil {
systemstack(func() {
mCache_Refill(c, int32(sizeclass))
})
s = c.alloc[sizeclass]
v = s.freelist
}
- s.freelist = v.next
+ s.freelist = v.ptr().next
s.ref++
//TODO: prefetch
x = unsafe.Pointer(v)
if flags&flagNoZero == 0 {
- v.next = nil
+ v.ptr().next = 0
if size > 2*ptrSize && ((*[2]uintptr)(x))[1] != 0 {
memclr(unsafe.Pointer(v), size)
}
masksize = masksize * pointersPerByte / 8 // 4 bits per word
masksize++ // unroll flag in the beginning
if masksize > maxGCMask && typ.gc[1] != 0 {
+ // write barriers have not been updated to deal with this case yet.
+ gothrow("maxGCMask too small for now")
// If the mask is too large, unroll the program directly
// into the GC bitmap. It's 7 times slower than copying
// from the pre-unrolled mask, but saves 1/16 of type size
}
}
marked:
+
+ // GCmarkterminate allocates black
+ // All slots hold nil so no scanning is needed.
+ // This may be racing with GC so do it atomically if there can be
+ // a race marking the bit.
+ if gcphase == _GCmarktermination {
+ systemstack(func() {
+ gcmarknewobject_m(uintptr(x))
+ })
+ }
+
if raceenabled {
racemalloc(x, size)
}
}
}
- if memstats.heap_alloc >= memstats.next_gc {
+ if memstats.heap_alloc >= memstats.next_gc/2 {
gogc(0)
}
return x
}
+func loadPtrMask(typ *_type) []uint8 {
+ var ptrmask *uint8
+ nptr := (uintptr(typ.size) + ptrSize - 1) / ptrSize
+ if typ.kind&kindGCProg != 0 {
+ masksize := nptr
+ if masksize%2 != 0 {
+ masksize *= 2 // repeated
+ }
+ masksize = masksize * pointersPerByte / 8 // 4 bits per word
+ masksize++ // unroll flag in the beginning
+ if masksize > maxGCMask && typ.gc[1] != 0 {
+ // write barriers have not been updated to deal with this case yet.
+ gothrow("maxGCMask too small for now")
+ }
+ ptrmask = (*uint8)(unsafe.Pointer(uintptr(typ.gc[0])))
+ // Check whether the program is already unrolled
+ // by checking if the unroll flag byte is set
+ maskword := uintptr(atomicloadp(unsafe.Pointer(ptrmask)))
+ if *(*uint8)(unsafe.Pointer(&maskword)) == 0 {
+ systemstack(func() {
+ unrollgcprog_m(typ)
+ })
+ }
+ ptrmask = (*uint8)(add(unsafe.Pointer(ptrmask), 1)) // skip the unroll flag byte
+ } else {
+ ptrmask = (*uint8)(unsafe.Pointer(typ.gc[0])) // pointer to unrolled mask
+ }
+ return (*[1 << 30]byte)(unsafe.Pointer(ptrmask))[:(nptr+1)/2]
+}
+
// implementation of new builtin
func newobject(typ *_type) unsafe.Pointer {
flags := uint32(0)
mp = acquirem()
mp.gcing = 1
releasem(mp)
+
systemstack(stoptheworld)
+ systemstack(finishsweep_m) // finish sweep before we start concurrent scan.
+ if true { // To turn on concurrent scan and mark set to true...
+ systemstack(starttheworld)
+ // Do a concurrent heap scan before we stop the world.
+ systemstack(gcscan_m)
+ systemstack(stoptheworld)
+ systemstack(gcinstallmarkwb_m)
+ systemstack(starttheworld)
+ systemstack(gcmark_m)
+ systemstack(stoptheworld)
+ systemstack(gcinstalloffwb_m)
+ }
+
if mp != acquirem() {
gothrow("gogc: rescheduled")
}
if debug.gctrace > 1 {
n = 2
}
+ eagersweep := force >= 2
for i := 0; i < n; i++ {
if i > 0 {
startTime = nanotime()
}
// switch to g0, call gc, then switch back
- eagersweep := force >= 2
systemstack(func() {
gc_m(startTime, eagersweep)
})
}
+ systemstack(func() {
+ gccheckmark_m(startTime, eagersweep)
+ })
+
// all done
mp.gcing = 0
semrelease(&worldsema)
}
}
+func GCcheckmarkenable() {
+ systemstack(gccheckmarkenable_m)
+}
+
+func GCcheckmarkdisable() {
+ systemstack(gccheckmarkdisable_m)
+}
+
// GC runs a garbage collection.
func GC() {
gogc(2)
// linker-provided
var noptrdata struct{}
+ var enoptrdata struct{}
+ var noptrbss struct{}
var enoptrbss struct{}
// SetFinalizer sets the finalizer associated with x to f.
// func main() {
// runtime.SetFinalizer(Foo, nil)
// }
- // The segments are, in order: text, rodata, noptrdata, data, bss, noptrbss.
- if uintptr(unsafe.Pointer(&noptrdata)) <= uintptr(e.data) && uintptr(e.data) < uintptr(unsafe.Pointer(&enoptrbss)) {
+ // The relevant segments are: noptrdata, data, bss, noptrbss.
+ // We cannot assume they are in any order or even contiguous,
+ // due to external linking.
+ if uintptr(unsafe.Pointer(&noptrdata)) <= uintptr(e.data) && uintptr(e.data) < uintptr(unsafe.Pointer(&enoptrdata)) ||
+ uintptr(unsafe.Pointer(&data)) <= uintptr(e.data) && uintptr(e.data) < uintptr(unsafe.Pointer(&edata)) ||
+ uintptr(unsafe.Pointer(&bss)) <= uintptr(e.data) && uintptr(e.data) < uintptr(unsafe.Pointer(&ebss)) ||
+ uintptr(unsafe.Pointer(&noptrbss)) <= uintptr(e.data) && uintptr(e.data) < uintptr(unsafe.Pointer(&enoptrbss)) {
return
}
gothrow("runtime.SetFinalizer: pointer not in allocated block")
// See http://golang.org/issue/5402 and http://golang.org/issue/5236.
// On other 64-bit platforms, we limit the arena to 128GB, or 37 bits.
// On 32-bit, we don't bother limiting anything, so we use the full 32-bit address.
- _MHeapMap_TotalBits = (_64bit*_Windows)*35 + (_64bit*(1-_Windows))*37 + (1-_64bit)*32
+ _MHeapMap_TotalBits = (_64bit*goos_windows)*35 + (_64bit*(1-goos_windows))*37 + (1-_64bit)*32
_MHeapMap_Bits = _MHeapMap_TotalBits - _PageShift
_MaxMem = uintptr(1<<_MHeapMap_TotalBits - 1)
)
// A generic linked list of blocks. (Typically the block is bigger than sizeof(MLink).)
+// Since assignments to mlink.next will result in a write barrier being preformed
+// this can not be used by some of the internal GC structures. For example when
+// the sweeper is placing an unmarked object on the free list it does not want the
+// write barrier to be called since that could result in the object being reachable.
type mlink struct {
next *mlink
}
+// A gclink is a node in a linked list of blocks, like mlink,
+// but it is opaque to the garbage collector.
+// The GC does not trace the pointers during collection,
+// and the compiler does not emit write barriers for assignments
+// of gclinkptr values. Code should store references to gclinks
+// as gclinkptr, not as *gclink.
+type gclink struct {
+ next gclinkptr
+}
+
+// A gclinkptr is a pointer to a gclink, but it is opaque
+// to the garbage collector.
+type gclinkptr uintptr
+
+// ptr returns the *gclink form of p.
+// The result should be used for accessing fields, not stored
+// in other data structures.
+func (p gclinkptr) ptr() *gclink {
+ return (*gclink)(unsafe.Pointer(p))
+}
+
// sysAlloc obtains a large chunk of zeroed memory from the
// operating system, typically on the order of a hundred kilobytes
// or a megabyte.
}
type stackfreelist struct {
- list *mlink // linked list of free stacks
- size uintptr // total size of stacks in list
+ list gclinkptr // linked list of free stacks
+ size uintptr // total size of stacks in list
}
// Per-thread (in Go, per-P) cache for small objects.
sudogcache *sudog
- gcworkbuf unsafe.Pointer
-
// Local allocator stats, flushed during GC.
local_nlookup uintptr // number of pointer lookups
local_largefree uintptr // bytes freed for large objects (>maxsmallsize)
)
type mspan struct {
- next *mspan // in a span linked list
- prev *mspan // in a span linked list
- start pageID // starting page number
- npages uintptr // number of pages in span
- freelist *mlink // list of free objects
+ next *mspan // in a span linked list
+ prev *mspan // in a span linked list
+ start pageID // starting page number
+ npages uintptr // number of pages in span
+ freelist gclinkptr // list of free objects
// sweep generation:
// if sweepgen == h->sweepgen - 2, the span needs sweeping
// if sweepgen == h->sweepgen - 1, the span is currently being swept
if c := p.mcache; c != nil {
c.tiny = nil
c.tinysize = 0
+
+ // disconnect cached list before dropping it on the floor,
+ // so that a dangling ref to one entry does not pin all of them.
+ var sg, sgnext *sudog
+ for sg = c.sudogcache; sg != nil; sg = sgnext {
+ sgnext = sg.next
+ sg.next = nil
+ }
c.sudogcache = nil
}
+
// clear defer pools
for i := range p.deferpool {
+ // disconnect cached list before dropping it on the floor,
+ // so that a dangling ref to one entry does not pin all of them.
+ var d, dlink *_defer
+ for d = p.deferpool[i]; d != nil; d = dlink {
+ dlink = d.link
+ d.link = nil
+ }
p.deferpool[i] = nil
}
}
}
}
+const (
+ _PoisonGC = 0xf969696969696969 & (1<<(8*ptrSize) - 1)
+ _PoisonStack = 0x6868686868686868 & (1<<(8*ptrSize) - 1)
+)
+
// NOTE: Really dst *unsafe.Pointer, src unsafe.Pointer,
// but if we do that, Go inserts a write barrier on *dst = src.
//go:nosplit
func writebarrierptr(dst *uintptr, src uintptr) {
*dst = src
+ writebarrierptr_nostore(dst, src)
+}
+
+// Like writebarrierptr, but the store has already been applied.
+// Do not reapply.
+//go:nosplit
+func writebarrierptr_nostore(dst *uintptr, src uintptr) {
+ if getg() == nil { // very low-level startup
+ return
+ }
+
+ if src != 0 && (src < _PageSize || src == _PoisonGC || src == _PoisonStack) {
+ systemstack(func() { gothrow("bad pointer in write barrier") })
+ }
+
+ mp := acquirem()
+ if mp.inwb || mp.dying > 0 {
+ releasem(mp)
+ return
+ }
+ mp.inwb = true
+ systemstack(func() {
+ gcmarkwb_m(dst, src)
+ })
+ mp.inwb = false
+ releasem(mp)
}
//go:nosplit
func writebarrierstring(dst *[2]uintptr, src [2]uintptr) {
- dst[0] = src[0]
+ writebarrierptr(&dst[0], src[0])
dst[1] = src[1]
}
//go:nosplit
func writebarrierslice(dst *[3]uintptr, src [3]uintptr) {
- dst[0] = src[0]
+ writebarrierptr(&dst[0], src[0])
dst[1] = src[1]
dst[2] = src[2]
}
//go:nosplit
func writebarrieriface(dst *[2]uintptr, src [2]uintptr) {
- dst[0] = src[0]
- dst[1] = src[1]
-}
-
-//go:nosplit
-func writebarrierfat2(dst *[2]uintptr, _ *byte, src [2]uintptr) {
- dst[0] = src[0]
- dst[1] = src[1]
+ writebarrierptr(&dst[0], src[0])
+ writebarrierptr(&dst[1], src[1])
}
-//go:nosplit
-func writebarrierfat3(dst *[3]uintptr, _ *byte, src [3]uintptr) {
- dst[0] = src[0]
- dst[1] = src[1]
- dst[2] = src[2]
-}
+//go:generate go run wbfat_gen.go -- wbfat.go
+//
+// The above line generates multiword write barriers for
+// all the combinations of ptr+scalar up to four words.
+// The implementations are written to wbfat.go.
//go:nosplit
-func writebarrierfat4(dst *[4]uintptr, _ *byte, src [4]uintptr) {
- dst[0] = src[0]
- dst[1] = src[1]
- dst[2] = src[2]
- dst[3] = src[3]
+func writebarrierfat(typ *_type, dst, src unsafe.Pointer) {
+ mask := loadPtrMask(typ)
+ nptr := typ.size / ptrSize
+ for i := uintptr(0); i < nptr; i += 2 {
+ bits := mask[i/2]
+ if (bits>>2)&_BitsMask == _BitsPointer {
+ writebarrierptr((*uintptr)(dst), *(*uintptr)(src))
+ } else {
+ *(*uintptr)(dst) = *(*uintptr)(src)
+ }
+ dst = add(dst, ptrSize)
+ src = add(src, ptrSize)
+ if i+1 == nptr {
+ break
+ }
+ bits >>= 4
+ if (bits>>2)&_BitsMask == _BitsPointer {
+ writebarrierptr((*uintptr)(dst), *(*uintptr)(src))
+ } else {
+ *(*uintptr)(dst) = *(*uintptr)(src)
+ }
+ dst = add(dst, ptrSize)
+ src = add(src, ptrSize)
+ }
}
//go:nosplit
-func writebarrierfat(typ *_type, dst, src unsafe.Pointer) {
- memmove(dst, src, typ.size)
+func writebarriercopy(typ *_type, dst, src slice) int {
+ n := dst.len
+ if n > src.len {
+ n = src.len
+ }
+ if n == 0 {
+ return 0
+ }
+ dstp := unsafe.Pointer(dst.array)
+ srcp := unsafe.Pointer(src.array)
+
+ if uintptr(srcp) < uintptr(dstp) && uintptr(srcp)+uintptr(n)*typ.size > uintptr(dstp) {
+ // Overlap with src before dst.
+ // Copy backward, being careful not to move dstp/srcp
+ // out of the array they point into.
+ dstp = add(dstp, uintptr(n-1)*typ.size)
+ srcp = add(srcp, uintptr(n-1)*typ.size)
+ i := uint(0)
+ for {
+ writebarrierfat(typ, dstp, srcp)
+ if i++; i >= n {
+ break
+ }
+ dstp = add(dstp, -typ.size)
+ srcp = add(srcp, -typ.size)
+ }
+ } else {
+ // Copy forward, being careful not to move dstp/srcp
+ // out of the array they point into.
+ i := uint(0)
+ for {
+ writebarrierfat(typ, dstp, srcp)
+ if i++; i >= n {
+ break
+ }
+ dstp = add(dstp, typ.size)
+ srcp = add(srcp, typ.size)
+ }
+ }
+ return int(n)
}
if _cgo_free == nil {
gothrow("_cgo_free missing")
}
- if _cgo_setenv == nil {
- gothrow("_cgo_setenv missing")
- }
- if _cgo_unsetenv == nil {
- gothrow("_cgo_unsetenv missing")
+ if GOOS != "windows" {
+ if _cgo_setenv == nil {
+ gothrow("_cgo_setenv missing")
+ }
+ if _cgo_unsetenv == nil {
+ gothrow("_cgo_unsetenv missing")
+ }
}
}
gothrow("acquireSudog: found s.elem != nil in cache")
}
c.sudogcache = s.next
+ s.next = nil
return s
}
// which keeps the garbage collector from being invoked.
mp := acquirem()
p := new(sudog)
+ if p.elem != nil {
+ gothrow("acquireSudog: found p.elem != nil after new")
+ }
releasem(mp)
return p
}
if s.selectdone != nil {
gothrow("runtime: sudog with non-nil selectdone")
}
+ if s.next != nil {
+ gothrow("runtime: sudog with non-nil next")
+ }
+ if s.prev != nil {
+ gothrow("runtime: sudog with non-nil prev")
+ }
+ if s.waitlink != nil {
+ gothrow("runtime: sudog with non-nil waitlink")
+ }
gp := getg()
if gp.param != nil {
gothrow("runtime: releaseSudog with non-nil gp.param")
_Pdead
)
- // XXX inserting below here
+ // The next line makes 'go generate' write the zgen_*.go files with
+ // per-OS and per-arch information, including constants
+ // named goos_$GOOS and goarch_$GOARCH for every
+ // known GOOS and GOARCH. The constant is 1 on the
+ // current system, 0 otherwise; multiplying by them is
+ // useful for defining GOOS- or GOARCH-specific constants.
+ //go:generate go run gengoos.go
type mutex struct {
// Futex-based impl treats it as uint32 key,
helpgc int32
spinning bool // m is out of work and is actively looking for work
blocked bool // m is blocked on a note
+ inwb bool // m is executing a write barrier
+ printlock int8
fastrand uint32
ncgocall uint64 // number of cgo calls in total
ncgo int32 // number of cgo calls currently in progress
fun [0]uintptr
}
- const (
- // TODO: Generate in cmd/dist.
- _NaCl = 0
- _Windows = 0
- _Solaris = 0
- _Plan9 = 0
- )
-
// Lock-free stack node.
+// // Also known to export_test.go.
type lfnode struct {
- next *lfnode
+ next uint64
pushcnt uintptr
}
// Indicates to write barrier and sychronization task to preform.
const (
- _GCoff = iota // stop and start nop
- _GCquiesce // stop and start nop
- _GCstw // stop the ps nop
- _GCmark // scan the stacks and start no white to black
- _GCsweep // stop and start nop
+ _GCoff = iota // GC not running, write barrier disabled
+ _GCquiesce // unused state
+ _GCstw // unused state
+ _GCscan // GC collecting roots into workbufs, write barrier disabled
+ _GCmark // GC marking from workbufs, write barrier ENABLED
+ _GCmarktermination // GC mark termination: allocate black, P's help GC, write barrier ENABLED
+ _GCsweep // GC mark completed; sweeping in background, write barrier disabled
)
type forcegcstate struct {
// iterating through the linked list they are in reverse order.
cas = nil
sglist = gp.waiting
- // Clear all selectdone and elem before unlinking from gp.waiting.
- // They must be cleared before being put back into the sudog cache.
- // Clear before unlinking, because if a stack copy happens after the unlink,
- // they will not be updated, they will be left pointing to the old stack,
- // which creates dangling pointers, which may be detected by the
- // garbage collector.
+ // Clear all elem before unlinking from gp.waiting.
for sg1 := gp.waiting; sg1 != nil; sg1 = sg1.waitlink {
sg1.selectdone = nil
sg1.elem = nil
}
}
sgnext = sglist.waitlink
+ sglist.waitlink = nil
releaseSudog(sglist)
sglist = sgnext
}
if q.last == sgp {
q.last = prevsgp
}
+ s.next = nil
return
}
l = &sgp.next
// Allocates a stack from the free pool. Must be called with
// stackpoolmu held.
-func stackpoolalloc(order uint8) *mlink {
+func stackpoolalloc(order uint8) gclinkptr {
list := &stackpool[order]
s := list.next
if s == list {
if s.ref != 0 {
gothrow("bad ref")
}
- if s.freelist != nil {
+ if s.freelist.ptr() != nil {
gothrow("bad freelist")
}
for i := uintptr(0); i < _StackCacheSize; i += _FixedStack << order {
- x := (*mlink)(unsafe.Pointer(uintptr(s.start)<<_PageShift + i))
- x.next = s.freelist
+ x := gclinkptr(uintptr(s.start)<<_PageShift + i)
+ x.ptr().next = s.freelist
s.freelist = x
}
mSpanList_Insert(list, s)
}
x := s.freelist
- if x == nil {
+ if x.ptr() == nil {
gothrow("span has no free stacks")
}
- s.freelist = x.next
+ s.freelist = x.ptr().next
s.ref++
- if s.freelist == nil {
+ if s.freelist.ptr() == nil {
// all stacks in s are allocated.
mSpanList_Remove(s)
}
}
// Adds stack x to the free pool. Must be called with stackpoolmu held.
-func stackpoolfree(x *mlink, order uint8) {
+func stackpoolfree(x gclinkptr, order uint8) {
s := mHeap_Lookup(&mheap_, (unsafe.Pointer)(x))
if s.state != _MSpanStack {
gothrow("freeing stack not in a stack span")
}
- if s.freelist == nil {
+ if s.freelist.ptr() == nil {
// s will now have a free stack
mSpanList_Insert(&stackpool[order], s)
}
- x.next = s.freelist
+ x.ptr().next = s.freelist
s.freelist = x
s.ref--
if s.ref == 0 {
// span is completely free - return to heap
mSpanList_Remove(s)
- s.freelist = nil
+ s.freelist = 0
mHeap_FreeStack(&mheap_, s)
}
}
// Grab some stacks from the global cache.
// Grab half of the allowed capacity (to prevent thrashing).
- var list *mlink
+ var list gclinkptr
var size uintptr
lock(&stackpoolmu)
for size < _StackCacheSize/2 {
x := stackpoolalloc(order)
- x.next = list
+ x.ptr().next = list
list = x
size += _FixedStack << order
}
size := c.stackcache[order].size
lock(&stackpoolmu)
for size > _StackCacheSize/2 {
- y := x.next
+ y := x.ptr().next
stackpoolfree(x, order)
x = y
size -= _FixedStack << order
lock(&stackpoolmu)
for order := uint8(0); order < _NumStackOrders; order++ {
x := c.stackcache[order].list
- for x != nil {
- y := x.next
+ for x.ptr() != nil {
+ y := x.ptr().next
stackpoolfree(x, order)
x = y
}
- c.stackcache[order].list = nil
+ c.stackcache[order].list = 0
c.stackcache[order].size = 0
}
unlock(&stackpoolmu)
order++
n2 >>= 1
}
- var x *mlink
+ var x gclinkptr
c := thisg.m.mcache
if c == nil || thisg.m.gcing != 0 || thisg.m.helpgc != 0 {
// c == nil can happen in the guts of exitsyscall or
unlock(&stackpoolmu)
} else {
x = c.stackcache[order].list
- if x == nil {
+ if x.ptr() == nil {
stackcacherefill(c, order)
x = c.stackcache[order].list
}
- c.stackcache[order].list = x.next
+ c.stackcache[order].list = x.ptr().next
c.stackcache[order].size -= uintptr(n)
}
v = (unsafe.Pointer)(x)
order++
n2 >>= 1
}
- x := (*mlink)(v)
+ x := gclinkptr(v)
c := gp.m.mcache
if c == nil || gp.m.gcing != 0 || gp.m.helpgc != 0 {
lock(&stackpoolmu)
if c.stackcache[order].size >= _StackCacheSize {
stackcacherelease(c, order)
}
- x.next = c.stackcache[order].list
+ x.ptr().next = c.stackcache[order].list
c.stackcache[order].list = x
c.stackcache[order].size += n
}
}
// Copies gp's stack to a new stack of a different size.
+// Caller must have changed gp status to Gcopystack.
func copystack(gp *g, newsize uintptr) {
if gp.syscallsp != 0 {
gothrow("stack growth not allowed in system call")
}
memmove(unsafe.Pointer(new.hi-used), unsafe.Pointer(old.hi-used), used)
- oldstatus := readgstatus(gp)
- oldstatus &^= _Gscan
- if oldstatus == _Gwaiting || oldstatus == _Grunnable {
- casgstatus(gp, oldstatus, _Gcopystack) // oldstatus is Gwaiting or Grunnable
- } else {
- gothrow("copystack: bad status, not Gwaiting or Grunnable")
- }
-
// Swap out old stack for new one
gp.stack = new
gp.stackguard0 = new.lo + _StackGuard // NOTE: might clobber a preempt request
gp.sched.sp = new.hi - used
- casgstatus(gp, _Gcopystack, oldstatus) // oldstatus is Gwaiting or Grunnable
-
// free old stack
if stackPoisonCopy != 0 {
fillstack(old, 0xfc)
gothrow("runtime: split stack overflow")
}
+ if gp.sched.ctxt != nil {
+ // morestack wrote sched.ctxt on its way in here,
+ // without a write barrier. Run the write barrier now.
+ // It is not possible to be preempted between then
+ // and now, so it's okay.
+ writebarrierptr_nostore((*uintptr)(unsafe.Pointer(&gp.sched.ctxt)), uintptr(gp.sched.ctxt))
+ }
+
if gp.stackguard0 == stackPreempt {
if gp == thisg.m.g0 {
gothrow("runtime: preempt g0")
gothrow("runtime: g is running but p is not")
}
if gp.preemptscan {
+ for !castogscanstatus(gp, _Gwaiting, _Gscanwaiting) {
+ // Likely to be racing with the GC as it sees a _Gwaiting and does the stack scan.
+ // If so this stack will be scanned twice which does not change correctness.
+ }
gcphasework(gp)
+ casfrom_Gscanstatus(gp, _Gscanwaiting, _Gwaiting)
casgstatus(gp, _Gwaiting, _Grunning)
gp.stackguard0 = gp.stack.lo + _StackGuard
gp.preempt = false
gothrow("stack overflow")
}
- // Note that the concurrent GC might be scanning the stack as we try to replace it.
- // copystack takes care of the appropriate coordination with the stack scanner.
+ oldstatus := readgstatus(gp)
+ oldstatus &^= _Gscan
+ casgstatus(gp, oldstatus, _Gcopystack) // oldstatus is Gwaiting or Grunnable
+
+ // The concurrent GC will not scan the stack while we are doing the copy since
+ // the gp is in a Gcopystack status.
copystack(gp, uintptr(newsize))
if stackDebug >= 1 {
print("stack grow done\n")
}
- casgstatus(gp, _Gwaiting, _Grunning)
+ casgstatus(gp, _Gcopystack, _Grunning)
gogo(&gp.sched)
}
if gp.syscallsp != 0 {
return
}
- if _Windows != 0 && gp.m != nil && gp.m.libcallsp != 0 {
-
- /* TODO
- if goos_windows && gp.m != nil && gp.m.libcallsp != 0 {
++ if goos_windows != 0 && gp.m != nil && gp.m.libcallsp != 0 {
return
}
- */
if stackDebug > 0 {
print("shrinking stack ", oldsize, "->", newsize, "\n")
}
+
+ // This is being done in a Gscan state and was initiated by the GC so no need to move to
+ // the Gcopystate.
+ // The world is stopped, so the goroutine must be Gwaiting or Grunnable,
+ // and what it is is not changing underfoot.
+ oldstatus := readgstatus(gp) &^ _Gscan
+ if oldstatus != _Gwaiting && oldstatus != _Grunnable {
+ gothrow("status is not Gwaiting or Grunnable")
+ }
+ casgstatus(gp, oldstatus, _Gcopystack)
copystack(gp, newsize)
+ casgstatus(gp, _Gcopystack, oldstatus)
}
// Do any delayed stack freeing that was queued up during GC.
//
// func f(arg1, arg2, arg3 int) {
// pc := getcallerpc(unsafe.Pointer(&arg1))
- // sp := getcallerpc(unsafe.Pointer(&arg2))
+ // sp := getcallersp(unsafe.Pointer(&arg1))
// }
//
// These two lines find the PC and SP immediately following
func call1073741824(fn, arg unsafe.Pointer, n, retoffset uint32)
func systemstack_switch()
+
+func prefetcht0(addr uintptr)
+func prefetcht1(addr uintptr)
+func prefetcht2(addr uintptr)
+func prefetchnta(addr uintptr)