runtime: access modules via a slice

The introduction of -buildmode=plugin means modules can be added to a
Go program while it is running. This means there exists some time
while the program is running with the module is on the moduledata
linked list, but it has not been initialized to the satisfaction of
other parts of the runtime. Notably, the GC.

This CL adds a new way of access modules, an activeModules function.
It returns a slice of modules that is built in the background and
atomically swapped in. The parts of the runtime that need to wait on
module initialization can use this slice instead of the linked list.

Fixes #17455

Change-Id: I04790fd07e40c7295beb47cea202eb439206d33d
Reviewed-on: https://go-review.googlesource.com/32357
Reviewed-by: Ian Lance Taylor <iant@golang.org>

diff --git a/src/runtime/cgocall.go b/src/runtime/cgocall.go
index 4542cb7b098ece1d0946d3cd7b8d3159a62a7704..86091c7a4d5f4dc9b6d7116312ca209c113a1e3c 100644 (file)
--- a/src/runtime/cgocall.go
+++ b/src/runtime/cgocall.go
@@ -569,7 +569,7 @@ func cgoCheckUnknownPointer(p unsafe.Pointer, msg string) (base, i uintptr) {
                return
        }
 
-       for datap := &firstmoduledata; datap != nil; datap = datap.next {
+       for _, datap := range activeModules() {
                if cgoInRange(p, datap.data, datap.edata) || cgoInRange(p, datap.bss, datap.ebss) {
                        // We have no way to know the size of the object.
                        // We have to assume that it might contain a pointer.
@@ -596,7 +596,7 @@ func cgoIsGoPointer(p unsafe.Pointer) bool {
                return true
        }
 
-       for datap := &firstmoduledata; datap != nil; datap = datap.next {
+       for _, datap := range activeModules() {
                if cgoInRange(p, datap.data, datap.edata) || cgoInRange(p, datap.bss, datap.ebss) {
                        return true
                }

diff --git a/src/runtime/cgocheck.go b/src/runtime/cgocheck.go
index cdec4f816fb2fe8b95a4268f0151c07ed18bab21..8cac5d994d5445550f1d790f74a4b63255604f63 100644 (file)
--- a/src/runtime/cgocheck.go
+++ b/src/runtime/cgocheck.go
@@ -108,7 +108,7 @@ func cgoCheckTypedBlock(typ *_type, src unsafe.Pointer, off, size uintptr) {
        }
 
        // The type has a GC program. Try to find GC bits somewhere else.
-       for datap := &firstmoduledata; datap != nil; datap = datap.next {
+       for _, datap := range activeModules() {
                if cgoInRange(src, datap.data, datap.edata) {
                        doff := uintptr(src) - datap.data
                        cgoCheckBits(add(src, -doff), datap.gcdatamask.bytedata, off+doff, size)

diff --git a/src/runtime/iface.go b/src/runtime/iface.go
index 721ac6924f1daf1387a64c464bff5cddad961167..26e2956eea013e13eb013895e22cbdbecba1eb0d 100644 (file)
--- a/src/runtime/iface.go
+++ b/src/runtime/iface.go
@@ -148,8 +148,8 @@ func additab(m *itab, locked, canfail bool) {
 
 func itabsinit() {
        lock(&ifaceLock)
-       for m := &firstmoduledata; m != nil; m = m.next {
-               for _, i := range m.itablinks {
+       for _, md := range activeModules() {
+               for _, i := range md.itablinks {
                        additab(i, true, false)
                }
        }

diff --git a/src/runtime/mbitmap.go b/src/runtime/mbitmap.go
index d32a8889d0a5f660c49081079a49f8953ba10a92..89d8a4cc76e43c1431c5721ad1fac9c64af65fe8 100644 (file)
--- a/src/runtime/mbitmap.go
+++ b/src/runtime/mbitmap.go
@@ -606,13 +606,13 @@ func bulkBarrierPreWrite(dst, src, size uintptr) {
 
                // If dst is a global, use the data or BSS bitmaps to
                // execute write barriers.
-               for datap := &firstmoduledata; datap != nil; datap = datap.next {
+               for _, datap := range activeModules() {
                        if datap.data <= dst && dst < datap.edata {
                                bulkBarrierBitmap(dst, src, size, dst-datap.data, datap.gcdatamask.bytedata)
                                return
                        }
                }
-               for datap := &firstmoduledata; datap != nil; datap = datap.next {
+               for _, datap := range activeModules() {
                        if datap.bss <= dst && dst < datap.ebss {
                                bulkBarrierBitmap(dst, src, size, dst-datap.bss, datap.gcbssmask.bytedata)
                                return
@@ -1852,7 +1852,7 @@ func getgcmask(ep interface{}) (mask []byte) {
        p := e.data
        t := e._type
        // data or bss
-       for datap := &firstmoduledata; datap != nil; datap = datap.next {
+       for _, datap := range activeModules() {
                // data
                if datap.data <= uintptr(p) && uintptr(p) < datap.edata {
                        bitmap := datap.gcdatamask.bytedata

diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index c625b75ea9eb20f82ff3d37800cfde96a8313770..430b7aa657a82fe07eafb8e37616ea8e3a4e704c 100644 (file)
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -176,10 +176,6 @@ func gcinit() {
        }
 
        _ = setGCPercent(readgogc())
-       for datap := &firstmoduledata; datap != nil; datap = datap.next {
-               datap.gcdatamask = progToPointerMask((*byte)(unsafe.Pointer(datap.gcdata)), datap.edata-datap.data)
-               datap.gcbssmask = progToPointerMask((*byte)(unsafe.Pointer(datap.gcbss)), datap.ebss-datap.bss)
-       }
        memstats.gc_trigger = heapminimum
        // Compute the goal heap size based on the trigger:
        //   trigger = marked * (1 + triggerRatio)

diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go
index 71092cb19dcc44bed2fe05aa543ab123717029be..00787ade04ac447b9348b222a895504a0fa7e507 100644 (file)
--- a/src/runtime/mgcmark.go
+++ b/src/runtime/mgcmark.go
@@ -60,14 +60,14 @@ func gcMarkRootPrepare() {
 
        // Only scan globals once per cycle; preferably concurrently.
        if !work.markrootDone {
-               for datap := &firstmoduledata; datap != nil; datap = datap.next {
+               for _, datap := range activeModules() {
                        nDataRoots := nBlocks(datap.edata - datap.data)
                        if nDataRoots > work.nDataRoots {
                                work.nDataRoots = nDataRoots
                        }
                }
 
-               for datap := &firstmoduledata; datap != nil; datap = datap.next {
+               for _, datap := range activeModules() {
                        nBSSRoots := nBlocks(datap.ebss - datap.bss)
                        if nBSSRoots > work.nBSSRoots {
                                work.nBSSRoots = nBSSRoots
@@ -175,12 +175,12 @@ func markroot(gcw *gcWork, i uint32) {
                flushmcache(int(i - baseFlushCache))
 
        case baseData <= i && i < baseBSS:
-               for datap := &firstmoduledata; datap != nil; datap = datap.next {
+               for _, datap := range activeModules() {
                        markrootBlock(datap.data, datap.edata-datap.data, datap.gcdatamask.bytedata, gcw, int(i-baseData))
                }
 
        case baseBSS <= i && i < baseSpans:
-               for datap := &firstmoduledata; datap != nil; datap = datap.next {
+               for _, datap := range activeModules() {
                        markrootBlock(datap.bss, datap.ebss-datap.bss, datap.gcbssmask.bytedata, gcw, int(i-baseBSS))
                }
 

diff --git a/src/runtime/plugin.go b/src/runtime/plugin.go
index 91fc275a653910726676c3ef65452329199c9aa1..7907936e1447cf8dae5dc80516141df95fe232b5 100644 (file)
--- a/src/runtime/plugin.go
+++ b/src/runtime/plugin.go
@@ -19,7 +19,7 @@ func plugin_lastmoduleinit() (path string, syms map[string]interface{}) {
                throw("runtime: plugin already initialized")
        }
 
-       for pmd := &firstmoduledata; pmd != md; pmd = pmd.next {
+       for _, pmd := range activeModules() {
                if pmd.pluginpath == md.pluginpath {
                        println("plugin: plugin", md.pluginpath, "already loaded")
                        throw("plugin: plugin already loaded")
@@ -43,9 +43,8 @@ func plugin_lastmoduleinit() (path string, syms map[string]interface{}) {
        }
 
        // Initialize the freshly loaded module.
+       modulesinit()
        typelinksinit()
-       md.gcdatamask = progToPointerMask((*byte)(unsafe.Pointer(md.gcdata)), md.edata-md.data)
-       md.gcbssmask = progToPointerMask((*byte)(unsafe.Pointer(md.gcbss)), md.ebss-md.bss)
 
        lock(&ifaceLock)
        for _, i := range md.itablinks {

diff --git a/src/runtime/proc.go b/src/runtime/proc.go
index c83644e8103d159cbc181bcfa993d819d767a139..1f47dc4de402e5f39cd566966ed309a66c565a8b 100644 (file)
--- a/src/runtime/proc.go
+++ b/src/runtime/proc.go
@@ -465,8 +465,9 @@ func schedinit() {
        mallocinit()
        mcommoninit(_g_.m)
        alginit()       // maps must not be used before this call
-       typelinksinit() // uses maps
-       itabsinit()
+       modulesinit()   // provides activeModules
+       typelinksinit() // uses maps, activeModules
+       itabsinit()     // uses activeModules
 
        msigsave(_g_.m)
        initSigmask = _g_.m.sigmask
@@ -474,7 +475,7 @@ func schedinit() {
        goargs()
        goenvs()
        parsedebugvars()
-       gcinit()
+       gcinit() // requires modulesinit
 
        sched.lastpoll = uint64(nanotime())
        procs := ncpu

diff --git a/src/runtime/runtime1.go b/src/runtime/runtime1.go
index 780e1d907a94fa536a0e5090c95609fb1a67492b..40c0e8579cb5e3c0de6638465abae7dffaade24e 100644 (file)
--- a/src/runtime/runtime1.go
+++ b/src/runtime/runtime1.go
@@ -493,11 +493,12 @@ func gomcache() *mcache {
 
 //go:linkname reflect_typelinks reflect.typelinks
 func reflect_typelinks() ([]unsafe.Pointer, [][]int32) {
-       sections := []unsafe.Pointer{unsafe.Pointer(firstmoduledata.types)}
-       ret := [][]int32{firstmoduledata.typelinks}
-       for datap := firstmoduledata.next; datap != nil; datap = datap.next {
-               sections = append(sections, unsafe.Pointer(datap.types))
-               ret = append(ret, datap.typelinks)
+       modules := activeModules()
+       sections := []unsafe.Pointer{unsafe.Pointer(modules[0].types)}
+       ret := [][]int32{modules[0].typelinks}
+       for _, md := range modules[1:] {
+               sections = append(sections, unsafe.Pointer(md.types))
+               ret = append(ret, md.typelinks)
        }
        return sections, ret
 }

diff --git a/src/runtime/symtab.go b/src/runtime/symtab.go
index d69f610ebb072ab40b6d3e806ff070a23a6dd6c2..9ec95d7a0cafbac54ca86c9e5944d44433117a53 100644 (file)
--- a/src/runtime/symtab.go
+++ b/src/runtime/symtab.go
@@ -5,6 +5,7 @@
 package runtime
 
 import (
+       "runtime/internal/atomic"
        "runtime/internal/sys"
        "unsafe"
 )
@@ -233,6 +234,52 @@ var pinnedTypemaps []map[typeOff]*_type
 
 var firstmoduledata moduledata  // linker symbol
 var lastmoduledatap *moduledata // linker symbol
+var modulesSlice unsafe.Pointer // see activeModules
+
+// activeModules returns a slice of active modules.
+//
+// A module is active once its gcdatamask and gcbssmask have been
+// assembled and it is usable by the GC.
+func activeModules() []*moduledata {
+       p := (*[]*moduledata)(atomic.Loadp(unsafe.Pointer(&modulesSlice)))
+       if p == nil {
+               return nil
+       }
+       return *p
+}
+
+// modulesinit creates the active modules slice out of all loaded modules.
+//
+// When a module is first loaded by the dynamic linker, an .init_array
+// function (written by cmd/link) is invoked to call addmoduledata,
+// appending to the module to the linked list that starts with
+// firstmoduledata.
+//
+// There are two times this can happen in the lifecycle of a Go
+// program. First, if compiled with -linkshared, a number of modules
+// built with -buildmode=shared can be loaded at program initialization.
+// Second, a Go program can load a module while running that was built
+// with -buildmode=plugin.
+//
+// After loading, this function is called which initializes the
+// moduledata so it is usable by the GC and creates a new activeModules
+// list.
+//
+// Only one goroutine may call modulesinit at a time.
+func modulesinit() {
+       oldNum := len(activeModules())
+       modules := new([]*moduledata)
+       num := 0
+       for md := &firstmoduledata; md != nil; md = md.next {
+               *modules = append(*modules, md)
+               num++
+               if num > oldNum {
+                       md.gcdatamask = progToPointerMask((*byte)(unsafe.Pointer(md.gcdata)), md.edata-md.data)
+                       md.gcbssmask = progToPointerMask((*byte)(unsafe.Pointer(md.gcbss)), md.ebss-md.bss)
+               }
+       }
+       atomicstorep(unsafe.Pointer(&modulesSlice), unsafe.Pointer(modules))
+}
 
 type functab struct {
        entry   uintptr

diff --git a/src/runtime/type.go b/src/runtime/type.go
index cacf880e9e2b0e7a3825c5809a3aaf7b3c55b06a..a3a19b9be03205553423cea01efbba189a2ed423 100644 (file)
--- a/src/runtime/type.go
+++ b/src/runtime/type.go
@@ -471,9 +471,9 @@ func typelinksinit() {
        }
        typehash := make(map[uint32][]*_type, len(firstmoduledata.typelinks))
 
-       prev := &firstmoduledata
-       md := firstmoduledata.next
-       for md != nil {
+       modules := activeModules()
+       prev := modules[0]
+       for _, md := range modules[1:] {
                // Collect types from the previous module into typehash.
        collect:
                for _, tl := range prev.typelinks {
@@ -513,7 +513,6 @@ func typelinksinit() {
                }
 
                prev = md
-               md = md.next
        }
 }