[release-branch.go1.21] runtime: properly model rwmutex in lock ranking

(This cherry-pick combines CL 549536 and the follow-up fix CL 555055.)

Currently, lock ranking doesn't really try to model rwmutex. It records
the internal locks rLock and wLock, but in a subpar fashion:

1. wLock is held from lock to unlock, so it works OK, but it conflates
   write locks of all rwmutexes as rwmutexW, rather than allowing
   different rwmutexes to have different rankings.
2. rLock is an internal implementation detail that is only taken when
   there is contention in rlock. As as result, the reader lock path is
   almost never checked.

Add proper modeling. rwmutexR and rwmutexW remain as the ranks of the
internal locks, which have their own ordering. The new init method is
passed the ranks of the higher level lock that this represents, just
like lockInit for mutex.

execW ordered before MALLOC captures the case from #64722. i.e., there
can be allocation between BeforeFork and AfterFork.

For #64722.
Fixes #64761.

------

runtime: replace rwmutexR/W with per-rwmutex lock rank

CL 549536 intended to decouple the internal implementation of rwmutex
from the semantic meaning of an rwmutex read/write lock in the static
lock ranking.

Unfortunately, it was not thought through well enough. The internals
were represented with the rwmutexR and rwmutexW lock ranks. The idea was
that the internal lock ranks need not model the higher-level ordering,
since those have separate rankings. That is incorrect; rwmutexW is held
for the duration of a write lock, so it must be ranked before any lock
taken while any write lock is held, which is precisely what we were
trying to avoid.

This is visible in violations like:

        0 : execW 11 0x0
        1 : rwmutexW 51 0x111d9c8
        2 : fin 30 0x111d3a0
        fatal error: lock ordering problem

execW < fin is modeled, but rwmutexW < fin is missing.

Fix this by eliminating the rwmutexR/W lock ranks shared across
different types of rwmutex. Instead require users to define an
additional "internal" lock rank to represent the implementation details
of rwmutex.rLock. We can avoid an additional "internal" lock rank for
rwmutex.wLock because the existing writeRank has the same semantics for
semantic and internal locking. i.e., writeRank is held for the duration
of a write lock, which is exactly how rwmutex.wLock is used, so we can
use writeRank directly on wLock.

For #64722.

Cq-Include-Trybots: luci.golang.try:go1.21-linux-amd64-staticlockranking
Change-Id: I23335b28faa42fb04f1bc9da02fdf54d1616cd28
Reviewed-on: https://go-review.googlesource.com/c/go/+/549536
Reviewed-by: Michael Knyszek <mknyszek@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
(cherry picked from commit 9b4b3e5acca2dabe107fa2c3ed963097d78a4562)
(cherry picked from commit dcbe77246922fe7ef41f07df228f47a37803f360)
Reviewed-on: https://go-review.googlesource.com/c/go/+/554976

diff --git a/src/runtime/export_test.go b/src/runtime/export_test.go
index a740993c58785f5a9a9fc4304ae1ea379a9c1b0f..34dd8904b64cf94729305c8438f10c4a49a61084 100644 (file)
--- a/src/runtime/export_test.go
+++ b/src/runtime/export_test.go
@@ -556,6 +556,10 @@ type RWMutex struct {
        rw rwmutex
 }
 
+func (rw *RWMutex) Init() {
+       rw.rw.init(lockRankTestR, lockRankTestRInternal, lockRankTestW)
+}
+
 func (rw *RWMutex) RLock() {
        rw.rw.rlock()
 }

diff --git a/src/runtime/lockrank.go b/src/runtime/lockrank.go
index 4d661e93dcd5c6d174987084d13983d248673d3a..fb99536a570f8010361a0a8bab77c0a322348100 100644 (file)
--- a/src/runtime/lockrank.go
+++ b/src/runtime/lockrank.go
@@ -18,8 +18,15 @@ const (
        lockRankSweepWaiters
        lockRankAssistQueue
        lockRankSweep
-       lockRankPollDesc
+       lockRankTestR
+       lockRankTestW
+       lockRankAllocmW
+       lockRankExecW
        lockRankCpuprof
+       lockRankPollDesc
+       // SCHED
+       lockRankAllocmR
+       lockRankExecR
        lockRankSched
        lockRankAllg
        lockRankAllp
@@ -28,8 +35,6 @@ const (
        lockRankHchan
        lockRankNotifyList
        lockRankSudog
-       lockRankRwmutexW
-       lockRankRwmutexR
        lockRankRoot
        lockRankItab
        lockRankReflectOffs
@@ -63,6 +68,9 @@ const (
        lockRankPanic
        lockRankDeadlock
        lockRankRaceFini
+       lockRankAllocmRInternal
+       lockRankExecRInternal
+       lockRankTestRInternal
 )
 
 // lockRankLeafRank is the rank of lock that does not have a declared rank,
@@ -71,52 +79,59 @@ const lockRankLeafRank lockRank = 1000
 
 // lockNames gives the names associated with each of the above ranks.
 var lockNames = []string{
-       lockRankSysmon:         "sysmon",
-       lockRankScavenge:       "scavenge",
-       lockRankForcegc:        "forcegc",
-       lockRankDefer:          "defer",
-       lockRankSweepWaiters:   "sweepWaiters",
-       lockRankAssistQueue:    "assistQueue",
-       lockRankSweep:          "sweep",
-       lockRankPollDesc:       "pollDesc",
-       lockRankCpuprof:        "cpuprof",
-       lockRankSched:          "sched",
-       lockRankAllg:           "allg",
-       lockRankAllp:           "allp",
-       lockRankTimers:         "timers",
-       lockRankNetpollInit:    "netpollInit",
-       lockRankHchan:          "hchan",
-       lockRankNotifyList:     "notifyList",
-       lockRankSudog:          "sudog",
-       lockRankRwmutexW:       "rwmutexW",
-       lockRankRwmutexR:       "rwmutexR",
-       lockRankRoot:           "root",
-       lockRankItab:           "itab",
-       lockRankReflectOffs:    "reflectOffs",
-       lockRankUserArenaState: "userArenaState",
-       lockRankTraceBuf:       "traceBuf",
-       lockRankTraceStrings:   "traceStrings",
-       lockRankFin:            "fin",
-       lockRankSpanSetSpine:   "spanSetSpine",
-       lockRankMspanSpecial:   "mspanSpecial",
-       lockRankGcBitsArenas:   "gcBitsArenas",
-       lockRankProfInsert:     "profInsert",
-       lockRankProfBlock:      "profBlock",
-       lockRankProfMemActive:  "profMemActive",
-       lockRankProfMemFuture:  "profMemFuture",
-       lockRankGscan:          "gscan",
-       lockRankStackpool:      "stackpool",
-       lockRankStackLarge:     "stackLarge",
-       lockRankHchanLeaf:      "hchanLeaf",
-       lockRankWbufSpans:      "wbufSpans",
-       lockRankMheap:          "mheap",
-       lockRankMheapSpecial:   "mheapSpecial",
-       lockRankGlobalAlloc:    "globalAlloc",
-       lockRankTrace:          "trace",
-       lockRankTraceStackTab:  "traceStackTab",
-       lockRankPanic:          "panic",
-       lockRankDeadlock:       "deadlock",
-       lockRankRaceFini:       "raceFini",
+       lockRankSysmon:          "sysmon",
+       lockRankScavenge:        "scavenge",
+       lockRankForcegc:         "forcegc",
+       lockRankDefer:           "defer",
+       lockRankSweepWaiters:    "sweepWaiters",
+       lockRankAssistQueue:     "assistQueue",
+       lockRankSweep:           "sweep",
+       lockRankTestR:           "testR",
+       lockRankTestW:           "testW",
+       lockRankAllocmW:         "allocmW",
+       lockRankExecW:           "execW",
+       lockRankCpuprof:         "cpuprof",
+       lockRankPollDesc:        "pollDesc",
+       lockRankAllocmR:         "allocmR",
+       lockRankExecR:           "execR",
+       lockRankSched:           "sched",
+       lockRankAllg:            "allg",
+       lockRankAllp:            "allp",
+       lockRankTimers:          "timers",
+       lockRankNetpollInit:     "netpollInit",
+       lockRankHchan:           "hchan",
+       lockRankNotifyList:      "notifyList",
+       lockRankSudog:           "sudog",
+       lockRankRoot:            "root",
+       lockRankItab:            "itab",
+       lockRankReflectOffs:     "reflectOffs",
+       lockRankUserArenaState:  "userArenaState",
+       lockRankTraceBuf:        "traceBuf",
+       lockRankTraceStrings:    "traceStrings",
+       lockRankFin:             "fin",
+       lockRankSpanSetSpine:    "spanSetSpine",
+       lockRankMspanSpecial:    "mspanSpecial",
+       lockRankGcBitsArenas:    "gcBitsArenas",
+       lockRankProfInsert:      "profInsert",
+       lockRankProfBlock:       "profBlock",
+       lockRankProfMemActive:   "profMemActive",
+       lockRankProfMemFuture:   "profMemFuture",
+       lockRankGscan:           "gscan",
+       lockRankStackpool:       "stackpool",
+       lockRankStackLarge:      "stackLarge",
+       lockRankHchanLeaf:       "hchanLeaf",
+       lockRankWbufSpans:       "wbufSpans",
+       lockRankMheap:           "mheap",
+       lockRankMheapSpecial:    "mheapSpecial",
+       lockRankGlobalAlloc:     "globalAlloc",
+       lockRankTrace:           "trace",
+       lockRankTraceStackTab:   "traceStackTab",
+       lockRankPanic:           "panic",
+       lockRankDeadlock:        "deadlock",
+       lockRankRaceFini:        "raceFini",
+       lockRankAllocmRInternal: "allocmRInternal",
+       lockRankExecRInternal:   "execRInternal",
+       lockRankTestRInternal:   "testRInternal",
 }
 
 func (rank lockRank) String() string {
@@ -138,50 +153,57 @@ func (rank lockRank) String() string {
 //
 // Lock ranks that allow self-cycles list themselves.
 var lockPartialOrder [][]lockRank = [][]lockRank{
-       lockRankSysmon:         {},
-       lockRankScavenge:       {lockRankSysmon},
-       lockRankForcegc:        {lockRankSysmon},
-       lockRankDefer:          {},
-       lockRankSweepWaiters:   {},
-       lockRankAssistQueue:    {},
-       lockRankSweep:          {},
-       lockRankPollDesc:       {},
-       lockRankCpuprof:        {},
-       lockRankSched:          {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof},
-       lockRankAllg:           {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched},
-       lockRankAllp:           {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched},
-       lockRankTimers:         {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllp, lockRankTimers},
-       lockRankNetpollInit:    {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllp, lockRankTimers},
-       lockRankHchan:          {lockRankSysmon, lockRankScavenge, lockRankSweep, lockRankHchan},
-       lockRankNotifyList:     {},
-       lockRankSudog:          {lockRankSysmon, lockRankScavenge, lockRankSweep, lockRankHchan, lockRankNotifyList},
-       lockRankRwmutexW:       {},
-       lockRankRwmutexR:       {lockRankSysmon, lockRankRwmutexW},
-       lockRankRoot:           {},
-       lockRankItab:           {},
-       lockRankReflectOffs:    {lockRankItab},
-       lockRankUserArenaState: {},
-       lockRankTraceBuf:       {lockRankSysmon, lockRankScavenge},
-       lockRankTraceStrings:   {lockRankSysmon, lockRankScavenge, lockRankTraceBuf},
-       lockRankFin:            {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
-       lockRankSpanSetSpine:   {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
-       lockRankMspanSpecial:   {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
-       lockRankGcBitsArenas:   {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankMspanSpecial},
-       lockRankProfInsert:     {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
-       lockRankProfBlock:      {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
-       lockRankProfMemActive:  {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
-       lockRankProfMemFuture:  {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankProfMemActive},
-       lockRankGscan:          {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture},
-       lockRankStackpool:      {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankRwmutexW, lockRankRwmutexR, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan},
-       lockRankStackLarge:     {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan},
-       lockRankHchanLeaf:      {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankHchanLeaf},
-       lockRankWbufSpans:      {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan},
-       lockRankMheap:          {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRwmutexW, lockRankRwmutexR, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans},
-       lockRankMheapSpecial:   {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRwmutexW, lockRankRwmutexR, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap},
-       lockRankGlobalAlloc:    {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRwmutexW, lockRankRwmutexR, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap, lockRankMheapSpecial},
-       lockRankTrace:          {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRwmutexW, lockRankRwmutexR, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap},
-       lockRankTraceStackTab:  {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankPollDesc, lockRankCpuprof, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRwmutexW, lockRankRwmutexR, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap, lockRankTrace},
-       lockRankPanic:          {},
-       lockRankDeadlock:       {lockRankPanic, lockRankDeadlock},
-       lockRankRaceFini:       {lockRankPanic},
+       lockRankSysmon:          {},
+       lockRankScavenge:        {lockRankSysmon},
+       lockRankForcegc:         {lockRankSysmon},
+       lockRankDefer:           {},
+       lockRankSweepWaiters:    {},
+       lockRankAssistQueue:     {},
+       lockRankSweep:           {},
+       lockRankTestR:           {},
+       lockRankTestW:           {},
+       lockRankAllocmW:         {},
+       lockRankExecW:           {},
+       lockRankCpuprof:         {},
+       lockRankPollDesc:        {},
+       lockRankAllocmR:         {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankCpuprof, lockRankPollDesc},
+       lockRankExecR:           {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankCpuprof, lockRankPollDesc},
+       lockRankSched:           {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR},
+       lockRankAllg:            {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched},
+       lockRankAllp:            {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched},
+       lockRankTimers:          {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllp, lockRankTimers},
+       lockRankNetpollInit:     {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllp, lockRankTimers},
+       lockRankHchan:           {lockRankSysmon, lockRankScavenge, lockRankSweep, lockRankTestR, lockRankHchan},
+       lockRankNotifyList:      {},
+       lockRankSudog:           {lockRankSysmon, lockRankScavenge, lockRankSweep, lockRankTestR, lockRankHchan, lockRankNotifyList},
+       lockRankRoot:            {},
+       lockRankItab:            {},
+       lockRankReflectOffs:     {lockRankItab},
+       lockRankUserArenaState:  {},
+       lockRankTraceBuf:        {lockRankSysmon, lockRankScavenge},
+       lockRankTraceStrings:    {lockRankSysmon, lockRankScavenge, lockRankTraceBuf},
+       lockRankFin:             {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
+       lockRankSpanSetSpine:    {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
+       lockRankMspanSpecial:    {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
+       lockRankGcBitsArenas:    {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankMspanSpecial},
+       lockRankProfInsert:      {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
+       lockRankProfBlock:       {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
+       lockRankProfMemActive:   {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings},
+       lockRankProfMemFuture:   {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankHchan, lockRankNotifyList, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankProfMemActive},
+       lockRankGscan:           {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture},
+       lockRankStackpool:       {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan},
+       lockRankStackLarge:      {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan},
+       lockRankHchanLeaf:       {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankHchanLeaf},
+       lockRankWbufSpans:       {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan},
+       lockRankMheap:           {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans},
+       lockRankMheapSpecial:    {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap},
+       lockRankGlobalAlloc:     {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap, lockRankMheapSpecial},
+       lockRankTrace:           {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap},
+       lockRankTraceStackTab:   {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankDefer, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR, lockRankExecR, lockRankSched, lockRankAllg, lockRankAllp, lockRankTimers, lockRankNetpollInit, lockRankHchan, lockRankNotifyList, lockRankSudog, lockRankRoot, lockRankItab, lockRankReflectOffs, lockRankUserArenaState, lockRankTraceBuf, lockRankTraceStrings, lockRankFin, lockRankSpanSetSpine, lockRankMspanSpecial, lockRankGcBitsArenas, lockRankProfInsert, lockRankProfBlock, lockRankProfMemActive, lockRankProfMemFuture, lockRankGscan, lockRankStackpool, lockRankStackLarge, lockRankWbufSpans, lockRankMheap, lockRankTrace},
+       lockRankPanic:           {},
+       lockRankDeadlock:        {lockRankPanic, lockRankDeadlock},
+       lockRankRaceFini:        {lockRankPanic},
+       lockRankAllocmRInternal: {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankAllocmW, lockRankCpuprof, lockRankPollDesc, lockRankAllocmR},
+       lockRankExecRInternal:   {lockRankSysmon, lockRankScavenge, lockRankForcegc, lockRankSweepWaiters, lockRankAssistQueue, lockRankSweep, lockRankTestR, lockRankExecW, lockRankCpuprof, lockRankPollDesc, lockRankExecR},
+       lockRankTestRInternal:   {lockRankTestR, lockRankTestW},
 }

diff --git a/src/runtime/mklockrank.go b/src/runtime/mklockrank.go
index c0d5a02146066eb626d773fd5195f747e567d338..744dc9290c5f34fac87910050e3ee42ca2c23a44 100644 (file)
--- a/src/runtime/mklockrank.go
+++ b/src/runtime/mklockrank.go
@@ -52,29 +52,38 @@ NONE <
   assistQueue,
   sweep;
 
+# Test only
+NONE < testR, testW;
+
 # Scheduler, timers, netpoll
-NONE < pollDesc, cpuprof;
+NONE <
+  allocmW,
+  execW,
+  cpuprof,
+  pollDesc;
 assistQueue,
   cpuprof,
   forcegc,
   pollDesc, # pollDesc can interact with timers, which can lock sched.
   scavenge,
   sweep,
-  sweepWaiters
+  sweepWaiters,
+  testR
+# Above SCHED are things that can call into the scheduler.
+< SCHED
+# Below SCHED is the scheduler implementation.
+< allocmR,
+  execR
 < sched;
 sched < allg, allp;
 allp < timers;
 timers < netpollInit;
 
 # Channels
-scavenge, sweep < hchan;
+scavenge, sweep, testR < hchan;
 NONE < notifyList;
 hchan, notifyList < sudog;
 
-# RWMutex
-NONE < rwmutexW;
-rwmutexW, sysmon < rwmutexR;
-
 # Semaphores
 NONE < root;
 
@@ -99,6 +108,9 @@ traceBuf < traceStrings;
 
 # Malloc
 allg,
+  allocmR,
+  execR, # May grow stack
+  execW, # May allocate after BeforeFork
   hchan,
   notifyList,
   reflectOffs,
@@ -135,7 +147,7 @@ gcBitsArenas,
 < STACKGROW
 # Below STACKGROW is the stack allocator/copying implementation.
 < gscan;
-gscan, rwmutexR < stackpool;
+gscan < stackpool;
 gscan < stackLarge;
 # Generally, hchan must be acquired before gscan. But in one case,
 # where we suspend a G and then shrink its stack, syncadjustsudogs
@@ -188,6 +200,20 @@ NONE < panic;
 panic < deadlock;
 # raceFini is only held while exiting.
 panic < raceFini;
+
+# RWMutex internal read lock
+
+allocmR,
+  allocmW
+< allocmRInternal;
+
+execR,
+  execW
+< execRInternal;
+
+testR,
+  testW
+< testRInternal;
 `
 
 // cyclicRanks lists lock ranks that allow multiple locks of the same

diff --git a/src/runtime/proc.go b/src/runtime/proc.go
index 0425aede23d429a7a8df24396b2cbcc508bd520f..b408337015a758458ec2ecf9fb6e9fbd2e33b922 100644 (file)
--- a/src/runtime/proc.go
+++ b/src/runtime/proc.go
@@ -703,6 +703,8 @@ func schedinit() {
        lockInit(&reflectOffs.lock, lockRankReflectOffs)
        lockInit(&finlock, lockRankFin)
        lockInit(&cpuprof.lock, lockRankCpuprof)
+       allocmLock.init(lockRankAllocmR, lockRankAllocmRInternal, lockRankAllocmW)
+       execLock.init(lockRankExecR, lockRankExecRInternal, lockRankExecW)
        traceLockInit()
        // Enforce that this lock is always a leaf lock.
        // All of this lock's critical sections should be

diff --git a/src/runtime/rwmutex.go b/src/runtime/rwmutex.go
index ede3d13599c4d59817a9c830ef1f434ea2e9283d..34d8f675c1798525af797e7b4ad2c465b01e7cfc 100644 (file)
--- a/src/runtime/rwmutex.go
+++ b/src/runtime/rwmutex.go
@@ -25,6 +25,43 @@ type rwmutex struct {
 
        readerCount atomic.Int32 // number of pending readers
        readerWait  atomic.Int32 // number of departing readers
+
+       readRank  lockRank // semantic lock rank for read locking
+}
+
+// Lock ranking an rwmutex has two aspects:
+//
+// Semantic ranking: this rwmutex represents some higher level lock that
+// protects some resource (e.g., allocmLock protects creation of new Ms). The
+// read and write locks of that resource need to be represented in the lock
+// rank.
+//
+// Internal ranking: as an implementation detail, rwmutex uses two mutexes:
+// rLock and wLock. These have lock order requirements: wLock must be locked
+// before rLock. This also needs to be represented in the lock rank.
+//
+// Semantic ranking is represented by acquiring readRank during read lock and
+// writeRank during write lock.
+//
+// wLock is held for the duration of a write lock, so it uses writeRank
+// directly, both for semantic and internal ranking. rLock is only held
+// temporarily inside the rlock/lock methods, so it uses readRankInternal to
+// represent internal ranking. Semantic ranking is represented by a separate
+// acquire of readRank for the duration of a read lock.
+//
+// The lock ranking must document this ordering:
+// - readRankInternal is a leaf lock.
+// - readRank is taken before readRankInternal.
+// - writeRank is taken before readRankInternal.
+// - readRank is placed in the lock order wherever a read lock of this rwmutex
+//   belongs.
+// - writeRank is placed in the lock order wherever a write lock of this
+//   rwmutex belongs.
+func (rw *rwmutex) init(readRank, readRankInternal, writeRank lockRank) {
+       rw.readRank = readRank
+
+       lockInit(&rw.rLock, readRankInternal)
+       lockInit(&rw.wLock, writeRank)
 }
 
 const rwmutexMaxReaders = 1 << 30
@@ -36,10 +73,14 @@ func (rw *rwmutex) rlock() {
        // deadlock (issue #20903). Alternatively, we could drop the P
        // while sleeping.
        acquirem()
+
+       acquireLockRank(rw.readRank)
+       lockWithRankMayAcquire(&rw.rLock, getLockRank(&rw.rLock))
+
        if rw.readerCount.Add(1) < 0 {
                // A writer is pending. Park on the reader queue.
                systemstack(func() {
-                       lockWithRank(&rw.rLock, lockRankRwmutexR)
+                       lock(&rw.rLock)
                        if rw.readerPass > 0 {
                                // Writer finished.
                                rw.readerPass -= 1
@@ -67,7 +108,7 @@ func (rw *rwmutex) runlock() {
                // A writer is pending.
                if rw.readerWait.Add(-1) == 0 {
                        // The last reader unblocks the writer.
-                       lockWithRank(&rw.rLock, lockRankRwmutexR)
+                       lock(&rw.rLock)
                        w := rw.writer.ptr()
                        if w != nil {
                                notewakeup(&w.park)
@@ -75,18 +116,19 @@ func (rw *rwmutex) runlock() {
                        unlock(&rw.rLock)
                }
        }
+       releaseLockRank(rw.readRank)
        releasem(getg().m)
 }
 
 // lock locks rw for writing.
 func (rw *rwmutex) lock() {
        // Resolve competition with other writers and stick to our P.
-       lockWithRank(&rw.wLock, lockRankRwmutexW)
+       lock(&rw.wLock)
        m := getg().m
        // Announce that there is a pending writer.
        r := rw.readerCount.Add(-rwmutexMaxReaders) + rwmutexMaxReaders
        // Wait for any active readers to complete.
-       lockWithRank(&rw.rLock, lockRankRwmutexR)
+       lock(&rw.rLock)
        if r != 0 && rw.readerWait.Add(r) != 0 {
                // Wait for reader to wake us up.
                systemstack(func() {
@@ -108,7 +150,7 @@ func (rw *rwmutex) unlock() {
                throw("unlock of unlocked rwmutex")
        }
        // Unblock blocked readers.
-       lockWithRank(&rw.rLock, lockRankRwmutexR)
+       lock(&rw.rLock)
        for rw.readers.ptr() != nil {
                reader := rw.readers.ptr()
                rw.readers = reader.schedlink

diff --git a/src/runtime/rwmutex_test.go b/src/runtime/rwmutex_test.go
index ddb16aead4382b615f16dbac8045fc947320220e..bdeb9c4901ddfc61162a4e6f5d8e40f6b292dbeb 100644 (file)
--- a/src/runtime/rwmutex_test.go
+++ b/src/runtime/rwmutex_test.go
@@ -29,6 +29,7 @@ func parallelReader(m *RWMutex, clocked chan bool, cunlock *atomic.Bool, cdone c
 func doTestParallelReaders(numReaders int) {
        GOMAXPROCS(numReaders + 1)
        var m RWMutex
+       m.Init()
        clocked := make(chan bool, numReaders)
        var cunlock atomic.Bool
        cdone := make(chan bool)
@@ -100,6 +101,7 @@ func HammerRWMutex(gomaxprocs, numReaders, num_iterations int) {
        // Number of active readers + 10000 * number of active writers.
        var activity int32
        var rwm RWMutex
+       rwm.Init()
        cdone := make(chan bool)
        go writer(&rwm, num_iterations, &activity, cdone)
        var i int
@@ -141,6 +143,7 @@ func BenchmarkRWMutexUncontended(b *testing.B) {
        }
        b.RunParallel(func(pb *testing.PB) {
                var rwm PaddedRWMutex
+               rwm.Init()
                for pb.Next() {
                        rwm.RLock()
                        rwm.RLock()
@@ -154,6 +157,7 @@ func BenchmarkRWMutexUncontended(b *testing.B) {
 
 func benchmarkRWMutex(b *testing.B, localWork, writeRatio int) {
        var rwm RWMutex
+       rwm.Init()
        b.RunParallel(func(pb *testing.PB) {
                foo := 0
                for pb.Next() {