diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 43afbc330bb..febcd9558c0 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -1727,7 +1727,13 @@ func gcBgMarkWorker(ready chan struct{}) { // the stack (see gopark). Prevent deadlock from recursively // starting GC by disabling preemption. gp.m.preemptoff = "GC worker init" - node := &new(gcBgMarkWorkerNodePadded).gcBgMarkWorkerNode // TODO: technically not allowed in the heap. See comment in tagptr.go. + // TODO: This is technically not allowed in the heap. See comment in tagptr.go. + // + // It is kept alive simply by virtue of being used in the infinite loop + // below. gcBgMarkWorkerPool keeps pointers to nodes that are not + // GC-visible, so this must be kept alive indefinitely (even if + // GOMAXPROCS decreases). + node := &new(gcBgMarkWorkerNodePadded).gcBgMarkWorkerNode gp.m.preemptoff = "" node.gp.set(gp) diff --git a/src/runtime/mgcpacer.go b/src/runtime/mgcpacer.go index 32c1b941e53..f78fb6f6367 100644 --- a/src/runtime/mgcpacer.go +++ b/src/runtime/mgcpacer.go @@ -10,7 +10,7 @@ import ( "internal/runtime/atomic" "internal/runtime/math" "internal/strconv" - _ "unsafe" // for go:linkname + _ "unsafe" ) const ( @@ -749,30 +749,33 @@ func (c *gcControllerState) enlistWorker() { } } -// findRunnableGCWorker returns a background mark worker for pp if it -// should be run. This must only be called when gcBlackenEnabled != 0. -func (c *gcControllerState) findRunnableGCWorker(pp *p, now int64) (*g, int64) { +// assignWaitingGCWorker assigns a background mark worker to pp if one should +// be run. +// +// If a worker is selected, it is assigned to pp.nextMarkGCWorker and the P is +// wired as a GC mark worker. The G is still in _Gwaiting. If no worker is +// selected, ok returns false. +// +// If assignedWaitingGCWorker returns true, this P must either: +// - Mark the G as runnable and run it, clearing pp.nextMarkGCWorker. +// - Or, call c.releaseNextGCMarkWorker. +// +// This must only be called when gcBlackenEnabled != 0. +func (c *gcControllerState) assignWaitingGCWorker(pp *p, now int64) (bool, int64) { if gcBlackenEnabled == 0 { throw("gcControllerState.findRunnable: blackening not enabled") } - // Since we have the current time, check if the GC CPU limiter - // hasn't had an update in a while. This check is necessary in - // case the limiter is on but hasn't been checked in a while and - // so may have left sufficient headroom to turn off again. if now == 0 { now = nanotime() } - if gcCPULimiter.needUpdate(now) { - gcCPULimiter.update(now) - } if !gcShouldScheduleWorker(pp) { // No good reason to schedule a worker. This can happen at // the end of the mark phase when there are still // assists tapering off. Don't bother running a worker // now because it'll just return immediately. - return nil, now + return false, now } if c.dedicatedMarkWorkersNeeded.Load() <= 0 && c.fractionalUtilizationGoal == 0 { @@ -783,7 +786,7 @@ func (c *gcControllerState) findRunnableGCWorker(pp *p, now int64) (*g, int64) { // When a dedicated worker stops running, the gcBgMarkWorker loop notes // the need for the worker before returning it to the pool. If we don't // see the need now, we wouldn't have found it in the pool anyway. - return nil, now + return false, now } // Grab a worker before we commit to running below. @@ -800,7 +803,7 @@ func (c *gcControllerState) findRunnableGCWorker(pp *p, now int64) (*g, int64) { // it will always do so with queued global work. Thus, that P // will be immediately eligible to re-run the worker G it was // just using, ensuring work can complete. - return nil, now + return false, now } decIfPositive := func(val *atomic.Int64) bool { @@ -823,7 +826,7 @@ func (c *gcControllerState) findRunnableGCWorker(pp *p, now int64) (*g, int64) { } else if c.fractionalUtilizationGoal == 0 { // No need for fractional workers. gcBgMarkWorkerPool.push(&node.node) - return nil, now + return false, now } else { // Is this P behind on the fractional utilization // goal? @@ -833,12 +836,48 @@ func (c *gcControllerState) findRunnableGCWorker(pp *p, now int64) (*g, int64) { if delta > 0 && float64(pp.gcFractionalMarkTime.Load())/float64(delta) > c.fractionalUtilizationGoal { // Nope. No need to run a fractional worker. gcBgMarkWorkerPool.push(&node.node) - return nil, now + return false, now } // Run a fractional worker. pp.gcMarkWorkerMode = gcMarkWorkerFractionalMode } + pp.nextGCMarkWorker = node + return true, now +} + +// findRunnableGCWorker returns a background mark worker for pp if it +// should be run. +// +// If findRunnableGCWorker returns a G, this P is wired as a GC mark worker and +// must run the G. +// +// This must only be called when gcBlackenEnabled != 0. +// +// This function is allowed to have write barriers because it is called from +// the portion of findRunnable that always has a P. +// +//go:yeswritebarrierrec +func (c *gcControllerState) findRunnableGCWorker(pp *p, now int64) (*g, int64) { + // Since we have the current time, check if the GC CPU limiter + // hasn't had an update in a while. This check is necessary in + // case the limiter is on but hasn't been checked in a while and + // so may have left sufficient headroom to turn off again. + if now == 0 { + now = nanotime() + } + if gcCPULimiter.needUpdate(now) { + gcCPULimiter.update(now) + } + + ok, now := c.assignWaitingGCWorker(pp, now) + if !ok { + return nil, now + } + + node := pp.nextGCMarkWorker + pp.nextGCMarkWorker = nil + // Run the background mark worker. gp := node.gp.ptr() trace := traceAcquire() @@ -850,6 +889,23 @@ func (c *gcControllerState) findRunnableGCWorker(pp *p, now int64) (*g, int64) { return gp, now } +// Release an unused pp.nextGCMarkWorker, if any. +// +// This function is allowed to have write barriers because it is called from +// the portion of schedule. +// +//go:yeswritebarrierrec +func (c *gcControllerState) releaseNextGCMarkWorker(pp *p) { + node := pp.nextGCMarkWorker + if node == nil { + return + } + + c.markWorkerStop(pp.gcMarkWorkerMode, 0) + gcBgMarkWorkerPool.push(&node.node) + pp.nextGCMarkWorker = nil +} + // resetLive sets up the controller state for the next mark phase after the end // of the previous one. Must be called after endCycle and before commit, before // the world is started. diff --git a/src/runtime/runtime2.go b/src/runtime/runtime2.go index e415df6ec1a..56082bf7f50 100644 --- a/src/runtime/runtime2.go +++ b/src/runtime/runtime2.go @@ -854,6 +854,18 @@ type p struct { // mark worker started. gcMarkWorkerStartTime int64 + // nextGCMarkWorker is the next mark worker to run. This may be set + // during start-the-world to assign a worker to this P. The P runs this + // worker on the next call to gcController.findRunnableGCWorker. If the + // P runs something else or stops, it must release this worker via + // gcController.releaseNextGCMarkWorker. + // + // See comment in gcBgMarkWorker about the lifetime of + // gcBgMarkWorkerNode. + // + // Only accessed by this P or during STW. + nextGCMarkWorker *gcBgMarkWorkerNode + // gcw is this P's GC work buffer cache. The work buffer is // filled by write barriers, drained by mutator assists, and // disposed on certain GC state transitions.