sync: yield to the waiter when unlocking a starving mutex

When we have already assigned the semaphore ticket to a specific
waiter, we want to get the waiter running as fast as possible since
no other G waiting on the semaphore can acquire it optimistically.

The net effect is that, when a sync.Mutex is contented, the code in
the critical section guarded by the Mutex gets a priority boost.

Fixes golang#33747

Change-Id: I9967f0f763c25504010651bdd7f944ee0189cd45

Author: CAFxX

src/runtime/export_test.go

@@ -718,3 +718,11 @@ func RunGetgThreadSwitchTest() {
 		panic("g1 != g3")
 	}
 }
+
+var Semacquire = semacquire
+var Semrelease1 = semrelease1
+
+func SemNwait(addr *uint32) uint32 {
+	root := semroot(addr)
+	return atomic.Load(&root.nwait)
+}

src/runtime/sema.go

@@ -192,6 +192,15 @@ func semrelease1(addr *uint32, handoff bool, skipframes int) {
 			s.ticket = 1
 		}
 		readyWithTime(s, 5+skipframes)
+		if s.ticket == 1 {
+			// Direct G handoff
+			// readyWithTime has added the waiter G as runnext in the current P; we now call
+			// the scheduler so that we start running the waiter G immediately. Note that
+			// waiter inherits our time slice: this is desirable to avoid having a highly
+			// contended semaphore hog the P indefinitely.
+			// See https://github.com/golang/go/issues/33747 for discussion.
+			Gosched()
+		}
 	}
 }
 

src/runtime/sema_test.go

@@ -0,0 +1,44 @@
+package runtime_test
+
+import (
+	. "runtime"
+	"sync/atomic"
+	"testing"
+)
+
+// TestSemaHandoff checks that when semrelease+handoff is
+// requested, the G that releases the semaphore yields its
+// P directly to the first waiter in line.
+// See https://github.com/golang/go/issues/33747 for discussion.
+func TestSemaHandoff(t *testing.T) {
+	const iter = 10000
+	fail := 0
+	for i := 0; i < iter; i++ {
+		res := testSemaHandoff()
+		if res != 1 {
+			fail++
+		}
+	}
+	// As long as >= 95% of handoffs are direct, we
+	// consider the test successful.
+	if fail > iter*5/100 {
+		t.Fatal("direct handoff failing:", fail)
+	}
+}
+
+func testSemaHandoff() (res uint32) {
+	sema := uint32(1)
+
+	Semacquire(&sema)
+	go func() {
+		Semacquire(&sema)
+		atomic.CompareAndSwapUint32(&res, 0, 1)
+	}()
+	for SemNwait(&sema) == 0 {
+		Gosched()
+	}
+	Semrelease1(&sema, true, 0)
+	atomic.CompareAndSwapUint32(&res, 0, 2)
+
+	return
+}

src/sync/mutex.go

@@ -216,7 +216,8 @@ func (m *Mutex) unlockSlow(new int32) {
 			old = m.state
 		}
 	} else {
-		// Starving mode: handoff mutex ownership to the next waiter.
+		// Starving mode: handoff mutex ownership to the next waiter, and yield our time slice
+		// so that the next waiter can start to run immediately.
 		// Note: mutexLocked is not set, the waiter will set it after wakeup.
 		// But mutex is still considered locked if mutexStarving is set,
 		// so new coming goroutines won't acquire it.