@@ -37,7 +37,34 @@ type Locker interface {
3737const (
3838 mutexLocked = 1 << iota // mutex is locked
3939 mutexWoken
40+ mutexStarving
4041 mutexWaiterShift = iota
42+
43+ // Mutex fairness.
44+ //
45+ // Mutex can be in 2 modes of operations: normal and starvation.
46+ // In normal mode waiters are queued in FIFO order, but a woken up waiter
47+ // does not own the mutex and competes with new arriving goroutines over
48+ // the ownership. New arriving goroutines have an advantage -- they are
49+ // already running on CPU and there can be lots of them, so a woken up
50+ // waiter has good chances of losing. In such case it is queued at front
51+ // of the wait queue. If a waiter fails to acquire the mutex for more than 1ms,
52+ // it switches mutex to the starvation mode.
53+ //
54+ // In starvation mode ownership of the mutex is directly handed off from
55+ // the unlocking goroutine to the waiter at the front of the queue.
56+ // New arriving goroutines don't try to acquire the mutex even if it appears
57+ // to be unlocked, and don't try to spin. Instead they queue themselves at
58+ // the tail of the wait queue.
59+ //
60+ // If a waiter receives ownership of the mutex and sees that either
61+ // (1) it is the last waiter in the queue, or (2) it waited for less than 1 ms,
62+ // it switches mutex back to normal operation mode.
63+ //
64+ // Normal mode has considerably better performance as a goroutine can acquire
65+ // a mutex several times in a row even if there are blocked waiters.
66+ // Starvation mode is important to prevent pathological cases of tail latency.
67+ starvationThresholdNs = 1e6
4168)
4269
4370// Lock locks m.
@@ -52,41 +79,86 @@ func (m *Mutex) Lock() {
5279 return
5380 }
5481
82+ var waitStartTime int64
83+ starving := false
5584 awoke := false
5685 iter := 0
86+ old := m .state
5787 for {
58- old := m .state
59- new := old | mutexLocked
60- if old & mutexLocked != 0 {
61- if runtime_canSpin (iter ) {
62- // Active spinning makes sense.
63- // Try to set mutexWoken flag to inform Unlock
64- // to not wake other blocked goroutines.
65- if ! awoke && old & mutexWoken == 0 && old >> mutexWaiterShift != 0 &&
66- atomic .CompareAndSwapInt32 (& m .state , old , old | mutexWoken ) {
67- awoke = true
68- }
69- runtime_doSpin ()
70- iter ++
71- continue
88+ // Don't spin in starvation mode, ownership is handed off to waiters
89+ // so we won't be able to acquire the mutex anyway.
90+ if old & (mutexLocked | mutexStarving ) == mutexLocked && runtime_canSpin (iter ) {
91+ // Active spinning makes sense.
92+ // Try to set mutexWoken flag to inform Unlock
93+ // to not wake other blocked goroutines.
94+ if ! awoke && old & mutexWoken == 0 && old >> mutexWaiterShift != 0 &&
95+ atomic .CompareAndSwapInt32 (& m .state , old , old | mutexWoken ) {
96+ awoke = true
7297 }
73- new = old + 1 << mutexWaiterShift
98+ runtime_doSpin ()
99+ iter ++
100+ old = m .state
101+ continue
102+ }
103+ new := old
104+ // Don't try to acquire starving mutex, new arriving goroutines must queue.
105+ if old & mutexStarving == 0 {
106+ new |= mutexLocked
107+ }
108+ if old & (mutexLocked | mutexStarving ) != 0 {
109+ new += 1 << mutexWaiterShift
110+ }
111+ // The current goroutine switches mutex to starvation mode.
112+ // But if the mutex is currently unlocked, don't do the switch.
113+ // Unlock expects that starving mutex has waiters, which will not
114+ // be true in this case.
115+ if starving && old & mutexLocked != 0 {
116+ new |= mutexStarving
74117 }
75118 if awoke {
76119 // The goroutine has been woken from sleep,
77120 // so we need to reset the flag in either case.
78121 if new & mutexWoken == 0 {
79- throw ("sync: inconsistent mutex state" )
122+ panic ("sync: inconsistent mutex state" )
80123 }
81124 new &^= mutexWoken
82125 }
83126 if atomic .CompareAndSwapInt32 (& m .state , old , new ) {
84- if old & mutexLocked == 0 {
127+ if old & (mutexLocked | mutexStarving ) == 0 {
128+ break // locked the mutex with CAS
129+ }
130+ // If we were already waiting before, queue at the front of the queue.
131+ queueLifo := waitStartTime != 0
132+ if waitStartTime == 0 {
133+ waitStartTime = runtime_nanotime ()
134+ }
135+ runtime_SemacquireMutex (& m .sema , queueLifo )
136+ starving = starving || runtime_nanotime ()- waitStartTime > starvationThresholdNs
137+ old = m .state
138+ if old & mutexStarving != 0 {
139+ // If this goroutine was woken and mutex is in starvation mode,
140+ // ownership was handed off to us but mutex is in somewhat
141+ // inconsistent state: mutexLocked is not set and we are still
Has comments. Original line has comments. 142+ // accounted as waiter. Fix that.
143+ if old & (mutexLocked | mutexWoken ) != 0 || old >> mutexWaiterShift == 0 {
144+ panic ("sync: inconsistent mutex state" )
145+ }
146+ delta := int32 (mutexLocked - 1 << mutexWaiterShift )
147+ if ! starving || old >> mutexWaiterShift == 1 {
148+ // Exit starvation mode.
149+ // Critical to do it here and consider wait time.
150+ // Starvation mode is so inefficient, that two goroutines
151+ // can go lock-step infinitely once they switch mutex
152+ // to starvation mode.
153+ delta -= mutexStarving
154+ }
155+ atomic .AddInt32 (& m .state , delta )
85156 break
86157 }
87- runtime_SemacquireMutex (& m .sema )
88158 awoke = true
89159 iter = 0
160+ } else {
161+ old = m .state
90162 }
91163 }
92164
@@ -110,22 +182,33 @@ func (m *Mutex) Unlock() {
110182 // Fast path: drop lock bit.
111183 new := atomic .AddInt32 (& m .state , - mutexLocked )
112184 if (new + mutexLocked )& mutexLocked == 0 {
113- throw ("sync: unlock of unlocked mutex" )
185+ panic ("sync: unlock of unlocked mutex" )
114186 }
115-
116- old := new
117- for {
118- // If there are no waiters or a goroutine has already
119- // been woken or grabbed the lock, no need to wake anyone.
120- if old >> mutexWaiterShift == 0 || old & (mutexLocked | mutexWoken ) != 0 {
121- return
122- }
123- // Grab the right to wake someone.
124- new = (old - 1 << mutexWaiterShift ) | mutexWoken
125- if atomic .CompareAndSwapInt32 (& m .state , old , new ) {
126- runtime_Semrelease (& m .sema )
127- return
187+ if new & mutexStarving == 0 {
188+ old := new
189+ for {
190+ // If there are no waiters or a goroutine has already
191+ // been woken or grabbed the lock, no need to wake anyone.
192+ // In starvation mode ownership is directly handed off from unlocking
193+ // goroutine to the next waiter. We are not part of this chain,
194+ // since we did not observe mutexStarving when we unlocked the mutex above.
195+ // So get off the way.
196+ if old >> mutexWaiterShift == 0 || old & (mutexLocked | mutexWoken | mutexStarving ) != 0 {
197+ return
198+ }
199+ // Grab the right to wake someone.
200+ new = (old - 1 << mutexWaiterShift ) | mutexWoken
201+ if atomic .CompareAndSwapInt32 (& m .state , old , new ) {
202+ runtime_Semrelease (& m .sema , false )
203+ return
204+ }
205+ old = m .state
128206 }
129- old = m .state
207+ } else {
208+ // Starving mode: handoff mutex ownership to the next waiter.
209+ // Note: mutexLocked is not set, the waiter will set it after wakeup.
210+ // But mutex is still considered locked if mutexStarving is set,
211+ // so new coming goroutines won't acquire it.
212+ runtime_Semrelease (& m .sema , true )
130213 }
131214}
