Skip to content

Commit 8e496f1

Browse files
committedNov 5, 2015
runtime: simplify buffered channels.
This change removes the retry mechanism we use for buffered channels. Instead, any sender waking up a receiver or vice versa completes the full protocol with its counterpart. This means the counterpart does not need to relock the channel when it wakes up. (Currently buffered channels need to relock on wakeup.) For sends on a channel with waiting receivers, this change replaces two copies (sender->queue, queue->receiver) with one (sender->receiver). For receives on channels with a waiting sender, two copies are still required. This change unifies to a large degree the algorithm for buffered and unbuffered channels, simplifying the overall implementation. Fixes #11506 benchmark old ns/op new ns/op delta BenchmarkChanProdCons10 125 110 -12.00% BenchmarkChanProdCons0 303 284 -6.27% BenchmarkChanProdCons100 75.5 71.3 -5.56% BenchmarkChanContended 6452 6125 -5.07% BenchmarkChanNonblocking 11.5 11.0 -4.35% BenchmarkChanCreation 149 143 -4.03% BenchmarkChanSem 63.6 61.6 -3.14% BenchmarkChanUncontended 6390 6212 -2.79% BenchmarkChanSync 282 276 -2.13% BenchmarkChanProdConsWork10 516 506 -1.94% BenchmarkChanProdConsWork0 696 685 -1.58% BenchmarkChanProdConsWork100 470 469 -0.21% BenchmarkChanPopular 660427 660012 -0.06% Change-Id: I164113a56432fbc7cace0786e49c5a6e6a708ea4 Reviewed-on: https://go-review.googlesource.com/9345 Run-TryBot: Keith Randall <khr@golang.org> Reviewed-by: Austin Clements <austin@google.com> Reviewed-by: Dmitry Vyukov <dvyukov@google.com>
1 parent 7bb2a7d commit 8e496f1

File tree

2 files changed

+264
-349
lines changed

2 files changed

+264
-349
lines changed
 

‎src/runtime/chan.go

Lines changed: 232 additions & 263 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@ package runtime
66

77
// This file contains the implementation of Go channels.
88

9+
// Invariants:
10+
// At least one of c.sendq and c.recvq is empty.
11+
// For buffered channels, also:
12+
// c.qcount > 0 implies that c.recvq is empty.
13+
// c.qcount < c.dataqsiz implies that c.sendq is empty.
914
import "unsafe"
1015

1116
const (
@@ -153,135 +158,117 @@ func chansend(t *chantype, c *hchan, ep unsafe.Pointer, block bool, callerpc uin
153158
}
154159

155160
lock(&c.lock)
161+
156162
if c.closed != 0 {
157163
unlock(&c.lock)
158164
panic("send on closed channel")
159165
}
160166

161-
if c.dataqsiz == 0 { // synchronous channel
162-
sg := c.recvq.dequeue()
163-
if sg != nil { // found a waiting receiver
164-
if raceenabled {
165-
racesync(c, sg)
166-
}
167-
unlock(&c.lock)
168-
169-
recvg := sg.g
170-
if sg.elem != nil {
171-
syncsend(c, sg, ep)
172-
}
173-
recvg.param = unsafe.Pointer(sg)
174-
if sg.releasetime != 0 {
175-
sg.releasetime = cputicks()
176-
}
177-
goready(recvg, 3)
178-
return true
179-
}
180-
181-
if !block {
182-
unlock(&c.lock)
183-
return false
184-
}
185-
186-
// no receiver available: block on this channel.
187-
gp := getg()
188-
mysg := acquireSudog()
189-
mysg.releasetime = 0
190-
if t0 != 0 {
191-
mysg.releasetime = -1
192-
}
193-
mysg.elem = ep
194-
mysg.waitlink = nil
195-
gp.waiting = mysg
196-
mysg.g = gp
197-
mysg.selectdone = nil
198-
gp.param = nil
199-
c.sendq.enqueue(mysg)
200-
goparkunlock(&c.lock, "chan send", traceEvGoBlockSend, 3)
167+
if sg := c.recvq.dequeue(); sg != nil {
168+
// Found a waiting receiver. We pass the value we want to send
169+
// directly to the receiver, bypassing the channel buffer (if any).
170+
send(c, sg, ep, func() { unlock(&c.lock) })
171+
return true
172+
}
201173

202-
// someone woke us up.
203-
if mysg != gp.waiting {
204-
throw("G waiting list is corrupted!")
205-
}
206-
gp.waiting = nil
207-
if gp.param == nil {
208-
if c.closed == 0 {
209-
throw("chansend: spurious wakeup")
210-
}
211-
panic("send on closed channel")
174+
if c.qcount < c.dataqsiz {
175+
// Space is available in the channel buffer. Enqueue the element to send.
176+
qp := chanbuf(c, c.sendx)
177+
if raceenabled {
178+
raceacquire(qp)
179+
racerelease(qp)
212180
}
213-
gp.param = nil
214-
if mysg.releasetime > 0 {
215-
blockevent(int64(mysg.releasetime)-t0, 2)
181+
typedmemmove(c.elemtype, qp, ep)
182+
c.sendx++
183+
if c.sendx == c.dataqsiz {
184+
c.sendx = 0
216185
}
217-
releaseSudog(mysg)
186+
c.qcount++
187+
unlock(&c.lock)
218188
return true
219189
}
220190

221-
// asynchronous channel
222-
// wait for some space to write our data
223-
var t1 int64
224-
for futile := byte(0); c.qcount >= c.dataqsiz; futile = traceFutileWakeup {
225-
if !block {
226-
unlock(&c.lock)
227-
return false
228-
}
229-
gp := getg()
230-
mysg := acquireSudog()
231-
mysg.releasetime = 0
232-
if t0 != 0 {
233-
mysg.releasetime = -1
234-
}
235-
mysg.g = gp
236-
mysg.elem = nil
237-
mysg.selectdone = nil
238-
c.sendq.enqueue(mysg)
239-
goparkunlock(&c.lock, "chan send", traceEvGoBlockSend|futile, 3)
240-
241-
// someone woke us up - try again
242-
if mysg.releasetime > 0 {
243-
t1 = mysg.releasetime
244-
}
245-
releaseSudog(mysg)
246-
lock(&c.lock)
247-
if c.closed != 0 {
248-
unlock(&c.lock)
249-
panic("send on closed channel")
250-
}
191+
if !block {
192+
unlock(&c.lock)
193+
return false
251194
}
252195

253-
// write our data into the channel buffer
254-
if raceenabled {
255-
raceacquire(chanbuf(c, c.sendx))
256-
racerelease(chanbuf(c, c.sendx))
196+
// Block on the channel. Some receiver will complete our operation for us.
197+
gp := getg()
198+
mysg := acquireSudog()
199+
mysg.releasetime = 0
200+
if t0 != 0 {
201+
mysg.releasetime = -1
202+
}
203+
mysg.elem = ep
204+
mysg.waitlink = nil
205+
mysg.g = gp
206+
mysg.selectdone = nil
207+
gp.waiting = mysg
208+
gp.param = nil
209+
c.sendq.enqueue(mysg)
210+
goparkunlock(&c.lock, "chan send", traceEvGoBlockSend, 3)
211+
212+
// someone woke us up.
213+
if mysg != gp.waiting {
214+
throw("G waiting list is corrupted")
215+
}
216+
gp.waiting = nil
217+
if gp.param == nil {
218+
if c.closed == 0 {
219+
throw("chansend: spurious wakeup")
220+
}
221+
panic("send on closed channel")
257222
}
258-
typedmemmove(c.elemtype, chanbuf(c, c.sendx), ep)
259-
c.sendx++
260-
if c.sendx == c.dataqsiz {
261-
c.sendx = 0
223+
gp.param = nil
224+
if mysg.releasetime > 0 {
225+
blockevent(int64(mysg.releasetime)-t0, 2)
262226
}
263-
c.qcount++
227+
releaseSudog(mysg)
228+
return true
229+
}
264230

265-
// wake up a waiting receiver
266-
sg := c.recvq.dequeue()
267-
if sg != nil {
268-
recvg := sg.g
269-
unlock(&c.lock)
270-
if sg.releasetime != 0 {
271-
sg.releasetime = cputicks()
231+
// send processes a send operation on an empty channel c.
232+
// The value ep sent by the sender is copied to the receiver sg.
233+
// The receiver is then woken up to go on its merry way.
234+
// Channel c must be empty and locked. send unlocks c with unlockf.
235+
// sg must already be dequeued from c.
236+
// ep must be non-nil and point to the heap or the caller's stack.
237+
func send(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func()) {
238+
if raceenabled {
239+
if c.dataqsiz == 0 {
240+
racesync(c, sg)
241+
} else {
242+
// Pretend we go through the buffer, even though
243+
// we copy directly. Note that we need to increment
244+
// the head/tail locations only when raceenabled.
245+
qp := chanbuf(c, c.recvx)
246+
raceacquire(qp)
247+
racerelease(qp)
248+
raceacquireg(sg.g, qp)
249+
racereleaseg(sg.g, qp)
250+
c.recvx++
251+
if c.recvx == c.dataqsiz {
252+
c.recvx = 0
253+
}
254+
c.sendx = c.recvx // c.sendx = (c.sendx+1) % c.dataqsiz
272255
}
273-
goready(recvg, 3)
274-
} else {
275-
unlock(&c.lock)
276256
}
277-
if t1 > 0 {
278-
blockevent(t1-t0, 2)
257+
unlockf()
258+
if sg.elem != nil {
259+
sendDirect(c.elemtype, sg.elem, ep)
260+
sg.elem = nil
279261
}
280-
return true
262+
gp := sg.g
263+
gp.param = unsafe.Pointer(sg)
264+
if sg.releasetime != 0 {
265+
sg.releasetime = cputicks()
266+
}
267+
goready(gp, 4)
281268
}
282269

283-
func syncsend(c *hchan, sg *sudog, elem unsafe.Pointer) {
284-
// Send on unbuffered channel is the only operation
270+
func sendDirect(t *_type, dst, src unsafe.Pointer) {
271+
// Send on an unbuffered or empty-buffered channel is the only operation
285272
// in the entire runtime where one goroutine
286273
// writes to the stack of another goroutine. The GC assumes that
287274
// stack writes only happen when the goroutine is running and are
@@ -290,9 +277,8 @@ func syncsend(c *hchan, sg *sudog, elem unsafe.Pointer) {
290277
// typedmemmove will call heapBitsBulkBarrier, but the target bytes
291278
// are not in the heap, so that will not help. We arrange to call
292279
// memmove and typeBitsBulkBarrier instead.
293-
memmove(sg.elem, elem, c.elemtype.size)
294-
typeBitsBulkBarrier(c.elemtype, uintptr(sg.elem), c.elemtype.size)
295-
sg.elem = nil
280+
memmove(dst, src, t.size)
281+
typeBitsBulkBarrier(t, uintptr(dst), t.size)
296282
}
297283

298284
func closechan(c *hchan) {
@@ -320,27 +306,36 @@ func closechan(c *hchan) {
320306
if sg == nil {
321307
break
322308
}
323-
gp := sg.g
324-
sg.elem = nil
325-
gp.param = nil
309+
if sg.elem != nil {
310+
memclr(sg.elem, uintptr(c.elemsize))
311+
sg.elem = nil
312+
}
326313
if sg.releasetime != 0 {
327314
sg.releasetime = cputicks()
328315
}
316+
gp := sg.g
317+
gp.param = nil
318+
if raceenabled {
319+
raceacquireg(gp, unsafe.Pointer(c))
320+
}
329321
goready(gp, 3)
330322
}
331323

332-
// release all writers
324+
// release all writers (they will panic)
333325
for {
334326
sg := c.sendq.dequeue()
335327
if sg == nil {
336328
break
337329
}
338-
gp := sg.g
339330
sg.elem = nil
340-
gp.param = nil
341331
if sg.releasetime != 0 {
342332
sg.releasetime = cputicks()
343333
}
334+
gp := sg.g
335+
gp.param = nil
336+
if raceenabled {
337+
raceacquireg(gp, unsafe.Pointer(c))
338+
}
344339
goready(gp, 3)
345340
}
346341
unlock(&c.lock)
@@ -363,8 +358,10 @@ func chanrecv2(t *chantype, c *hchan, elem unsafe.Pointer) (received bool) {
363358
// If block == false and no elements are available, returns (false, false).
364359
// Otherwise, if c is closed, zeros *ep and returns (true, false).
365360
// Otherwise, fills in *ep with an element and returns (true, true).
361+
// A non-nil ep must point to the heap or the caller's stack.
366362
func chanrecv(t *chantype, c *hchan, ep unsafe.Pointer, block bool) (selected, received bool) {
367-
// raceenabled: don't need to check ep, as it is always on the stack.
363+
// raceenabled: don't need to check ep, as it is always on the stack
364+
// or is new memory allocated by reflect.
368365

369366
if debugChan {
370367
print("chanrecv: chan=", c, "\n")
@@ -402,167 +399,139 @@ func chanrecv(t *chantype, c *hchan, ep unsafe.Pointer, block bool) (selected, r
402399
}
403400

404401
lock(&c.lock)
405-
if c.dataqsiz == 0 { // synchronous channel
406-
if c.closed != 0 {
407-
return recvclosed(c, ep)
408-
}
409-
410-
sg := c.sendq.dequeue()
411-
if sg != nil {
412-
if raceenabled {
413-
racesync(c, sg)
414-
}
415-
unlock(&c.lock)
416-
417-
if ep != nil {
418-
typedmemmove(c.elemtype, ep, sg.elem)
419-
}
420-
sg.elem = nil
421-
gp := sg.g
422-
gp.param = unsafe.Pointer(sg)
423-
if sg.releasetime != 0 {
424-
sg.releasetime = cputicks()
425-
}
426-
goready(gp, 3)
427-
selected = true
428-
received = true
429-
return
430-
}
431-
432-
if !block {
433-
unlock(&c.lock)
434-
return
435-
}
436402

437-
// no sender available: block on this channel.
438-
gp := getg()
439-
mysg := acquireSudog()
440-
mysg.releasetime = 0
441-
if t0 != 0 {
442-
mysg.releasetime = -1
403+
if c.closed != 0 && c.qcount == 0 {
404+
if raceenabled {
405+
raceacquire(unsafe.Pointer(c))
443406
}
444-
mysg.elem = ep
445-
mysg.waitlink = nil
446-
gp.waiting = mysg
447-
mysg.g = gp
448-
mysg.selectdone = nil
449-
gp.param = nil
450-
c.recvq.enqueue(mysg)
451-
goparkunlock(&c.lock, "chan receive", traceEvGoBlockRecv, 3)
452-
453-
// someone woke us up
454-
if mysg != gp.waiting {
455-
throw("G waiting list is corrupted!")
456-
}
457-
gp.waiting = nil
458-
if mysg.releasetime > 0 {
459-
blockevent(mysg.releasetime-t0, 2)
460-
}
461-
haveData := gp.param != nil
462-
gp.param = nil
463-
releaseSudog(mysg)
464-
465-
if haveData {
466-
// a sender sent us some data. It already wrote to ep.
467-
selected = true
468-
received = true
469-
return
470-
}
471-
472-
lock(&c.lock)
473-
if c.closed == 0 {
474-
throw("chanrecv: spurious wakeup")
407+
unlock(&c.lock)
408+
if ep != nil {
409+
memclr(ep, uintptr(c.elemsize))
475410
}
476-
return recvclosed(c, ep)
411+
return true, false
477412
}
478413

479-
// asynchronous channel
480-
// wait for some data to appear
481-
var t1 int64
482-
for futile := byte(0); c.qcount <= 0; futile = traceFutileWakeup {
483-
if c.closed != 0 {
484-
selected, received = recvclosed(c, ep)
485-
if t1 > 0 {
486-
blockevent(t1-t0, 2)
487-
}
488-
return
489-
}
490-
491-
if !block {
492-
unlock(&c.lock)
493-
return
494-
}
414+
if sg := c.sendq.dequeue(); sg != nil {
415+
// Found a waiting sender. If buffer is size 0, receive value
416+
// directly from sender. Otherwise, recieve from head of queue
417+
// and add sender's value to the tail of the queue (both map to
418+
// the same buffer slot because the queue is full).
419+
recv(c, sg, ep, func() { unlock(&c.lock) })
420+
return true, true
421+
}
495422

496-
// wait for someone to send an element
497-
gp := getg()
498-
mysg := acquireSudog()
499-
mysg.releasetime = 0
500-
if t0 != 0 {
501-
mysg.releasetime = -1
423+
if c.qcount > 0 {
424+
// Receive directly from queue
425+
qp := chanbuf(c, c.recvx)
426+
if raceenabled {
427+
raceacquire(qp)
428+
racerelease(qp)
502429
}
503-
mysg.elem = nil
504-
mysg.g = gp
505-
mysg.selectdone = nil
506-
507-
c.recvq.enqueue(mysg)
508-
goparkunlock(&c.lock, "chan receive", traceEvGoBlockRecv|futile, 3)
509-
510-
// someone woke us up - try again
511-
if mysg.releasetime > 0 {
512-
t1 = mysg.releasetime
430+
if ep != nil {
431+
typedmemmove(c.elemtype, ep, qp)
513432
}
514-
releaseSudog(mysg)
515-
lock(&c.lock)
516-
}
517-
518-
if raceenabled {
519-
raceacquire(chanbuf(c, c.recvx))
520-
racerelease(chanbuf(c, c.recvx))
521-
}
522-
if ep != nil {
523-
typedmemmove(c.elemtype, ep, chanbuf(c, c.recvx))
524-
}
525-
memclr(chanbuf(c, c.recvx), uintptr(c.elemsize))
526-
527-
c.recvx++
528-
if c.recvx == c.dataqsiz {
529-
c.recvx = 0
530-
}
531-
c.qcount--
532-
533-
// ping a sender now that there is space
534-
sg := c.sendq.dequeue()
535-
if sg != nil {
536-
gp := sg.g
537-
unlock(&c.lock)
538-
if sg.releasetime != 0 {
539-
sg.releasetime = cputicks()
433+
memclr(qp, uintptr(c.elemsize))
434+
c.recvx++
435+
if c.recvx == c.dataqsiz {
436+
c.recvx = 0
540437
}
541-
goready(gp, 3)
542-
} else {
438+
c.qcount--
543439
unlock(&c.lock)
440+
return true, true
544441
}
545442

546-
if t1 > 0 {
547-
blockevent(t1-t0, 2)
548-
}
549-
selected = true
550-
received = true
551-
return
443+
if !block {
444+
unlock(&c.lock)
445+
return false, false
446+
}
447+
448+
// no sender available: block on this channel.
449+
gp := getg()
450+
mysg := acquireSudog()
451+
mysg.releasetime = 0
452+
if t0 != 0 {
453+
mysg.releasetime = -1
454+
}
455+
mysg.elem = ep
456+
mysg.waitlink = nil
457+
gp.waiting = mysg
458+
mysg.g = gp
459+
mysg.selectdone = nil
460+
gp.param = nil
461+
c.recvq.enqueue(mysg)
462+
goparkunlock(&c.lock, "chan receive", traceEvGoBlockRecv, 3)
463+
464+
// someone woke us up
465+
if mysg != gp.waiting {
466+
throw("G waiting list is corrupted")
467+
}
468+
gp.waiting = nil
469+
if mysg.releasetime > 0 {
470+
blockevent(mysg.releasetime-t0, 2)
471+
}
472+
closed := gp.param == nil
473+
gp.param = nil
474+
releaseSudog(mysg)
475+
return true, !closed
552476
}
553477

554-
// recvclosed is a helper function for chanrecv. Handles cleanup
555-
// when the receiver encounters a closed channel.
556-
// Caller must hold c.lock, recvclosed will release the lock.
557-
func recvclosed(c *hchan, ep unsafe.Pointer) (selected, recevied bool) {
558-
if raceenabled {
559-
raceacquire(unsafe.Pointer(c))
478+
// recv processes a receive operation on a full channel c.
479+
// There are 2 parts:
480+
// 1) The value sent by the sender sg is put into the channel
481+
// and the sender is woken up to go on its merry way.
482+
// 2) The value received by the receiver (the current G) is
483+
// written to ep.
484+
// For synchronous channels, both values are the same.
485+
// For asynchronous channels, the receiver gets its data from
486+
// the channel buffer and the sender's data is put in the
487+
// channel buffer.
488+
// Channel c must be full and locked. recv unlocks c with unlockf.
489+
// sg must already be dequeued from c.
490+
// A non-nil ep must point to the heap or the caller's stack.
491+
func recv(c *hchan, sg *sudog, ep unsafe.Pointer, unlockf func()) {
492+
if c.dataqsiz == 0 {
493+
if raceenabled {
494+
racesync(c, sg)
495+
}
496+
unlockf()
497+
if ep != nil {
498+
// copy data from sender
499+
// ep points to our own stack or heap, so nothing
500+
// special (ala sendDirect) needed here.
501+
typedmemmove(c.elemtype, ep, sg.elem)
502+
}
503+
} else {
504+
// Queue is full. Take the item at the
505+
// head of the queue. Make the sender enqueue
506+
// its item at the tail of the queue. Since the
507+
// queue is full, those are both the same slot.
508+
qp := chanbuf(c, c.recvx)
509+
if raceenabled {
510+
raceacquire(qp)
511+
racerelease(qp)
512+
raceacquireg(sg.g, qp)
513+
racereleaseg(sg.g, qp)
514+
}
515+
// copy data from queue to receiver
516+
if ep != nil {
517+
typedmemmove(c.elemtype, ep, qp)
518+
}
519+
// copy data from sender to queue
520+
typedmemmove(c.elemtype, qp, sg.elem)
521+
c.recvx++
522+
if c.recvx == c.dataqsiz {
523+
c.recvx = 0
524+
}
525+
c.sendx = c.recvx // c.sendx = (c.sendx+1) % c.dataqsiz
526+
unlockf()
560527
}
561-
unlock(&c.lock)
562-
if ep != nil {
563-
memclr(ep, uintptr(c.elemsize))
528+
sg.elem = nil
529+
gp := sg.g
530+
gp.param = unsafe.Pointer(sg)
531+
if sg.releasetime != 0 {
532+
sg.releasetime = cputicks()
564533
}
565-
return true, false
534+
goready(gp, 4)
566535
}
567536

568537
// compiler implements

‎src/runtime/select.go

Lines changed: 32 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -304,7 +304,7 @@ func selectgoImpl(sel *hselect) (uintptr, uint16) {
304304
k *scase
305305
sglist *sudog
306306
sgnext *sudog
307-
futile byte
307+
qp unsafe.Pointer
308308
)
309309

310310
loop:
@@ -317,15 +317,12 @@ loop:
317317

318318
switch cas.kind {
319319
case caseRecv:
320-
if c.dataqsiz > 0 {
321-
if c.qcount > 0 {
322-
goto asyncrecv
323-
}
324-
} else {
325-
sg = c.sendq.dequeue()
326-
if sg != nil {
327-
goto syncrecv
328-
}
320+
sg = c.sendq.dequeue()
321+
if sg != nil {
322+
goto recv
323+
}
324+
if c.qcount > 0 {
325+
goto bufrecv
329326
}
330327
if c.closed != 0 {
331328
goto rclose
@@ -338,15 +335,12 @@ loop:
338335
if c.closed != 0 {
339336
goto sclose
340337
}
341-
if c.dataqsiz > 0 {
342-
if c.qcount < c.dataqsiz {
343-
goto asyncsend
344-
}
345-
} else {
346-
sg = c.recvq.dequeue()
347-
if sg != nil {
348-
goto syncsend
349-
}
338+
sg = c.recvq.dequeue()
339+
if sg != nil {
340+
goto send
341+
}
342+
if c.qcount < c.dataqsiz {
343+
goto bufsend
350344
}
351345

352346
case caseDefault:
@@ -363,6 +357,9 @@ loop:
363357
// pass 2 - enqueue on all chans
364358
gp = getg()
365359
done = 0
360+
if gp.waiting != nil {
361+
throw("gp.waiting != nil")
362+
}
366363
for i := 0; i < int(sel.ncase); i++ {
367364
cas = &scases[pollorder[i]]
368365
c = cas.c
@@ -389,7 +386,7 @@ loop:
389386

390387
// wait for someone to wake us up
391388
gp.param = nil
392-
gopark(selparkcommit, unsafe.Pointer(sel), "select", traceEvGoBlockSelect|futile, 2)
389+
gopark(selparkcommit, unsafe.Pointer(sel), "select", traceEvGoBlockSelect, 2)
393390

394391
// someone woke us up
395392
sellock(sel)
@@ -432,16 +429,13 @@ loop:
432429
}
433430

434431
if cas == nil {
435-
futile = traceFutileWakeup
432+
// This can happen if we were woken up by a close().
433+
// TODO: figure that out explicitly so we don't need this loop.
436434
goto loop
437435
}
438436

439437
c = cas.c
440438

441-
if c.dataqsiz > 0 {
442-
throw("selectgo: shouldn't happen")
443-
}
444-
445439
if debugSelect {
446440
print("wait-return: sel=", sel, " c=", c, " cas=", cas, " kind=", cas.kind, "\n")
447441
}
@@ -470,7 +464,7 @@ loop:
470464
selunlock(sel)
471465
goto retc
472466

473-
asyncrecv:
467+
bufrecv:
474468
// can receive from buffer
475469
if raceenabled {
476470
if cas.elem != nil {
@@ -485,29 +479,20 @@ asyncrecv:
485479
if cas.receivedp != nil {
486480
*cas.receivedp = true
487481
}
482+
qp = chanbuf(c, c.recvx)
488483
if cas.elem != nil {
489-
typedmemmove(c.elemtype, cas.elem, chanbuf(c, c.recvx))
484+
typedmemmove(c.elemtype, cas.elem, qp)
490485
}
491-
memclr(chanbuf(c, c.recvx), uintptr(c.elemsize))
486+
memclr(qp, uintptr(c.elemsize))
492487
c.recvx++
493488
if c.recvx == c.dataqsiz {
494489
c.recvx = 0
495490
}
496491
c.qcount--
497-
sg = c.sendq.dequeue()
498-
if sg != nil {
499-
gp = sg.g
500-
selunlock(sel)
501-
if sg.releasetime != 0 {
502-
sg.releasetime = cputicks()
503-
}
504-
goready(gp, 3)
505-
} else {
506-
selunlock(sel)
507-
}
492+
selunlock(sel)
508493
goto retc
509494

510-
asyncsend:
495+
bufsend:
511496
// can send to buffer
512497
if raceenabled {
513498
raceacquire(chanbuf(c, c.sendx))
@@ -523,47 +508,18 @@ asyncsend:
523508
c.sendx = 0
524509
}
525510
c.qcount++
526-
sg = c.recvq.dequeue()
527-
if sg != nil {
528-
gp = sg.g
529-
selunlock(sel)
530-
if sg.releasetime != 0 {
531-
sg.releasetime = cputicks()
532-
}
533-
goready(gp, 3)
534-
} else {
535-
selunlock(sel)
536-
}
511+
selunlock(sel)
537512
goto retc
538513

539-
syncrecv:
514+
recv:
540515
// can receive from sleeping sender (sg)
541-
if raceenabled {
542-
if cas.elem != nil {
543-
raceWriteObjectPC(c.elemtype, cas.elem, cas.pc, chanrecvpc)
544-
}
545-
racesync(c, sg)
546-
}
547-
if msanenabled && cas.elem != nil {
548-
msanwrite(cas.elem, c.elemtype.size)
549-
}
550-
selunlock(sel)
516+
recv(c, sg, cas.elem, func() { selunlock(sel) })
551517
if debugSelect {
552518
print("syncrecv: sel=", sel, " c=", c, "\n")
553519
}
554520
if cas.receivedp != nil {
555521
*cas.receivedp = true
556522
}
557-
if cas.elem != nil {
558-
typedmemmove(c.elemtype, cas.elem, sg.elem)
559-
}
560-
sg.elem = nil
561-
gp = sg.g
562-
gp.param = unsafe.Pointer(sg)
563-
if sg.releasetime != 0 {
564-
sg.releasetime = cputicks()
565-
}
566-
goready(gp, 3)
567523
goto retc
568524

569525
rclose:
@@ -580,29 +536,19 @@ rclose:
580536
}
581537
goto retc
582538

583-
syncsend:
584-
// can send to sleeping receiver (sg)
539+
send:
540+
// can send to a sleeping receiver (sg)
585541
if raceenabled {
586542
raceReadObjectPC(c.elemtype, cas.elem, cas.pc, chansendpc)
587-
racesync(c, sg)
588543
}
589544
if msanenabled {
590545
msanread(cas.elem, c.elemtype.size)
591546
}
592-
selunlock(sel)
547+
send(c, sg, cas.elem, func() { selunlock(sel) })
593548
if debugSelect {
594549
print("syncsend: sel=", sel, " c=", c, "\n")
595550
}
596-
if sg.elem != nil {
597-
syncsend(c, sg, cas.elem)
598-
}
599-
sg.elem = nil
600-
gp = sg.g
601-
gp.param = unsafe.Pointer(sg)
602-
if sg.releasetime != 0 {
603-
sg.releasetime = cputicks()
604-
}
605-
goready(gp, 3)
551+
goto retc
606552

607553
retc:
608554
if cas.releasetime > 0 {

0 commit comments

Comments
 (0)
Please sign in to comment.