@@ -6,6 +6,11 @@ package runtime
6
6
7
7
// This file contains the implementation of Go channels.
8
8
9
+ // Invariants:
10
+ // At least one of c.sendq and c.recvq is empty.
11
+ // For buffered channels, also:
12
+ // c.qcount > 0 implies that c.recvq is empty.
13
+ // c.qcount < c.dataqsiz implies that c.sendq is empty.
9
14
import "unsafe"
10
15
11
16
const (
@@ -153,135 +158,117 @@ func chansend(t *chantype, c *hchan, ep unsafe.Pointer, block bool, callerpc uin
153
158
}
154
159
155
160
lock (& c .lock )
161
+
156
162
if c .closed != 0 {
157
163
unlock (& c .lock )
158
164
panic ("send on closed channel" )
159
165
}
160
166
161
- if c .dataqsiz == 0 { // synchronous channel
162
- sg := c .recvq .dequeue ()
163
- if sg != nil { // found a waiting receiver
164
- if raceenabled {
165
- racesync (c , sg )
166
- }
167
- unlock (& c .lock )
168
-
169
- recvg := sg .g
170
- if sg .elem != nil {
171
- syncsend (c , sg , ep )
172
- }
173
- recvg .param = unsafe .Pointer (sg )
174
- if sg .releasetime != 0 {
175
- sg .releasetime = cputicks ()
176
- }
177
- goready (recvg , 3 )
178
- return true
179
- }
180
-
181
- if ! block {
182
- unlock (& c .lock )
183
- return false
184
- }
185
-
186
- // no receiver available: block on this channel.
187
- gp := getg ()
188
- mysg := acquireSudog ()
189
- mysg .releasetime = 0
190
- if t0 != 0 {
191
- mysg .releasetime = - 1
192
- }
193
- mysg .elem = ep
194
- mysg .waitlink = nil
195
- gp .waiting = mysg
196
- mysg .g = gp
197
- mysg .selectdone = nil
198
- gp .param = nil
199
- c .sendq .enqueue (mysg )
200
- goparkunlock (& c .lock , "chan send" , traceEvGoBlockSend , 3 )
167
+ if sg := c .recvq .dequeue (); sg != nil {
168
+ // Found a waiting receiver. We pass the value we want to send
169
+ // directly to the receiver, bypassing the channel buffer (if any).
170
+ send (c , sg , ep , func () { unlock (& c .lock ) })
171
+ return true
172
+ }
201
173
202
- // someone woke us up.
203
- if mysg != gp .waiting {
204
- throw ("G waiting list is corrupted!" )
205
- }
206
- gp .waiting = nil
207
- if gp .param == nil {
208
- if c .closed == 0 {
209
- throw ("chansend: spurious wakeup" )
210
- }
211
- panic ("send on closed channel" )
174
+ if c .qcount < c .dataqsiz {
175
+ // Space is available in the channel buffer. Enqueue the element to send.
176
+ qp := chanbuf (c , c .sendx )
177
+ if raceenabled {
178
+ raceacquire (qp )
179
+ racerelease (qp )
212
180
}
213
- gp .param = nil
214
- if mysg .releasetime > 0 {
215
- blockevent (int64 (mysg .releasetime )- t0 , 2 )
181
+ typedmemmove (c .elemtype , qp , ep )
182
+ c .sendx ++
183
+ if c .sendx == c .dataqsiz {
184
+ c .sendx = 0
216
185
}
217
- releaseSudog (mysg )
186
+ c .qcount ++
187
+ unlock (& c .lock )
218
188
return true
219
189
}
220
190
221
- // asynchronous channel
222
- // wait for some space to write our data
223
- var t1 int64
224
- for futile := byte (0 ); c .qcount >= c .dataqsiz ; futile = traceFutileWakeup {
225
- if ! block {
226
- unlock (& c .lock )
227
- return false
228
- }
229
- gp := getg ()
230
- mysg := acquireSudog ()
231
- mysg .releasetime = 0
232
- if t0 != 0 {
233
- mysg .releasetime = - 1
234
- }
235
- mysg .g = gp
236
- mysg .elem = nil
237
- mysg .selectdone = nil
238
- c .sendq .enqueue (mysg )
239
- goparkunlock (& c .lock , "chan send" , traceEvGoBlockSend | futile , 3 )
240
-
241
- // someone woke us up - try again
242
- if mysg .releasetime > 0 {
243
- t1 = mysg .releasetime
244
- }
245
- releaseSudog (mysg )
246
- lock (& c .lock )
247
- if c .closed != 0 {
248
- unlock (& c .lock )
249
- panic ("send on closed channel" )
250
- }
191
+ if ! block {
192
+ unlock (& c .lock )
193
+ return false
251
194
}
252
195
253
- // write our data into the channel buffer
254
- if raceenabled {
255
- raceacquire (chanbuf (c , c .sendx ))
256
- racerelease (chanbuf (c , c .sendx ))
196
+ // Block on the channel. Some receiver will complete our operation for us.
197
+ gp := getg ()
198
+ mysg := acquireSudog ()
199
+ mysg .releasetime = 0
200
+ if t0 != 0 {
201
+ mysg .releasetime = - 1
202
+ }
203
+ mysg .elem = ep
204
+ mysg .waitlink = nil
205
+ mysg .g = gp
206
+ mysg .selectdone = nil
207
+ gp .waiting = mysg
208
+ gp .param = nil
209
+ c .sendq .enqueue (mysg )
210
+ goparkunlock (& c .lock , "chan send" , traceEvGoBlockSend , 3 )
211
+
212
+ // someone woke us up.
213
+ if mysg != gp .waiting {
214
+ throw ("G waiting list is corrupted" )
215
+ }
216
+ gp .waiting = nil
217
+ if gp .param == nil {
218
+ if c .closed == 0 {
219
+ throw ("chansend: spurious wakeup" )
220
+ }
221
+ panic ("send on closed channel" )
257
222
}
258
- typedmemmove (c .elemtype , chanbuf (c , c .sendx ), ep )
259
- c .sendx ++
260
- if c .sendx == c .dataqsiz {
261
- c .sendx = 0
223
+ gp .param = nil
224
+ if mysg .releasetime > 0 {
225
+ blockevent (int64 (mysg .releasetime )- t0 , 2 )
262
226
}
263
- c .qcount ++
227
+ releaseSudog (mysg )
228
+ return true
229
+ }
264
230
265
- // wake up a waiting receiver
266
- sg := c .recvq .dequeue ()
267
- if sg != nil {
268
- recvg := sg .g
269
- unlock (& c .lock )
270
- if sg .releasetime != 0 {
271
- sg .releasetime = cputicks ()
231
+ // send processes a send operation on an empty channel c.
232
+ // The value ep sent by the sender is copied to the receiver sg.
233
+ // The receiver is then woken up to go on its merry way.
234
+ // Channel c must be empty and locked. send unlocks c with unlockf.
235
+ // sg must already be dequeued from c.
236
+ // ep must be non-nil and point to the heap or the caller's stack.
237
+ func send (c * hchan , sg * sudog , ep unsafe.Pointer , unlockf func ()) {
238
+ if raceenabled {
239
+ if c .dataqsiz == 0 {
240
+ racesync (c , sg )
241
+ } else {
242
+ // Pretend we go through the buffer, even though
243
+ // we copy directly. Note that we need to increment
244
+ // the head/tail locations only when raceenabled.
245
+ qp := chanbuf (c , c .recvx )
246
+ raceacquire (qp )
247
+ racerelease (qp )
248
+ raceacquireg (sg .g , qp )
249
+ racereleaseg (sg .g , qp )
250
+ c .recvx ++
251
+ if c .recvx == c .dataqsiz {
252
+ c .recvx = 0
253
+ }
254
+ c .sendx = c .recvx // c.sendx = (c.sendx+1) % c.dataqsiz
272
255
}
273
- goready (recvg , 3 )
274
- } else {
275
- unlock (& c .lock )
276
256
}
277
- if t1 > 0 {
278
- blockevent (t1 - t0 , 2 )
257
+ unlockf ()
258
+ if sg .elem != nil {
259
+ sendDirect (c .elemtype , sg .elem , ep )
260
+ sg .elem = nil
279
261
}
280
- return true
262
+ gp := sg .g
263
+ gp .param = unsafe .Pointer (sg )
264
+ if sg .releasetime != 0 {
265
+ sg .releasetime = cputicks ()
266
+ }
267
+ goready (gp , 4 )
281
268
}
282
269
283
- func syncsend ( c * hchan , sg * sudog , elem unsafe.Pointer ) {
284
- // Send on unbuffered channel is the only operation
270
+ func sendDirect ( t * _type , dst , src unsafe.Pointer ) {
271
+ // Send on an unbuffered or empty-buffered channel is the only operation
285
272
// in the entire runtime where one goroutine
286
273
// writes to the stack of another goroutine. The GC assumes that
287
274
// stack writes only happen when the goroutine is running and are
@@ -290,9 +277,8 @@ func syncsend(c *hchan, sg *sudog, elem unsafe.Pointer) {
290
277
// typedmemmove will call heapBitsBulkBarrier, but the target bytes
291
278
// are not in the heap, so that will not help. We arrange to call
292
279
// memmove and typeBitsBulkBarrier instead.
293
- memmove (sg .elem , elem , c .elemtype .size )
294
- typeBitsBulkBarrier (c .elemtype , uintptr (sg .elem ), c .elemtype .size )
295
- sg .elem = nil
280
+ memmove (dst , src , t .size )
281
+ typeBitsBulkBarrier (t , uintptr (dst ), t .size )
296
282
}
297
283
298
284
func closechan (c * hchan ) {
@@ -320,27 +306,36 @@ func closechan(c *hchan) {
320
306
if sg == nil {
321
307
break
322
308
}
323
- gp := sg .g
324
- sg .elem = nil
325
- gp .param = nil
309
+ if sg .elem != nil {
310
+ memclr (sg .elem , uintptr (c .elemsize ))
311
+ sg .elem = nil
312
+ }
326
313
if sg .releasetime != 0 {
327
314
sg .releasetime = cputicks ()
328
315
}
316
+ gp := sg .g
317
+ gp .param = nil
318
+ if raceenabled {
319
+ raceacquireg (gp , unsafe .Pointer (c ))
320
+ }
329
321
goready (gp , 3 )
330
322
}
331
323
332
- // release all writers
324
+ // release all writers (they will panic)
333
325
for {
334
326
sg := c .sendq .dequeue ()
335
327
if sg == nil {
336
328
break
337
329
}
338
- gp := sg .g
339
330
sg .elem = nil
340
- gp .param = nil
341
331
if sg .releasetime != 0 {
342
332
sg .releasetime = cputicks ()
343
333
}
334
+ gp := sg .g
335
+ gp .param = nil
336
+ if raceenabled {
337
+ raceacquireg (gp , unsafe .Pointer (c ))
338
+ }
344
339
goready (gp , 3 )
345
340
}
346
341
unlock (& c .lock )
@@ -363,8 +358,10 @@ func chanrecv2(t *chantype, c *hchan, elem unsafe.Pointer) (received bool) {
363
358
// If block == false and no elements are available, returns (false, false).
364
359
// Otherwise, if c is closed, zeros *ep and returns (true, false).
365
360
// Otherwise, fills in *ep with an element and returns (true, true).
361
+ // A non-nil ep must point to the heap or the caller's stack.
366
362
func chanrecv (t * chantype , c * hchan , ep unsafe.Pointer , block bool ) (selected , received bool ) {
367
- // raceenabled: don't need to check ep, as it is always on the stack.
363
+ // raceenabled: don't need to check ep, as it is always on the stack
364
+ // or is new memory allocated by reflect.
368
365
369
366
if debugChan {
370
367
print ("chanrecv: chan=" , c , "\n " )
@@ -402,167 +399,139 @@ func chanrecv(t *chantype, c *hchan, ep unsafe.Pointer, block bool) (selected, r
402
399
}
403
400
404
401
lock (& c .lock )
405
- if c .dataqsiz == 0 { // synchronous channel
406
- if c .closed != 0 {
407
- return recvclosed (c , ep )
408
- }
409
-
410
- sg := c .sendq .dequeue ()
411
- if sg != nil {
412
- if raceenabled {
413
- racesync (c , sg )
414
- }
415
- unlock (& c .lock )
416
-
417
- if ep != nil {
418
- typedmemmove (c .elemtype , ep , sg .elem )
419
- }
420
- sg .elem = nil
421
- gp := sg .g
422
- gp .param = unsafe .Pointer (sg )
423
- if sg .releasetime != 0 {
424
- sg .releasetime = cputicks ()
425
- }
426
- goready (gp , 3 )
427
- selected = true
428
- received = true
429
- return
430
- }
431
-
432
- if ! block {
433
- unlock (& c .lock )
434
- return
435
- }
436
402
437
- // no sender available: block on this channel.
438
- gp := getg ()
439
- mysg := acquireSudog ()
440
- mysg .releasetime = 0
441
- if t0 != 0 {
442
- mysg .releasetime = - 1
403
+ if c .closed != 0 && c .qcount == 0 {
404
+ if raceenabled {
405
+ raceacquire (unsafe .Pointer (c ))
443
406
}
444
- mysg .elem = ep
445
- mysg .waitlink = nil
446
- gp .waiting = mysg
447
- mysg .g = gp
448
- mysg .selectdone = nil
449
- gp .param = nil
450
- c .recvq .enqueue (mysg )
451
- goparkunlock (& c .lock , "chan receive" , traceEvGoBlockRecv , 3 )
452
-
453
- // someone woke us up
454
- if mysg != gp .waiting {
455
- throw ("G waiting list is corrupted!" )
456
- }
457
- gp .waiting = nil
458
- if mysg .releasetime > 0 {
459
- blockevent (mysg .releasetime - t0 , 2 )
460
- }
461
- haveData := gp .param != nil
462
- gp .param = nil
463
- releaseSudog (mysg )
464
-
465
- if haveData {
466
- // a sender sent us some data. It already wrote to ep.
467
- selected = true
468
- received = true
469
- return
470
- }
471
-
472
- lock (& c .lock )
473
- if c .closed == 0 {
474
- throw ("chanrecv: spurious wakeup" )
407
+ unlock (& c .lock )
408
+ if ep != nil {
409
+ memclr (ep , uintptr (c .elemsize ))
475
410
}
476
- return recvclosed ( c , ep )
411
+ return true , false
477
412
}
478
413
479
- // asynchronous channel
480
- // wait for some data to appear
481
- var t1 int64
482
- for futile := byte (0 ); c .qcount <= 0 ; futile = traceFutileWakeup {
483
- if c .closed != 0 {
484
- selected , received = recvclosed (c , ep )
485
- if t1 > 0 {
486
- blockevent (t1 - t0 , 2 )
487
- }
488
- return
489
- }
490
-
491
- if ! block {
492
- unlock (& c .lock )
493
- return
494
- }
414
+ if sg := c .sendq .dequeue (); sg != nil {
415
+ // Found a waiting sender. If buffer is size 0, receive value
416
+ // directly from sender. Otherwise, recieve from head of queue
417
+ // and add sender's value to the tail of the queue (both map to
418
+ // the same buffer slot because the queue is full).
419
+ recv (c , sg , ep , func () { unlock (& c .lock ) })
420
+ return true , true
421
+ }
495
422
496
- // wait for someone to send an element
497
- gp := getg ()
498
- mysg := acquireSudog ( )
499
- mysg . releasetime = 0
500
- if t0 != 0 {
501
- mysg . releasetime = - 1
423
+ if c . qcount > 0 {
424
+ // Receive directly from queue
425
+ qp := chanbuf ( c , c . recvx )
426
+ if raceenabled {
427
+ raceacquire ( qp )
428
+ racerelease ( qp )
502
429
}
503
- mysg .elem = nil
504
- mysg .g = gp
505
- mysg .selectdone = nil
506
-
507
- c .recvq .enqueue (mysg )
508
- goparkunlock (& c .lock , "chan receive" , traceEvGoBlockRecv | futile , 3 )
509
-
510
- // someone woke us up - try again
511
- if mysg .releasetime > 0 {
512
- t1 = mysg .releasetime
430
+ if ep != nil {
431
+ typedmemmove (c .elemtype , ep , qp )
513
432
}
514
- releaseSudog (mysg )
515
- lock (& c .lock )
516
- }
517
-
518
- if raceenabled {
519
- raceacquire (chanbuf (c , c .recvx ))
520
- racerelease (chanbuf (c , c .recvx ))
521
- }
522
- if ep != nil {
523
- typedmemmove (c .elemtype , ep , chanbuf (c , c .recvx ))
524
- }
525
- memclr (chanbuf (c , c .recvx ), uintptr (c .elemsize ))
526
-
527
- c .recvx ++
528
- if c .recvx == c .dataqsiz {
529
- c .recvx = 0
530
- }
531
- c .qcount --
532
-
533
- // ping a sender now that there is space
534
- sg := c .sendq .dequeue ()
535
- if sg != nil {
536
- gp := sg .g
537
- unlock (& c .lock )
538
- if sg .releasetime != 0 {
539
- sg .releasetime = cputicks ()
433
+ memclr (qp , uintptr (c .elemsize ))
434
+ c .recvx ++
435
+ if c .recvx == c .dataqsiz {
436
+ c .recvx = 0
540
437
}
541
- goready (gp , 3 )
542
- } else {
438
+ c .qcount --
543
439
unlock (& c .lock )
440
+ return true , true
544
441
}
545
442
546
- if t1 > 0 {
547
- blockevent (t1 - t0 , 2 )
548
- }
549
- selected = true
550
- received = true
551
- return
443
+ if ! block {
444
+ unlock (& c .lock )
445
+ return false , false
446
+ }
447
+
448
+ // no sender available: block on this channel.
449
+ gp := getg ()
450
+ mysg := acquireSudog ()
451
+ mysg .releasetime = 0
452
+ if t0 != 0 {
453
+ mysg .releasetime = - 1
454
+ }
455
+ mysg .elem = ep
456
+ mysg .waitlink = nil
457
+ gp .waiting = mysg
458
+ mysg .g = gp
459
+ mysg .selectdone = nil
460
+ gp .param = nil
461
+ c .recvq .enqueue (mysg )
462
+ goparkunlock (& c .lock , "chan receive" , traceEvGoBlockRecv , 3 )
463
+
464
+ // someone woke us up
465
+ if mysg != gp .waiting {
466
+ throw ("G waiting list is corrupted" )
467
+ }
468
+ gp .waiting = nil
469
+ if mysg .releasetime > 0 {
470
+ blockevent (mysg .releasetime - t0 , 2 )
471
+ }
472
+ closed := gp .param == nil
473
+ gp .param = nil
474
+ releaseSudog (mysg )
475
+ return true , ! closed
552
476
}
553
477
554
- // recvclosed is a helper function for chanrecv. Handles cleanup
555
- // when the receiver encounters a closed channel.
556
- // Caller must hold c.lock, recvclosed will release the lock.
557
- func recvclosed (c * hchan , ep unsafe.Pointer ) (selected , recevied bool ) {
558
- if raceenabled {
559
- raceacquire (unsafe .Pointer (c ))
478
+ // recv processes a receive operation on a full channel c.
479
+ // There are 2 parts:
480
+ // 1) The value sent by the sender sg is put into the channel
481
+ // and the sender is woken up to go on its merry way.
482
+ // 2) The value received by the receiver (the current G) is
483
+ // written to ep.
484
+ // For synchronous channels, both values are the same.
485
+ // For asynchronous channels, the receiver gets its data from
486
+ // the channel buffer and the sender's data is put in the
487
+ // channel buffer.
488
+ // Channel c must be full and locked. recv unlocks c with unlockf.
489
+ // sg must already be dequeued from c.
490
+ // A non-nil ep must point to the heap or the caller's stack.
491
+ func recv (c * hchan , sg * sudog , ep unsafe.Pointer , unlockf func ()) {
492
+ if c .dataqsiz == 0 {
493
+ if raceenabled {
494
+ racesync (c , sg )
495
+ }
496
+ unlockf ()
497
+ if ep != nil {
498
+ // copy data from sender
499
+ // ep points to our own stack or heap, so nothing
500
+ // special (ala sendDirect) needed here.
501
+ typedmemmove (c .elemtype , ep , sg .elem )
502
+ }
503
+ } else {
504
+ // Queue is full. Take the item at the
505
+ // head of the queue. Make the sender enqueue
506
+ // its item at the tail of the queue. Since the
507
+ // queue is full, those are both the same slot.
508
+ qp := chanbuf (c , c .recvx )
509
+ if raceenabled {
510
+ raceacquire (qp )
511
+ racerelease (qp )
512
+ raceacquireg (sg .g , qp )
513
+ racereleaseg (sg .g , qp )
514
+ }
515
+ // copy data from queue to receiver
516
+ if ep != nil {
517
+ typedmemmove (c .elemtype , ep , qp )
518
+ }
519
+ // copy data from sender to queue
520
+ typedmemmove (c .elemtype , qp , sg .elem )
521
+ c .recvx ++
522
+ if c .recvx == c .dataqsiz {
523
+ c .recvx = 0
524
+ }
525
+ c .sendx = c .recvx // c.sendx = (c.sendx+1) % c.dataqsiz
526
+ unlockf ()
560
527
}
561
- unlock (& c .lock )
562
- if ep != nil {
563
- memclr (ep , uintptr (c .elemsize ))
528
+ sg .elem = nil
529
+ gp := sg .g
530
+ gp .param = unsafe .Pointer (sg )
531
+ if sg .releasetime != 0 {
532
+ sg .releasetime = cputicks ()
564
533
}
565
- return true , false
534
+ goready ( gp , 4 )
566
535
}
567
536
568
537
// compiler implements
0 commit comments