Source file
src/runtime/mgcmark.go
Documentation: runtime
1
2
3
4
5
6
7 package runtime
8
9 import (
10 "runtime/internal/atomic"
11 "runtime/internal/sys"
12 "unsafe"
13 )
14
15 const (
16 fixedRootFinalizers = iota
17 fixedRootFreeGStacks
18 fixedRootCount
19
20
21
22 rootBlockBytes = 256 << 10
23
24
25
26
27
28
29
30
31 maxObletBytes = 128 << 10
32
33
34
35
36
37
38
39 drainCheckThreshold = 100000
40
41
42
43
44
45
46
47
48
49 pagesPerSpanRoot = 512
50 )
51
52
53
54
55
56 func gcMarkRootPrepare() {
57 assertWorldStopped()
58
59 work.nFlushCacheRoots = 0
60
61
62 nBlocks := func(bytes uintptr) int {
63 return int(divRoundUp(bytes, rootBlockBytes))
64 }
65
66 work.nDataRoots = 0
67 work.nBSSRoots = 0
68
69
70 for _, datap := range activeModules() {
71 nDataRoots := nBlocks(datap.edata - datap.data)
72 if nDataRoots > work.nDataRoots {
73 work.nDataRoots = nDataRoots
74 }
75 }
76
77 for _, datap := range activeModules() {
78 nBSSRoots := nBlocks(datap.ebss - datap.bss)
79 if nBSSRoots > work.nBSSRoots {
80 work.nBSSRoots = nBSSRoots
81 }
82 }
83
84
85
86
87
88
89
90
91
92
93
94
95
96 mheap_.markArenas = mheap_.allArenas[:len(mheap_.allArenas):len(mheap_.allArenas)]
97 work.nSpanRoots = len(mheap_.markArenas) * (pagesPerArena / pagesPerSpanRoot)
98
99
100
101
102
103
104
105 work.nStackRoots = int(atomic.Loaduintptr(&allglen))
106
107 work.markrootNext = 0
108 work.markrootJobs = uint32(fixedRootCount + work.nFlushCacheRoots + work.nDataRoots + work.nBSSRoots + work.nSpanRoots + work.nStackRoots)
109 }
110
111
112
113 func gcMarkRootCheck() {
114 if work.markrootNext < work.markrootJobs {
115 print(work.markrootNext, " of ", work.markrootJobs, " markroot jobs done\n")
116 throw("left over markroot jobs")
117 }
118
119 lock(&allglock)
120
121 var gp *g
122 for i := 0; i < work.nStackRoots; i++ {
123 gp = allgs[i]
124 if !gp.gcscandone {
125 goto fail
126 }
127 }
128 unlock(&allglock)
129 return
130
131 fail:
132 println("gp", gp, "goid", gp.goid,
133 "status", readgstatus(gp),
134 "gcscandone", gp.gcscandone)
135 throw("scan missed a g")
136 }
137
138
139 var oneptrmask = [...]uint8{1}
140
141
142
143
144
145
146
147
148 func markroot(gcw *gcWork, i uint32) {
149
150
151 baseFlushCache := uint32(fixedRootCount)
152 baseData := baseFlushCache + uint32(work.nFlushCacheRoots)
153 baseBSS := baseData + uint32(work.nDataRoots)
154 baseSpans := baseBSS + uint32(work.nBSSRoots)
155 baseStacks := baseSpans + uint32(work.nSpanRoots)
156 end := baseStacks + uint32(work.nStackRoots)
157
158
159 switch {
160 case baseFlushCache <= i && i < baseData:
161 flushmcache(int(i - baseFlushCache))
162
163 case baseData <= i && i < baseBSS:
164 for _, datap := range activeModules() {
165 markrootBlock(datap.data, datap.edata-datap.data, datap.gcdatamask.bytedata, gcw, int(i-baseData))
166 }
167
168 case baseBSS <= i && i < baseSpans:
169 for _, datap := range activeModules() {
170 markrootBlock(datap.bss, datap.ebss-datap.bss, datap.gcbssmask.bytedata, gcw, int(i-baseBSS))
171 }
172
173 case i == fixedRootFinalizers:
174 for fb := allfin; fb != nil; fb = fb.alllink {
175 cnt := uintptr(atomic.Load(&fb.cnt))
176 scanblock(uintptr(unsafe.Pointer(&fb.fin[0])), cnt*unsafe.Sizeof(fb.fin[0]), &finptrmask[0], gcw, nil)
177 }
178
179 case i == fixedRootFreeGStacks:
180
181
182 systemstack(markrootFreeGStacks)
183
184 case baseSpans <= i && i < baseStacks:
185
186 markrootSpans(gcw, int(i-baseSpans))
187
188 default:
189
190 var gp *g
191 if baseStacks <= i && i < end {
192 gp = allgs[i-baseStacks]
193 } else {
194 throw("markroot: bad index")
195 }
196
197
198
199 status := readgstatus(gp)
200 if (status == _Gwaiting || status == _Gsyscall) && gp.waitsince == 0 {
201 gp.waitsince = work.tstart
202 }
203
204
205
206 systemstack(func() {
207
208
209
210
211 userG := getg().m.curg
212 selfScan := gp == userG && readgstatus(userG) == _Grunning
213 if selfScan {
214 casgstatus(userG, _Grunning, _Gwaiting)
215 userG.waitreason = waitReasonGarbageCollectionScan
216 }
217
218
219
220
221
222
223
224
225 stopped := suspendG(gp)
226 if stopped.dead {
227 gp.gcscandone = true
228 return
229 }
230 if gp.gcscandone {
231 throw("g already scanned")
232 }
233 scanstack(gp, gcw)
234 gp.gcscandone = true
235 resumeG(stopped)
236
237 if selfScan {
238 casgstatus(userG, _Gwaiting, _Grunning)
239 }
240 })
241 }
242 }
243
244
245
246
247
248 func markrootBlock(b0, n0 uintptr, ptrmask0 *uint8, gcw *gcWork, shard int) {
249 if rootBlockBytes%(8*sys.PtrSize) != 0 {
250
251 throw("rootBlockBytes must be a multiple of 8*ptrSize")
252 }
253
254
255
256
257 off := uintptr(shard) * rootBlockBytes
258 if off >= n0 {
259 return
260 }
261 b := b0 + off
262 ptrmask := (*uint8)(add(unsafe.Pointer(ptrmask0), uintptr(shard)*(rootBlockBytes/(8*sys.PtrSize))))
263 n := uintptr(rootBlockBytes)
264 if off+n > n0 {
265 n = n0 - off
266 }
267
268
269 scanblock(b, n, ptrmask, gcw, nil)
270 }
271
272
273
274
275
276 func markrootFreeGStacks() {
277
278 lock(&sched.gFree.lock)
279 list := sched.gFree.stack
280 sched.gFree.stack = gList{}
281 unlock(&sched.gFree.lock)
282 if list.empty() {
283 return
284 }
285
286
287 q := gQueue{list.head, list.head}
288 for gp := list.head.ptr(); gp != nil; gp = gp.schedlink.ptr() {
289 stackfree(gp.stack)
290 gp.stack.lo = 0
291 gp.stack.hi = 0
292
293
294 q.tail.set(gp)
295 }
296
297
298 lock(&sched.gFree.lock)
299 sched.gFree.noStack.pushAll(q)
300 unlock(&sched.gFree.lock)
301 }
302
303
304
305
306 func markrootSpans(gcw *gcWork, shard int) {
307
308
309
310
311
312
313
314
315
316 sg := mheap_.sweepgen
317
318
319 ai := mheap_.markArenas[shard/(pagesPerArena/pagesPerSpanRoot)]
320 ha := mheap_.arenas[ai.l1()][ai.l2()]
321 arenaPage := uint(uintptr(shard) * pagesPerSpanRoot % pagesPerArena)
322
323
324 specialsbits := ha.pageSpecials[arenaPage/8:]
325 specialsbits = specialsbits[:pagesPerSpanRoot/8]
326 for i := range specialsbits {
327
328 specials := atomic.Load8(&specialsbits[i])
329 if specials == 0 {
330 continue
331 }
332 for j := uint(0); j < 8; j++ {
333 if specials&(1<<j) == 0 {
334 continue
335 }
336
337
338
339
340
341
342 s := ha.spans[arenaPage+uint(i)*8+j]
343
344
345
346 if state := s.state.get(); state != mSpanInUse {
347 print("s.state = ", state, "\n")
348 throw("non in-use span found with specials bit set")
349 }
350
351 if !useCheckmark && !(s.sweepgen == sg || s.sweepgen == sg+3) {
352
353 print("sweep ", s.sweepgen, " ", sg, "\n")
354 throw("gc: unswept span")
355 }
356
357
358
359 lock(&s.speciallock)
360 for sp := s.specials; sp != nil; sp = sp.next {
361 if sp.kind != _KindSpecialFinalizer {
362 continue
363 }
364
365
366 spf := (*specialfinalizer)(unsafe.Pointer(sp))
367
368 p := s.base() + uintptr(spf.special.offset)/s.elemsize*s.elemsize
369
370
371
372
373 scanobject(p, gcw)
374
375
376 scanblock(uintptr(unsafe.Pointer(&spf.fn)), sys.PtrSize, &oneptrmask[0], gcw, nil)
377 }
378 unlock(&s.speciallock)
379 }
380 }
381 }
382
383
384
385
386
387 func gcAssistAlloc(gp *g) {
388
389
390 if getg() == gp.m.g0 {
391 return
392 }
393 if mp := getg().m; mp.locks > 0 || mp.preemptoff != "" {
394 return
395 }
396
397 traced := false
398 retry:
399
400
401
402
403 assistWorkPerByte := float64frombits(atomic.Load64(&gcController.assistWorkPerByte))
404 assistBytesPerWork := float64frombits(atomic.Load64(&gcController.assistBytesPerWork))
405 debtBytes := -gp.gcAssistBytes
406 scanWork := int64(assistWorkPerByte * float64(debtBytes))
407 if scanWork < gcOverAssistWork {
408 scanWork = gcOverAssistWork
409 debtBytes = int64(assistBytesPerWork * float64(scanWork))
410 }
411
412
413
414
415
416
417
418 bgScanCredit := atomic.Loadint64(&gcController.bgScanCredit)
419 stolen := int64(0)
420 if bgScanCredit > 0 {
421 if bgScanCredit < scanWork {
422 stolen = bgScanCredit
423 gp.gcAssistBytes += 1 + int64(assistBytesPerWork*float64(stolen))
424 } else {
425 stolen = scanWork
426 gp.gcAssistBytes += debtBytes
427 }
428 atomic.Xaddint64(&gcController.bgScanCredit, -stolen)
429
430 scanWork -= stolen
431
432 if scanWork == 0 {
433
434
435 if traced {
436 traceGCMarkAssistDone()
437 }
438 return
439 }
440 }
441
442 if trace.enabled && !traced {
443 traced = true
444 traceGCMarkAssistStart()
445 }
446
447
448 systemstack(func() {
449 gcAssistAlloc1(gp, scanWork)
450
451
452 })
453
454 completed := gp.param != nil
455 gp.param = nil
456 if completed {
457 gcMarkDone()
458 }
459
460 if gp.gcAssistBytes < 0 {
461
462
463
464
465
466
467
468 if gp.preempt {
469 Gosched()
470 goto retry
471 }
472
473
474
475
476
477
478
479
480
481
482 if !gcParkAssist() {
483 goto retry
484 }
485
486
487
488 }
489 if traced {
490 traceGCMarkAssistDone()
491 }
492 }
493
494
495
496
497
498
499
500
501
502
503
504 func gcAssistAlloc1(gp *g, scanWork int64) {
505
506
507 gp.param = nil
508
509 if atomic.Load(&gcBlackenEnabled) == 0 {
510
511
512
513
514
515
516
517 gp.gcAssistBytes = 0
518 return
519 }
520
521
522
523 startTime := nanotime()
524
525 decnwait := atomic.Xadd(&work.nwait, -1)
526 if decnwait == work.nproc {
527 println("runtime: work.nwait =", decnwait, "work.nproc=", work.nproc)
528 throw("nwait > work.nprocs")
529 }
530
531
532 casgstatus(gp, _Grunning, _Gwaiting)
533 gp.waitreason = waitReasonGCAssistMarking
534
535
536
537 gcw := &getg().m.p.ptr().gcw
538 workDone := gcDrainN(gcw, scanWork)
539
540 casgstatus(gp, _Gwaiting, _Grunning)
541
542
543
544
545
546
547
548 assistBytesPerWork := float64frombits(atomic.Load64(&gcController.assistBytesPerWork))
549 gp.gcAssistBytes += 1 + int64(assistBytesPerWork*float64(workDone))
550
551
552
553 incnwait := atomic.Xadd(&work.nwait, +1)
554 if incnwait > work.nproc {
555 println("runtime: work.nwait=", incnwait,
556 "work.nproc=", work.nproc)
557 throw("work.nwait > work.nproc")
558 }
559
560 if incnwait == work.nproc && !gcMarkWorkAvailable(nil) {
561
562
563
564
565 gp.param = unsafe.Pointer(gp)
566 }
567 duration := nanotime() - startTime
568 _p_ := gp.m.p.ptr()
569 _p_.gcAssistTime += duration
570 if _p_.gcAssistTime > gcAssistTimeSlack {
571 atomic.Xaddint64(&gcController.assistTime, _p_.gcAssistTime)
572 _p_.gcAssistTime = 0
573 }
574 }
575
576
577
578
579 func gcWakeAllAssists() {
580 lock(&work.assistQueue.lock)
581 list := work.assistQueue.q.popList()
582 injectglist(&list)
583 unlock(&work.assistQueue.lock)
584 }
585
586
587
588
589
590
591
592 func gcParkAssist() bool {
593 lock(&work.assistQueue.lock)
594
595
596
597 if atomic.Load(&gcBlackenEnabled) == 0 {
598 unlock(&work.assistQueue.lock)
599 return true
600 }
601
602 gp := getg()
603 oldList := work.assistQueue.q
604 work.assistQueue.q.pushBack(gp)
605
606
607
608
609
610 if atomic.Loadint64(&gcController.bgScanCredit) > 0 {
611 work.assistQueue.q = oldList
612 if oldList.tail != 0 {
613 oldList.tail.ptr().schedlink.set(nil)
614 }
615 unlock(&work.assistQueue.lock)
616 return false
617 }
618
619 goparkunlock(&work.assistQueue.lock, waitReasonGCAssistWait, traceEvGoBlockGC, 2)
620 return true
621 }
622
623
624
625
626
627
628
629
630
631
632
633 func gcFlushBgCredit(scanWork int64) {
634 if work.assistQueue.q.empty() {
635
636
637
638
639 atomic.Xaddint64(&gcController.bgScanCredit, scanWork)
640 return
641 }
642
643 assistBytesPerWork := float64frombits(atomic.Load64(&gcController.assistBytesPerWork))
644 scanBytes := int64(float64(scanWork) * assistBytesPerWork)
645
646 lock(&work.assistQueue.lock)
647 for !work.assistQueue.q.empty() && scanBytes > 0 {
648 gp := work.assistQueue.q.pop()
649
650
651 if scanBytes+gp.gcAssistBytes >= 0 {
652
653 scanBytes += gp.gcAssistBytes
654 gp.gcAssistBytes = 0
655
656
657
658
659
660
661 ready(gp, 0, false)
662 } else {
663
664 gp.gcAssistBytes += scanBytes
665 scanBytes = 0
666
667
668
669
670 work.assistQueue.q.pushBack(gp)
671 break
672 }
673 }
674
675 if scanBytes > 0 {
676
677 assistWorkPerByte := float64frombits(atomic.Load64(&gcController.assistWorkPerByte))
678 scanWork = int64(float64(scanBytes) * assistWorkPerByte)
679 atomic.Xaddint64(&gcController.bgScanCredit, scanWork)
680 }
681 unlock(&work.assistQueue.lock)
682 }
683
684
685
686
687
688
689
690
691
692
693
694
695 func scanstack(gp *g, gcw *gcWork) {
696 if readgstatus(gp)&_Gscan == 0 {
697 print("runtime:scanstack: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", hex(readgstatus(gp)), "\n")
698 throw("scanstack - bad status")
699 }
700
701 switch readgstatus(gp) &^ _Gscan {
702 default:
703 print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n")
704 throw("mark - bad status")
705 case _Gdead:
706 return
707 case _Grunning:
708 print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n")
709 throw("scanstack: goroutine not stopped")
710 case _Grunnable, _Gsyscall, _Gwaiting:
711
712 }
713
714 if gp == getg() {
715 throw("can't scan our own stack")
716 }
717
718 if isShrinkStackSafe(gp) {
719
720 shrinkstack(gp)
721 } else {
722
723 gp.preemptShrink = true
724 }
725
726 var state stackScanState
727 state.stack = gp.stack
728
729 if stackTraceDebug {
730 println("stack trace goroutine", gp.goid)
731 }
732
733 if debugScanConservative && gp.asyncSafePoint {
734 print("scanning async preempted goroutine ", gp.goid, " stack [", hex(gp.stack.lo), ",", hex(gp.stack.hi), ")\n")
735 }
736
737
738
739
740 if gp.sched.ctxt != nil {
741 scanblock(uintptr(unsafe.Pointer(&gp.sched.ctxt)), sys.PtrSize, &oneptrmask[0], gcw, &state)
742 }
743
744
745 scanframe := func(frame *stkframe, unused unsafe.Pointer) bool {
746 scanframeworker(frame, &state, gcw)
747 return true
748 }
749 gentraceback(^uintptr(0), ^uintptr(0), 0, gp, 0, nil, 0x7fffffff, scanframe, nil, 0)
750
751
752
753
754
755 tracebackdefers(gp, scanframe, nil)
756
757
758 for d := gp._defer; d != nil; d = d.link {
759 if d.fn != nil {
760
761
762 scanblock(uintptr(unsafe.Pointer(&d.fn)), sys.PtrSize, &oneptrmask[0], gcw, &state)
763 }
764 if d.link != nil {
765
766
767 scanblock(uintptr(unsafe.Pointer(&d.link)), sys.PtrSize, &oneptrmask[0], gcw, &state)
768 }
769
770
771
772 if d.heap {
773 scanblock(uintptr(unsafe.Pointer(&d)), sys.PtrSize, &oneptrmask[0], gcw, &state)
774 }
775 }
776 if gp._panic != nil {
777
778 state.putPtr(uintptr(unsafe.Pointer(gp._panic)), false)
779 }
780
781
782
783
784
785
786 state.buildIndex()
787 for {
788 p, conservative := state.getPtr()
789 if p == 0 {
790 break
791 }
792 obj := state.findObject(p)
793 if obj == nil {
794 continue
795 }
796 t := obj.typ
797 if t == nil {
798
799 continue
800 }
801 obj.setType(nil)
802 if stackTraceDebug {
803 printlock()
804 print(" live stkobj at", hex(state.stack.lo+uintptr(obj.off)), "of type", t.string())
805 if conservative {
806 print(" (conservative)")
807 }
808 println()
809 printunlock()
810 }
811 gcdata := t.gcdata
812 var s *mspan
813 if t.kind&kindGCProg != 0 {
814
815
816
817
818
819
820
821
822
823 s = materializeGCProg(t.ptrdata, gcdata)
824 gcdata = (*byte)(unsafe.Pointer(s.startAddr))
825 }
826
827 b := state.stack.lo + uintptr(obj.off)
828 if conservative {
829 scanConservative(b, t.ptrdata, gcdata, gcw, &state)
830 } else {
831 scanblock(b, t.ptrdata, gcdata, gcw, &state)
832 }
833
834 if s != nil {
835 dematerializeGCProg(s)
836 }
837 }
838
839
840
841 for state.head != nil {
842 x := state.head
843 state.head = x.next
844 if stackTraceDebug {
845 for i := 0; i < x.nobj; i++ {
846 obj := &x.obj[i]
847 if obj.typ == nil {
848 continue
849 }
850 println(" dead stkobj at", hex(gp.stack.lo+uintptr(obj.off)), "of type", obj.typ.string())
851
852 }
853 }
854 x.nobj = 0
855 putempty((*workbuf)(unsafe.Pointer(x)))
856 }
857 if state.buf != nil || state.cbuf != nil || state.freeBuf != nil {
858 throw("remaining pointer buffers")
859 }
860 }
861
862
863
864 func scanframeworker(frame *stkframe, state *stackScanState, gcw *gcWork) {
865 if _DebugGC > 1 && frame.continpc != 0 {
866 print("scanframe ", funcname(frame.fn), "\n")
867 }
868
869 isAsyncPreempt := frame.fn.valid() && frame.fn.funcID == funcID_asyncPreempt
870 isDebugCall := frame.fn.valid() && frame.fn.funcID == funcID_debugCallV1
871 if state.conservative || isAsyncPreempt || isDebugCall {
872 if debugScanConservative {
873 println("conservatively scanning function", funcname(frame.fn), "at PC", hex(frame.continpc))
874 }
875
876
877
878
879
880
881
882
883
884 if frame.varp != 0 {
885 size := frame.varp - frame.sp
886 if size > 0 {
887 scanConservative(frame.sp, size, nil, gcw, state)
888 }
889 }
890
891
892 if frame.arglen != 0 {
893
894
895 scanConservative(frame.argp, frame.arglen, nil, gcw, state)
896 }
897
898 if isAsyncPreempt || isDebugCall {
899
900
901
902
903 state.conservative = true
904 } else {
905
906
907
908 state.conservative = false
909 }
910 return
911 }
912
913 locals, args, objs := getStackMap(frame, &state.cache, false)
914
915
916 if locals.n > 0 {
917 size := uintptr(locals.n) * sys.PtrSize
918 scanblock(frame.varp-size, size, locals.bytedata, gcw, state)
919 }
920
921
922 if args.n > 0 {
923 scanblock(frame.argp, uintptr(args.n)*sys.PtrSize, args.bytedata, gcw, state)
924 }
925
926
927 if frame.varp != 0 {
928
929
930
931 for _, obj := range objs {
932 off := obj.off
933 base := frame.varp
934 if off >= 0 {
935 base = frame.argp
936 }
937 ptr := base + uintptr(off)
938 if ptr < frame.sp {
939
940 continue
941 }
942 if stackTraceDebug {
943 println("stkobj at", hex(ptr), "of type", obj.typ.string())
944 }
945 state.addObject(ptr, obj.typ)
946 }
947 }
948 }
949
950 type gcDrainFlags int
951
952 const (
953 gcDrainUntilPreempt gcDrainFlags = 1 << iota
954 gcDrainFlushBgCredit
955 gcDrainIdle
956 gcDrainFractional
957 )
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981 func gcDrain(gcw *gcWork, flags gcDrainFlags) {
982 if !writeBarrier.needed {
983 throw("gcDrain phase incorrect")
984 }
985
986 gp := getg().m.curg
987 preemptible := flags&gcDrainUntilPreempt != 0
988 flushBgCredit := flags&gcDrainFlushBgCredit != 0
989 idle := flags&gcDrainIdle != 0
990
991 initScanWork := gcw.scanWork
992
993
994
995 checkWork := int64(1<<63 - 1)
996 var check func() bool
997 if flags&(gcDrainIdle|gcDrainFractional) != 0 {
998 checkWork = initScanWork + drainCheckThreshold
999 if idle {
1000 check = pollWork
1001 } else if flags&gcDrainFractional != 0 {
1002 check = pollFractionalWorkerExit
1003 }
1004 }
1005
1006
1007 if work.markrootNext < work.markrootJobs {
1008
1009 for !(gp.preempt && (preemptible || atomic.Load(&sched.gcwaiting) != 0)) {
1010 job := atomic.Xadd(&work.markrootNext, +1) - 1
1011 if job >= work.markrootJobs {
1012 break
1013 }
1014 markroot(gcw, job)
1015 if check != nil && check() {
1016 goto done
1017 }
1018 }
1019 }
1020
1021
1022
1023 for !(gp.preempt && (preemptible || atomic.Load(&sched.gcwaiting) != 0)) {
1024
1025
1026
1027
1028
1029 if work.full == 0 {
1030 gcw.balance()
1031 }
1032
1033 b := gcw.tryGetFast()
1034 if b == 0 {
1035 b = gcw.tryGet()
1036 if b == 0 {
1037
1038
1039
1040 wbBufFlush(nil, 0)
1041 b = gcw.tryGet()
1042 }
1043 }
1044 if b == 0 {
1045
1046 break
1047 }
1048 scanobject(b, gcw)
1049
1050
1051
1052
1053 if gcw.scanWork >= gcCreditSlack {
1054 atomic.Xaddint64(&gcController.scanWork, gcw.scanWork)
1055 if flushBgCredit {
1056 gcFlushBgCredit(gcw.scanWork - initScanWork)
1057 initScanWork = 0
1058 }
1059 checkWork -= gcw.scanWork
1060 gcw.scanWork = 0
1061
1062 if checkWork <= 0 {
1063 checkWork += drainCheckThreshold
1064 if check != nil && check() {
1065 break
1066 }
1067 }
1068 }
1069 }
1070
1071 done:
1072
1073 if gcw.scanWork > 0 {
1074 atomic.Xaddint64(&gcController.scanWork, gcw.scanWork)
1075 if flushBgCredit {
1076 gcFlushBgCredit(gcw.scanWork - initScanWork)
1077 }
1078 gcw.scanWork = 0
1079 }
1080 }
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095 func gcDrainN(gcw *gcWork, scanWork int64) int64 {
1096 if !writeBarrier.needed {
1097 throw("gcDrainN phase incorrect")
1098 }
1099
1100
1101
1102 workFlushed := -gcw.scanWork
1103
1104 gp := getg().m.curg
1105 for !gp.preempt && workFlushed+gcw.scanWork < scanWork {
1106
1107 if work.full == 0 {
1108 gcw.balance()
1109 }
1110
1111
1112
1113
1114
1115
1116 b := gcw.tryGetFast()
1117 if b == 0 {
1118 b = gcw.tryGet()
1119 if b == 0 {
1120
1121
1122 wbBufFlush(nil, 0)
1123 b = gcw.tryGet()
1124 }
1125 }
1126
1127 if b == 0 {
1128
1129
1130
1131
1132 if work.markrootNext < work.markrootJobs {
1133 job := atomic.Xadd(&work.markrootNext, +1) - 1
1134 if job < work.markrootJobs {
1135 markroot(gcw, job)
1136 continue
1137 }
1138 }
1139
1140 break
1141 }
1142 scanobject(b, gcw)
1143
1144
1145 if gcw.scanWork >= gcCreditSlack {
1146 atomic.Xaddint64(&gcController.scanWork, gcw.scanWork)
1147 workFlushed += gcw.scanWork
1148 gcw.scanWork = 0
1149 }
1150 }
1151
1152
1153
1154
1155
1156 return workFlushed + gcw.scanWork
1157 }
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167 func scanblock(b0, n0 uintptr, ptrmask *uint8, gcw *gcWork, stk *stackScanState) {
1168
1169
1170
1171 b := b0
1172 n := n0
1173
1174 for i := uintptr(0); i < n; {
1175
1176 bits := uint32(*addb(ptrmask, i/(sys.PtrSize*8)))
1177 if bits == 0 {
1178 i += sys.PtrSize * 8
1179 continue
1180 }
1181 for j := 0; j < 8 && i < n; j++ {
1182 if bits&1 != 0 {
1183
1184 p := *(*uintptr)(unsafe.Pointer(b + i))
1185 if p != 0 {
1186 if obj, span, objIndex := findObject(p, b, i); obj != 0 {
1187 greyobject(obj, b, i, span, gcw, objIndex)
1188 } else if stk != nil && p >= stk.stack.lo && p < stk.stack.hi {
1189 stk.putPtr(p, false)
1190 }
1191 }
1192 }
1193 bits >>= 1
1194 i += sys.PtrSize
1195 }
1196 }
1197 }
1198
1199
1200
1201
1202
1203
1204
1205 func scanobject(b uintptr, gcw *gcWork) {
1206
1207
1208
1209
1210
1211 hbits := heapBitsForAddr(b)
1212 s := spanOfUnchecked(b)
1213 n := s.elemsize
1214 if n == 0 {
1215 throw("scanobject n == 0")
1216 }
1217
1218 if n > maxObletBytes {
1219
1220
1221 if b == s.base() {
1222
1223
1224
1225
1226
1227 if s.spanclass.noscan() {
1228
1229 gcw.bytesMarked += uint64(n)
1230 return
1231 }
1232
1233
1234
1235
1236
1237
1238 for oblet := b + maxObletBytes; oblet < s.base()+s.elemsize; oblet += maxObletBytes {
1239 if !gcw.putFast(oblet) {
1240 gcw.put(oblet)
1241 }
1242 }
1243 }
1244
1245
1246
1247
1248 n = s.base() + s.elemsize - b
1249 if n > maxObletBytes {
1250 n = maxObletBytes
1251 }
1252 }
1253
1254 var i uintptr
1255 for i = 0; i < n; i += sys.PtrSize {
1256
1257 if i != 0 {
1258
1259 hbits = hbits.next()
1260 }
1261
1262 bits := hbits.bits()
1263 if bits&bitScan == 0 {
1264 break
1265 }
1266 if bits&bitPointer == 0 {
1267 continue
1268 }
1269
1270
1271
1272 obj := *(*uintptr)(unsafe.Pointer(b + i))
1273
1274
1275
1276 if obj != 0 && obj-b >= n {
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286 if obj, span, objIndex := findObject(obj, b, i); obj != 0 {
1287 greyobject(obj, b, i, span, gcw, objIndex)
1288 }
1289 }
1290 }
1291 gcw.bytesMarked += uint64(n)
1292 gcw.scanWork += int64(i)
1293 }
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303 func scanConservative(b, n uintptr, ptrmask *uint8, gcw *gcWork, state *stackScanState) {
1304 if debugScanConservative {
1305 printlock()
1306 print("conservatively scanning [", hex(b), ",", hex(b+n), ")\n")
1307 hexdumpWords(b, b+n, func(p uintptr) byte {
1308 if ptrmask != nil {
1309 word := (p - b) / sys.PtrSize
1310 bits := *addb(ptrmask, word/8)
1311 if (bits>>(word%8))&1 == 0 {
1312 return '$'
1313 }
1314 }
1315
1316 val := *(*uintptr)(unsafe.Pointer(p))
1317 if state != nil && state.stack.lo <= val && val < state.stack.hi {
1318 return '@'
1319 }
1320
1321 span := spanOfHeap(val)
1322 if span == nil {
1323 return ' '
1324 }
1325 idx := span.objIndex(val)
1326 if span.isFree(idx) {
1327 return ' '
1328 }
1329 return '*'
1330 })
1331 printunlock()
1332 }
1333
1334 for i := uintptr(0); i < n; i += sys.PtrSize {
1335 if ptrmask != nil {
1336 word := i / sys.PtrSize
1337 bits := *addb(ptrmask, word/8)
1338 if bits == 0 {
1339
1340
1341
1342
1343
1344
1345 if i%(sys.PtrSize*8) != 0 {
1346 throw("misaligned mask")
1347 }
1348 i += sys.PtrSize*8 - sys.PtrSize
1349 continue
1350 }
1351 if (bits>>(word%8))&1 == 0 {
1352 continue
1353 }
1354 }
1355
1356 val := *(*uintptr)(unsafe.Pointer(b + i))
1357
1358
1359 if state != nil && state.stack.lo <= val && val < state.stack.hi {
1360
1361
1362
1363
1364
1365
1366
1367
1368 state.putPtr(val, true)
1369 continue
1370 }
1371
1372
1373 span := spanOfHeap(val)
1374 if span == nil {
1375 continue
1376 }
1377
1378
1379 idx := span.objIndex(val)
1380 if span.isFree(idx) {
1381 continue
1382 }
1383
1384
1385 obj := span.base() + idx*span.elemsize
1386 greyobject(obj, b, i, span, gcw, idx)
1387 }
1388 }
1389
1390
1391
1392
1393
1394 func shade(b uintptr) {
1395 if obj, span, objIndex := findObject(b, 0, 0); obj != 0 {
1396 gcw := &getg().m.p.ptr().gcw
1397 greyobject(obj, 0, 0, span, gcw, objIndex)
1398 }
1399 }
1400
1401
1402
1403
1404
1405
1406
1407
1408 func greyobject(obj, base, off uintptr, span *mspan, gcw *gcWork, objIndex uintptr) {
1409
1410 if obj&(sys.PtrSize-1) != 0 {
1411 throw("greyobject: obj not pointer-aligned")
1412 }
1413 mbits := span.markBitsForIndex(objIndex)
1414
1415 if useCheckmark {
1416 if setCheckmark(obj, base, off, mbits) {
1417
1418 return
1419 }
1420 } else {
1421 if debug.gccheckmark > 0 && span.isFree(objIndex) {
1422 print("runtime: marking free object ", hex(obj), " found at *(", hex(base), "+", hex(off), ")\n")
1423 gcDumpObject("base", base, off)
1424 gcDumpObject("obj", obj, ^uintptr(0))
1425 getg().m.traceback = 2
1426 throw("marking free object")
1427 }
1428
1429
1430 if mbits.isMarked() {
1431 return
1432 }
1433 mbits.setMarked()
1434
1435
1436 arena, pageIdx, pageMask := pageIndexOf(span.base())
1437 if arena.pageMarks[pageIdx]&pageMask == 0 {
1438 atomic.Or8(&arena.pageMarks[pageIdx], pageMask)
1439 }
1440
1441
1442
1443 if span.spanclass.noscan() {
1444 gcw.bytesMarked += uint64(span.elemsize)
1445 return
1446 }
1447 }
1448
1449
1450
1451
1452
1453
1454
1455 if !gcw.putFast(obj) {
1456 gcw.put(obj)
1457 }
1458 }
1459
1460
1461
1462 func gcDumpObject(label string, obj, off uintptr) {
1463 s := spanOf(obj)
1464 print(label, "=", hex(obj))
1465 if s == nil {
1466 print(" s=nil\n")
1467 return
1468 }
1469 print(" s.base()=", hex(s.base()), " s.limit=", hex(s.limit), " s.spanclass=", s.spanclass, " s.elemsize=", s.elemsize, " s.state=")
1470 if state := s.state.get(); 0 <= state && int(state) < len(mSpanStateNames) {
1471 print(mSpanStateNames[state], "\n")
1472 } else {
1473 print("unknown(", state, ")\n")
1474 }
1475
1476 skipped := false
1477 size := s.elemsize
1478 if s.state.get() == mSpanManual && size == 0 {
1479
1480
1481
1482 size = off + sys.PtrSize
1483 }
1484 for i := uintptr(0); i < size; i += sys.PtrSize {
1485
1486
1487
1488 if !(i < 128*sys.PtrSize || off-16*sys.PtrSize < i && i < off+16*sys.PtrSize) {
1489 skipped = true
1490 continue
1491 }
1492 if skipped {
1493 print(" ...\n")
1494 skipped = false
1495 }
1496 print(" *(", label, "+", i, ") = ", hex(*(*uintptr)(unsafe.Pointer(obj + i))))
1497 if i == off {
1498 print(" <==")
1499 }
1500 print("\n")
1501 }
1502 if skipped {
1503 print(" ...\n")
1504 }
1505 }
1506
1507
1508
1509
1510
1511
1512
1513
1514 func gcmarknewobject(span *mspan, obj, size, scanSize uintptr) {
1515 if useCheckmark {
1516 throw("gcmarknewobject called while doing checkmark")
1517 }
1518
1519
1520 objIndex := span.objIndex(obj)
1521 span.markBitsForIndex(objIndex).setMarked()
1522
1523
1524 arena, pageIdx, pageMask := pageIndexOf(span.base())
1525 if arena.pageMarks[pageIdx]&pageMask == 0 {
1526 atomic.Or8(&arena.pageMarks[pageIdx], pageMask)
1527 }
1528
1529 gcw := &getg().m.p.ptr().gcw
1530 gcw.bytesMarked += uint64(size)
1531 gcw.scanWork += int64(scanSize)
1532 }
1533
1534
1535
1536
1537 func gcMarkTinyAllocs() {
1538 assertWorldStopped()
1539
1540 for _, p := range allp {
1541 c := p.mcache
1542 if c == nil || c.tiny == 0 {
1543 continue
1544 }
1545 _, span, objIndex := findObject(c.tiny, 0, 0)
1546 gcw := &p.gcw
1547 greyobject(c.tiny, 0, 0, span, gcw, objIndex)
1548 }
1549 }
1550
View as plain text