// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Memory statistics package runtime import ( "runtime/internal/atomic" "unsafe" ) type mstats struct { // Statistics about malloc heap. heapStats consistentHeapStats // Statistics about stacks. stacks_sys sysMemStat // only counts newosproc0 stack in mstats; differs from MemStats.StackSys // Statistics about allocation of low-level fixed-size structures. mspan_sys sysMemStat mcache_sys sysMemStat buckhash_sys sysMemStat // profiling bucket hash table // Statistics about GC overhead. gcMiscSys sysMemStat // updated atomically or during STW // Miscellaneous statistics. other_sys sysMemStat // updated atomically or during STW // Statistics about the garbage collector. // Protected by mheap or worldsema during GC. last_gc_unix uint64 // last gc (in unix time) pause_total_ns uint64 pause_ns [256]uint64 // circular buffer of recent gc pause lengths pause_end [256]uint64 // circular buffer of recent gc end times (nanoseconds since 1970) numgc uint32 numforcedgc uint32 // number of user-forced GCs gc_cpu_fraction float64 // fraction of CPU time used by GC last_gc_nanotime uint64 // last gc (monotonic time) lastHeapInUse uint64 // heapInUse at mark termination of the previous GC enablegc bool } var memstats mstats // A MemStats records statistics about the memory allocator. type MemStats struct { // General statistics. // Alloc is bytes of allocated heap objects. // // This is the same as HeapAlloc (see below). Alloc uint64 // TotalAlloc is cumulative bytes allocated for heap objects. // // TotalAlloc increases as heap objects are allocated, but // unlike Alloc and HeapAlloc, it does not decrease when // objects are freed. TotalAlloc uint64 // Sys is the total bytes of memory obtained from the OS. // // Sys is the sum of the XSys fields below. Sys measures the // virtual address space reserved by the Go runtime for the // heap, stacks, and other internal data structures. It's // likely that not all of the virtual address space is backed // by physical memory at any given moment, though in general // it all was at some point. Sys uint64 // Lookups is the number of pointer lookups performed by the // runtime. // // This is primarily useful for debugging runtime internals. Lookups uint64 // Mallocs is the cumulative count of heap objects allocated. // The number of live objects is Mallocs - Frees. Mallocs uint64 // Frees is the cumulative count of heap objects freed. Frees uint64 // Heap memory statistics. // // Interpreting the heap statistics requires some knowledge of // how Go organizes memory. Go divides the virtual address // space of the heap into "spans", which are contiguous // regions of memory 8K or larger. A span may be in one of // three states: // // An "idle" span contains no objects or other data. The // physical memory backing an idle span can be released back // to the OS (but the virtual address space never is), or it // can be converted into an "in use" or "stack" span. // // An "in use" span contains at least one heap object and may // have free space available to allocate more heap objects. // // A "stack" span is used for goroutine stacks. Stack spans // are not considered part of the heap. A span can change // between heap and stack memory; it is never used for both // simultaneously. // HeapAlloc is bytes of allocated heap objects. // // "Allocated" heap objects include all reachable objects, as // well as unreachable objects that the garbage collector has // not yet freed. Specifically, HeapAlloc increases as heap // objects are allocated and decreases as the heap is swept // and unreachable objects are freed. Sweeping occurs // incrementally between GC cycles, so these two processes // occur simultaneously, and as a result HeapAlloc tends to // change smoothly (in contrast with the sawtooth that is // typical of stop-the-world garbage collectors). HeapAlloc uint64 // HeapSys is bytes of heap memory obtained from the OS. // // HeapSys measures the amount of virtual address space // reserved for the heap. This includes virtual address space // that has been reserved but not yet used, which consumes no // physical memory, but tends to be small, as well as virtual // address space for which the physical memory has been // returned to the OS after it became unused (see HeapReleased // for a measure of the latter). // // HeapSys estimates the largest size the heap has had. HeapSys uint64 // HeapIdle is bytes in idle (unused) spans. // // Idle spans have no objects in them. These spans could be // (and may already have been) returned to the OS, or they can // be reused for heap allocations, or they can be reused as // stack memory. // // HeapIdle minus HeapReleased estimates the amount of memory // that could be returned to the OS, but is being retained by // the runtime so it can grow the heap without requesting more // memory from the OS. If this difference is significantly // larger than the heap size, it indicates there was a recent // transient spike in live heap size. HeapIdle uint64 // HeapInuse is bytes in in-use spans. // // In-use spans have at least one object in them. These spans // can only be used for other objects of roughly the same // size. // // HeapInuse minus HeapAlloc estimates the amount of memory // that has been dedicated to particular size classes, but is // not currently being used. This is an upper bound on // fragmentation, but in general this memory can be reused // efficiently. HeapInuse uint64 // HeapReleased is bytes of physical memory returned to the OS. // // This counts heap memory from idle spans that was returned // to the OS and has not yet been reacquired for the heap. HeapReleased uint64 // HeapObjects is the number of allocated heap objects. // // Like HeapAlloc, this increases as objects are allocated and // decreases as the heap is swept and unreachable objects are // freed. HeapObjects uint64 // Stack memory statistics. // // Stacks are not considered part of the heap, but the runtime // can reuse a span of heap memory for stack memory, and // vice-versa. // StackInuse is bytes in stack spans. // // In-use stack spans have at least one stack in them. These // spans can only be used for other stacks of the same size. // // There is no StackIdle because unused stack spans are // returned to the heap (and hence counted toward HeapIdle). StackInuse uint64 // StackSys is bytes of stack memory obtained from the OS. // // StackSys is StackInuse, plus any memory obtained directly // from the OS for OS thread stacks. // // In non-cgo programs this metric is currently equal to StackInuse // (but this should not be relied upon, and the value may change in // the future). // // In cgo programs this metric includes OS thread stacks allocated // directly from the OS. Currently, this only accounts for one stack in // c-shared and c-archive build modes and other sources of stacks from // the OS (notably, any allocated by C code) are not currently measured. // Note this too may change in the future. StackSys uint64 // Off-heap memory statistics. // // The following statistics measure runtime-internal // structures that are not allocated from heap memory (usually // because they are part of implementing the heap). Unlike // heap or stack memory, any memory allocated to these // structures is dedicated to these structures. // // These are primarily useful for debugging runtime memory // overheads. // MSpanInuse is bytes of allocated mspan structures. MSpanInuse uint64 // MSpanSys is bytes of memory obtained from the OS for mspan // structures. MSpanSys uint64 // MCacheInuse is bytes of allocated mcache structures. MCacheInuse uint64 // MCacheSys is bytes of memory obtained from the OS for // mcache structures. MCacheSys uint64 // BuckHashSys is bytes of memory in profiling bucket hash tables. BuckHashSys uint64 // GCSys is bytes of memory in garbage collection metadata. GCSys uint64 // OtherSys is bytes of memory in miscellaneous off-heap // runtime allocations. OtherSys uint64 // Garbage collector statistics. // NextGC is the target heap size of the next GC cycle. // // The garbage collector's goal is to keep HeapAlloc ≤ NextGC. // At the end of each GC cycle, the target for the next cycle // is computed based on the amount of reachable data and the // value of GOGC. NextGC uint64 // LastGC is the time the last garbage collection finished, as // nanoseconds since 1970 (the UNIX epoch). LastGC uint64 // PauseTotalNs is the cumulative nanoseconds in GC // stop-the-world pauses since the program started. // // During a stop-the-world pause, all goroutines are paused // and only the garbage collector can run. PauseTotalNs uint64 // PauseNs is a circular buffer of recent GC stop-the-world // pause times in nanoseconds. // // The most recent pause is at PauseNs[(NumGC+255)%256]. In // general, PauseNs[N%256] records the time paused in the most // recent N%256th GC cycle. There may be multiple pauses per // GC cycle; this is the sum of all pauses during a cycle. PauseNs [256]uint64 // PauseEnd is a circular buffer of recent GC pause end times, // as nanoseconds since 1970 (the UNIX epoch). // // This buffer is filled the same way as PauseNs. There may be // multiple pauses per GC cycle; this records the end of the // last pause in a cycle. PauseEnd [256]uint64 // NumGC is the number of completed GC cycles. NumGC uint32 // NumForcedGC is the number of GC cycles that were forced by // the application calling the GC function. NumForcedGC uint32 // GCCPUFraction is the fraction of this program's available // CPU time used by the GC since the program started. // // GCCPUFraction is expressed as a number between 0 and 1, // where 0 means GC has consumed none of this program's CPU. A // program's available CPU time is defined as the integral of // GOMAXPROCS since the program started. That is, if // GOMAXPROCS is 2 and a program has been running for 10 // seconds, its "available CPU" is 20 seconds. GCCPUFraction // does not include CPU time used for write barrier activity. // // This is the same as the fraction of CPU reported by // GODEBUG=gctrace=1. GCCPUFraction float64 // EnableGC indicates that GC is enabled. It is always true, // even if GOGC=off. EnableGC bool // DebugGC is currently unused. DebugGC bool // BySize reports per-size class allocation statistics. // // BySize[N] gives statistics for allocations of size S where // BySize[N-1].Size < S ≤ BySize[N].Size. // // This does not report allocations larger than BySize[60].Size. BySize [61]struct { // Size is the maximum byte size of an object in this // size class. Size uint32 // Mallocs is the cumulative count of heap objects // allocated in this size class. The cumulative bytes // of allocation is Size*Mallocs. The number of live // objects in this size class is Mallocs - Frees. Mallocs uint64 // Frees is the cumulative count of heap objects freed // in this size class. Frees uint64 } } func init() { if offset := unsafe.Offsetof(memstats.heapStats); offset%8 != 0 { println(offset) throw("memstats.heapStats not aligned to 8 bytes") } // Ensure the size of heapStatsDelta causes adjacent fields/slots (e.g. // [3]heapStatsDelta) to be 8-byte aligned. if size := unsafe.Sizeof(heapStatsDelta{}); size%8 != 0 { println(size) throw("heapStatsDelta not a multiple of 8 bytes in size") } } // ReadMemStats populates m with memory allocator statistics. // // The returned memory allocator statistics are up to date as of the // call to ReadMemStats. This is in contrast with a heap profile, // which is a snapshot as of the most recently completed garbage // collection cycle. func ReadMemStats(m *MemStats) { _ = m.Alloc // nil check test before we switch stacks, see issue 61158 stw := stopTheWorld(stwReadMemStats) systemstack(func() { readmemstats_m(m) }) startTheWorld(stw) } // doubleCheckReadMemStats controls a double-check mode for ReadMemStats that // ensures consistency between the values that ReadMemStats is using and the // runtime-internal stats. var doubleCheckReadMemStats = false // readmemstats_m populates stats for internal runtime values. // // The world must be stopped. func readmemstats_m(stats *MemStats) { assertWorldStopped() // Flush mcaches to mcentral before doing anything else. // // Flushing to the mcentral may in general cause stats to // change as mcentral data structures are manipulated. systemstack(flushallmcaches) // Calculate memory allocator stats. // During program execution we only count number of frees and amount of freed memory. // Current number of alive objects in the heap and amount of alive heap memory // are calculated by scanning all spans. // Total number of mallocs is calculated as number of frees plus number of alive objects. // Similarly, total amount of allocated memory is calculated as amount of freed memory // plus amount of alive heap memory. // Collect consistent stats, which are the source-of-truth in some cases. var consStats heapStatsDelta memstats.heapStats.unsafeRead(&consStats) // Collect large allocation stats. totalAlloc := consStats.largeAlloc nMalloc := consStats.largeAllocCount totalFree := consStats.largeFree nFree := consStats.largeFreeCount // Collect per-sizeclass stats. var bySize [_NumSizeClasses]struct { Size uint32 Mallocs uint64 Frees uint64 } for i := range bySize { bySize[i].Size = uint32(class_to_size[i]) // Malloc stats. a := consStats.smallAllocCount[i] totalAlloc += a * uint64(class_to_size[i]) nMalloc += a bySize[i].Mallocs = a // Free stats. f := consStats.smallFreeCount[i] totalFree += f * uint64(class_to_size[i]) nFree += f bySize[i].Frees = f } // Account for tiny allocations. // For historical reasons, MemStats includes tiny allocations // in both the total free and total alloc count. This double-counts // memory in some sense because their tiny allocation block is also // counted. Tracking the lifetime of individual tiny allocations is // currently not done because it would be too expensive. nFree += consStats.tinyAllocCount nMalloc += consStats.tinyAllocCount // Calculate derived stats. stackInUse := uint64(consStats.inStacks) gcWorkBufInUse := uint64(consStats.inWorkBufs) gcProgPtrScalarBitsInUse := uint64(consStats.inPtrScalarBits) totalMapped := gcController.heapInUse.load() + gcController.heapFree.load() + gcController.heapReleased.load() + memstats.stacks_sys.load() + memstats.mspan_sys.load() + memstats.mcache_sys.load() + memstats.buckhash_sys.load() + memstats.gcMiscSys.load() + memstats.other_sys.load() + stackInUse + gcWorkBufInUse + gcProgPtrScalarBitsInUse heapGoal := gcController.heapGoal() if doubleCheckReadMemStats { // Only check this if we're debugging. It would be bad to crash an application // just because the debugging stats are wrong. We mostly rely on tests to catch // these issues, and we enable the double check mode for tests. // // The world is stopped, so the consistent stats (after aggregation) // should be identical to some combination of memstats. In particular: // // * memstats.heapInUse == inHeap // * memstats.heapReleased == released // * memstats.heapInUse + memstats.heapFree == committed - inStacks - inWorkBufs - inPtrScalarBits // * memstats.totalAlloc == totalAlloc // * memstats.totalFree == totalFree // // Check if that's actually true. // // Prevent sysmon and the tracer from skewing the stats since they can // act without synchronizing with a STW. See #64401. lock(&sched.sysmonlock) lock(&trace.lock) if gcController.heapInUse.load() != uint64(consStats.inHeap) { print("runtime: heapInUse=", gcController.heapInUse.load(), "\n") print("runtime: consistent value=", consStats.inHeap, "\n") throw("heapInUse and consistent stats are not equal") } if gcController.heapReleased.load() != uint64(consStats.released) { print("runtime: heapReleased=", gcController.heapReleased.load(), "\n") print("runtime: consistent value=", consStats.released, "\n") throw("heapReleased and consistent stats are not equal") } heapRetained := gcController.heapInUse.load() + gcController.heapFree.load() consRetained := uint64(consStats.committed - consStats.inStacks - consStats.inWorkBufs - consStats.inPtrScalarBits) if heapRetained != consRetained { print("runtime: global value=", heapRetained, "\n") print("runtime: consistent value=", consRetained, "\n") throw("measures of the retained heap are not equal") } if gcController.totalAlloc.Load() != totalAlloc { print("runtime: totalAlloc=", gcController.totalAlloc.Load(), "\n") print("runtime: consistent value=", totalAlloc, "\n") throw("totalAlloc and consistent stats are not equal") } if gcController.totalFree.Load() != totalFree { print("runtime: totalFree=", gcController.totalFree.Load(), "\n") print("runtime: consistent value=", totalFree, "\n") throw("totalFree and consistent stats are not equal") } // Also check that mappedReady lines up with totalMapped - released. // This isn't really the same type of "make sure consistent stats line up" situation, // but this is an opportune time to check. if gcController.mappedReady.Load() != totalMapped-uint64(consStats.released) { print("runtime: mappedReady=", gcController.mappedReady.Load(), "\n") print("runtime: totalMapped=", totalMapped, "\n") print("runtime: released=", uint64(consStats.released), "\n") print("runtime: totalMapped-released=", totalMapped-uint64(consStats.released), "\n") throw("mappedReady and other memstats are not equal") } unlock(&trace.lock) unlock(&sched.sysmonlock) } // We've calculated all the values we need. Now, populate stats. stats.Alloc = totalAlloc - totalFree stats.TotalAlloc = totalAlloc stats.Sys = totalMapped stats.Mallocs = nMalloc stats.Frees = nFree stats.HeapAlloc = totalAlloc - totalFree stats.HeapSys = gcController.heapInUse.load() + gcController.heapFree.load() + gcController.heapReleased.load() // By definition, HeapIdle is memory that was mapped // for the heap but is not currently used to hold heap // objects. It also specifically is memory that can be // used for other purposes, like stacks, but this memory // is subtracted out of HeapSys before it makes that // transition. Put another way: // // HeapSys = bytes allocated from the OS for the heap - bytes ultimately used for non-heap purposes // HeapIdle = bytes allocated from the OS for the heap - bytes ultimately used for any purpose // // or // // HeapSys = sys - stacks_inuse - gcWorkBufInUse - gcProgPtrScalarBitsInUse // HeapIdle = sys - stacks_inuse - gcWorkBufInUse - gcProgPtrScalarBitsInUse - heapInUse // // => HeapIdle = HeapSys - heapInUse = heapFree + heapReleased stats.HeapIdle = gcController.heapFree.load() + gcController.heapReleased.load() stats.HeapInuse = gcController.heapInUse.load() stats.HeapReleased = gcController.heapReleased.load() stats.HeapObjects = nMalloc - nFree stats.StackInuse = stackInUse // memstats.stacks_sys is only memory mapped directly for OS stacks. // Add in heap-allocated stack memory for user consumption. stats.StackSys = stackInUse + memstats.stacks_sys.load() stats.MSpanInuse = uint64(mheap_.spanalloc.inuse) stats.MSpanSys = memstats.mspan_sys.load() stats.MCacheInuse = uint64(mheap_.cachealloc.inuse) stats.MCacheSys = memstats.mcache_sys.load() stats.BuckHashSys = memstats.buckhash_sys.load() // MemStats defines GCSys as an aggregate of all memory related // to the memory management system, but we track this memory // at a more granular level in the runtime. stats.GCSys = memstats.gcMiscSys.load() + gcWorkBufInUse + gcProgPtrScalarBitsInUse stats.OtherSys = memstats.other_sys.load() stats.NextGC = heapGoal stats.LastGC = memstats.last_gc_unix stats.PauseTotalNs = memstats.pause_total_ns stats.PauseNs = memstats.pause_ns stats.PauseEnd = memstats.pause_end stats.NumGC = memstats.numgc stats.NumForcedGC = memstats.numforcedgc stats.GCCPUFraction = memstats.gc_cpu_fraction stats.EnableGC = true // stats.BySize and bySize might not match in length. // That's OK, stats.BySize cannot change due to backwards // compatibility issues. copy will copy the minimum amount // of values between the two of them. copy(stats.BySize[:], bySize[:]) } //go:linkname readGCStats runtime/debug.readGCStats func readGCStats(pauses *[]uint64) { systemstack(func() { readGCStats_m(pauses) }) } // readGCStats_m must be called on the system stack because it acquires the heap // lock. See mheap for details. // //go:systemstack func readGCStats_m(pauses *[]uint64) { p := *pauses // Calling code in runtime/debug should make the slice large enough. if cap(p) < len(memstats.pause_ns)+3 { throw("short slice passed to readGCStats") } // Pass back: pauses, pause ends, last gc (absolute time), number of gc, total pause ns. lock(&mheap_.lock) n := memstats.numgc if n > uint32(len(memstats.pause_ns)) { n = uint32(len(memstats.pause_ns)) } // The pause buffer is circular. The most recent pause is at // pause_ns[(numgc-1)%len(pause_ns)], and then backward // from there to go back farther in time. We deliver the times // most recent first (in p[0]). p = p[:cap(p)] for i := uint32(0); i < n; i++ { j := (memstats.numgc - 1 - i) % uint32(len(memstats.pause_ns)) p[i] = memstats.pause_ns[j] p[n+i] = memstats.pause_end[j] } p[n+n] = memstats.last_gc_unix p[n+n+1] = uint64(memstats.numgc) p[n+n+2] = memstats.pause_total_ns unlock(&mheap_.lock) *pauses = p[:n+n+3] } // flushmcache flushes the mcache of allp[i]. // // The world must be stopped. // //go:nowritebarrier func flushmcache(i int) { assertWorldStopped() p := allp[i] c := p.mcache if c == nil { return } c.releaseAll() stackcache_clear(c) } // flushallmcaches flushes the mcaches of all Ps. // // The world must be stopped. // //go:nowritebarrier func flushallmcaches() { assertWorldStopped() for i := 0; i < int(gomaxprocs); i++ { flushmcache(i) } } // sysMemStat represents a global system statistic that is managed atomically. // // This type must structurally be a uint64 so that mstats aligns with MemStats. type sysMemStat uint64 // load atomically reads the value of the stat. // // Must be nosplit as it is called in runtime initialization, e.g. newosproc0. // //go:nosplit func (s *sysMemStat) load() uint64 { return atomic.Load64((*uint64)(s)) } // add atomically adds the sysMemStat by n. // // Must be nosplit as it is called in runtime initialization, e.g. newosproc0. // //go:nosplit func (s *sysMemStat) add(n int64) { val := atomic.Xadd64((*uint64)(s), n) if (n > 0 && int64(val) < n) || (n < 0 && int64(val)+n < n) { print("runtime: val=", val, " n=", n, "\n") throw("sysMemStat overflow") } } // heapStatsDelta contains deltas of various runtime memory statistics // that need to be updated together in order for them to be kept // consistent with one another. type heapStatsDelta struct { // Memory stats. committed int64 // byte delta of memory committed released int64 // byte delta of released memory generated inHeap int64 // byte delta of memory placed in the heap inStacks int64 // byte delta of memory reserved for stacks inWorkBufs int64 // byte delta of memory reserved for work bufs inPtrScalarBits int64 // byte delta of memory reserved for unrolled GC prog bits // Allocator stats. // // These are all uint64 because they're cumulative, and could quickly wrap // around otherwise. tinyAllocCount uint64 // number of tiny allocations largeAlloc uint64 // bytes allocated for large objects largeAllocCount uint64 // number of large object allocations smallAllocCount [_NumSizeClasses]uint64 // number of allocs for small objects largeFree uint64 // bytes freed for large objects (>maxSmallSize) largeFreeCount uint64 // number of frees for large objects (>maxSmallSize) smallFreeCount [_NumSizeClasses]uint64 // number of frees for small objects (<=maxSmallSize) // NOTE: This struct must be a multiple of 8 bytes in size because it // is stored in an array. If it's not, atomic accesses to the above // fields may be unaligned and fail on 32-bit platforms. } // merge adds in the deltas from b into a. func (a *heapStatsDelta) merge(b *heapStatsDelta) { a.committed += b.committed a.released += b.released a.inHeap += b.inHeap a.inStacks += b.inStacks a.inWorkBufs += b.inWorkBufs a.inPtrScalarBits += b.inPtrScalarBits a.tinyAllocCount += b.tinyAllocCount a.largeAlloc += b.largeAlloc a.largeAllocCount += b.largeAllocCount for i := range b.smallAllocCount { a.smallAllocCount[i] += b.smallAllocCount[i] } a.largeFree += b.largeFree a.largeFreeCount += b.largeFreeCount for i := range b.smallFreeCount { a.smallFreeCount[i] += b.smallFreeCount[i] } } // consistentHeapStats represents a set of various memory statistics // whose updates must be viewed completely to get a consistent // state of the world. // // To write updates to memory stats use the acquire and release // methods. To obtain a consistent global snapshot of these statistics, // use read. type consistentHeapStats struct { // stats is a ring buffer of heapStatsDelta values. // Writers always atomically update the delta at index gen. // // Readers operate by rotating gen (0 -> 1 -> 2 -> 0 -> ...) // and synchronizing with writers by observing each P's // statsSeq field. If the reader observes a P not writing, // it can be sure that it will pick up the new gen value the // next time it writes. // // The reader then takes responsibility by clearing space // in the ring buffer for the next reader to rotate gen to // that space (i.e. it merges in values from index (gen-2) mod 3 // to index (gen-1) mod 3, then clears the former). // // Note that this means only one reader can be reading at a time. // There is no way for readers to synchronize. // // This process is why we need a ring buffer of size 3 instead // of 2: one is for the writers, one contains the most recent // data, and the last one is clear so writers can begin writing // to it the moment gen is updated. stats [3]heapStatsDelta // gen represents the current index into which writers // are writing, and can take on the value of 0, 1, or 2. gen atomic.Uint32 // noPLock is intended to provide mutual exclusion for updating // stats when no P is available. It does not block other writers // with a P, only other writers without a P and the reader. Because // stats are usually updated when a P is available, contention on // this lock should be minimal. noPLock mutex } // acquire returns a heapStatsDelta to be updated. In effect, // it acquires the shard for writing. release must be called // as soon as the relevant deltas are updated. // // The returned heapStatsDelta must be updated atomically. // // The caller's P must not change between acquire and // release. This also means that the caller should not // acquire a P or release its P in between. A P also must // not acquire a given consistentHeapStats if it hasn't // yet released it. // // nosplit because a stack growth in this function could // lead to a stack allocation that could reenter the // function. // //go:nosplit func (m *consistentHeapStats) acquire() *heapStatsDelta { if pp := getg().m.p.ptr(); pp != nil { seq := pp.statsSeq.Add(1) if seq%2 == 0 { // Should have been incremented to odd. print("runtime: seq=", seq, "\n") throw("bad sequence number") } } else { lock(&m.noPLock) } gen := m.gen.Load() % 3 return &m.stats[gen] } // release indicates that the writer is done modifying // the delta. The value returned by the corresponding // acquire must no longer be accessed or modified after // release is called. // // The caller's P must not change between acquire and // release. This also means that the caller should not // acquire a P or release its P in between. // // nosplit because a stack growth in this function could // lead to a stack allocation that causes another acquire // before this operation has completed. // //go:nosplit func (m *consistentHeapStats) release() { if pp := getg().m.p.ptr(); pp != nil { seq := pp.statsSeq.Add(1) if seq%2 != 0 { // Should have been incremented to even. print("runtime: seq=", seq, "\n") throw("bad sequence number") } } else { unlock(&m.noPLock) } } // unsafeRead aggregates the delta for this shard into out. // // Unsafe because it does so without any synchronization. The // world must be stopped. func (m *consistentHeapStats) unsafeRead(out *heapStatsDelta) { assertWorldStopped() for i := range m.stats { out.merge(&m.stats[i]) } } // unsafeClear clears the shard. // // Unsafe because the world must be stopped and values should // be donated elsewhere before clearing. func (m *consistentHeapStats) unsafeClear() { assertWorldStopped() for i := range m.stats { m.stats[i] = heapStatsDelta{} } } // read takes a globally consistent snapshot of m // and puts the aggregated value in out. Even though out is a // heapStatsDelta, the resulting values should be complete and // valid statistic values. // // Not safe to call concurrently. The world must be stopped // or metricsSema must be held. func (m *consistentHeapStats) read(out *heapStatsDelta) { // Getting preempted after this point is not safe because // we read allp. We need to make sure a STW can't happen // so it doesn't change out from under us. mp := acquirem() // Get the current generation. We can be confident that this // will not change since read is serialized and is the only // one that modifies currGen. currGen := m.gen.Load() prevGen := currGen - 1 if currGen == 0 { prevGen = 2 } // Prevent writers without a P from writing while we update gen. lock(&m.noPLock) // Rotate gen, effectively taking a snapshot of the state of // these statistics at the point of the exchange by moving // writers to the next set of deltas. // // This exchange is safe to do because we won't race // with anyone else trying to update this value. m.gen.Swap((currGen + 1) % 3) // Allow P-less writers to continue. They'll be writing to the // next generation now. unlock(&m.noPLock) for _, p := range allp { // Spin until there are no more writers. for p.statsSeq.Load()%2 != 0 { } } // At this point we've observed that each sequence // number is even, so any future writers will observe // the new gen value. That means it's safe to read from // the other deltas in the stats buffer. // Perform our responsibilities and free up // stats[prevGen] for the next time we want to take // a snapshot. m.stats[currGen].merge(&m.stats[prevGen]) m.stats[prevGen] = heapStatsDelta{} // Finally, copy out the complete delta. *out = m.stats[currGen] releasem(mp) } type cpuStats struct { // All fields are CPU time in nanoseconds computed by comparing // calls of nanotime. This means they're all overestimates, because // they don't accurately compute on-CPU time (so some of the time // could be spent scheduled away by the OS). gcAssistTime int64 // GC assists gcDedicatedTime int64 // GC dedicated mark workers + pauses gcIdleTime int64 // GC idle mark workers gcPauseTime int64 // GC pauses (all GOMAXPROCS, even if just 1 is running) gcTotalTime int64 scavengeAssistTime int64 // background scavenger scavengeBgTime int64 // scavenge assists scavengeTotalTime int64 idleTime int64 // Time Ps spent in _Pidle. userTime int64 // Time Ps spent in _Prunning or _Psyscall that's not any of the above. totalTime int64 // GOMAXPROCS * (monotonic wall clock time elapsed) } // accumulate takes a cpuStats and adds in the current state of all GC CPU // counters. // // gcMarkPhase indicates that we're in the mark phase and that certain counter // values should be used. func (s *cpuStats) accumulate(now int64, gcMarkPhase bool) { // N.B. Mark termination and sweep termination pauses are // accumulated in work.cpuStats at the end of their respective pauses. var ( markAssistCpu int64 markDedicatedCpu int64 markFractionalCpu int64 markIdleCpu int64 ) if gcMarkPhase { // N.B. These stats may have stale values if the GC is not // currently in the mark phase. markAssistCpu = gcController.assistTime.Load() markDedicatedCpu = gcController.dedicatedMarkTime.Load() markFractionalCpu = gcController.fractionalMarkTime.Load() markIdleCpu = gcController.idleMarkTime.Load() } // The rest of the stats below are either derived from the above or // are reset on each mark termination. scavAssistCpu := scavenge.assistTime.Load() scavBgCpu := scavenge.backgroundTime.Load() // Update cumulative GC CPU stats. s.gcAssistTime += markAssistCpu s.gcDedicatedTime += markDedicatedCpu + markFractionalCpu s.gcIdleTime += markIdleCpu s.gcTotalTime += markAssistCpu + markDedicatedCpu + markFractionalCpu + markIdleCpu // Update cumulative scavenge CPU stats. s.scavengeAssistTime += scavAssistCpu s.scavengeBgTime += scavBgCpu s.scavengeTotalTime += scavAssistCpu + scavBgCpu // Update total CPU. s.totalTime = sched.totaltime + (now-sched.procresizetime)*int64(gomaxprocs) s.idleTime += sched.idleTime.Load() // Compute userTime. We compute this indirectly as everything that's not the above. // // Since time spent in _Pgcstop is covered by gcPauseTime, and time spent in _Pidle // is covered by idleTime, what we're left with is time spent in _Prunning and _Psyscall, // the latter of which is fine because the P will either go idle or get used for something // else via sysmon. Meanwhile if we subtract GC time from whatever's left, we get non-GC // _Prunning time. Note that this still leaves time spent in sweeping and in the scheduler, // but that's fine. The overwhelming majority of this time will be actual user time. s.userTime = s.totalTime - (s.gcTotalTime + s.scavengeTotalTime + s.idleTime) }