Black Lives Matter. Support the Equal Justice Initiative.

Source file src/runtime/metrics.go

Documentation: runtime

     1  // Copyright 2020 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  // Metrics implementation exported to runtime/metrics.
     8  
     9  import (
    10  	"runtime/internal/atomic"
    11  	"unsafe"
    12  )
    13  
    14  var (
    15  	// metrics is a map of runtime/metrics keys to
    16  	// data used by the runtime to sample each metric's
    17  	// value.
    18  	metricsSema uint32 = 1
    19  	metricsInit bool
    20  	metrics     map[string]metricData
    21  
    22  	sizeClassBuckets []float64
    23  	timeHistBuckets  []float64
    24  )
    25  
    26  type metricData struct {
    27  	// deps is the set of runtime statistics that this metric
    28  	// depends on. Before compute is called, the statAggregate
    29  	// which will be passed must ensure() these dependencies.
    30  	deps statDepSet
    31  
    32  	// compute is a function that populates a metricValue
    33  	// given a populated statAggregate structure.
    34  	compute func(in *statAggregate, out *metricValue)
    35  }
    36  
    37  // initMetrics initializes the metrics map if it hasn't been yet.
    38  //
    39  // metricsSema must be held.
    40  func initMetrics() {
    41  	if metricsInit {
    42  		return
    43  	}
    44  
    45  	sizeClassBuckets = make([]float64, _NumSizeClasses, _NumSizeClasses+1)
    46  	// Skip size class 0 which is a stand-in for large objects, but large
    47  	// objects are tracked separately (and they actually get placed in
    48  	// the last bucket, not the first).
    49  	sizeClassBuckets[0] = 1 // The smallest allocation is 1 byte in size.
    50  	for i := 1; i < _NumSizeClasses; i++ {
    51  		// Size classes have an inclusive upper-bound
    52  		// and exclusive lower bound (e.g. 48-byte size class is
    53  		// (32, 48]) whereas we want and inclusive lower-bound
    54  		// and exclusive upper-bound (e.g. 48-byte size class is
    55  		// [33, 49). We can achieve this by shifting all bucket
    56  		// boundaries up by 1.
    57  		//
    58  		// Also, a float64 can precisely represent integers with
    59  		// value up to 2^53 and size classes are relatively small
    60  		// (nowhere near 2^48 even) so this will give us exact
    61  		// boundaries.
    62  		sizeClassBuckets[i] = float64(class_to_size[i] + 1)
    63  	}
    64  	sizeClassBuckets = append(sizeClassBuckets, float64Inf())
    65  
    66  	timeHistBuckets = timeHistogramMetricsBuckets()
    67  	metrics = map[string]metricData{
    68  		"/gc/cycles/automatic:gc-cycles": {
    69  			deps: makeStatDepSet(sysStatsDep),
    70  			compute: func(in *statAggregate, out *metricValue) {
    71  				out.kind = metricKindUint64
    72  				out.scalar = in.sysStats.gcCyclesDone - in.sysStats.gcCyclesForced
    73  			},
    74  		},
    75  		"/gc/cycles/forced:gc-cycles": {
    76  			deps: makeStatDepSet(sysStatsDep),
    77  			compute: func(in *statAggregate, out *metricValue) {
    78  				out.kind = metricKindUint64
    79  				out.scalar = in.sysStats.gcCyclesForced
    80  			},
    81  		},
    82  		"/gc/cycles/total:gc-cycles": {
    83  			deps: makeStatDepSet(sysStatsDep),
    84  			compute: func(in *statAggregate, out *metricValue) {
    85  				out.kind = metricKindUint64
    86  				out.scalar = in.sysStats.gcCyclesDone
    87  			},
    88  		},
    89  		"/gc/heap/allocs-by-size:bytes": {
    90  			deps: makeStatDepSet(heapStatsDep),
    91  			compute: func(in *statAggregate, out *metricValue) {
    92  				hist := out.float64HistOrInit(sizeClassBuckets)
    93  				hist.counts[len(hist.counts)-1] = uint64(in.heapStats.largeAllocCount)
    94  				// Cut off the first index which is ostensibly for size class 0,
    95  				// but large objects are tracked separately so it's actually unused.
    96  				for i, count := range in.heapStats.smallAllocCount[1:] {
    97  					hist.counts[i] = uint64(count)
    98  				}
    99  			},
   100  		},
   101  		"/gc/heap/frees-by-size:bytes": {
   102  			deps: makeStatDepSet(heapStatsDep),
   103  			compute: func(in *statAggregate, out *metricValue) {
   104  				hist := out.float64HistOrInit(sizeClassBuckets)
   105  				hist.counts[len(hist.counts)-1] = uint64(in.heapStats.largeFreeCount)
   106  				// Cut off the first index which is ostensibly for size class 0,
   107  				// but large objects are tracked separately so it's actually unused.
   108  				for i, count := range in.heapStats.smallFreeCount[1:] {
   109  					hist.counts[i] = uint64(count)
   110  				}
   111  			},
   112  		},
   113  		"/gc/heap/goal:bytes": {
   114  			deps: makeStatDepSet(sysStatsDep),
   115  			compute: func(in *statAggregate, out *metricValue) {
   116  				out.kind = metricKindUint64
   117  				out.scalar = in.sysStats.heapGoal
   118  			},
   119  		},
   120  		"/gc/heap/objects:objects": {
   121  			deps: makeStatDepSet(heapStatsDep),
   122  			compute: func(in *statAggregate, out *metricValue) {
   123  				out.kind = metricKindUint64
   124  				out.scalar = in.heapStats.numObjects
   125  			},
   126  		},
   127  		"/gc/pauses:seconds": {
   128  			compute: func(_ *statAggregate, out *metricValue) {
   129  				hist := out.float64HistOrInit(timeHistBuckets)
   130  				// The bottom-most bucket, containing negative values, is tracked
   131  				// as a separately as underflow, so fill that in manually and then
   132  				// iterate over the rest.
   133  				hist.counts[0] = atomic.Load64(&memstats.gcPauseDist.underflow)
   134  				for i := range memstats.gcPauseDist.counts {
   135  					hist.counts[i+1] = atomic.Load64(&memstats.gcPauseDist.counts[i])
   136  				}
   137  			},
   138  		},
   139  		"/memory/classes/heap/free:bytes": {
   140  			deps: makeStatDepSet(heapStatsDep),
   141  			compute: func(in *statAggregate, out *metricValue) {
   142  				out.kind = metricKindUint64
   143  				out.scalar = uint64(in.heapStats.committed - in.heapStats.inHeap -
   144  					in.heapStats.inStacks - in.heapStats.inWorkBufs -
   145  					in.heapStats.inPtrScalarBits)
   146  			},
   147  		},
   148  		"/memory/classes/heap/objects:bytes": {
   149  			deps: makeStatDepSet(heapStatsDep),
   150  			compute: func(in *statAggregate, out *metricValue) {
   151  				out.kind = metricKindUint64
   152  				out.scalar = in.heapStats.inObjects
   153  			},
   154  		},
   155  		"/memory/classes/heap/released:bytes": {
   156  			deps: makeStatDepSet(heapStatsDep),
   157  			compute: func(in *statAggregate, out *metricValue) {
   158  				out.kind = metricKindUint64
   159  				out.scalar = uint64(in.heapStats.released)
   160  			},
   161  		},
   162  		"/memory/classes/heap/stacks:bytes": {
   163  			deps: makeStatDepSet(heapStatsDep),
   164  			compute: func(in *statAggregate, out *metricValue) {
   165  				out.kind = metricKindUint64
   166  				out.scalar = uint64(in.heapStats.inStacks)
   167  			},
   168  		},
   169  		"/memory/classes/heap/unused:bytes": {
   170  			deps: makeStatDepSet(heapStatsDep),
   171  			compute: func(in *statAggregate, out *metricValue) {
   172  				out.kind = metricKindUint64
   173  				out.scalar = uint64(in.heapStats.inHeap) - in.heapStats.inObjects
   174  			},
   175  		},
   176  		"/memory/classes/metadata/mcache/free:bytes": {
   177  			deps: makeStatDepSet(sysStatsDep),
   178  			compute: func(in *statAggregate, out *metricValue) {
   179  				out.kind = metricKindUint64
   180  				out.scalar = in.sysStats.mCacheSys - in.sysStats.mCacheInUse
   181  			},
   182  		},
   183  		"/memory/classes/metadata/mcache/inuse:bytes": {
   184  			deps: makeStatDepSet(sysStatsDep),
   185  			compute: func(in *statAggregate, out *metricValue) {
   186  				out.kind = metricKindUint64
   187  				out.scalar = in.sysStats.mCacheInUse
   188  			},
   189  		},
   190  		"/memory/classes/metadata/mspan/free:bytes": {
   191  			deps: makeStatDepSet(sysStatsDep),
   192  			compute: func(in *statAggregate, out *metricValue) {
   193  				out.kind = metricKindUint64
   194  				out.scalar = in.sysStats.mSpanSys - in.sysStats.mSpanInUse
   195  			},
   196  		},
   197  		"/memory/classes/metadata/mspan/inuse:bytes": {
   198  			deps: makeStatDepSet(sysStatsDep),
   199  			compute: func(in *statAggregate, out *metricValue) {
   200  				out.kind = metricKindUint64
   201  				out.scalar = in.sysStats.mSpanInUse
   202  			},
   203  		},
   204  		"/memory/classes/metadata/other:bytes": {
   205  			deps: makeStatDepSet(heapStatsDep, sysStatsDep),
   206  			compute: func(in *statAggregate, out *metricValue) {
   207  				out.kind = metricKindUint64
   208  				out.scalar = uint64(in.heapStats.inWorkBufs+in.heapStats.inPtrScalarBits) + in.sysStats.gcMiscSys
   209  			},
   210  		},
   211  		"/memory/classes/os-stacks:bytes": {
   212  			deps: makeStatDepSet(sysStatsDep),
   213  			compute: func(in *statAggregate, out *metricValue) {
   214  				out.kind = metricKindUint64
   215  				out.scalar = in.sysStats.stacksSys
   216  			},
   217  		},
   218  		"/memory/classes/other:bytes": {
   219  			deps: makeStatDepSet(sysStatsDep),
   220  			compute: func(in *statAggregate, out *metricValue) {
   221  				out.kind = metricKindUint64
   222  				out.scalar = in.sysStats.otherSys
   223  			},
   224  		},
   225  		"/memory/classes/profiling/buckets:bytes": {
   226  			deps: makeStatDepSet(sysStatsDep),
   227  			compute: func(in *statAggregate, out *metricValue) {
   228  				out.kind = metricKindUint64
   229  				out.scalar = in.sysStats.buckHashSys
   230  			},
   231  		},
   232  		"/memory/classes/total:bytes": {
   233  			deps: makeStatDepSet(heapStatsDep, sysStatsDep),
   234  			compute: func(in *statAggregate, out *metricValue) {
   235  				out.kind = metricKindUint64
   236  				out.scalar = uint64(in.heapStats.committed+in.heapStats.released) +
   237  					in.sysStats.stacksSys + in.sysStats.mSpanSys +
   238  					in.sysStats.mCacheSys + in.sysStats.buckHashSys +
   239  					in.sysStats.gcMiscSys + in.sysStats.otherSys
   240  			},
   241  		},
   242  		"/sched/goroutines:goroutines": {
   243  			compute: func(_ *statAggregate, out *metricValue) {
   244  				out.kind = metricKindUint64
   245  				out.scalar = uint64(gcount())
   246  			},
   247  		},
   248  	}
   249  	metricsInit = true
   250  }
   251  
   252  // statDep is a dependency on a group of statistics
   253  // that a metric might have.
   254  type statDep uint
   255  
   256  const (
   257  	heapStatsDep statDep = iota // corresponds to heapStatsAggregate
   258  	sysStatsDep                 // corresponds to sysStatsAggregate
   259  	numStatsDeps
   260  )
   261  
   262  // statDepSet represents a set of statDeps.
   263  //
   264  // Under the hood, it's a bitmap.
   265  type statDepSet [1]uint64
   266  
   267  // makeStatDepSet creates a new statDepSet from a list of statDeps.
   268  func makeStatDepSet(deps ...statDep) statDepSet {
   269  	var s statDepSet
   270  	for _, d := range deps {
   271  		s[d/64] |= 1 << (d % 64)
   272  	}
   273  	return s
   274  }
   275  
   276  // differennce returns set difference of s from b as a new set.
   277  func (s statDepSet) difference(b statDepSet) statDepSet {
   278  	var c statDepSet
   279  	for i := range s {
   280  		c[i] = s[i] &^ b[i]
   281  	}
   282  	return c
   283  }
   284  
   285  // union returns the union of the two sets as a new set.
   286  func (s statDepSet) union(b statDepSet) statDepSet {
   287  	var c statDepSet
   288  	for i := range s {
   289  		c[i] = s[i] | b[i]
   290  	}
   291  	return c
   292  }
   293  
   294  // empty returns true if there are no dependencies in the set.
   295  func (s *statDepSet) empty() bool {
   296  	for _, c := range s {
   297  		if c != 0 {
   298  			return false
   299  		}
   300  	}
   301  	return true
   302  }
   303  
   304  // has returns true if the set contains a given statDep.
   305  func (s *statDepSet) has(d statDep) bool {
   306  	return s[d/64]&(1<<(d%64)) != 0
   307  }
   308  
   309  // heapStatsAggregate represents memory stats obtained from the
   310  // runtime. This set of stats is grouped together because they
   311  // depend on each other in some way to make sense of the runtime's
   312  // current heap memory use. They're also sharded across Ps, so it
   313  // makes sense to grab them all at once.
   314  type heapStatsAggregate struct {
   315  	heapStatsDelta
   316  
   317  	// Derived from values in heapStatsDelta.
   318  
   319  	// inObjects is the bytes of memory occupied by objects,
   320  	inObjects uint64
   321  
   322  	// numObjects is the number of live objects in the heap.
   323  	numObjects uint64
   324  }
   325  
   326  // compute populates the heapStatsAggregate with values from the runtime.
   327  func (a *heapStatsAggregate) compute() {
   328  	memstats.heapStats.read(&a.heapStatsDelta)
   329  
   330  	// Calculate derived stats.
   331  	a.inObjects = uint64(a.largeAlloc - a.largeFree)
   332  	a.numObjects = uint64(a.largeAllocCount - a.largeFreeCount)
   333  	for i := range a.smallAllocCount {
   334  		n := uint64(a.smallAllocCount[i] - a.smallFreeCount[i])
   335  		a.inObjects += n * uint64(class_to_size[i])
   336  		a.numObjects += n
   337  	}
   338  }
   339  
   340  // sysStatsAggregate represents system memory stats obtained
   341  // from the runtime. This set of stats is grouped together because
   342  // they're all relatively cheap to acquire and generally independent
   343  // of one another and other runtime memory stats. The fact that they
   344  // may be acquired at different times, especially with respect to
   345  // heapStatsAggregate, means there could be some skew, but because of
   346  // these stats are independent, there's no real consistency issue here.
   347  type sysStatsAggregate struct {
   348  	stacksSys      uint64
   349  	mSpanSys       uint64
   350  	mSpanInUse     uint64
   351  	mCacheSys      uint64
   352  	mCacheInUse    uint64
   353  	buckHashSys    uint64
   354  	gcMiscSys      uint64
   355  	otherSys       uint64
   356  	heapGoal       uint64
   357  	gcCyclesDone   uint64
   358  	gcCyclesForced uint64
   359  }
   360  
   361  // compute populates the sysStatsAggregate with values from the runtime.
   362  func (a *sysStatsAggregate) compute() {
   363  	a.stacksSys = memstats.stacks_sys.load()
   364  	a.buckHashSys = memstats.buckhash_sys.load()
   365  	a.gcMiscSys = memstats.gcMiscSys.load()
   366  	a.otherSys = memstats.other_sys.load()
   367  	a.heapGoal = atomic.Load64(&memstats.next_gc)
   368  	a.gcCyclesDone = uint64(memstats.numgc)
   369  	a.gcCyclesForced = uint64(memstats.numforcedgc)
   370  
   371  	systemstack(func() {
   372  		lock(&mheap_.lock)
   373  		a.mSpanSys = memstats.mspan_sys.load()
   374  		a.mSpanInUse = uint64(mheap_.spanalloc.inuse)
   375  		a.mCacheSys = memstats.mcache_sys.load()
   376  		a.mCacheInUse = uint64(mheap_.cachealloc.inuse)
   377  		unlock(&mheap_.lock)
   378  	})
   379  }
   380  
   381  // statAggregate is the main driver of the metrics implementation.
   382  //
   383  // It contains multiple aggregates of runtime statistics, as well
   384  // as a set of these aggregates that it has populated. The aggergates
   385  // are populated lazily by its ensure method.
   386  type statAggregate struct {
   387  	ensured   statDepSet
   388  	heapStats heapStatsAggregate
   389  	sysStats  sysStatsAggregate
   390  }
   391  
   392  // ensure populates statistics aggregates determined by deps if they
   393  // haven't yet been populated.
   394  func (a *statAggregate) ensure(deps *statDepSet) {
   395  	missing := deps.difference(a.ensured)
   396  	if missing.empty() {
   397  		return
   398  	}
   399  	for i := statDep(0); i < numStatsDeps; i++ {
   400  		if !missing.has(i) {
   401  			continue
   402  		}
   403  		switch i {
   404  		case heapStatsDep:
   405  			a.heapStats.compute()
   406  		case sysStatsDep:
   407  			a.sysStats.compute()
   408  		}
   409  	}
   410  	a.ensured = a.ensured.union(missing)
   411  }
   412  
   413  // metricValidKind is a runtime copy of runtime/metrics.ValueKind and
   414  // must be kept structurally identical to that type.
   415  type metricKind int
   416  
   417  const (
   418  	// These values must be kept identical to their corresponding Kind* values
   419  	// in the runtime/metrics package.
   420  	metricKindBad metricKind = iota
   421  	metricKindUint64
   422  	metricKindFloat64
   423  	metricKindFloat64Histogram
   424  )
   425  
   426  // metricSample is a runtime copy of runtime/metrics.Sample and
   427  // must be kept structurally identical to that type.
   428  type metricSample struct {
   429  	name  string
   430  	value metricValue
   431  }
   432  
   433  // metricValue is a runtime copy of runtime/metrics.Sample and
   434  // must be kept structurally identical to that type.
   435  type metricValue struct {
   436  	kind    metricKind
   437  	scalar  uint64         // contains scalar values for scalar Kinds.
   438  	pointer unsafe.Pointer // contains non-scalar values.
   439  }
   440  
   441  // float64HistOrInit tries to pull out an existing float64Histogram
   442  // from the value, but if none exists, then it allocates one with
   443  // the given buckets.
   444  func (v *metricValue) float64HistOrInit(buckets []float64) *metricFloat64Histogram {
   445  	var hist *metricFloat64Histogram
   446  	if v.kind == metricKindFloat64Histogram && v.pointer != nil {
   447  		hist = (*metricFloat64Histogram)(v.pointer)
   448  	} else {
   449  		v.kind = metricKindFloat64Histogram
   450  		hist = new(metricFloat64Histogram)
   451  		v.pointer = unsafe.Pointer(hist)
   452  	}
   453  	hist.buckets = buckets
   454  	if len(hist.counts) != len(hist.buckets)-1 {
   455  		hist.counts = make([]uint64, len(buckets)-1)
   456  	}
   457  	return hist
   458  }
   459  
   460  // metricFloat64Histogram is a runtime copy of runtime/metrics.Float64Histogram
   461  // and must be kept structurally identical to that type.
   462  type metricFloat64Histogram struct {
   463  	counts  []uint64
   464  	buckets []float64
   465  }
   466  
   467  // agg is used by readMetrics, and is protected by metricsSema.
   468  //
   469  // Managed as a global variable because its pointer will be
   470  // an argument to a dynamically-defined function, and we'd
   471  // like to avoid it escaping to the heap.
   472  var agg statAggregate
   473  
   474  // readMetrics is the implementation of runtime/metrics.Read.
   475  //
   476  //go:linkname readMetrics runtime/metrics.runtime_readMetrics
   477  func readMetrics(samplesp unsafe.Pointer, len int, cap int) {
   478  	// Construct a slice from the args.
   479  	sl := slice{samplesp, len, cap}
   480  	samples := *(*[]metricSample)(unsafe.Pointer(&sl))
   481  
   482  	// Acquire the metricsSema but with handoff. This operation
   483  	// is expensive enough that queueing up goroutines and handing
   484  	// off between them will be noticably better-behaved.
   485  	semacquire1(&metricsSema, true, 0, 0)
   486  
   487  	// Ensure the map is initialized.
   488  	initMetrics()
   489  
   490  	// Clear agg defensively.
   491  	agg = statAggregate{}
   492  
   493  	// Sample.
   494  	for i := range samples {
   495  		sample := &samples[i]
   496  		data, ok := metrics[sample.name]
   497  		if !ok {
   498  			sample.value.kind = metricKindBad
   499  			continue
   500  		}
   501  		// Ensure we have all the stats we need.
   502  		// agg is populated lazily.
   503  		agg.ensure(&data.deps)
   504  
   505  		// Compute the value based on the stats we have.
   506  		data.compute(&agg, &sample.value)
   507  	}
   508  
   509  	semrelease(&metricsSema)
   510  }
   511  

View as plain text