Source file src/runtime/mprof.go

Documentation: runtime

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Malloc profiling.
     6  // Patterned after tcmalloc's algorithms; shorter code.
     7  
     8  package runtime
     9  
    10  import (
    11  	"internal/abi"
    12  	"runtime/internal/atomic"
    13  	"unsafe"
    14  )
    15  
    16  // NOTE(rsc): Everything here could use cas if contention became an issue.
    17  var (
    18  	// profInsertLock protects changes to the start of all *bucket linked lists
    19  	profInsertLock mutex
    20  	// profBlockLock protects the contents of every blockRecord struct
    21  	profBlockLock mutex
    22  	// profMemActiveLock protects the active field of every memRecord struct
    23  	profMemActiveLock mutex
    24  	// profMemFutureLock is a set of locks that protect the respective elements
    25  	// of the future array of every memRecord struct
    26  	profMemFutureLock [len(memRecord{}.future)]mutex
    27  )
    28  
    29  // All memory allocations are local and do not escape outside of the profiler.
    30  // The profiler is forbidden from referring to garbage-collected memory.
    31  
    32  const (
    33  	// profile types
    34  	memProfile bucketType = 1 + iota
    35  	blockProfile
    36  	mutexProfile
    37  
    38  	// size of bucket hash table
    39  	buckHashSize = 179999
    40  
    41  	// max depth of stack to record in bucket
    42  	maxStack = 32
    43  )
    44  
    45  type bucketType int
    46  
    47  // A bucket holds per-call-stack profiling information.
    48  // The representation is a bit sleazy, inherited from C.
    49  // This struct defines the bucket header. It is followed in
    50  // memory by the stack words and then the actual record
    51  // data, either a memRecord or a blockRecord.
    52  //
    53  // Per-call-stack profiling information.
    54  // Lookup by hashing call stack into a linked-list hash table.
    55  //
    56  // None of the fields in this bucket header are modified after
    57  // creation, including its next and allnext links.
    58  //
    59  // No heap pointers.
    60  //
    61  //go:notinheap
    62  type bucket struct {
    63  	next    *bucket
    64  	allnext *bucket
    65  	typ     bucketType // memBucket or blockBucket (includes mutexProfile)
    66  	hash    uintptr
    67  	size    uintptr
    68  	nstk    uintptr
    69  }
    70  
    71  // A memRecord is the bucket data for a bucket of type memProfile,
    72  // part of the memory profile.
    73  type memRecord struct {
    74  	// The following complex 3-stage scheme of stats accumulation
    75  	// is required to obtain a consistent picture of mallocs and frees
    76  	// for some point in time.
    77  	// The problem is that mallocs come in real time, while frees
    78  	// come only after a GC during concurrent sweeping. So if we would
    79  	// naively count them, we would get a skew toward mallocs.
    80  	//
    81  	// Hence, we delay information to get consistent snapshots as
    82  	// of mark termination. Allocations count toward the next mark
    83  	// termination's snapshot, while sweep frees count toward the
    84  	// previous mark termination's snapshot:
    85  	//
    86  	//              MT          MT          MT          MT
    87  	//             .·|         .·|         .·|         .·|
    88  	//          .·˙  |      .·˙  |      .·˙  |      .·˙  |
    89  	//       .·˙     |   .·˙     |   .·˙     |   .·˙     |
    90  	//    .·˙        |.·˙        |.·˙        |.·˙        |
    91  	//
    92  	//       alloc → ▲ ← free
    93  	//               ┠┅┅┅┅┅┅┅┅┅┅┅P
    94  	//       C+2     →    C+1    →  C
    95  	//
    96  	//                   alloc → ▲ ← free
    97  	//                           ┠┅┅┅┅┅┅┅┅┅┅┅P
    98  	//                   C+2     →    C+1    →  C
    99  	//
   100  	// Since we can't publish a consistent snapshot until all of
   101  	// the sweep frees are accounted for, we wait until the next
   102  	// mark termination ("MT" above) to publish the previous mark
   103  	// termination's snapshot ("P" above). To do this, allocation
   104  	// and free events are accounted to *future* heap profile
   105  	// cycles ("C+n" above) and we only publish a cycle once all
   106  	// of the events from that cycle must be done. Specifically:
   107  	//
   108  	// Mallocs are accounted to cycle C+2.
   109  	// Explicit frees are accounted to cycle C+2.
   110  	// GC frees (done during sweeping) are accounted to cycle C+1.
   111  	//
   112  	// After mark termination, we increment the global heap
   113  	// profile cycle counter and accumulate the stats from cycle C
   114  	// into the active profile.
   115  
   116  	// active is the currently published profile. A profiling
   117  	// cycle can be accumulated into active once its complete.
   118  	active memRecordCycle
   119  
   120  	// future records the profile events we're counting for cycles
   121  	// that have not yet been published. This is ring buffer
   122  	// indexed by the global heap profile cycle C and stores
   123  	// cycles C, C+1, and C+2. Unlike active, these counts are
   124  	// only for a single cycle; they are not cumulative across
   125  	// cycles.
   126  	//
   127  	// We store cycle C here because there's a window between when
   128  	// C becomes the active cycle and when we've flushed it to
   129  	// active.
   130  	future [3]memRecordCycle
   131  }
   132  
   133  // memRecordCycle
   134  type memRecordCycle struct {
   135  	allocs, frees           uintptr
   136  	alloc_bytes, free_bytes uintptr
   137  }
   138  
   139  // add accumulates b into a. It does not zero b.
   140  func (a *memRecordCycle) add(b *memRecordCycle) {
   141  	a.allocs += b.allocs
   142  	a.frees += b.frees
   143  	a.alloc_bytes += b.alloc_bytes
   144  	a.free_bytes += b.free_bytes
   145  }
   146  
   147  // A blockRecord is the bucket data for a bucket of type blockProfile,
   148  // which is used in blocking and mutex profiles.
   149  type blockRecord struct {
   150  	count  float64
   151  	cycles int64
   152  }
   153  
   154  var (
   155  	mbuckets atomic.UnsafePointer // *bucket, memory profile buckets
   156  	bbuckets atomic.UnsafePointer // *bucket, blocking profile buckets
   157  	xbuckets atomic.UnsafePointer // *bucket, mutex profile buckets
   158  	buckhash atomic.UnsafePointer // *buckhashArray
   159  
   160  	mProfCycle mProfCycleHolder
   161  )
   162  
   163  type buckhashArray [buckHashSize]atomic.UnsafePointer // *bucket
   164  
   165  const mProfCycleWrap = uint32(len(memRecord{}.future)) * (2 << 24)
   166  
   167  // mProfCycleHolder holds the global heap profile cycle number (wrapped at
   168  // mProfCycleWrap, stored starting at bit 1), and a flag (stored at bit 0) to
   169  // indicate whether future[cycle] in all buckets has been queued to flush into
   170  // the active profile.
   171  type mProfCycleHolder struct {
   172  	value atomic.Uint32
   173  }
   174  
   175  // read returns the current cycle count.
   176  func (c *mProfCycleHolder) read() (cycle uint32) {
   177  	v := c.value.Load()
   178  	cycle = v >> 1
   179  	return cycle
   180  }
   181  
   182  // setFlushed sets the flushed flag. It returns the current cycle count and the
   183  // previous value of the flushed flag.
   184  func (c *mProfCycleHolder) setFlushed() (cycle uint32, alreadyFlushed bool) {
   185  	for {
   186  		prev := c.value.Load()
   187  		cycle = prev >> 1
   188  		alreadyFlushed = (prev & 0x1) != 0
   189  		next := prev | 0x1
   190  		if c.value.CompareAndSwap(prev, next) {
   191  			return cycle, alreadyFlushed
   192  		}
   193  	}
   194  }
   195  
   196  // increment increases the cycle count by one, wrapping the value at
   197  // mProfCycleWrap. It clears the flushed flag.
   198  func (c *mProfCycleHolder) increment() {
   199  	// We explicitly wrap mProfCycle rather than depending on
   200  	// uint wraparound because the memRecord.future ring does not
   201  	// itself wrap at a power of two.
   202  	for {
   203  		prev := c.value.Load()
   204  		cycle := prev >> 1
   205  		cycle = (cycle + 1) % mProfCycleWrap
   206  		next := cycle << 1
   207  		if c.value.CompareAndSwap(prev, next) {
   208  			break
   209  		}
   210  	}
   211  }
   212  
   213  // newBucket allocates a bucket with the given type and number of stack entries.
   214  func newBucket(typ bucketType, nstk int) *bucket {
   215  	size := unsafe.Sizeof(bucket{}) + uintptr(nstk)*unsafe.Sizeof(uintptr(0))
   216  	switch typ {
   217  	default:
   218  		throw("invalid profile bucket type")
   219  	case memProfile:
   220  		size += unsafe.Sizeof(memRecord{})
   221  	case blockProfile, mutexProfile:
   222  		size += unsafe.Sizeof(blockRecord{})
   223  	}
   224  
   225  	b := (*bucket)(persistentalloc(size, 0, &memstats.buckhash_sys))
   226  	b.typ = typ
   227  	b.nstk = uintptr(nstk)
   228  	return b
   229  }
   230  
   231  // stk returns the slice in b holding the stack.
   232  func (b *bucket) stk() []uintptr {
   233  	stk := (*[maxStack]uintptr)(add(unsafe.Pointer(b), unsafe.Sizeof(*b)))
   234  	return stk[:b.nstk:b.nstk]
   235  }
   236  
   237  // mp returns the memRecord associated with the memProfile bucket b.
   238  func (b *bucket) mp() *memRecord {
   239  	if b.typ != memProfile {
   240  		throw("bad use of bucket.mp")
   241  	}
   242  	data := add(unsafe.Pointer(b), unsafe.Sizeof(*b)+b.nstk*unsafe.Sizeof(uintptr(0)))
   243  	return (*memRecord)(data)
   244  }
   245  
   246  // bp returns the blockRecord associated with the blockProfile bucket b.
   247  func (b *bucket) bp() *blockRecord {
   248  	if b.typ != blockProfile && b.typ != mutexProfile {
   249  		throw("bad use of bucket.bp")
   250  	}
   251  	data := add(unsafe.Pointer(b), unsafe.Sizeof(*b)+b.nstk*unsafe.Sizeof(uintptr(0)))
   252  	return (*blockRecord)(data)
   253  }
   254  
   255  // Return the bucket for stk[0:nstk], allocating new bucket if needed.
   256  func stkbucket(typ bucketType, size uintptr, stk []uintptr, alloc bool) *bucket {
   257  	bh := (*buckhashArray)(buckhash.Load())
   258  	if bh == nil {
   259  		lock(&profInsertLock)
   260  		// check again under the lock
   261  		bh = (*buckhashArray)(buckhash.Load())
   262  		if bh == nil {
   263  			bh = (*buckhashArray)(sysAlloc(unsafe.Sizeof(buckhashArray{}), &memstats.buckhash_sys))
   264  			if bh == nil {
   265  				throw("runtime: cannot allocate memory")
   266  			}
   267  			buckhash.StoreNoWB(unsafe.Pointer(bh))
   268  		}
   269  		unlock(&profInsertLock)
   270  	}
   271  
   272  	// Hash stack.
   273  	var h uintptr
   274  	for _, pc := range stk {
   275  		h += pc
   276  		h += h << 10
   277  		h ^= h >> 6
   278  	}
   279  	// hash in size
   280  	h += size
   281  	h += h << 10
   282  	h ^= h >> 6
   283  	// finalize
   284  	h += h << 3
   285  	h ^= h >> 11
   286  
   287  	i := int(h % buckHashSize)
   288  	// first check optimistically, without the lock
   289  	for b := (*bucket)(bh[i].Load()); b != nil; b = b.next {
   290  		if b.typ == typ && b.hash == h && b.size == size && eqslice(b.stk(), stk) {
   291  			return b
   292  		}
   293  	}
   294  
   295  	if !alloc {
   296  		return nil
   297  	}
   298  
   299  	lock(&profInsertLock)
   300  	// check again under the insertion lock
   301  	for b := (*bucket)(bh[i].Load()); b != nil; b = b.next {
   302  		if b.typ == typ && b.hash == h && b.size == size && eqslice(b.stk(), stk) {
   303  			unlock(&profInsertLock)
   304  			return b
   305  		}
   306  	}
   307  
   308  	// Create new bucket.
   309  	b := newBucket(typ, len(stk))
   310  	copy(b.stk(), stk)
   311  	b.hash = h
   312  	b.size = size
   313  
   314  	var allnext *atomic.UnsafePointer
   315  	if typ == memProfile {
   316  		allnext = &mbuckets
   317  	} else if typ == mutexProfile {
   318  		allnext = &xbuckets
   319  	} else {
   320  		allnext = &bbuckets
   321  	}
   322  
   323  	b.next = (*bucket)(bh[i].Load())
   324  	b.allnext = (*bucket)(allnext.Load())
   325  
   326  	bh[i].StoreNoWB(unsafe.Pointer(b))
   327  	allnext.StoreNoWB(unsafe.Pointer(b))
   328  
   329  	unlock(&profInsertLock)
   330  	return b
   331  }
   332  
   333  func eqslice(x, y []uintptr) bool {
   334  	if len(x) != len(y) {
   335  		return false
   336  	}
   337  	for i, xi := range x {
   338  		if xi != y[i] {
   339  			return false
   340  		}
   341  	}
   342  	return true
   343  }
   344  
   345  // mProf_NextCycle publishes the next heap profile cycle and creates a
   346  // fresh heap profile cycle. This operation is fast and can be done
   347  // during STW. The caller must call mProf_Flush before calling
   348  // mProf_NextCycle again.
   349  //
   350  // This is called by mark termination during STW so allocations and
   351  // frees after the world is started again count towards a new heap
   352  // profiling cycle.
   353  func mProf_NextCycle() {
   354  	mProfCycle.increment()
   355  }
   356  
   357  // mProf_Flush flushes the events from the current heap profiling
   358  // cycle into the active profile. After this it is safe to start a new
   359  // heap profiling cycle with mProf_NextCycle.
   360  //
   361  // This is called by GC after mark termination starts the world. In
   362  // contrast with mProf_NextCycle, this is somewhat expensive, but safe
   363  // to do concurrently.
   364  func mProf_Flush() {
   365  	cycle, alreadyFlushed := mProfCycle.setFlushed()
   366  	if alreadyFlushed {
   367  		return
   368  	}
   369  
   370  	index := cycle % uint32(len(memRecord{}.future))
   371  	lock(&profMemActiveLock)
   372  	lock(&profMemFutureLock[index])
   373  	mProf_FlushLocked(index)
   374  	unlock(&profMemFutureLock[index])
   375  	unlock(&profMemActiveLock)
   376  }
   377  
   378  // mProf_FlushLocked flushes the events from the heap profiling cycle at index
   379  // into the active profile. The caller must hold the lock for the active profile
   380  // (profMemActiveLock) and for the profiling cycle at index
   381  // (profMemFutureLock[index]).
   382  func mProf_FlushLocked(index uint32) {
   383  	assertLockHeld(&profMemActiveLock)
   384  	assertLockHeld(&profMemFutureLock[index])
   385  	head := (*bucket)(mbuckets.Load())
   386  	for b := head; b != nil; b = b.allnext {
   387  		mp := b.mp()
   388  
   389  		// Flush cycle C into the published profile and clear
   390  		// it for reuse.
   391  		mpc := &mp.future[index]
   392  		mp.active.add(mpc)
   393  		*mpc = memRecordCycle{}
   394  	}
   395  }
   396  
   397  // mProf_PostSweep records that all sweep frees for this GC cycle have
   398  // completed. This has the effect of publishing the heap profile
   399  // snapshot as of the last mark termination without advancing the heap
   400  // profile cycle.
   401  func mProf_PostSweep() {
   402  	// Flush cycle C+1 to the active profile so everything as of
   403  	// the last mark termination becomes visible. *Don't* advance
   404  	// the cycle, since we're still accumulating allocs in cycle
   405  	// C+2, which have to become C+1 in the next mark termination
   406  	// and so on.
   407  	cycle := mProfCycle.read() + 1
   408  
   409  	index := cycle % uint32(len(memRecord{}.future))
   410  	lock(&profMemActiveLock)
   411  	lock(&profMemFutureLock[index])
   412  	mProf_FlushLocked(index)
   413  	unlock(&profMemFutureLock[index])
   414  	unlock(&profMemActiveLock)
   415  }
   416  
   417  // Called by malloc to record a profiled block.
   418  func mProf_Malloc(p unsafe.Pointer, size uintptr) {
   419  	var stk [maxStack]uintptr
   420  	nstk := callers(4, stk[:])
   421  
   422  	index := (mProfCycle.read() + 2) % uint32(len(memRecord{}.future))
   423  
   424  	b := stkbucket(memProfile, size, stk[:nstk], true)
   425  	mp := b.mp()
   426  	mpc := &mp.future[index]
   427  
   428  	lock(&profMemFutureLock[index])
   429  	mpc.allocs++
   430  	mpc.alloc_bytes += size
   431  	unlock(&profMemFutureLock[index])
   432  
   433  	// Setprofilebucket locks a bunch of other mutexes, so we call it outside of
   434  	// the profiler locks. This reduces potential contention and chances of
   435  	// deadlocks. Since the object must be alive during the call to
   436  	// mProf_Malloc, it's fine to do this non-atomically.
   437  	systemstack(func() {
   438  		setprofilebucket(p, b)
   439  	})
   440  }
   441  
   442  // Called when freeing a profiled block.
   443  func mProf_Free(b *bucket, size uintptr) {
   444  	index := (mProfCycle.read() + 1) % uint32(len(memRecord{}.future))
   445  
   446  	mp := b.mp()
   447  	mpc := &mp.future[index]
   448  
   449  	lock(&profMemFutureLock[index])
   450  	mpc.frees++
   451  	mpc.free_bytes += size
   452  	unlock(&profMemFutureLock[index])
   453  }
   454  
   455  var blockprofilerate uint64 // in CPU ticks
   456  
   457  // SetBlockProfileRate controls the fraction of goroutine blocking events
   458  // that are reported in the blocking profile. The profiler aims to sample
   459  // an average of one blocking event per rate nanoseconds spent blocked.
   460  //
   461  // To include every blocking event in the profile, pass rate = 1.
   462  // To turn off profiling entirely, pass rate <= 0.
   463  func SetBlockProfileRate(rate int) {
   464  	var r int64
   465  	if rate <= 0 {
   466  		r = 0 // disable profiling
   467  	} else if rate == 1 {
   468  		r = 1 // profile everything
   469  	} else {
   470  		// convert ns to cycles, use float64 to prevent overflow during multiplication
   471  		r = int64(float64(rate) * float64(tickspersecond()) / (1000 * 1000 * 1000))
   472  		if r == 0 {
   473  			r = 1
   474  		}
   475  	}
   476  
   477  	atomic.Store64(&blockprofilerate, uint64(r))
   478  }
   479  
   480  func blockevent(cycles int64, skip int) {
   481  	if cycles <= 0 {
   482  		cycles = 1
   483  	}
   484  
   485  	rate := int64(atomic.Load64(&blockprofilerate))
   486  	if blocksampled(cycles, rate) {
   487  		saveblockevent(cycles, rate, skip+1, blockProfile)
   488  	}
   489  }
   490  
   491  // blocksampled returns true for all events where cycles >= rate. Shorter
   492  // events have a cycles/rate random chance of returning true.
   493  func blocksampled(cycles, rate int64) bool {
   494  	if rate <= 0 || (rate > cycles && int64(fastrand())%rate > cycles) {
   495  		return false
   496  	}
   497  	return true
   498  }
   499  
   500  func saveblockevent(cycles, rate int64, skip int, which bucketType) {
   501  	gp := getg()
   502  	var nstk int
   503  	var stk [maxStack]uintptr
   504  	if gp.m.curg == nil || gp.m.curg == gp {
   505  		nstk = callers(skip, stk[:])
   506  	} else {
   507  		nstk = gcallers(gp.m.curg, skip, stk[:])
   508  	}
   509  	b := stkbucket(which, 0, stk[:nstk], true)
   510  	bp := b.bp()
   511  
   512  	lock(&profBlockLock)
   513  	if which == blockProfile && cycles < rate {
   514  		// Remove sampling bias, see discussion on http://golang.org/cl/299991.
   515  		bp.count += float64(rate) / float64(cycles)
   516  		bp.cycles += rate
   517  	} else {
   518  		bp.count++
   519  		bp.cycles += cycles
   520  	}
   521  	unlock(&profBlockLock)
   522  }
   523  
   524  var mutexprofilerate uint64 // fraction sampled
   525  
   526  // SetMutexProfileFraction controls the fraction of mutex contention events
   527  // that are reported in the mutex profile. On average 1/rate events are
   528  // reported. The previous rate is returned.
   529  //
   530  // To turn off profiling entirely, pass rate 0.
   531  // To just read the current rate, pass rate < 0.
   532  // (For n>1 the details of sampling may change.)
   533  func SetMutexProfileFraction(rate int) int {
   534  	if rate < 0 {
   535  		return int(mutexprofilerate)
   536  	}
   537  	old := mutexprofilerate
   538  	atomic.Store64(&mutexprofilerate, uint64(rate))
   539  	return int(old)
   540  }
   541  
   542  //go:linkname mutexevent sync.event
   543  func mutexevent(cycles int64, skip int) {
   544  	if cycles < 0 {
   545  		cycles = 0
   546  	}
   547  	rate := int64(atomic.Load64(&mutexprofilerate))
   548  	// TODO(pjw): measure impact of always calling fastrand vs using something
   549  	// like malloc.go:nextSample()
   550  	if rate > 0 && int64(fastrand())%rate == 0 {
   551  		saveblockevent(cycles, rate, skip+1, mutexProfile)
   552  	}
   553  }
   554  
   555  // Go interface to profile data.
   556  
   557  // A StackRecord describes a single execution stack.
   558  type StackRecord struct {
   559  	Stack0 [32]uintptr // stack trace for this record; ends at first 0 entry
   560  }
   561  
   562  // Stack returns the stack trace associated with the record,
   563  // a prefix of r.Stack0.
   564  func (r *StackRecord) Stack() []uintptr {
   565  	for i, v := range r.Stack0 {
   566  		if v == 0 {
   567  			return r.Stack0[0:i]
   568  		}
   569  	}
   570  	return r.Stack0[0:]
   571  }
   572  
   573  // MemProfileRate controls the fraction of memory allocations
   574  // that are recorded and reported in the memory profile.
   575  // The profiler aims to sample an average of
   576  // one allocation per MemProfileRate bytes allocated.
   577  //
   578  // To include every allocated block in the profile, set MemProfileRate to 1.
   579  // To turn off profiling entirely, set MemProfileRate to 0.
   580  //
   581  // The tools that process the memory profiles assume that the
   582  // profile rate is constant across the lifetime of the program
   583  // and equal to the current value. Programs that change the
   584  // memory profiling rate should do so just once, as early as
   585  // possible in the execution of the program (for example,
   586  // at the beginning of main).
   587  var MemProfileRate int = defaultMemProfileRate(512 * 1024)
   588  
   589  // defaultMemProfileRate returns 0 if disableMemoryProfiling is set.
   590  // It exists primarily for the godoc rendering of MemProfileRate
   591  // above.
   592  func defaultMemProfileRate(v int) int {
   593  	if disableMemoryProfiling {
   594  		return 0
   595  	}
   596  	return v
   597  }
   598  
   599  // disableMemoryProfiling is set by the linker if runtime.MemProfile
   600  // is not used and the link type guarantees nobody else could use it
   601  // elsewhere.
   602  var disableMemoryProfiling bool
   603  
   604  // A MemProfileRecord describes the live objects allocated
   605  // by a particular call sequence (stack trace).
   606  type MemProfileRecord struct {
   607  	AllocBytes, FreeBytes     int64       // number of bytes allocated, freed
   608  	AllocObjects, FreeObjects int64       // number of objects allocated, freed
   609  	Stack0                    [32]uintptr // stack trace for this record; ends at first 0 entry
   610  }
   611  
   612  // InUseBytes returns the number of bytes in use (AllocBytes - FreeBytes).
   613  func (r *MemProfileRecord) InUseBytes() int64 { return r.AllocBytes - r.FreeBytes }
   614  
   615  // InUseObjects returns the number of objects in use (AllocObjects - FreeObjects).
   616  func (r *MemProfileRecord) InUseObjects() int64 {
   617  	return r.AllocObjects - r.FreeObjects
   618  }
   619  
   620  // Stack returns the stack trace associated with the record,
   621  // a prefix of r.Stack0.
   622  func (r *MemProfileRecord) Stack() []uintptr {
   623  	for i, v := range r.Stack0 {
   624  		if v == 0 {
   625  			return r.Stack0[0:i]
   626  		}
   627  	}
   628  	return r.Stack0[0:]
   629  }
   630  
   631  // MemProfile returns a profile of memory allocated and freed per allocation
   632  // site.
   633  //
   634  // MemProfile returns n, the number of records in the current memory profile.
   635  // If len(p) >= n, MemProfile copies the profile into p and returns n, true.
   636  // If len(p) < n, MemProfile does not change p and returns n, false.
   637  //
   638  // If inuseZero is true, the profile includes allocation records
   639  // where r.AllocBytes > 0 but r.AllocBytes == r.FreeBytes.
   640  // These are sites where memory was allocated, but it has all
   641  // been released back to the runtime.
   642  //
   643  // The returned profile may be up to two garbage collection cycles old.
   644  // This is to avoid skewing the profile toward allocations; because
   645  // allocations happen in real time but frees are delayed until the garbage
   646  // collector performs sweeping, the profile only accounts for allocations
   647  // that have had a chance to be freed by the garbage collector.
   648  //
   649  // Most clients should use the runtime/pprof package or
   650  // the testing package's -test.memprofile flag instead
   651  // of calling MemProfile directly.
   652  func MemProfile(p []MemProfileRecord, inuseZero bool) (n int, ok bool) {
   653  	cycle := mProfCycle.read()
   654  	// If we're between mProf_NextCycle and mProf_Flush, take care
   655  	// of flushing to the active profile so we only have to look
   656  	// at the active profile below.
   657  	index := cycle % uint32(len(memRecord{}.future))
   658  	lock(&profMemActiveLock)
   659  	lock(&profMemFutureLock[index])
   660  	mProf_FlushLocked(index)
   661  	unlock(&profMemFutureLock[index])
   662  	clear := true
   663  	head := (*bucket)(mbuckets.Load())
   664  	for b := head; b != nil; b = b.allnext {
   665  		mp := b.mp()
   666  		if inuseZero || mp.active.alloc_bytes != mp.active.free_bytes {
   667  			n++
   668  		}
   669  		if mp.active.allocs != 0 || mp.active.frees != 0 {
   670  			clear = false
   671  		}
   672  	}
   673  	if clear {
   674  		// Absolutely no data, suggesting that a garbage collection
   675  		// has not yet happened. In order to allow profiling when
   676  		// garbage collection is disabled from the beginning of execution,
   677  		// accumulate all of the cycles, and recount buckets.
   678  		n = 0
   679  		for b := head; b != nil; b = b.allnext {
   680  			mp := b.mp()
   681  			for c := range mp.future {
   682  				lock(&profMemFutureLock[c])
   683  				mp.active.add(&mp.future[c])
   684  				mp.future[c] = memRecordCycle{}
   685  				unlock(&profMemFutureLock[c])
   686  			}
   687  			if inuseZero || mp.active.alloc_bytes != mp.active.free_bytes {
   688  				n++
   689  			}
   690  		}
   691  	}
   692  	if n <= len(p) {
   693  		ok = true
   694  		idx := 0
   695  		for b := head; b != nil; b = b.allnext {
   696  			mp := b.mp()
   697  			if inuseZero || mp.active.alloc_bytes != mp.active.free_bytes {
   698  				record(&p[idx], b)
   699  				idx++
   700  			}
   701  		}
   702  	}
   703  	unlock(&profMemActiveLock)
   704  	return
   705  }
   706  
   707  // Write b's data to r.
   708  func record(r *MemProfileRecord, b *bucket) {
   709  	mp := b.mp()
   710  	r.AllocBytes = int64(mp.active.alloc_bytes)
   711  	r.FreeBytes = int64(mp.active.free_bytes)
   712  	r.AllocObjects = int64(mp.active.allocs)
   713  	r.FreeObjects = int64(mp.active.frees)
   714  	if raceenabled {
   715  		racewriterangepc(unsafe.Pointer(&r.Stack0[0]), unsafe.Sizeof(r.Stack0), getcallerpc(), abi.FuncPCABIInternal(MemProfile))
   716  	}
   717  	if msanenabled {
   718  		msanwrite(unsafe.Pointer(&r.Stack0[0]), unsafe.Sizeof(r.Stack0))
   719  	}
   720  	if asanenabled {
   721  		asanwrite(unsafe.Pointer(&r.Stack0[0]), unsafe.Sizeof(r.Stack0))
   722  	}
   723  	copy(r.Stack0[:], b.stk())
   724  	for i := int(b.nstk); i < len(r.Stack0); i++ {
   725  		r.Stack0[i] = 0
   726  	}
   727  }
   728  
   729  func iterate_memprof(fn func(*bucket, uintptr, *uintptr, uintptr, uintptr, uintptr)) {
   730  	lock(&profMemActiveLock)
   731  	head := (*bucket)(mbuckets.Load())
   732  	for b := head; b != nil; b = b.allnext {
   733  		mp := b.mp()
   734  		fn(b, b.nstk, &b.stk()[0], b.size, mp.active.allocs, mp.active.frees)
   735  	}
   736  	unlock(&profMemActiveLock)
   737  }
   738  
   739  // BlockProfileRecord describes blocking events originated
   740  // at a particular call sequence (stack trace).
   741  type BlockProfileRecord struct {
   742  	Count  int64
   743  	Cycles int64
   744  	StackRecord
   745  }
   746  
   747  // BlockProfile returns n, the number of records in the current blocking profile.
   748  // If len(p) >= n, BlockProfile copies the profile into p and returns n, true.
   749  // If len(p) < n, BlockProfile does not change p and returns n, false.
   750  //
   751  // Most clients should use the runtime/pprof package or
   752  // the testing package's -test.blockprofile flag instead
   753  // of calling BlockProfile directly.
   754  func BlockProfile(p []BlockProfileRecord) (n int, ok bool) {
   755  	lock(&profBlockLock)
   756  	head := (*bucket)(bbuckets.Load())
   757  	for b := head; b != nil; b = b.allnext {
   758  		n++
   759  	}
   760  	if n <= len(p) {
   761  		ok = true
   762  		for b := head; b != nil; b = b.allnext {
   763  			bp := b.bp()
   764  			r := &p[0]
   765  			r.Count = int64(bp.count)
   766  			// Prevent callers from having to worry about division by zero errors.
   767  			// See discussion on http://golang.org/cl/299991.
   768  			if r.Count == 0 {
   769  				r.Count = 1
   770  			}
   771  			r.Cycles = bp.cycles
   772  			if raceenabled {
   773  				racewriterangepc(unsafe.Pointer(&r.Stack0[0]), unsafe.Sizeof(r.Stack0), getcallerpc(), abi.FuncPCABIInternal(BlockProfile))
   774  			}
   775  			if msanenabled {
   776  				msanwrite(unsafe.Pointer(&r.Stack0[0]), unsafe.Sizeof(r.Stack0))
   777  			}
   778  			if asanenabled {
   779  				asanwrite(unsafe.Pointer(&r.Stack0[0]), unsafe.Sizeof(r.Stack0))
   780  			}
   781  			i := copy(r.Stack0[:], b.stk())
   782  			for ; i < len(r.Stack0); i++ {
   783  				r.Stack0[i] = 0
   784  			}
   785  			p = p[1:]
   786  		}
   787  	}
   788  	unlock(&profBlockLock)
   789  	return
   790  }
   791  
   792  // MutexProfile returns n, the number of records in the current mutex profile.
   793  // If len(p) >= n, MutexProfile copies the profile into p and returns n, true.
   794  // Otherwise, MutexProfile does not change p, and returns n, false.
   795  //
   796  // Most clients should use the runtime/pprof package
   797  // instead of calling MutexProfile directly.
   798  func MutexProfile(p []BlockProfileRecord) (n int, ok bool) {
   799  	lock(&profBlockLock)
   800  	head := (*bucket)(xbuckets.Load())
   801  	for b := head; b != nil; b = b.allnext {
   802  		n++
   803  	}
   804  	if n <= len(p) {
   805  		ok = true
   806  		for b := head; b != nil; b = b.allnext {
   807  			bp := b.bp()
   808  			r := &p[0]
   809  			r.Count = int64(bp.count)
   810  			r.Cycles = bp.cycles
   811  			i := copy(r.Stack0[:], b.stk())
   812  			for ; i < len(r.Stack0); i++ {
   813  				r.Stack0[i] = 0
   814  			}
   815  			p = p[1:]
   816  		}
   817  	}
   818  	unlock(&profBlockLock)
   819  	return
   820  }
   821  
   822  // ThreadCreateProfile returns n, the number of records in the thread creation profile.
   823  // If len(p) >= n, ThreadCreateProfile copies the profile into p and returns n, true.
   824  // If len(p) < n, ThreadCreateProfile does not change p and returns n, false.
   825  //
   826  // Most clients should use the runtime/pprof package instead
   827  // of calling ThreadCreateProfile directly.
   828  func ThreadCreateProfile(p []StackRecord) (n int, ok bool) {
   829  	first := (*m)(atomic.Loadp(unsafe.Pointer(&allm)))
   830  	for mp := first; mp != nil; mp = mp.alllink {
   831  		n++
   832  	}
   833  	if n <= len(p) {
   834  		ok = true
   835  		i := 0
   836  		for mp := first; mp != nil; mp = mp.alllink {
   837  			p[i].Stack0 = mp.createstack
   838  			i++
   839  		}
   840  	}
   841  	return
   842  }
   843  
   844  //go:linkname runtime_goroutineProfileWithLabels runtime/pprof.runtime_goroutineProfileWithLabels
   845  func runtime_goroutineProfileWithLabels(p []StackRecord, labels []unsafe.Pointer) (n int, ok bool) {
   846  	return goroutineProfileWithLabels(p, labels)
   847  }
   848  
   849  const go119ConcurrentGoroutineProfile = true
   850  
   851  // labels may be nil. If labels is non-nil, it must have the same length as p.
   852  func goroutineProfileWithLabels(p []StackRecord, labels []unsafe.Pointer) (n int, ok bool) {
   853  	if labels != nil && len(labels) != len(p) {
   854  		labels = nil
   855  	}
   856  
   857  	if go119ConcurrentGoroutineProfile {
   858  		return goroutineProfileWithLabelsConcurrent(p, labels)
   859  	}
   860  	return goroutineProfileWithLabelsSync(p, labels)
   861  }
   862  
   863  var goroutineProfile = struct {
   864  	sema    uint32
   865  	active  bool
   866  	offset  atomic.Int64
   867  	records []StackRecord
   868  	labels  []unsafe.Pointer
   869  }{
   870  	sema: 1,
   871  }
   872  
   873  // goroutineProfileState indicates the status of a goroutine's stack for the
   874  // current in-progress goroutine profile. Goroutines' stacks are initially
   875  // "Absent" from the profile, and end up "Satisfied" by the time the profile is
   876  // complete. While a goroutine's stack is being captured, its
   877  // goroutineProfileState will be "InProgress" and it will not be able to run
   878  // until the capture completes and the state moves to "Satisfied".
   879  //
   880  // Some goroutines (the finalizer goroutine, which at various times can be
   881  // either a "system" or a "user" goroutine, and the goroutine that is
   882  // coordinating the profile, any goroutines created during the profile) move
   883  // directly to the "Satisfied" state.
   884  type goroutineProfileState uint32
   885  
   886  const (
   887  	goroutineProfileAbsent goroutineProfileState = iota
   888  	goroutineProfileInProgress
   889  	goroutineProfileSatisfied
   890  )
   891  
   892  type goroutineProfileStateHolder atomic.Uint32
   893  
   894  func (p *goroutineProfileStateHolder) Load() goroutineProfileState {
   895  	return goroutineProfileState((*atomic.Uint32)(p).Load())
   896  }
   897  
   898  func (p *goroutineProfileStateHolder) Store(value goroutineProfileState) {
   899  	(*atomic.Uint32)(p).Store(uint32(value))
   900  }
   901  
   902  func (p *goroutineProfileStateHolder) CompareAndSwap(old, new goroutineProfileState) bool {
   903  	return (*atomic.Uint32)(p).CompareAndSwap(uint32(old), uint32(new))
   904  }
   905  
   906  func goroutineProfileWithLabelsConcurrent(p []StackRecord, labels []unsafe.Pointer) (n int, ok bool) {
   907  	semacquire(&goroutineProfile.sema)
   908  
   909  	ourg := getg()
   910  
   911  	stopTheWorld("profile")
   912  	// Using gcount while the world is stopped should give us a consistent view
   913  	// of the number of live goroutines, minus the number of goroutines that are
   914  	// alive and permanently marked as "system". But to make this count agree
   915  	// with what we'd get from isSystemGoroutine, we need special handling for
   916  	// goroutines that can vary between user and system to ensure that the count
   917  	// doesn't change during the collection. So, check the finalizer goroutine
   918  	// in particular.
   919  	n = int(gcount())
   920  	if fingRunning {
   921  		n++
   922  	}
   923  
   924  	if n > len(p) {
   925  		// There's not enough space in p to store the whole profile, so (per the
   926  		// contract of runtime.GoroutineProfile) we're not allowed to write to p
   927  		// at all and must return n, false.
   928  		startTheWorld()
   929  		semrelease(&goroutineProfile.sema)
   930  		return n, false
   931  	}
   932  
   933  	// Save current goroutine.
   934  	sp := getcallersp()
   935  	pc := getcallerpc()
   936  	systemstack(func() {
   937  		saveg(pc, sp, ourg, &p[0])
   938  	})
   939  	ourg.goroutineProfiled.Store(goroutineProfileSatisfied)
   940  	goroutineProfile.offset.Store(1)
   941  
   942  	// Prepare for all other goroutines to enter the profile. Aside from ourg,
   943  	// every goroutine struct in the allgs list has its goroutineProfiled field
   944  	// cleared. Any goroutine created from this point on (while
   945  	// goroutineProfile.active is set) will start with its goroutineProfiled
   946  	// field set to goroutineProfileSatisfied.
   947  	goroutineProfile.active = true
   948  	goroutineProfile.records = p
   949  	goroutineProfile.labels = labels
   950  	// The finalizer goroutine needs special handling because it can vary over
   951  	// time between being a user goroutine (eligible for this profile) and a
   952  	// system goroutine (to be excluded). Pick one before restarting the world.
   953  	if fing != nil {
   954  		fing.goroutineProfiled.Store(goroutineProfileSatisfied)
   955  		if readgstatus(fing) != _Gdead && !isSystemGoroutine(fing, false) {
   956  			doRecordGoroutineProfile(fing)
   957  		}
   958  	}
   959  	startTheWorld()
   960  
   961  	// Visit each goroutine that existed as of the startTheWorld call above.
   962  	//
   963  	// New goroutines may not be in this list, but we didn't want to know about
   964  	// them anyway. If they do appear in this list (via reusing a dead goroutine
   965  	// struct, or racing to launch between the world restarting and us getting
   966  	// the list), they will already have their goroutineProfiled field set to
   967  	// goroutineProfileSatisfied before their state transitions out of _Gdead.
   968  	//
   969  	// Any goroutine that the scheduler tries to execute concurrently with this
   970  	// call will start by adding itself to the profile (before the act of
   971  	// executing can cause any changes in its stack).
   972  	forEachGRace(func(gp1 *g) {
   973  		tryRecordGoroutineProfile(gp1, Gosched)
   974  	})
   975  
   976  	stopTheWorld("profile cleanup")
   977  	endOffset := goroutineProfile.offset.Swap(0)
   978  	goroutineProfile.active = false
   979  	goroutineProfile.records = nil
   980  	goroutineProfile.labels = nil
   981  	startTheWorld()
   982  
   983  	// Restore the invariant that every goroutine struct in allgs has its
   984  	// goroutineProfiled field cleared.
   985  	forEachGRace(func(gp1 *g) {
   986  		gp1.goroutineProfiled.Store(goroutineProfileAbsent)
   987  	})
   988  
   989  	if raceenabled {
   990  		raceacquire(unsafe.Pointer(&labelSync))
   991  	}
   992  
   993  	if n != int(endOffset) {
   994  		// It's a big surprise that the number of goroutines changed while we
   995  		// were collecting the profile. But probably better to return a
   996  		// truncated profile than to crash the whole process.
   997  		//
   998  		// For instance, needm moves a goroutine out of the _Gdead state and so
   999  		// might be able to change the goroutine count without interacting with
  1000  		// the scheduler. For code like that, the race windows are small and the
  1001  		// combination of features is uncommon, so it's hard to be (and remain)
  1002  		// sure we've caught them all.
  1003  	}
  1004  
  1005  	semrelease(&goroutineProfile.sema)
  1006  	return n, true
  1007  }
  1008  
  1009  // tryRecordGoroutineProfileWB asserts that write barriers are allowed and calls
  1010  // tryRecordGoroutineProfile.
  1011  //
  1012  //go:yeswritebarrierrec
  1013  func tryRecordGoroutineProfileWB(gp1 *g) {
  1014  	if getg().m.p.ptr() == nil {
  1015  		throw("no P available, write barriers are forbidden")
  1016  	}
  1017  	tryRecordGoroutineProfile(gp1, osyield)
  1018  }
  1019  
  1020  // tryRecordGoroutineProfile ensures that gp1 has the appropriate representation
  1021  // in the current goroutine profile: either that it should not be profiled, or
  1022  // that a snapshot of its call stack and labels are now in the profile.
  1023  func tryRecordGoroutineProfile(gp1 *g, yield func()) {
  1024  	if readgstatus(gp1) == _Gdead {
  1025  		// Dead goroutines should not appear in the profile. Goroutines that
  1026  		// start while profile collection is active will get goroutineProfiled
  1027  		// set to goroutineProfileSatisfied before transitioning out of _Gdead,
  1028  		// so here we check _Gdead first.
  1029  		return
  1030  	}
  1031  	if isSystemGoroutine(gp1, true) {
  1032  		// System goroutines should not appear in the profile. (The finalizer
  1033  		// goroutine is marked as "already profiled".)
  1034  		return
  1035  	}
  1036  
  1037  	for {
  1038  		prev := gp1.goroutineProfiled.Load()
  1039  		if prev == goroutineProfileSatisfied {
  1040  			// This goroutine is already in the profile (or is new since the
  1041  			// start of collection, so shouldn't appear in the profile).
  1042  			break
  1043  		}
  1044  		if prev == goroutineProfileInProgress {
  1045  			// Something else is adding gp1 to the goroutine profile right now.
  1046  			// Give that a moment to finish.
  1047  			yield()
  1048  			continue
  1049  		}
  1050  
  1051  		// While we have gp1.goroutineProfiled set to
  1052  		// goroutineProfileInProgress, gp1 may appear _Grunnable but will not
  1053  		// actually be able to run. Disable preemption for ourselves, to make
  1054  		// sure we finish profiling gp1 right away instead of leaving it stuck
  1055  		// in this limbo.
  1056  		mp := acquirem()
  1057  		if gp1.goroutineProfiled.CompareAndSwap(goroutineProfileAbsent, goroutineProfileInProgress) {
  1058  			doRecordGoroutineProfile(gp1)
  1059  			gp1.goroutineProfiled.Store(goroutineProfileSatisfied)
  1060  		}
  1061  		releasem(mp)
  1062  	}
  1063  }
  1064  
  1065  // doRecordGoroutineProfile writes gp1's call stack and labels to an in-progress
  1066  // goroutine profile. Preemption is disabled.
  1067  //
  1068  // This may be called via tryRecordGoroutineProfile in two ways: by the
  1069  // goroutine that is coordinating the goroutine profile (running on its own
  1070  // stack), or from the scheduler in preparation to execute gp1 (running on the
  1071  // system stack).
  1072  func doRecordGoroutineProfile(gp1 *g) {
  1073  	if readgstatus(gp1) == _Grunning {
  1074  		print("doRecordGoroutineProfile gp1=", gp1.goid, "\n")
  1075  		throw("cannot read stack of running goroutine")
  1076  	}
  1077  
  1078  	offset := int(goroutineProfile.offset.Add(1)) - 1
  1079  
  1080  	if offset >= len(goroutineProfile.records) {
  1081  		// Should be impossible, but better to return a truncated profile than
  1082  		// to crash the entire process at this point. Instead, deal with it in
  1083  		// goroutineProfileWithLabelsConcurrent where we have more context.
  1084  		return
  1085  	}
  1086  
  1087  	// saveg calls gentraceback, which may call cgo traceback functions. When
  1088  	// called from the scheduler, this is on the system stack already so
  1089  	// traceback.go:cgoContextPCs will avoid calling back into the scheduler.
  1090  	//
  1091  	// When called from the goroutine coordinating the profile, we still have
  1092  	// set gp1.goroutineProfiled to goroutineProfileInProgress and so are still
  1093  	// preventing it from being truly _Grunnable. So we'll use the system stack
  1094  	// to avoid schedule delays.
  1095  	systemstack(func() { saveg(^uintptr(0), ^uintptr(0), gp1, &goroutineProfile.records[offset]) })
  1096  
  1097  	if goroutineProfile.labels != nil {
  1098  		goroutineProfile.labels[offset] = gp1.labels
  1099  	}
  1100  }
  1101  
  1102  func goroutineProfileWithLabelsSync(p []StackRecord, labels []unsafe.Pointer) (n int, ok bool) {
  1103  	gp := getg()
  1104  
  1105  	isOK := func(gp1 *g) bool {
  1106  		// Checking isSystemGoroutine here makes GoroutineProfile
  1107  		// consistent with both NumGoroutine and Stack.
  1108  		return gp1 != gp && readgstatus(gp1) != _Gdead && !isSystemGoroutine(gp1, false)
  1109  	}
  1110  
  1111  	stopTheWorld("profile")
  1112  
  1113  	// World is stopped, no locking required.
  1114  	n = 1
  1115  	forEachGRace(func(gp1 *g) {
  1116  		if isOK(gp1) {
  1117  			n++
  1118  		}
  1119  	})
  1120  
  1121  	if n <= len(p) {
  1122  		ok = true
  1123  		r, lbl := p, labels
  1124  
  1125  		// Save current goroutine.
  1126  		sp := getcallersp()
  1127  		pc := getcallerpc()
  1128  		systemstack(func() {
  1129  			saveg(pc, sp, gp, &r[0])
  1130  		})
  1131  		r = r[1:]
  1132  
  1133  		// If we have a place to put our goroutine labelmap, insert it there.
  1134  		if labels != nil {
  1135  			lbl[0] = gp.labels
  1136  			lbl = lbl[1:]
  1137  		}
  1138  
  1139  		// Save other goroutines.
  1140  		forEachGRace(func(gp1 *g) {
  1141  			if !isOK(gp1) {
  1142  				return
  1143  			}
  1144  
  1145  			if len(r) == 0 {
  1146  				// Should be impossible, but better to return a
  1147  				// truncated profile than to crash the entire process.
  1148  				return
  1149  			}
  1150  			// saveg calls gentraceback, which may call cgo traceback functions.
  1151  			// The world is stopped, so it cannot use cgocall (which will be
  1152  			// blocked at exitsyscall). Do it on the system stack so it won't
  1153  			// call into the schedular (see traceback.go:cgoContextPCs).
  1154  			systemstack(func() { saveg(^uintptr(0), ^uintptr(0), gp1, &r[0]) })
  1155  			if labels != nil {
  1156  				lbl[0] = gp1.labels
  1157  				lbl = lbl[1:]
  1158  			}
  1159  			r = r[1:]
  1160  		})
  1161  	}
  1162  
  1163  	if raceenabled {
  1164  		raceacquire(unsafe.Pointer(&labelSync))
  1165  	}
  1166  
  1167  	startTheWorld()
  1168  	return n, ok
  1169  }
  1170  
  1171  // GoroutineProfile returns n, the number of records in the active goroutine stack profile.
  1172  // If len(p) >= n, GoroutineProfile copies the profile into p and returns n, true.
  1173  // If len(p) < n, GoroutineProfile does not change p and returns n, false.
  1174  //
  1175  // Most clients should use the runtime/pprof package instead
  1176  // of calling GoroutineProfile directly.
  1177  func GoroutineProfile(p []StackRecord) (n int, ok bool) {
  1178  
  1179  	return goroutineProfileWithLabels(p, nil)
  1180  }
  1181  
  1182  func saveg(pc, sp uintptr, gp *g, r *StackRecord) {
  1183  	n := gentraceback(pc, sp, 0, gp, 0, &r.Stack0[0], len(r.Stack0), nil, nil, 0)
  1184  	if n < len(r.Stack0) {
  1185  		r.Stack0[n] = 0
  1186  	}
  1187  }
  1188  
  1189  // Stack formats a stack trace of the calling goroutine into buf
  1190  // and returns the number of bytes written to buf.
  1191  // If all is true, Stack formats stack traces of all other goroutines
  1192  // into buf after the trace for the current goroutine.
  1193  func Stack(buf []byte, all bool) int {
  1194  	if all {
  1195  		stopTheWorld("stack trace")
  1196  	}
  1197  
  1198  	n := 0
  1199  	if len(buf) > 0 {
  1200  		gp := getg()
  1201  		sp := getcallersp()
  1202  		pc := getcallerpc()
  1203  		systemstack(func() {
  1204  			g0 := getg()
  1205  			// Force traceback=1 to override GOTRACEBACK setting,
  1206  			// so that Stack's results are consistent.
  1207  			// GOTRACEBACK is only about crash dumps.
  1208  			g0.m.traceback = 1
  1209  			g0.writebuf = buf[0:0:len(buf)]
  1210  			goroutineheader(gp)
  1211  			traceback(pc, sp, 0, gp)
  1212  			if all {
  1213  				tracebackothers(gp)
  1214  			}
  1215  			g0.m.traceback = 0
  1216  			n = len(g0.writebuf)
  1217  			g0.writebuf = nil
  1218  		})
  1219  	}
  1220  
  1221  	if all {
  1222  		startTheWorld()
  1223  	}
  1224  	return n
  1225  }
  1226  
  1227  // Tracing of alloc/free/gc.
  1228  
  1229  var tracelock mutex
  1230  
  1231  func tracealloc(p unsafe.Pointer, size uintptr, typ *_type) {
  1232  	lock(&tracelock)
  1233  	gp := getg()
  1234  	gp.m.traceback = 2
  1235  	if typ == nil {
  1236  		print("tracealloc(", p, ", ", hex(size), ")\n")
  1237  	} else {
  1238  		print("tracealloc(", p, ", ", hex(size), ", ", typ.string(), ")\n")
  1239  	}
  1240  	if gp.m.curg == nil || gp == gp.m.curg {
  1241  		goroutineheader(gp)
  1242  		pc := getcallerpc()
  1243  		sp := getcallersp()
  1244  		systemstack(func() {
  1245  			traceback(pc, sp, 0, gp)
  1246  		})
  1247  	} else {
  1248  		goroutineheader(gp.m.curg)
  1249  		traceback(^uintptr(0), ^uintptr(0), 0, gp.m.curg)
  1250  	}
  1251  	print("\n")
  1252  	gp.m.traceback = 0
  1253  	unlock(&tracelock)
  1254  }
  1255  
  1256  func tracefree(p unsafe.Pointer, size uintptr) {
  1257  	lock(&tracelock)
  1258  	gp := getg()
  1259  	gp.m.traceback = 2
  1260  	print("tracefree(", p, ", ", hex(size), ")\n")
  1261  	goroutineheader(gp)
  1262  	pc := getcallerpc()
  1263  	sp := getcallersp()
  1264  	systemstack(func() {
  1265  		traceback(pc, sp, 0, gp)
  1266  	})
  1267  	print("\n")
  1268  	gp.m.traceback = 0
  1269  	unlock(&tracelock)
  1270  }
  1271  
  1272  func tracegc() {
  1273  	lock(&tracelock)
  1274  	gp := getg()
  1275  	gp.m.traceback = 2
  1276  	print("tracegc()\n")
  1277  	// running on m->g0 stack; show all non-g0 goroutines
  1278  	tracebackothers(gp)
  1279  	print("end tracegc\n")
  1280  	print("\n")
  1281  	gp.m.traceback = 0
  1282  	unlock(&tracelock)
  1283  }
  1284  

View as plain text