1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "runtime/internal/atomic" 9 "unsafe" 10 ) 11 12 // Per-thread (in Go, per-P) cache for small objects. 13 // This includes a small object cache and local allocation stats. 14 // No locking needed because it is per-thread (per-P). 15 // 16 // mcaches are allocated from non-GC'd memory, so any heap pointers 17 // must be specially handled. 18 // 19 //go:notinheap 20 type mcache struct { 21 // The following members are accessed on every malloc, 22 // so they are grouped here for better caching. 23 nextSample uintptr // trigger heap sample after allocating this many bytes 24 scanAlloc uintptr // bytes of scannable heap allocated 25 26 // Allocator cache for tiny objects w/o pointers. 27 // See "Tiny allocator" comment in malloc.go. 28 29 // tiny points to the beginning of the current tiny block, or 30 // nil if there is no current tiny block. 31 // 32 // tiny is a heap pointer. Since mcache is in non-GC'd memory, 33 // we handle it by clearing it in releaseAll during mark 34 // termination. 35 // 36 // tinyAllocs is the number of tiny allocations performed 37 // by the P that owns this mcache. 38 tiny uintptr 39 tinyoffset uintptr 40 tinyAllocs uintptr 41 42 // The rest is not accessed on every malloc. 43 44 alloc [numSpanClasses]*mspan // spans to allocate from, indexed by spanClass 45 46 stackcache [_NumStackOrders]stackfreelist 47 48 // flushGen indicates the sweepgen during which this mcache 49 // was last flushed. If flushGen != mheap_.sweepgen, the spans 50 // in this mcache are stale and need to the flushed so they 51 // can be swept. This is done in acquirep. 52 flushGen uint32 53 } 54 55 // A gclink is a node in a linked list of blocks, like mlink, 56 // but it is opaque to the garbage collector. 57 // The GC does not trace the pointers during collection, 58 // and the compiler does not emit write barriers for assignments 59 // of gclinkptr values. Code should store references to gclinks 60 // as gclinkptr, not as *gclink. 61 type gclink struct { 62 next gclinkptr 63 } 64 65 // A gclinkptr is a pointer to a gclink, but it is opaque 66 // to the garbage collector. 67 type gclinkptr uintptr 68 69 // ptr returns the *gclink form of p. 70 // The result should be used for accessing fields, not stored 71 // in other data structures. 72 func (p gclinkptr) ptr() *gclink { 73 return (*gclink)(unsafe.Pointer(p)) 74 } 75 76 type stackfreelist struct { 77 list gclinkptr // linked list of free stacks 78 size uintptr // total size of stacks in list 79 } 80 81 // dummy mspan that contains no free objects. 82 var emptymspan mspan 83 84 func allocmcache() *mcache { 85 var c *mcache 86 systemstack(func() { 87 lock(&mheap_.lock) 88 c = (*mcache)(mheap_.cachealloc.alloc()) 89 c.flushGen = mheap_.sweepgen 90 unlock(&mheap_.lock) 91 }) 92 for i := range c.alloc { 93 c.alloc[i] = &emptymspan 94 } 95 c.nextSample = nextSample() 96 return c 97 } 98 99 // freemcache releases resources associated with this 100 // mcache and puts the object onto a free list. 101 // 102 // In some cases there is no way to simply release 103 // resources, such as statistics, so donate them to 104 // a different mcache (the recipient). 105 func freemcache(c *mcache) { 106 systemstack(func() { 107 c.releaseAll() 108 stackcache_clear(c) 109 110 // NOTE(rsc,rlh): If gcworkbuffree comes back, we need to coordinate 111 // with the stealing of gcworkbufs during garbage collection to avoid 112 // a race where the workbuf is double-freed. 113 // gcworkbuffree(c.gcworkbuf) 114 115 lock(&mheap_.lock) 116 mheap_.cachealloc.free(unsafe.Pointer(c)) 117 unlock(&mheap_.lock) 118 }) 119 } 120 121 // getMCache is a convenience function which tries to obtain an mcache. 122 // 123 // Returns nil if we're not bootstrapping or we don't have a P. The caller's 124 // P must not change, so we must be in a non-preemptible state. 125 func getMCache(mp *m) *mcache { 126 // Grab the mcache, since that's where stats live. 127 pp := mp.p.ptr() 128 var c *mcache 129 if pp == nil { 130 // We will be called without a P while bootstrapping, 131 // in which case we use mcache0, which is set in mallocinit. 132 // mcache0 is cleared when bootstrapping is complete, 133 // by procresize. 134 c = mcache0 135 } else { 136 c = pp.mcache 137 } 138 return c 139 } 140 141 // refill acquires a new span of span class spc for c. This span will 142 // have at least one free object. The current span in c must be full. 143 // 144 // Must run in a non-preemptible context since otherwise the owner of 145 // c could change. 146 func (c *mcache) refill(spc spanClass) { 147 // Return the current cached span to the central lists. 148 s := c.alloc[spc] 149 150 if uintptr(s.allocCount) != s.nelems { 151 throw("refill of span with free space remaining") 152 } 153 if s != &emptymspan { 154 // Mark this span as no longer cached. 155 if s.sweepgen != mheap_.sweepgen+3 { 156 throw("bad sweepgen in refill") 157 } 158 mheap_.central[spc].mcentral.uncacheSpan(s) 159 160 // Count up how many slots were used and record it. 161 stats := memstats.heapStats.acquire() 162 slotsUsed := int64(s.allocCount) - int64(s.allocCountBeforeCache) 163 atomic.Xadd64(&stats.smallAllocCount[spc.sizeclass()], slotsUsed) 164 165 // Flush tinyAllocs. 166 if spc == tinySpanClass { 167 atomic.Xadd64(&stats.tinyAllocCount, int64(c.tinyAllocs)) 168 c.tinyAllocs = 0 169 } 170 memstats.heapStats.release() 171 172 // Count the allocs in inconsistent, internal stats. 173 bytesAllocated := slotsUsed * int64(s.elemsize) 174 gcController.totalAlloc.Add(bytesAllocated) 175 176 // Clear the second allocCount just to be safe. 177 s.allocCountBeforeCache = 0 178 } 179 180 // Get a new cached span from the central lists. 181 s = mheap_.central[spc].mcentral.cacheSpan() 182 if s == nil { 183 throw("out of memory") 184 } 185 186 if uintptr(s.allocCount) == s.nelems { 187 throw("span has no free space") 188 } 189 190 // Indicate that this span is cached and prevent asynchronous 191 // sweeping in the next sweep phase. 192 s.sweepgen = mheap_.sweepgen + 3 193 194 // Store the current alloc count for accounting later. 195 s.allocCountBeforeCache = s.allocCount 196 197 // Update heapLive and flush scanAlloc. 198 // 199 // We have not yet allocated anything new into the span, but we 200 // assume that all of its slots will get used, so this makes 201 // heapLive an overestimate. 202 // 203 // When the span gets uncached, we'll fix up this overestimate 204 // if necessary (see releaseAll). 205 // 206 // We pick an overestimate here because an underestimate leads 207 // the pacer to believe that it's in better shape than it is, 208 // which appears to lead to more memory used. See #53738 for 209 // more details. 210 usedBytes := uintptr(s.allocCount) * s.elemsize 211 gcController.update(int64(s.npages*pageSize)-int64(usedBytes), int64(c.scanAlloc)) 212 c.scanAlloc = 0 213 214 c.alloc[spc] = s 215 } 216 217 // allocLarge allocates a span for a large object. 218 func (c *mcache) allocLarge(size uintptr, noscan bool) *mspan { 219 if size+_PageSize < size { 220 throw("out of memory") 221 } 222 npages := size >> _PageShift 223 if size&_PageMask != 0 { 224 npages++ 225 } 226 227 // Deduct credit for this span allocation and sweep if 228 // necessary. mHeap_Alloc will also sweep npages, so this only 229 // pays the debt down to npage pages. 230 deductSweepCredit(npages*_PageSize, npages) 231 232 spc := makeSpanClass(0, noscan) 233 s := mheap_.alloc(npages, spc) 234 if s == nil { 235 throw("out of memory") 236 } 237 238 // Count the alloc in consistent, external stats. 239 stats := memstats.heapStats.acquire() 240 atomic.Xadd64(&stats.largeAlloc, int64(npages*pageSize)) 241 atomic.Xadd64(&stats.largeAllocCount, 1) 242 memstats.heapStats.release() 243 244 // Count the alloc in inconsistent, internal stats. 245 gcController.totalAlloc.Add(int64(npages * pageSize)) 246 247 // Update heapLive. 248 gcController.update(int64(s.npages*pageSize), 0) 249 250 // Put the large span in the mcentral swept list so that it's 251 // visible to the background sweeper. 252 mheap_.central[spc].mcentral.fullSwept(mheap_.sweepgen).push(s) 253 s.limit = s.base() + size 254 heapBitsForAddr(s.base()).initSpan(s) 255 return s 256 } 257 258 func (c *mcache) releaseAll() { 259 // Take this opportunity to flush scanAlloc. 260 scanAlloc := int64(c.scanAlloc) 261 c.scanAlloc = 0 262 263 sg := mheap_.sweepgen 264 dHeapLive := int64(0) 265 for i := range c.alloc { 266 s := c.alloc[i] 267 if s != &emptymspan { 268 slotsUsed := int64(s.allocCount) - int64(s.allocCountBeforeCache) 269 s.allocCountBeforeCache = 0 270 271 // Adjust smallAllocCount for whatever was allocated. 272 stats := memstats.heapStats.acquire() 273 atomic.Xadd64(&stats.smallAllocCount[spanClass(i).sizeclass()], slotsUsed) 274 memstats.heapStats.release() 275 276 // Adjust the actual allocs in inconsistent, internal stats. 277 // We assumed earlier that the full span gets allocated. 278 gcController.totalAlloc.Add(slotsUsed * int64(s.elemsize)) 279 280 if s.sweepgen != sg+1 { 281 // refill conservatively counted unallocated slots in gcController.heapLive. 282 // Undo this. 283 // 284 // If this span was cached before sweep, then gcController.heapLive was totally 285 // recomputed since caching this span, so we don't do this for stale spans. 286 dHeapLive -= int64(uintptr(s.nelems)-uintptr(s.allocCount)) * int64(s.elemsize) 287 } 288 289 // Release the span to the mcentral. 290 mheap_.central[i].mcentral.uncacheSpan(s) 291 c.alloc[i] = &emptymspan 292 } 293 } 294 // Clear tinyalloc pool. 295 c.tiny = 0 296 c.tinyoffset = 0 297 298 // Flush tinyAllocs. 299 stats := memstats.heapStats.acquire() 300 atomic.Xadd64(&stats.tinyAllocCount, int64(c.tinyAllocs)) 301 c.tinyAllocs = 0 302 memstats.heapStats.release() 303 304 // Update heapLive and heapScan. 305 gcController.update(dHeapLive, scanAlloc) 306 } 307 308 // prepareForSweep flushes c if the system has entered a new sweep phase 309 // since c was populated. This must happen between the sweep phase 310 // starting and the first allocation from c. 311 func (c *mcache) prepareForSweep() { 312 // Alternatively, instead of making sure we do this on every P 313 // between starting the world and allocating on that P, we 314 // could leave allocate-black on, allow allocation to continue 315 // as usual, use a ragged barrier at the beginning of sweep to 316 // ensure all cached spans are swept, and then disable 317 // allocate-black. However, with this approach it's difficult 318 // to avoid spilling mark bits into the *next* GC cycle. 319 sg := mheap_.sweepgen 320 if c.flushGen == sg { 321 return 322 } else if c.flushGen != sg-2 { 323 println("bad flushGen", c.flushGen, "in prepareForSweep; sweepgen", sg) 324 throw("bad flushGen") 325 } 326 c.releaseAll() 327 stackcache_clear(c) 328 atomic.Store(&c.flushGen, mheap_.sweepgen) // Synchronizes with gcStart 329 } 330