...
Source file
src/runtime/mranges.go
Documentation: runtime
1
2
3
4
5
6
7
8
9
10 package runtime
11
12 import (
13 "internal/goarch"
14 "runtime/internal/atomic"
15 "unsafe"
16 )
17
18
19
20
21 type addrRange struct {
22
23
24
25
26
27 base, limit offAddr
28 }
29
30
31
32
33 func makeAddrRange(base, limit uintptr) addrRange {
34 r := addrRange{offAddr{base}, offAddr{limit}}
35 if (base-arenaBaseOffset >= base) != (limit-arenaBaseOffset >= limit) {
36 throw("addr range base and limit are not in the same memory segment")
37 }
38 return r
39 }
40
41
42 func (a addrRange) size() uintptr {
43 if !a.base.lessThan(a.limit) {
44 return 0
45 }
46
47
48 return a.limit.diff(a.base)
49 }
50
51
52 func (a addrRange) contains(addr uintptr) bool {
53 return a.base.lessEqual(offAddr{addr}) && (offAddr{addr}).lessThan(a.limit)
54 }
55
56
57
58
59
60 func (a addrRange) subtract(b addrRange) addrRange {
61 if b.base.lessEqual(a.base) && a.limit.lessEqual(b.limit) {
62 return addrRange{}
63 } else if a.base.lessThan(b.base) && b.limit.lessThan(a.limit) {
64 throw("bad prune")
65 } else if b.limit.lessThan(a.limit) && a.base.lessThan(b.limit) {
66 a.base = b.limit
67 } else if a.base.lessThan(b.base) && b.base.lessThan(a.limit) {
68 a.limit = b.base
69 }
70 return a
71 }
72
73
74
75 func (a addrRange) removeGreaterEqual(addr uintptr) addrRange {
76 if (offAddr{addr}).lessEqual(a.base) {
77 return addrRange{}
78 }
79 if a.limit.lessEqual(offAddr{addr}) {
80 return a
81 }
82 return makeAddrRange(a.base.addr(), addr)
83 }
84
85 var (
86
87
88 minOffAddr = offAddr{arenaBaseOffset}
89
90
91
92
93 maxOffAddr = offAddr{(((1 << heapAddrBits) - 1) + arenaBaseOffset) & uintptrMask}
94 )
95
96
97
98
99 type offAddr struct {
100
101
102 a uintptr
103 }
104
105
106 func (l offAddr) add(bytes uintptr) offAddr {
107 return offAddr{a: l.a + bytes}
108 }
109
110
111 func (l offAddr) sub(bytes uintptr) offAddr {
112 return offAddr{a: l.a - bytes}
113 }
114
115
116
117 func (l1 offAddr) diff(l2 offAddr) uintptr {
118 return l1.a - l2.a
119 }
120
121
122
123 func (l1 offAddr) lessThan(l2 offAddr) bool {
124 return (l1.a - arenaBaseOffset) < (l2.a - arenaBaseOffset)
125 }
126
127
128
129 func (l1 offAddr) lessEqual(l2 offAddr) bool {
130 return (l1.a - arenaBaseOffset) <= (l2.a - arenaBaseOffset)
131 }
132
133
134 func (l1 offAddr) equal(l2 offAddr) bool {
135
136
137 return l1 == l2
138 }
139
140
141 func (l offAddr) addr() uintptr {
142 return l.a
143 }
144
145
146
147
148 type atomicOffAddr struct {
149
150 a atomic.Int64
151 }
152
153
154
155 func (b *atomicOffAddr) Clear() {
156 for {
157 old := b.a.Load()
158 if old < 0 {
159 return
160 }
161 if b.a.CompareAndSwap(old, int64(minOffAddr.addr()-arenaBaseOffset)) {
162 return
163 }
164 }
165 }
166
167
168
169 func (b *atomicOffAddr) StoreMin(addr uintptr) {
170 new := int64(addr - arenaBaseOffset)
171 for {
172 old := b.a.Load()
173 if old < new {
174 return
175 }
176 if b.a.CompareAndSwap(old, new) {
177 return
178 }
179 }
180 }
181
182
183
184
185
186 func (b *atomicOffAddr) StoreUnmark(markedAddr, newAddr uintptr) {
187 b.a.CompareAndSwap(-int64(markedAddr-arenaBaseOffset), int64(newAddr-arenaBaseOffset))
188 }
189
190
191
192 func (b *atomicOffAddr) StoreMarked(addr uintptr) {
193 b.a.Store(-int64(addr - arenaBaseOffset))
194 }
195
196
197
198 func (b *atomicOffAddr) Load() (uintptr, bool) {
199 v := b.a.Load()
200 wasMarked := false
201 if v < 0 {
202 wasMarked = true
203 v = -v
204 }
205 return uintptr(v) + arenaBaseOffset, wasMarked
206 }
207
208
209
210
211
212
213
214
215
216
217
218 type addrRanges struct {
219
220 ranges []addrRange
221
222
223
224 totalBytes uintptr
225
226
227 sysStat *sysMemStat
228 }
229
230 func (a *addrRanges) init(sysStat *sysMemStat) {
231 ranges := (*notInHeapSlice)(unsafe.Pointer(&a.ranges))
232 ranges.len = 0
233 ranges.cap = 16
234 ranges.array = (*notInHeap)(persistentalloc(unsafe.Sizeof(addrRange{})*uintptr(ranges.cap), goarch.PtrSize, sysStat))
235 a.sysStat = sysStat
236 a.totalBytes = 0
237 }
238
239
240
241 func (a *addrRanges) findSucc(addr uintptr) int {
242 base := offAddr{addr}
243
244
245
246
247 const iterMax = 8
248 bot, top := 0, len(a.ranges)
249 for top-bot > iterMax {
250 i := ((top - bot) / 2) + bot
251 if a.ranges[i].contains(base.addr()) {
252
253
254 return i + 1
255 }
256 if base.lessThan(a.ranges[i].base) {
257
258
259
260 top = i
261 } else {
262
263
264
265
266
267 bot = i + 1
268 }
269 }
270
271
272
273 for i := bot; i < top; i++ {
274 if base.lessThan(a.ranges[i].base) {
275 return i
276 }
277 }
278 return top
279 }
280
281
282
283
284
285
286 func (a *addrRanges) findAddrGreaterEqual(addr uintptr) (uintptr, bool) {
287 i := a.findSucc(addr)
288 if i == 0 {
289 return a.ranges[0].base.addr(), true
290 }
291 if a.ranges[i-1].contains(addr) {
292 return addr, true
293 }
294 if i < len(a.ranges) {
295 return a.ranges[i].base.addr(), true
296 }
297 return 0, false
298 }
299
300
301 func (a *addrRanges) contains(addr uintptr) bool {
302 i := a.findSucc(addr)
303 if i == 0 {
304 return false
305 }
306 return a.ranges[i-1].contains(addr)
307 }
308
309
310
311
312 func (a *addrRanges) add(r addrRange) {
313
314
315
316
317
318
319
320
321
322
323
324 if r.size() == 0 {
325 print("runtime: range = {", hex(r.base.addr()), ", ", hex(r.limit.addr()), "}\n")
326 throw("attempted to add zero-sized address range")
327 }
328
329
330 i := a.findSucc(r.base.addr())
331 coalescesDown := i > 0 && a.ranges[i-1].limit.equal(r.base)
332 coalescesUp := i < len(a.ranges) && r.limit.equal(a.ranges[i].base)
333 if coalescesUp && coalescesDown {
334
335
336 a.ranges[i-1].limit = a.ranges[i].limit
337
338
339 copy(a.ranges[i:], a.ranges[i+1:])
340 a.ranges = a.ranges[:len(a.ranges)-1]
341 } else if coalescesDown {
342
343
344 a.ranges[i-1].limit = r.limit
345 } else if coalescesUp {
346
347
348 a.ranges[i].base = r.base
349 } else {
350
351
352 if len(a.ranges)+1 > cap(a.ranges) {
353
354
355
356
357 oldRanges := a.ranges
358 ranges := (*notInHeapSlice)(unsafe.Pointer(&a.ranges))
359 ranges.len = len(oldRanges) + 1
360 ranges.cap = cap(oldRanges) * 2
361 ranges.array = (*notInHeap)(persistentalloc(unsafe.Sizeof(addrRange{})*uintptr(ranges.cap), goarch.PtrSize, a.sysStat))
362
363
364 copy(a.ranges[:i], oldRanges[:i])
365 copy(a.ranges[i+1:], oldRanges[i:])
366 } else {
367 a.ranges = a.ranges[:len(a.ranges)+1]
368 copy(a.ranges[i+1:], a.ranges[i:])
369 }
370 a.ranges[i] = r
371 }
372 a.totalBytes += r.size()
373 }
374
375
376
377
378 func (a *addrRanges) removeLast(nBytes uintptr) addrRange {
379 if len(a.ranges) == 0 {
380 return addrRange{}
381 }
382 r := a.ranges[len(a.ranges)-1]
383 size := r.size()
384 if size > nBytes {
385 newEnd := r.limit.sub(nBytes)
386 a.ranges[len(a.ranges)-1].limit = newEnd
387 a.totalBytes -= nBytes
388 return addrRange{newEnd, r.limit}
389 }
390 a.ranges = a.ranges[:len(a.ranges)-1]
391 a.totalBytes -= size
392 return r
393 }
394
395
396
397 func (a *addrRanges) removeGreaterEqual(addr uintptr) {
398 pivot := a.findSucc(addr)
399 if pivot == 0 {
400
401 a.totalBytes = 0
402 a.ranges = a.ranges[:0]
403 return
404 }
405 removed := uintptr(0)
406 for _, r := range a.ranges[pivot:] {
407 removed += r.size()
408 }
409 if r := a.ranges[pivot-1]; r.contains(addr) {
410 removed += r.size()
411 r = r.removeGreaterEqual(addr)
412 if r.size() == 0 {
413 pivot--
414 } else {
415 removed -= r.size()
416 a.ranges[pivot-1] = r
417 }
418 }
419 a.ranges = a.ranges[:pivot]
420 a.totalBytes -= removed
421 }
422
423
424
425 func (a *addrRanges) cloneInto(b *addrRanges) {
426 if len(a.ranges) > cap(b.ranges) {
427
428 ranges := (*notInHeapSlice)(unsafe.Pointer(&b.ranges))
429 ranges.len = 0
430 ranges.cap = cap(a.ranges)
431 ranges.array = (*notInHeap)(persistentalloc(unsafe.Sizeof(addrRange{})*uintptr(ranges.cap), goarch.PtrSize, b.sysStat))
432 }
433 b.ranges = b.ranges[:len(a.ranges)]
434 b.totalBytes = a.totalBytes
435 copy(b.ranges, a.ranges)
436 }
437
View as plain text