...
Source file
src/runtime/mpagecache.go
Documentation: runtime
1
2
3
4
5 package runtime
6
7 import (
8 "runtime/internal/sys"
9 "unsafe"
10 )
11
12 const pageCachePages = 8 * unsafe.Sizeof(pageCache{}.cache)
13
14
15
16
17
18 type pageCache struct {
19 base uintptr
20 cache uint64
21 scav uint64
22 }
23
24
25
26 func (c *pageCache) empty() bool {
27 return c.cache == 0
28 }
29
30
31
32
33
34
35
36
37
38 func (c *pageCache) alloc(npages uintptr) (uintptr, uintptr) {
39 if c.cache == 0 {
40 return 0, 0
41 }
42 if npages == 1 {
43 i := uintptr(sys.TrailingZeros64(c.cache))
44 scav := (c.scav >> i) & 1
45 c.cache &^= 1 << i
46 c.scav &^= 1 << i
47 return c.base + i*pageSize, uintptr(scav) * pageSize
48 }
49 return c.allocN(npages)
50 }
51
52
53
54
55
56
57
58 func (c *pageCache) allocN(npages uintptr) (uintptr, uintptr) {
59 i := findBitRange64(c.cache, uint(npages))
60 if i >= 64 {
61 return 0, 0
62 }
63 mask := ((uint64(1) << npages) - 1) << i
64 scav := sys.OnesCount64(c.scav & mask)
65 c.cache &^= mask
66 c.scav &^= mask
67 return c.base + uintptr(i*pageSize), uintptr(scav) * pageSize
68 }
69
70
71
72
73
74
75
76
77
78
79 func (c *pageCache) flush(p *pageAlloc) {
80 assertLockHeld(p.mheapLock)
81
82 if c.empty() {
83 return
84 }
85 ci := chunkIndex(c.base)
86 pi := chunkPageIndex(c.base)
87
88
89
90 for i := uint(0); i < 64; i++ {
91 if c.cache&(1<<i) != 0 {
92 p.chunkOf(ci).free1(pi + i)
93 }
94 if c.scav&(1<<i) != 0 {
95 p.chunkOf(ci).scavenged.setRange(pi+i, 1)
96 }
97 }
98
99
100 if b := (offAddr{c.base}); b.lessThan(p.searchAddr) {
101 p.searchAddr = b
102 }
103 p.update(c.base, pageCachePages, false, false)
104 *c = pageCache{}
105 }
106
107
108
109
110
111
112
113
114
115
116 func (p *pageAlloc) allocToCache() pageCache {
117 assertLockHeld(p.mheapLock)
118
119
120
121 if chunkIndex(p.searchAddr.addr()) >= p.end {
122 return pageCache{}
123 }
124 c := pageCache{}
125 ci := chunkIndex(p.searchAddr.addr())
126 var chunk *pallocData
127 if p.summary[len(p.summary)-1][ci] != 0 {
128
129 chunk = p.chunkOf(ci)
130 j, _ := chunk.find(1, chunkPageIndex(p.searchAddr.addr()))
131 if j == ^uint(0) {
132 throw("bad summary data")
133 }
134 c = pageCache{
135 base: chunkBase(ci) + alignDown(uintptr(j), 64)*pageSize,
136 cache: ^chunk.pages64(j),
137 scav: chunk.scavenged.block64(j),
138 }
139 } else {
140
141
142 addr, _ := p.find(1)
143 if addr == 0 {
144
145
146 p.searchAddr = maxSearchAddr()
147 return pageCache{}
148 }
149 ci := chunkIndex(addr)
150 chunk = p.chunkOf(ci)
151 c = pageCache{
152 base: alignDown(addr, 64*pageSize),
153 cache: ^chunk.pages64(chunkPageIndex(addr)),
154 scav: chunk.scavenged.block64(chunkPageIndex(addr)),
155 }
156 }
157
158
159
160 cpi := chunkPageIndex(c.base)
161 chunk.allocPages64(cpi, c.cache)
162 chunk.scavenged.clearBlock64(cpi, c.cache&c.scav )
163
164
165 p.update(c.base, pageCachePages, false, true)
166
167
168
169
170
171
172
173
174
175 p.searchAddr = offAddr{c.base + pageSize*(pageCachePages-1)}
176 return c
177 }
178
View as plain text