Documentation: runtime
1
2
3
4
5 package runtime
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56 import (
57 "internal/abi"
58 "internal/goarch"
59 "runtime/internal/atomic"
60 "runtime/internal/math"
61 "unsafe"
62 )
63
64 const (
65
66 bucketCntBits = 3
67 bucketCnt = 1 << bucketCntBits
68
69
70
71 loadFactorNum = 13
72 loadFactorDen = 2
73
74
75
76
77
78 maxKeySize = 128
79 maxElemSize = 128
80
81
82
83
84 dataOffset = unsafe.Offsetof(struct {
85 b bmap
86 v int64
87 }{}.v)
88
89
90
91
92
93 emptyRest = 0
94 emptyOne = 1
95 evacuatedX = 2
96 evacuatedY = 3
97 evacuatedEmpty = 4
98 minTopHash = 5
99
100
101 iterator = 1
102 oldIterator = 2
103 hashWriting = 4
104 sameSizeGrow = 8
105
106
107 noCheck = 1<<(8*goarch.PtrSize) - 1
108 )
109
110
111 func isEmpty(x uint8) bool {
112 return x <= emptyOne
113 }
114
115
116 type hmap struct {
117
118
119 count int
120 flags uint8
121 B uint8
122 noverflow uint16
123 hash0 uint32
124
125 buckets unsafe.Pointer
126 oldbuckets unsafe.Pointer
127 nevacuate uintptr
128
129 extra *mapextra
130 }
131
132
133 type mapextra struct {
134
135
136
137
138
139
140
141
142 overflow *[]*bmap
143 oldoverflow *[]*bmap
144
145
146 nextOverflow *bmap
147 }
148
149
150 type bmap struct {
151
152
153
154 tophash [bucketCnt]uint8
155
156
157
158
159
160 }
161
162
163
164
165 type hiter struct {
166 key unsafe.Pointer
167 elem unsafe.Pointer
168 t *maptype
169 h *hmap
170 buckets unsafe.Pointer
171 bptr *bmap
172 overflow *[]*bmap
173 oldoverflow *[]*bmap
174 startBucket uintptr
175 offset uint8
176 wrapped bool
177 B uint8
178 i uint8
179 bucket uintptr
180 checkBucket uintptr
181 }
182
183
184 func bucketShift(b uint8) uintptr {
185
186 return uintptr(1) << (b & (goarch.PtrSize*8 - 1))
187 }
188
189
190 func bucketMask(b uint8) uintptr {
191 return bucketShift(b) - 1
192 }
193
194
195 func tophash(hash uintptr) uint8 {
196 top := uint8(hash >> (goarch.PtrSize*8 - 8))
197 if top < minTopHash {
198 top += minTopHash
199 }
200 return top
201 }
202
203 func evacuated(b *bmap) bool {
204 h := b.tophash[0]
205 return h > emptyOne && h < minTopHash
206 }
207
208 func (b *bmap) overflow(t *maptype) *bmap {
209 return *(**bmap)(add(unsafe.Pointer(b), uintptr(t.bucketsize)-goarch.PtrSize))
210 }
211
212 func (b *bmap) setoverflow(t *maptype, ovf *bmap) {
213 *(**bmap)(add(unsafe.Pointer(b), uintptr(t.bucketsize)-goarch.PtrSize)) = ovf
214 }
215
216 func (b *bmap) keys() unsafe.Pointer {
217 return add(unsafe.Pointer(b), dataOffset)
218 }
219
220
221
222
223
224
225
226
227 func (h *hmap) incrnoverflow() {
228
229
230
231 if h.B < 16 {
232 h.noverflow++
233 return
234 }
235
236
237
238 mask := uint32(1)<<(h.B-15) - 1
239
240
241 if fastrand()&mask == 0 {
242 h.noverflow++
243 }
244 }
245
246 func (h *hmap) newoverflow(t *maptype, b *bmap) *bmap {
247 var ovf *bmap
248 if h.extra != nil && h.extra.nextOverflow != nil {
249
250
251 ovf = h.extra.nextOverflow
252 if ovf.overflow(t) == nil {
253
254 h.extra.nextOverflow = (*bmap)(add(unsafe.Pointer(ovf), uintptr(t.bucketsize)))
255 } else {
256
257
258
259 ovf.setoverflow(t, nil)
260 h.extra.nextOverflow = nil
261 }
262 } else {
263 ovf = (*bmap)(newobject(t.bucket))
264 }
265 h.incrnoverflow()
266 if t.bucket.ptrdata == 0 {
267 h.createOverflow()
268 *h.extra.overflow = append(*h.extra.overflow, ovf)
269 }
270 b.setoverflow(t, ovf)
271 return ovf
272 }
273
274 func (h *hmap) createOverflow() {
275 if h.extra == nil {
276 h.extra = new(mapextra)
277 }
278 if h.extra.overflow == nil {
279 h.extra.overflow = new([]*bmap)
280 }
281 }
282
283 func makemap64(t *maptype, hint int64, h *hmap) *hmap {
284 if int64(int(hint)) != hint {
285 hint = 0
286 }
287 return makemap(t, int(hint), h)
288 }
289
290
291
292
293 func makemap_small() *hmap {
294 h := new(hmap)
295 h.hash0 = fastrand()
296 return h
297 }
298
299
300
301
302
303
304 func makemap(t *maptype, hint int, h *hmap) *hmap {
305 mem, overflow := math.MulUintptr(uintptr(hint), t.bucket.size)
306 if overflow || mem > maxAlloc {
307 hint = 0
308 }
309
310
311 if h == nil {
312 h = new(hmap)
313 }
314 h.hash0 = fastrand()
315
316
317
318 B := uint8(0)
319 for overLoadFactor(hint, B) {
320 B++
321 }
322 h.B = B
323
324
325
326
327 if h.B != 0 {
328 var nextOverflow *bmap
329 h.buckets, nextOverflow = makeBucketArray(t, h.B, nil)
330 if nextOverflow != nil {
331 h.extra = new(mapextra)
332 h.extra.nextOverflow = nextOverflow
333 }
334 }
335
336 return h
337 }
338
339
340
341
342
343
344
345 func makeBucketArray(t *maptype, b uint8, dirtyalloc unsafe.Pointer) (buckets unsafe.Pointer, nextOverflow *bmap) {
346 base := bucketShift(b)
347 nbuckets := base
348
349
350 if b >= 4 {
351
352
353
354 nbuckets += bucketShift(b - 4)
355 sz := t.bucket.size * nbuckets
356 up := roundupsize(sz)
357 if up != sz {
358 nbuckets = up / t.bucket.size
359 }
360 }
361
362 if dirtyalloc == nil {
363 buckets = newarray(t.bucket, int(nbuckets))
364 } else {
365
366
367
368 buckets = dirtyalloc
369 size := t.bucket.size * nbuckets
370 if t.bucket.ptrdata != 0 {
371 memclrHasPointers(buckets, size)
372 } else {
373 memclrNoHeapPointers(buckets, size)
374 }
375 }
376
377 if base != nbuckets {
378
379
380
381
382
383 nextOverflow = (*bmap)(add(buckets, base*uintptr(t.bucketsize)))
384 last := (*bmap)(add(buckets, (nbuckets-1)*uintptr(t.bucketsize)))
385 last.setoverflow(t, (*bmap)(buckets))
386 }
387 return buckets, nextOverflow
388 }
389
390
391
392
393
394
395 func mapaccess1(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer {
396 if raceenabled && h != nil {
397 callerpc := getcallerpc()
398 pc := abi.FuncPCABIInternal(mapaccess1)
399 racereadpc(unsafe.Pointer(h), callerpc, pc)
400 raceReadObjectPC(t.key, key, callerpc, pc)
401 }
402 if msanenabled && h != nil {
403 msanread(key, t.key.size)
404 }
405 if asanenabled && h != nil {
406 asanread(key, t.key.size)
407 }
408 if h == nil || h.count == 0 {
409 if t.hashMightPanic() {
410 t.hasher(key, 0)
411 }
412 return unsafe.Pointer(&zeroVal[0])
413 }
414 if h.flags&hashWriting != 0 {
415 fatal("concurrent map read and map write")
416 }
417 hash := t.hasher(key, uintptr(h.hash0))
418 m := bucketMask(h.B)
419 b := (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
420 if c := h.oldbuckets; c != nil {
421 if !h.sameSizeGrow() {
422
423 m >>= 1
424 }
425 oldb := (*bmap)(add(c, (hash&m)*uintptr(t.bucketsize)))
426 if !evacuated(oldb) {
427 b = oldb
428 }
429 }
430 top := tophash(hash)
431 bucketloop:
432 for ; b != nil; b = b.overflow(t) {
433 for i := uintptr(0); i < bucketCnt; i++ {
434 if b.tophash[i] != top {
435 if b.tophash[i] == emptyRest {
436 break bucketloop
437 }
438 continue
439 }
440 k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize))
441 if t.indirectkey() {
442 k = *((*unsafe.Pointer)(k))
443 }
444 if t.key.equal(key, k) {
445 e := add(unsafe.Pointer(b), dataOffset+bucketCnt*uintptr(t.keysize)+i*uintptr(t.elemsize))
446 if t.indirectelem() {
447 e = *((*unsafe.Pointer)(e))
448 }
449 return e
450 }
451 }
452 }
453 return unsafe.Pointer(&zeroVal[0])
454 }
455
456 func mapaccess2(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, bool) {
457 if raceenabled && h != nil {
458 callerpc := getcallerpc()
459 pc := abi.FuncPCABIInternal(mapaccess2)
460 racereadpc(unsafe.Pointer(h), callerpc, pc)
461 raceReadObjectPC(t.key, key, callerpc, pc)
462 }
463 if msanenabled && h != nil {
464 msanread(key, t.key.size)
465 }
466 if asanenabled && h != nil {
467 asanread(key, t.key.size)
468 }
469 if h == nil || h.count == 0 {
470 if t.hashMightPanic() {
471 t.hasher(key, 0)
472 }
473 return unsafe.Pointer(&zeroVal[0]), false
474 }
475 if h.flags&hashWriting != 0 {
476 fatal("concurrent map read and map write")
477 }
478 hash := t.hasher(key, uintptr(h.hash0))
479 m := bucketMask(h.B)
480 b := (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
481 if c := h.oldbuckets; c != nil {
482 if !h.sameSizeGrow() {
483
484 m >>= 1
485 }
486 oldb := (*bmap)(add(c, (hash&m)*uintptr(t.bucketsize)))
487 if !evacuated(oldb) {
488 b = oldb
489 }
490 }
491 top := tophash(hash)
492 bucketloop:
493 for ; b != nil; b = b.overflow(t) {
494 for i := uintptr(0); i < bucketCnt; i++ {
495 if b.tophash[i] != top {
496 if b.tophash[i] == emptyRest {
497 break bucketloop
498 }
499 continue
500 }
501 k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize))
502 if t.indirectkey() {
503 k = *((*unsafe.Pointer)(k))
504 }
505 if t.key.equal(key, k) {
506 e := add(unsafe.Pointer(b), dataOffset+bucketCnt*uintptr(t.keysize)+i*uintptr(t.elemsize))
507 if t.indirectelem() {
508 e = *((*unsafe.Pointer)(e))
509 }
510 return e, true
511 }
512 }
513 }
514 return unsafe.Pointer(&zeroVal[0]), false
515 }
516
517
518 func mapaccessK(t *maptype, h *hmap, key unsafe.Pointer) (unsafe.Pointer, unsafe.Pointer) {
519 if h == nil || h.count == 0 {
520 return nil, nil
521 }
522 hash := t.hasher(key, uintptr(h.hash0))
523 m := bucketMask(h.B)
524 b := (*bmap)(add(h.buckets, (hash&m)*uintptr(t.bucketsize)))
525 if c := h.oldbuckets; c != nil {
526 if !h.sameSizeGrow() {
527
528 m >>= 1
529 }
530 oldb := (*bmap)(add(c, (hash&m)*uintptr(t.bucketsize)))
531 if !evacuated(oldb) {
532 b = oldb
533 }
534 }
535 top := tophash(hash)
536 bucketloop:
537 for ; b != nil; b = b.overflow(t) {
538 for i := uintptr(0); i < bucketCnt; i++ {
539 if b.tophash[i] != top {
540 if b.tophash[i] == emptyRest {
541 break bucketloop
542 }
543 continue
544 }
545 k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize))
546 if t.indirectkey() {
547 k = *((*unsafe.Pointer)(k))
548 }
549 if t.key.equal(key, k) {
550 e := add(unsafe.Pointer(b), dataOffset+bucketCnt*uintptr(t.keysize)+i*uintptr(t.elemsize))
551 if t.indirectelem() {
552 e = *((*unsafe.Pointer)(e))
553 }
554 return k, e
555 }
556 }
557 }
558 return nil, nil
559 }
560
561 func mapaccess1_fat(t *maptype, h *hmap, key, zero unsafe.Pointer) unsafe.Pointer {
562 e := mapaccess1(t, h, key)
563 if e == unsafe.Pointer(&zeroVal[0]) {
564 return zero
565 }
566 return e
567 }
568
569 func mapaccess2_fat(t *maptype, h *hmap, key, zero unsafe.Pointer) (unsafe.Pointer, bool) {
570 e := mapaccess1(t, h, key)
571 if e == unsafe.Pointer(&zeroVal[0]) {
572 return zero, false
573 }
574 return e, true
575 }
576
577
578 func mapassign(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer {
579 if h == nil {
580 panic(plainError("assignment to entry in nil map"))
581 }
582 if raceenabled {
583 callerpc := getcallerpc()
584 pc := abi.FuncPCABIInternal(mapassign)
585 racewritepc(unsafe.Pointer(h), callerpc, pc)
586 raceReadObjectPC(t.key, key, callerpc, pc)
587 }
588 if msanenabled {
589 msanread(key, t.key.size)
590 }
591 if asanenabled {
592 asanread(key, t.key.size)
593 }
594 if h.flags&hashWriting != 0 {
595 fatal("concurrent map writes")
596 }
597 hash := t.hasher(key, uintptr(h.hash0))
598
599
600
601 h.flags ^= hashWriting
602
603 if h.buckets == nil {
604 h.buckets = newobject(t.bucket)
605 }
606
607 again:
608 bucket := hash & bucketMask(h.B)
609 if h.growing() {
610 growWork(t, h, bucket)
611 }
612 b := (*bmap)(add(h.buckets, bucket*uintptr(t.bucketsize)))
613 top := tophash(hash)
614
615 var inserti *uint8
616 var insertk unsafe.Pointer
617 var elem unsafe.Pointer
618 bucketloop:
619 for {
620 for i := uintptr(0); i < bucketCnt; i++ {
621 if b.tophash[i] != top {
622 if isEmpty(b.tophash[i]) && inserti == nil {
623 inserti = &b.tophash[i]
624 insertk = add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize))
625 elem = add(unsafe.Pointer(b), dataOffset+bucketCnt*uintptr(t.keysize)+i*uintptr(t.elemsize))
626 }
627 if b.tophash[i] == emptyRest {
628 break bucketloop
629 }
630 continue
631 }
632 k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize))
633 if t.indirectkey() {
634 k = *((*unsafe.Pointer)(k))
635 }
636 if !t.key.equal(key, k) {
637 continue
638 }
639
640 if t.needkeyupdate() {
641 typedmemmove(t.key, k, key)
642 }
643 elem = add(unsafe.Pointer(b), dataOffset+bucketCnt*uintptr(t.keysize)+i*uintptr(t.elemsize))
644 goto done
645 }
646 ovf := b.overflow(t)
647 if ovf == nil {
648 break
649 }
650 b = ovf
651 }
652
653
654
655
656
657 if !h.growing() && (overLoadFactor(h.count+1, h.B) || tooManyOverflowBuckets(h.noverflow, h.B)) {
658 hashGrow(t, h)
659 goto again
660 }
661
662 if inserti == nil {
663
664 newb := h.newoverflow(t, b)
665 inserti = &newb.tophash[0]
666 insertk = add(unsafe.Pointer(newb), dataOffset)
667 elem = add(insertk, bucketCnt*uintptr(t.keysize))
668 }
669
670
671 if t.indirectkey() {
672 kmem := newobject(t.key)
673 *(*unsafe.Pointer)(insertk) = kmem
674 insertk = kmem
675 }
676 if t.indirectelem() {
677 vmem := newobject(t.elem)
678 *(*unsafe.Pointer)(elem) = vmem
679 }
680 typedmemmove(t.key, insertk, key)
681 *inserti = top
682 h.count++
683
684 done:
685 if h.flags&hashWriting == 0 {
686 fatal("concurrent map writes")
687 }
688 h.flags &^= hashWriting
689 if t.indirectelem() {
690 elem = *((*unsafe.Pointer)(elem))
691 }
692 return elem
693 }
694
695 func mapdelete(t *maptype, h *hmap, key unsafe.Pointer) {
696 if raceenabled && h != nil {
697 callerpc := getcallerpc()
698 pc := abi.FuncPCABIInternal(mapdelete)
699 racewritepc(unsafe.Pointer(h), callerpc, pc)
700 raceReadObjectPC(t.key, key, callerpc, pc)
701 }
702 if msanenabled && h != nil {
703 msanread(key, t.key.size)
704 }
705 if asanenabled && h != nil {
706 asanread(key, t.key.size)
707 }
708 if h == nil || h.count == 0 {
709 if t.hashMightPanic() {
710 t.hasher(key, 0)
711 }
712 return
713 }
714 if h.flags&hashWriting != 0 {
715 fatal("concurrent map writes")
716 }
717
718 hash := t.hasher(key, uintptr(h.hash0))
719
720
721
722 h.flags ^= hashWriting
723
724 bucket := hash & bucketMask(h.B)
725 if h.growing() {
726 growWork(t, h, bucket)
727 }
728 b := (*bmap)(add(h.buckets, bucket*uintptr(t.bucketsize)))
729 bOrig := b
730 top := tophash(hash)
731 search:
732 for ; b != nil; b = b.overflow(t) {
733 for i := uintptr(0); i < bucketCnt; i++ {
734 if b.tophash[i] != top {
735 if b.tophash[i] == emptyRest {
736 break search
737 }
738 continue
739 }
740 k := add(unsafe.Pointer(b), dataOffset+i*uintptr(t.keysize))
741 k2 := k
742 if t.indirectkey() {
743 k2 = *((*unsafe.Pointer)(k2))
744 }
745 if !t.key.equal(key, k2) {
746 continue
747 }
748
749 if t.indirectkey() {
750 *(*unsafe.Pointer)(k) = nil
751 } else if t.key.ptrdata != 0 {
752 memclrHasPointers(k, t.key.size)
753 }
754 e := add(unsafe.Pointer(b), dataOffset+bucketCnt*uintptr(t.keysize)+i*uintptr(t.elemsize))
755 if t.indirectelem() {
756 *(*unsafe.Pointer)(e) = nil
757 } else if t.elem.ptrdata != 0 {
758 memclrHasPointers(e, t.elem.size)
759 } else {
760 memclrNoHeapPointers(e, t.elem.size)
761 }
762 b.tophash[i] = emptyOne
763
764
765
766
767 if i == bucketCnt-1 {
768 if b.overflow(t) != nil && b.overflow(t).tophash[0] != emptyRest {
769 goto notLast
770 }
771 } else {
772 if b.tophash[i+1] != emptyRest {
773 goto notLast
774 }
775 }
776 for {
777 b.tophash[i] = emptyRest
778 if i == 0 {
779 if b == bOrig {
780 break
781 }
782
783 c := b
784 for b = bOrig; b.overflow(t) != c; b = b.overflow(t) {
785 }
786 i = bucketCnt - 1
787 } else {
788 i--
789 }
790 if b.tophash[i] != emptyOne {
791 break
792 }
793 }
794 notLast:
795 h.count--
796
797
798 if h.count == 0 {
799 h.hash0 = fastrand()
800 }
801 break search
802 }
803 }
804
805 if h.flags&hashWriting == 0 {
806 fatal("concurrent map writes")
807 }
808 h.flags &^= hashWriting
809 }
810
811
812
813
814
815 func mapiterinit(t *maptype, h *hmap, it *hiter) {
816 if raceenabled && h != nil {
817 callerpc := getcallerpc()
818 racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapiterinit))
819 }
820
821 it.t = t
822 if h == nil || h.count == 0 {
823 return
824 }
825
826 if unsafe.Sizeof(hiter{})/goarch.PtrSize != 12 {
827 throw("hash_iter size incorrect")
828 }
829 it.h = h
830
831
832 it.B = h.B
833 it.buckets = h.buckets
834 if t.bucket.ptrdata == 0 {
835
836
837
838
839 h.createOverflow()
840 it.overflow = h.extra.overflow
841 it.oldoverflow = h.extra.oldoverflow
842 }
843
844
845 var r uintptr
846 if h.B > 31-bucketCntBits {
847 r = uintptr(fastrand64())
848 } else {
849 r = uintptr(fastrand())
850 }
851 it.startBucket = r & bucketMask(h.B)
852 it.offset = uint8(r >> h.B & (bucketCnt - 1))
853
854
855 it.bucket = it.startBucket
856
857
858
859 if old := h.flags; old&(iterator|oldIterator) != iterator|oldIterator {
860 atomic.Or8(&h.flags, iterator|oldIterator)
861 }
862
863 mapiternext(it)
864 }
865
866 func mapiternext(it *hiter) {
867 h := it.h
868 if raceenabled {
869 callerpc := getcallerpc()
870 racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(mapiternext))
871 }
872 if h.flags&hashWriting != 0 {
873 fatal("concurrent map iteration and map write")
874 }
875 t := it.t
876 bucket := it.bucket
877 b := it.bptr
878 i := it.i
879 checkBucket := it.checkBucket
880
881 next:
882 if b == nil {
883 if bucket == it.startBucket && it.wrapped {
884
885 it.key = nil
886 it.elem = nil
887 return
888 }
889 if h.growing() && it.B == h.B {
890
891
892
893
894 oldbucket := bucket & it.h.oldbucketmask()
895 b = (*bmap)(add(h.oldbuckets, oldbucket*uintptr(t.bucketsize)))
896 if !evacuated(b) {
897 checkBucket = bucket
898 } else {
899 b = (*bmap)(add(it.buckets, bucket*uintptr(t.bucketsize)))
900 checkBucket = noCheck
901 }
902 } else {
903 b = (*bmap)(add(it.buckets, bucket*uintptr(t.bucketsize)))
904 checkBucket = noCheck
905 }
906 bucket++
907 if bucket == bucketShift(it.B) {
908 bucket = 0
909 it.wrapped = true
910 }
911 i = 0
912 }
913 for ; i < bucketCnt; i++ {
914 offi := (i + it.offset) & (bucketCnt - 1)
915 if isEmpty(b.tophash[offi]) || b.tophash[offi] == evacuatedEmpty {
916
917
918 continue
919 }
920 k := add(unsafe.Pointer(b), dataOffset+uintptr(offi)*uintptr(t.keysize))
921 if t.indirectkey() {
922 k = *((*unsafe.Pointer)(k))
923 }
924 e := add(unsafe.Pointer(b), dataOffset+bucketCnt*uintptr(t.keysize)+uintptr(offi)*uintptr(t.elemsize))
925 if checkBucket != noCheck && !h.sameSizeGrow() {
926
927
928
929
930
931
932
933 if t.reflexivekey() || t.key.equal(k, k) {
934
935
936 hash := t.hasher(k, uintptr(h.hash0))
937 if hash&bucketMask(it.B) != checkBucket {
938 continue
939 }
940 } else {
941
942
943
944
945
946
947
948 if checkBucket>>(it.B-1) != uintptr(b.tophash[offi]&1) {
949 continue
950 }
951 }
952 }
953 if (b.tophash[offi] != evacuatedX && b.tophash[offi] != evacuatedY) ||
954 !(t.reflexivekey() || t.key.equal(k, k)) {
955
956
957
958
959 it.key = k
960 if t.indirectelem() {
961 e = *((*unsafe.Pointer)(e))
962 }
963 it.elem = e
964 } else {
965
966
967
968
969
970
971
972 rk, re := mapaccessK(t, h, k)
973 if rk == nil {
974 continue
975 }
976 it.key = rk
977 it.elem = re
978 }
979 it.bucket = bucket
980 if it.bptr != b {
981 it.bptr = b
982 }
983 it.i = i + 1
984 it.checkBucket = checkBucket
985 return
986 }
987 b = b.overflow(t)
988 i = 0
989 goto next
990 }
991
992
993 func mapclear(t *maptype, h *hmap) {
994 if raceenabled && h != nil {
995 callerpc := getcallerpc()
996 pc := abi.FuncPCABIInternal(mapclear)
997 racewritepc(unsafe.Pointer(h), callerpc, pc)
998 }
999
1000 if h == nil || h.count == 0 {
1001 return
1002 }
1003
1004 if h.flags&hashWriting != 0 {
1005 fatal("concurrent map writes")
1006 }
1007
1008 h.flags ^= hashWriting
1009
1010 h.flags &^= sameSizeGrow
1011 h.oldbuckets = nil
1012 h.nevacuate = 0
1013 h.noverflow = 0
1014 h.count = 0
1015
1016
1017
1018 h.hash0 = fastrand()
1019
1020
1021 if h.extra != nil {
1022 *h.extra = mapextra{}
1023 }
1024
1025
1026
1027
1028 _, nextOverflow := makeBucketArray(t, h.B, h.buckets)
1029 if nextOverflow != nil {
1030
1031
1032 h.extra.nextOverflow = nextOverflow
1033 }
1034
1035 if h.flags&hashWriting == 0 {
1036 fatal("concurrent map writes")
1037 }
1038 h.flags &^= hashWriting
1039 }
1040
1041 func hashGrow(t *maptype, h *hmap) {
1042
1043
1044
1045 bigger := uint8(1)
1046 if !overLoadFactor(h.count+1, h.B) {
1047 bigger = 0
1048 h.flags |= sameSizeGrow
1049 }
1050 oldbuckets := h.buckets
1051 newbuckets, nextOverflow := makeBucketArray(t, h.B+bigger, nil)
1052
1053 flags := h.flags &^ (iterator | oldIterator)
1054 if h.flags&iterator != 0 {
1055 flags |= oldIterator
1056 }
1057
1058 h.B += bigger
1059 h.flags = flags
1060 h.oldbuckets = oldbuckets
1061 h.buckets = newbuckets
1062 h.nevacuate = 0
1063 h.noverflow = 0
1064
1065 if h.extra != nil && h.extra.overflow != nil {
1066
1067 if h.extra.oldoverflow != nil {
1068 throw("oldoverflow is not nil")
1069 }
1070 h.extra.oldoverflow = h.extra.overflow
1071 h.extra.overflow = nil
1072 }
1073 if nextOverflow != nil {
1074 if h.extra == nil {
1075 h.extra = new(mapextra)
1076 }
1077 h.extra.nextOverflow = nextOverflow
1078 }
1079
1080
1081
1082 }
1083
1084
1085 func overLoadFactor(count int, B uint8) bool {
1086 return count > bucketCnt && uintptr(count) > loadFactorNum*(bucketShift(B)/loadFactorDen)
1087 }
1088
1089
1090
1091
1092 func tooManyOverflowBuckets(noverflow uint16, B uint8) bool {
1093
1094
1095
1096
1097 if B > 15 {
1098 B = 15
1099 }
1100
1101 return noverflow >= uint16(1)<<(B&15)
1102 }
1103
1104
1105 func (h *hmap) growing() bool {
1106 return h.oldbuckets != nil
1107 }
1108
1109
1110 func (h *hmap) sameSizeGrow() bool {
1111 return h.flags&sameSizeGrow != 0
1112 }
1113
1114
1115 func (h *hmap) noldbuckets() uintptr {
1116 oldB := h.B
1117 if !h.sameSizeGrow() {
1118 oldB--
1119 }
1120 return bucketShift(oldB)
1121 }
1122
1123
1124 func (h *hmap) oldbucketmask() uintptr {
1125 return h.noldbuckets() - 1
1126 }
1127
1128 func growWork(t *maptype, h *hmap, bucket uintptr) {
1129
1130
1131 evacuate(t, h, bucket&h.oldbucketmask())
1132
1133
1134 if h.growing() {
1135 evacuate(t, h, h.nevacuate)
1136 }
1137 }
1138
1139 func bucketEvacuated(t *maptype, h *hmap, bucket uintptr) bool {
1140 b := (*bmap)(add(h.oldbuckets, bucket*uintptr(t.bucketsize)))
1141 return evacuated(b)
1142 }
1143
1144
1145 type evacDst struct {
1146 b *bmap
1147 i int
1148 k unsafe.Pointer
1149 e unsafe.Pointer
1150 }
1151
1152 func evacuate(t *maptype, h *hmap, oldbucket uintptr) {
1153 b := (*bmap)(add(h.oldbuckets, oldbucket*uintptr(t.bucketsize)))
1154 newbit := h.noldbuckets()
1155 if !evacuated(b) {
1156
1157
1158
1159
1160 var xy [2]evacDst
1161 x := &xy[0]
1162 x.b = (*bmap)(add(h.buckets, oldbucket*uintptr(t.bucketsize)))
1163 x.k = add(unsafe.Pointer(x.b), dataOffset)
1164 x.e = add(x.k, bucketCnt*uintptr(t.keysize))
1165
1166 if !h.sameSizeGrow() {
1167
1168
1169 y := &xy[1]
1170 y.b = (*bmap)(add(h.buckets, (oldbucket+newbit)*uintptr(t.bucketsize)))
1171 y.k = add(unsafe.Pointer(y.b), dataOffset)
1172 y.e = add(y.k, bucketCnt*uintptr(t.keysize))
1173 }
1174
1175 for ; b != nil; b = b.overflow(t) {
1176 k := add(unsafe.Pointer(b), dataOffset)
1177 e := add(k, bucketCnt*uintptr(t.keysize))
1178 for i := 0; i < bucketCnt; i, k, e = i+1, add(k, uintptr(t.keysize)), add(e, uintptr(t.elemsize)) {
1179 top := b.tophash[i]
1180 if isEmpty(top) {
1181 b.tophash[i] = evacuatedEmpty
1182 continue
1183 }
1184 if top < minTopHash {
1185 throw("bad map state")
1186 }
1187 k2 := k
1188 if t.indirectkey() {
1189 k2 = *((*unsafe.Pointer)(k2))
1190 }
1191 var useY uint8
1192 if !h.sameSizeGrow() {
1193
1194
1195 hash := t.hasher(k2, uintptr(h.hash0))
1196 if h.flags&iterator != 0 && !t.reflexivekey() && !t.key.equal(k2, k2) {
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208 useY = top & 1
1209 top = tophash(hash)
1210 } else {
1211 if hash&newbit != 0 {
1212 useY = 1
1213 }
1214 }
1215 }
1216
1217 if evacuatedX+1 != evacuatedY || evacuatedX^1 != evacuatedY {
1218 throw("bad evacuatedN")
1219 }
1220
1221 b.tophash[i] = evacuatedX + useY
1222 dst := &xy[useY]
1223
1224 if dst.i == bucketCnt {
1225 dst.b = h.newoverflow(t, dst.b)
1226 dst.i = 0
1227 dst.k = add(unsafe.Pointer(dst.b), dataOffset)
1228 dst.e = add(dst.k, bucketCnt*uintptr(t.keysize))
1229 }
1230 dst.b.tophash[dst.i&(bucketCnt-1)] = top
1231 if t.indirectkey() {
1232 *(*unsafe.Pointer)(dst.k) = k2
1233 } else {
1234 typedmemmove(t.key, dst.k, k)
1235 }
1236 if t.indirectelem() {
1237 *(*unsafe.Pointer)(dst.e) = *(*unsafe.Pointer)(e)
1238 } else {
1239 typedmemmove(t.elem, dst.e, e)
1240 }
1241 dst.i++
1242
1243
1244
1245
1246 dst.k = add(dst.k, uintptr(t.keysize))
1247 dst.e = add(dst.e, uintptr(t.elemsize))
1248 }
1249 }
1250
1251 if h.flags&oldIterator == 0 && t.bucket.ptrdata != 0 {
1252 b := add(h.oldbuckets, oldbucket*uintptr(t.bucketsize))
1253
1254
1255 ptr := add(b, dataOffset)
1256 n := uintptr(t.bucketsize) - dataOffset
1257 memclrHasPointers(ptr, n)
1258 }
1259 }
1260
1261 if oldbucket == h.nevacuate {
1262 advanceEvacuationMark(h, t, newbit)
1263 }
1264 }
1265
1266 func advanceEvacuationMark(h *hmap, t *maptype, newbit uintptr) {
1267 h.nevacuate++
1268
1269
1270 stop := h.nevacuate + 1024
1271 if stop > newbit {
1272 stop = newbit
1273 }
1274 for h.nevacuate != stop && bucketEvacuated(t, h, h.nevacuate) {
1275 h.nevacuate++
1276 }
1277 if h.nevacuate == newbit {
1278
1279 h.oldbuckets = nil
1280
1281
1282
1283 if h.extra != nil {
1284 h.extra.oldoverflow = nil
1285 }
1286 h.flags &^= sameSizeGrow
1287 }
1288 }
1289
1290
1291
1292
1293 func reflect_makemap(t *maptype, cap int) *hmap {
1294
1295 if t.key.equal == nil {
1296 throw("runtime.reflect_makemap: unsupported map key type")
1297 }
1298 if t.key.size > maxKeySize && (!t.indirectkey() || t.keysize != uint8(goarch.PtrSize)) ||
1299 t.key.size <= maxKeySize && (t.indirectkey() || t.keysize != uint8(t.key.size)) {
1300 throw("key size wrong")
1301 }
1302 if t.elem.size > maxElemSize && (!t.indirectelem() || t.elemsize != uint8(goarch.PtrSize)) ||
1303 t.elem.size <= maxElemSize && (t.indirectelem() || t.elemsize != uint8(t.elem.size)) {
1304 throw("elem size wrong")
1305 }
1306 if t.key.align > bucketCnt {
1307 throw("key align too big")
1308 }
1309 if t.elem.align > bucketCnt {
1310 throw("elem align too big")
1311 }
1312 if t.key.size%uintptr(t.key.align) != 0 {
1313 throw("key size not a multiple of key align")
1314 }
1315 if t.elem.size%uintptr(t.elem.align) != 0 {
1316 throw("elem size not a multiple of elem align")
1317 }
1318 if bucketCnt < 8 {
1319 throw("bucketsize too small for proper alignment")
1320 }
1321 if dataOffset%uintptr(t.key.align) != 0 {
1322 throw("need padding in bucket (key)")
1323 }
1324 if dataOffset%uintptr(t.elem.align) != 0 {
1325 throw("need padding in bucket (elem)")
1326 }
1327
1328 return makemap(t, cap, nil)
1329 }
1330
1331
1332 func reflect_mapaccess(t *maptype, h *hmap, key unsafe.Pointer) unsafe.Pointer {
1333 elem, ok := mapaccess2(t, h, key)
1334 if !ok {
1335
1336 elem = nil
1337 }
1338 return elem
1339 }
1340
1341
1342 func reflect_mapaccess_faststr(t *maptype, h *hmap, key string) unsafe.Pointer {
1343 elem, ok := mapaccess2_faststr(t, h, key)
1344 if !ok {
1345
1346 elem = nil
1347 }
1348 return elem
1349 }
1350
1351
1352 func reflect_mapassign(t *maptype, h *hmap, key unsafe.Pointer, elem unsafe.Pointer) {
1353 p := mapassign(t, h, key)
1354 typedmemmove(t.elem, p, elem)
1355 }
1356
1357
1358 func reflect_mapassign_faststr(t *maptype, h *hmap, key string, elem unsafe.Pointer) {
1359 p := mapassign_faststr(t, h, key)
1360 typedmemmove(t.elem, p, elem)
1361 }
1362
1363
1364 func reflect_mapdelete(t *maptype, h *hmap, key unsafe.Pointer) {
1365 mapdelete(t, h, key)
1366 }
1367
1368
1369 func reflect_mapdelete_faststr(t *maptype, h *hmap, key string) {
1370 mapdelete_faststr(t, h, key)
1371 }
1372
1373
1374 func reflect_mapiterinit(t *maptype, h *hmap, it *hiter) {
1375 mapiterinit(t, h, it)
1376 }
1377
1378
1379 func reflect_mapiternext(it *hiter) {
1380 mapiternext(it)
1381 }
1382
1383
1384 func reflect_mapiterkey(it *hiter) unsafe.Pointer {
1385 return it.key
1386 }
1387
1388
1389 func reflect_mapiterelem(it *hiter) unsafe.Pointer {
1390 return it.elem
1391 }
1392
1393
1394 func reflect_maplen(h *hmap) int {
1395 if h == nil {
1396 return 0
1397 }
1398 if raceenabled {
1399 callerpc := getcallerpc()
1400 racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(reflect_maplen))
1401 }
1402 return h.count
1403 }
1404
1405
1406 func reflectlite_maplen(h *hmap) int {
1407 if h == nil {
1408 return 0
1409 }
1410 if raceenabled {
1411 callerpc := getcallerpc()
1412 racereadpc(unsafe.Pointer(h), callerpc, abi.FuncPCABIInternal(reflect_maplen))
1413 }
1414 return h.count
1415 }
1416
1417 const maxZero = 1024
1418 var zeroVal [maxZero]byte
1419
View as plain text