1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "internal/abi" 9 "internal/goarch" 10 "runtime/internal/math" 11 "unsafe" 12 ) 13 14 // Should be a built-in for unsafe.Pointer? 15 // 16 //go:nosplit 17 func add(p unsafe.Pointer, x uintptr) unsafe.Pointer { 18 return unsafe.Pointer(uintptr(p) + x) 19 } 20 21 // getg returns the pointer to the current g. 22 // The compiler rewrites calls to this function into instructions 23 // that fetch the g directly (from TLS or from the dedicated register). 24 func getg() *g 25 26 // mcall switches from the g to the g0 stack and invokes fn(g), 27 // where g is the goroutine that made the call. 28 // mcall saves g's current PC/SP in g->sched so that it can be restored later. 29 // It is up to fn to arrange for that later execution, typically by recording 30 // g in a data structure, causing something to call ready(g) later. 31 // mcall returns to the original goroutine g later, when g has been rescheduled. 32 // fn must not return at all; typically it ends by calling schedule, to let the m 33 // run other goroutines. 34 // 35 // mcall can only be called from g stacks (not g0, not gsignal). 36 // 37 // This must NOT be go:noescape: if fn is a stack-allocated closure, 38 // fn puts g on a run queue, and g executes before fn returns, the 39 // closure will be invalidated while it is still executing. 40 func mcall(fn func(*g)) 41 42 // systemstack runs fn on a system stack. 43 // If systemstack is called from the per-OS-thread (g0) stack, or 44 // if systemstack is called from the signal handling (gsignal) stack, 45 // systemstack calls fn directly and returns. 46 // Otherwise, systemstack is being called from the limited stack 47 // of an ordinary goroutine. In this case, systemstack switches 48 // to the per-OS-thread stack, calls fn, and switches back. 49 // It is common to use a func literal as the argument, in order 50 // to share inputs and outputs with the code around the call 51 // to system stack: 52 // 53 // ... set up y ... 54 // systemstack(func() { 55 // x = bigcall(y) 56 // }) 57 // ... use x ... 58 // 59 //go:noescape 60 func systemstack(fn func()) 61 62 var badsystemstackMsg = "fatal: systemstack called from unexpected goroutine" 63 64 //go:nosplit 65 //go:nowritebarrierrec 66 func badsystemstack() { 67 sp := stringStructOf(&badsystemstackMsg) 68 write(2, sp.str, int32(sp.len)) 69 } 70 71 // memclrNoHeapPointers clears n bytes starting at ptr. 72 // 73 // Usually you should use typedmemclr. memclrNoHeapPointers should be 74 // used only when the caller knows that *ptr contains no heap pointers 75 // because either: 76 // 77 // *ptr is initialized memory and its type is pointer-free, or 78 // 79 // *ptr is uninitialized memory (e.g., memory that's being reused 80 // for a new allocation) and hence contains only "junk". 81 // 82 // memclrNoHeapPointers ensures that if ptr is pointer-aligned, and n 83 // is a multiple of the pointer size, then any pointer-aligned, 84 // pointer-sized portion is cleared atomically. Despite the function 85 // name, this is necessary because this function is the underlying 86 // implementation of typedmemclr and memclrHasPointers. See the doc of 87 // memmove for more details. 88 // 89 // The (CPU-specific) implementations of this function are in memclr_*.s. 90 // 91 //go:noescape 92 func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) 93 94 //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers 95 func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) { 96 memclrNoHeapPointers(ptr, n) 97 } 98 99 // memmove copies n bytes from "from" to "to". 100 // 101 // memmove ensures that any pointer in "from" is written to "to" with 102 // an indivisible write, so that racy reads cannot observe a 103 // half-written pointer. This is necessary to prevent the garbage 104 // collector from observing invalid pointers, and differs from memmove 105 // in unmanaged languages. However, memmove is only required to do 106 // this if "from" and "to" may contain pointers, which can only be the 107 // case if "from", "to", and "n" are all be word-aligned. 108 // 109 // Implementations are in memmove_*.s. 110 // 111 //go:noescape 112 func memmove(to, from unsafe.Pointer, n uintptr) 113 114 // Outside assembly calls memmove. Make sure it has ABI wrappers. 115 // 116 //go:linkname memmove 117 118 //go:linkname reflect_memmove reflect.memmove 119 func reflect_memmove(to, from unsafe.Pointer, n uintptr) { 120 memmove(to, from, n) 121 } 122 123 // exported value for testing 124 const hashLoad = float32(loadFactorNum) / float32(loadFactorDen) 125 126 //go:nosplit 127 func fastrand() uint32 { 128 mp := getg().m 129 // Implement wyrand: https://github.com/wangyi-fudan/wyhash 130 // Only the platform that math.Mul64 can be lowered 131 // by the compiler should be in this list. 132 if goarch.IsAmd64|goarch.IsArm64|goarch.IsPpc64| 133 goarch.IsPpc64le|goarch.IsMips64|goarch.IsMips64le| 134 goarch.IsS390x|goarch.IsRiscv64 == 1 { 135 mp.fastrand += 0xa0761d6478bd642f 136 hi, lo := math.Mul64(mp.fastrand, mp.fastrand^0xe7037ed1a0b428db) 137 return uint32(hi ^ lo) 138 } 139 140 // Implement xorshift64+: 2 32-bit xorshift sequences added together. 141 // Shift triplet [17,7,16] was calculated as indicated in Marsaglia's 142 // Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf 143 // This generator passes the SmallCrush suite, part of TestU01 framework: 144 // http://simul.iro.umontreal.ca/testu01/tu01.html 145 t := (*[2]uint32)(unsafe.Pointer(&mp.fastrand)) 146 s1, s0 := t[0], t[1] 147 s1 ^= s1 << 17 148 s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16 149 t[0], t[1] = s0, s1 150 return s0 + s1 151 } 152 153 //go:nosplit 154 func fastrandn(n uint32) uint32 { 155 // This is similar to fastrand() % n, but faster. 156 // See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ 157 return uint32(uint64(fastrand()) * uint64(n) >> 32) 158 } 159 160 func fastrand64() uint64 { 161 mp := getg().m 162 // Implement wyrand: https://github.com/wangyi-fudan/wyhash 163 // Only the platform that math.Mul64 can be lowered 164 // by the compiler should be in this list. 165 if goarch.IsAmd64|goarch.IsArm64|goarch.IsPpc64| 166 goarch.IsPpc64le|goarch.IsMips64|goarch.IsMips64le| 167 goarch.IsS390x|goarch.IsRiscv64 == 1 { 168 mp.fastrand += 0xa0761d6478bd642f 169 hi, lo := math.Mul64(mp.fastrand, mp.fastrand^0xe7037ed1a0b428db) 170 return hi ^ lo 171 } 172 173 // Implement xorshift64+: 2 32-bit xorshift sequences added together. 174 // Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf 175 // This generator passes the SmallCrush suite, part of TestU01 framework: 176 // http://simul.iro.umontreal.ca/testu01/tu01.html 177 t := (*[2]uint32)(unsafe.Pointer(&mp.fastrand)) 178 s1, s0 := t[0], t[1] 179 s1 ^= s1 << 17 180 s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16 181 r := uint64(s0 + s1) 182 183 s0, s1 = s1, s0 184 s1 ^= s1 << 17 185 s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16 186 r += uint64(s0+s1) << 32 187 188 t[0], t[1] = s0, s1 189 return r 190 } 191 192 func fastrandu() uint { 193 if goarch.PtrSize == 4 { 194 return uint(fastrand()) 195 } 196 return uint(fastrand64()) 197 } 198 199 //go:linkname sync_fastrandn sync.fastrandn 200 func sync_fastrandn(n uint32) uint32 { return fastrandn(n) } 201 202 //go:linkname net_fastrandu net.fastrandu 203 func net_fastrandu() uint { return fastrandu() } 204 205 //go:linkname os_fastrand os.fastrand 206 func os_fastrand() uint32 { return fastrand() } 207 208 // in internal/bytealg/equal_*.s 209 // 210 //go:noescape 211 func memequal(a, b unsafe.Pointer, size uintptr) bool 212 213 // noescape hides a pointer from escape analysis. noescape is 214 // the identity function but escape analysis doesn't think the 215 // output depends on the input. noescape is inlined and currently 216 // compiles down to zero instructions. 217 // USE CAREFULLY! 218 // 219 //go:nosplit 220 func noescape(p unsafe.Pointer) unsafe.Pointer { 221 x := uintptr(p) 222 return unsafe.Pointer(x ^ 0) 223 } 224 225 // Not all cgocallback frames are actually cgocallback, 226 // so not all have these arguments. Mark them uintptr so that the GC 227 // does not misinterpret memory when the arguments are not present. 228 // cgocallback is not called from Go, only from crosscall2. 229 // This in turn calls cgocallbackg, which is where we'll find 230 // pointer-declared arguments. 231 func cgocallback(fn, frame, ctxt uintptr) 232 233 func gogo(buf *gobuf) 234 235 func asminit() 236 func setg(gg *g) 237 func breakpoint() 238 239 // reflectcall calls fn with arguments described by stackArgs, stackArgsSize, 240 // frameSize, and regArgs. 241 // 242 // Arguments passed on the stack and space for return values passed on the stack 243 // must be laid out at the space pointed to by stackArgs (with total length 244 // stackArgsSize) according to the ABI. 245 // 246 // stackRetOffset must be some value <= stackArgsSize that indicates the 247 // offset within stackArgs where the return value space begins. 248 // 249 // frameSize is the total size of the argument frame at stackArgs and must 250 // therefore be >= stackArgsSize. It must include additional space for spilling 251 // register arguments for stack growth and preemption. 252 // 253 // TODO(mknyszek): Once we don't need the additional spill space, remove frameSize, 254 // since frameSize will be redundant with stackArgsSize. 255 // 256 // Arguments passed in registers must be laid out in regArgs according to the ABI. 257 // regArgs will hold any return values passed in registers after the call. 258 // 259 // reflectcall copies stack arguments from stackArgs to the goroutine stack, and 260 // then copies back stackArgsSize-stackRetOffset bytes back to the return space 261 // in stackArgs once fn has completed. It also "unspills" argument registers from 262 // regArgs before calling fn, and spills them back into regArgs immediately 263 // following the call to fn. If there are results being returned on the stack, 264 // the caller should pass the argument frame type as stackArgsType so that 265 // reflectcall can execute appropriate write barriers during the copy. 266 // 267 // reflectcall expects regArgs.ReturnIsPtr to be populated indicating which 268 // registers on the return path will contain Go pointers. It will then store 269 // these pointers in regArgs.Ptrs such that they are visible to the GC. 270 // 271 // Package reflect passes a frame type. In package runtime, there is only 272 // one call that copies results back, in callbackWrap in syscall_windows.go, and it 273 // does NOT pass a frame type, meaning there are no write barriers invoked. See that 274 // call site for justification. 275 // 276 // Package reflect accesses this symbol through a linkname. 277 // 278 // Arguments passed through to reflectcall do not escape. The type is used 279 // only in a very limited callee of reflectcall, the stackArgs are copied, and 280 // regArgs is only used in the reflectcall frame. 281 // 282 //go:noescape 283 func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 284 285 func procyield(cycles uint32) 286 287 type neverCallThisFunction struct{} 288 289 // goexit is the return stub at the top of every goroutine call stack. 290 // Each goroutine stack is constructed as if goexit called the 291 // goroutine's entry point function, so that when the entry point 292 // function returns, it will return to goexit, which will call goexit1 293 // to perform the actual exit. 294 // 295 // This function must never be called directly. Call goexit1 instead. 296 // gentraceback assumes that goexit terminates the stack. A direct 297 // call on the stack will cause gentraceback to stop walking the stack 298 // prematurely and if there is leftover state it may panic. 299 func goexit(neverCallThisFunction) 300 301 // publicationBarrier performs a store/store barrier (a "publication" 302 // or "export" barrier). Some form of synchronization is required 303 // between initializing an object and making that object accessible to 304 // another processor. Without synchronization, the initialization 305 // writes and the "publication" write may be reordered, allowing the 306 // other processor to follow the pointer and observe an uninitialized 307 // object. In general, higher-level synchronization should be used, 308 // such as locking or an atomic pointer write. publicationBarrier is 309 // for when those aren't an option, such as in the implementation of 310 // the memory manager. 311 // 312 // There's no corresponding barrier for the read side because the read 313 // side naturally has a data dependency order. All architectures that 314 // Go supports or seems likely to ever support automatically enforce 315 // data dependency ordering. 316 func publicationBarrier() 317 318 // getcallerpc returns the program counter (PC) of its caller's caller. 319 // getcallersp returns the stack pointer (SP) of its caller's caller. 320 // The implementation may be a compiler intrinsic; there is not 321 // necessarily code implementing this on every platform. 322 // 323 // For example: 324 // 325 // func f(arg1, arg2, arg3 int) { 326 // pc := getcallerpc() 327 // sp := getcallersp() 328 // } 329 // 330 // These two lines find the PC and SP immediately following 331 // the call to f (where f will return). 332 // 333 // The call to getcallerpc and getcallersp must be done in the 334 // frame being asked about. 335 // 336 // The result of getcallersp is correct at the time of the return, 337 // but it may be invalidated by any subsequent call to a function 338 // that might relocate the stack in order to grow or shrink it. 339 // A general rule is that the result of getcallersp should be used 340 // immediately and can only be passed to nosplit functions. 341 342 //go:noescape 343 func getcallerpc() uintptr 344 345 //go:noescape 346 func getcallersp() uintptr // implemented as an intrinsic on all platforms 347 348 // getclosureptr returns the pointer to the current closure. 349 // getclosureptr can only be used in an assignment statement 350 // at the entry of a function. Moreover, go:nosplit directive 351 // must be specified at the declaration of caller function, 352 // so that the function prolog does not clobber the closure register. 353 // for example: 354 // 355 // //go:nosplit 356 // func f(arg1, arg2, arg3 int) { 357 // dx := getclosureptr() 358 // } 359 // 360 // The compiler rewrites calls to this function into instructions that fetch the 361 // pointer from a well-known register (DX on x86 architecture, etc.) directly. 362 func getclosureptr() uintptr 363 364 //go:noescape 365 func asmcgocall(fn, arg unsafe.Pointer) int32 366 367 func morestack() 368 func morestack_noctxt() 369 func rt0_go() 370 371 // return0 is a stub used to return 0 from deferproc. 372 // It is called at the very end of deferproc to signal 373 // the calling Go function that it should not jump 374 // to deferreturn. 375 // in asm_*.s 376 func return0() 377 378 // in asm_*.s 379 // not called directly; definitions here supply type information for traceback. 380 // These must have the same signature (arg pointer map) as reflectcall. 381 func call16(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 382 func call32(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 383 func call64(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 384 func call128(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 385 func call256(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 386 func call512(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 387 func call1024(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 388 func call2048(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 389 func call4096(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 390 func call8192(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 391 func call16384(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 392 func call32768(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 393 func call65536(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 394 func call131072(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 395 func call262144(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 396 func call524288(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 397 func call1048576(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 398 func call2097152(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 399 func call4194304(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 400 func call8388608(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 401 func call16777216(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 402 func call33554432(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 403 func call67108864(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 404 func call134217728(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 405 func call268435456(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 406 func call536870912(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 407 func call1073741824(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 408 409 func systemstack_switch() 410 411 // alignUp rounds n up to a multiple of a. a must be a power of 2. 412 func alignUp(n, a uintptr) uintptr { 413 return (n + a - 1) &^ (a - 1) 414 } 415 416 // alignDown rounds n down to a multiple of a. a must be a power of 2. 417 func alignDown(n, a uintptr) uintptr { 418 return n &^ (a - 1) 419 } 420 421 // divRoundUp returns ceil(n / a). 422 func divRoundUp(n, a uintptr) uintptr { 423 // a is generally a power of two. This will get inlined and 424 // the compiler will optimize the division. 425 return (n + a - 1) / a 426 } 427 428 // checkASM reports whether assembly runtime checks have passed. 429 func checkASM() bool 430 431 func memequal_varlen(a, b unsafe.Pointer) bool 432 433 // bool2int returns 0 if x is false or 1 if x is true. 434 func bool2int(x bool) int { 435 // Avoid branches. In the SSA compiler, this compiles to 436 // exactly what you would want it to. 437 return int(uint8(*(*uint8)(unsafe.Pointer(&x)))) 438 } 439 440 // abort crashes the runtime in situations where even throw might not 441 // work. In general it should do something a debugger will recognize 442 // (e.g., an INT3 on x86). A crash in abort is recognized by the 443 // signal handler, which will attempt to tear down the runtime 444 // immediately. 445 func abort() 446 447 // Called from compiled code; declared for vet; do NOT call from Go. 448 func gcWriteBarrier() 449 func duffzero() 450 func duffcopy() 451 452 // Called from linker-generated .initarray; declared for go vet; do NOT call from Go. 453 func addmoduledata() 454 455 // Injected by the signal handler for panicking signals. 456 // Initializes any registers that have fixed meaning at calls but 457 // are scratch in bodies and calls sigpanic. 458 // On many platforms it just jumps to sigpanic. 459 func sigpanic0() 460 461 // intArgRegs is used by the various register assignment 462 // algorithm implementations in the runtime. These include:. 463 // - Finalizers (mfinal.go) 464 // - Windows callbacks (syscall_windows.go) 465 // 466 // Both are stripped-down versions of the algorithm since they 467 // only have to deal with a subset of cases (finalizers only 468 // take a pointer or interface argument, Go Windows callbacks 469 // don't support floating point). 470 // 471 // It should be modified with care and are generally only 472 // modified when testing this package. 473 // 474 // It should never be set higher than its internal/abi 475 // constant counterparts, because the system relies on a 476 // structure that is at least large enough to hold the 477 // registers the system supports. 478 // 479 // Protected by finlock. 480 var intArgRegs = abi.IntArgRegs 481