github.com/mtsmfm/go/src@v0.0.0-20221020090648-44bdcb9f8fde/runtime/iface.go (about) 1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "internal/abi" 9 "internal/goarch" 10 "runtime/internal/atomic" 11 "unsafe" 12 ) 13 14 const itabInitSize = 512 15 16 var ( 17 itabLock mutex // lock for accessing itab table 18 itabTable = &itabTableInit // pointer to current table 19 itabTableInit = itabTableType{size: itabInitSize} // starter table 20 ) 21 22 // Note: change the formula in the mallocgc call in itabAdd if you change these fields. 23 type itabTableType struct { 24 size uintptr // length of entries array. Always a power of 2. 25 count uintptr // current number of filled entries. 26 entries [itabInitSize]*itab // really [size] large 27 } 28 29 func itabHashFunc(inter *interfacetype, typ *_type) uintptr { 30 // compiler has provided some good hash codes for us. 31 return uintptr(inter.typ.hash ^ typ.hash) 32 } 33 34 func getitab(inter *interfacetype, typ *_type, canfail bool) *itab { 35 if len(inter.mhdr) == 0 { 36 throw("internal error - misuse of itab") 37 } 38 39 // easy case 40 if typ.tflag&tflagUncommon == 0 { 41 if canfail { 42 return nil 43 } 44 name := inter.typ.nameOff(inter.mhdr[0].name) 45 panic(&TypeAssertionError{nil, typ, &inter.typ, name.name()}) 46 } 47 48 var m *itab 49 50 // First, look in the existing table to see if we can find the itab we need. 51 // This is by far the most common case, so do it without locks. 52 // Use atomic to ensure we see any previous writes done by the thread 53 // that updates the itabTable field (with atomic.Storep in itabAdd). 54 t := (*itabTableType)(atomic.Loadp(unsafe.Pointer(&itabTable))) 55 if m = t.find(inter, typ); m != nil { 56 goto finish 57 } 58 59 // Not found. Grab the lock and try again. 60 lock(&itabLock) 61 if m = itabTable.find(inter, typ); m != nil { 62 unlock(&itabLock) 63 goto finish 64 } 65 66 // Entry doesn't exist yet. Make a new entry & add it. 67 m = (*itab)(persistentalloc(unsafe.Sizeof(itab{})+uintptr(len(inter.mhdr)-1)*goarch.PtrSize, 0, &memstats.other_sys)) 68 m.inter = inter 69 m._type = typ 70 // The hash is used in type switches. However, compiler statically generates itab's 71 // for all interface/type pairs used in switches (which are added to itabTable 72 // in itabsinit). The dynamically-generated itab's never participate in type switches, 73 // and thus the hash is irrelevant. 74 // Note: m.hash is _not_ the hash used for the runtime itabTable hash table. 75 m.hash = 0 76 m.init() 77 itabAdd(m) 78 unlock(&itabLock) 79 finish: 80 if m.fun[0] != 0 { 81 return m 82 } 83 if canfail { 84 return nil 85 } 86 // this can only happen if the conversion 87 // was already done once using the , ok form 88 // and we have a cached negative result. 89 // The cached result doesn't record which 90 // interface function was missing, so initialize 91 // the itab again to get the missing function name. 92 panic(&TypeAssertionError{concrete: typ, asserted: &inter.typ, missingMethod: m.init()}) 93 } 94 95 // find finds the given interface/type pair in t. 96 // Returns nil if the given interface/type pair isn't present. 97 func (t *itabTableType) find(inter *interfacetype, typ *_type) *itab { 98 // Implemented using quadratic probing. 99 // Probe sequence is h(i) = h0 + i*(i+1)/2 mod 2^k. 100 // We're guaranteed to hit all table entries using this probe sequence. 101 mask := t.size - 1 102 h := itabHashFunc(inter, typ) & mask 103 for i := uintptr(1); ; i++ { 104 p := (**itab)(add(unsafe.Pointer(&t.entries), h*goarch.PtrSize)) 105 // Use atomic read here so if we see m != nil, we also see 106 // the initializations of the fields of m. 107 // m := *p 108 m := (*itab)(atomic.Loadp(unsafe.Pointer(p))) 109 if m == nil { 110 return nil 111 } 112 if m.inter == inter && m._type == typ { 113 return m 114 } 115 h += i 116 h &= mask 117 } 118 } 119 120 // itabAdd adds the given itab to the itab hash table. 121 // itabLock must be held. 122 func itabAdd(m *itab) { 123 // Bugs can lead to calling this while mallocing is set, 124 // typically because this is called while panicing. 125 // Crash reliably, rather than only when we need to grow 126 // the hash table. 127 if getg().m.mallocing != 0 { 128 throw("malloc deadlock") 129 } 130 131 t := itabTable 132 if t.count >= 3*(t.size/4) { // 75% load factor 133 // Grow hash table. 134 // t2 = new(itabTableType) + some additional entries 135 // We lie and tell malloc we want pointer-free memory because 136 // all the pointed-to values are not in the heap. 137 t2 := (*itabTableType)(mallocgc((2+2*t.size)*goarch.PtrSize, nil, true)) 138 t2.size = t.size * 2 139 140 // Copy over entries. 141 // Note: while copying, other threads may look for an itab and 142 // fail to find it. That's ok, they will then try to get the itab lock 143 // and as a consequence wait until this copying is complete. 144 iterate_itabs(t2.add) 145 if t2.count != t.count { 146 throw("mismatched count during itab table copy") 147 } 148 // Publish new hash table. Use an atomic write: see comment in getitab. 149 atomicstorep(unsafe.Pointer(&itabTable), unsafe.Pointer(t2)) 150 // Adopt the new table as our own. 151 t = itabTable 152 // Note: the old table can be GC'ed here. 153 } 154 t.add(m) 155 } 156 157 // add adds the given itab to itab table t. 158 // itabLock must be held. 159 func (t *itabTableType) add(m *itab) { 160 // See comment in find about the probe sequence. 161 // Insert new itab in the first empty spot in the probe sequence. 162 mask := t.size - 1 163 h := itabHashFunc(m.inter, m._type) & mask 164 for i := uintptr(1); ; i++ { 165 p := (**itab)(add(unsafe.Pointer(&t.entries), h*goarch.PtrSize)) 166 m2 := *p 167 if m2 == m { 168 // A given itab may be used in more than one module 169 // and thanks to the way global symbol resolution works, the 170 // pointed-to itab may already have been inserted into the 171 // global 'hash'. 172 return 173 } 174 if m2 == nil { 175 // Use atomic write here so if a reader sees m, it also 176 // sees the correctly initialized fields of m. 177 // NoWB is ok because m is not in heap memory. 178 // *p = m 179 atomic.StorepNoWB(unsafe.Pointer(p), unsafe.Pointer(m)) 180 t.count++ 181 return 182 } 183 h += i 184 h &= mask 185 } 186 } 187 188 // init fills in the m.fun array with all the code pointers for 189 // the m.inter/m._type pair. If the type does not implement the interface, 190 // it sets m.fun[0] to 0 and returns the name of an interface function that is missing. 191 // It is ok to call this multiple times on the same m, even concurrently. 192 func (m *itab) init() string { 193 inter := m.inter 194 typ := m._type 195 x := typ.uncommon() 196 197 // both inter and typ have method sorted by name, 198 // and interface names are unique, 199 // so can iterate over both in lock step; 200 // the loop is O(ni+nt) not O(ni*nt). 201 ni := len(inter.mhdr) 202 nt := int(x.mcount) 203 xmhdr := (*[1 << 16]method)(add(unsafe.Pointer(x), uintptr(x.moff)))[:nt:nt] 204 j := 0 205 methods := (*[1 << 16]unsafe.Pointer)(unsafe.Pointer(&m.fun[0]))[:ni:ni] 206 var fun0 unsafe.Pointer 207 imethods: 208 for k := 0; k < ni; k++ { 209 i := &inter.mhdr[k] 210 itype := inter.typ.typeOff(i.ityp) 211 name := inter.typ.nameOff(i.name) 212 iname := name.name() 213 ipkg := name.pkgPath() 214 if ipkg == "" { 215 ipkg = inter.pkgpath.name() 216 } 217 for ; j < nt; j++ { 218 t := &xmhdr[j] 219 tname := typ.nameOff(t.name) 220 if typ.typeOff(t.mtyp) == itype && tname.name() == iname { 221 pkgPath := tname.pkgPath() 222 if pkgPath == "" { 223 pkgPath = typ.nameOff(x.pkgpath).name() 224 } 225 if tname.isExported() || pkgPath == ipkg { 226 if m != nil { 227 ifn := typ.textOff(t.ifn) 228 if k == 0 { 229 fun0 = ifn // we'll set m.fun[0] at the end 230 } else { 231 methods[k] = ifn 232 } 233 } 234 continue imethods 235 } 236 } 237 } 238 // didn't find method 239 m.fun[0] = 0 240 return iname 241 } 242 m.fun[0] = uintptr(fun0) 243 return "" 244 } 245 246 func itabsinit() { 247 lockInit(&itabLock, lockRankItab) 248 lock(&itabLock) 249 for _, md := range activeModules() { 250 for _, i := range md.itablinks { 251 itabAdd(i) 252 } 253 } 254 unlock(&itabLock) 255 } 256 257 // panicdottypeE is called when doing an e.(T) conversion and the conversion fails. 258 // have = the dynamic type we have. 259 // want = the static type we're trying to convert to. 260 // iface = the static type we're converting from. 261 func panicdottypeE(have, want, iface *_type) { 262 panic(&TypeAssertionError{iface, have, want, ""}) 263 } 264 265 // panicdottypeI is called when doing an i.(T) conversion and the conversion fails. 266 // Same args as panicdottypeE, but "have" is the dynamic itab we have. 267 func panicdottypeI(have *itab, want, iface *_type) { 268 var t *_type 269 if have != nil { 270 t = have._type 271 } 272 panicdottypeE(t, want, iface) 273 } 274 275 // panicnildottype is called when doing a i.(T) conversion and the interface i is nil. 276 // want = the static type we're trying to convert to. 277 func panicnildottype(want *_type) { 278 panic(&TypeAssertionError{nil, nil, want, ""}) 279 // TODO: Add the static type we're converting from as well. 280 // It might generate a better error message. 281 // Just to match other nil conversion errors, we don't for now. 282 } 283 284 // The specialized convTx routines need a type descriptor to use when calling mallocgc. 285 // We don't need the type to be exact, just to have the correct size, alignment, and pointer-ness. 286 // However, when debugging, it'd be nice to have some indication in mallocgc where the types came from, 287 // so we use named types here. 288 // We then construct interface values of these types, 289 // and then extract the type word to use as needed. 290 type ( 291 uint16InterfacePtr uint16 292 uint32InterfacePtr uint32 293 uint64InterfacePtr uint64 294 stringInterfacePtr string 295 sliceInterfacePtr []byte 296 ) 297 298 var ( 299 uint16Eface any = uint16InterfacePtr(0) 300 uint32Eface any = uint32InterfacePtr(0) 301 uint64Eface any = uint64InterfacePtr(0) 302 stringEface any = stringInterfacePtr("") 303 sliceEface any = sliceInterfacePtr(nil) 304 305 uint16Type *_type = efaceOf(&uint16Eface)._type 306 uint32Type *_type = efaceOf(&uint32Eface)._type 307 uint64Type *_type = efaceOf(&uint64Eface)._type 308 stringType *_type = efaceOf(&stringEface)._type 309 sliceType *_type = efaceOf(&sliceEface)._type 310 ) 311 312 // The conv and assert functions below do very similar things. 313 // The convXXX functions are guaranteed by the compiler to succeed. 314 // The assertXXX functions may fail (either panicking or returning false, 315 // depending on whether they are 1-result or 2-result). 316 // The convXXX functions succeed on a nil input, whereas the assertXXX 317 // functions fail on a nil input. 318 319 // convT converts a value of type t, which is pointed to by v, to a pointer that can 320 // be used as the second word of an interface value. 321 func convT(t *_type, v unsafe.Pointer) unsafe.Pointer { 322 if raceenabled { 323 raceReadObjectPC(t, v, getcallerpc(), abi.FuncPCABIInternal(convT)) 324 } 325 if msanenabled { 326 msanread(v, t.size) 327 } 328 if asanenabled { 329 asanread(v, t.size) 330 } 331 x := mallocgc(t.size, t, true) 332 typedmemmove(t, x, v) 333 return x 334 } 335 func convTnoptr(t *_type, v unsafe.Pointer) unsafe.Pointer { 336 // TODO: maybe take size instead of type? 337 if raceenabled { 338 raceReadObjectPC(t, v, getcallerpc(), abi.FuncPCABIInternal(convTnoptr)) 339 } 340 if msanenabled { 341 msanread(v, t.size) 342 } 343 if asanenabled { 344 asanread(v, t.size) 345 } 346 347 x := mallocgc(t.size, t, false) 348 memmove(x, v, t.size) 349 return x 350 } 351 352 func convT16(val uint16) (x unsafe.Pointer) { 353 if val < uint16(len(staticuint64s)) { 354 x = unsafe.Pointer(&staticuint64s[val]) 355 if goarch.BigEndian { 356 x = add(x, 6) 357 } 358 } else { 359 x = mallocgc(2, uint16Type, false) 360 *(*uint16)(x) = val 361 } 362 return 363 } 364 365 func convT32(val uint32) (x unsafe.Pointer) { 366 if val < uint32(len(staticuint64s)) { 367 x = unsafe.Pointer(&staticuint64s[val]) 368 if goarch.BigEndian { 369 x = add(x, 4) 370 } 371 } else { 372 x = mallocgc(4, uint32Type, false) 373 *(*uint32)(x) = val 374 } 375 return 376 } 377 378 func convT64(val uint64) (x unsafe.Pointer) { 379 if val < uint64(len(staticuint64s)) { 380 x = unsafe.Pointer(&staticuint64s[val]) 381 } else { 382 x = mallocgc(8, uint64Type, false) 383 *(*uint64)(x) = val 384 } 385 return 386 } 387 388 func convTstring(val string) (x unsafe.Pointer) { 389 if val == "" { 390 x = unsafe.Pointer(&zeroVal[0]) 391 } else { 392 x = mallocgc(unsafe.Sizeof(val), stringType, true) 393 *(*string)(x) = val 394 } 395 return 396 } 397 398 func convTslice(val []byte) (x unsafe.Pointer) { 399 // Note: this must work for any element type, not just byte. 400 if (*slice)(unsafe.Pointer(&val)).array == nil { 401 x = unsafe.Pointer(&zeroVal[0]) 402 } else { 403 x = mallocgc(unsafe.Sizeof(val), sliceType, true) 404 *(*[]byte)(x) = val 405 } 406 return 407 } 408 409 // convI2I returns the new itab to be used for the destination value 410 // when converting a value with itab src to the dst interface. 411 func convI2I(dst *interfacetype, src *itab) *itab { 412 if src == nil { 413 return nil 414 } 415 if src.inter == dst { 416 return src 417 } 418 return getitab(dst, src._type, false) 419 } 420 421 func assertI2I(inter *interfacetype, tab *itab) *itab { 422 if tab == nil { 423 // explicit conversions require non-nil interface value. 424 panic(&TypeAssertionError{nil, nil, &inter.typ, ""}) 425 } 426 if tab.inter == inter { 427 return tab 428 } 429 return getitab(inter, tab._type, false) 430 } 431 432 func assertI2I2(inter *interfacetype, i iface) (r iface) { 433 tab := i.tab 434 if tab == nil { 435 return 436 } 437 if tab.inter != inter { 438 tab = getitab(inter, tab._type, true) 439 if tab == nil { 440 return 441 } 442 } 443 r.tab = tab 444 r.data = i.data 445 return 446 } 447 448 func assertE2I(inter *interfacetype, t *_type) *itab { 449 if t == nil { 450 // explicit conversions require non-nil interface value. 451 panic(&TypeAssertionError{nil, nil, &inter.typ, ""}) 452 } 453 return getitab(inter, t, false) 454 } 455 456 func assertE2I2(inter *interfacetype, e eface) (r iface) { 457 t := e._type 458 if t == nil { 459 return 460 } 461 tab := getitab(inter, t, true) 462 if tab == nil { 463 return 464 } 465 r.tab = tab 466 r.data = e.data 467 return 468 } 469 470 //go:linkname reflect_ifaceE2I reflect.ifaceE2I 471 func reflect_ifaceE2I(inter *interfacetype, e eface, dst *iface) { 472 *dst = iface{assertE2I(inter, e._type), e.data} 473 } 474 475 //go:linkname reflectlite_ifaceE2I internal/reflectlite.ifaceE2I 476 func reflectlite_ifaceE2I(inter *interfacetype, e eface, dst *iface) { 477 *dst = iface{assertE2I(inter, e._type), e.data} 478 } 479 480 func iterate_itabs(fn func(*itab)) { 481 // Note: only runs during stop the world or with itabLock held, 482 // so no other locks/atomics needed. 483 t := itabTable 484 for i := uintptr(0); i < t.size; i++ { 485 m := *(**itab)(add(unsafe.Pointer(&t.entries), i*goarch.PtrSize)) 486 if m != nil { 487 fn(m) 488 } 489 } 490 } 491 492 // staticuint64s is used to avoid allocating in convTx for small integer values. 493 var staticuint64s = [...]uint64{ 494 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 495 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 496 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 497 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 498 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 499 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 500 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 501 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 502 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 503 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 504 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 505 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 506 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 507 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 508 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 509 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 510 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 511 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 512 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 513 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 514 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 515 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 516 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 517 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 518 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 519 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 520 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 521 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 522 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 523 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 524 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 525 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 526 } 527 528 // The linker redirects a reference of a method that it determined 529 // unreachable to a reference to this function, so it will throw if 530 // ever called. 531 func unreachableMethod() { 532 throw("unreachable method called. linker bug?") 533 }