github.com/karrick/go@v0.0.0-20170817181416-d5b0ec858b37/src/runtime/iface.go (about) 1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "runtime/internal/atomic" 9 "runtime/internal/sys" 10 "unsafe" 11 ) 12 13 const itabInitSize = 512 14 15 var ( 16 itabLock mutex // lock for accessing itab table 17 itabTable = &itabTableInit // pointer to current table 18 itabTableInit = itabTableType{size: itabInitSize} // starter table 19 ) 20 21 //Note: change the formula in the mallocgc call in itabAdd if you change these fields. 22 type itabTableType struct { 23 size uintptr // length of entries array. Always a power of 2. 24 count uintptr // current number of filled entries. 25 entries [itabInitSize]*itab // really [size] large 26 } 27 28 func itabHashFunc(inter *interfacetype, typ *_type) uintptr { 29 // compiler has provided some good hash codes for us. 30 return uintptr(inter.typ.hash ^ typ.hash) 31 } 32 33 func getitab(inter *interfacetype, typ *_type, canfail bool) *itab { 34 if len(inter.mhdr) == 0 { 35 throw("internal error - misuse of itab") 36 } 37 38 // easy case 39 if typ.tflag&tflagUncommon == 0 { 40 if canfail { 41 return nil 42 } 43 name := inter.typ.nameOff(inter.mhdr[0].name) 44 panic(&TypeAssertionError{"", typ.string(), inter.typ.string(), name.name()}) 45 } 46 47 var m *itab 48 49 // First, look in the existing table to see if we can find the itab we need. 50 // This is by far the most common case, so do it without locks. 51 // Use atomic to ensure we see any previous writes done by the thread 52 // that updates the itabTable field (with atomic.Storep in addItab). 53 t := (*itabTableType)(atomic.Loadp(unsafe.Pointer(&itabTable))) 54 if m = t.find(inter, typ); m != nil { 55 goto finish 56 } 57 58 // Not found. Grab the lock and try again. 59 lock(&itabLock) 60 if m = itabTable.find(inter, typ); m != nil { 61 unlock(&itabLock) 62 goto finish 63 } 64 65 // Entry doesn't exist yet. Make a new entry & add it. 66 m = (*itab)(persistentalloc(unsafe.Sizeof(itab{})+uintptr(len(inter.mhdr)-1)*sys.PtrSize, 0, &memstats.other_sys)) 67 m.inter = inter 68 m._type = typ 69 m.init() 70 itabAdd(m) 71 unlock(&itabLock) 72 finish: 73 if m.fun[0] != 0 { 74 return m 75 } 76 if canfail { 77 return nil 78 } 79 // this can only happen if the conversion 80 // was already done once using the , ok form 81 // and we have a cached negative result. 82 // The cached result doesn't record which 83 // interface function was missing, so initialize 84 // the itab again to get the missing function name. 85 panic(&TypeAssertionError{concreteString: typ.string(), assertedString: inter.typ.string(), missingMethod: m.init()}) 86 } 87 88 // itabFind finds the given interface/type pair in t. 89 // Returns nil if the given interface/type pair isn't present. 90 func (t *itabTableType) find(inter *interfacetype, typ *_type) *itab { 91 // Implemented using quadratic probing. 92 // Probe sequence is h(i) = h0 + i*(i+1)/2 mod 2^k. 93 // We're guaranteed to hit all table entries using this probe sequence. 94 mask := t.size - 1 95 h := itabHashFunc(inter, typ) & mask 96 for i := uintptr(1); ; i++ { 97 p := (**itab)(add(unsafe.Pointer(&t.entries), h*sys.PtrSize)) 98 // Use atomic read here so if we see m != nil, we also see 99 // the initializations of the fields of m. 100 // m := *p 101 m := (*itab)(atomic.Loadp(unsafe.Pointer(p))) 102 if m == nil { 103 return nil 104 } 105 if m.inter == inter && m._type == typ { 106 return m 107 } 108 h += i 109 h &= mask 110 } 111 } 112 113 // itabAdd adds the given itab to the itab hash table. 114 // itabLock must be held. 115 func itabAdd(m *itab) { 116 t := itabTable 117 if t.count >= 3*(t.size/4) { // 75% load factor 118 // Grow hash table. Use an atomic write: see comment in getitab. 119 // t2 = new(itabTableType) + some additional entries 120 // We lie and tell malloc we want pointer-free memory because 121 // all the pointed-to values are not in the heap. 122 t2 := (*itabTableType)(mallocgc((2+2*t.size)*sys.PtrSize, nil, true)) 123 t2.size = t.size * 2 124 atomicstorep(unsafe.Pointer(&itabTable), unsafe.Pointer(t2)) 125 126 // Copy over entries. 127 // Note: while copying, other threads may look for an itab and 128 // fail to find it. That's ok, they will then try to get the itab lock 129 // and as a consequence wait until this copying is complete. 130 for i := uintptr(0); i < t.size; i++ { 131 if m2 := *(**itab)(add(unsafe.Pointer(&t.entries), i*sys.PtrSize)); m2 != nil { 132 itabAdd(m2) 133 } 134 } 135 if itabTable.count != t.count { 136 throw("mismatched count during itab table copy") 137 } 138 // Adopt the new table as our own. 139 t = itabTable 140 // Note: the old table can be GC'ed here. 141 } 142 // See comment in itabFind about the probe sequence. 143 // Insert new itab in the first empty spot in the probe sequence. 144 mask := t.size - 1 145 h := itabHashFunc(m.inter, m._type) & mask 146 for i := uintptr(1); ; i++ { 147 p := (**itab)(add(unsafe.Pointer(&t.entries), h*sys.PtrSize)) 148 m2 := *p 149 if m2 == m { 150 // A given itab may be used in more than one module 151 // and thanks to the way global symbol resolution works, the 152 // pointed-to itab may already have been inserted into the 153 // global 'hash'. 154 return 155 } 156 if m2 == nil { 157 // Use atomic write here so if a reader sees m, it also 158 // sees the correctly initialized fields of m. 159 // NoWB is ok because m is not in heap memory. 160 // *p = m 161 atomic.StorepNoWB(unsafe.Pointer(p), unsafe.Pointer(m)) 162 t.count++ 163 return 164 } 165 h += i 166 h &= mask 167 } 168 } 169 170 // init fills in the m.fun array with all the code pointers for 171 // the m.inter/m._type pair. If the type does not implement the interface, 172 // it sets m.fun[0] to 0 and returns the name of an interface function that is missing. 173 // It is ok to call this multiple times on the same m, even concurrently. 174 func (m *itab) init() string { 175 inter := m.inter 176 typ := m._type 177 x := typ.uncommon() 178 179 // both inter and typ have method sorted by name, 180 // and interface names are unique, 181 // so can iterate over both in lock step; 182 // the loop is O(ni+nt) not O(ni*nt). 183 ni := len(inter.mhdr) 184 nt := int(x.mcount) 185 xmhdr := (*[1 << 16]method)(add(unsafe.Pointer(x), uintptr(x.moff)))[:nt:nt] 186 j := 0 187 imethods: 188 for k := 0; k < ni; k++ { 189 i := &inter.mhdr[k] 190 itype := inter.typ.typeOff(i.ityp) 191 name := inter.typ.nameOff(i.name) 192 iname := name.name() 193 ipkg := name.pkgPath() 194 if ipkg == "" { 195 ipkg = inter.pkgpath.name() 196 } 197 for ; j < nt; j++ { 198 t := &xmhdr[j] 199 tname := typ.nameOff(t.name) 200 if typ.typeOff(t.mtyp) == itype && tname.name() == iname { 201 pkgPath := tname.pkgPath() 202 if pkgPath == "" { 203 pkgPath = typ.nameOff(x.pkgpath).name() 204 } 205 if tname.isExported() || pkgPath == ipkg { 206 if m != nil { 207 ifn := typ.textOff(t.ifn) 208 *(*unsafe.Pointer)(add(unsafe.Pointer(&m.fun[0]), uintptr(k)*sys.PtrSize)) = ifn 209 } 210 continue imethods 211 } 212 } 213 } 214 // didn't find method 215 m.fun[0] = 0 216 return iname 217 } 218 m.hash = typ.hash 219 return "" 220 } 221 222 func itabsinit() { 223 lock(&itabLock) 224 for _, md := range activeModules() { 225 for _, i := range md.itablinks { 226 itabAdd(i) 227 } 228 } 229 unlock(&itabLock) 230 } 231 232 // panicdottypeE is called when doing an e.(T) conversion and the conversion fails. 233 // have = the dynamic type we have. 234 // want = the static type we're trying to convert to. 235 // iface = the static type we're converting from. 236 func panicdottypeE(have, want, iface *_type) { 237 haveString := "" 238 if have != nil { 239 haveString = have.string() 240 } 241 panic(&TypeAssertionError{iface.string(), haveString, want.string(), ""}) 242 } 243 244 // panicdottypeI is called when doing an i.(T) conversion and the conversion fails. 245 // Same args as panicdottypeE, but "have" is the dynamic itab we have. 246 func panicdottypeI(have *itab, want, iface *_type) { 247 var t *_type 248 if have != nil { 249 t = have._type 250 } 251 panicdottypeE(t, want, iface) 252 } 253 254 // panicnildottype is called when doing a i.(T) conversion and the interface i is nil. 255 // want = the static type we're trying to convert to. 256 func panicnildottype(want *_type) { 257 panic(&TypeAssertionError{"", "", want.string(), ""}) 258 // TODO: Add the static type we're converting from as well. 259 // It might generate a better error message. 260 // Just to match other nil conversion errors, we don't for now. 261 } 262 263 // The conv and assert functions below do very similar things. 264 // The convXXX functions are guaranteed by the compiler to succeed. 265 // The assertXXX functions may fail (either panicking or returning false, 266 // depending on whether they are 1-result or 2-result). 267 // The convXXX functions succeed on a nil input, whereas the assertXXX 268 // functions fail on a nil input. 269 270 func convT2E(t *_type, elem unsafe.Pointer) (e eface) { 271 if raceenabled { 272 raceReadObjectPC(t, elem, getcallerpc(unsafe.Pointer(&t)), funcPC(convT2E)) 273 } 274 if msanenabled { 275 msanread(elem, t.size) 276 } 277 x := mallocgc(t.size, t, true) 278 // TODO: We allocate a zeroed object only to overwrite it with actual data. 279 // Figure out how to avoid zeroing. Also below in convT2Eslice, convT2I, convT2Islice. 280 typedmemmove(t, x, elem) 281 e._type = t 282 e.data = x 283 return 284 } 285 286 func convT2E16(t *_type, elem unsafe.Pointer) (e eface) { 287 if raceenabled { 288 raceReadObjectPC(t, elem, getcallerpc(unsafe.Pointer(&t)), funcPC(convT2E16)) 289 } 290 if msanenabled { 291 msanread(elem, t.size) 292 } 293 var x unsafe.Pointer 294 if *(*uint16)(elem) == 0 { 295 x = unsafe.Pointer(&zeroVal[0]) 296 } else { 297 x = mallocgc(2, t, false) 298 *(*uint16)(x) = *(*uint16)(elem) 299 } 300 e._type = t 301 e.data = x 302 return 303 } 304 305 func convT2E32(t *_type, elem unsafe.Pointer) (e eface) { 306 if raceenabled { 307 raceReadObjectPC(t, elem, getcallerpc(unsafe.Pointer(&t)), funcPC(convT2E32)) 308 } 309 if msanenabled { 310 msanread(elem, t.size) 311 } 312 var x unsafe.Pointer 313 if *(*uint32)(elem) == 0 { 314 x = unsafe.Pointer(&zeroVal[0]) 315 } else { 316 x = mallocgc(4, t, false) 317 *(*uint32)(x) = *(*uint32)(elem) 318 } 319 e._type = t 320 e.data = x 321 return 322 } 323 324 func convT2E64(t *_type, elem unsafe.Pointer) (e eface) { 325 if raceenabled { 326 raceReadObjectPC(t, elem, getcallerpc(unsafe.Pointer(&t)), funcPC(convT2E64)) 327 } 328 if msanenabled { 329 msanread(elem, t.size) 330 } 331 var x unsafe.Pointer 332 if *(*uint64)(elem) == 0 { 333 x = unsafe.Pointer(&zeroVal[0]) 334 } else { 335 x = mallocgc(8, t, false) 336 *(*uint64)(x) = *(*uint64)(elem) 337 } 338 e._type = t 339 e.data = x 340 return 341 } 342 343 func convT2Estring(t *_type, elem unsafe.Pointer) (e eface) { 344 if raceenabled { 345 raceReadObjectPC(t, elem, getcallerpc(unsafe.Pointer(&t)), funcPC(convT2Estring)) 346 } 347 if msanenabled { 348 msanread(elem, t.size) 349 } 350 var x unsafe.Pointer 351 if *(*string)(elem) == "" { 352 x = unsafe.Pointer(&zeroVal[0]) 353 } else { 354 x = mallocgc(t.size, t, true) 355 *(*string)(x) = *(*string)(elem) 356 } 357 e._type = t 358 e.data = x 359 return 360 } 361 362 func convT2Eslice(t *_type, elem unsafe.Pointer) (e eface) { 363 if raceenabled { 364 raceReadObjectPC(t, elem, getcallerpc(unsafe.Pointer(&t)), funcPC(convT2Eslice)) 365 } 366 if msanenabled { 367 msanread(elem, t.size) 368 } 369 var x unsafe.Pointer 370 if v := *(*slice)(elem); uintptr(v.array) == 0 { 371 x = unsafe.Pointer(&zeroVal[0]) 372 } else { 373 x = mallocgc(t.size, t, true) 374 *(*slice)(x) = *(*slice)(elem) 375 } 376 e._type = t 377 e.data = x 378 return 379 } 380 381 func convT2Enoptr(t *_type, elem unsafe.Pointer) (e eface) { 382 if raceenabled { 383 raceReadObjectPC(t, elem, getcallerpc(unsafe.Pointer(&t)), funcPC(convT2Enoptr)) 384 } 385 if msanenabled { 386 msanread(elem, t.size) 387 } 388 x := mallocgc(t.size, t, false) 389 memmove(x, elem, t.size) 390 e._type = t 391 e.data = x 392 return 393 } 394 395 func convT2I(tab *itab, elem unsafe.Pointer) (i iface) { 396 t := tab._type 397 if raceenabled { 398 raceReadObjectPC(t, elem, getcallerpc(unsafe.Pointer(&tab)), funcPC(convT2I)) 399 } 400 if msanenabled { 401 msanread(elem, t.size) 402 } 403 x := mallocgc(t.size, t, true) 404 typedmemmove(t, x, elem) 405 i.tab = tab 406 i.data = x 407 return 408 } 409 410 func convT2I16(tab *itab, elem unsafe.Pointer) (i iface) { 411 t := tab._type 412 if raceenabled { 413 raceReadObjectPC(t, elem, getcallerpc(unsafe.Pointer(&tab)), funcPC(convT2I16)) 414 } 415 if msanenabled { 416 msanread(elem, t.size) 417 } 418 var x unsafe.Pointer 419 if *(*uint16)(elem) == 0 { 420 x = unsafe.Pointer(&zeroVal[0]) 421 } else { 422 x = mallocgc(2, t, false) 423 *(*uint16)(x) = *(*uint16)(elem) 424 } 425 i.tab = tab 426 i.data = x 427 return 428 } 429 430 func convT2I32(tab *itab, elem unsafe.Pointer) (i iface) { 431 t := tab._type 432 if raceenabled { 433 raceReadObjectPC(t, elem, getcallerpc(unsafe.Pointer(&tab)), funcPC(convT2I32)) 434 } 435 if msanenabled { 436 msanread(elem, t.size) 437 } 438 var x unsafe.Pointer 439 if *(*uint32)(elem) == 0 { 440 x = unsafe.Pointer(&zeroVal[0]) 441 } else { 442 x = mallocgc(4, t, false) 443 *(*uint32)(x) = *(*uint32)(elem) 444 } 445 i.tab = tab 446 i.data = x 447 return 448 } 449 450 func convT2I64(tab *itab, elem unsafe.Pointer) (i iface) { 451 t := tab._type 452 if raceenabled { 453 raceReadObjectPC(t, elem, getcallerpc(unsafe.Pointer(&tab)), funcPC(convT2I64)) 454 } 455 if msanenabled { 456 msanread(elem, t.size) 457 } 458 var x unsafe.Pointer 459 if *(*uint64)(elem) == 0 { 460 x = unsafe.Pointer(&zeroVal[0]) 461 } else { 462 x = mallocgc(8, t, false) 463 *(*uint64)(x) = *(*uint64)(elem) 464 } 465 i.tab = tab 466 i.data = x 467 return 468 } 469 470 func convT2Istring(tab *itab, elem unsafe.Pointer) (i iface) { 471 t := tab._type 472 if raceenabled { 473 raceReadObjectPC(t, elem, getcallerpc(unsafe.Pointer(&tab)), funcPC(convT2Istring)) 474 } 475 if msanenabled { 476 msanread(elem, t.size) 477 } 478 var x unsafe.Pointer 479 if *(*string)(elem) == "" { 480 x = unsafe.Pointer(&zeroVal[0]) 481 } else { 482 x = mallocgc(t.size, t, true) 483 *(*string)(x) = *(*string)(elem) 484 } 485 i.tab = tab 486 i.data = x 487 return 488 } 489 490 func convT2Islice(tab *itab, elem unsafe.Pointer) (i iface) { 491 t := tab._type 492 if raceenabled { 493 raceReadObjectPC(t, elem, getcallerpc(unsafe.Pointer(&tab)), funcPC(convT2Islice)) 494 } 495 if msanenabled { 496 msanread(elem, t.size) 497 } 498 var x unsafe.Pointer 499 if v := *(*slice)(elem); uintptr(v.array) == 0 { 500 x = unsafe.Pointer(&zeroVal[0]) 501 } else { 502 x = mallocgc(t.size, t, true) 503 *(*slice)(x) = *(*slice)(elem) 504 } 505 i.tab = tab 506 i.data = x 507 return 508 } 509 510 func convT2Inoptr(tab *itab, elem unsafe.Pointer) (i iface) { 511 t := tab._type 512 if raceenabled { 513 raceReadObjectPC(t, elem, getcallerpc(unsafe.Pointer(&tab)), funcPC(convT2Inoptr)) 514 } 515 if msanenabled { 516 msanread(elem, t.size) 517 } 518 x := mallocgc(t.size, t, false) 519 memmove(x, elem, t.size) 520 i.tab = tab 521 i.data = x 522 return 523 } 524 525 func convI2I(inter *interfacetype, i iface) (r iface) { 526 tab := i.tab 527 if tab == nil { 528 return 529 } 530 if tab.inter == inter { 531 r.tab = tab 532 r.data = i.data 533 return 534 } 535 r.tab = getitab(inter, tab._type, false) 536 r.data = i.data 537 return 538 } 539 540 func assertI2I(inter *interfacetype, i iface) (r iface) { 541 tab := i.tab 542 if tab == nil { 543 // explicit conversions require non-nil interface value. 544 panic(&TypeAssertionError{"", "", inter.typ.string(), ""}) 545 } 546 if tab.inter == inter { 547 r.tab = tab 548 r.data = i.data 549 return 550 } 551 r.tab = getitab(inter, tab._type, false) 552 r.data = i.data 553 return 554 } 555 556 func assertI2I2(inter *interfacetype, i iface) (r iface, b bool) { 557 tab := i.tab 558 if tab == nil { 559 return 560 } 561 if tab.inter != inter { 562 tab = getitab(inter, tab._type, true) 563 if tab == nil { 564 return 565 } 566 } 567 r.tab = tab 568 r.data = i.data 569 b = true 570 return 571 } 572 573 func assertE2I(inter *interfacetype, e eface) (r iface) { 574 t := e._type 575 if t == nil { 576 // explicit conversions require non-nil interface value. 577 panic(&TypeAssertionError{"", "", inter.typ.string(), ""}) 578 } 579 r.tab = getitab(inter, t, false) 580 r.data = e.data 581 return 582 } 583 584 func assertE2I2(inter *interfacetype, e eface) (r iface, b bool) { 585 t := e._type 586 if t == nil { 587 return 588 } 589 tab := getitab(inter, t, true) 590 if tab == nil { 591 return 592 } 593 r.tab = tab 594 r.data = e.data 595 b = true 596 return 597 } 598 599 //go:linkname reflect_ifaceE2I reflect.ifaceE2I 600 func reflect_ifaceE2I(inter *interfacetype, e eface, dst *iface) { 601 *dst = assertE2I(inter, e) 602 } 603 604 func iterate_itabs(fn func(*itab)) { 605 // Note: only runs during stop the world, so no locks/atomics needed. 606 t := itabTable 607 for i := uintptr(0); i < t.size; i++ { 608 m := *(**itab)(add(unsafe.Pointer(&t.entries), i*sys.PtrSize)) 609 if m != nil { 610 fn(m) 611 } 612 } 613 } 614 615 // staticbytes is used to avoid convT2E for byte-sized values. 616 var staticbytes = [...]byte{ 617 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 618 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 619 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 620 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, 621 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 622 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 623 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 624 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 625 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 626 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 627 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 628 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, 629 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 630 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 631 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 632 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, 633 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 634 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 635 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 636 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 637 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 638 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, 639 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 640 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 641 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 642 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, 643 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, 644 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, 645 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 646 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, 647 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 648 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, 649 }