github.com/lrita/numa@v1.0.2/vdso_linux_amd64.go (about) 1 package numa 2 3 import ( 4 "os" 5 "unsafe" 6 ) 7 8 const ( 9 _AT_NULL = 0 // End of vector 10 _AT_SYSINFO_EHDR = 33 11 12 _PT_LOAD = 1 /* Loadable program segment */ 13 _PT_DYNAMIC = 2 /* Dynamic linking information */ 14 15 _DT_NULL = 0 /* Marks end of dynamic section */ 16 _DT_HASH = 4 /* Dynamic symbol hash table */ 17 _DT_STRTAB = 5 /* Address of string table */ 18 _DT_SYMTAB = 6 /* Address of symbol table */ 19 _DT_GNU_HASH = 0x6ffffef5 /* GNU-style dynamic symbol hash table */ 20 _DT_VERSYM = 0x6ffffff0 21 _DT_VERDEF = 0x6ffffffc 22 23 _VER_FLG_BASE = 0x1 /* Version definition of file itself */ 24 25 _SHN_UNDEF = 0 /* Undefined section */ 26 27 _SHT_DYNSYM = 11 /* Dynamic linker symbol table */ 28 29 _STT_FUNC = 2 /* Symbol is a code object */ 30 31 _STT_NOTYPE = 0 /* Symbol type is not specified */ 32 33 _STB_GLOBAL = 1 /* Global symbol */ 34 _STB_WEAK = 2 /* Weak symbol */ 35 36 _EI_NIDENT = 16 37 38 // vdsoArrayMax is the byte-size of a maximally sized array on this architecture. 39 // See cmd/compile/internal/amd64/galign.go arch.MAXWIDTH initialization. 40 vdsoArrayMax = 1<<50 - 1 41 42 // Maximum indices for the array types used when traversing the vDSO ELF structures. 43 // Computed from architecture-specific max provided by vdso_linux_*.go 44 vdsoSymTabSize = vdsoArrayMax / unsafe.Sizeof(elfSym{}) 45 vdsoDynSize = vdsoArrayMax / unsafe.Sizeof(elfDyn{}) 46 vdsoSymStringsSize = vdsoArrayMax // byte 47 vdsoVerSymSize = vdsoArrayMax / 2 // uint16 48 vdsoHashSize = vdsoArrayMax / 4 // uint32 49 50 // vdsoBloomSizeScale is a scaling factor for gnuhash tables which are uint32 indexed, 51 // but contain uintptrs 52 vdsoBloomSizeScale = unsafe.Sizeof(uintptr(0)) / 4 // uint32 53 ) 54 55 type vdsoVersionKey struct { 56 version string 57 verHash uint32 58 } 59 60 // ELF64 structure definitions for use by the vDSO loader 61 62 type elfSym struct { 63 st_name uint32 64 st_info byte 65 st_other byte 66 st_shndx uint16 67 st_value uint64 68 st_size uint64 69 } 70 71 type elfVerdef struct { 72 vd_version uint16 /* Version revision */ 73 vd_flags uint16 /* Version information */ 74 vd_ndx uint16 /* Version Index */ 75 vd_cnt uint16 /* Number of associated aux entries */ 76 vd_hash uint32 /* Version name hash value */ 77 vd_aux uint32 /* Offset in bytes to verdaux array */ 78 vd_next uint32 /* Offset in bytes to next verdef entry */ 79 } 80 81 type elfEhdr struct { 82 e_ident [_EI_NIDENT]byte /* Magic number and other info */ 83 e_type uint16 /* Object file type */ 84 e_machine uint16 /* Architecture */ 85 e_version uint32 /* Object file version */ 86 e_entry uint64 /* Entry point virtual address */ 87 e_phoff uint64 /* Program header table file offset */ 88 e_shoff uint64 /* Section header table file offset */ 89 e_flags uint32 /* Processor-specific flags */ 90 e_ehsize uint16 /* ELF header size in bytes */ 91 e_phentsize uint16 /* Program header table entry size */ 92 e_phnum uint16 /* Program header table entry count */ 93 e_shentsize uint16 /* Section header table entry size */ 94 e_shnum uint16 /* Section header table entry count */ 95 e_shstrndx uint16 /* Section header string table index */ 96 } 97 98 type elfPhdr struct { 99 p_type uint32 /* Segment type */ 100 p_flags uint32 /* Segment flags */ 101 p_offset uint64 /* Segment file offset */ 102 p_vaddr uint64 /* Segment virtual address */ 103 p_paddr uint64 /* Segment physical address */ 104 p_filesz uint64 /* Segment size in file */ 105 p_memsz uint64 /* Segment size in memory */ 106 p_align uint64 /* Segment alignment */ 107 } 108 109 type elfShdr struct { 110 sh_name uint32 /* Section name (string tbl index) */ 111 sh_type uint32 /* Section type */ 112 sh_flags uint64 /* Section flags */ 113 sh_addr uint64 /* Section virtual addr at execution */ 114 sh_offset uint64 /* Section file offset */ 115 sh_size uint64 /* Section size in bytes */ 116 sh_link uint32 /* Link to another section */ 117 sh_info uint32 /* Additional section information */ 118 sh_addralign uint64 /* Section alignment */ 119 sh_entsize uint64 /* Entry size if section holds table */ 120 } 121 122 type elfDyn struct { 123 d_tag int64 /* Dynamic entry type */ 124 d_val uint64 /* Integer value */ 125 } 126 127 type elfVerdaux struct { 128 vda_name uint32 /* Version or dependency names */ 129 vda_next uint32 /* Offset in bytes to next verdaux entry */ 130 } 131 132 type vdsoInfo struct { 133 valid bool 134 135 /* Load information */ 136 loadAddr unsafe.Pointer 137 loadOffset unsafe.Pointer /* loadAddr - recorded vaddr */ 138 139 /* Symbol table */ 140 symtab *[vdsoSymTabSize]elfSym 141 symstrings *[vdsoSymStringsSize]byte 142 chain []uint32 143 bucket []uint32 144 symOff uint32 145 isGNUHash bool 146 147 /* Version table */ 148 versym *[vdsoVerSymSize]uint16 149 verdef *elfVerdef 150 } 151 152 var ( 153 vdsoLinuxVersion = vdsoVersionKey{"LINUX_2.6", 0x3ae75f6} 154 vdsoinfo vdsoInfo 155 vdsoVersion int32 156 157 vdsoGetCPU uintptr 158 ) 159 160 /* How to extract and insert information held in the st_info field. */ 161 func _ELF_ST_BIND(val byte) byte { return val >> 4 } 162 func _ELF_ST_TYPE(val byte) byte { return val & 0xf } 163 164 //go:nosplit 165 func add(p unsafe.Pointer, x uintptr) unsafe.Pointer { 166 return unsafe.Pointer(uintptr(p) + x) 167 } 168 169 //go:linkname gostringnocopy runtime.gostringnocopy 170 //go:nosplit 171 func gostringnocopy(str *byte) string 172 173 func vdsoInitFromSysinfoEhdr(info *vdsoInfo, hdr *elfEhdr) { 174 info.valid = false 175 info.loadAddr = unsafe.Pointer(hdr) 176 177 pt := unsafe.Pointer(uintptr(info.loadAddr) + uintptr(hdr.e_phoff)) 178 179 // We need two things from the segment table: the load offset 180 // and the dynamic table. 181 var foundVaddr bool 182 var dyn *[vdsoDynSize]elfDyn 183 for i := uint16(0); i < hdr.e_phnum; i++ { 184 pt := (*elfPhdr)(add(pt, uintptr(i)*unsafe.Sizeof(elfPhdr{}))) 185 switch pt.p_type { 186 case _PT_LOAD: 187 if !foundVaddr { 188 foundVaddr = true 189 info.loadOffset = unsafe.Pointer(uintptr(info.loadAddr) + uintptr(pt.p_offset-pt.p_vaddr)) 190 } 191 192 case _PT_DYNAMIC: 193 dyn = (*[vdsoDynSize]elfDyn)(unsafe.Pointer(uintptr(info.loadAddr) + uintptr(pt.p_offset))) 194 } 195 } 196 197 if !foundVaddr || dyn == nil { 198 return // Failed 199 } 200 201 // Fish out the useful bits of the dynamic table. 202 203 var hash, gnuhash *[vdsoHashSize]uint32 204 info.symstrings = nil 205 info.symtab = nil 206 info.versym = nil 207 info.verdef = nil 208 for i := 0; dyn[i].d_tag != _DT_NULL; i++ { 209 dt := &dyn[i] 210 p := unsafe.Pointer(uintptr(info.loadOffset) + uintptr(dt.d_val)) 211 switch dt.d_tag { 212 case _DT_STRTAB: 213 info.symstrings = (*[vdsoSymStringsSize]byte)(unsafe.Pointer(p)) 214 case _DT_SYMTAB: 215 info.symtab = (*[vdsoSymTabSize]elfSym)(unsafe.Pointer(p)) 216 case _DT_HASH: 217 hash = (*[vdsoHashSize]uint32)(unsafe.Pointer(p)) 218 case _DT_GNU_HASH: 219 gnuhash = (*[vdsoHashSize]uint32)(unsafe.Pointer(p)) 220 case _DT_VERSYM: 221 info.versym = (*[vdsoVerSymSize]uint16)(unsafe.Pointer(p)) 222 case _DT_VERDEF: 223 info.verdef = (*elfVerdef)(unsafe.Pointer(p)) 224 } 225 } 226 227 if info.symstrings == nil || info.symtab == nil || (hash == nil && gnuhash == nil) { 228 return // Failed 229 } 230 231 if info.verdef == nil { 232 info.versym = nil 233 } 234 235 if gnuhash != nil { 236 // Parse the GNU hash table header. 237 nbucket := gnuhash[0] 238 info.symOff = gnuhash[1] 239 bloomSize := gnuhash[2] 240 info.bucket = gnuhash[4+bloomSize*uint32(vdsoBloomSizeScale):][:nbucket] 241 info.chain = gnuhash[4+bloomSize*uint32(vdsoBloomSizeScale)+nbucket:] 242 info.isGNUHash = true 243 } else { 244 // Parse the hash table header. 245 nbucket := hash[0] 246 nchain := hash[1] 247 info.bucket = hash[2 : 2+nbucket] 248 info.chain = hash[2+nbucket : 2+nbucket+nchain] 249 } 250 251 // That's all we need. 252 info.valid = true 253 } 254 255 func vdsoFindVersion(info *vdsoInfo, ver *vdsoVersionKey) int32 { 256 if !info.valid { 257 return 0 258 } 259 260 def := info.verdef 261 for { 262 if def.vd_flags&_VER_FLG_BASE == 0 { 263 aux := (*elfVerdaux)(add(unsafe.Pointer(def), uintptr(def.vd_aux))) 264 if def.vd_hash == ver.verHash && ver.version == gostringnocopy(&info.symstrings[aux.vda_name]) { 265 return int32(def.vd_ndx & 0x7fff) 266 } 267 } 268 269 if def.vd_next == 0 { 270 break 271 } 272 def = (*elfVerdef)(add(unsafe.Pointer(def), uintptr(def.vd_next))) 273 } 274 275 return -1 // cannot match any version 276 } 277 278 func vdsoParseSymbols(name string, info *vdsoInfo, version int32) uintptr { 279 if !info.valid { 280 return 0 281 } 282 283 load := func(symIndex uint32, name string) uintptr { 284 sym := &info.symtab[symIndex] 285 typ := _ELF_ST_TYPE(sym.st_info) 286 bind := _ELF_ST_BIND(sym.st_info) 287 // On ppc64x, VDSO functions are of type _STT_NOTYPE. 288 if typ != _STT_FUNC && typ != _STT_NOTYPE || bind != _STB_GLOBAL && bind != _STB_WEAK || sym.st_shndx == _SHN_UNDEF { 289 return 0 290 } 291 if name != gostringnocopy(&info.symstrings[sym.st_name]) { 292 return 0 293 } 294 // Check symbol version. 295 if info.versym != nil && version != 0 && int32(info.versym[symIndex]&0x7fff) != version { 296 return 0 297 } 298 299 return uintptr(info.loadOffset) + uintptr(sym.st_value) 300 } 301 302 if !info.isGNUHash { 303 // Old-style DT_HASH table. 304 hash := elfHash(name) 305 for chain := info.bucket[hash%uint32(len(info.bucket))]; chain != 0; chain = info.chain[chain] { 306 if p := load(chain, name); p != 0 { 307 return p 308 } 309 } 310 return 0 311 } 312 313 // New-style DT_GNU_HASH table. 314 gnuhash := elfGNUHash(name) 315 symIndex := info.bucket[gnuhash%uint32(len(info.bucket))] 316 if symIndex < info.symOff { 317 return 0 318 } 319 for ; ; symIndex++ { 320 hash := info.chain[symIndex-info.symOff] 321 if hash|1 == gnuhash|1 { 322 // Found a hash match. 323 if p := load(symIndex, name); p != 0 { 324 return p 325 } 326 } 327 if hash&1 != 0 { 328 // End of chain. 329 break 330 } 331 } 332 return 0 333 } 334 335 func elfHash(name string) (h uint32) { 336 for i := 0; i < len(name); i++ { 337 h = h<<4 + uint32(name[i]) 338 g := h & 0xf0000000 339 if g != 0 { 340 h ^= g >> 24 341 } 342 h &= ^g 343 } 344 return 345 } 346 347 func elfGNUHash(name string) (h uint32) { 348 h = 5381 349 for i := 0; i < len(name); i++ { 350 h = h*33 + uint32(name[i]) 351 } 352 return 353 } 354 355 func init() { 356 fd, err := os.Open("/proc/self/auxv") 357 if err != nil { 358 panic(err) 359 } 360 var auxv [128]uintptr 361 n, err := fd.Read((*(*[128 * unsafe.Sizeof(auxv[0])]byte)(unsafe.Pointer(&auxv)))[:]) 362 if err != nil || n <= 0 { 363 panic(err) 364 } 365 var base unsafe.Pointer 366 auxv[len(auxv)-2] = _AT_NULL // Make sure auxv is terminated, even if we didn't read the whole file. 367 for i := 0; auxv[i] != _AT_NULL; i += 2 { 368 tag, val := auxv[i], auxv[i+1] 369 if tag != _AT_SYSINFO_EHDR || val == 0 { 370 continue 371 } 372 vdsoInitFromSysinfoEhdr(&vdsoinfo, (*elfEhdr)(unsafe.Pointer(uintptr(base)+val))) 373 } 374 vdsoVersion = vdsoFindVersion(&vdsoinfo, &vdsoLinuxVersion) 375 initVDSOAll() 376 } 377 378 func vdsoSym(name string) uintptr { 379 return vdsoParseSymbols(name, &vdsoinfo, vdsoVersion) 380 } 381 382 func initVDSODefault(name string, def uintptr) uintptr { 383 if p := vdsoSym(name); p != 0 { 384 def = p 385 } 386 return def 387 } 388 389 func initVDSOAll() { 390 vdsoGetCPU = initVDSODefault("__vdso_getcpu", 0xffffffffff600800) 391 }