github.com/bananabytelabs/wazero@v0.0.0-20240105073314-54b22a776da8/internal/engine/wazevo/engine.go (about) 1 package wazevo 2 3 import ( 4 "context" 5 "encoding/hex" 6 "errors" 7 "fmt" 8 "runtime" 9 "sort" 10 "sync" 11 "unsafe" 12 13 "github.com/bananabytelabs/wazero/api" 14 "github.com/bananabytelabs/wazero/experimental" 15 "github.com/bananabytelabs/wazero/internal/engine/wazevo/backend" 16 "github.com/bananabytelabs/wazero/internal/engine/wazevo/frontend" 17 "github.com/bananabytelabs/wazero/internal/engine/wazevo/ssa" 18 "github.com/bananabytelabs/wazero/internal/engine/wazevo/wazevoapi" 19 "github.com/bananabytelabs/wazero/internal/filecache" 20 "github.com/bananabytelabs/wazero/internal/platform" 21 "github.com/bananabytelabs/wazero/internal/version" 22 "github.com/bananabytelabs/wazero/internal/wasm" 23 ) 24 25 type ( 26 // engine implements wasm.Engine. 27 engine struct { 28 wazeroVersion string 29 fileCache filecache.Cache 30 compiledModules map[wasm.ModuleID]*compiledModule 31 // sortedCompiledModules is a list of compiled modules sorted by the initial address of the executable. 32 sortedCompiledModules []*compiledModule 33 mux sync.RWMutex 34 // rels is a list of relocations to be resolved. This is reused for each compilation to avoid allocation. 35 rels []backend.RelocationInfo 36 // refToBinaryOffset is reused for each compilation to avoid allocation. 37 refToBinaryOffset map[ssa.FuncRef]int 38 // sharedFunctions is compiled functions shared by all modules. 39 sharedFunctions *sharedFunctions 40 // setFinalizer defaults to runtime.SetFinalizer, but overridable for tests. 41 setFinalizer func(obj interface{}, finalizer interface{}) 42 43 // The followings are reused for compiling shared functions. 44 machine backend.Machine 45 be backend.Compiler 46 } 47 48 sharedFunctions struct { 49 // memoryGrowExecutable is a compiled trampoline executable for memory.grow builtin function. 50 memoryGrowExecutable []byte 51 // checkModuleExitCode is a compiled trampoline executable for checking module instance exit code. This 52 // is used when ensureTermination is true. 53 checkModuleExitCode []byte 54 // stackGrowExecutable is a compiled executable for growing stack builtin function. 55 stackGrowExecutable []byte 56 // tableGrowExecutable is a compiled trampoline executable for table.grow builtin function. 57 tableGrowExecutable []byte 58 // refFuncExecutable is a compiled trampoline executable for ref.func builtin function. 59 refFuncExecutable []byte 60 listenerBeforeTrampolines map[*wasm.FunctionType][]byte 61 listenerAfterTrampolines map[*wasm.FunctionType][]byte 62 } 63 64 // compiledModule is a compiled variant of a wasm.Module and ready to be used for instantiation. 65 compiledModule struct { 66 *executables 67 // functionOffsets maps a local function index to the offset in the executable. 68 functionOffsets []int 69 parent *engine 70 module *wasm.Module 71 ensureTermination bool 72 listeners []experimental.FunctionListener 73 listenerBeforeTrampolines []*byte 74 listenerAfterTrampolines []*byte 75 76 // The followings are only available for non host modules. 77 78 offsets wazevoapi.ModuleContextOffsetData 79 sharedFunctions *sharedFunctions 80 sourceMap sourceMap 81 } 82 83 executables struct { 84 executable []byte 85 entryPreambles [][]byte 86 } 87 ) 88 89 // sourceMap is a mapping from the offset of the executable to the offset of the original wasm binary. 90 type sourceMap struct { 91 // executableOffsets is a sorted list of offsets of the executable. This is index-correlated with wasmBinaryOffsets, 92 // in other words executableOffsets[i] is the offset of the executable which corresponds to the offset of a Wasm 93 // binary pointed by wasmBinaryOffsets[i]. 94 executableOffsets []uintptr 95 // wasmBinaryOffsets is the counterpart of executableOffsets. 96 wasmBinaryOffsets []uint64 97 } 98 99 var _ wasm.Engine = (*engine)(nil) 100 101 // NewEngine returns the implementation of wasm.Engine. 102 func NewEngine(ctx context.Context, _ api.CoreFeatures, fc filecache.Cache) wasm.Engine { 103 machine := newMachine() 104 be := backend.NewCompiler(ctx, machine, ssa.NewBuilder()) 105 e := &engine{ 106 compiledModules: make(map[wasm.ModuleID]*compiledModule), refToBinaryOffset: make(map[ssa.FuncRef]int), 107 setFinalizer: runtime.SetFinalizer, 108 machine: machine, 109 be: be, 110 fileCache: fc, 111 wazeroVersion: version.GetWazeroVersion(), 112 } 113 e.compileSharedFunctions() 114 return e 115 } 116 117 // CompileModule implements wasm.Engine. 118 func (e *engine) CompileModule(ctx context.Context, module *wasm.Module, listeners []experimental.FunctionListener, ensureTermination bool) (err error) { 119 if wazevoapi.PerfMapEnabled { 120 wazevoapi.PerfMap.Lock() 121 defer wazevoapi.PerfMap.Unlock() 122 } 123 124 if _, ok, err := e.getCompiledModule(module, listeners, ensureTermination); ok { // cache hit! 125 return nil 126 } else if err != nil { 127 return err 128 } 129 130 if wazevoapi.DeterministicCompilationVerifierEnabled { 131 ctx = wazevoapi.NewDeterministicCompilationVerifierContext(ctx, len(module.CodeSection)) 132 } 133 cm, err := e.compileModule(ctx, module, listeners, ensureTermination) 134 if err != nil { 135 return err 136 } 137 if err = e.addCompiledModule(module, cm); err != nil { 138 return err 139 } 140 141 if wazevoapi.DeterministicCompilationVerifierEnabled { 142 for i := 0; i < wazevoapi.DeterministicCompilationVerifyingIter; i++ { 143 _, err := e.compileModule(ctx, module, listeners, ensureTermination) 144 if err != nil { 145 return err 146 } 147 } 148 } 149 150 if len(listeners) > 0 { 151 cm.listeners = listeners 152 cm.listenerBeforeTrampolines = make([]*byte, len(module.TypeSection)) 153 cm.listenerAfterTrampolines = make([]*byte, len(module.TypeSection)) 154 for i := range module.TypeSection { 155 typ := &module.TypeSection[i] 156 before, after := e.getListenerTrampolineForType(typ) 157 cm.listenerBeforeTrampolines[i] = before 158 cm.listenerAfterTrampolines[i] = after 159 } 160 } 161 return nil 162 } 163 164 func (exec *executables) compileEntryPreambles(m *wasm.Module, machine backend.Machine, be backend.Compiler) { 165 exec.entryPreambles = make([][]byte, len(m.TypeSection)) 166 for i := range m.TypeSection { 167 typ := &m.TypeSection[i] 168 sig := frontend.SignatureForWasmFunctionType(typ) 169 be.Init() 170 buf := machine.CompileEntryPreamble(&sig) 171 executable := mmapExecutable(buf) 172 exec.entryPreambles[i] = executable 173 174 if wazevoapi.PerfMapEnabled { 175 wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&executable[0])), 176 uint64(len(executable)), fmt.Sprintf("entry_preamble::type=%s", typ.String())) 177 } 178 } 179 } 180 181 func (e *engine) compileModule(ctx context.Context, module *wasm.Module, listeners []experimental.FunctionListener, ensureTermination bool) (*compiledModule, error) { 182 withListener := len(listeners) > 0 183 e.rels = e.rels[:0] 184 cm := &compiledModule{ 185 offsets: wazevoapi.NewModuleContextOffsetData(module, withListener), parent: e, module: module, 186 ensureTermination: ensureTermination, 187 executables: &executables{}, 188 } 189 190 if module.IsHostModule { 191 return e.compileHostModule(ctx, module, listeners) 192 } 193 194 importedFns, localFns := int(module.ImportFunctionCount), len(module.FunctionSection) 195 if localFns == 0 { 196 return cm, nil 197 } 198 199 if wazevoapi.DeterministicCompilationVerifierEnabled { 200 // The compilation must be deterministic regardless of the order of functions being compiled. 201 wazevoapi.DeterministicCompilationVerifierRandomizeIndexes(ctx) 202 } 203 204 needSourceInfo := module.DWARFLines != nil 205 206 // Creates new compiler instances which are reused for each function. 207 ssaBuilder := ssa.NewBuilder() 208 fe := frontend.NewFrontendCompiler(module, ssaBuilder, &cm.offsets, ensureTermination, withListener, needSourceInfo) 209 machine := newMachine() 210 be := backend.NewCompiler(ctx, machine, ssaBuilder) 211 212 cm.executables.compileEntryPreambles(module, machine, be) 213 214 totalSize := 0 // Total binary size of the executable. 215 cm.functionOffsets = make([]int, localFns) 216 bodies := make([][]byte, localFns) 217 for i := range module.CodeSection { 218 if wazevoapi.DeterministicCompilationVerifierEnabled { 219 i = wazevoapi.DeterministicCompilationVerifierGetRandomizedLocalFunctionIndex(ctx, i) 220 } 221 222 fidx := wasm.Index(i + importedFns) 223 224 if wazevoapi.NeedFunctionNameInContext { 225 def := module.FunctionDefinition(fidx) 226 name := def.DebugName() 227 if len(def.ExportNames()) > 0 { 228 name = def.ExportNames()[0] 229 } 230 ctx = wazevoapi.SetCurrentFunctionName(ctx, i, fmt.Sprintf("[%d/%d]%s", i, len(module.CodeSection)-1, name)) 231 } 232 233 needListener := len(listeners) > 0 && listeners[i] != nil 234 body, rels, err := e.compileLocalWasmFunction(ctx, module, wasm.Index(i), fe, ssaBuilder, be, needListener) 235 if err != nil { 236 return nil, fmt.Errorf("compile function %d/%d: %v", i, len(module.CodeSection)-1, err) 237 } 238 239 // Align 16-bytes boundary. 240 totalSize = (totalSize + 15) &^ 15 241 cm.functionOffsets[i] = totalSize 242 243 if needSourceInfo { 244 // At the beginning of the function, we add the offset of the function body so that 245 // we can resolve the source location of the call site of before listener call. 246 cm.sourceMap.executableOffsets = append(cm.sourceMap.executableOffsets, uintptr(totalSize)) 247 cm.sourceMap.wasmBinaryOffsets = append(cm.sourceMap.wasmBinaryOffsets, module.CodeSection[i].BodyOffsetInCodeSection) 248 249 for _, info := range be.SourceOffsetInfo() { 250 cm.sourceMap.executableOffsets = append(cm.sourceMap.executableOffsets, uintptr(totalSize)+uintptr(info.ExecutableOffset)) 251 cm.sourceMap.wasmBinaryOffsets = append(cm.sourceMap.wasmBinaryOffsets, uint64(info.SourceOffset)) 252 } 253 } 254 255 fref := frontend.FunctionIndexToFuncRef(fidx) 256 e.refToBinaryOffset[fref] = totalSize 257 258 // At this point, relocation offsets are relative to the start of the function body, 259 // so we adjust it to the start of the executable. 260 for _, r := range rels { 261 r.Offset += int64(totalSize) 262 e.rels = append(e.rels, r) 263 } 264 265 bodies[i] = body 266 totalSize += len(body) 267 if wazevoapi.PrintMachineCodeHexPerFunction { 268 fmt.Printf("[[[machine code for %s]]]\n%s\n\n", wazevoapi.GetCurrentFunctionName(ctx), hex.EncodeToString(body)) 269 } 270 } 271 272 // Allocate executable memory and then copy the generated machine code. 273 executable, err := platform.MmapCodeSegment(totalSize) 274 if err != nil { 275 panic(err) 276 } 277 cm.executable = executable 278 279 for i, b := range bodies { 280 offset := cm.functionOffsets[i] 281 copy(executable[offset:], b) 282 } 283 284 if wazevoapi.PerfMapEnabled { 285 wazevoapi.PerfMap.Flush(uintptr(unsafe.Pointer(&executable[0])), cm.functionOffsets) 286 } 287 288 if needSourceInfo { 289 for i := range cm.sourceMap.executableOffsets { 290 cm.sourceMap.executableOffsets[i] += uintptr(unsafe.Pointer(&cm.executable[0])) 291 } 292 } 293 294 // Resolve relocations for local function calls. 295 machine.ResolveRelocations(e.refToBinaryOffset, executable, e.rels) 296 297 if runtime.GOARCH == "arm64" { 298 // On arm64, we cannot give all of rwx at the same time, so we change it to exec. 299 if err = platform.MprotectRX(executable); err != nil { 300 return nil, err 301 } 302 } 303 cm.sharedFunctions = e.sharedFunctions 304 e.setFinalizer(cm.executables, executablesFinalizer) 305 return cm, nil 306 } 307 308 func (e *engine) compileLocalWasmFunction( 309 ctx context.Context, 310 module *wasm.Module, 311 localFunctionIndex wasm.Index, 312 fe *frontend.Compiler, 313 ssaBuilder ssa.Builder, 314 be backend.Compiler, 315 needListener bool, 316 ) (body []byte, rels []backend.RelocationInfo, err error) { 317 typIndex := module.FunctionSection[localFunctionIndex] 318 typ := &module.TypeSection[typIndex] 319 codeSeg := &module.CodeSection[localFunctionIndex] 320 321 // Initializes both frontend and backend compilers. 322 fe.Init(localFunctionIndex, typIndex, typ, codeSeg.LocalTypes, codeSeg.Body, needListener, codeSeg.BodyOffsetInCodeSection) 323 be.Init() 324 325 // Lower Wasm to SSA. 326 fe.LowerToSSA() 327 if wazevoapi.PrintSSA && wazevoapi.PrintEnabledIndex(ctx) { 328 fmt.Printf("[[[SSA for %s]]]%s\n", wazevoapi.GetCurrentFunctionName(ctx), ssaBuilder.Format()) 329 } 330 331 if wazevoapi.DeterministicCompilationVerifierEnabled { 332 wazevoapi.VerifyOrSetDeterministicCompilationContextValue(ctx, "SSA", ssaBuilder.Format()) 333 } 334 335 // Run SSA-level optimization passes. 336 ssaBuilder.RunPasses() 337 338 if wazevoapi.PrintOptimizedSSA && wazevoapi.PrintEnabledIndex(ctx) { 339 fmt.Printf("[[[Optimized SSA for %s]]]%s\n", wazevoapi.GetCurrentFunctionName(ctx), ssaBuilder.Format()) 340 } 341 342 if wazevoapi.DeterministicCompilationVerifierEnabled { 343 wazevoapi.VerifyOrSetDeterministicCompilationContextValue(ctx, "Optimized SSA", ssaBuilder.Format()) 344 } 345 346 // Finalize the layout of SSA blocks which might use the optimization results. 347 ssaBuilder.LayoutBlocks() 348 349 if wazevoapi.PrintBlockLaidOutSSA && wazevoapi.PrintEnabledIndex(ctx) { 350 fmt.Printf("[[[Laidout SSA for %s]]]%s\n", wazevoapi.GetCurrentFunctionName(ctx), ssaBuilder.Format()) 351 } 352 353 if wazevoapi.DeterministicCompilationVerifierEnabled { 354 wazevoapi.VerifyOrSetDeterministicCompilationContextValue(ctx, "Block laid out SSA", ssaBuilder.Format()) 355 } 356 357 // Now our ssaBuilder contains the necessary information to further lower them to 358 // machine code. 359 original, rels, err := be.Compile(ctx) 360 if err != nil { 361 return nil, nil, fmt.Errorf("ssa->machine code: %v", err) 362 } 363 364 // TODO: optimize as zero copy. 365 copied := make([]byte, len(original)) 366 copy(copied, original) 367 return copied, rels, nil 368 } 369 370 func (e *engine) compileHostModule(ctx context.Context, module *wasm.Module, listeners []experimental.FunctionListener) (*compiledModule, error) { 371 machine := newMachine() 372 be := backend.NewCompiler(ctx, machine, ssa.NewBuilder()) 373 374 num := len(module.CodeSection) 375 cm := &compiledModule{module: module, listeners: listeners, executables: &executables{}} 376 cm.functionOffsets = make([]int, num) 377 totalSize := 0 // Total binary size of the executable. 378 bodies := make([][]byte, num) 379 var sig ssa.Signature 380 for i := range module.CodeSection { 381 totalSize = (totalSize + 15) &^ 15 382 cm.functionOffsets[i] = totalSize 383 384 typIndex := module.FunctionSection[i] 385 typ := &module.TypeSection[typIndex] 386 387 // We can relax until the index fits together in ExitCode as we do in wazevoapi.ExitCodeCallGoModuleFunctionWithIndex. 388 // However, 1 << 16 should be large enough for a real use case. 389 const hostFunctionNumMaximum = 1 << 16 390 if i >= hostFunctionNumMaximum { 391 return nil, fmt.Errorf("too many host functions (maximum %d)", hostFunctionNumMaximum) 392 } 393 394 sig.ID = ssa.SignatureID(typIndex) // This is important since we reuse the `machine` which caches the ABI based on the SignatureID. 395 sig.Params = append(sig.Params[:0], 396 ssa.TypeI64, // First argument must be exec context. 397 ssa.TypeI64, // The second argument is the moduleContextOpaque of this host module. 398 ) 399 for _, t := range typ.Params { 400 sig.Params = append(sig.Params, frontend.WasmTypeToSSAType(t)) 401 } 402 403 sig.Results = sig.Results[:0] 404 for _, t := range typ.Results { 405 sig.Results = append(sig.Results, frontend.WasmTypeToSSAType(t)) 406 } 407 408 c := &module.CodeSection[i] 409 if c.GoFunc == nil { 410 panic("BUG: GoFunc must be set for host module") 411 } 412 413 withListener := len(listeners) > 0 && listeners[i] != nil 414 var exitCode wazevoapi.ExitCode 415 fn := c.GoFunc 416 switch fn.(type) { 417 case api.GoModuleFunction: 418 exitCode = wazevoapi.ExitCodeCallGoModuleFunctionWithIndex(i, withListener) 419 case api.GoFunction: 420 exitCode = wazevoapi.ExitCodeCallGoFunctionWithIndex(i, withListener) 421 } 422 423 be.Init() 424 machine.CompileGoFunctionTrampoline(exitCode, &sig, true) 425 be.Encode() 426 body := be.Buf() 427 428 if wazevoapi.PerfMapEnabled { 429 name := module.FunctionDefinition(wasm.Index(i)).DebugName() 430 wazevoapi.PerfMap.AddModuleEntry(i, 431 int64(totalSize), 432 uint64(len(body)), 433 fmt.Sprintf("trampoline:%s", name)) 434 } 435 436 // TODO: optimize as zero copy. 437 copied := make([]byte, len(body)) 438 copy(copied, body) 439 bodies[i] = copied 440 totalSize += len(body) 441 } 442 443 // Allocate executable memory and then copy the generated machine code. 444 executable, err := platform.MmapCodeSegment(totalSize) 445 if err != nil { 446 panic(err) 447 } 448 cm.executable = executable 449 450 for i, b := range bodies { 451 offset := cm.functionOffsets[i] 452 copy(executable[offset:], b) 453 } 454 455 if wazevoapi.PerfMapEnabled { 456 wazevoapi.PerfMap.Flush(uintptr(unsafe.Pointer(&executable[0])), cm.functionOffsets) 457 } 458 459 if runtime.GOARCH == "arm64" { 460 // On arm64, we cannot give all of rwx at the same time, so we change it to exec. 461 if err = platform.MprotectRX(executable); err != nil { 462 return nil, err 463 } 464 } 465 e.setFinalizer(cm.executables, executablesFinalizer) 466 return cm, nil 467 } 468 469 // Close implements wasm.Engine. 470 func (e *engine) Close() (err error) { 471 e.mux.Lock() 472 defer e.mux.Unlock() 473 474 for _, cm := range e.compiledModules { 475 cm.functionOffsets = nil 476 cm.module = nil 477 cm.parent = nil 478 cm.executables = nil 479 } 480 e.sortedCompiledModules = nil 481 e.compiledModules = nil 482 e.sharedFunctions = nil 483 return nil 484 } 485 486 // CompiledModuleCount implements wasm.Engine. 487 func (e *engine) CompiledModuleCount() uint32 { 488 e.mux.RLock() 489 defer e.mux.RUnlock() 490 return uint32(len(e.compiledModules)) 491 } 492 493 // DeleteCompiledModule implements wasm.Engine. 494 func (e *engine) DeleteCompiledModule(m *wasm.Module) { 495 e.mux.Lock() 496 defer e.mux.Unlock() 497 cm, ok := e.compiledModules[m.ID] 498 if ok { 499 if len(cm.executable) > 0 { 500 e.deleteCompiledModuleFromSortedList(cm) 501 } 502 delete(e.compiledModules, m.ID) 503 } 504 } 505 506 func (e *engine) addCompiledModuleToSortedList(cm *compiledModule) { 507 ptr := uintptr(unsafe.Pointer(&cm.executable[0])) 508 509 index := sort.Search(len(e.sortedCompiledModules), func(i int) bool { 510 return uintptr(unsafe.Pointer(&e.sortedCompiledModules[i].executable[0])) >= ptr 511 }) 512 e.sortedCompiledModules = append(e.sortedCompiledModules, nil) 513 copy(e.sortedCompiledModules[index+1:], e.sortedCompiledModules[index:]) 514 e.sortedCompiledModules[index] = cm 515 } 516 517 func (e *engine) deleteCompiledModuleFromSortedList(cm *compiledModule) { 518 ptr := uintptr(unsafe.Pointer(&cm.executable[0])) 519 520 index := sort.Search(len(e.sortedCompiledModules), func(i int) bool { 521 return uintptr(unsafe.Pointer(&e.sortedCompiledModules[i].executable[0])) >= ptr 522 }) 523 if index >= len(e.sortedCompiledModules) { 524 return 525 } 526 copy(e.sortedCompiledModules[index:], e.sortedCompiledModules[index+1:]) 527 e.sortedCompiledModules = e.sortedCompiledModules[:len(e.sortedCompiledModules)-1] 528 } 529 530 func (e *engine) compiledModuleOfAddr(addr uintptr) *compiledModule { 531 e.mux.RLock() 532 defer e.mux.RUnlock() 533 534 index := sort.Search(len(e.sortedCompiledModules), func(i int) bool { 535 return uintptr(unsafe.Pointer(&e.sortedCompiledModules[i].executable[0])) > addr 536 }) 537 index -= 1 538 if index < 0 { 539 return nil 540 } 541 return e.sortedCompiledModules[index] 542 } 543 544 // NewModuleEngine implements wasm.Engine. 545 func (e *engine) NewModuleEngine(m *wasm.Module, mi *wasm.ModuleInstance) (wasm.ModuleEngine, error) { 546 me := &moduleEngine{} 547 548 // Note: imported functions are resolved in moduleEngine.ResolveImportedFunction. 549 me.importedFunctions = make([]importedFunction, m.ImportFunctionCount) 550 551 compiled, ok := e.compiledModules[m.ID] 552 if !ok { 553 return nil, errors.New("source module must be compiled before instantiation") 554 } 555 me.parent = compiled 556 me.module = mi 557 me.listeners = compiled.listeners 558 559 if m.IsHostModule { 560 me.opaque = buildHostModuleOpaque(m, compiled.listeners) 561 me.opaquePtr = &me.opaque[0] 562 } else { 563 if size := compiled.offsets.TotalSize; size != 0 { 564 opaque := make([]byte, size) 565 me.opaque = opaque 566 me.opaquePtr = &opaque[0] 567 } 568 } 569 return me, nil 570 } 571 572 func (e *engine) compileSharedFunctions() { 573 e.sharedFunctions = &sharedFunctions{ 574 listenerBeforeTrampolines: make(map[*wasm.FunctionType][]byte), 575 listenerAfterTrampolines: make(map[*wasm.FunctionType][]byte), 576 } 577 578 e.be.Init() 579 { 580 src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeGrowMemory, &ssa.Signature{ 581 Params: []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI32}, 582 Results: []ssa.Type{ssa.TypeI32}, 583 }, false) 584 e.sharedFunctions.memoryGrowExecutable = mmapExecutable(src) 585 if wazevoapi.PerfMapEnabled { 586 exe := e.sharedFunctions.memoryGrowExecutable 587 wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "memory_grow_trampoline") 588 } 589 } 590 591 e.be.Init() 592 { 593 src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeTableGrow, &ssa.Signature{ 594 Params: []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI32 /* table index */, ssa.TypeI32 /* num */, ssa.TypeI64 /* ref */}, 595 Results: []ssa.Type{ssa.TypeI32}, 596 }, false) 597 e.sharedFunctions.tableGrowExecutable = mmapExecutable(src) 598 if wazevoapi.PerfMapEnabled { 599 exe := e.sharedFunctions.tableGrowExecutable 600 wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "table_grow_trampoline") 601 } 602 } 603 604 e.be.Init() 605 { 606 src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeCheckModuleExitCode, &ssa.Signature{ 607 Params: []ssa.Type{ssa.TypeI32 /* exec context */}, 608 Results: []ssa.Type{ssa.TypeI32}, 609 }, false) 610 e.sharedFunctions.checkModuleExitCode = mmapExecutable(src) 611 if wazevoapi.PerfMapEnabled { 612 exe := e.sharedFunctions.checkModuleExitCode 613 wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "check_module_exit_code_trampoline") 614 } 615 } 616 617 e.be.Init() 618 { 619 src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeRefFunc, &ssa.Signature{ 620 Params: []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI32 /* function index */}, 621 Results: []ssa.Type{ssa.TypeI64}, // returns the function reference. 622 }, false) 623 e.sharedFunctions.refFuncExecutable = mmapExecutable(src) 624 if wazevoapi.PerfMapEnabled { 625 exe := e.sharedFunctions.refFuncExecutable 626 wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "ref_func_trampoline") 627 } 628 } 629 630 e.be.Init() 631 { 632 src := e.machine.CompileStackGrowCallSequence() 633 e.sharedFunctions.stackGrowExecutable = mmapExecutable(src) 634 if wazevoapi.PerfMapEnabled { 635 exe := e.sharedFunctions.stackGrowExecutable 636 wazevoapi.PerfMap.AddEntry(uintptr(unsafe.Pointer(&exe[0])), uint64(len(exe)), "stack_grow_trampoline") 637 } 638 } 639 640 e.setFinalizer(e.sharedFunctions, sharedFunctionsFinalizer) 641 } 642 643 func sharedFunctionsFinalizer(sf *sharedFunctions) { 644 if err := platform.MunmapCodeSegment(sf.memoryGrowExecutable); err != nil { 645 panic(err) 646 } 647 if err := platform.MunmapCodeSegment(sf.checkModuleExitCode); err != nil { 648 panic(err) 649 } 650 if err := platform.MunmapCodeSegment(sf.stackGrowExecutable); err != nil { 651 panic(err) 652 } 653 if err := platform.MunmapCodeSegment(sf.tableGrowExecutable); err != nil { 654 panic(err) 655 } 656 if err := platform.MunmapCodeSegment(sf.refFuncExecutable); err != nil { 657 panic(err) 658 } 659 for _, f := range sf.listenerBeforeTrampolines { 660 if err := platform.MunmapCodeSegment(f); err != nil { 661 panic(err) 662 } 663 } 664 for _, f := range sf.listenerAfterTrampolines { 665 if err := platform.MunmapCodeSegment(f); err != nil { 666 panic(err) 667 } 668 } 669 670 sf.memoryGrowExecutable = nil 671 sf.checkModuleExitCode = nil 672 sf.stackGrowExecutable = nil 673 sf.tableGrowExecutable = nil 674 sf.refFuncExecutable = nil 675 sf.listenerBeforeTrampolines = nil 676 sf.listenerAfterTrampolines = nil 677 } 678 679 func executablesFinalizer(exec *executables) { 680 if len(exec.executable) > 0 { 681 if err := platform.MunmapCodeSegment(exec.executable); err != nil { 682 panic(err) 683 } 684 } 685 exec.executable = nil 686 687 for _, f := range exec.entryPreambles { 688 if err := platform.MunmapCodeSegment(f); err != nil { 689 panic(err) 690 } 691 } 692 exec.entryPreambles = nil 693 } 694 695 func mmapExecutable(src []byte) []byte { 696 executable, err := platform.MmapCodeSegment(len(src)) 697 if err != nil { 698 panic(err) 699 } 700 701 copy(executable, src) 702 703 if runtime.GOARCH == "arm64" { 704 // On arm64, we cannot give all of rwx at the same time, so we change it to exec. 705 if err = platform.MprotectRX(executable); err != nil { 706 panic(err) 707 } 708 } 709 return executable 710 } 711 712 func (cm *compiledModule) functionIndexOf(addr uintptr) wasm.Index { 713 addr -= uintptr(unsafe.Pointer(&cm.executable[0])) 714 offset := cm.functionOffsets 715 index := sort.Search(len(offset), func(i int) bool { 716 return offset[i] > int(addr) 717 }) 718 index-- 719 if index < 0 { 720 panic("BUG") 721 } 722 return wasm.Index(index) 723 } 724 725 func (e *engine) getListenerTrampolineForType(functionType *wasm.FunctionType) (before, after *byte) { 726 e.mux.Lock() 727 defer e.mux.Unlock() 728 729 beforeBuf, ok := e.sharedFunctions.listenerBeforeTrampolines[functionType] 730 afterBuf := e.sharedFunctions.listenerAfterTrampolines[functionType] 731 if ok { 732 return &beforeBuf[0], &afterBuf[0] 733 } 734 735 beforeSig, afterSig := frontend.SignatureForListener(functionType) 736 737 e.be.Init() 738 buf := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeCallListenerBefore, beforeSig, false) 739 beforeBuf = mmapExecutable(buf) 740 741 e.be.Init() 742 buf = e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeCallListenerAfter, afterSig, false) 743 afterBuf = mmapExecutable(buf) 744 745 e.sharedFunctions.listenerBeforeTrampolines[functionType] = beforeBuf 746 e.sharedFunctions.listenerAfterTrampolines[functionType] = afterBuf 747 return &beforeBuf[0], &afterBuf[0] 748 } 749 750 func (cm *compiledModule) getSourceOffset(pc uintptr) uint64 { 751 offsets := cm.sourceMap.executableOffsets 752 if len(offsets) == 0 { 753 return 0 754 } 755 756 index := sort.Search(len(offsets), func(i int) bool { 757 return offsets[i] >= pc 758 }) 759 760 index-- 761 if index < 0 { 762 return 0 763 } 764 return cm.sourceMap.wasmBinaryOffsets[index] 765 }