github.com/wasilibs/wazerox@v0.0.0-20240124024944-4923be63ab5f/internal/engine/wazevo/engine.go (about)

     1  package wazevo
     2  
     3  import (
     4  	"context"
     5  	"encoding/hex"
     6  	"errors"
     7  	"fmt"
     8  	"runtime"
     9  	"sort"
    10  	"sync"
    11  	"unsafe"
    12  
    13  	"github.com/wasilibs/wazerox/api"
    14  	"github.com/wasilibs/wazerox/experimental"
    15  	"github.com/wasilibs/wazerox/internal/engine/wazevo/backend"
    16  	"github.com/wasilibs/wazerox/internal/engine/wazevo/frontend"
    17  	"github.com/wasilibs/wazerox/internal/engine/wazevo/ssa"
    18  	"github.com/wasilibs/wazerox/internal/engine/wazevo/wazevoapi"
    19  	"github.com/wasilibs/wazerox/internal/filecache"
    20  	"github.com/wasilibs/wazerox/internal/platform"
    21  	"github.com/wasilibs/wazerox/internal/version"
    22  	"github.com/wasilibs/wazerox/internal/wasm"
    23  )
    24  
    25  type (
    26  	// engine implements wasm.Engine.
    27  	engine struct {
    28  		wazeroVersion   string
    29  		fileCache       filecache.Cache
    30  		compiledModules map[wasm.ModuleID]*compiledModule
    31  		// sortedCompiledModules is a list of compiled modules sorted by the initial address of the executable.
    32  		sortedCompiledModules []*compiledModule
    33  		mux                   sync.RWMutex
    34  		// rels is a list of relocations to be resolved. This is reused for each compilation to avoid allocation.
    35  		rels []backend.RelocationInfo
    36  		// refToBinaryOffset is reused for each compilation to avoid allocation.
    37  		refToBinaryOffset map[ssa.FuncRef]int
    38  		// sharedFunctions is compiled functions shared by all modules.
    39  		sharedFunctions *sharedFunctions
    40  		// setFinalizer defaults to runtime.SetFinalizer, but overridable for tests.
    41  		setFinalizer func(obj interface{}, finalizer interface{})
    42  
    43  		// The followings are reused for compiling shared functions.
    44  		machine backend.Machine
    45  		be      backend.Compiler
    46  	}
    47  
    48  	sharedFunctions struct {
    49  		// memoryGrowExecutable is a compiled trampoline executable for memory.grow builtin function.
    50  		memoryGrowExecutable []byte
    51  		// checkModuleExitCode is a compiled trampoline executable for checking module instance exit code. This
    52  		// is used when ensureTermination is true.
    53  		checkModuleExitCode []byte
    54  		// stackGrowExecutable is a compiled executable for growing stack builtin function.
    55  		stackGrowExecutable []byte
    56  		// tableGrowExecutable is a compiled trampoline executable for table.grow builtin function.
    57  		tableGrowExecutable []byte
    58  		// refFuncExecutable is a compiled trampoline executable for ref.func builtin function.
    59  		refFuncExecutable         []byte
    60  		listenerBeforeTrampolines map[*wasm.FunctionType][]byte
    61  		listenerAfterTrampolines  map[*wasm.FunctionType][]byte
    62  	}
    63  
    64  	// compiledModule is a compiled variant of a wasm.Module and ready to be used for instantiation.
    65  	compiledModule struct {
    66  		*executables
    67  		// functionOffsets maps a local function index to the offset in the executable.
    68  		functionOffsets           []int
    69  		parent                    *engine
    70  		module                    *wasm.Module
    71  		ensureTermination         bool
    72  		listeners                 []experimental.FunctionListener
    73  		listenerBeforeTrampolines []*byte
    74  		listenerAfterTrampolines  []*byte
    75  
    76  		// The followings are only available for non host modules.
    77  
    78  		offsets         wazevoapi.ModuleContextOffsetData
    79  		sharedFunctions *sharedFunctions
    80  		sourceMap       sourceMap
    81  	}
    82  
    83  	executables struct {
    84  		executable     []byte
    85  		entryPreambles [][]byte
    86  	}
    87  )
    88  
    89  // sourceMap is a mapping from the offset of the executable to the offset of the original wasm binary.
    90  type sourceMap struct {
    91  	// executableOffsets is a sorted list of offsets of the executable. This is index-correlated with wasmBinaryOffsets,
    92  	// in other words executableOffsets[i] is the offset of the executable which corresponds to the offset of a Wasm
    93  	// binary pointed by wasmBinaryOffsets[i].
    94  	executableOffsets []uintptr
    95  	// wasmBinaryOffsets is the counterpart of executableOffsets.
    96  	wasmBinaryOffsets []uint64
    97  }
    98  
    99  var _ wasm.Engine = (*engine)(nil)
   100  
   101  // NewEngine returns the implementation of wasm.Engine.
   102  func NewEngine(ctx context.Context, _ api.CoreFeatures, fc filecache.Cache) wasm.Engine {
   103  	machine := newMachine()
   104  	be := backend.NewCompiler(ctx, machine, ssa.NewBuilder())
   105  	e := &engine{
   106  		compiledModules: make(map[wasm.ModuleID]*compiledModule), refToBinaryOffset: make(map[ssa.FuncRef]int),
   107  		setFinalizer:  runtime.SetFinalizer,
   108  		machine:       machine,
   109  		be:            be,
   110  		fileCache:     fc,
   111  		wazeroVersion: version.GetWazeroVersion(),
   112  	}
   113  	e.compileSharedFunctions()
   114  	return e
   115  }
   116  
   117  // CompileModule implements wasm.Engine.
   118  func (e *engine) CompileModule(ctx context.Context, module *wasm.Module, listeners []experimental.FunctionListener, ensureTermination bool) (err error) {
   119  	if _, ok, err := e.getCompiledModule(module, listeners, ensureTermination); ok { // cache hit!
   120  		return nil
   121  	} else if err != nil {
   122  		return err
   123  	}
   124  
   125  	if wazevoapi.DeterministicCompilationVerifierEnabled {
   126  		ctx = wazevoapi.NewDeterministicCompilationVerifierContext(ctx, len(module.CodeSection))
   127  	}
   128  	cm, err := e.compileModule(ctx, module, listeners, ensureTermination)
   129  	if err != nil {
   130  		return err
   131  	}
   132  	if err = e.addCompiledModule(module, cm); err != nil {
   133  		return err
   134  	}
   135  
   136  	if wazevoapi.DeterministicCompilationVerifierEnabled {
   137  		for i := 0; i < wazevoapi.DeterministicCompilationVerifyingIter; i++ {
   138  			_, err := e.compileModule(ctx, module, listeners, ensureTermination)
   139  			if err != nil {
   140  				return err
   141  			}
   142  		}
   143  	}
   144  
   145  	if len(listeners) > 0 {
   146  		cm.listeners = listeners
   147  		cm.listenerBeforeTrampolines = make([]*byte, len(module.TypeSection))
   148  		cm.listenerAfterTrampolines = make([]*byte, len(module.TypeSection))
   149  		for i := range module.TypeSection {
   150  			typ := &module.TypeSection[i]
   151  			before, after := e.getListenerTrampolineForType(typ)
   152  			cm.listenerBeforeTrampolines[i] = before
   153  			cm.listenerAfterTrampolines[i] = after
   154  		}
   155  	}
   156  	return nil
   157  }
   158  
   159  func (exec *executables) compileEntryPreambles(m *wasm.Module, machine backend.Machine, be backend.Compiler) {
   160  	exec.entryPreambles = make([][]byte, len(m.TypeSection))
   161  	for i := range m.TypeSection {
   162  		typ := &m.TypeSection[i]
   163  		sig := frontend.SignatureForWasmFunctionType(typ)
   164  		be.Init()
   165  		buf := machine.CompileEntryPreamble(&sig)
   166  		executable := mmapExecutable(buf)
   167  		exec.entryPreambles[i] = executable
   168  	}
   169  }
   170  
   171  func (e *engine) compileModule(ctx context.Context, module *wasm.Module, listeners []experimental.FunctionListener, ensureTermination bool) (*compiledModule, error) {
   172  	withListener := len(listeners) > 0
   173  	e.rels = e.rels[:0]
   174  	cm := &compiledModule{
   175  		offsets: wazevoapi.NewModuleContextOffsetData(module, withListener), parent: e, module: module,
   176  		ensureTermination: ensureTermination,
   177  		executables:       &executables{},
   178  	}
   179  
   180  	if module.IsHostModule {
   181  		return e.compileHostModule(ctx, module, listeners)
   182  	}
   183  
   184  	importedFns, localFns := int(module.ImportFunctionCount), len(module.FunctionSection)
   185  	if localFns == 0 {
   186  		return cm, nil
   187  	}
   188  
   189  	if wazevoapi.DeterministicCompilationVerifierEnabled {
   190  		// The compilation must be deterministic regardless of the order of functions being compiled.
   191  		wazevoapi.DeterministicCompilationVerifierRandomizeIndexes(ctx)
   192  	}
   193  
   194  	needSourceInfo := module.DWARFLines != nil
   195  
   196  	// Creates new compiler instances which are reused for each function.
   197  	ssaBuilder := ssa.NewBuilder()
   198  	fe := frontend.NewFrontendCompiler(module, ssaBuilder, &cm.offsets, ensureTermination, withListener, needSourceInfo)
   199  	machine := newMachine()
   200  	be := backend.NewCompiler(ctx, machine, ssaBuilder)
   201  
   202  	cm.executables.compileEntryPreambles(module, machine, be)
   203  
   204  	totalSize := 0 // Total binary size of the executable.
   205  	cm.functionOffsets = make([]int, localFns)
   206  	bodies := make([][]byte, localFns)
   207  	for i := range module.CodeSection {
   208  		if wazevoapi.DeterministicCompilationVerifierEnabled {
   209  			i = wazevoapi.DeterministicCompilationVerifierGetRandomizedLocalFunctionIndex(ctx, i)
   210  		}
   211  
   212  		fidx := wasm.Index(i + importedFns)
   213  
   214  		if wazevoapi.NeedFunctionNameInContext {
   215  			def := module.FunctionDefinition(fidx)
   216  			name := def.DebugName()
   217  			if len(def.ExportNames()) > 0 {
   218  				name = def.ExportNames()[0]
   219  			}
   220  			ctx = wazevoapi.SetCurrentFunctionName(ctx, fmt.Sprintf("[%d/%d] \"%s\"", i, len(module.CodeSection)-1, name))
   221  		}
   222  
   223  		needListener := len(listeners) > 0 && listeners[i] != nil
   224  		body, rels, err := e.compileLocalWasmFunction(ctx, module, wasm.Index(i), fe, ssaBuilder, be, needListener)
   225  		if err != nil {
   226  			return nil, fmt.Errorf("compile function %d/%d: %v", i, len(module.CodeSection)-1, err)
   227  		}
   228  
   229  		// Align 16-bytes boundary.
   230  		totalSize = (totalSize + 15) &^ 15
   231  		cm.functionOffsets[i] = totalSize
   232  
   233  		if needSourceInfo {
   234  			// At the beginning of the function, we add the offset of the function body so that
   235  			// we can resolve the source location of the call site of before listener call.
   236  			cm.sourceMap.executableOffsets = append(cm.sourceMap.executableOffsets, uintptr(totalSize))
   237  			cm.sourceMap.wasmBinaryOffsets = append(cm.sourceMap.wasmBinaryOffsets, module.CodeSection[i].BodyOffsetInCodeSection)
   238  
   239  			for _, info := range be.SourceOffsetInfo() {
   240  				cm.sourceMap.executableOffsets = append(cm.sourceMap.executableOffsets, uintptr(totalSize)+uintptr(info.ExecutableOffset))
   241  				cm.sourceMap.wasmBinaryOffsets = append(cm.sourceMap.wasmBinaryOffsets, uint64(info.SourceOffset))
   242  			}
   243  		}
   244  
   245  		fref := frontend.FunctionIndexToFuncRef(fidx)
   246  		e.refToBinaryOffset[fref] = totalSize
   247  
   248  		// At this point, relocation offsets are relative to the start of the function body,
   249  		// so we adjust it to the start of the executable.
   250  		for _, r := range rels {
   251  			r.Offset += int64(totalSize)
   252  			e.rels = append(e.rels, r)
   253  		}
   254  
   255  		bodies[i] = body
   256  		totalSize += len(body)
   257  		if wazevoapi.PrintMachineCodeHexPerFunction {
   258  			fmt.Printf("[[[machine code for %s]]]\n%s\n\n", wazevoapi.GetCurrentFunctionName(ctx), hex.EncodeToString(body))
   259  		}
   260  	}
   261  
   262  	// Allocate executable memory and then copy the generated machine code.
   263  	executable, err := platform.MmapCodeSegment(totalSize)
   264  	if err != nil {
   265  		panic(err)
   266  	}
   267  	cm.executable = executable
   268  
   269  	for i, b := range bodies {
   270  		offset := cm.functionOffsets[i]
   271  		copy(executable[offset:], b)
   272  	}
   273  
   274  	if needSourceInfo {
   275  		for i := range cm.sourceMap.executableOffsets {
   276  			cm.sourceMap.executableOffsets[i] += uintptr(unsafe.Pointer(&cm.executable[0]))
   277  		}
   278  	}
   279  
   280  	// Resolve relocations for local function calls.
   281  	machine.ResolveRelocations(e.refToBinaryOffset, executable, e.rels)
   282  
   283  	if runtime.GOARCH == "arm64" {
   284  		// On arm64, we cannot give all of rwx at the same time, so we change it to exec.
   285  		if err = platform.MprotectRX(executable); err != nil {
   286  			return nil, err
   287  		}
   288  	}
   289  	cm.sharedFunctions = e.sharedFunctions
   290  	e.setFinalizer(cm.executables, executablesFinalizer)
   291  	return cm, nil
   292  }
   293  
   294  func (e *engine) compileLocalWasmFunction(
   295  	ctx context.Context,
   296  	module *wasm.Module,
   297  	localFunctionIndex wasm.Index,
   298  	fe *frontend.Compiler,
   299  	ssaBuilder ssa.Builder,
   300  	be backend.Compiler,
   301  	needListener bool,
   302  ) (body []byte, rels []backend.RelocationInfo, err error) {
   303  	typIndex := module.FunctionSection[localFunctionIndex]
   304  	typ := &module.TypeSection[typIndex]
   305  	codeSeg := &module.CodeSection[localFunctionIndex]
   306  
   307  	// Initializes both frontend and backend compilers.
   308  	fe.Init(localFunctionIndex, typIndex, typ, codeSeg.LocalTypes, codeSeg.Body, needListener, codeSeg.BodyOffsetInCodeSection)
   309  	be.Init()
   310  
   311  	// Lower Wasm to SSA.
   312  	fe.LowerToSSA()
   313  	if wazevoapi.PrintSSA {
   314  		fmt.Printf("[[[SSA for %s]]]%s\n", wazevoapi.GetCurrentFunctionName(ctx), ssaBuilder.Format())
   315  	}
   316  
   317  	if wazevoapi.DeterministicCompilationVerifierEnabled {
   318  		wazevoapi.VerifyOrSetDeterministicCompilationContextValue(ctx, "SSA", ssaBuilder.Format())
   319  	}
   320  
   321  	// Run SSA-level optimization passes.
   322  	ssaBuilder.RunPasses()
   323  
   324  	if wazevoapi.PrintOptimizedSSA {
   325  		fmt.Printf("[[[Optimized SSA for %s]]]%s\n", wazevoapi.GetCurrentFunctionName(ctx), ssaBuilder.Format())
   326  	}
   327  
   328  	if wazevoapi.DeterministicCompilationVerifierEnabled {
   329  		wazevoapi.VerifyOrSetDeterministicCompilationContextValue(ctx, "Optimized SSA", ssaBuilder.Format())
   330  	}
   331  
   332  	// Finalize the layout of SSA blocks which might use the optimization results.
   333  	ssaBuilder.LayoutBlocks()
   334  
   335  	if wazevoapi.PrintBlockLaidOutSSA {
   336  		fmt.Printf("[[[Laidout SSA for %s]]]%s\n", wazevoapi.GetCurrentFunctionName(ctx), ssaBuilder.Format())
   337  	}
   338  
   339  	if wazevoapi.DeterministicCompilationVerifierEnabled {
   340  		wazevoapi.VerifyOrSetDeterministicCompilationContextValue(ctx, "Block laid out SSA", ssaBuilder.Format())
   341  	}
   342  
   343  	// Now our ssaBuilder contains the necessary information to further lower them to
   344  	// machine code.
   345  	original, rels, err := be.Compile(ctx)
   346  	if err != nil {
   347  		return nil, nil, fmt.Errorf("ssa->machine code: %v", err)
   348  	}
   349  
   350  	// TODO: optimize as zero copy.
   351  	copied := make([]byte, len(original))
   352  	copy(copied, original)
   353  	return copied, rels, nil
   354  }
   355  
   356  func (e *engine) compileHostModule(ctx context.Context, module *wasm.Module, listeners []experimental.FunctionListener) (*compiledModule, error) {
   357  	machine := newMachine()
   358  	be := backend.NewCompiler(ctx, machine, ssa.NewBuilder())
   359  
   360  	num := len(module.CodeSection)
   361  	cm := &compiledModule{module: module, listeners: listeners, executables: &executables{}}
   362  	cm.functionOffsets = make([]int, num)
   363  	totalSize := 0 // Total binary size of the executable.
   364  	bodies := make([][]byte, num)
   365  	var sig ssa.Signature
   366  	for i := range module.CodeSection {
   367  		totalSize = (totalSize + 15) &^ 15
   368  		cm.functionOffsets[i] = totalSize
   369  
   370  		typIndex := module.FunctionSection[i]
   371  		typ := &module.TypeSection[typIndex]
   372  
   373  		// We can relax until the index fits together in ExitCode as we do in wazevoapi.ExitCodeCallGoModuleFunctionWithIndex.
   374  		// However, 1 << 16 should be large enough for a real use case.
   375  		const hostFunctionNumMaximum = 1 << 16
   376  		if i >= hostFunctionNumMaximum {
   377  			return nil, fmt.Errorf("too many host functions (maximum %d)", hostFunctionNumMaximum)
   378  		}
   379  
   380  		sig.ID = ssa.SignatureID(typIndex) // This is important since we reuse the `machine` which caches the ABI based on the SignatureID.
   381  		sig.Params = append(sig.Params[:0],
   382  			ssa.TypeI64, // First argument must be exec context.
   383  			ssa.TypeI64, // The second argument is the moduleContextOpaque of this host module.
   384  		)
   385  		for _, t := range typ.Params {
   386  			sig.Params = append(sig.Params, frontend.WasmTypeToSSAType(t))
   387  		}
   388  
   389  		sig.Results = sig.Results[:0]
   390  		for _, t := range typ.Results {
   391  			sig.Results = append(sig.Results, frontend.WasmTypeToSSAType(t))
   392  		}
   393  
   394  		c := &module.CodeSection[i]
   395  		if c.GoFunc == nil {
   396  			panic("BUG: GoFunc must be set for host module")
   397  		}
   398  
   399  		withListener := len(listeners) > 0 && listeners[i] != nil
   400  		var exitCode wazevoapi.ExitCode
   401  		fn := c.GoFunc
   402  		switch fn.(type) {
   403  		case api.GoModuleFunction:
   404  			exitCode = wazevoapi.ExitCodeCallGoModuleFunctionWithIndex(i, withListener)
   405  		case api.GoFunction:
   406  			exitCode = wazevoapi.ExitCodeCallGoFunctionWithIndex(i, withListener)
   407  		}
   408  
   409  		be.Init()
   410  		machine.CompileGoFunctionTrampoline(exitCode, &sig, true)
   411  		be.Encode()
   412  		body := be.Buf()
   413  
   414  		// TODO: optimize as zero copy.
   415  		copied := make([]byte, len(body))
   416  		copy(copied, body)
   417  		bodies[i] = copied
   418  		totalSize += len(body)
   419  	}
   420  
   421  	// Allocate executable memory and then copy the generated machine code.
   422  	executable, err := platform.MmapCodeSegment(totalSize)
   423  	if err != nil {
   424  		panic(err)
   425  	}
   426  	cm.executable = executable
   427  
   428  	for i, b := range bodies {
   429  		offset := cm.functionOffsets[i]
   430  		copy(executable[offset:], b)
   431  	}
   432  
   433  	if runtime.GOARCH == "arm64" {
   434  		// On arm64, we cannot give all of rwx at the same time, so we change it to exec.
   435  		if err = platform.MprotectRX(executable); err != nil {
   436  			return nil, err
   437  		}
   438  	}
   439  	e.setFinalizer(cm.executables, executablesFinalizer)
   440  	return cm, nil
   441  }
   442  
   443  // Close implements wasm.Engine.
   444  func (e *engine) Close() (err error) {
   445  	e.mux.Lock()
   446  	defer e.mux.Unlock()
   447  
   448  	for _, cm := range e.compiledModules {
   449  		cm.functionOffsets = nil
   450  		cm.module = nil
   451  		cm.parent = nil
   452  		cm.executables = nil
   453  	}
   454  	e.sortedCompiledModules = nil
   455  	e.compiledModules = nil
   456  	e.sharedFunctions = nil
   457  	return nil
   458  }
   459  
   460  // CompiledModuleCount implements wasm.Engine.
   461  func (e *engine) CompiledModuleCount() uint32 {
   462  	e.mux.RLock()
   463  	defer e.mux.RUnlock()
   464  	return uint32(len(e.compiledModules))
   465  }
   466  
   467  // DeleteCompiledModule implements wasm.Engine.
   468  func (e *engine) DeleteCompiledModule(m *wasm.Module) {
   469  	e.mux.Lock()
   470  	defer e.mux.Unlock()
   471  	cm, ok := e.compiledModules[m.ID]
   472  	if ok {
   473  		if len(cm.executable) > 0 {
   474  			e.deleteCompiledModuleFromSortedList(cm)
   475  		}
   476  		delete(e.compiledModules, m.ID)
   477  	}
   478  }
   479  
   480  func (e *engine) addCompiledModuleToSortedList(cm *compiledModule) {
   481  	ptr := uintptr(unsafe.Pointer(&cm.executable[0]))
   482  
   483  	index := sort.Search(len(e.sortedCompiledModules), func(i int) bool {
   484  		return uintptr(unsafe.Pointer(&e.sortedCompiledModules[i].executable[0])) >= ptr
   485  	})
   486  	e.sortedCompiledModules = append(e.sortedCompiledModules, nil)
   487  	copy(e.sortedCompiledModules[index+1:], e.sortedCompiledModules[index:])
   488  	e.sortedCompiledModules[index] = cm
   489  }
   490  
   491  func (e *engine) deleteCompiledModuleFromSortedList(cm *compiledModule) {
   492  	ptr := uintptr(unsafe.Pointer(&cm.executable[0]))
   493  
   494  	index := sort.Search(len(e.sortedCompiledModules), func(i int) bool {
   495  		return uintptr(unsafe.Pointer(&e.sortedCompiledModules[i].executable[0])) >= ptr
   496  	})
   497  	if index >= len(e.sortedCompiledModules) {
   498  		return
   499  	}
   500  	copy(e.sortedCompiledModules[index:], e.sortedCompiledModules[index+1:])
   501  	e.sortedCompiledModules = e.sortedCompiledModules[:len(e.sortedCompiledModules)-1]
   502  }
   503  
   504  func (e *engine) compiledModuleOfAddr(addr uintptr) *compiledModule {
   505  	e.mux.RLock()
   506  	defer e.mux.RUnlock()
   507  
   508  	index := sort.Search(len(e.sortedCompiledModules), func(i int) bool {
   509  		return uintptr(unsafe.Pointer(&e.sortedCompiledModules[i].executable[0])) > addr
   510  	})
   511  	index -= 1
   512  	if index < 0 {
   513  		return nil
   514  	}
   515  	return e.sortedCompiledModules[index]
   516  }
   517  
   518  // NewModuleEngine implements wasm.Engine.
   519  func (e *engine) NewModuleEngine(m *wasm.Module, mi *wasm.ModuleInstance) (wasm.ModuleEngine, error) {
   520  	me := &moduleEngine{}
   521  
   522  	// Note: imported functions are resolved in moduleEngine.ResolveImportedFunction.
   523  	me.importedFunctions = make([]importedFunction, m.ImportFunctionCount)
   524  
   525  	compiled, ok := e.compiledModules[m.ID]
   526  	if !ok {
   527  		return nil, errors.New("source module must be compiled before instantiation")
   528  	}
   529  	me.parent = compiled
   530  	me.module = mi
   531  	me.listeners = compiled.listeners
   532  
   533  	if m.IsHostModule {
   534  		me.opaque = buildHostModuleOpaque(m, compiled.listeners)
   535  		me.opaquePtr = &me.opaque[0]
   536  	} else {
   537  		if size := compiled.offsets.TotalSize; size != 0 {
   538  			opaque := make([]byte, size)
   539  			me.opaque = opaque
   540  			me.opaquePtr = &opaque[0]
   541  		}
   542  	}
   543  	return me, nil
   544  }
   545  
   546  func (e *engine) compileSharedFunctions() {
   547  	e.sharedFunctions = &sharedFunctions{
   548  		listenerBeforeTrampolines: make(map[*wasm.FunctionType][]byte),
   549  		listenerAfterTrampolines:  make(map[*wasm.FunctionType][]byte),
   550  	}
   551  
   552  	e.be.Init()
   553  	{
   554  		src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeGrowMemory, &ssa.Signature{
   555  			Params:  []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI32},
   556  			Results: []ssa.Type{ssa.TypeI32},
   557  		}, false)
   558  		e.sharedFunctions.memoryGrowExecutable = mmapExecutable(src)
   559  	}
   560  
   561  	e.be.Init()
   562  	{
   563  		src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeTableGrow, &ssa.Signature{
   564  			Params:  []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI32 /* table index */, ssa.TypeI32 /* num */, ssa.TypeI64 /* ref */},
   565  			Results: []ssa.Type{ssa.TypeI32},
   566  		}, false)
   567  		e.sharedFunctions.tableGrowExecutable = mmapExecutable(src)
   568  	}
   569  
   570  	e.be.Init()
   571  	{
   572  		src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeCheckModuleExitCode, &ssa.Signature{
   573  			Params:  []ssa.Type{ssa.TypeI32 /* exec context */},
   574  			Results: []ssa.Type{ssa.TypeI32},
   575  		}, false)
   576  		e.sharedFunctions.checkModuleExitCode = mmapExecutable(src)
   577  	}
   578  
   579  	e.be.Init()
   580  	{
   581  		src := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeRefFunc, &ssa.Signature{
   582  			Params:  []ssa.Type{ssa.TypeI64 /* exec context */, ssa.TypeI32 /* function index */},
   583  			Results: []ssa.Type{ssa.TypeI64}, // returns the function reference.
   584  		}, false)
   585  		e.sharedFunctions.refFuncExecutable = mmapExecutable(src)
   586  	}
   587  
   588  	e.be.Init()
   589  	{
   590  		src := e.machine.CompileStackGrowCallSequence()
   591  		e.sharedFunctions.stackGrowExecutable = mmapExecutable(src)
   592  	}
   593  
   594  	e.setFinalizer(e.sharedFunctions, sharedFunctionsFinalizer)
   595  }
   596  
   597  func sharedFunctionsFinalizer(sf *sharedFunctions) {
   598  	if err := platform.MunmapCodeSegment(sf.memoryGrowExecutable); err != nil {
   599  		panic(err)
   600  	}
   601  	if err := platform.MunmapCodeSegment(sf.checkModuleExitCode); err != nil {
   602  		panic(err)
   603  	}
   604  	if err := platform.MunmapCodeSegment(sf.stackGrowExecutable); err != nil {
   605  		panic(err)
   606  	}
   607  	if err := platform.MunmapCodeSegment(sf.tableGrowExecutable); err != nil {
   608  		panic(err)
   609  	}
   610  	if err := platform.MunmapCodeSegment(sf.refFuncExecutable); err != nil {
   611  		panic(err)
   612  	}
   613  	for _, f := range sf.listenerBeforeTrampolines {
   614  		if err := platform.MunmapCodeSegment(f); err != nil {
   615  			panic(err)
   616  		}
   617  	}
   618  	for _, f := range sf.listenerAfterTrampolines {
   619  		if err := platform.MunmapCodeSegment(f); err != nil {
   620  			panic(err)
   621  		}
   622  	}
   623  
   624  	sf.memoryGrowExecutable = nil
   625  	sf.checkModuleExitCode = nil
   626  	sf.stackGrowExecutable = nil
   627  	sf.tableGrowExecutable = nil
   628  	sf.refFuncExecutable = nil
   629  	sf.listenerBeforeTrampolines = nil
   630  	sf.listenerAfterTrampolines = nil
   631  }
   632  
   633  func executablesFinalizer(exec *executables) {
   634  	if len(exec.executable) > 0 {
   635  		if err := platform.MunmapCodeSegment(exec.executable); err != nil {
   636  			panic(err)
   637  		}
   638  	}
   639  	exec.executable = nil
   640  
   641  	for _, f := range exec.entryPreambles {
   642  		if err := platform.MunmapCodeSegment(f); err != nil {
   643  			panic(err)
   644  		}
   645  	}
   646  	exec.entryPreambles = nil
   647  }
   648  
   649  func mmapExecutable(src []byte) []byte {
   650  	executable, err := platform.MmapCodeSegment(len(src))
   651  	if err != nil {
   652  		panic(err)
   653  	}
   654  
   655  	copy(executable, src)
   656  
   657  	if runtime.GOARCH == "arm64" {
   658  		// On arm64, we cannot give all of rwx at the same time, so we change it to exec.
   659  		if err = platform.MprotectRX(executable); err != nil {
   660  			panic(err)
   661  		}
   662  	}
   663  	return executable
   664  }
   665  
   666  func (cm *compiledModule) functionIndexOf(addr uintptr) wasm.Index {
   667  	addr -= uintptr(unsafe.Pointer(&cm.executable[0]))
   668  	offset := cm.functionOffsets
   669  	index := sort.Search(len(offset), func(i int) bool {
   670  		return offset[i] > int(addr)
   671  	})
   672  	index--
   673  	if index < 0 {
   674  		panic("BUG")
   675  	}
   676  	return wasm.Index(index)
   677  }
   678  
   679  func (e *engine) getListenerTrampolineForType(functionType *wasm.FunctionType) (before, after *byte) {
   680  	e.mux.Lock()
   681  	defer e.mux.Unlock()
   682  
   683  	beforeBuf, ok := e.sharedFunctions.listenerBeforeTrampolines[functionType]
   684  	afterBuf := e.sharedFunctions.listenerAfterTrampolines[functionType]
   685  	if ok {
   686  		return &beforeBuf[0], &afterBuf[0]
   687  	}
   688  
   689  	beforeSig, afterSig := frontend.SignatureForListener(functionType)
   690  
   691  	e.be.Init()
   692  	buf := e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeCallListenerBefore, beforeSig, false)
   693  	beforeBuf = mmapExecutable(buf)
   694  
   695  	e.be.Init()
   696  	buf = e.machine.CompileGoFunctionTrampoline(wazevoapi.ExitCodeCallListenerAfter, afterSig, false)
   697  	afterBuf = mmapExecutable(buf)
   698  
   699  	e.sharedFunctions.listenerBeforeTrampolines[functionType] = beforeBuf
   700  	e.sharedFunctions.listenerAfterTrampolines[functionType] = afterBuf
   701  	return &beforeBuf[0], &afterBuf[0]
   702  }
   703  
   704  func (cm *compiledModule) getSourceOffset(pc uintptr) uint64 {
   705  	offsets := cm.sourceMap.executableOffsets
   706  	if len(offsets) == 0 {
   707  		return 0
   708  	}
   709  
   710  	index := sort.Search(len(offsets), func(i int) bool {
   711  		return offsets[i] >= pc
   712  	})
   713  
   714  	index--
   715  	if index < 0 {
   716  		return 0
   717  	}
   718  	return cm.sourceMap.wasmBinaryOffsets[index]
   719  }